From 406e1f7fabad267b50f9da62b3d1527461b2598a Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Sun, 30 Jun 2019 17:42:09 +0000 Subject: [PATCH 01/27] fix and improve star_trans on SST --- fastNLP/io/data_loader/sst.py | 7 ++- fastNLP/models/star_transformer.py | 6 +-- fastNLP/modules/encoder/star_transformer.py | 13 +++-- reproduction/Star_transformer/datasets.py | 11 ++-- reproduction/Star_transformer/run.sh | 4 +- reproduction/Star_transformer/train.py | 58 ++++++++++++++------- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/fastNLP/io/data_loader/sst.py b/fastNLP/io/data_loader/sst.py index 1e1b8bef..021a79b7 100644 --- a/fastNLP/io/data_loader/sst.py +++ b/fastNLP/io/data_loader/sst.py @@ -1,11 +1,14 @@ from typing import Iterable from nltk import Tree +import spacy from ..base_loader import DataInfo, DataSetLoader from ...core.vocabulary import VocabularyOption, Vocabulary from ...core.dataset import DataSet from ...core.instance import Instance from ..embed_loader import EmbeddingOption, EmbedLoader +spacy.prefer_gpu() +sptk = spacy.load('en') class SSTLoader(DataSetLoader): URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' @@ -56,8 +59,8 @@ class SSTLoader(DataSetLoader): def _get_one(data, subtree): tree = Tree.fromstring(data) if subtree: - return [(t.leaves(), t.label()) for t in tree.subtrees()] - return [(tree.leaves(), tree.label())] + return [([x.text for x in sptk.tokenizer(' '.join(t.leaves()))], t.label()) for t in tree.subtrees() ] + return [([x.text for x in sptk.tokenizer(' '.join(tree.leaves()))], tree.label())] def process(self, paths, diff --git a/fastNLP/models/star_transformer.py b/fastNLP/models/star_transformer.py index 4c944a54..1aba5a8c 100644 --- a/fastNLP/models/star_transformer.py +++ b/fastNLP/models/star_transformer.py @@ -46,7 +46,7 @@ class StarTransEnc(nn.Module): super(StarTransEnc, self).__init__() self.embedding = get_embeddings(init_embed) emb_dim = self.embedding.embedding_dim - self.emb_fc = nn.Linear(emb_dim, hidden_size) + #self.emb_fc = nn.Linear(emb_dim, hidden_size) self.emb_drop = nn.Dropout(emb_dropout) self.encoder = StarTransformer(hidden_size=hidden_size, num_layers=num_layers, @@ -65,7 +65,7 @@ class StarTransEnc(nn.Module): [batch, hidden] 全局 relay 节点, 详见论文 """ x = self.embedding(x) - x = self.emb_fc(self.emb_drop(x)) + #x = self.emb_fc(self.emb_drop(x)) nodes, relay = self.encoder(x, mask) return nodes, relay @@ -205,7 +205,7 @@ class STSeqCls(nn.Module): max_len=max_len, emb_dropout=emb_dropout, dropout=dropout) - self.cls = _Cls(hidden_size, num_cls, cls_hidden_size) + self.cls = _Cls(hidden_size, num_cls, cls_hidden_size, dropout=dropout) def forward(self, words, seq_len): """ diff --git a/fastNLP/modules/encoder/star_transformer.py b/fastNLP/modules/encoder/star_transformer.py index 1eec7c13..76b7e922 100644 --- a/fastNLP/modules/encoder/star_transformer.py +++ b/fastNLP/modules/encoder/star_transformer.py @@ -35,11 +35,13 @@ class StarTransformer(nn.Module): self.iters = num_layers self.norm = nn.ModuleList([nn.LayerNorm(hidden_size) for _ in range(self.iters)]) + self.emb_fc = nn.Conv2d(hidden_size, hidden_size, 1) + self.emb_drop = nn.Dropout(dropout) self.ring_att = nn.ModuleList( - [_MSA1(hidden_size, nhead=num_head, head_dim=head_dim, dropout=dropout) + [_MSA1(hidden_size, nhead=num_head, head_dim=head_dim, dropout=0.0) for _ in range(self.iters)]) self.star_att = nn.ModuleList( - [_MSA2(hidden_size, nhead=num_head, head_dim=head_dim, dropout=dropout) + [_MSA2(hidden_size, nhead=num_head, head_dim=head_dim, dropout=0.0) for _ in range(self.iters)]) if max_len is not None: @@ -66,18 +68,19 @@ class StarTransformer(nn.Module): smask = torch.cat([torch.zeros(B, 1, ).byte().to(mask), mask], 1) embs = data.permute(0, 2, 1)[:, :, :, None] # B H L 1 - if self.pos_emb: + if self.pos_emb and False: P = self.pos_emb(torch.arange(L, dtype=torch.long, device=embs.device) \ .view(1, L)).permute(0, 2, 1).contiguous()[:, :, :, None] # 1 H L 1 embs = embs + P - + embs = norm_func(self.emb_drop, embs) nodes = embs relay = embs.mean(2, keepdim=True) ex_mask = mask[:, None, :, None].expand(B, H, L, 1) r_embs = embs.view(B, H, 1, L) for i in range(self.iters): ax = torch.cat([r_embs, relay.expand(B, H, 1, L)], 2) - nodes = nodes + F.leaky_relu(self.ring_att[i](norm_func(self.norm[i], nodes), ax=ax)) + nodes = F.leaky_relu(self.ring_att[i](norm_func(self.norm[i], nodes), ax=ax)) + #nodes = F.leaky_relu(self.ring_att[i](nodes, ax=ax)) relay = F.leaky_relu(self.star_att[i](relay, torch.cat([relay, nodes], 2), smask)) nodes = nodes.masked_fill_(ex_mask, 0) diff --git a/reproduction/Star_transformer/datasets.py b/reproduction/Star_transformer/datasets.py index a9257fd4..5597dcbf 100644 --- a/reproduction/Star_transformer/datasets.py +++ b/reproduction/Star_transformer/datasets.py @@ -50,13 +50,15 @@ def load_sst(path, files): for sub in [True, False, False]] ds_list = [loader.load(os.path.join(path, fn)) for fn, loader in zip(files, loaders)] - word_v = Vocabulary(min_freq=2) + word_v = Vocabulary(min_freq=0) tag_v = Vocabulary(unknown=None, padding=None) for ds in ds_list: ds.apply(lambda x: [w.lower() for w in x['words']], new_field_name='words') - ds_list[0].drop(lambda x: len(x['words']) < 3) + #ds_list[0].drop(lambda x: len(x['words']) < 3) update_v(word_v, ds_list[0], 'words') + update_v(word_v, ds_list[1], 'words') + update_v(word_v, ds_list[2], 'words') ds_list[0].apply(lambda x: tag_v.add_word( x['target']), new_field_name=None) @@ -151,7 +153,10 @@ class EmbedLoader: # some words from vocab are missing in pre-trained embedding # we normally sample each dimension vocab_embed = embedding_matrix[np.where(hit_flags)] - sampled_vectors = np.random.normal(vocab_embed.mean(axis=0), vocab_embed.std(axis=0), + #sampled_vectors = np.random.normal(vocab_embed.mean(axis=0), vocab_embed.std(axis=0), + # size=(len(vocab) - np.sum(hit_flags), emb_dim)) + sampled_vectors = np.random.uniform(-0.01, 0.01, size=(len(vocab) - np.sum(hit_flags), emb_dim)) + embedding_matrix[np.where(1 - hit_flags)] = sampled_vectors return embedding_matrix diff --git a/reproduction/Star_transformer/run.sh b/reproduction/Star_transformer/run.sh index 0972c662..5cd6954b 100644 --- a/reproduction/Star_transformer/run.sh +++ b/reproduction/Star_transformer/run.sh @@ -1,5 +1,5 @@ #python -u train.py --task pos --ds conll --mode train --gpu 1 --lr 3e-4 --w_decay 2e-5 --lr_decay .95 --drop 0.3 --ep 25 --bsz 64 > conll_pos102.log 2>&1 & #python -u train.py --task pos --ds ctb --mode train --gpu 1 --lr 3e-4 --w_decay 2e-5 --lr_decay .95 --drop 0.3 --ep 25 --bsz 64 > ctb_pos101.log 2>&1 & -#python -u train.py --task cls --ds sst --mode train --gpu 2 --lr 1e-4 --w_decay 1e-5 --lr_decay 0.9 --drop 0.5 --ep 50 --bsz 128 > sst_cls201.log & +python -u train.py --task cls --ds sst --mode train --gpu 0 --lr 1e-4 --w_decay 5e-5 --lr_decay 1.0 --drop 0.4 --ep 20 --bsz 64 > sst_cls.log & #python -u train.py --task nli --ds snli --mode train --gpu 1 --lr 1e-4 --w_decay 1e-5 --lr_decay 0.9 --drop 0.4 --ep 120 --bsz 128 > snli_nli201.log & -python -u train.py --task ner --ds conll --mode train --gpu 0 --lr 1e-4 --w_decay 1e-5 --lr_decay 0.9 --drop 0.4 --ep 120 --bsz 64 > conll_ner201.log & +#python -u train.py --task ner --ds conll --mode train --gpu 0 --lr 1e-4 --w_decay 1e-5 --lr_decay 0.9 --drop 0.4 --ep 120 --bsz 64 > conll_ner201.log & diff --git a/reproduction/Star_transformer/train.py b/reproduction/Star_transformer/train.py index 6fb58daf..edb7fe27 100644 --- a/reproduction/Star_transformer/train.py +++ b/reproduction/Star_transformer/train.py @@ -7,8 +7,8 @@ import fastNLP as FN from fastNLP.models.star_transformer import STSeqLabel, STSeqCls, STNLICls from fastNLP.core.const import Const as C import sys -sys.path.append('/remote-home/yfshao/workdir/dev_fastnlp/') - +#sys.path.append('/remote-home/yfshao/workdir/dev_fastnlp/') +pre_dir = '/home/ec2-user/fast_data/' g_model_select = { 'pos': STSeqLabel, @@ -17,8 +17,8 @@ g_model_select = { 'nli': STNLICls, } -g_emb_file_path = {'en': '/remote-home/yfshao/workdir/datasets/word_vector/glove.840B.300d.txt', - 'zh': '/remote-home/yfshao/workdir/datasets/word_vector/cc.zh.300.vec'} +g_emb_file_path = {'en': pre_dir + 'glove.840B.300d.txt', + 'zh': pre_dir + 'cc.zh.300.vec'} g_args = None g_model_cfg = None @@ -53,7 +53,7 @@ def get_conll2012_ner(): def get_sst(): - path = '/remote-home/yfshao/workdir/datasets/SST' + path = pre_dir + 'sst' files = ['train.txt', 'dev.txt', 'test.txt'] return load_sst(path, files) @@ -94,6 +94,7 @@ class MyCallback(FN.core.callback.Callback): nn.utils.clip_grad.clip_grad_norm_(self.model.parameters(), 5.0) def on_step_end(self): + return warm_steps = 6000 # learning rate warm-up & decay if self.step <= warm_steps: @@ -108,12 +109,14 @@ class MyCallback(FN.core.callback.Callback): def train(): - seed = set_rng_seeds(1234) + #seed = set_rng_seeds(1234) + seed = set_rng_seeds(np.random.randint(65536)) print('RNG SEED {}'.format(seed)) print('loading data') ds_list, word_v, tag_v = g_datasets['{}-{}'.format( g_args.ds, g_args.task)]() print(ds_list[0][:2]) + print(len(ds_list[0]), len(ds_list[1]), len(ds_list[2])) embed = load_pretrain_emb(word_v, lang='zh' if g_args.ds == 'ctb' else 'en') g_model_cfg['num_cls'] = len(tag_v) print(g_model_cfg) @@ -123,11 +126,14 @@ def train(): def init_model(model): for p in model.parameters(): if p.size(0) != len(word_v): - nn.init.normal_(p, 0.0, 0.05) + if len(p.size())<2: + nn.init.constant_(p, 0.0) + else: + nn.init.normal_(p, 0.0, 0.05) init_model(model) train_data = ds_list[0] - dev_data = ds_list[2] - test_data = ds_list[1] + dev_data = ds_list[1] + test_data = ds_list[2] print(tag_v.word2idx) if g_args.task in ['pos', 'ner']: @@ -145,14 +151,26 @@ def train(): } metric_key, metric = metrics[g_args.task] device = 'cuda' if torch.cuda.is_available() else 'cpu' - ex_param = [x for x in model.parameters( - ) if x.requires_grad and x.size(0) != len(word_v)] - optim_cfg = [{'params': model.enc.embedding.parameters(), 'lr': g_args.lr*0.1}, - {'params': ex_param, 'lr': g_args.lr, 'weight_decay': g_args.w_decay}, ] - trainer = FN.Trainer(train_data=train_data, model=model, optimizer=torch.optim.Adam(optim_cfg), loss=loss, - batch_size=g_args.bsz, n_epochs=g_args.ep, print_every=10, dev_data=dev_data, metrics=metric, - metric_key=metric_key, validate_every=3000, save_path=g_args.log, use_tqdm=False, - device=device, callbacks=[MyCallback()]) + + params = [(x,y) for x,y in list(model.named_parameters()) if y.requires_grad and y.size(0) != len(word_v)] + no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] + print([n for n,p in params]) + optim_cfg = [ + #{'params': model.enc.embedding.parameters(), 'lr': g_args.lr*0.1}, + {'params': [p for n, p in params if not any(nd in n for nd in no_decay)], 'lr': g_args.lr, 'weight_decay': 1.0*g_args.w_decay}, + {'params': [p for n, p in params if any(nd in n for nd in no_decay)], 'lr': g_args.lr, 'weight_decay': 0.0*g_args.w_decay} + ] + + print(model) + trainer = FN.Trainer(model=model, train_data=train_data, dev_data=dev_data, + loss=loss, metrics=metric, metric_key=metric_key, + optimizer=torch.optim.Adam(optim_cfg), + n_epochs=g_args.ep, batch_size=g_args.bsz, print_every=100, validate_every=1000, + device=device, + use_tqdm=False, prefetch=False, + save_path=g_args.log, + sampler=FN.BucketSampler(100, g_args.bsz, C.INPUT_LEN), + callbacks=[MyCallback()]) trainer.train() tester = FN.Tester(data=test_data, model=model, metrics=metric, @@ -195,12 +213,12 @@ def main(): 'init_embed': (None, 300), 'num_cls': None, 'hidden_size': g_args.hidden, - 'num_layers': 4, + 'num_layers': 2, 'num_head': g_args.nhead, 'head_dim': g_args.hdim, 'max_len': MAX_LEN, - 'cls_hidden_size': 600, - 'emb_dropout': 0.3, + 'cls_hidden_size': 200, + 'emb_dropout': g_args.drop, 'dropout': g_args.drop, } run_select[g_args.mode.lower()]() From 205f8fe6af65c0d6716839eef7ed8b2da2f00b34 Mon Sep 17 00:00:00 2001 From: QipengGuo Date: Sun, 30 Jun 2019 17:45:09 +0000 Subject: [PATCH 02/27] set random seed --- reproduction/Star_transformer/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reproduction/Star_transformer/train.py b/reproduction/Star_transformer/train.py index edb7fe27..f125ed02 100644 --- a/reproduction/Star_transformer/train.py +++ b/reproduction/Star_transformer/train.py @@ -109,8 +109,8 @@ class MyCallback(FN.core.callback.Callback): def train(): - #seed = set_rng_seeds(1234) - seed = set_rng_seeds(np.random.randint(65536)) + seed = set_rng_seeds(28848) + #seed = set_rng_seeds(np.random.randint(65536)) print('RNG SEED {}'.format(seed)) print('loading data') ds_list, word_v, tag_v = g_datasets['{}-{}'.format( From 204bb067697c37ee077c05bf324d92a0c35a3f59 Mon Sep 17 00:00:00 2001 From: QipengGuo Date: Mon, 1 Jul 2019 02:24:18 +0000 Subject: [PATCH 03/27] fix random process --- reproduction/Star_transformer/train.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/reproduction/Star_transformer/train.py b/reproduction/Star_transformer/train.py index f125ed02..480748df 100644 --- a/reproduction/Star_transformer/train.py +++ b/reproduction/Star_transformer/train.py @@ -1,4 +1,6 @@ from util import get_argparser, set_gpu, set_rng_seeds, add_model_args +seed = set_rng_seeds(15360) +print('RNG SEED {}'.format(seed)) from datasets import load_seqtag, load_sst, load_snli, EmbedLoader, MAX_LEN import torch.nn as nn import torch @@ -109,9 +111,6 @@ class MyCallback(FN.core.callback.Callback): def train(): - seed = set_rng_seeds(28848) - #seed = set_rng_seeds(np.random.randint(65536)) - print('RNG SEED {}'.format(seed)) print('loading data') ds_list, word_v, tag_v = g_datasets['{}-{}'.format( g_args.ds, g_args.task)]() From 32917dac7f8e87adbad72c69d3b269dd48606b8a Mon Sep 17 00:00:00 2001 From: yunfan Date: Thu, 27 Jun 2019 21:56:49 +0800 Subject: [PATCH 04/27] add dpcnn --- .../text_classification/model/dpcnn.py | 92 ++++++++++++++++++- .../text_classification/train_dpcnn.py | 80 ++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) diff --git a/reproduction/text_classification/model/dpcnn.py b/reproduction/text_classification/model/dpcnn.py index f87f5c14..2da7b3e5 100644 --- a/reproduction/text_classification/model/dpcnn.py +++ b/reproduction/text_classification/model/dpcnn.py @@ -1 +1,91 @@ -# TODO \ No newline at end of file +import torch +import torch.nn as nn +from fastNLP.modules.utils import get_embeddings +from fastNLP.core import Const as C + +class DPCNN(nn.Module): + def __init__(self, init_embed, num_cls, n_filters=256, kernel_size=3, n_layers=7, embed_dropout=0.1, dropout=0.1): + super().__init__() + self.region_embed = RegionEmbedding(init_embed, out_dim=n_filters, kernel_sizes=[3, 5, 9]) + embed_dim = self.region_embed.embedding_dim + self.conv_list = nn.ModuleList() + for i in range(n_layers): + self.conv_list.append(nn.Sequential( + nn.ReLU(), + nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + )) + self.pool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1) + self.embed_drop = nn.Dropout(embed_dropout) + self.classfier = nn.Sequential( + nn.Dropout(dropout), + nn.Linear(n_filters, num_cls), + ) + self.reset_parameters() + + def reset_parameters(self): + for m in self.modules(): + if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)): + nn.init.normal_(m.weight, mean=0, std=0.01) + if m.bias is not None: + nn.init.normal_(m.bias, mean=0, std=0.01) + + def forward(self, words, seq_len=None): + words = words.long() + # get region embeddings + x = self.region_embed(words) + x = self.embed_drop(x) + + # not pooling on first conv + x = self.conv_list[0](x) + x + for conv in self.conv_list[1:]: + x = self.pool(x) + x = conv(x) + x + + # B, C, L => B, C + x, _ = torch.max(x, dim=2) + x = self.classfier(x) + return {C.OUTPUT: x} + + def predict(self, words, seq_len=None): + x = self.forward(words, seq_len)[C.OUTPUT] + return {C.OUTPUT: torch.argmax(x, 1)} + + +class RegionEmbedding(nn.Module): + def __init__(self, init_embed, out_dim=300, kernel_sizes=None): + super().__init__() + if kernel_sizes is None: + kernel_sizes = [5, 9] + assert isinstance(kernel_sizes, list), 'kernel_sizes should be List(int)' + self.embed = get_embeddings(init_embed) + try: + embed_dim = self.embed.embedding_dim + except Exception: + embed_dim = self.embed.embed_size + self.region_embeds = nn.ModuleList() + for ksz in kernel_sizes: + self.region_embeds.append(nn.Sequential( + nn.Conv1d(embed_dim, embed_dim, ksz, padding=ksz // 2), + )) + self.linears = nn.ModuleList([nn.Conv1d(embed_dim, out_dim, 1) + for _ in range(len(kernel_sizes) + 1)]) + self.embedding_dim = embed_dim + + def forward(self, x): + x = self.embed(x) + x = x.transpose(1, 2) + # B, C, L + out = self.linears[0](x) + for conv, fc in zip(self.region_embeds, self.linears[1:]): + conv_i = conv(x) + out = out + fc(conv_i) + # B, C, L + return out + + +if __name__ == '__main__': + x = torch.randint(0, 10000, size=(5, 15), dtype=torch.long) + model = DPCNN((10000, 300), 20) + y = model(x) + print(y.size(), y.mean(1), y.std(1)) diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index e69de29b..13ff4fc1 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -0,0 +1,80 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +from fastNLP.core.const import Const as C +from fastNLP.core import LRScheduler +import torch.nn as nn +from fastNLP.io.dataset_loader import SSTLoader +from reproduction.text_classification.model.dpcnn import DPCNN +from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.core.trainer import Trainer +from torch.optim import SGD +import torch.cuda +from torch.optim.lr_scheduler import CosineAnnealingLR + +##hyper +class Config(): + model_dir_or_name="en-base-uncased" + embedding_grad= False, + train_epoch= 30 + batch_size = 100 + num_classes=5 + task= "SST" + datadir = '/remote-home/yfshao/workdir/datasets/SST' + datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} + lr=1e-3 + def __init__(self): + self.datapath = {k:os.path.join(self.datadir, v) + for k, v in self.datafile.items()} + +ops=Config() + + +##1.task相关信息：利用dataloader载入dataInfo +datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds='train') + +print(len(datainfo.datasets['train'])) +print(len(datainfo.datasets['dev'])) + + +## 2.或直接复用fastNLP的模型 +vocab = datainfo.vocabs['words'] + +# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) +embedding = StaticEmbedding(vocab) +print(len(vocab)) +print(len(datainfo.vocabs['target'])) +model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) + +## 3. 声明loss,metric,optimizer +loss=CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) +metric=AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) +optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], + lr=ops.lr, momentum=0.9, weight_decay=0) + +callbacks = [] +callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) + +device = 'cuda:0' if torch.cuda.is_available() else 'cpu' +print(device) + +for ds in datainfo.datasets.values(): + ds.apply_field(len, C.INPUT, C.INPUT_LEN) + ds.set_input(C.INPUT, C.INPUT_LEN) + ds.set_target(C.TARGET) + +## 4.定义train方法 +def train(model,datainfo,loss,metrics,optimizer,num_epochs=ops.train_epoch): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=[metrics], dev_data=datainfo.datasets['dev'], device=device, + check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, + n_epochs=num_epochs) + print(trainer.train()) + + +if __name__=="__main__": + train(model,datainfo,loss,metric,optimizer) \ No newline at end of file From f1adb0f9156357944183fdb692d5206aad7e3b0d Mon Sep 17 00:00:00 2001 From: yunfan Date: Thu, 4 Jul 2019 13:56:37 +0800 Subject: [PATCH 05/27] add ID-CNN --- .../ner/model/dilated_cnn.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 reproduction/seqence_labelling/ner/model/dilated_cnn.py diff --git a/reproduction/seqence_labelling/ner/model/dilated_cnn.py b/reproduction/seqence_labelling/ner/model/dilated_cnn.py new file mode 100644 index 00000000..cd2fa64b --- /dev/null +++ b/reproduction/seqence_labelling/ner/model/dilated_cnn.py @@ -0,0 +1,111 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from fastNLP.modules.decoder import ConditionalRandomField +from fastNLP.modules.encoder import Embedding +from fastNLP.core.utils import seq_len_to_mask +from fastNLP.core.const import Const as C + + +class IDCNN(nn.Module): + def __init__(self, init_embed, char_embed, + num_cls, + repeats, num_layers, num_filters, kernel_size, + use_crf=False, use_projection=False, block_loss=False, + input_dropout=0.3, hidden_dropout=0.2, inner_dropout=0.0): + super(IDCNN, self).__init__() + self.word_embeddings = Embedding(init_embed) + self.char_embeddings = Embedding(char_embed) + embedding_size = self.word_embeddings.embedding_dim + \ + self.char_embeddings.embedding_dim + + self.conv0 = nn.Sequential( + nn.Conv1d(in_channels=embedding_size, + out_channels=num_filters, + kernel_size=kernel_size, + stride=1, dilation=1, + padding=kernel_size//2, + bias=True), + nn.ReLU(), + ) + + block = [] + for layer_i in range(num_layers): + dilated = 2 ** layer_i + block.append(nn.Conv1d( + in_channels=num_filters, + out_channels=num_filters, + kernel_size=kernel_size, + stride=1, dilation=dilated, + padding=(kernel_size//2) * dilated, + bias=True)) + block.append(nn.ReLU()) + self.block = nn.Sequential(*block) + + if use_projection: + self.projection = nn.Sequential( + nn.Conv1d( + in_channels=num_filters, + out_channels=num_filters//2, + kernel_size=1, + bias=True), + nn.ReLU(),) + encode_dim = num_filters // 2 + else: + self.projection = None + encode_dim = num_filters + + self.input_drop = nn.Dropout(input_dropout) + self.hidden_drop = nn.Dropout(hidden_dropout) + self.inner_drop = nn.Dropout(inner_dropout) + self.repeats = repeats + self.out_fc = nn.Conv1d( + in_channels=encode_dim, + out_channels=num_cls, + kernel_size=1, + bias=True) + self.crf = ConditionalRandomField( + num_tags=num_cls) if use_crf else None + self.block_loss = block_loss + + def forward(self, words, chars, seq_len, target=None): + e1 = self.word_embeddings(words) + e2 = self.char_embeddings(chars) + x = torch.cat((e1, e2), dim=-1) # b,l,h + mask = seq_len_to_mask(seq_len) + + x = x.transpose(1, 2) # b,h,l + last_output = self.conv0(x) + output = [] + for repeat in range(self.repeats): + last_output = self.block(last_output) + hidden = self.projection(last_output) if self.projection is not None else last_output + output.append(self.out_fc(hidden)) + + def compute_loss(y, t, mask): + if self.crf is not None and target is not None: + loss = self.crf(y, t, mask) + else: + t.masked_fill_(mask == 0, -100) + loss = F.cross_entropy(y, t, ignore_index=-100) + return loss + + if self.block_loss: + losses = [compute_loss(o, target, mask) for o in output] + loss = sum(losses) + else: + loss = compute_loss(output[-1], target, mask) + + scores = output[-1] + if self.crf is not None: + pred = self.crf.viterbi_decode(scores, target, mask) + else: + pred = scores.max(1)[1] * mask.long() + + return { + C.LOSS: loss, + C.OUTPUT: pred, + } + + def predict(self, words, chars, seq_len): + return self.forward(words, chars, seq_len)[C.OUTPUT] From 372496ca32a5015c19563c82ca1b29d6071a6e81 Mon Sep 17 00:00:00 2001 From: yunfan Date: Thu, 4 Jul 2019 14:03:53 +0800 Subject: [PATCH 06/27] update model & dataloader in text_classification --- .../text_classification/data/IMDBLoader.py | 82 +++++++++ .../text_classification/data/yelpLoader.py | 164 +++++++++++++++--- .../text_classification/train_dpcnn.py | 97 +++++++---- 3 files changed, 283 insertions(+), 60 deletions(-) create mode 100644 reproduction/text_classification/data/IMDBLoader.py diff --git a/reproduction/text_classification/data/IMDBLoader.py b/reproduction/text_classification/data/IMDBLoader.py new file mode 100644 index 00000000..2df87e26 --- /dev/null +++ b/reproduction/text_classification/data/IMDBLoader.py @@ -0,0 +1,82 @@ +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader +from fastNLP.core.vocabulary import VocabularyOption +from fastNLP.io.base_loader import DataSetLoader, DataInfo +from typing import Union, Dict, List, Iterator +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP import Vocabulary +from fastNLP import Const +# from reproduction.utils import check_dataloader_paths +from functools import partial + + +class IMDBLoader(DataSetLoader): + """ + 读取IMDB数据集，DataSet包含以下fields: + + words: list(str), 需要分类的文本 + target: str, 文本的标签 + + + """ + + def __init__(self): + super(IMDBLoader, self).__init__() + + def _load(self, path): + dataset = DataSet() + with open(path, 'r', encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split('\t') + target = parts[0] + words = parts[1].split() + dataset.append(Instance(words=words, target=target)) + if len(dataset) == 0: + raise RuntimeError(f"{path} has no valid data.") + + return dataset + + def process(self, + paths: Union[str, Dict[str, str]], + src_vocab_opt: VocabularyOption = None, + tgt_vocab_opt: VocabularyOption = None, + src_embed_opt: EmbeddingOption = None): + + # paths = check_dataloader_paths(paths) + datasets = {} + info = DataInfo() + for name, path in paths.items(): + dataset = self.load(path) + datasets[name] = dataset + + datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False) + + src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) + src_vocab.from_dataset(datasets['train'], field_name='words') + # src_vocab.from_dataset(datasets['train'], datasets["dev"], datasets["test"], field_name='words') + src_vocab.index_dataset(*datasets.values(), field_name='words') + + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) + tgt_vocab.from_dataset(datasets['train'], field_name='target') + tgt_vocab.index_dataset(*datasets.values(), field_name='target') + + info.vocabs = { + "words": src_vocab, + "target": tgt_vocab + } + + info.datasets = datasets + + if src_embed_opt is not None: + embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) + info.embeddings['words'] = embed + + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + + return info diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index c47d48fd..63605ecf 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -1,4 +1,6 @@ import ast +import csv +from typing import Iterable from fastNLP import DataSet, Instance, Vocabulary from fastNLP.core.vocabulary import VocabularyOption from fastNLP.io import JsonLoader @@ -10,11 +12,34 @@ from reproduction.Star_transformer.datasets import EmbedLoader from reproduction.utils import check_dataloader_paths +def clean_str(sentence, char_lower=False): + """ + heavily borrowed from github + https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb + :param sentence: is a str + :return: + """ + if char_lower: + sentence = sentence.lower() + import re + nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') + words = sentence.split() + words_collection = [] + for word in words: + if word in ['-lrb-', '-rrb-', '', '-r', '-l', 'b-']: + continue + tt = nonalpnum.split(word) + t = ''.join(tt) + if t != '': + words_collection.append(t) + + return words_collection + + class yelpLoader(JsonLoader): - """ 读取Yelp数据集, DataSet包含fields: - + review_id: str, 22 character unique review id user_id: str, 22 character unique user id business_id: str, 22 character business id @@ -24,23 +49,25 @@ class yelpLoader(JsonLoader): date: str, date formatted YYYY-MM-DD words: list(str), 需要分类的文本 target: str, 文本的标签 - + 数据来源: https://www.yelp.com/dataset/download - + :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` """ - - def __init__(self, fine_grained=False): + + def __init__(self, fine_grained=False, lower=False): super(yelpLoader, self).__init__() tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral', - '4.0': 'positive', '5.0': 'very positive'} + '4.0': 'positive', '5.0': 'very positive'} if not fine_grained: tag_v['1.0'] = tag_v['2.0'] tag_v['5.0'] = tag_v['4.0'] self.fine_grained = fine_grained self.tag_v = tag_v - - def _load(self, path): + self.lower = lower + + ''' + def _load_json(self, path): ds = DataSet() for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): d = ast.literal_eval(d) @@ -49,20 +76,113 @@ class yelpLoader(JsonLoader): ds.append(Instance(**d)) return ds - def process(self, paths: Union[str, Dict[str, str]], vocab_opt: VocabularyOption = None, - embed_opt: EmbeddingOption = None): + def _load_yelp2015_broken(self,path): + ds = DataSet() + with open (path,encoding='ISO 8859-1') as f: + row=f.readline() + all_count=0 + exp_count=0 + while row: + row=row.split("\t\t") + all_count+=1 + if len(row)>=3: + words=row[-1].split() + try: + target=self.tag_v[str(row[-2])+".0"] + ds.append(Instance(words=words, target=target)) + except KeyError: + exp_count+=1 + else: + exp_count+=1 + row = f.readline() + print("error sample count:",exp_count) + print("all count:",all_count) + return ds + ''' + + def _load(self, path): + ds = DataSet() + csv_reader = csv.reader(open(path, encoding='utf-8')) + all_count = 0 + real_count = 0 + for row in csv_reader: + all_count += 1 + if len(row) == 2: + target = self.tag_v[row[0] + ".0"] + words = clean_str(row[1], self.lower) + if len(words) != 0: + ds.append(Instance(words=words, target=target)) + real_count += 1 + print("all count:", all_count) + print("real count:", real_count) + return ds + + def process(self, paths: Union[str, Dict[str, str]], + train_ds: Iterable[str] = None, + src_vocab_op: VocabularyOption = None, + tgt_vocab_op: VocabularyOption = None, + embed_opt: EmbeddingOption = None, + char_level_op=False): paths = check_dataloader_paths(paths) datasets = {} - info = DataInfo() - vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) - for name, path in paths.items(): - dataset = self.load(path) - datasets[name] = dataset - vocab.from_dataset(dataset, field_name="words") - info.vocabs = vocab - info.datasets = datasets - if embed_opt is not None: - embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) - info.embeddings['words'] = embed + info = DataInfo(datasets=self.load(paths)) + src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) + _train_ds = [info.datasets[name] + for name in train_ds] if train_ds else info.datasets.values() + + # vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) + # for name, path in paths.items(): + # dataset = self.load(path) + # datasets[name] = dataset + # vocab.from_dataset(dataset, field_name="words") + # info.vocabs = vocab + # info.datasets = datasets + + def wordtochar(words): + chars = [] + for word in words: + word = word.lower() + for char in word: + chars.append(char) + return chars + + input_name, target_name = 'words', 'target' + info.vocabs = {} + # 就分隔为char形式 + if char_level_op: + for dataset in info.datasets.values(): + dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') + # if embed_opt is not None: + # embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) + # info.embeddings['words'] = embed + else: + src_vocab.from_dataset(*_train_ds, field_name=input_name) + src_vocab.index_dataset(*info.datasets.values(), field_name=input_name, new_field_name=input_name) + info.vocabs[input_name] = src_vocab + + tgt_vocab.from_dataset(*_train_ds, field_name=target_name) + tgt_vocab.index_dataset( + *info.datasets.values(), + field_name=target_name, new_field_name=target_name) + info.vocabs[target_name] = tgt_vocab + return info + +if __name__ == "__main__": + testloader = yelpLoader() + # datapath = {"train": "/remote-home/ygwang/yelp_full/train.csv", + # "test": "/remote-home/ygwang/yelp_full/test.csv"} + # datapath={"train": "/remote-home/ygwang/yelp_full/test.csv"} + datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv", + "test": "/remote-home/ygwang/yelp_polarity/test.csv"} + datainfo = testloader.process(datapath, char_level_op=True) + + len_count = 0 + for instance in datainfo.datasets["train"]: + len_count += len(instance["chars"]) + + ave_len = len_count / len(datainfo.datasets["train"]) + print(ave_len) \ No newline at end of file diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index 13ff4fc1..bf243ffb 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -1,65 +1,83 @@ # 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 + +from torch.optim.lr_scheduler import CosineAnnealingLR +import torch.cuda +from torch.optim import SGD +from fastNLP.core.trainer import Trainer +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding +from reproduction.text_classification.model.dpcnn import DPCNN +from .data.yelpLoader import yelpLoader +from fastNLP.io.dataset_loader import SSTLoader +import torch.nn as nn +from fastNLP.core import LRScheduler +from fastNLP.core.const import Const as C +import sys import os os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" -from fastNLP.core.const import Const as C -from fastNLP.core import LRScheduler -import torch.nn as nn -from fastNLP.io.dataset_loader import SSTLoader -from reproduction.text_classification.model.dpcnn import DPCNN -from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding -from fastNLP import CrossEntropyLoss, AccuracyMetric -from fastNLP.core.trainer import Trainer -from torch.optim import SGD -import torch.cuda -from torch.optim.lr_scheduler import CosineAnnealingLR +sys.path.append('../..') + + +# hyper -##hyper class Config(): - model_dir_or_name="en-base-uncased" - embedding_grad= False, - train_epoch= 30 + model_dir_or_name = "en-base-uncased" + embedding_grad = False, + train_epoch = 30 batch_size = 100 - num_classes=5 - task= "SST" - datadir = '/remote-home/yfshao/workdir/datasets/SST' - datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} - lr=1e-3 + num_classes = 2 + task = "yelp_p" + #datadir = '/remote-home/yfshao/workdir/datasets/SST' + datadir = '/remote-home/ygwang/yelp_polarity' + #datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} + datafile = {"train": "train.csv", "test": "test.csv"} + lr = 1e-3 + def __init__(self): - self.datapath = {k:os.path.join(self.datadir, v) + self.datapath = {k: os.path.join(self.datadir, v) for k, v in self.datafile.items()} -ops=Config() + +ops = Config() -##1.task相关信息：利用dataloader载入dataInfo -datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds='train') +# 1.task相关信息：利用dataloader载入dataInfo +#datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) +datainfo = yelpLoader(fine_grained=True, lower=True).process( + paths=ops.datapath, train_ds=['train']) print(len(datainfo.datasets['train'])) -print(len(datainfo.datasets['dev'])) +print(len(datainfo.datasets['test'])) -## 2.或直接复用fastNLP的模型 -vocab = datainfo.vocabs['words'] +# 2.或直接复用fastNLP的模型 +vocab = datainfo.vocabs['words'] # embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) -embedding = StaticEmbedding(vocab) +#embedding = StaticEmbedding(vocab) +embedding = StaticEmbedding( + vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) + print(len(vocab)) print(len(datainfo.vocabs['target'])) + model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) -## 3. 声明loss,metric,optimizer -loss=CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) -metric=AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) -optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], - lr=ops.lr, momentum=0.9, weight_decay=0) + +# 3. 声明loss,metric,optimizer +loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) +metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) +optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], + lr=ops.lr, momentum=0.9, weight_decay=0) callbacks = [] callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + print(device) for ds in datainfo.datasets.values(): @@ -67,14 +85,17 @@ for ds in datainfo.datasets.values(): ds.set_input(C.INPUT, C.INPUT_LEN) ds.set_target(C.TARGET) -## 4.定义train方法 -def train(model,datainfo,loss,metrics,optimizer,num_epochs=ops.train_epoch): + +# 4.定义train方法 +def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch): trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=[metrics], dev_data=datainfo.datasets['dev'], device=device, + metrics=[metrics], + dev_data=datainfo.datasets['test'], device=device, check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, n_epochs=num_epochs) + print(trainer.train()) -if __name__=="__main__": - train(model,datainfo,loss,metric,optimizer) \ No newline at end of file +if __name__ == "__main__": + train(model, datainfo, loss, metric, optimizer) From c5fc29dfef2a48729aafed42ccf6309aa5bee17e Mon Sep 17 00:00:00 2001 From: yunfan Date: Sat, 6 Jul 2019 13:15:40 +0800 Subject: [PATCH 07/27] -update DPCNN & train script -use spacy tokenizer for yelp data -add set_rng_seed --- fastNLP/modules/aggregator/attention.py | 9 +- .../text_classification/data/yelpLoader.py | 18 +++- .../text_classification/model/dpcnn.py | 22 +++-- .../text_classification/train_dpcnn.py | 83 ++++++++++++------- .../text_classification/utils/util_init.py | 11 +++ 5 files changed, 94 insertions(+), 49 deletions(-) create mode 100644 reproduction/text_classification/utils/util_init.py diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index 4101b033..2bee7f2e 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -19,7 +19,7 @@ class DotAttention(nn.Module): 补上文档 """ - def __init__(self, key_size, value_size, dropout=0): + def __init__(self, key_size, value_size, dropout=0.0): super(DotAttention, self).__init__() self.key_size = key_size self.value_size = value_size @@ -37,7 +37,7 @@ class DotAttention(nn.Module): """ output = torch.matmul(Q, K.transpose(1, 2)) / self.scale if mask_out is not None: - output.masked_fill_(mask_out, -1e8) + output.masked_fill_(mask_out, -1e18) output = self.softmax(output) output = self.drop(output) return torch.matmul(output, V) @@ -67,9 +67,8 @@ class MultiHeadAttention(nn.Module): self.k_in = nn.Linear(input_size, in_size) self.v_in = nn.Linear(input_size, in_size) # follow the paper, do not apply dropout within dot-product - self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=0) + self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=dropout) self.out = nn.Linear(value_size * num_head, input_size) - self.drop = TimestepDropout(dropout) self.reset_parameters() def reset_parameters(self): @@ -105,7 +104,7 @@ class MultiHeadAttention(nn.Module): # concat all heads, do output linear atte = atte.permute(1, 2, 0, 3).contiguous().view(batch, sq, -1) - output = self.drop(self.out(atte)) + output = self.out(atte) return output diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index 63605ecf..d97f9399 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -8,11 +8,20 @@ from fastNLP.io.base_loader import DataInfo from fastNLP.io.embed_loader import EmbeddingOption from fastNLP.io.file_reader import _read_json from typing import Union, Dict -from reproduction.Star_transformer.datasets import EmbedLoader from reproduction.utils import check_dataloader_paths -def clean_str(sentence, char_lower=False): +def get_tokenizer(): + try: + import spacy + en = spacy.load('en') + print('use spacy tokenizer') + return lambda x: [w.text for w in en.tokenizer(x)] + except Exception as e: + print('use raw tokenizer') + return lambda x: x.split() + +def clean_str(sentence, tokenizer, char_lower=False): """ heavily borrowed from github https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb @@ -23,7 +32,7 @@ def clean_str(sentence, char_lower=False): sentence = sentence.lower() import re nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') - words = sentence.split() + words = tokenizer(sentence) words_collection = [] for word in words: if word in ['-lrb-', '-rrb-', '', '-r', '-l', 'b-']: @@ -65,6 +74,7 @@ class yelpLoader(JsonLoader): self.fine_grained = fine_grained self.tag_v = tag_v self.lower = lower + self.tokenizer = get_tokenizer() ''' def _load_json(self, path): @@ -109,7 +119,7 @@ class yelpLoader(JsonLoader): all_count += 1 if len(row) == 2: target = self.tag_v[row[0] + ".0"] - words = clean_str(row[1], self.lower) + words = clean_str(row[1], self.tokenizer, self.lower) if len(words) != 0: ds.append(Instance(words=words, target=target)) real_count += 1 diff --git a/reproduction/text_classification/model/dpcnn.py b/reproduction/text_classification/model/dpcnn.py index 2da7b3e5..dafe62bc 100644 --- a/reproduction/text_classification/model/dpcnn.py +++ b/reproduction/text_classification/model/dpcnn.py @@ -3,22 +3,27 @@ import torch.nn as nn from fastNLP.modules.utils import get_embeddings from fastNLP.core import Const as C + class DPCNN(nn.Module): - def __init__(self, init_embed, num_cls, n_filters=256, kernel_size=3, n_layers=7, embed_dropout=0.1, dropout=0.1): + def __init__(self, init_embed, num_cls, n_filters=256, + kernel_size=3, n_layers=7, embed_dropout=0.1, cls_dropout=0.1): super().__init__() - self.region_embed = RegionEmbedding(init_embed, out_dim=n_filters, kernel_sizes=[3, 5, 9]) + self.region_embed = RegionEmbedding( + init_embed, out_dim=n_filters, kernel_sizes=[1, 3, 5]) embed_dim = self.region_embed.embedding_dim self.conv_list = nn.ModuleList() for i in range(n_layers): self.conv_list.append(nn.Sequential( nn.ReLU(), - nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), - nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + nn.Conv1d(n_filters, n_filters, kernel_size, + padding=kernel_size//2), + nn.Conv1d(n_filters, n_filters, kernel_size, + padding=kernel_size//2), )) self.pool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1) self.embed_drop = nn.Dropout(embed_dropout) self.classfier = nn.Sequential( - nn.Dropout(dropout), + nn.Dropout(cls_dropout), nn.Linear(n_filters, num_cls), ) self.reset_parameters() @@ -57,7 +62,8 @@ class RegionEmbedding(nn.Module): super().__init__() if kernel_sizes is None: kernel_sizes = [5, 9] - assert isinstance(kernel_sizes, list), 'kernel_sizes should be List(int)' + assert isinstance( + kernel_sizes, list), 'kernel_sizes should be List(int)' self.embed = get_embeddings(init_embed) try: embed_dim = self.embed.embedding_dim @@ -69,14 +75,14 @@ class RegionEmbedding(nn.Module): nn.Conv1d(embed_dim, embed_dim, ksz, padding=ksz // 2), )) self.linears = nn.ModuleList([nn.Conv1d(embed_dim, out_dim, 1) - for _ in range(len(kernel_sizes) + 1)]) + for _ in range(len(kernel_sizes))]) self.embedding_dim = embed_dim def forward(self, x): x = self.embed(x) x = x.transpose(1, 2) # B, C, L - out = self.linears[0](x) + out = 0 for conv, fc in zip(self.region_embeds, self.linears[1:]): conv_i = conv(x) out = out + fc(conv_i) diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index bf243ffb..294a0742 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -1,40 +1,44 @@ # 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 -from torch.optim.lr_scheduler import CosineAnnealingLR import torch.cuda +from fastNLP.core.utils import cache_results from torch.optim import SGD +from torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR from fastNLP.core.trainer import Trainer from fastNLP import CrossEntropyLoss, AccuracyMetric from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding from reproduction.text_classification.model.dpcnn import DPCNN -from .data.yelpLoader import yelpLoader -from fastNLP.io.dataset_loader import SSTLoader +from data.yelpLoader import yelpLoader import torch.nn as nn from fastNLP.core import LRScheduler from fastNLP.core.const import Const as C -import sys +from fastNLP.core.vocabulary import VocabularyOption +from utils.util_init import set_rng_seeds import os os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" -sys.path.append('../..') - # hyper class Config(): - model_dir_or_name = "en-base-uncased" - embedding_grad = False, + seed = 12345 + model_dir_or_name = "dpcnn-yelp-p" + embedding_grad = True train_epoch = 30 batch_size = 100 num_classes = 2 task = "yelp_p" #datadir = '/remote-home/yfshao/workdir/datasets/SST' - datadir = '/remote-home/ygwang/yelp_polarity' + datadir = '/remote-home/yfshao/workdir/datasets/yelp_polarity' #datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} datafile = {"train": "train.csv", "test": "test.csv"} lr = 1e-3 + src_vocab_op = VocabularyOption() + embed_dropout = 0.3 + cls_dropout = 0.1 + weight_decay = 1e-4 def __init__(self): self.datapath = {k: os.path.join(self.datadir, v) @@ -43,15 +47,23 @@ class Config(): ops = Config() +set_rng_seeds(ops.seed) +print('RNG SEED: {}'.format(ops.seed)) # 1.task相关信息：利用dataloader载入dataInfo #datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) -datainfo = yelpLoader(fine_grained=True, lower=True).process( - paths=ops.datapath, train_ds=['train']) -print(len(datainfo.datasets['train'])) -print(len(datainfo.datasets['test'])) - +@cache_results(ops.model_dir_or_name+'-data-cache') +def load_data(): + datainfo = yelpLoader(fine_grained=True, lower=True).process( + paths=ops.datapath, train_ds=['train'], src_vocab_op=ops.src_vocab_op) + for ds in datainfo.datasets.values(): + ds.apply_field(len, C.INPUT, C.INPUT_LEN) + ds.set_input(C.INPUT, C.INPUT_LEN) + ds.set_target(C.TARGET) + return datainfo + +datainfo = load_data() # 2.或直接复用fastNLP的模型 @@ -59,43 +71,50 @@ vocab = datainfo.vocabs['words'] # embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) #embedding = StaticEmbedding(vocab) embedding = StaticEmbedding( - vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) + vocab, model_dir_or_name='en-word2vec-300', requires_grad=ops.embedding_grad, + normalize=False +) + +print(len(datainfo.datasets['train'])) +print(len(datainfo.datasets['test'])) +print(datainfo.datasets['train'][0]) print(len(vocab)) print(len(datainfo.vocabs['target'])) -model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) - +model = DPCNN(init_embed=embedding, num_cls=ops.num_classes, + embed_dropout=ops.embed_dropout, cls_dropout=ops.cls_dropout) +print(model) # 3. 声明loss,metric,optimizer loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], - lr=ops.lr, momentum=0.9, weight_decay=0) + lr=ops.lr, momentum=0.9, weight_decay=ops.weight_decay) callbacks = [] callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) +# callbacks.append +# LRScheduler(LambdaLR(optimizer, lambda epoch: ops.lr if epoch < +# ops.train_epoch * 0.8 else ops.lr * 0.1)) +# ) + +# callbacks.append( +# FitlogCallback(data=datainfo.datasets, verbose=1) +# ) device = 'cuda:0' if torch.cuda.is_available() else 'cpu' print(device) -for ds in datainfo.datasets.values(): - ds.apply_field(len, C.INPUT, C.INPUT_LEN) - ds.set_input(C.INPUT, C.INPUT_LEN) - ds.set_target(C.TARGET) - - # 4.定义train方法 -def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch): - trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=[metrics], - dev_data=datainfo.datasets['test'], device=device, - check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, - n_epochs=num_epochs) +trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=[metric], + dev_data=datainfo.datasets['test'], device=device, + check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, + n_epochs=ops.train_epoch, num_workers=4) - print(trainer.train()) if __name__ == "__main__": - train(model, datainfo, loss, metric, optimizer) + print(trainer.train()) diff --git a/reproduction/text_classification/utils/util_init.py b/reproduction/text_classification/utils/util_init.py new file mode 100644 index 00000000..fcb8fffb --- /dev/null +++ b/reproduction/text_classification/utils/util_init.py @@ -0,0 +1,11 @@ +import numpy +import torch +import random + + +def set_rng_seeds(seed): + random.seed(seed) + numpy.random.seed(seed) + torch.random.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + # print('RNG_SEED {}'.format(seed)) From b02a91ea0117e49707476ce599159723efe36595 Mon Sep 17 00:00:00 2001 From: wyg <1505116161@qq.com> Date: Sat, 6 Jul 2019 15:36:45 +0800 Subject: [PATCH 08/27] =?UTF-8?q?[add]=20dataloader:=20yelp/sst2/IMDB/MTL1?= =?UTF-8?q?6=20[add]=20model:=20char=5Fcnn=20dpcnn=20[test]=20train=5Fchar?= =?UTF-8?q?=5Fcnn=20train=5Fdpcnn,dataloader=E7=9A=84=E5=9D=87=E5=9C=A8?= =?UTF-8?q?=E8=87=AA=E5=B7=B1=E7=9A=84main=E6=96=B9=E6=B3=95=E5=86=85?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../text_classification/data/IMDBLoader.py | 107 +++++++++ .../text_classification/data/sstLoader.py | 98 +++++++++ .../text_classification/data/yelpLoader.py | 184 +++++++++++++--- .../text_classification/model/char_cnn.py | 91 +++++++- .../text_classification/model/dpcnn.py | 112 +++++++++- .../text_classification/train_char_cnn.py | 206 ++++++++++++++++++ .../text_classification/train_dpcnn.py | 101 +++++++++ 7 files changed, 868 insertions(+), 31 deletions(-) create mode 100644 reproduction/text_classification/data/IMDBLoader.py create mode 100644 reproduction/text_classification/data/sstLoader.py diff --git a/reproduction/text_classification/data/IMDBLoader.py b/reproduction/text_classification/data/IMDBLoader.py new file mode 100644 index 00000000..cb422524 --- /dev/null +++ b/reproduction/text_classification/data/IMDBLoader.py @@ -0,0 +1,107 @@ +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader +from fastNLP.core.vocabulary import VocabularyOption +from fastNLP.io.base_loader import DataSetLoader, DataInfo +from typing import Union, Dict, List, Iterator +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP import Vocabulary +from fastNLP import Const +# from reproduction.utils import check_dataloader_paths +from functools import partial + +class IMDBLoader(DataSetLoader): + """ + 读取IMDB数据集，DataSet包含以下fields: + + words: list(str), 需要分类的文本 + target: str, 文本的标签 + + + """ + + def __init__(self): + super(IMDBLoader, self).__init__() + + def _load(self, path): + dataset = DataSet() + with open(path, 'r', encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split('\t') + target = parts[0] + words = parts[1].split() + dataset.append(Instance(words=words, target=target)) + if len(dataset)==0: + raise RuntimeError(f"{path} has no valid data.") + + return dataset + + def process(self, + paths: Union[str, Dict[str, str]], + src_vocab_opt: VocabularyOption = None, + tgt_vocab_opt: VocabularyOption = None, + src_embed_opt: EmbeddingOption = None, + char_level_op=False): + + # paths = check_dataloader_paths(paths) + + datasets = {} + info = DataInfo() + for name, path in paths.items(): + dataset = self.load(path) + datasets[name] = dataset + + def wordtochar(words): + chars = [] + for word in words: + word = word.lower() + for char in word: + chars.append(char) + return chars + + if char_level_op: + for dataset in datasets.values(): + dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') + + datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False) + + src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) + src_vocab.from_dataset(datasets['train'], field_name='words') + # src_vocab.from_dataset(datasets['train'], datasets["dev"], datasets["test"], field_name='words') + src_vocab.index_dataset(*datasets.values(), field_name='words') + + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) + tgt_vocab.from_dataset(datasets['train'], field_name='target') + tgt_vocab.index_dataset(*datasets.values(), field_name='target') + + info.vocabs = { + "words": src_vocab, + "target": tgt_vocab + } + + info.datasets = datasets + + if src_embed_opt is not None: + embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) + info.embeddings['words'] = embed + + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + + return info + +if __name__=="__main__": + datapath = {"train": "/remote-home/ygwang/IMDB_data/train.csv", + "test": "/remote-home/ygwang/IMDB_data/test.csv"} + datainfo=IMDBLoader().process(datapath,char_level_op=True) + #print(datainfo.datasets["train"]) + len_count = 0 + for instance in datainfo.datasets["train"]: + len_count += len(instance["chars"]) + + ave_len = len_count / len(datainfo.datasets["train"]) + print(ave_len) \ No newline at end of file diff --git a/reproduction/text_classification/data/sstLoader.py b/reproduction/text_classification/data/sstLoader.py new file mode 100644 index 00000000..bffb67fd --- /dev/null +++ b/reproduction/text_classification/data/sstLoader.py @@ -0,0 +1,98 @@ +import csv +from typing import Iterable +from fastNLP import DataSet, Instance, Vocabulary +from fastNLP.core.vocabulary import VocabularyOption +from fastNLP.io.base_loader import DataInfo,DataSetLoader +from fastNLP.io.embed_loader import EmbeddingOption +from fastNLP.io.file_reader import _read_json +from typing import Union, Dict +from reproduction.Star_transformer.datasets import EmbedLoader +from reproduction.utils import check_dataloader_paths + +class sst2Loader(DataSetLoader): + ''' + 数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', + ''' + def __init__(self): + super(sst2Loader, self).__init__() + + def _load(self, path: str) -> DataSet: + ds = DataSet() + all_count=0 + csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t') + skip_row = 0 + for idx,row in enumerate(csv_reader): + if idx<=skip_row: + continue + target = row[1] + words = row[0].split() + ds.append(Instance(words=words,target=target)) + all_count+=1 + print("all count:", all_count) + return ds + + def process(self, + paths: Union[str, Dict[str, str]], + src_vocab_opt: VocabularyOption = None, + tgt_vocab_opt: VocabularyOption = None, + src_embed_opt: EmbeddingOption = None, + char_level_op=False): + + paths = check_dataloader_paths(paths) + datasets = {} + info = DataInfo() + for name, path in paths.items(): + dataset = self.load(path) + datasets[name] = dataset + + def wordtochar(words): + chars=[] + for word in words: + word=word.lower() + for char in word: + chars.append(char) + return chars + + input_name, target_name = 'words', 'target' + info.vocabs={} + + # 就分隔为char形式 + if char_level_op: + for dataset in datasets.values(): + dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') + + src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) + src_vocab.from_dataset(datasets['train'], field_name='words') + src_vocab.index_dataset(*datasets.values(), field_name='words') + + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) + tgt_vocab.from_dataset(datasets['train'], field_name='target') + tgt_vocab.index_dataset(*datasets.values(), field_name='target') + + + info.vocabs = { + "words": src_vocab, + "target": tgt_vocab + } + + info.datasets = datasets + + + if src_embed_opt is not None: + embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) + info.embeddings['words'] = embed + + return info + +if __name__=="__main__": + datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv", + "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"} + datainfo=sst2Loader().process(datapath,char_level_op=True) + #print(datainfo.datasets["train"]) + len_count = 0 + for instance in datainfo.datasets["train"]: + len_count += len(instance["chars"]) + + ave_len = len_count / len(datainfo.datasets["train"]) + print(ave_len) \ No newline at end of file diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index c47d48fd..9d34004d 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -1,8 +1,10 @@ import ast +import csv +from typing import Iterable from fastNLP import DataSet, Instance, Vocabulary from fastNLP.core.vocabulary import VocabularyOption from fastNLP.io import JsonLoader -from fastNLP.io.base_loader import DataInfo +from fastNLP.io.base_loader import DataInfo,DataSetLoader from fastNLP.io.embed_loader import EmbeddingOption from fastNLP.io.file_reader import _read_json from typing import Union, Dict @@ -10,27 +12,44 @@ from reproduction.Star_transformer.datasets import EmbedLoader from reproduction.utils import check_dataloader_paths -class yelpLoader(JsonLoader): - +def clean_str(sentence,char_lower=False): """ - 读取Yelp数据集, DataSet包含fields: + heavily borrowed from github + https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb + :param sentence: is a str + :return: + """ + if char_lower: + sentence=sentence.lower() + import re + nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') + words = sentence.split() + words_collection = [] + for word in words: + if word in ['-lrb-', '-rrb-', '', '-r', '-l', 'b-']: + continue + tt = nonalpnum.split(word) + t = ''.join(tt) + if t != '': + words_collection.append(t) + + return words_collection + + +class yelpLoader(DataSetLoader): - review_id: str, 22 character unique review id - user_id: str, 22 character unique user id - business_id: str, 22 character business id - useful: int, number of useful votes received - funny: int, number of funny votes received - cool: int, number of cool votes received - date: str, date formatted YYYY-MM-DD + """ + 读取Yelp_full/Yelp_polarity数据集, DataSet包含fields: + words: list(str), 需要分类的文本 target: str, 文本的标签 - - 数据来源: https://www.yelp.com/dataset/download - + chars:list(str),未index的字符列表 + + 数据集：yelp_full/yelp_polarity :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` """ - def __init__(self, fine_grained=False): + def __init__(self, fine_grained=False,lower=False): super(yelpLoader, self).__init__() tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral', '4.0': 'positive', '5.0': 'very positive'} @@ -39,8 +58,24 @@ class yelpLoader(JsonLoader): tag_v['5.0'] = tag_v['4.0'] self.fine_grained = fine_grained self.tag_v = tag_v + self.lower=lower + + ''' + 读取Yelp数据集, DataSet包含fields: - def _load(self, path): + review_id: str, 22 character unique review id + user_id: str, 22 character unique user id + business_id: str, 22 character business id + useful: int, number of useful votes received + funny: int, number of funny votes received + cool: int, number of cool votes received + date: str, date formatted YYYY-MM-DD + words: list(str), 需要分类的文本 + target: str, 文本的标签 + + 数据来源: https://www.yelp.com/dataset/download + + def _load_json(self, path): ds = DataSet() for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): d = ast.literal_eval(d) @@ -48,21 +83,112 @@ class yelpLoader(JsonLoader): d["target"] = self.tag_v[str(d.pop("stars"))] ds.append(Instance(**d)) return ds + + def _load_yelp2015_broken(self,path): + ds = DataSet() + with open (path,encoding='ISO 8859-1') as f: + row=f.readline() + all_count=0 + exp_count=0 + while row: + row=row.split("\t\t") + all_count+=1 + if len(row)>=3: + words=row[-1].split() + try: + target=self.tag_v[str(row[-2])+".0"] + ds.append(Instance(words=words, target=target)) + except KeyError: + exp_count+=1 + else: + exp_count+=1 + row = f.readline() + print("error sample count:",exp_count) + print("all count:",all_count) + return ds + ''' + def _load(self, path): + ds = DataSet() + csv_reader=csv.reader(open(path,encoding='utf-8')) + all_count=0 + real_count=0 + for row in csv_reader: + all_count+=1 + if len(row)==2: + target=self.tag_v[row[0]+".0"] + words=clean_str(row[1],self.lower) + if len(words)!=0: + ds.append(Instance(words=words,target=target)) + real_count += 1 + print("all count:", all_count) + print("real count:", real_count) + return ds + - def process(self, paths: Union[str, Dict[str, str]], vocab_opt: VocabularyOption = None, - embed_opt: EmbeddingOption = None): + def process(self, paths: Union[str, Dict[str, str]], + train_ds: Iterable[str] = None, + src_vocab_op: VocabularyOption = None, + tgt_vocab_op: VocabularyOption = None, + embed_opt: EmbeddingOption = None, + char_level_op=False): paths = check_dataloader_paths(paths) datasets = {} - info = DataInfo() - vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) - for name, path in paths.items(): - dataset = self.load(path) - datasets[name] = dataset - vocab.from_dataset(dataset, field_name="words") - info.vocabs = vocab - info.datasets = datasets - if embed_opt is not None: - embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) - info.embeddings['words'] = embed + info = DataInfo(datasets=self.load(paths)) + src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) + _train_ds = [info.datasets[name] + for name in train_ds] if train_ds else info.datasets.values() + #vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) + # for name, path in paths.items(): + # dataset = self.load(path) + # datasets[name] = dataset + # vocab.from_dataset(dataset, field_name="words") + # info.vocabs = vocab + # info.datasets = datasets + + def wordtochar(words): + chars=[] + for word in words: + word=word.lower() + for char in word: + chars.append(char) + return chars + + input_name, target_name = 'words', 'target' + info.vocabs={} + #就分隔为char形式 + if char_level_op: + for dataset in info.datasets.values(): + dataset.apply_field(wordtochar, field_name="words",new_field_name='chars') + # if embed_opt is not None: + # embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) + # info.embeddings['words'] = embed + else: + src_vocab.from_dataset(*_train_ds, field_name=input_name) + src_vocab.index_dataset(*info.datasets.values(),field_name=input_name, new_field_name=input_name) + info.vocabs[input_name]=src_vocab + + tgt_vocab.from_dataset(*_train_ds, field_name=target_name) + tgt_vocab.index_dataset( + *info.datasets.values(), + field_name=target_name, new_field_name=target_name) + info.vocabs[target_name]=tgt_vocab + return info +if __name__=="__main__": + testloader=yelpLoader() + # datapath = {"train": "/remote-home/ygwang/yelp_full/train.csv", + # "test": "/remote-home/ygwang/yelp_full/test.csv"} + #datapath={"train": "/remote-home/ygwang/yelp_full/test.csv"} + datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv", + "test": "/remote-home/ygwang/yelp_polarity/test.csv"} + datainfo=testloader.process(datapath,char_level_op=True) + + len_count=0 + for instance in datainfo.datasets["train"]: + len_count+=len(instance["chars"]) + + ave_len=len_count/len(datainfo.datasets["train"]) + print(ave_len) \ No newline at end of file diff --git a/reproduction/text_classification/model/char_cnn.py b/reproduction/text_classification/model/char_cnn.py index f87f5c14..ac370082 100644 --- a/reproduction/text_classification/model/char_cnn.py +++ b/reproduction/text_classification/model/char_cnn.py @@ -1 +1,90 @@ -# TODO \ No newline at end of file +''' +@author: https://github.com/ahmedbesbes/character-based-cnn +这里借鉴了上述链接中char-cnn model的代码，改动主要为将其改动为符合fastnlp的pipline +''' +import torch +import torch.nn as nn +from fastNLP.core.const import Const as C + +class CharacterLevelCNN(nn.Module): + def __init__(self, args,embedding): + super(CharacterLevelCNN, self).__init__() + + self.config=args.char_cnn_config + self.embedding=embedding + + conv_layers = [] + for i, conv_layer_parameter in enumerate(self.config['model_parameters'][args.model_size]['conv']): + if i == 0: + #in_channels = args.number_of_characters + len(args.extra_characters) + in_channels = args.embedding_dim + out_channels = conv_layer_parameter[0] + else: + in_channels, out_channels = conv_layer_parameter[0], conv_layer_parameter[0] + + if conv_layer_parameter[2] != -1: + conv_layer = nn.Sequential(nn.Conv1d(in_channels, + out_channels, + kernel_size=conv_layer_parameter[1], padding=0), + nn.ReLU(), + nn.MaxPool1d(conv_layer_parameter[2])) + else: + conv_layer = nn.Sequential(nn.Conv1d(in_channels, + out_channels, + kernel_size=conv_layer_parameter[1], padding=0), + nn.ReLU()) + conv_layers.append(conv_layer) + self.conv_layers = nn.ModuleList(conv_layers) + + input_shape = (args.batch_size, args.max_length, + args.number_of_characters + len(args.extra_characters)) + dimension = self._get_conv_output(input_shape) + + print('dimension :', dimension) + + fc_layer_parameter = self.config['model_parameters'][args.model_size]['fc'][0] + fc_layers = nn.ModuleList([ + nn.Sequential( + nn.Linear(dimension, fc_layer_parameter), nn.Dropout(0.5)), + nn.Sequential(nn.Linear(fc_layer_parameter, + fc_layer_parameter), nn.Dropout(0.5)), + nn.Linear(fc_layer_parameter, args.num_classes), + ]) + + self.fc_layers = fc_layers + + if args.model_size == 'small': + self._create_weights(mean=0.0, std=0.05) + elif args.model_size == 'large': + self._create_weights(mean=0.0, std=0.02) + + def _create_weights(self, mean=0.0, std=0.05): + for module in self.modules(): + if isinstance(module, nn.Conv1d) or isinstance(module, nn.Linear): + module.weight.data.normal_(mean, std) + + def _get_conv_output(self, shape): + input = torch.rand(shape) + output = input.transpose(1, 2) + # forward pass through conv layers + for i in range(len(self.conv_layers)): + output = self.conv_layers[i](output) + + output = output.view(output.size(0), -1) + n_size = output.size(1) + return n_size + + def forward(self, chars): + input=self.embedding(chars) + output = input.transpose(1, 2) + # forward pass through conv layers + for i in range(len(self.conv_layers)): + output = self.conv_layers[i](output) + + output = output.view(output.size(0), -1) + + # forward pass through fc layers + for i in range(len(self.fc_layers)): + output = self.fc_layers[i](output) + + return {C.OUTPUT: output} \ No newline at end of file diff --git a/reproduction/text_classification/model/dpcnn.py b/reproduction/text_classification/model/dpcnn.py index f87f5c14..a846af72 100644 --- a/reproduction/text_classification/model/dpcnn.py +++ b/reproduction/text_classification/model/dpcnn.py @@ -1 +1,111 @@ -# TODO \ No newline at end of file + + +import torch +import torch.nn as nn +from fastNLP.modules.utils import get_embeddings +from fastNLP.core import Const as C + + + +class DPCNN(nn.Module): + + def __init__(self, init_embed, num_cls, n_filters=256, kernel_size=3, n_layers=7, embed_dropout=0.1, dropout=0.1): + super().__init__() + self.region_embed = RegionEmbedding(init_embed, out_dim=n_filters, kernel_sizes=[3, 5, 9]) + embed_dim = self.region_embed.embedding_dim + self.conv_list = nn.ModuleList() + for i in range(n_layers): + self.conv_list.append(nn.Sequential( + nn.ReLU(), + nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + )) + + self.pool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1) + self.embed_drop = nn.Dropout(embed_dropout) + self.classfier = nn.Sequential( + nn.Dropout(dropout), + nn.Linear(n_filters, num_cls), + ) + self.reset_parameters() + + + + def reset_parameters(self): + for m in self.modules(): + if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)): + nn.init.normal_(m.weight, mean=0, std=0.01) + if m.bias is not None: + nn.init.normal_(m.bias, mean=0, std=0.01) + + + + def forward(self, words, seq_len=None): + words = words.long() + # get region embeddings + x = self.region_embed(words) + x = self.embed_drop(x) + + # not pooling on first conv + x = self.conv_list[0](x) + x + for conv in self.conv_list[1:]: + x = self.pool(x) + x = conv(x) + x + + # B, C, L => B, C + x, _ = torch.max(x, dim=2) + x = self.classfier(x) + return {C.OUTPUT: x} + + + + def predict(self, words, seq_len=None): + x = self.forward(words, seq_len)[C.OUTPUT] + return {C.OUTPUT: torch.argmax(x, 1)} + + + + + +class RegionEmbedding(nn.Module): + def __init__(self, init_embed, out_dim=300, kernel_sizes=None): + super().__init__() + if kernel_sizes is None: + kernel_sizes = [5, 9] + assert isinstance(kernel_sizes, list), 'kernel_sizes should be List(int)' + self.embed = get_embeddings(init_embed) + try: + embed_dim = self.embed.embedding_dim + except Exception: + embed_dim = self.embed.embed_size + self.region_embeds = nn.ModuleList() + for ksz in kernel_sizes: + self.region_embeds.append(nn.Sequential( + nn.Conv1d(embed_dim, embed_dim, ksz, padding=ksz // 2), + )) + self.linears = nn.ModuleList([nn.Conv1d(embed_dim, out_dim, 1) + for _ in range(len(kernel_sizes) + 1)]) + self.embedding_dim = embed_dim + + + def forward(self, x): + x = self.embed(x) + x = x.transpose(1, 2) + # B, C, L + out = self.linears[0](x) + for conv, fc in zip(self.region_embeds, self.linears[1:]): + conv_i = conv(x) + out = out + fc(conv_i) + # B, C, L + + return out + + + + + +if __name__ == '__main__': + x = torch.randint(0, 10000, size=(5, 15), dtype=torch.long) + model = DPCNN((10000, 300), 20) + y = model(x) + print(y.size(), y.mean(1), y.std(1)) \ No newline at end of file diff --git a/reproduction/text_classification/train_char_cnn.py b/reproduction/text_classification/train_char_cnn.py index e69de29b..c2c983a4 100644 --- a/reproduction/text_classification/train_char_cnn.py +++ b/reproduction/text_classification/train_char_cnn.py @@ -0,0 +1,206 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' + +import sys +sys.path.append('../..') +from fastNLP.core.const import Const as C +import torch.nn as nn +from fastNLP.io.dataset_loader import SSTLoader +from data.yelpLoader import yelpLoader +from data.sstLoader import sst2Loader +from data.IMDBLoader import IMDBLoader +from model.char_cnn import CharacterLevelCNN +from fastNLP.core.vocabulary import Vocabulary +from fastNLP.models.cnn_text_classification import CNNText +from fastNLP.modules.encoder.embedding import CNNCharEmbedding,StaticEmbedding,StackEmbedding,LSTMCharEmbedding +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.core.trainer import Trainer +from torch.optim import SGD +from torch.autograd import Variable +import torch +from fastNLP import BucketSampler + +##hyper +#todo 这里加入fastnlp的记录 +class Config(): + model_dir_or_name="en-base-uncased" + embedding_grad= False, + bert_embedding_larers= '4,-2,-1' + train_epoch= 50 + num_classes=2 + task= "IMDB" + #yelp_p + datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv", + "test": "/remote-home/ygwang/yelp_polarity/test.csv"} + #IMDB + #datapath = {"train": "/remote-home/ygwang/IMDB_data/train.csv", + # "test": "/remote-home/ygwang/IMDB_data/test.csv"} + # sst + # datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv", + # "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"} + + lr=0.01 + batch_size=128 + model_size="large" + number_of_characters=69 + extra_characters='' + max_length=1014 + + char_cnn_config={ + "alphabet": { + "en": { + "lower": { + "alphabet": "abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}", + "number_of_characters": 69 + }, + "both": { + "alphabet": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}", + "number_of_characters": 95 + } + } + }, + "model_parameters": { + "small": { + "conv": [ + #依次是channel，kennnel_size，maxpooling_size + [256,7,3], + [256,7,3], + [256,3,-1], + [256,3,-1], + [256,3,-1], + [256,3,3] + ], + "fc": [1024,1024] + }, + "large":{ + "conv":[ + [1024, 7, 3], + [1024, 7, 3], + [1024, 3, -1], + [1024, 3, -1], + [1024, 3, -1], + [1024, 3, 3] + ], + "fc": [2048,2048] + } + }, + "data": { + "text_column": "SentimentText", + "label_column": "Sentiment", + "max_length": 1014, + "num_of_classes": 2, + "encoding": None, + "chunksize": 50000, + "max_rows": 100000, + "preprocessing_steps": ["lower", "remove_hashtags", "remove_urls", "remove_user_mentions"] + }, + "training": { + "batch_size": 128, + "learning_rate": 0.01, + "epochs": 10, + "optimizer": "sgd" + } + } +ops=Config + + +##1.task相关信息：利用dataloader载入dataInfo +dataloader=sst2Loader() +dataloader=IMDBLoader() +#dataloader=yelpLoader(fine_grained=True) +datainfo=dataloader.process(ops.datapath,char_level_op=True) +char_vocab=ops.char_cnn_config["alphabet"]["en"]["lower"]["alphabet"] +ops.number_of_characters=len(char_vocab) +ops.embedding_dim=ops.number_of_characters + +#chartoindex +def chartoindex(chars): + max_seq_len=ops.max_length + zero_index=len(char_vocab) + char_index_list=[] + for char in chars: + if char in char_vocab: + char_index_list.append(char_vocab.index(char)) + else: + #和均使用最后一个作为embbeding + char_index_list.append(zero_index) + if len(char_index_list) > max_seq_len: + char_index_list = char_index_list[:max_seq_len] + elif 0 < len(char_index_list) < max_seq_len: + char_index_list = char_index_list+[zero_index]*(max_seq_len-len(char_index_list)) + elif len(char_index_list) == 0: + char_index_list=[zero_index]*max_seq_len + return char_index_list + +for dataset in datainfo.datasets.values(): + dataset.apply_field(chartoindex,field_name='chars',new_field_name='chars') + +datainfo.datasets['train'].set_input('chars') +datainfo.datasets['test'].set_input('chars') +datainfo.datasets['train'].set_target('target') +datainfo.datasets['test'].set_target('target') + +##2. 定义/组装模型，这里可以随意，就如果是fastNLP封装好的，类似CNNText就直接用初始化调用就好了，这里只是给出一个伪框架表示占位，在这里建立符合fastNLP输入输出规范的model +class ModelFactory(nn.Module): + """ + 用于拼装embedding，encoder，decoder 以及设计forward过程 + + :param embedding: embbeding model + :param encoder: encoder model + :param decoder: decoder model + + """ + def __int__(self,embedding,encoder,decoder,**kwargs): + super(ModelFactory,self).__init__() + self.embedding=embedding + self.encoder=encoder + self.decoder=decoder + + def forward(self,x): + return {C.OUTPUT:None} + +## 2.或直接复用fastNLP的模型 +#vocab=datainfo.vocabs['words'] +vocab_label=datainfo.vocabs['target'] +''' +# emded_char=CNNCharEmbedding(vocab) +# embed_word = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50', requires_grad=True) +# embedding=StackEmbedding([emded_char, embed_word]) +# cnn_char_embed = CNNCharEmbedding(vocab) +# lstm_char_embed = LSTMCharEmbedding(vocab) +# embedding = StackEmbedding([cnn_char_embed, lstm_char_embed]) +''' +#one-hot embedding +embedding_weight= Variable(torch.zeros(len(char_vocab)+1, len(char_vocab))) + +for i in range(len(char_vocab)): + embedding_weight[i][i]=1 +embedding=nn.Embedding(num_embeddings=len(char_vocab)+1,embedding_dim=len(char_vocab),padding_idx=len(char_vocab),_weight=embedding_weight) +for para in embedding.parameters(): + para.requires_grad=False +#CNNText太过于简单 +#model=CNNText(init_embed=embedding, num_classes=ops.num_classes) +model=CharacterLevelCNN(ops,embedding) + +## 3. 声明loss,metric,optimizer +loss=CrossEntropyLoss +metric=AccuracyMetric +optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], lr=ops.lr) + +## 4.定义train方法 +def train(model,datainfo,loss,metrics,optimizer,num_epochs=100): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss(target='target'), + metrics=[metrics(target='target')], dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, + n_epochs=num_epochs) + print(trainer.train()) + + + +if __name__=="__main__": + #print(vocab_label) + + #print(datainfo.datasets["train"]) + train(model,datainfo,loss,metric,optimizer,num_epochs=ops.train_epoch) + \ No newline at end of file diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index e69de29b..8ddea1a3 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -0,0 +1,101 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 + +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +import sys +sys.path.append('../..') +from fastNLP.core.const import Const as C +from fastNLP.core import LRScheduler +import torch.nn as nn +from fastNLP.io.dataset_loader import SSTLoader +from data.yelpLoader import yelpLoader +from reproduction.text_classification.model.dpcnn import DPCNN +from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.core.trainer import Trainer +from torch.optim import SGD +import torch.cuda +from torch.optim.lr_scheduler import CosineAnnealingLR + + +##hyper + +class Config(): + model_dir_or_name="en-base-uncased" + embedding_grad= False, + train_epoch= 30 + batch_size = 100 + num_classes=2 + task= "yelp_p" + #datadir = '/remote-home/yfshao/workdir/datasets/SST' + datadir = '/remote-home/ygwang/yelp_polarity' + #datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} + datafile = {"train": "train.csv", "test": "test.csv"} + lr=1e-3 + + def __init__(self): + self.datapath = {k:os.path.join(self.datadir, v) + for k, v in self.datafile.items()} + +ops=Config() + + + +##1.task相关信息：利用dataloader载入dataInfo + +#datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) +datainfo=yelpLoader(fine_grained=True,lower=True).process(paths=ops.datapath, train_ds=['train']) +print(len(datainfo.datasets['train'])) +print(len(datainfo.datasets['test'])) + + +## 2.或直接复用fastNLP的模型 + +vocab = datainfo.vocabs['words'] +# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) +#embedding = StaticEmbedding(vocab) +embedding = StaticEmbedding(vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) + +print(len(vocab)) +print(len(datainfo.vocabs['target'])) + +model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) + + + +## 3. 声明loss,metric,optimizer +loss=CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) +metric=AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) +optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], + lr=ops.lr, momentum=0.9, weight_decay=0) + +callbacks = [] +callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) + + +device = 'cuda:3' if torch.cuda.is_available() else 'cpu' + +print(device) + +for ds in datainfo.datasets.values(): + ds.apply_field(len, C.INPUT, C.INPUT_LEN) + ds.set_input(C.INPUT, C.INPUT_LEN) + ds.set_target(C.TARGET) + + +## 4.定义train方法 +def train(model,datainfo,loss,metrics,optimizer,num_epochs=ops.train_epoch): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=[metrics], dev_data=datainfo.datasets['test'], device=3, + check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, + n_epochs=num_epochs) + + print(trainer.train()) + + + +if __name__=="__main__": + train(model,datainfo,loss,metric,optimizer) \ No newline at end of file From c7463cf0907b49f20007b3fd8d487d9ee84051e1 Mon Sep 17 00:00:00 2001 From: wyg <1505116161@qq.com> Date: Sat, 6 Jul 2019 16:45:37 +0800 Subject: [PATCH 09/27] [verify] yelpdataloader --- reproduction/text_classification/data/yelpLoader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index 9e1e1c6b..90a80301 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -128,7 +128,7 @@ class yelpLoader(DataSetLoader): all_count+=1 if len(row)==2: target=self.tag_v[row[0]+".0"] - words=clean_str(row[1],self.lower) + words=clean_str(row[1],self.tokenizer,self.lower) if len(words)!=0: ds.append(Instance(words=words,target=target)) real_count += 1 From c07cbd72f696564a77eb3e888b4d14618fe5b526 Mon Sep 17 00:00:00 2001 From: xxliu Date: Sat, 6 Jul 2019 18:28:46 +0800 Subject: [PATCH 10/27] =?UTF-8?q?=E6=8C=87=E4=BB=A3=E6=B6=88=E8=A7=A3?= =?UTF-8?q?=E6=BA=90=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../coreference_resolution/__init__.py | 0 .../data_load/__init__.py | 0 .../data_load/cr_loader.py | 68 +++ .../coreference_resolution/model/__init__.py | 0 .../coreference_resolution/model/config.py | 54 ++ .../coreference_resolution/model/metric.py | 163 +++++ .../coreference_resolution/model/model_re.py | 576 ++++++++++++++++++ .../model/preprocess.py | 225 +++++++ .../model/softmax_loss.py | 32 + .../coreference_resolution/model/util.py | 101 +++ reproduction/coreference_resolution/readme.md | 49 ++ .../coreference_resolution/test/__init__.py | 0 .../test/test_dataloader.py | 14 + reproduction/coreference_resolution/train.py | 69 +++ reproduction/coreference_resolution/valid.py | 24 + 15 files changed, 1375 insertions(+) create mode 100644 reproduction/coreference_resolution/__init__.py create mode 100644 reproduction/coreference_resolution/data_load/__init__.py create mode 100644 reproduction/coreference_resolution/data_load/cr_loader.py create mode 100644 reproduction/coreference_resolution/model/__init__.py create mode 100644 reproduction/coreference_resolution/model/config.py create mode 100644 reproduction/coreference_resolution/model/metric.py create mode 100644 reproduction/coreference_resolution/model/model_re.py create mode 100644 reproduction/coreference_resolution/model/preprocess.py create mode 100644 reproduction/coreference_resolution/model/softmax_loss.py create mode 100644 reproduction/coreference_resolution/model/util.py create mode 100644 reproduction/coreference_resolution/readme.md create mode 100644 reproduction/coreference_resolution/test/__init__.py create mode 100644 reproduction/coreference_resolution/test/test_dataloader.py create mode 100644 reproduction/coreference_resolution/train.py create mode 100644 reproduction/coreference_resolution/valid.py diff --git a/reproduction/coreference_resolution/__init__.py b/reproduction/coreference_resolution/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reproduction/coreference_resolution/data_load/__init__.py b/reproduction/coreference_resolution/data_load/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reproduction/coreference_resolution/data_load/cr_loader.py b/reproduction/coreference_resolution/data_load/cr_loader.py new file mode 100644 index 00000000..986afcd5 --- /dev/null +++ b/reproduction/coreference_resolution/data_load/cr_loader.py @@ -0,0 +1,68 @@ +from fastNLP.io.dataset_loader import JsonLoader,DataSet,Instance +from fastNLP.io.file_reader import _read_json +from fastNLP.core.vocabulary import Vocabulary +from fastNLP.io.base_loader import DataInfo +from reproduction.coreference_resolution.model.config import Config +import reproduction.coreference_resolution.model.preprocess as preprocess + + +class CRLoader(JsonLoader): + def __init__(self, fields=None, dropna=False): + super().__init__(fields, dropna) + + def _load(self, path): + """ + 加载数据 + :param path: + :return: + """ + dataset = DataSet() + for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): + if self.fields: + ins = {self.fields[k]: v for k, v in d.items()} + else: + ins = d + dataset.append(Instance(**ins)) + return dataset + + def process(self, paths, **kwargs): + data_info = DataInfo() + for name in ['train', 'test', 'dev']: + data_info.datasets[name] = self.load(paths[name]) + + config = Config() + vocab = Vocabulary().from_dataset(*data_info.datasets.values(), field_name='sentences') + vocab.build_vocab() + word2id = vocab.word2idx + + char_dict = preprocess.get_char_dict(config.char_path) + data_info.vocabs = vocab + + genres = {g: i for i, g in enumerate(["bc", "bn", "mz", "nw", "pt", "tc", "wb"])} + + for name, ds in data_info.datasets.items(): + ds.apply(lambda x: preprocess.doc2numpy(x['sentences'], word2id, char_dict, max(config.filter), + config.max_sentences, is_train=name=='train')[0], + new_field_name='doc_np') + ds.apply(lambda x: preprocess.doc2numpy(x['sentences'], word2id, char_dict, max(config.filter), + config.max_sentences, is_train=name=='train')[1], + new_field_name='char_index') + ds.apply(lambda x: preprocess.doc2numpy(x['sentences'], word2id, char_dict, max(config.filter), + config.max_sentences, is_train=name=='train')[2], + new_field_name='seq_len') + ds.apply(lambda x: preprocess.speaker2numpy(x["speakers"], config.max_sentences, is_train=name=='train'), + new_field_name='speaker_ids_np') + ds.apply(lambda x: genres[x["doc_key"][:2]], new_field_name='genre') + + ds.set_ignore_type('clusters') + ds.set_padder('clusters', None) + ds.set_input("sentences", "doc_np", "speaker_ids_np", "genre", "char_index", "seq_len") + ds.set_target("clusters") + + # train_dev, test = self.ds.split(348 / (2802 + 343 + 348), shuffle=False) + # train, dev = train_dev.split(343 / (2802 + 343), shuffle=False) + + return data_info + + + diff --git a/reproduction/coreference_resolution/model/__init__.py b/reproduction/coreference_resolution/model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reproduction/coreference_resolution/model/config.py b/reproduction/coreference_resolution/model/config.py new file mode 100644 index 00000000..6011257b --- /dev/null +++ b/reproduction/coreference_resolution/model/config.py @@ -0,0 +1,54 @@ +class Config(): + def __init__(self): + self.is_training = True + # path + self.glove = 'data/glove.840B.300d.txt.filtered' + self.turian = 'data/turian.50d.txt' + self.train_path = "data/train.english.jsonlines" + self.dev_path = "data/dev.english.jsonlines" + self.test_path = "data/test.english.jsonlines" + self.char_path = "data/char_vocab.english.txt" + + self.cuda = "0" + self.max_word = 1500 + self.epoch = 200 + + # config + # self.use_glove = True + # self.use_turian = True #No + self.use_elmo = False + self.use_CNN = True + self.model_heads = True #Yes + self.use_width = True # Yes + self.use_distance = True #Yes + self.use_metadata = True #Yes + + self.mention_ratio = 0.4 + self.max_sentences = 50 + self.span_width = 10 + self.feature_size = 20 #宽度信息emb的size + self.lr = 0.001 + self.lr_decay = 1e-3 + self.max_antecedents = 100 # 这个参数在mention detection中没有用 + self.atten_hidden_size = 150 + self.mention_hidden_size = 150 + self.sa_hidden_size = 150 + + self.char_emb_size = 8 + self.filter = [3,4,5] + + + # decay = 1e-5 + + def __str__(self): + d = self.__dict__ + out = 'config==============\n' + for i in list(d): + out += i+":" + out += str(d[i])+"\n" + out+="config==============\n" + return out + +if __name__=="__main__": + config = Config() + print(config) diff --git a/reproduction/coreference_resolution/model/metric.py b/reproduction/coreference_resolution/model/metric.py new file mode 100644 index 00000000..2c924660 --- /dev/null +++ b/reproduction/coreference_resolution/model/metric.py @@ -0,0 +1,163 @@ +from fastNLP.core.metrics import MetricBase + +import numpy as np + +from collections import Counter +from sklearn.utils.linear_assignment_ import linear_assignment + +""" +Mostly borrowed from https://github.com/clarkkev/deep-coref/blob/master/evaluation.py +""" + + + +class CRMetric(MetricBase): + def __init__(self): + super().__init__() + self.evaluators = [Evaluator(m) for m in (muc, b_cubed, ceafe)] + + # TODO 改名为evaluate，输入也 + def evaluate(self, predicted, mention_to_predicted,clusters): + for e in self.evaluators: + e.update(predicted,mention_to_predicted, clusters) + + def get_f1(self): + return sum(e.get_f1() for e in self.evaluators) / len(self.evaluators) + + def get_recall(self): + return sum(e.get_recall() for e in self.evaluators) / len(self.evaluators) + + def get_precision(self): + return sum(e.get_precision() for e in self.evaluators) / len(self.evaluators) + + # TODO 原本的getprf + def get_metric(self,reset=False): + res = {"pre":self.get_precision(), "rec":self.get_recall(), "f":self.get_f1()} + self.evaluators = [Evaluator(m) for m in (muc, b_cubed, ceafe)] + return res + + + + + + +class Evaluator(): + def __init__(self, metric, beta=1): + self.p_num = 0 + self.p_den = 0 + self.r_num = 0 + self.r_den = 0 + self.metric = metric + self.beta = beta + + def update(self, predicted,mention_to_predicted,gold): + gold = gold[0].tolist() + gold = [tuple(tuple(m) for m in gc) for gc in gold] + mention_to_gold = {} + for gc in gold: + for mention in gc: + mention_to_gold[mention] = gc + + if self.metric == ceafe: + pn, pd, rn, rd = self.metric(predicted, gold) + else: + pn, pd = self.metric(predicted, mention_to_gold) + rn, rd = self.metric(gold, mention_to_predicted) + self.p_num += pn + self.p_den += pd + self.r_num += rn + self.r_den += rd + + def get_f1(self): + return f1(self.p_num, self.p_den, self.r_num, self.r_den, beta=self.beta) + + def get_recall(self): + return 0 if self.r_num == 0 else self.r_num / float(self.r_den) + + def get_precision(self): + return 0 if self.p_num == 0 else self.p_num / float(self.p_den) + + def get_prf(self): + return self.get_precision(), self.get_recall(), self.get_f1() + + def get_counts(self): + return self.p_num, self.p_den, self.r_num, self.r_den + + + +def b_cubed(clusters, mention_to_gold): + num, dem = 0, 0 + + for c in clusters: + if len(c) == 1: + continue + + gold_counts = Counter() + correct = 0 + for m in c: + if m in mention_to_gold: + gold_counts[tuple(mention_to_gold[m])] += 1 + for c2, count in gold_counts.items(): + if len(c2) != 1: + correct += count * count + + num += correct / float(len(c)) + dem += len(c) + + return num, dem + + +def muc(clusters, mention_to_gold): + tp, p = 0, 0 + for c in clusters: + p += len(c) - 1 + tp += len(c) + linked = set() + for m in c: + if m in mention_to_gold: + linked.add(mention_to_gold[m]) + else: + tp -= 1 + tp -= len(linked) + return tp, p + + +def phi4(c1, c2): + return 2 * len([m for m in c1 if m in c2]) / float(len(c1) + len(c2)) + + +def ceafe(clusters, gold_clusters): + clusters = [c for c in clusters if len(c) != 1] + scores = np.zeros((len(gold_clusters), len(clusters))) + for i in range(len(gold_clusters)): + for j in range(len(clusters)): + scores[i, j] = phi4(gold_clusters[i], clusters[j]) + matching = linear_assignment(-scores) + similarity = sum(scores[matching[:, 0], matching[:, 1]]) + return similarity, len(clusters), similarity, len(gold_clusters) + + +def lea(clusters, mention_to_gold): + num, dem = 0, 0 + + for c in clusters: + if len(c) == 1: + continue + + common_links = 0 + all_links = len(c) * (len(c) - 1) / 2.0 + for i, m in enumerate(c): + if m in mention_to_gold: + for m2 in c[i + 1:]: + if m2 in mention_to_gold and mention_to_gold[m] == mention_to_gold[m2]: + common_links += 1 + + num += len(c) * common_links / float(all_links) + dem += len(c) + + return num, dem + +def f1(p_num, p_den, r_num, r_den, beta=1): + p = 0 if p_den == 0 else p_num / float(p_den) + r = 0 if r_den == 0 else r_num / float(r_den) + return 0 if p + r == 0 else (1 + beta * beta) * p * r / (beta * beta * p + r) diff --git a/reproduction/coreference_resolution/model/model_re.py b/reproduction/coreference_resolution/model/model_re.py new file mode 100644 index 00000000..9dd90ec4 --- /dev/null +++ b/reproduction/coreference_resolution/model/model_re.py @@ -0,0 +1,576 @@ +import torch +import numpy as np +import torch.nn as nn +import torch.nn.functional as F + +from allennlp.commands.elmo import ElmoEmbedder +from fastNLP.models.base_model import BaseModel +from fastNLP.modules.encoder.variational_rnn import VarLSTM +from reproduction.coreference_resolution.model import preprocess +from fastNLP.io.embed_loader import EmbedLoader +import random + +# 设置seed +torch.manual_seed(0) # cpu +torch.cuda.manual_seed(0) # gpu +np.random.seed(0) # numpy +random.seed(0) + + +class ffnn(nn.Module): + def __init__(self, input_size, hidden_size, output_size): + super(ffnn, self).__init__() + + self.f = nn.Sequential( + # 多少层数 + nn.Linear(input_size, hidden_size), + nn.ReLU(inplace=True), + nn.Dropout(p=0.2), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(inplace=True), + nn.Dropout(p=0.2), + nn.Linear(hidden_size, output_size) + ) + self.reset_param() + + def reset_param(self): + for name, param in self.named_parameters(): + if param.dim() > 1: + nn.init.xavier_normal_(param) + # param.data = torch.tensor(np.random.randn(*param.shape)).float() + else: + nn.init.zeros_(param) + + def forward(self, input): + return self.f(input).squeeze() + + +class Model(BaseModel): + def __init__(self, vocab, config): + word2id = vocab.word2idx + super(Model, self).__init__() + vocab_num = len(word2id) + self.word2id = word2id + self.config = config + self.char_dict = preprocess.get_char_dict('data/char_vocab.english.txt') + self.genres = {g: i for i, g in enumerate(["bc", "bn", "mz", "nw", "pt", "tc", "wb"])} + self.device = torch.device("cuda:" + config.cuda) + + self.emb = nn.Embedding(vocab_num, 350) + + emb1 = EmbedLoader().load_with_vocab(config.glove, vocab,normalize=False) + emb2 = EmbedLoader().load_with_vocab(config.turian, vocab ,normalize=False) + pre_emb = np.concatenate((emb1, emb2), axis=1) + pre_emb /= (np.linalg.norm(pre_emb, axis=1, keepdims=True) + 1e-12) + + if pre_emb is not None: + self.emb.weight = nn.Parameter(torch.from_numpy(pre_emb).float()) + for param in self.emb.parameters(): + param.requires_grad = False + self.emb_dropout = nn.Dropout(inplace=True) + + + if config.use_elmo: + self.elmo = ElmoEmbedder(options_file='data/elmo/elmo_2x4096_512_2048cnn_2xhighway_options.json', + weight_file='data/elmo/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5', + cuda_device=int(config.cuda)) + print("elmo load over.") + self.elmo_args = torch.randn((3), requires_grad=True).to(self.device) + + self.char_emb = nn.Embedding(len(self.char_dict), config.char_emb_size) + self.conv1 = nn.Conv1d(config.char_emb_size, 50, 3) + self.conv2 = nn.Conv1d(config.char_emb_size, 50, 4) + self.conv3 = nn.Conv1d(config.char_emb_size, 50, 5) + + self.feature_emb = nn.Embedding(config.span_width, config.feature_size) + self.feature_emb_dropout = nn.Dropout(p=0.2, inplace=True) + + self.mention_distance_emb = nn.Embedding(10, config.feature_size) + self.distance_drop = nn.Dropout(p=0.2, inplace=True) + + self.genre_emb = nn.Embedding(7, config.feature_size) + self.speaker_emb = nn.Embedding(2, config.feature_size) + + self.bilstm = VarLSTM(input_size=350+150*config.use_CNN+config.use_elmo*1024,hidden_size=200,bidirectional=True,batch_first=True,hidden_dropout=0.2) + # self.bilstm = nn.LSTM(input_size=500, hidden_size=200, bidirectional=True, batch_first=True) + self.h0 = nn.init.orthogonal_(torch.empty(2, 1, 200)).to(self.device) + self.c0 = nn.init.orthogonal_(torch.empty(2, 1, 200)).to(self.device) + self.bilstm_drop = nn.Dropout(p=0.2, inplace=True) + + self.atten = ffnn(input_size=400, hidden_size=config.atten_hidden_size, output_size=1) + self.mention_score = ffnn(input_size=1320, hidden_size=config.mention_hidden_size, output_size=1) + self.sa = ffnn(input_size=3980+40*config.use_metadata, hidden_size=config.sa_hidden_size, output_size=1) + self.mention_start_np = None + self.mention_end_np = None + + def _reorder_lstm(self, word_emb, seq_lens): + sort_ind = sorted(range(len(seq_lens)), key=lambda i: seq_lens[i], reverse=True) + seq_lens_re = [seq_lens[i] for i in sort_ind] + emb_seq = self.reorder_sequence(word_emb, sort_ind, batch_first=True) + packed_seq = nn.utils.rnn.pack_padded_sequence(emb_seq, seq_lens_re, batch_first=True) + + h0 = self.h0.repeat(1, len(seq_lens), 1) + c0 = self.c0.repeat(1, len(seq_lens), 1) + packed_out, final_states = self.bilstm(packed_seq, (h0, c0)) + + lstm_out, _ = nn.utils.rnn.pad_packed_sequence(packed_out, batch_first=True) + back_map = {ind: i for i, ind in enumerate(sort_ind)} + reorder_ind = [back_map[i] for i in range(len(seq_lens_re))] + lstm_out = self.reorder_sequence(lstm_out, reorder_ind, batch_first=True) + return lstm_out + + def reorder_sequence(self, sequence_emb, order, batch_first=True): + """ + sequence_emb: [T, B, D] if not batch_first + order: list of sequence length + """ + batch_dim = 0 if batch_first else 1 + assert len(order) == sequence_emb.size()[batch_dim] + + order = torch.LongTensor(order) + order = order.to(sequence_emb).long() + + sorted_ = sequence_emb.index_select(index=order, dim=batch_dim) + + del order + return sorted_ + + def flat_lstm(self, lstm_out, seq_lens): + batch = lstm_out.shape[0] + seq = lstm_out.shape[1] + dim = lstm_out.shape[2] + l = [j + i * seq for i, seq_len in enumerate(seq_lens) for j in range(seq_len)] + flatted = torch.index_select(lstm_out.view(batch * seq, dim), 0, torch.LongTensor(l).to(self.device)) + return flatted + + def potential_mention_index(self, word_index, max_sent_len): + # get mention index [3,2]:the first sentence is 3 and secend 2 + # [0,0,0,1,1] --> [[0, 0], [0, 1], [1, 1], [1, 2], [2, 2], [3, 3], [3, 4], [4, 4]] (max =2) + potential_mention = [] + for i in range(len(word_index)): + for j in range(i, i + max_sent_len): + if (j < len(word_index) and word_index[i] == word_index[j]): + potential_mention.append([i, j]) + return potential_mention + + def get_mention_start_end(self, seq_lens): + # 序列长度转换成mention + # [3,2] --> [0,0,0,1,1] + word_index = [0] * sum(seq_lens) + sent_index = 0 + index = 0 + for length in seq_lens: + for l in range(length): + word_index[index] = sent_index + index += 1 + sent_index += 1 + + # [0,0,0,1,1]-->[[0,0],[0,1],[0,2]....] + mention_id = self.potential_mention_index(word_index, self.config.span_width) + mention_start = np.array(mention_id, dtype=int)[:, 0] + mention_end = np.array(mention_id, dtype=int)[:, 1] + return mention_start, mention_end + + def get_mention_emb(self, flatten_lstm, mention_start, mention_end): + mention_start_tensor = torch.from_numpy(mention_start).to(self.device) + mention_end_tensor = torch.from_numpy(mention_end).to(self.device) + emb_start = flatten_lstm.index_select(dim=0, index=mention_start_tensor) # [mention_num,embed] + emb_end = flatten_lstm.index_select(dim=0, index=mention_end_tensor) # [mention_num,embed] + return emb_start, emb_end + + def get_mask(self, mention_start, mention_end): + # big mask for attention + mention_num = mention_start.shape[0] + mask = np.zeros((mention_num, self.config.span_width)) # [mention_num,span_width] + for i in range(mention_num): + start = mention_start[i] + end = mention_end[i] + # 实际上是宽度 + for j in range(end - start + 1): + mask[i][j] = 1 + mask = torch.from_numpy(mask) # [mention_num,max_mention] + # 0-->-inf 1-->0 + log_mask = torch.log(mask) + return log_mask + + def get_mention_index(self, mention_start, max_mention): + # TODO 后面可能要改 + assert len(mention_start.shape) == 1 + mention_start_tensor = torch.from_numpy(mention_start) + num_mention = mention_start_tensor.shape[0] + mention_index = mention_start_tensor.expand(max_mention, num_mention).transpose(0, + 1) # [num_mention,max_mention] + assert mention_index.shape[0] == num_mention + assert mention_index.shape[1] == max_mention + range_add = torch.arange(0, max_mention).expand(num_mention, max_mention).long() # [num_mention,max_mention] + mention_index = mention_index + range_add + mention_index = torch.min(mention_index, torch.LongTensor([mention_start[-1]]).expand(num_mention, max_mention)) + return mention_index.to(self.device) + + def sort_mention(self, mention_start, mention_end, candidate_mention_emb, candidate_mention_score, seq_lens): + # 排序记录,高分段在前面 + mention_score, mention_ids = torch.sort(candidate_mention_score, descending=True) + preserve_mention_num = int(self.config.mention_ratio * sum(seq_lens)) + mention_ids = mention_ids[0:preserve_mention_num] + mention_score = mention_score[0:preserve_mention_num] + + mention_start_tensor = torch.from_numpy(mention_start).to(self.device).index_select(dim=0, + index=mention_ids) # [lamda*word_num] + mention_end_tensor = torch.from_numpy(mention_end).to(self.device).index_select(dim=0, + index=mention_ids) # [lamda*word_num] + mention_emb = candidate_mention_emb.index_select(index=mention_ids, dim=0) # [lamda*word_num,emb] + assert mention_score.shape[0] == preserve_mention_num + assert mention_start_tensor.shape[0] == preserve_mention_num + assert mention_end_tensor.shape[0] == preserve_mention_num + assert mention_emb.shape[0] == preserve_mention_num + # TODO 不交叉没做处理 + + # 对start进行再排序，实际位置在前面 + # TODO 这里只考虑了start没有考虑end + mention_start_tensor, temp_index = torch.sort(mention_start_tensor) + mention_end_tensor = mention_end_tensor.index_select(dim=0, index=temp_index) + mention_emb = mention_emb.index_select(dim=0, index=temp_index) + mention_score = mention_score.index_select(dim=0, index=temp_index) + return mention_start_tensor, mention_end_tensor, mention_score, mention_emb + + def get_antecedents(self, mention_starts, max_antecedents): + num_mention = mention_starts.shape[0] + max_antecedents = min(max_antecedents, num_mention) + # mention和它是第几个mention之间的对应关系 + antecedents = np.zeros((num_mention, max_antecedents), dtype=int) # [num_mention,max_an] + # 记录长度 + antecedents_len = [0] * num_mention + for i in range(num_mention): + ante_count = 0 + for j in range(max(0, i - max_antecedents), i): + antecedents[i, ante_count] = j + ante_count += 1 + # 补位操作 + for j in range(ante_count, max_antecedents): + antecedents[i, j] = 0 + antecedents_len[i] = ante_count + assert antecedents.shape[1] == max_antecedents + return antecedents, antecedents_len + + def get_antecedents_score(self, span_represent, mention_score, antecedents, antecedents_len, mention_speakers_ids, + genre): + num_mention = mention_score.shape[0] + max_antecedent = antecedents.shape[1] + + pair_emb = self.get_pair_emb(span_represent, antecedents, mention_speakers_ids, genre) # [span_num,max_ant,emb] + antecedent_scores = self.sa(pair_emb) + mask01 = self.sequence_mask(antecedents_len, max_antecedent) + maskinf = torch.log(mask01).to(self.device) + assert maskinf.shape[1] <= max_antecedent + assert antecedent_scores.shape[0] == num_mention + antecedent_scores = antecedent_scores + maskinf + antecedents = torch.from_numpy(antecedents).to(self.device) + mention_scoreij = mention_score.unsqueeze(1) + torch.gather( + mention_score.unsqueeze(0).expand(num_mention, num_mention), dim=1, index=antecedents) + antecedent_scores += mention_scoreij + + antecedent_scores = torch.cat([torch.zeros([mention_score.shape[0], 1]).to(self.device), antecedent_scores], + 1) # [num_mentions, max_ant + 1] + return antecedent_scores + + ############################## + def distance_bin(self, mention_distance): + bins = torch.zeros(mention_distance.size()).byte().to(self.device) + rg = [[1, 1], [2, 2], [3, 3], [4, 4], [5, 7], [8, 15], [16, 31], [32, 63], [64, 300]] + for t, k in enumerate(rg): + i, j = k[0], k[1] + b = torch.LongTensor([i]).unsqueeze(-1).expand(mention_distance.size()).to(self.device) + m1 = torch.ge(mention_distance, b) + e = torch.LongTensor([j]).unsqueeze(-1).expand(mention_distance.size()).to(self.device) + m2 = torch.le(mention_distance, e) + bins = bins + (t + 1) * (m1 & m2) + return bins.long() + + def get_distance_emb(self, antecedents_tensor): + num_mention = antecedents_tensor.shape[0] + max_ant = antecedents_tensor.shape[1] + + assert max_ant <= self.config.max_antecedents + source = torch.arange(0, num_mention).expand(max_ant, num_mention).transpose(0,1).to(self.device) # [num_mention,max_ant] + mention_distance = source - antecedents_tensor + mention_distance_bin = self.distance_bin(mention_distance) + distance_emb = self.mention_distance_emb(mention_distance_bin) + distance_emb = self.distance_drop(distance_emb) + return distance_emb + + def get_pair_emb(self, span_emb, antecedents, mention_speakers_ids, genre): + emb_dim = span_emb.shape[1] + num_span = span_emb.shape[0] + max_ant = antecedents.shape[1] + assert span_emb.shape[0] == antecedents.shape[0] + antecedents = torch.from_numpy(antecedents).to(self.device) + + # [num_span,max_ant,emb] + antecedent_emb = torch.gather(span_emb.unsqueeze(0).expand(num_span, num_span, emb_dim), dim=1, + index=antecedents.unsqueeze(2).expand(num_span, max_ant, emb_dim)) + # [num_span,max_ant,emb] + target_emb_tiled = span_emb.expand((max_ant, num_span, emb_dim)) + target_emb_tiled = target_emb_tiled.transpose(0, 1) + + similarity_emb = antecedent_emb * target_emb_tiled + + pair_emb_list = [target_emb_tiled, antecedent_emb, similarity_emb] + + # get speakers and genre + if self.config.use_metadata: + antecedent_speaker_ids = mention_speakers_ids.unsqueeze(0).expand(num_span, num_span).gather(dim=1, + index=antecedents) + same_speaker = torch.eq(mention_speakers_ids.unsqueeze(1).expand(num_span, max_ant), + antecedent_speaker_ids) # [num_mention,max_ant] + speaker_embedding = self.speaker_emb(same_speaker.long().to(self.device)) # [mention_num.max_ant,emb] + genre_embedding = self.genre_emb( + torch.LongTensor([genre]).expand(num_span, max_ant).to(self.device)) # [mention_num,max_ant,emb] + pair_emb_list.append(speaker_embedding) + pair_emb_list.append(genre_embedding) + + # get distance emb + if self.config.use_distance: + distance_emb = self.get_distance_emb(antecedents) + pair_emb_list.append(distance_emb) + + pair_emb = torch.cat(pair_emb_list, 2) + return pair_emb + + def sequence_mask(self, len_list, max_len): + x = np.zeros((len(len_list), max_len)) + for i in range(len(len_list)): + l = len_list[i] + for j in range(l): + x[i][j] = 1 + return torch.from_numpy(x).float() + + def logsumexp(self, value, dim=None, keepdim=False): + """Numerically stable implementation of the operation + + value.exp().sum(dim, keepdim).log() + """ + # TODO: torch.max(value, dim=None) threw an error at time of writing + if dim is not None: + m, _ = torch.max(value, dim=dim, keepdim=True) + value0 = value - m + if keepdim is False: + m = m.squeeze(dim) + return m + torch.log(torch.sum(torch.exp(value0), + dim=dim, keepdim=keepdim)) + else: + m = torch.max(value) + sum_exp = torch.sum(torch.exp(value - m)) + + return m + torch.log(sum_exp) + + def softmax_loss(self, antecedent_scores, antecedent_labels): + antecedent_labels = torch.from_numpy(antecedent_labels * 1).to(self.device) + gold_scores = antecedent_scores + torch.log(antecedent_labels.float()) # [num_mentions, max_ant + 1] + marginalized_gold_scores = self.logsumexp(gold_scores, 1) # [num_mentions] + log_norm = self.logsumexp(antecedent_scores, 1) # [num_mentions] + return torch.sum(log_norm - marginalized_gold_scores) # [num_mentions]reduce_logsumexp + + def get_predicted_antecedents(self, antecedents, antecedent_scores): + predicted_antecedents = [] + for i, index in enumerate(np.argmax(antecedent_scores.detach(), axis=1) - 1): + if index < 0: + predicted_antecedents.append(-1) + else: + predicted_antecedents.append(antecedents[i, index]) + return predicted_antecedents + + def get_predicted_clusters(self, mention_starts, mention_ends, predicted_antecedents): + mention_to_predicted = {} + predicted_clusters = [] + for i, predicted_index in enumerate(predicted_antecedents): + if predicted_index < 0: + continue + assert i > predicted_index + predicted_antecedent = (int(mention_starts[predicted_index]), int(mention_ends[predicted_index])) + if predicted_antecedent in mention_to_predicted: + predicted_cluster = mention_to_predicted[predicted_antecedent] + else: + predicted_cluster = len(predicted_clusters) + predicted_clusters.append([predicted_antecedent]) + mention_to_predicted[predicted_antecedent] = predicted_cluster + + mention = (int(mention_starts[i]), int(mention_ends[i])) + predicted_clusters[predicted_cluster].append(mention) + mention_to_predicted[mention] = predicted_cluster + + predicted_clusters = [tuple(pc) for pc in predicted_clusters] + mention_to_predicted = {m: predicted_clusters[i] for m, i in mention_to_predicted.items()} + + return predicted_clusters, mention_to_predicted + + def evaluate_coref(self, mention_starts, mention_ends, predicted_antecedents, gold_clusters, evaluator): + gold_clusters = [tuple(tuple(m) for m in gc) for gc in gold_clusters] + mention_to_gold = {} + for gc in gold_clusters: + for mention in gc: + mention_to_gold[mention] = gc + predicted_clusters, mention_to_predicted = self.get_predicted_clusters(mention_starts, mention_ends, + predicted_antecedents) + evaluator.update(predicted_clusters, gold_clusters, mention_to_predicted, mention_to_gold) + return predicted_clusters + + + def forward(self, sentences, doc_np, speaker_ids_np, genre, char_index, seq_len): + """ + 实际输入都是tensor + :param sentences: 句子，被fastNLP转化成了numpy， + :param doc_np: 被fastNLP转化成了Tensor + :param speaker_ids_np: 被fastNLP转化成了Tensor + :param genre: 被fastNLP转化成了Tensor + :param char_index: 被fastNLP转化成了Tensor + :param seq_len: 被fastNLP转化成了Tensor + :return: + """ + # change for fastNLP + sentences = sentences[0].tolist() + doc_tensor = doc_np[0] + speakers_tensor = speaker_ids_np[0] + genre = genre[0].item() + char_index = char_index[0] + seq_len = seq_len[0].cpu().numpy() + + # 类型 + + # doc_tensor = torch.from_numpy(doc_np).to(self.device) + # speakers_tensor = torch.from_numpy(speaker_ids_np).to(self.device) + mention_emb_list = [] + + word_emb = self.emb(doc_tensor) + word_emb_list = [word_emb] + if self.config.use_CNN: + # [batch, length, char_length, char_dim] + char = self.char_emb(char_index) + char_size = char.size() + # first transform to [batch *length, char_length, char_dim] + # then transpose to [batch * length, char_dim, char_length] + char = char.view(char_size[0] * char_size[1], char_size[2], char_size[3]).transpose(1, 2) + + # put into cnn [batch*length, char_filters, char_length] + # then put into maxpooling [batch * length, char_filters] + char_over_cnn, _ = self.conv1(char).max(dim=2) + # reshape to [batch, length, char_filters] + char_over_cnn = torch.tanh(char_over_cnn).view(char_size[0], char_size[1], -1) + word_emb_list.append(char_over_cnn) + + char_over_cnn, _ = self.conv2(char).max(dim=2) + char_over_cnn = torch.tanh(char_over_cnn).view(char_size[0], char_size[1], -1) + word_emb_list.append(char_over_cnn) + + char_over_cnn, _ = self.conv3(char).max(dim=2) + char_over_cnn = torch.tanh(char_over_cnn).view(char_size[0], char_size[1], -1) + word_emb_list.append(char_over_cnn) + + # word_emb = torch.cat(word_emb_list, dim=2) + + # use elmo or not + if self.config.use_elmo: + # 如果确实被截断了 + if doc_tensor.shape[0] == 50 and len(sentences) > 50: + sentences = sentences[0:50] + elmo_embedding, elmo_mask = self.elmo.batch_to_embeddings(sentences) + elmo_embedding = elmo_embedding.to( + self.device) # [sentence_num,max_sent_len,3,1024]--[sentence_num,max_sent,1024] + elmo_embedding = elmo_embedding[:, 0, :, :] * self.elmo_args[0] + elmo_embedding[:, 1, :, :] * \ + self.elmo_args[1] + elmo_embedding[:, 2, :, :] * self.elmo_args[2] + word_emb_list.append(elmo_embedding) + # print(word_emb_list[0].shape) + # print(word_emb_list[1].shape) + # print(word_emb_list[2].shape) + # print(word_emb_list[3].shape) + # print(word_emb_list[4].shape) + + word_emb = torch.cat(word_emb_list, dim=2) + + word_emb = self.emb_dropout(word_emb) + # word_emb_elmo = self.emb_dropout(word_emb_elmo) + lstm_out = self._reorder_lstm(word_emb, seq_len) + flatten_lstm = self.flat_lstm(lstm_out, seq_len) # [word_num,emb] + flatten_lstm = self.bilstm_drop(flatten_lstm) + # TODO 没有按照论文写 + flatten_word_emb = self.flat_lstm(word_emb, seq_len) # [word_num,emb] + + mention_start, mention_end = self.get_mention_start_end(seq_len) # [mention_num] + self.mention_start_np = mention_start # [mention_num] np + self.mention_end_np = mention_end + mention_num = mention_start.shape[0] + emb_start, emb_end = self.get_mention_emb(flatten_lstm, mention_start, mention_end) # [mention_num,emb] + + # list + mention_emb_list.append(emb_start) + mention_emb_list.append(emb_end) + + if self.config.use_width: + mention_width_index = mention_end - mention_start + mention_width_tensor = torch.from_numpy(mention_width_index).to(self.device) # [mention_num] + mention_width_emb = self.feature_emb(mention_width_tensor) + mention_width_emb = self.feature_emb_dropout(mention_width_emb) + mention_emb_list.append(mention_width_emb) + + if self.config.model_heads: + mention_index = self.get_mention_index(mention_start, self.config.span_width) # [mention_num,max_mention] + log_mask_tensor = self.get_mask(mention_start, mention_end).float().to( + self.device) # [mention_num,max_mention] + alpha = self.atten(flatten_lstm).to(self.device) # [word_num] + + # 得到attention + mention_head_score = torch.gather(alpha.expand(mention_num, -1), 1, + mention_index).float().to(self.device) # [mention_num,max_mention] + mention_attention = F.softmax(mention_head_score + log_mask_tensor, dim=1) # [mention_num,max_mention] + + # TODO flatte lstm + word_num = flatten_lstm.shape[0] + lstm_emb = flatten_lstm.shape[1] + emb_num = flatten_word_emb.shape[1] + + # [num_mentions, max_mention_width, emb] + mention_text_emb = torch.gather( + flatten_word_emb.unsqueeze(1).expand(word_num, self.config.span_width, emb_num), + 0, mention_index.unsqueeze(2).expand(mention_num, self.config.span_width, + emb_num)) + # [mention_num,emb] + mention_head_emb = torch.sum( + mention_attention.unsqueeze(2).expand(mention_num, self.config.span_width, emb_num) * mention_text_emb, + dim=1) + mention_emb_list.append(mention_head_emb) + + candidate_mention_emb = torch.cat(mention_emb_list, 1) # [candidate_mention_num,emb] + candidate_mention_score = self.mention_score(candidate_mention_emb) # [candidate_mention_num] + + antecedent_scores, antecedents, mention_start_tensor, mention_end_tensor = (None, None, None, None) + mention_start_tensor, mention_end_tensor, mention_score, mention_emb = \ + self.sort_mention(mention_start, mention_end, candidate_mention_emb, candidate_mention_score, seq_len) + mention_speakers_ids = speakers_tensor.index_select(dim=0, index=mention_start_tensor) # num_mention + + antecedents, antecedents_len = self.get_antecedents(mention_start_tensor, self.config.max_antecedents) + antecedent_scores = self.get_antecedents_score(mention_emb, mention_score, antecedents, antecedents_len, + mention_speakers_ids, genre) + + ans = {"candidate_mention_score": candidate_mention_score, "antecedent_scores": antecedent_scores, + "antecedents": antecedents, "mention_start_tensor": mention_start_tensor, + "mention_end_tensor": mention_end_tensor} + + return ans + + def predict(self, sentences, doc_np, speaker_ids_np, genre, char_index, seq_len): + ans = self(sentences, + doc_np, + speaker_ids_np, + genre, + char_index, + seq_len) + + predicted_antecedents = self.get_predicted_antecedents(ans["antecedents"], ans["antecedent_scores"]) + predicted_clusters, mention_to_predicted = self.get_predicted_clusters(ans["mention_start_tensor"], + ans["mention_end_tensor"], + predicted_antecedents) + + return {'predicted':predicted_clusters,"mention_to_predicted":mention_to_predicted} + + +if __name__ == '__main__': + pass diff --git a/reproduction/coreference_resolution/model/preprocess.py b/reproduction/coreference_resolution/model/preprocess.py new file mode 100644 index 00000000..d97fcb4d --- /dev/null +++ b/reproduction/coreference_resolution/model/preprocess.py @@ -0,0 +1,225 @@ +import json +import numpy as np +from . import util +import collections + +def load(path): + """ + load the file from jsonline + :param path: + :return: examples with many example(dict): {"clusters":[[[mention],[mention]],[another cluster]], + "doc_key":"str","speakers":[[,,,],[]...],"sentence":[[][]]} + """ + with open(path) as f: + train_examples = [json.loads(jsonline) for jsonline in f.readlines()] + return train_examples + +def get_vocab(): + """ + 从所有的句子中得到最终的字典,被main调用,不止是train，还有dev和test + :param examples: + :return: word2id & id2word + """ + word2id = {'PAD':0,'UNK':1} + id2word = {0:'PAD',1:'UNK'} + index = 2 + data = [load("../data/train.english.jsonlines"),load("../data/dev.english.jsonlines"),load("../data/test.english.jsonlines")] + for examples in data: + for example in examples: + for sent in example["sentences"]: + for word in sent: + if(word not in word2id): + word2id[word]=index + id2word[index] = word + index += 1 + return word2id,id2word + +def normalize(v): + norm = np.linalg.norm(v) + if norm > 0: + return v / norm + else: + return v + +# 加载glove得到embedding +def get_emb(id2word,embedding_size): + glove_oov = 0 + turian_oov = 0 + both = 0 + glove_emb_path = "../data/glove.840B.300d.txt.filtered" + turian_emb_path = "../data/turian.50d.txt" + word_num = len(id2word) + emb = np.zeros((word_num,embedding_size)) + glove_emb_dict = util.load_embedding_dict(glove_emb_path,300,"txt") + turian_emb_dict = util.load_embedding_dict(turian_emb_path,50,"txt") + for i in range(word_num): + if id2word[i] in glove_emb_dict: + word_embedding = glove_emb_dict.get(id2word[i]) + emb[i][0:300] = np.array(word_embedding) + else: + # print(id2word[i]) + glove_oov += 1 + if id2word[i] in turian_emb_dict: + word_embedding = turian_emb_dict.get(id2word[i]) + emb[i][300:350] = np.array(word_embedding) + else: + # print(id2word[i]) + turian_oov += 1 + if id2word[i] not in glove_emb_dict and id2word[i] not in turian_emb_dict: + both += 1 + emb[i] = normalize(emb[i]) + print("embedding num:"+str(word_num)) + print("glove num:"+str(glove_oov)) + print("glove oov rate:"+str(glove_oov/word_num)) + print("turian num:"+str(turian_oov)) + print("turian oov rate:"+str(turian_oov/word_num)) + print("both num:"+str(both)) + return emb + + +def _doc2vec(doc,word2id,char_dict,max_filter,max_sentences,is_train): + max_len = 0 + max_word_length = 0 + docvex = [] + length = [] + if is_train: + sent_num = min(max_sentences,len(doc)) + else: + sent_num = len(doc) + + for i in range(sent_num): + sent = doc[i] + length.append(len(sent)) + if (len(sent) > max_len): + max_len = len(sent) + sent_vec =[] + for j,word in enumerate(sent): + if len(word)>max_word_length: + max_word_length = len(word) + if word in word2id: + sent_vec.append(word2id[word]) + else: + sent_vec.append(word2id["UNK"]) + docvex.append(sent_vec) + + char_index = np.zeros((sent_num, max_len, max_word_length),dtype=int) + for i in range(sent_num): + sent = doc[i] + for j,word in enumerate(sent): + char_index[i, j, :len(word)] = [char_dict[c] for c in word] + + return docvex,char_index,length,max_len + +# TODO 修改了接口，确认所有该修改的地方都修改好 +def doc2numpy(doc,word2id,chardict,max_filter,max_sentences,is_train): + docvec, char_index, length, max_len = _doc2vec(doc,word2id,chardict,max_filter,max_sentences,is_train) + assert max(length) == max_len + assert char_index.shape[0]==len(length) + assert char_index.shape[1]==max_len + doc_np = np.zeros((len(docvec), max_len), int) + for i in range(len(docvec)): + for j in range(len(docvec[i])): + doc_np[i][j] = docvec[i][j] + return doc_np,char_index,length + +# TODO 没有测试 +def speaker2numpy(speakers_raw,max_sentences,is_train): + if is_train and len(speakers_raw)> max_sentences: + speakers_raw = speakers_raw[0:max_sentences] + speakers = flatten(speakers_raw) + speaker_dict = {s: i for i, s in enumerate(set(speakers))} + speaker_ids = np.array([speaker_dict[s] for s in speakers]) + return speaker_ids + + +def flat_cluster(clusters): + flatted = [] + for cluster in clusters: + for item in cluster: + flatted.append(item) + return flatted + +def get_right_mention(clusters,mention_start_np,mention_end_np): + flatted = flat_cluster(clusters) + cluster_num = len(flatted) + mention_num = mention_start_np.shape[0] + right_mention = np.zeros(mention_num,dtype=int) + for i in range(mention_num): + if [mention_start_np[i],mention_end_np[i]] in flatted: + right_mention[i]=1 + return right_mention,cluster_num + +def handle_cluster(clusters): + gold_mentions = sorted(tuple(m) for m in flatten(clusters)) + gold_mention_map = {m: i for i, m in enumerate(gold_mentions)} + cluster_ids = np.zeros(len(gold_mentions), dtype=int) + for cluster_id, cluster in enumerate(clusters): + for mention in cluster: + cluster_ids[gold_mention_map[tuple(mention)]] = cluster_id + gold_starts, gold_ends = tensorize_mentions(gold_mentions) + return cluster_ids, gold_starts, gold_ends + +# 展平 +def flatten(l): + return [item for sublist in l for item in sublist] + +# 把mention分成start end +def tensorize_mentions(mentions): + if len(mentions) > 0: + starts, ends = zip(*mentions) + else: + starts, ends = [], [] + return np.array(starts), np.array(ends) + +def get_char_dict(path): + vocab = [""] + with open(path) as f: + vocab.extend(c.strip() for c in f.readlines()) + char_dict = collections.defaultdict(int) + char_dict.update({c: i for i, c in enumerate(vocab)}) + return char_dict + +def get_labels(clusters,mention_starts,mention_ends,max_antecedents): + cluster_ids, gold_starts, gold_ends = handle_cluster(clusters) + num_mention = mention_starts.shape[0] + num_gold = gold_starts.shape[0] + max_antecedents = min(max_antecedents, num_mention) + mention_indices = {} + + for i in range(num_mention): + mention_indices[(mention_starts[i].detach().item(), mention_ends[i].detach().item())] = i + # 用来记录哪些mention是对的，-1表示错误，正数代表这个mention实际上对应哪个gold cluster的id + mention_cluster_ids = [-1] * num_mention + # test + right_mention_count = 0 + for i in range(num_gold): + right_mention = mention_indices.get((gold_starts[i], gold_ends[i])) + if (right_mention != None): + right_mention_count += 1 + mention_cluster_ids[right_mention] = cluster_ids[i] + + # i j 是否属于同一个cluster + labels = np.zeros((num_mention, max_antecedents + 1), dtype=bool) # [num_mention,max_an+1] + for i in range(num_mention): + ante_count = 0 + null_label = True + for j in range(max(0, i - max_antecedents), i): + if (mention_cluster_ids[i] >= 0 and mention_cluster_ids[i] == mention_cluster_ids[j]): + labels[i, ante_count + 1] = True + null_label = False + else: + labels[i, ante_count + 1] = False + ante_count += 1 + for j in range(ante_count, max_antecedents): + labels[i, j + 1] = False + labels[i, 0] = null_label + return labels + +# test=========================== + + +if __name__=="__main__": + word2id,id2word = get_vocab() + get_emb(id2word,350) + + diff --git a/reproduction/coreference_resolution/model/softmax_loss.py b/reproduction/coreference_resolution/model/softmax_loss.py new file mode 100644 index 00000000..c75a31d6 --- /dev/null +++ b/reproduction/coreference_resolution/model/softmax_loss.py @@ -0,0 +1,32 @@ +from fastNLP.core.losses import LossBase + +from reproduction.coreference_resolution.model.preprocess import get_labels +from reproduction.coreference_resolution.model.config import Config +import torch + + +class SoftmaxLoss(LossBase): + """ + 交叉熵loss + 允许多标签分类 + """ + + def __init__(self, antecedent_scores=None, clusters=None, mention_start_tensor=None, mention_end_tensor=None): + """ + + :param pred: + :param target: + """ + super().__init__() + self._init_param_map(antecedent_scores=antecedent_scores, clusters=clusters, + mention_start_tensor=mention_start_tensor, mention_end_tensor=mention_end_tensor) + + def get_loss(self, antecedent_scores, clusters, mention_start_tensor, mention_end_tensor): + antecedent_labels = get_labels(clusters[0], mention_start_tensor, mention_end_tensor, + Config().max_antecedents) + + antecedent_labels = torch.from_numpy(antecedent_labels*1).to(torch.device("cuda:" + Config().cuda)) + gold_scores = antecedent_scores + torch.log(antecedent_labels.float()).to(torch.device("cuda:" + Config().cuda)) # [num_mentions, max_ant + 1] + marginalized_gold_scores = gold_scores.logsumexp(dim=1) # [num_mentions] + log_norm = antecedent_scores.logsumexp(dim=1) # [num_mentions] + return torch.sum(log_norm - marginalized_gold_scores) diff --git a/reproduction/coreference_resolution/model/util.py b/reproduction/coreference_resolution/model/util.py new file mode 100644 index 00000000..42cd09fe --- /dev/null +++ b/reproduction/coreference_resolution/model/util.py @@ -0,0 +1,101 @@ +import os +import errno +import collections +import torch +import numpy as np +import pyhocon + + + +# flatten the list +def flatten(l): + return [item for sublist in l for item in sublist] + + +def get_config(filename): + return pyhocon.ConfigFactory.parse_file(filename) + + +# safe make directions +def mkdirs(path): + try: + os.makedirs(path) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + return path + + +def load_char_dict(char_vocab_path): + vocab = [""] + with open(char_vocab_path) as f: + vocab.extend(c.strip() for c in f.readlines()) + char_dict = collections.defaultdict(int) + char_dict.update({c: i for i, c in enumerate(vocab)}) + return char_dict + +# 加载embedding +def load_embedding_dict(embedding_path, embedding_size, embedding_format): + print("Loading word embeddings from {}...".format(embedding_path)) + default_embedding = np.zeros(embedding_size) + embedding_dict = collections.defaultdict(lambda: default_embedding) + skip_first = embedding_format == "vec" + with open(embedding_path) as f: + for i, line in enumerate(f.readlines()): + if skip_first and i == 0: + continue + splits = line.split() + assert len(splits) == embedding_size + 1 + word = splits[0] + embedding = np.array([float(s) for s in splits[1:]]) + embedding_dict[word] = embedding + print("Done loading word embeddings.") + return embedding_dict + + +# safe devide +def maybe_divide(x, y): + return 0 if y == 0 else x / float(y) + + +def shape(x, dim): + return x.get_shape()[dim].value or torch.shape(x)[dim] + + +def normalize(v): + norm = np.linalg.norm(v) + if norm > 0: + return v / norm + else: + return v + + +class RetrievalEvaluator(object): + def __init__(self): + self._num_correct = 0 + self._num_gold = 0 + self._num_predicted = 0 + + def update(self, gold_set, predicted_set): + self._num_correct += len(gold_set & predicted_set) + self._num_gold += len(gold_set) + self._num_predicted += len(predicted_set) + + def recall(self): + return maybe_divide(self._num_correct, self._num_gold) + + def precision(self): + return maybe_divide(self._num_correct, self._num_predicted) + + def metrics(self): + recall = self.recall() + precision = self.precision() + f1 = maybe_divide(2 * recall * precision, precision + recall) + return recall, precision, f1 + + + +if __name__=="__main__": + print(load_char_dict("../data/char_vocab.english.txt")) + embedding_dict = load_embedding_dict("../data/glove.840B.300d.txt.filtered",300,"txt") + print("hello") diff --git a/reproduction/coreference_resolution/readme.md b/reproduction/coreference_resolution/readme.md new file mode 100644 index 00000000..67d8cdc7 --- /dev/null +++ b/reproduction/coreference_resolution/readme.md @@ -0,0 +1,49 @@ +# 共指消解复现 +## 介绍 +Coreference resolution是查找文本中指向同一现实实体的所有表达式的任务。 +对于涉及自然语言理解的许多更高级别的NLP任务来说， +这是一个重要的步骤，例如文档摘要，问题回答和信息提取。 +代码的实现主要基于[ End-to-End Coreference Resolution (Lee et al, 2017)](https://arxiv.org/pdf/1707.07045). + + +## 数据获取与预处理 +论文在[OntoNote5.0](https://allennlp.org/models)数据集上取得了当时的sota结果。 +由于版权问题，本文无法提供数据集的下载，请自行下载。 +原始数据集的格式为conll格式，详细介绍参考数据集给出的官方介绍页面。 + +代码实现采用了论文作者Lee的预处理方法，具体细节参加[链接](https://github.com/kentonl/e2e-coref/blob/e2e/setup_training.sh)。 +处理之后的数据集为json格式，例子： +``` +{ + "clusters": [], + "doc_key": "nw", + "sentences": [["This", "is", "the", "first", "sentence", "."], ["This", "is", "the", "second", "."]], + "speakers": [["spk1", "spk1", "spk1", "spk1", "spk1", "spk1"], ["spk2", "spk2", "spk2", "spk2", "spk2"]] +} +``` + +### embedding 数据集下载 +[turian emdedding](https://lil.cs.washington.edu/coref/turian.50d.txt) + +[glove embedding]( https://nlp.stanford.edu/data/glove.840B.300d.zip) + + + +## 运行 +```python +# 训练代码 +CUDA_VISIBLE_DEVICES=0 python train.py +# 测试代码 +CUDA_VISIBLE_DEVICES=0 python valid.py +``` + +## 结果 +原论文作者在测试集上取得了67.2%的结果，AllenNLP复现的结果为 [63.0%](https://allennlp.org/models)。 +其中allenNLP训练时没有加入speaker信息，没有variational dropout以及只使用了100的antecedents而不是250。 + +在与allenNLP使用同样的超参和配置时，本代码复现取得了63.6%的F1值。 + + +## 问题 +如果您有什么问题或者反馈，请提issue或者邮件联系我： +yexu_i@qq.com diff --git a/reproduction/coreference_resolution/test/__init__.py b/reproduction/coreference_resolution/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reproduction/coreference_resolution/test/test_dataloader.py b/reproduction/coreference_resolution/test/test_dataloader.py new file mode 100644 index 00000000..0d9dae52 --- /dev/null +++ b/reproduction/coreference_resolution/test/test_dataloader.py @@ -0,0 +1,14 @@ +import unittest +from ..data_load.cr_loader import CRLoader + +class Test_CRLoader(unittest.TestCase): + def test_cr_loader(self): + train_path = 'data/train.english.jsonlines.mini' + dev_path = 'data/dev.english.jsonlines.minid' + test_path = 'data/test.english.jsonlines' + cr = CRLoader() + data_info = cr.process({'train':train_path,'dev':dev_path,'test':test_path}) + + print(data_info.datasets['train'][0]) + print(data_info.datasets['dev'][0]) + print(data_info.datasets['test'][0]) diff --git a/reproduction/coreference_resolution/train.py b/reproduction/coreference_resolution/train.py new file mode 100644 index 00000000..a231a575 --- /dev/null +++ b/reproduction/coreference_resolution/train.py @@ -0,0 +1,69 @@ +import sys +sys.path.append('../..') + +import torch +from torch.optim import Adam + +from fastNLP.core.callback import Callback, GradientClipCallback +from fastNLP.core.trainer import Trainer + +from reproduction.coreference_resolution.data_load.cr_loader import CRLoader +from reproduction.coreference_resolution.model.config import Config +from reproduction.coreference_resolution.model.model_re import Model +from reproduction.coreference_resolution.model.softmax_loss import SoftmaxLoss +from reproduction.coreference_resolution.model.metric import CRMetric +from fastNLP import SequentialSampler +from fastNLP import cache_results + + +# torch.backends.cudnn.benchmark = False +# torch.backends.cudnn.deterministic = True + +class LRCallback(Callback): + def __init__(self, parameters, decay_rate=1e-3): + super().__init__() + self.paras = parameters + self.decay_rate = decay_rate + + def on_step_end(self): + if self.step % 100 == 0: + for para in self.paras: + para['lr'] = para['lr'] * (1 - self.decay_rate) + + +if __name__ == "__main__": + config = Config() + + print(config) + + @cache_results('cache.pkl') + def cache(): + cr_train_dev_test = CRLoader() + + data_info = cr_train_dev_test.process({'train': config.train_path, 'dev': config.dev_path, + 'test': config.test_path}) + return data_info + data_info = cache() + print("数据集划分：\ntrain:", str(len(data_info.datasets["train"])), + "\ndev:" + str(len(data_info.datasets["dev"])) + "\ntest:" + str(len(data_info.datasets["test"]))) + # print(data_info) + model = Model(data_info.vocabs, config) + print(model) + + loss = SoftmaxLoss() + + metric = CRMetric() + + optim = Adam(model.parameters(), lr=config.lr) + + lr_decay_callback = LRCallback(optim.param_groups, config.lr_decay) + + trainer = Trainer(model=model, train_data=data_info.datasets["train"], dev_data=data_info.datasets["dev"], + loss=loss, metrics=metric, check_code_level=-1,sampler=None, + batch_size=1, device=torch.device("cuda:" + config.cuda), metric_key='f', n_epochs=config.epoch, + optimizer=optim, + save_path='/remote-home/xxliu/pycharm/fastNLP/fastNLP/reproduction/coreference_resolution/save', + callbacks=[lr_decay_callback, GradientClipCallback(clip_value=5)]) + print() + + trainer.train() diff --git a/reproduction/coreference_resolution/valid.py b/reproduction/coreference_resolution/valid.py new file mode 100644 index 00000000..826332c6 --- /dev/null +++ b/reproduction/coreference_resolution/valid.py @@ -0,0 +1,24 @@ +import torch +from reproduction.coreference_resolution.model.config import Config +from reproduction.coreference_resolution.model.metric import CRMetric +from reproduction.coreference_resolution.data_load.cr_loader import CRLoader +from fastNLP import Tester +import argparse + + +if __name__=='__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--path') + args = parser.parse_args() + + cr_loader = CRLoader() + config = Config() + data_info = cr_loader.process({'train': config.train_path, 'dev': config.dev_path, + 'test': config.test_path}) + metirc = CRMetric() + model = torch.load(args.path) + tester = Tester(data_info.datasets['test'],model,metirc,batch_size=1,device="cuda:0") + tester.test() + print('test over') + + From d05aca6da62ecf1d7f41bbeeb03ea470a40ca165 Mon Sep 17 00:00:00 2001 From: lyhuang18 <42239874+lyhuang18@users.noreply.github.com> Date: Sun, 7 Jul 2019 08:21:51 +0800 Subject: [PATCH 11/27] TC/LSTM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LSTM、AWDLSTM和LSTM+self attention三个模型 --- .../text_classification/data/IMDBLoader.py | 80 +++++++++++++ .../text_classification/data/MTL16Loader.py | 6 +- .../text_classification/data/SSTLoader.py | 99 ++++++++++++++++ .../text_classification/data/yelpLoader.py | 111 ++++++++++-------- .../text_classification/model/awd_lstm.py | 31 +++++ .../model/awdlstm_module.py | 86 ++++++++++++++ .../text_classification/model/lstm.py | 30 +++++ .../model/lstm_self_attention.py | 35 ++++++ .../text_classification/model/weight_drop.py | 99 ++++++++++++++++ .../text_classification/results_LSTM.xlsx | Bin 0 -> 9944 bytes .../text_classification/train_awdlstm.py | 102 ++++++++++++++++ .../text_classification/train_lstm.py | 99 ++++++++++++++++ .../text_classification/train_lstm_att.py | 101 ++++++++++++++++ 13 files changed, 827 insertions(+), 52 deletions(-) create mode 100644 reproduction/text_classification/data/IMDBLoader.py create mode 100644 reproduction/text_classification/data/SSTLoader.py create mode 100644 reproduction/text_classification/model/awd_lstm.py create mode 100644 reproduction/text_classification/model/awdlstm_module.py create mode 100644 reproduction/text_classification/model/lstm.py create mode 100644 reproduction/text_classification/model/lstm_self_attention.py create mode 100644 reproduction/text_classification/model/weight_drop.py create mode 100644 reproduction/text_classification/results_LSTM.xlsx create mode 100644 reproduction/text_classification/train_awdlstm.py create mode 100644 reproduction/text_classification/train_lstm.py create mode 100644 reproduction/text_classification/train_lstm_att.py diff --git a/reproduction/text_classification/data/IMDBLoader.py b/reproduction/text_classification/data/IMDBLoader.py new file mode 100644 index 00000000..d591cdf8 --- /dev/null +++ b/reproduction/text_classification/data/IMDBLoader.py @@ -0,0 +1,80 @@ +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader +from fastNLP.core.vocabulary import VocabularyOption +from fastNLP.io.base_loader import DataSetLoader, DataInfo +from typing import Union, Dict, List, Iterator +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP import Vocabulary +from fastNLP import Const +# from reproduction.utils import check_dataloader_paths +from functools import partial + +class IMDBLoader(DataSetLoader): + """ + 读取IMDB数据集，DataSet包含以下fields: + + words: list(str), 需要分类的文本 + target: str, 文本的标签 + + + """ + + def __init__(self): + super(IMDBLoader, self).__init__() + + def _load(self, path): + dataset = DataSet() + with open(path, 'r', encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split('\t') + target = parts[0] + words = parts[1].lower().split() + dataset.append(Instance(words=words, target=target)) + if len(dataset)==0: + raise RuntimeError(f"{path} has no valid data.") + + return dataset + + def process(self, + paths: Union[str, Dict[str, str]], + src_vocab_opt: VocabularyOption = None, + tgt_vocab_opt: VocabularyOption = None, + src_embed_opt: EmbeddingOption = None): + + # paths = check_dataloader_paths(paths) + datasets = {} + info = DataInfo() + for name, path in paths.items(): + dataset = self.load(path) + datasets[name] = dataset + + datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False) + + src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) + src_vocab.from_dataset(datasets['train'], field_name='words') + src_vocab.index_dataset(*datasets.values(), field_name='words') + + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) + tgt_vocab.from_dataset(datasets['train'], field_name='target') + tgt_vocab.index_dataset(*datasets.values(), field_name='target') + + info.vocabs = { + "words": src_vocab, + "target": tgt_vocab + } + + info.datasets = datasets + + if src_embed_opt is not None: + embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) + info.embeddings['words'] = embed + + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + + return info diff --git a/reproduction/text_classification/data/MTL16Loader.py b/reproduction/text_classification/data/MTL16Loader.py index 1b3e6245..066b53b4 100644 --- a/reproduction/text_classification/data/MTL16Loader.py +++ b/reproduction/text_classification/data/MTL16Loader.py @@ -32,7 +32,7 @@ class MTL16Loader(DataSetLoader): continue parts = line.split('\t') target = parts[0] - words = parts[1].split() + words = parts[1].lower().split() dataset.append(Instance(words=words, target=target)) if len(dataset)==0: raise RuntimeError(f"{path} has no valid data.") @@ -72,4 +72,8 @@ class MTL16Loader(DataSetLoader): embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) info.embeddings['words'] = embed + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + return info diff --git a/reproduction/text_classification/data/SSTLoader.py b/reproduction/text_classification/data/SSTLoader.py new file mode 100644 index 00000000..b570994e --- /dev/null +++ b/reproduction/text_classification/data/SSTLoader.py @@ -0,0 +1,99 @@ +from typing import Iterable +from nltk import Tree +from fastNLP.io.base_loader import DataInfo, DataSetLoader +from fastNLP.core.vocabulary import VocabularyOption, Vocabulary +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader + + +class SSTLoader(DataSetLoader): + URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' + DATA_DIR = 'sst/' + + """ + 别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader` + + 读取SST数据集, DataSet包含fields:: + + words: list(str) 需要分类的文本 + target: str 文本的标签 + + 数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip + + :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False`` + :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` + """ + + def __init__(self, subtree=False, fine_grained=False): + self.subtree = subtree + + tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral', + '3': 'positive', '4': 'very positive'} + if not fine_grained: + tag_v['0'] = tag_v['1'] + tag_v['4'] = tag_v['3'] + self.tag_v = tag_v + + def _load(self, path): + """ + + :param str path: 存储数据的路径 + :return: 一个 :class:`~fastNLP.DataSet` 类型的对象 + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + datas = [] + for l in f: + datas.extend([(s, self.tag_v[t]) + for s, t in self._get_one(l, self.subtree)]) + ds = DataSet() + for words, tag in datas: + ds.append(Instance(words=words, target=tag)) + return ds + + @staticmethod + def _get_one(data, subtree): + tree = Tree.fromstring(data) + if subtree: + return [(t.leaves(), t.label()) for t in tree.subtrees()] + return [(tree.leaves(), tree.label())] + + def process(self, + paths, + train_ds: Iterable[str] = None, + src_vocab_op: VocabularyOption = None, + tgt_vocab_op: VocabularyOption = None, + src_embed_op: EmbeddingOption = None): + input_name, target_name = 'words', 'target' + src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) + + info = DataInfo(datasets=self.load(paths)) + _train_ds = [info.datasets[name] + for name in train_ds] if train_ds else info.datasets.values() + src_vocab.from_dataset(*_train_ds, field_name=input_name) + tgt_vocab.from_dataset(*_train_ds, field_name=target_name) + src_vocab.index_dataset( + *info.datasets.values(), + field_name=input_name, new_field_name=input_name) + tgt_vocab.index_dataset( + *info.datasets.values(), + field_name=target_name, new_field_name=target_name) + info.vocabs = { + input_name: src_vocab, + target_name: tgt_vocab + } + + if src_embed_op is not None: + src_embed_op.vocab = src_vocab + init_emb = EmbedLoader.load_with_vocab(**src_embed_op) + info.embeddings[input_name] = init_emb + + for name, dataset in info.datasets.items(): + dataset.set_input(input_name) + dataset.set_target(target_name) + + return info + diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index c47d48fd..680b3488 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -1,68 +1,77 @@ -import ast -from fastNLP import DataSet, Instance, Vocabulary +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader from fastNLP.core.vocabulary import VocabularyOption -from fastNLP.io import JsonLoader -from fastNLP.io.base_loader import DataInfo -from fastNLP.io.embed_loader import EmbeddingOption -from fastNLP.io.file_reader import _read_json -from typing import Union, Dict -from reproduction.Star_transformer.datasets import EmbedLoader -from reproduction.utils import check_dataloader_paths +from fastNLP.io.base_loader import DataSetLoader, DataInfo +from typing import Union, Dict, List, Iterator +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP import Vocabulary +from fastNLP import Const +# from reproduction.utils import check_dataloader_paths +from functools import partial +import pandas as pd - -class yelpLoader(JsonLoader): - +class yelpLoader(DataSetLoader): """ - 读取Yelp数据集, DataSet包含fields: - - review_id: str, 22 character unique review id - user_id: str, 22 character unique user id - business_id: str, 22 character business id - useful: int, number of useful votes received - funny: int, number of funny votes received - cool: int, number of cool votes received - date: str, date formatted YYYY-MM-DD + 读取IMDB数据集，DataSet包含以下fields: + words: list(str), 需要分类的文本 target: str, 文本的标签 - - 数据来源: https://www.yelp.com/dataset/download - - :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` + + """ - - def __init__(self, fine_grained=False): + + def __init__(self): super(yelpLoader, self).__init__() - tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral', - '4.0': 'positive', '5.0': 'very positive'} - if not fine_grained: - tag_v['1.0'] = tag_v['2.0'] - tag_v['5.0'] = tag_v['4.0'] - self.fine_grained = fine_grained - self.tag_v = tag_v - + def _load(self, path): - ds = DataSet() - for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): - d = ast.literal_eval(d) - d["words"] = d.pop("text").split() - d["target"] = self.tag_v[str(d.pop("stars"))] - ds.append(Instance(**d)) - return ds + dataset = DataSet() + data = pd.read_csv(path, header=None, sep=",").values + for line in data: + target = str(line[0]) + words = str(line[1]).lower().split() + dataset.append(Instance(words=words, target=target)) + if len(dataset)==0: + raise RuntimeError(f"{path} has no valid data.") - def process(self, paths: Union[str, Dict[str, str]], vocab_opt: VocabularyOption = None, - embed_opt: EmbeddingOption = None): - paths = check_dataloader_paths(paths) + return dataset + + def process(self, + paths: Union[str, Dict[str, str]], + src_vocab_opt: VocabularyOption = None, + tgt_vocab_opt: VocabularyOption = None, + src_embed_opt: EmbeddingOption = None): + + # paths = check_dataloader_paths(paths) datasets = {} info = DataInfo() - vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) for name, path in paths.items(): dataset = self.load(path) datasets[name] = dataset - vocab.from_dataset(dataset, field_name="words") - info.vocabs = vocab + + datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False) + + src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) + src_vocab.from_dataset(datasets['train'], field_name='words') + src_vocab.index_dataset(*datasets.values(), field_name='words') + + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) + tgt_vocab.from_dataset(datasets['train'], field_name='target') + tgt_vocab.index_dataset(*datasets.values(), field_name='target') + + info.vocabs = { + "words": src_vocab, + "target": tgt_vocab + } + info.datasets = datasets - if embed_opt is not None: - embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) + + if src_embed_opt is not None: + embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) info.embeddings['words'] = embed - return info + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + + return info diff --git a/reproduction/text_classification/model/awd_lstm.py b/reproduction/text_classification/model/awd_lstm.py new file mode 100644 index 00000000..0d8f711a --- /dev/null +++ b/reproduction/text_classification/model/awd_lstm.py @@ -0,0 +1,31 @@ +import torch +import torch.nn as nn +from fastNLP.core.const import Const as C +from .awdlstm_module import LSTM +from fastNLP.modules import encoder +from fastNLP.modules.decoder.mlp import MLP + + +class AWDLSTMSentiment(nn.Module): + def __init__(self, init_embed, + num_classes, + hidden_dim=256, + num_layers=1, + nfc=128, + wdrop=0.5): + super(AWDLSTMSentiment,self).__init__() + self.embed = encoder.Embedding(init_embed) + self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True, wdrop=wdrop) + self.mlp = MLP(size_layer=[hidden_dim* 2, nfc, num_classes]) + + def forward(self, words): + x_emb = self.embed(words) + output, _ = self.lstm(x_emb) + output = self.mlp(output[:,-1,:]) + return {C.OUTPUT: output} + + def predict(self, words): + output = self(words) + _, predict = output[C.OUTPUT].max(dim=1) + return {C.OUTPUT: predict} + diff --git a/reproduction/text_classification/model/awdlstm_module.py b/reproduction/text_classification/model/awdlstm_module.py new file mode 100644 index 00000000..87bfe730 --- /dev/null +++ b/reproduction/text_classification/model/awdlstm_module.py @@ -0,0 +1,86 @@ +""" +轻量封装的 Pytorch LSTM 模块. +可在 forward 时传入序列的长度, 自动对padding做合适的处理. +""" +__all__ = [ + "LSTM" +] + +import torch +import torch.nn as nn +import torch.nn.utils.rnn as rnn + +from fastNLP.modules.utils import initial_parameter +from torch import autograd +from .weight_drop import WeightDrop + + +class LSTM(nn.Module): + """ + 别名：:class:`fastNLP.modules.LSTM` :class:`fastNLP.modules.encoder.lstm.LSTM` + + LSTM 模块, 轻量封装的Pytorch LSTM. 在提供seq_len的情况下，将自动使用pack_padded_sequence; 同时默认将forget gate的bias初始化 + 为1; 且可以应对DataParallel中LSTM的使用问题。 + + :param input_size: 输入 `x` 的特征维度 + :param hidden_size: 隐状态 `h` 的特征维度. + :param num_layers: rnn的层数. Default: 1 + :param dropout: 层间dropout概率. Default: 0 + :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` + :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 + :(batch, seq, feature). Default: ``False`` + :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` + """ + + def __init__(self, input_size, hidden_size=100, num_layers=1, dropout=0.0, batch_first=True, + bidirectional=False, bias=True, wdrop=0.5): + super(LSTM, self).__init__() + self.batch_first = batch_first + self.lstm = nn.LSTM(input_size, hidden_size, num_layers, bias=bias, batch_first=batch_first, + dropout=dropout, bidirectional=bidirectional) + self.lstm = WeightDrop(self.lstm, ['weight_hh_l0'], dropout=wdrop) + self.init_param() + + def init_param(self): + for name, param in self.named_parameters(): + if 'bias' in name: + # based on https://github.com/pytorch/pytorch/issues/750#issuecomment-280671871 + param.data.fill_(0) + n = param.size(0) + start, end = n // 4, n // 2 + param.data[start:end].fill_(1) + else: + nn.init.xavier_uniform_(param) + + def forward(self, x, seq_len=None, h0=None, c0=None): + """ + + :param x: [batch, seq_len, input_size] 输入序列 + :param seq_len: [batch, ] 序列长度, 若为 ``None``, 所有输入看做一样长. Default: ``None`` + :param h0: [batch, hidden_size] 初始隐状态, 若为 ``None`` , 设为全0向量. Default: ``None`` + :param c0: [batch, hidden_size] 初始Cell状态, 若为 ``None`` , 设为全0向量. Default: ``None`` + :return (output, ht) 或 output: 若 ``get_hidden=True`` [batch, seq_len, hidden_size*num_direction] 输出序列 + 和 [batch, hidden_size*num_direction] 最后时刻隐状态. + """ + batch_size, max_len, _ = x.size() + if h0 is not None and c0 is not None: + hx = (h0, c0) + else: + hx = None + if seq_len is not None and not isinstance(x, rnn.PackedSequence): + sort_lens, sort_idx = torch.sort(seq_len, dim=0, descending=True) + if self.batch_first: + x = x[sort_idx] + else: + x = x[:, sort_idx] + x = rnn.pack_padded_sequence(x, sort_lens, batch_first=self.batch_first) + output, hx = self.lstm(x, hx) # -> [N,L,C] + output, _ = rnn.pad_packed_sequence(output, batch_first=self.batch_first, total_length=max_len) + _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) + if self.batch_first: + output = output[unsort_idx] + else: + output = output[:, unsort_idx] + else: + output, hx = self.lstm(x, hx) + return output, hx diff --git a/reproduction/text_classification/model/lstm.py b/reproduction/text_classification/model/lstm.py new file mode 100644 index 00000000..388f3f1c --- /dev/null +++ b/reproduction/text_classification/model/lstm.py @@ -0,0 +1,30 @@ +import torch +import torch.nn as nn +from fastNLP.core.const import Const as C +from fastNLP.modules.encoder.lstm import LSTM +from fastNLP.modules import encoder +from fastNLP.modules.decoder.mlp import MLP + + +class BiLSTMSentiment(nn.Module): + def __init__(self, init_embed, + num_classes, + hidden_dim=256, + num_layers=1, + nfc=128): + super(BiLSTMSentiment,self).__init__() + self.embed = encoder.Embedding(init_embed) + self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True) + self.mlp = MLP(size_layer=[hidden_dim* 2, nfc, num_classes]) + + def forward(self, words): + x_emb = self.embed(words) + output, _ = self.lstm(x_emb) + output = self.mlp(output[:,-1,:]) + return {C.OUTPUT: output} + + def predict(self, words): + output = self(words) + _, predict = output[C.OUTPUT].max(dim=1) + return {C.OUTPUT: predict} + diff --git a/reproduction/text_classification/model/lstm_self_attention.py b/reproduction/text_classification/model/lstm_self_attention.py new file mode 100644 index 00000000..239635fe --- /dev/null +++ b/reproduction/text_classification/model/lstm_self_attention.py @@ -0,0 +1,35 @@ +import torch +import torch.nn as nn +from fastNLP.core.const import Const as C +from fastNLP.modules.encoder.lstm import LSTM +from fastNLP.modules import encoder +from fastNLP.modules.aggregator.attention import SelfAttention +from fastNLP.modules.decoder.mlp import MLP + + +class BiLSTM_SELF_ATTENTION(nn.Module): + def __init__(self, init_embed, + num_classes, + hidden_dim=256, + num_layers=1, + attention_unit=256, + attention_hops=1, + nfc=128): + super(BiLSTM_SELF_ATTENTION,self).__init__() + self.embed = encoder.Embedding(init_embed) + self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True) + self.attention = SelfAttention(input_size=hidden_dim * 2 , attention_unit=attention_unit, attention_hops=attention_hops) + self.mlp = MLP(size_layer=[hidden_dim* 2*attention_hops, nfc, num_classes]) + + def forward(self, words): + x_emb = self.embed(words) + output, _ = self.lstm(x_emb) + after_attention, penalty = self.attention(output,words) + after_attention =after_attention.view(after_attention.size(0),-1) + output = self.mlp(after_attention) + return {C.OUTPUT: output} + + def predict(self, words): + output = self(words) + _, predict = output[C.OUTPUT].max(dim=1) + return {C.OUTPUT: predict} diff --git a/reproduction/text_classification/model/weight_drop.py b/reproduction/text_classification/model/weight_drop.py new file mode 100644 index 00000000..60fda179 --- /dev/null +++ b/reproduction/text_classification/model/weight_drop.py @@ -0,0 +1,99 @@ +import torch +from torch.nn import Parameter +from functools import wraps + +class WeightDrop(torch.nn.Module): + def __init__(self, module, weights, dropout=0, variational=False): + super(WeightDrop, self).__init__() + self.module = module + self.weights = weights + self.dropout = dropout + self.variational = variational + self._setup() + + def widget_demagnetizer_y2k_edition(*args, **kwargs): + # We need to replace flatten_parameters with a nothing function + # It must be a function rather than a lambda as otherwise pickling explodes + # We can't write boring code though, so ... WIDGET DEMAGNETIZER Y2K EDITION! + # (╯°□°）╯︵ ┻━┻ + return + + def _setup(self): + # Terrible temporary solution to an issue regarding compacting weights re: CUDNN RNN + if issubclass(type(self.module), torch.nn.RNNBase): + self.module.flatten_parameters = self.widget_demagnetizer_y2k_edition + + for name_w in self.weights: + print('Applying weight drop of {} to {}'.format(self.dropout, name_w)) + w = getattr(self.module, name_w) + del self.module._parameters[name_w] + self.module.register_parameter(name_w + '_raw', Parameter(w.data)) + + def _setweights(self): + for name_w in self.weights: + raw_w = getattr(self.module, name_w + '_raw') + w = None + if self.variational: + mask = torch.autograd.Variable(torch.ones(raw_w.size(0), 1)) + if raw_w.is_cuda: mask = mask.cuda() + mask = torch.nn.functional.dropout(mask, p=self.dropout, training=True) + w = mask.expand_as(raw_w) * raw_w + else: + w = torch.nn.functional.dropout(raw_w, p=self.dropout, training=self.training) + setattr(self.module, name_w, w) + + def forward(self, *args): + self._setweights() + return self.module.forward(*args) + +if __name__ == '__main__': + import torch + from weight_drop import WeightDrop + + # Input is (seq, batch, input) + x = torch.autograd.Variable(torch.randn(2, 1, 10)).cuda() + h0 = None + + ### + + print('Testing WeightDrop') + print('=-=-=-=-=-=-=-=-=-=') + + ### + + print('Testing WeightDrop with Linear') + + lin = WeightDrop(torch.nn.Linear(10, 10), ['weight'], dropout=0.9) + lin.cuda() + run1 = [x.sum() for x in lin(x).data] + run2 = [x.sum() for x in lin(x).data] + + print('All items should be different') + print('Run 1:', run1) + print('Run 2:', run2) + + assert run1[0] != run2[0] + assert run1[1] != run2[1] + + print('---') + + ### + + print('Testing WeightDrop with LSTM') + + wdrnn = WeightDrop(torch.nn.LSTM(10, 10), ['weight_hh_l0'], dropout=0.9) + wdrnn.cuda() + + run1 = [x.sum() for x in wdrnn(x, h0)[0].data] + run2 = [x.sum() for x in wdrnn(x, h0)[0].data] + + print('First timesteps should be equal, all others should differ') + print('Run 1:', run1) + print('Run 2:', run2) + + # First time step, not influenced by hidden to hidden weights, should be equal + assert run1[0] == run2[0] + # Second step should not + assert run1[1] != run2[1] + + print('---') diff --git a/reproduction/text_classification/results_LSTM.xlsx b/reproduction/text_classification/results_LSTM.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..0d7b841b12b43ee346c4d9db17032fade8c0b40a GIT binary patch literal 9944 zcmeHNWmFtlw{0APHPE=by9Rd+1a}G2xVu{jmf!?;OK^8dkYEYHgIj>0!ChY`GxIXT zSV;*Y zQ45hC)h0rUXS_C5!#Y0QjUTPKibgeJC$)<0upZ5^AYS<_jgGkJsx8Zv&Qq#q$Zw*` zoeza>WwWJ%3(7R3YGYq==is64@57JRBRpxEHDR%)xTm1T;jVkcxzTx~v?X2|E+RqQ z&hV`S`f_?lXrN06T7ljT&xFtOxhgY({GwJ{n+gY~CRJs+3$X@E;&~K3s9L$X?ut}y zdvFYtV^D30&o4nLYE=gH){cV|g)q^$+P==7=ZFW(zITKjq(H%dru&($dEKO zat7PFu(JHv|2IAVgLCq4uU;0f__Ui1C3Ii*CanK_av>HduHYdi+e)tPA1F7EULTcD zMY!;emI$a$@Ek@apxyswaBe{`YP+BO>?>EL&<93FpCxe5>_EmD2`BSkZ!M{&Z5ah1=OUt z+V}F%sz%4$4_?0QnCA8Oe|(9eEV(5PcI2H)GQxcRheWt^HX>k znc2QSmP~(#b?eKll+mwB!G(FjF)Y(ZopTzXTgP?yG1IGu7pc3n|6m~aCF$EmNRRdJ zB;mXrhQ)#b05;(P0Cb3FJZ)J$9Gq=T92{(Z^kZcj1`g?LAiwOo`@kEe&XCHyB5Wkd z(g`<}R(n?2*ESi;*le|m2_@SBOWgEZ4Q;NZh_)=a(ZlWM`Yz55cGf#Vav!7wq%QiU zaBPa{yjrvJ(ua`+q?5)9nB|ytd8A^V;N03_5M|;nC^2; zOxZ=FJUAbF5`H0_!!*AcxhV<*4>Hga4rGfQc-GlVfBWO%JQmI;^%R?><^<9ZmXbSTw}hEt4w`G*JdfoV%`SrmhYGM z*wFP!2i#Ad@osrt4zAF5>#V_LIb3^f&UD16`S5tC5n4b+J0*Tdkh-zW<8EER{miqi z!Q5NdVC1D2IRsKPLFc;TOY4KuPVYQ_o!k<9LE}^xa29mv1{ad=xrwrcZ>V;%)R#_&?kIbnXhQS+ut?#FV^t`Dm%`tEvaH~hb?izQlIGI`!*`>c-O z4P3OcGci(}TF7k*-4s+zTT6p1t#E+{aP-k-nTMt0q>$|Rcm4zkCng_5T3R5T3;_TU z3X(s6^fQ0YqQCkaCkTYI%N}VAdwcVI2dVRL( z%)dfeCn{ASDd86K4Rs|8>z5%7ppSQ1&r5C9*k5rKzaFH2H+RJJ@nqk8ZJ=**3G$`? z9Wr<6;B66D0N^_s0DuR%;?I)nVhIMjy0HE{u>YvN8CrAE`Fub>BddF;%l77wd2Eg} z{Xl3$v~Y|ss+y3fF%wz*hnwgqRGtnhg)jG6l7(Q8~Fh^6_D<2cJ3-d0siYP3gF3qSfYi>@HKWOxyAN8cvh?29yIIEVV2q>jy zXiF>TftbSLp2y;AI5Ng4>Bvf-Pl|wxIUN_O6I8OW^4XGOEW^8lXl*IuU3#6tl2i)C z!I#OjhKcpMNrDu~DwqB&hL~~KWYw$mTG_AZs%y;dyg$7tXa)HOcuJ&clVbWK9BfP( zGqmB-uSS3h`RE7}i8(hqZc{m3EXU|hZ$+q9QHcjw^l1-RxG$RNz4E1P9C42WRBh+O zx7Op2IhPRDT79m)f*hU(G@ZwI?~2su?>;uuZmppbfL$x)jcJE+N_trmc=v5rx1oYr z%EN&P9AN_Np-ie6&IInKT|A0Kjt5JP#B-_KbbVn`gsI6Ajkwo?3sOK_=E9hra;vK< z*<32Dw?Ob#nk4;7!09hYk0?+lv#g#x=zHI642I@@lxr7I;D!|!jOeS2KZQ;>fp^Q6 zuL!BEU@kv=|v(d!SjJ;=0eVEj@lTkV%8!z>j;C(?^h`WivdJp!6{3tSUMXpo@g0&7Xdw zbO?1CCiHIma_YH?3ZsV-f)~yPYz&NUJmma=fhzQrvD@t?s*gD z{=Ca{)=GN;i9?(5v%g+wjk)G3rA9~2+ z@|}E9Vf(O`5WWEqksWAEW)DT|KtcqY<(Y5eKD7XPH;THR#b38jRvHd{;YMjgO`foW^M<}nO1}TRX7{OsBr7tWD(|V`sS9Bx9 z&s(sjyFA3j`gE5vmnZtjaz_xb!l~5#T*m!OxAnE*m{WXLm&o`)g|S558{68rvEs7S z^jbaqPQcgJfIHr>3g!xJ>3TC8MJLsgDtI1LWAWzn$Q@JU3IRc zL_q?9RFHuOK3fiP3JP*@*mxrM7KaHNnfK|yXM=g?=LN;Mzg}5 zf{Cl1UXpA#RRdqPTfcsqgMVma8>HsRW2zGjJ8yox%_Xt1%$qyevhiXPHg3XAH8F5b zpcX&%nFWMu*|%pYijEaj#Nqjxqe1^us7v zmc{x`C$jYx-Wf{yyuiQ~u;p&ey<*&b^nA<8;52opFI~w`^}VbMOUrQ1)ZXcH=+dAgg_ryOwM%dDJ%<8JU%K$diojHgHU|VCix5czr6A{z zg{yiBbYgsep3Zmiw%ac`5O~5x(#t(<95Dp)r)Pb^60}|Qo7Q241Ge_PICtK+@c&|W^+8@Jo0M##c1e)nno_q%;+pQ%>g zQ*i@p&VKi1VXe|rvkenJX*wYc#>ki^-Ab2~xFQKlbwZ^2^m*Gw9Cm#iJs23?;n*4{ z4w?=s3gL)RXq;2-`qPZ#uG0(wEJ*5x$k`OTh=RDoa1DRVPV|6rdqTD(0C{;;_x#YvqFGsoB^Go5b*_Lnz< zHw&F(@yz54w1+42_~T%Abg>Ayp){1`L`S}(;`PxCOup`UL>-*~T$&`oRKWw9_vSJhDNu{^@z4I_M$*7psjea&`zIcxv^%KfY-Q2nKqtCXsnf4;f|y8 z4%n0p%1u0bt#O5&N#6U z9z+V_d?3K6$N)?8?_sfSUa1I$GZ-@Bso#X|Yv_tKs%*iF1s{2bpLMcD-f7TYwR~|U*}_7xUh-F z=>Kl2q%OZoYb#8pR%BO3tW?N%o8RKmt_NWpDVpx4+c*?=9C zhiFOrGdEoHcCTSazf0>PQ56_aRoH*6!?}wf@{q%ITJ1`3{MzxgYdmM29go&3eUgQ# z)zNDA@FLye8{u43Hg9k~a*0imS@10-IwgaP23xcILHhBh)Mx616v$n#3y>Da2t=M{ z3rE=0uAkT{7K;JMcVH`{ip?wZHf$JV{R$Wuj0@FufiP3b8IEmR_8P)Dn=H~purPE+z z0^c%s?q;g*gUypgR~8wgb`)I2__Td@i<~N&#`*(Ym7Bj%MC4@lolXr;Ep_Ns#v}GN zW^z}BYVIy7veC73wLb2hQt3tv646{?K53h44Bp+!c;27e$bPm|qyE<*-`_o?&r zR#H1ptUg&;<;16ze>t;JhkK_0Mmez~zG*q9AaMr2@yB7I?Xf#ac!T9GUf6*#mj3|LB3Q3V1E$VceH}Sk!`p~sI$w` zSi9?zXNfcyeUp>Jtl7(GThwA7QCC9;1PB0ZFTahbW+~21D>T_P_<=Udf|Ja;hc5lu zJHrkcb*rKR<4+7LXI^kHO9rE_CN~`4R!4~u9@m7X!*^Kb>jaBre9BhDrPWz}(stQ{ zB*W!gH0rw|xpL@R-Ctz5OdLQY2zP$~tYgKc+X<^p>vqY#HuE{MK+p?DNQm~I!xQ!C z_ak`KjvKfCc&7h2)A0$-XEVXb45p*!ANVq|C3U!#Nra9%5A>Jzvq%(#)9`8NES%p= zD&^*4O9pR|#yfh*H-)`}+hYAK6YNOxVnKnZ5DbtZDCp0mvNU!Eo2j`vTiILuky{BW z3f*kLz9qPK68%0&6~W;J_%0H7e87zU_sJuMQ=;AxLmIn{yj^rY?p{y2?mf;|S*%aT z1PQ$n^g$U=UplHiOD?WgbOar72A-K_$h^IPjjKHQ9u*m7+2T{TU6#pqs8wA6h#mNPL-0RoLGHK<7#JPWQ*+wX2Q>GLWX>EHR^QZ{JjKr zKdAC?cIHr7VYf^m&0M)XE0>w&rXR*vB(D$iUp*}AF4R1Vaq6V!blR)fC=+R*NS^yN zxXfG2!Do0CfmneZiWoaqvrYiE-<{#7JVRs&KO$IK@V|Cd>^Mhl8g?_dN2eG!ntJzJ z?+Ms(D0D(v#33G}{KI=cl%78a?|(=?zlQKXg`bd^&d4q{kkmQMorI@P!j~5)5^B5@ zO&S|8CRelYbIGyipj)3>BJlh5b;sd#p7d+MVnTDYIHrjf=of3IxRzB1KC{g4#H#tW z=Rru!g0aTqEFEbuN=ot(eX-iu$ki`YsBZDP{d;DrqmhbZg}=?nMR>6dT)re6DJ!wI zJ2FD3V^yl(XIk^qdd@GTs&ls+?MEtTpa1T&?9p(D+~e41!g;cf`=!1g1)7NMc${U~ z_@TiT*?szWe1`P9qO7~gbaf4R^H&l80PPf>qSsmZ??nIUfh!<2ZOM z*;XG5ybnF%R(~2NSk5)BslYrjs%`V#P6NBhX+&c~Y=ND_8hh1658q9(-1E3nFX_vX zU|L2|jJEZg!#D3h?LMs6RE)K)PFXSn*}H@K&Ar1=OE0^M7J@_pP_V%NznybXa+E_v zPE!kTS`XQ8-3>5lgxm&+k$=lS3%cQxT96h~h+>EVQ4Gx-OqHD-99>vV9h|{`&K>_( zD1-!-Z@i*XHybGATy~!PNZ85W(yE;5HNXuH6H|=LJ?VB3aBS}}z!8s!=Tp|k!7PE9gsLatGrh1 zu_tp!x}#MxxStTOWuwTCm0X2#LYFo5JzySlfVlME(a3Lbf{_M^Mj(V{0skJ2#*U8v z4MvE|{@gO+JMHIw%oM_1p@r^6W!Te&ifXD9FnUi{0pz^3!2CE23c)=D16kl?V)$KD z_t`nu_aymsQAJ8~Y7SHZU#DbLzxvKim%T`NlR=AN zhZtmw5Rv(Dm9jcqSX2^*o)S6vktSs*{)XJt$XhP*nK;$UX}SU~`=&~U{FSktYj~og zaY2#QDk(?mL|S8qe)v9}m;2nz?RytvL)N&6+L14TWuuR*THxnw^uG=P`!I8cU z-b~8b&+p9q1V{SDr1B2%L>rv%M=VI9g0|NYoZf+0nHyk?UNUi<&?1}rnzmC6ItD~&&Pk?`}8~Drc$MFp$N&dEq;GyBexzt~#pCHQHgPGNb z#{Zm1`eh0Lpgj6%{C`XixxaI|7Yd?yZI-wznTA8$CVY}ATbCDCdiKl62=)n(g*NA DxVh?K literal 0 HcmV?d00001 diff --git a/reproduction/text_classification/train_awdlstm.py b/reproduction/text_classification/train_awdlstm.py new file mode 100644 index 00000000..ce3e52bc --- /dev/null +++ b/reproduction/text_classification/train_awdlstm.py @@ -0,0 +1,102 @@ +# 这个模型需要在pytorch=0.4下运行，weight_drop不支持1.0 + +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' + + +import torch.nn as nn + +from data.SSTLoader import SSTLoader +from data.IMDBLoader import IMDBLoader +from data.yelpLoader import yelpLoader +from fastNLP.modules.encoder.embedding import StaticEmbedding +from model.awd_lstm import AWDLSTMSentiment + +from fastNLP.core.const import Const as C +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP import Trainer, Tester +from torch.optim import Adam +from fastNLP.io.model_io import ModelLoader, ModelSaver + +import argparse + + +class Config(): + train_epoch= 10 + lr=0.001 + + num_classes=2 + hidden_dim=256 + num_layers=1 + nfc=128 + wdrop=0.5 + + task_name = "IMDB" + datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} + load_model_path="./result_IMDB/best_BiLSTM_SELF_ATTENTION_acc_2019-07-07-04-16-51" + save_model_path="./result_IMDB_test/" +opt=Config + + +# load data +dataloaders = { + "IMDB":IMDBLoader(), + "YELP":yelpLoader(), + "SST-5":SSTLoader(subtree=True,fine_grained=True), + "SST-3":SSTLoader(subtree=True,fine_grained=False) +} + +if opt.task_name not in ["IMDB", "YELP", "SST-5", "SST-3"]: + raise ValueError("task name must in ['IMDB', 'YELP, 'SST-5', 'SST-3']") + +dataloader = dataloaders[opt.task_name] +datainfo=dataloader.process(opt.datapath) +# print(datainfo.datasets["train"]) +# print(datainfo) + + +# define model +vocab=datainfo.vocabs['words'] +embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) +model=AWDLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc, wdrop=opt.wdrop) + + +# define loss_function and metrics +loss=CrossEntropyLoss() +metrics=AccuracyMetric() +optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) + + +def train(datainfo, model, optimizer, loss, metrics, opt): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=metrics, dev_data=datainfo.datasets['dev'], device=0, check_code_level=-1, + n_epochs=opt.train_epoch, save_path=opt.save_model_path) + trainer.train() + + +def test(datainfo, metrics, opt): + # load model + model = ModelLoader.load_pytorch_model(opt.load_model_path) + print("model loaded!") + + # Tester + tester = Tester(datainfo.datasets['test'], model, metrics, batch_size=4, device=0) + acc = tester.test() + print("acc=",acc) + + + +parser = argparse.ArgumentParser() +parser.add_argument('--mode', required=True, dest="mode",help='set the model\'s model') + + +args = parser.parse_args() +if args.mode == 'train': + train(datainfo, model, optimizer, loss, metrics, opt) +elif args.mode == 'test': + test(datainfo, metrics, opt) +else: + print('no mode specified for model!') + parser.print_help() diff --git a/reproduction/text_classification/train_lstm.py b/reproduction/text_classification/train_lstm.py new file mode 100644 index 00000000..b320e79c --- /dev/null +++ b/reproduction/text_classification/train_lstm.py @@ -0,0 +1,99 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' + + +import torch.nn as nn + +from data.SSTLoader import SSTLoader +from data.IMDBLoader import IMDBLoader +from data.yelpLoader import yelpLoader +from fastNLP.modules.encoder.embedding import StaticEmbedding +from model.lstm import BiLSTMSentiment + +from fastNLP.core.const import Const as C +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP import Trainer, Tester +from torch.optim import Adam +from fastNLP.io.model_io import ModelLoader, ModelSaver + +import argparse + + +class Config(): + train_epoch= 10 + lr=0.001 + + num_classes=2 + hidden_dim=256 + num_layers=1 + nfc=128 + + task_name = "IMDB" + datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} + load_model_path="./result_IMDB/best_BiLSTM_SELF_ATTENTION_acc_2019-07-07-04-16-51" + save_model_path="./result_IMDB_test/" +opt=Config + + +# load data +dataloaders = { + "IMDB":IMDBLoader(), + "YELP":yelpLoader(), + "SST-5":SSTLoader(subtree=True,fine_grained=True), + "SST-3":SSTLoader(subtree=True,fine_grained=False) +} + +if opt.task_name not in ["IMDB", "YELP", "SST-5", "SST-3"]: + raise ValueError("task name must in ['IMDB', 'YELP, 'SST-5', 'SST-3']") + +dataloader = dataloaders[opt.task_name] +datainfo=dataloader.process(opt.datapath) +# print(datainfo.datasets["train"]) +# print(datainfo) + + +# define model +vocab=datainfo.vocabs['words'] +embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) +model=BiLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc) + + +# define loss_function and metrics +loss=CrossEntropyLoss() +metrics=AccuracyMetric() +optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) + + +def train(datainfo, model, optimizer, loss, metrics, opt): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=metrics, dev_data=datainfo.datasets['dev'], device=0, check_code_level=-1, + n_epochs=opt.train_epoch, save_path=opt.save_model_path) + trainer.train() + + +def test(datainfo, metrics, opt): + # load model + model = ModelLoader.load_pytorch_model(opt.load_model_path) + print("model loaded!") + + # Tester + tester = Tester(datainfo.datasets['test'], model, metrics, batch_size=4, device=0) + acc = tester.test() + print("acc=",acc) + + + +parser = argparse.ArgumentParser() +parser.add_argument('--mode', required=True, dest="mode",help='set the model\'s model') + + +args = parser.parse_args() +if args.mode == 'train': + train(datainfo, model, optimizer, loss, metrics, opt) +elif args.mode == 'test': + test(datainfo, metrics, opt) +else: + print('no mode specified for model!') + parser.print_help() diff --git a/reproduction/text_classification/train_lstm_att.py b/reproduction/text_classification/train_lstm_att.py new file mode 100644 index 00000000..8db27d09 --- /dev/null +++ b/reproduction/text_classification/train_lstm_att.py @@ -0,0 +1,101 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' + + +import torch.nn as nn + +from data.SSTLoader import SSTLoader +from data.IMDBLoader import IMDBLoader +from data.yelpLoader import yelpLoader +from fastNLP.modules.encoder.embedding import StaticEmbedding +from model.lstm_self_attention import BiLSTM_SELF_ATTENTION + +from fastNLP.core.const import Const as C +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP import Trainer, Tester +from torch.optim import Adam +from fastNLP.io.model_io import ModelLoader, ModelSaver + +import argparse + + +class Config(): + train_epoch= 10 + lr=0.001 + + num_classes=2 + hidden_dim=256 + num_layers=1 + attention_unit=256 + attention_hops=1 + nfc=128 + + task_name = "IMDB" + datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} + load_model_path="./result_IMDB/best_BiLSTM_SELF_ATTENTION_acc_2019-07-07-04-16-51" + save_model_path="./result_IMDB_test/" +opt=Config + + +# load data +dataloaders = { + "IMDB":IMDBLoader(), + "YELP":yelpLoader(), + "SST-5":SSTLoader(subtree=True,fine_grained=True), + "SST-3":SSTLoader(subtree=True,fine_grained=False) +} + +if opt.task_name not in ["IMDB", "YELP", "SST-5", "SST-3"]: + raise ValueError("task name must in ['IMDB', 'YELP, 'SST-5', 'SST-3']") + +dataloader = dataloaders[opt.task_name] +datainfo=dataloader.process(opt.datapath) +# print(datainfo.datasets["train"]) +# print(datainfo) + + +# define model +vocab=datainfo.vocabs['words'] +embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) +model=BiLSTM_SELF_ATTENTION(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, attention_unit=opt.attention_unit, attention_hops=opt.attention_hops, nfc=opt.nfc) + + +# define loss_function and metrics +loss=CrossEntropyLoss() +metrics=AccuracyMetric() +optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) + + +def train(datainfo, model, optimizer, loss, metrics, opt): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=metrics, dev_data=datainfo.datasets['dev'], device=0, check_code_level=-1, + n_epochs=opt.train_epoch, save_path=opt.save_model_path) + trainer.train() + + +def test(datainfo, metrics, opt): + # load model + model = ModelLoader.load_pytorch_model(opt.load_model_path) + print("model loaded!") + + # Tester + tester = Tester(datainfo.datasets['test'], model, metrics, batch_size=4, device=0) + acc = tester.test() + print("acc=",acc) + + + +parser = argparse.ArgumentParser() +parser.add_argument('--mode', required=True, dest="mode",help='set the model\'s model') + + +args = parser.parse_args() +if args.mode == 'train': + train(datainfo, model, optimizer, loss, metrics, opt) +elif args.mode == 'test': + test(datainfo, metrics, opt) +else: + print('no mode specified for model!') + parser.print_help() From f6bba93696b8266d27a657eceace16653df3525f Mon Sep 17 00:00:00 2001 From: wyg <1505116161@qq.com> Date: Sun, 7 Jul 2019 14:50:36 +0800 Subject: [PATCH 12/27] [verify] yelpdataloader [add] HAN train_HAN --- .../text_classification/data/sstLoader.py | 106 +++++++++++++++-- .../text_classification/data/yelpLoader.py | 6 + reproduction/text_classification/model/HAN.py | 109 ++++++++++++++++++ reproduction/text_classification/train_HAN.py | 109 ++++++++++++++++++ .../text_classification/train_char_cnn.py | 7 +- 5 files changed, 324 insertions(+), 13 deletions(-) create mode 100644 reproduction/text_classification/model/HAN.py create mode 100644 reproduction/text_classification/train_HAN.py diff --git a/reproduction/text_classification/data/sstLoader.py b/reproduction/text_classification/data/sstLoader.py index bffb67fd..0d1b647c 100644 --- a/reproduction/text_classification/data/sstLoader.py +++ b/reproduction/text_classification/data/sstLoader.py @@ -1,13 +1,101 @@ -import csv from typing import Iterable -from fastNLP import DataSet, Instance, Vocabulary -from fastNLP.core.vocabulary import VocabularyOption -from fastNLP.io.base_loader import DataInfo,DataSetLoader -from fastNLP.io.embed_loader import EmbeddingOption -from fastNLP.io.file_reader import _read_json -from typing import Union, Dict -from reproduction.Star_transformer.datasets import EmbedLoader -from reproduction.utils import check_dataloader_paths +from nltk import Tree +from fastNLP.io.base_loader import DataInfo, DataSetLoader +from fastNLP.core.vocabulary import VocabularyOption, Vocabulary +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader + + +class SSTLoader(DataSetLoader): + URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' + DATA_DIR = 'sst/' + + """ + 别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader` + + 读取SST数据集, DataSet包含fields:: + + words: list(str) 需要分类的文本 + target: str 文本的标签 + + 数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip + + :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False`` + :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` + """ + + def __init__(self, subtree=False, fine_grained=False): + self.subtree = subtree + + tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral', + '3': 'positive', '4': 'very positive'} + if not fine_grained: + tag_v['0'] = tag_v['1'] + tag_v['4'] = tag_v['3'] + self.tag_v = tag_v + + def _load(self, path): + """ + + :param str path: 存储数据的路径 + :return: 一个 :class:`~fastNLP.DataSet` 类型的对象 + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + datas = [] + for l in f: + datas.extend([(s, self.tag_v[t]) + for s, t in self._get_one(l, self.subtree)]) + ds = DataSet() + for words, tag in datas: + ds.append(Instance(words=words, target=tag)) + return ds + + @staticmethod + def _get_one(data, subtree): + tree = Tree.fromstring(data) + if subtree: + return [(t.leaves(), t.label()) for t in tree.subtrees()] + return [(tree.leaves(), tree.label())] + + def process(self, + paths, + train_ds: Iterable[str] = None, + src_vocab_op: VocabularyOption = None, + tgt_vocab_op: VocabularyOption = None, + src_embed_op: EmbeddingOption = None): + input_name, target_name = 'words', 'target' + src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) + + info = DataInfo(datasets=self.load(paths)) + _train_ds = [info.datasets[name] + for name in train_ds] if train_ds else info.datasets.values() + src_vocab.from_dataset(*_train_ds, field_name=input_name) + tgt_vocab.from_dataset(*_train_ds, field_name=target_name) + src_vocab.index_dataset( + *info.datasets.values(), + field_name=input_name, new_field_name=input_name) + tgt_vocab.index_dataset( + *info.datasets.values(), + field_name=target_name, new_field_name=target_name) + info.vocabs = { + input_name: src_vocab, + target_name: tgt_vocab + } + + if src_embed_op is not None: + src_embed_op.vocab = src_vocab + init_emb = EmbedLoader.load_with_vocab(**src_embed_op) + info.embeddings[input_name] = init_emb + + for name, dataset in info.datasets.items(): + dataset.set_input(input_name) + dataset.set_target(target_name) + + return info class sst2Loader(DataSetLoader): ''' diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index 0e65fb20..280e8be0 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -184,6 +184,12 @@ class yelpLoader(DataSetLoader): info.vocabs[target_name]=tgt_vocab + info.datasets['train'],info.datasets['dev']=info.datasets['train'].split(0.1, shuffle=False) + + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + return info if __name__=="__main__": diff --git a/reproduction/text_classification/model/HAN.py b/reproduction/text_classification/model/HAN.py new file mode 100644 index 00000000..0902d1e4 --- /dev/null +++ b/reproduction/text_classification/model/HAN.py @@ -0,0 +1,109 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable +from fastNLP.modules.utils import get_embeddings +from fastNLP.core import Const as C + + +def pack_sequence(tensor_seq, padding_value=0.0): + if len(tensor_seq) <= 0: + return + length = [v.size(0) for v in tensor_seq] + max_len = max(length) + size = [len(tensor_seq), max_len] + size.extend(list(tensor_seq[0].size()[1:])) + ans = torch.Tensor(*size).fill_(padding_value) + if tensor_seq[0].data.is_cuda: + ans = ans.cuda() + ans = Variable(ans) + for i, v in enumerate(tensor_seq): + ans[i, :length[i], :] = v + return ans + + +class HANCLS(nn.Module): + def __init__(self, init_embed, num_cls): + super(HANCLS, self).__init__() + + self.embed = get_embeddings(init_embed) + self.han = HAN(input_size=300, + output_size=num_cls, + word_hidden_size=50, word_num_layers=1, word_context_size=100, + sent_hidden_size=50, sent_num_layers=1, sent_context_size=100 + ) + + def forward(self, input_sents): + # input_sents [B, num_sents, seq-len] dtype long + # target + B, num_sents, seq_len = input_sents.size() + input_sents = input_sents.view(-1, seq_len) # flat + words_embed = self.embed(input_sents) # should be [B*num-sent, seqlen , word-dim] + words_embed = words_embed.view(B, num_sents, seq_len, -1) # recover # [B, num-sent, seqlen , word-dim] + out = self.han(words_embed) + + return {C.OUTPUT: out} + + def predict(self, input_sents): + x = self.forward(input_sents)[C.OUTPUT] + return {C.OUTPUT: torch.argmax(x, 1)} + + +class HAN(nn.Module): + def __init__(self, input_size, output_size, + word_hidden_size, word_num_layers, word_context_size, + sent_hidden_size, sent_num_layers, sent_context_size): + super(HAN, self).__init__() + + self.word_layer = AttentionNet(input_size, + word_hidden_size, + word_num_layers, + word_context_size) + self.sent_layer = AttentionNet(2 * word_hidden_size, + sent_hidden_size, + sent_num_layers, + sent_context_size) + self.output_layer = nn.Linear(2 * sent_hidden_size, output_size) + self.softmax = nn.LogSoftmax(dim=1) + + def forward(self, batch_doc): + # input is a sequence of matrix + doc_vec_list = [] + for doc in batch_doc: + sent_mat = self.word_layer(doc) # doc's dim (num_sent, seq_len, word_dim) + doc_vec_list.append(sent_mat) # sent_mat's dim (num_sent, vec_dim) + doc_vec = self.sent_layer(pack_sequence(doc_vec_list)) + output = self.softmax(self.output_layer(doc_vec)) + return output + + +class AttentionNet(nn.Module): + def __init__(self, input_size, gru_hidden_size, gru_num_layers, context_vec_size): + super(AttentionNet, self).__init__() + + self.input_size = input_size + self.gru_hidden_size = gru_hidden_size + self.gru_num_layers = gru_num_layers + self.context_vec_size = context_vec_size + + # Encoder + self.gru = nn.GRU(input_size=input_size, + hidden_size=gru_hidden_size, + num_layers=gru_num_layers, + batch_first=True, + bidirectional=True) + # Attention + self.fc = nn.Linear(2 * gru_hidden_size, context_vec_size) + self.tanh = nn.Tanh() + self.softmax = nn.Softmax(dim=1) + # context vector + self.context_vec = nn.Parameter(torch.Tensor(context_vec_size, 1)) + self.context_vec.data.uniform_(-0.1, 0.1) + + def forward(self, inputs): + # GRU part + h_t, hidden = self.gru(inputs) # inputs's dim (batch_size, seq_len, word_dim) + u = self.tanh(self.fc(h_t)) + # Attention part + alpha = self.softmax(torch.matmul(u, self.context_vec)) # u's dim (batch_size, seq_len, context_vec_size) + output = torch.bmm(torch.transpose(h_t, 1, 2), alpha) # alpha's dim (batch_size, seq_len, 1) + return torch.squeeze(output, dim=2) # output's dim (batch_size, 2*hidden_size, 1) diff --git a/reproduction/text_classification/train_HAN.py b/reproduction/text_classification/train_HAN.py new file mode 100644 index 00000000..b1135342 --- /dev/null +++ b/reproduction/text_classification/train_HAN.py @@ -0,0 +1,109 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 + +import os +import sys +sys.path.append('../../') +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +from fastNLP.core.const import Const as C +from fastNLP.core import LRScheduler +import torch.nn as nn +from fastNLP.io.dataset_loader import SSTLoader +from reproduction.text_classification.data.yelpLoader import yelpLoader +from reproduction.text_classification.model.HAN import HANCLS +from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.core.trainer import Trainer +from torch.optim import SGD +import torch.cuda +from torch.optim.lr_scheduler import CosineAnnealingLR + + +##hyper + +class Config(): + model_dir_or_name = "en-base-uncased" + embedding_grad = False, + train_epoch = 30 + batch_size = 100 + num_classes = 5 + task = "yelp" + #datadir = '/remote-home/lyli/fastNLP/yelp_polarity/' + datadir = '/remote-home/ygwang/yelp_polarity/' + datafile = {"train": "train.csv", "test": "test.csv"} + lr = 1e-3 + + def __init__(self): + self.datapath = {k: os.path.join(self.datadir, v) + for k, v in self.datafile.items()} + + +ops = Config() + +##1.task相关信息：利用dataloader载入dataInfo + +datainfo = yelpLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) +print(len(datainfo.datasets['train'])) +print(len(datainfo.datasets['test'])) + + +# post process +def make_sents(words): + sents = [words] + return sents + + +for dataset in datainfo.datasets.values(): + dataset.apply_field(make_sents, field_name='words', new_field_name='input_sents') + +datainfo = datainfo +datainfo.datasets['train'].set_input('input_sents') +datainfo.datasets['test'].set_input('input_sents') +datainfo.datasets['train'].set_target('target') +datainfo.datasets['test'].set_target('target') + +## 2.或直接复用fastNLP的模型 + +vocab = datainfo.vocabs['words'] +# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) +embedding = StaticEmbedding(vocab) + +print(len(vocab)) +print(len(datainfo.vocabs['target'])) + +# model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) +model = HANCLS(init_embed=embedding, num_cls=ops.num_classes) + +## 3. 声明loss,metric,optimizer +loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) +metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) +optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], + lr=ops.lr, momentum=0.9, weight_decay=0) + +callbacks = [] +callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) + +device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + +print(device) + +for ds in datainfo.datasets.values(): + ds.apply_field(len, C.INPUT, C.INPUT_LEN) + ds.set_input(C.INPUT, C.INPUT_LEN) + ds.set_target(C.TARGET) + + +## 4.定义train方法 +def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=[metrics], dev_data=datainfo.datasets['test'], device=device, + check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, + n_epochs=num_epochs) + + print(trainer.train()) + + +if __name__ == "__main__": + train(model, datainfo, loss, metric, optimizer) diff --git a/reproduction/text_classification/train_char_cnn.py b/reproduction/text_classification/train_char_cnn.py index c2c983a4..050527fe 100644 --- a/reproduction/text_classification/train_char_cnn.py +++ b/reproduction/text_classification/train_char_cnn.py @@ -7,7 +7,6 @@ import sys sys.path.append('../..') from fastNLP.core.const import Const as C import torch.nn as nn -from fastNLP.io.dataset_loader import SSTLoader from data.yelpLoader import yelpLoader from data.sstLoader import sst2Loader from data.IMDBLoader import IMDBLoader @@ -107,9 +106,9 @@ ops=Config ##1.task相关信息：利用dataloader载入dataInfo -dataloader=sst2Loader() -dataloader=IMDBLoader() -#dataloader=yelpLoader(fine_grained=True) +#dataloader=sst2Loader() +#dataloader=IMDBLoader() +dataloader=yelpLoader(fine_grained=True) datainfo=dataloader.process(ops.datapath,char_level_op=True) char_vocab=ops.char_cnn_config["alphabet"]["en"]["lower"]["alphabet"] ops.number_of_characters=len(char_vocab) From 2a1d5dc2a422b0066851d104b666adf0bcad00cc Mon Sep 17 00:00:00 2001 From: lionel <13307130318@fudan.edu.cn> Date: Sun, 7 Jul 2019 15:24:48 +0800 Subject: [PATCH 13/27] Add cntn model for matching. --- reproduction/matching/matching_cntn.py | 105 ++++++++++++++++++++++ reproduction/matching/model/cntn.py | 120 +++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 reproduction/matching/matching_cntn.py create mode 100644 reproduction/matching/model/cntn.py diff --git a/reproduction/matching/matching_cntn.py b/reproduction/matching/matching_cntn.py new file mode 100644 index 00000000..d813164d --- /dev/null +++ b/reproduction/matching/matching_cntn.py @@ -0,0 +1,105 @@ +import argparse +import torch +import os + +from fastNLP.core import Trainer, Tester, Adam, AccuracyMetric, Const +from fastNLP.modules.encoder.embedding import StaticEmbedding + +from reproduction.matching.data.MatchingDataLoader import QNLILoader, RTELoader, SNLILoader, MNLILoader +from reproduction.matching.model.cntn import CNTNModel + +# define hyper-parameters +argument = argparse.ArgumentParser() +argument.add_argument('--embedding', choices=['glove', 'word2vec'], default='glove') +argument.add_argument('--batch-size-per-gpu', type=int, default=256) +argument.add_argument('--n-epochs', type=int, default=200) +argument.add_argument('--lr', type=float, default=1e-5) +argument.add_argument('--seq-len-type', choices=['mask', 'seq_len'], default='mask') +argument.add_argument('--save-dir', type=str, default=None) +argument.add_argument('--cntn-depth', type=int, default=1) +argument.add_argument('--cntn-ns', type=int, default=200) +argument.add_argument('--cntn-k-top', type=int, default=10) +argument.add_argument('--cntn-r', type=int, default=5) +argument.add_argument('--dataset', choices=['qnli', 'rte', 'snli', 'mnli'], default='qnli') +argument.add_argument('--max-len', type=int, default=50) +arg = argument.parse_args() + +# dataset dict +dev_dict = { + 'qnli': 'dev', + 'rte': 'dev', + 'snli': 'dev', + 'mnli': 'dev_matched', +} + +test_dict = { + 'qnli': 'dev', + 'rte': 'dev', + 'snli': 'test', + 'mnli': 'dev_matched', +} + +# set num_labels +if arg.dataset == 'qnli' or arg.dataset == 'rte': + num_labels = 2 +else: + num_labels = 3 + +# load data set +if arg.dataset == 'qnli': + data_info = QNLILoader().process( + paths='path/to/qnli/data', to_lower=True, seq_len_type=arg.seq_len_type, bert_tokenizer=None, + get_index=True, concat=False, auto_pad_length=arg.max_len) +elif arg.dataset == 'rte': + data_info = RTELoader().process( + paths='path/to/rte/data', to_lower=True, seq_len_type=arg.seq_len_type, bert_tokenizer=None, + get_index=True, concat=False, auto_pad_length=arg.max_len) +elif arg.dataset == 'snli': + data_info = SNLILoader().process( + paths='path/to/snli/data', to_lower=True, seq_len_type=arg.seq_len_type, bert_tokenizer=None, + get_index=True, concat=False, auto_pad_length=arg.max_len) +elif arg.dataset == 'mnli': + data_info = MNLILoader().process( + paths='path/to/mnli/data', to_lower=True, seq_len_type=arg.seq_len_type, bert_tokenizer=None, + get_index=True, concat=False, auto_pad_length=arg.max_len) +else: + raise ValueError(f'now we only support [qnli,rte,snli,mnli] dataset for cntn model!') + +# load embedding +if arg.embedding == 'word2vec': + embedding = StaticEmbedding(data_info.vocabs[Const.INPUT], model_dir_or_name='en-word2vec-300', requires_grad=True) +elif arg.embedding == 'glove': + embedding = StaticEmbedding(data_info.vocabs[Const.INPUT], model_dir_or_name='en-glove-840b-300', + requires_grad=True) +else: + raise ValueError(f'now we only support word2vec or glove embedding for cntn model!') + +# define model +model = CNTNModel(embedding, ns=arg.cntn_ns, k_top=arg.cntn_k_top, num_labels=num_labels, depth=arg.cntn_depth, + r=arg.cntn_r) +print(model) + +# define trainer +trainer = Trainer(train_data=data_info.datasets['train'], model=model, + optimizer=Adam(lr=arg.lr, model_params=model.parameters()), + batch_size=torch.cuda.device_count() * arg.batch_size_per_gpu, + n_epochs=arg.n_epochs, print_every=-1, + dev_data=data_info.datasets[dev_dict[arg.dataset]], + metrics=AccuracyMetric(), metric_key='acc', + device=[i for i in range(torch.cuda.device_count())], + check_code_level=-1) + +# train model +trainer.train(load_best_model=True) + +# define tester +tester = Tester( + data=data_info.datasets[test_dict[arg.dataset]], + model=model, + metrics=AccuracyMetric(), + batch_size=torch.cuda.device_count() * arg.batch_size_per_gpu, + device=[i for i in range(torch.cuda.device_count())] +) + +# test model +tester.test() diff --git a/reproduction/matching/model/cntn.py b/reproduction/matching/model/cntn.py new file mode 100644 index 00000000..0b4803fa --- /dev/null +++ b/reproduction/matching/model/cntn.py @@ -0,0 +1,120 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np + +from torch.nn import CrossEntropyLoss + +from fastNLP.models import BaseModel +from fastNLP.modules.encoder.embedding import TokenEmbedding +from fastNLP.core.const import Const + + +class DynamicKMaxPooling(nn.Module): + """ + :param k_top: Fixed number of pooling output features for the topmost convolutional layer. + :param l: Number of convolutional layers. + """ + + def __init__(self, k_top, l): + super(DynamicKMaxPooling, self).__init__() + self.k_top = k_top + self.L = l + + def forward(self, x, l): + """ + :param x: Input sequence. + :param l: Current convolutional layers. + """ + s = x.size()[3] + k_ll = ((self.L - l) / self.L) * s + k_l = int(round(max(self.k_top, np.ceil(k_ll)))) + out = F.adaptive_max_pool2d(x, (x.size()[2], k_l)) + return out + + +class CNTNModel(BaseModel): + """ + 使用CNN进行问答匹配的模型 + 'Qiu, Xipeng, and Xuanjing Huang. + Convolutional neural tensor network architecture for community-based question answering. + Twenty-Fourth International Joint Conference on Artificial Intelligence. 2015.' + + :param init_embedding: Embedding. + :param ns: Sentence embedding size. + :param k_top: Fixed number of pooling output features for the topmost convolutional layer. + :param num_labels: Number of labels. + :param depth: Number of convolutional layers. + :param r: Number of weight tensor slices. + :param drop_rate: Dropout rate. + """ + + def __init__(self, init_embedding: TokenEmbedding, ns=200, k_top=10, num_labels=2, depth=2, r=5, + dropout_rate=0.3): + super(CNTNModel, self).__init__() + self.embedding = init_embedding + self.depth = depth + self.kmaxpooling = DynamicKMaxPooling(k_top, depth) + self.conv_q = nn.ModuleList() + self.conv_a = nn.ModuleList() + width = self.embedding.embed_size + for i in range(depth): + self.conv_q.append(nn.Sequential( + nn.Dropout(p=dropout_rate), + nn.Conv2d( + in_channels=1, + out_channels=width // 2, + kernel_size=(width, 3), + padding=(0, 2)) + )) + self.conv_a.append(nn.Sequential( + nn.Dropout(p=dropout_rate), + nn.Conv2d( + in_channels=1, + out_channels=width // 2, + kernel_size=(width, 3), + padding=(0, 2)) + )) + width = width // 2 + + self.fc_q = nn.Sequential(nn.Dropout(p=dropout_rate), nn.Linear(width * k_top, ns)) + self.fc_a = nn.Sequential(nn.Dropout(p=dropout_rate), nn.Linear(width * k_top, ns)) + self.weight_M = nn.Bilinear(ns, ns, r) + self.weight_V = nn.Linear(2 * ns, r) + self.weight_u = nn.Sequential(nn.Dropout(p=dropout_rate), nn.Linear(r, num_labels)) + + def forward(self, words1, words2, seq_len1, seq_len2, target=None): + """ + :param words1: [batch, seq_len, emb_size] Question. + :param words2: [batch, seq_len, emb_size] Answer. + :param seq_len1: [batch] + :param seq_len2: [batch] + :param target: [batch] Glod labels. + :return: + """ + in_q = self.embedding(words1) + in_a = self.embedding(words2) + in_q = in_q.permute(0, 2, 1).unsqueeze(1) + in_a = in_a.permute(0, 2, 1).unsqueeze(1) + + for i in range(self.depth): + in_q = F.relu(self.conv_q[i](in_q)) + in_q = in_q.squeeze().unsqueeze(1) + in_q = self.kmaxpooling(in_q, i + 1) + in_a = F.relu(self.conv_a[i](in_a)) + in_a = in_a.squeeze().unsqueeze(1) + in_a = self.kmaxpooling(in_a, i + 1) + + in_q = self.fc_q(in_q.view(in_q.size(0), -1)) + in_a = self.fc_q(in_a.view(in_a.size(0), -1)) + score = torch.tanh(self.weight_u(self.weight_M(in_q, in_a) + self.weight_V(torch.cat((in_q, in_a), -1)))) + + if target is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(score, target) + return {Const.LOSS: loss, Const.OUTPUT: score} + else: + return {Const.OUTPUT: score} + + def predict(self, **kwargs): + return self.forward(**kwargs) From f369778ab33aa91447821db0de37b1bce4be4b62 Mon Sep 17 00:00:00 2001 From: wyg <1505116161@qq.com> Date: Sun, 7 Jul 2019 15:33:08 +0800 Subject: [PATCH 14/27] [verify] sstdataloader add sst2 [add] readme --- reproduction/text_classification/README.md | 22 +++++++++++++++++++ .../text_classification/data/sstLoader.py | 3 ++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 reproduction/text_classification/README.md diff --git a/reproduction/text_classification/README.md b/reproduction/text_classification/README.md new file mode 100644 index 00000000..b058fbb2 --- /dev/null +++ b/reproduction/text_classification/README.md @@ -0,0 +1,22 @@ +# text_classification任务模型复现 +这里使用fastNLP复现以下模型： +char_cnn :论文链接[Character-level Convolutional Networks for Text Classification](https://arxiv.org/pdf/1509.01626v3.pdf) +dpcnn:论文链接[Deep Pyramid Convolutional Neural Networks for TextCategorization](https://ai.tencent.com/ailab/media/publications/ACL3-Brady.pdf) +HAN:论文链接[Hierarchical Attention Networks for Document Classification](https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf) +#待补充 +awd_lstm: +lstm_self_attention(BCN?): +awd-sltm: + +# 数据集及复现结果汇总 + +使用fastNLP复现的结果vs论文汇报结果(/前为fastNLP实现，后面为论文报道,-表示论文没有在该数据集上列出结果) + +model name | yelp_p | sst-2|IMDB| +:---: | :---: | :---: | :---: +char_cnn | 93.80/95.12 | - |- | +dpcnn | 95.50/97.36 | - |- | +HAN |- | - |-| +BCN| - |- |-| +awd-lstm| - |- |-| + diff --git a/reproduction/text_classification/data/sstLoader.py b/reproduction/text_classification/data/sstLoader.py index 0d1b647c..d8403b7a 100644 --- a/reproduction/text_classification/data/sstLoader.py +++ b/reproduction/text_classification/data/sstLoader.py @@ -5,7 +5,8 @@ from fastNLP.core.vocabulary import VocabularyOption, Vocabulary from fastNLP import DataSet from fastNLP import Instance from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader - +import csv +from typing import Union, Dict class SSTLoader(DataSetLoader): URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' From d4fa6986019e9ea857fb10591d7c1d647251033b Mon Sep 17 00:00:00 2001 From: yunfan Date: Thu, 27 Jun 2019 21:56:49 +0800 Subject: [PATCH 15/27] add dpcnn --- .../text_classification/model/dpcnn.py | 92 ++++++++++++++++++- .../text_classification/train_dpcnn.py | 80 ++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) diff --git a/reproduction/text_classification/model/dpcnn.py b/reproduction/text_classification/model/dpcnn.py index f87f5c14..2da7b3e5 100644 --- a/reproduction/text_classification/model/dpcnn.py +++ b/reproduction/text_classification/model/dpcnn.py @@ -1 +1,91 @@ -# TODO \ No newline at end of file +import torch +import torch.nn as nn +from fastNLP.modules.utils import get_embeddings +from fastNLP.core import Const as C + +class DPCNN(nn.Module): + def __init__(self, init_embed, num_cls, n_filters=256, kernel_size=3, n_layers=7, embed_dropout=0.1, dropout=0.1): + super().__init__() + self.region_embed = RegionEmbedding(init_embed, out_dim=n_filters, kernel_sizes=[3, 5, 9]) + embed_dim = self.region_embed.embedding_dim + self.conv_list = nn.ModuleList() + for i in range(n_layers): + self.conv_list.append(nn.Sequential( + nn.ReLU(), + nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + )) + self.pool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1) + self.embed_drop = nn.Dropout(embed_dropout) + self.classfier = nn.Sequential( + nn.Dropout(dropout), + nn.Linear(n_filters, num_cls), + ) + self.reset_parameters() + + def reset_parameters(self): + for m in self.modules(): + if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)): + nn.init.normal_(m.weight, mean=0, std=0.01) + if m.bias is not None: + nn.init.normal_(m.bias, mean=0, std=0.01) + + def forward(self, words, seq_len=None): + words = words.long() + # get region embeddings + x = self.region_embed(words) + x = self.embed_drop(x) + + # not pooling on first conv + x = self.conv_list[0](x) + x + for conv in self.conv_list[1:]: + x = self.pool(x) + x = conv(x) + x + + # B, C, L => B, C + x, _ = torch.max(x, dim=2) + x = self.classfier(x) + return {C.OUTPUT: x} + + def predict(self, words, seq_len=None): + x = self.forward(words, seq_len)[C.OUTPUT] + return {C.OUTPUT: torch.argmax(x, 1)} + + +class RegionEmbedding(nn.Module): + def __init__(self, init_embed, out_dim=300, kernel_sizes=None): + super().__init__() + if kernel_sizes is None: + kernel_sizes = [5, 9] + assert isinstance(kernel_sizes, list), 'kernel_sizes should be List(int)' + self.embed = get_embeddings(init_embed) + try: + embed_dim = self.embed.embedding_dim + except Exception: + embed_dim = self.embed.embed_size + self.region_embeds = nn.ModuleList() + for ksz in kernel_sizes: + self.region_embeds.append(nn.Sequential( + nn.Conv1d(embed_dim, embed_dim, ksz, padding=ksz // 2), + )) + self.linears = nn.ModuleList([nn.Conv1d(embed_dim, out_dim, 1) + for _ in range(len(kernel_sizes) + 1)]) + self.embedding_dim = embed_dim + + def forward(self, x): + x = self.embed(x) + x = x.transpose(1, 2) + # B, C, L + out = self.linears[0](x) + for conv, fc in zip(self.region_embeds, self.linears[1:]): + conv_i = conv(x) + out = out + fc(conv_i) + # B, C, L + return out + + +if __name__ == '__main__': + x = torch.randint(0, 10000, size=(5, 15), dtype=torch.long) + model = DPCNN((10000, 300), 20) + y = model(x) + print(y.size(), y.mean(1), y.std(1)) diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index e69de29b..13ff4fc1 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -0,0 +1,80 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +from fastNLP.core.const import Const as C +from fastNLP.core import LRScheduler +import torch.nn as nn +from fastNLP.io.dataset_loader import SSTLoader +from reproduction.text_classification.model.dpcnn import DPCNN +from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.core.trainer import Trainer +from torch.optim import SGD +import torch.cuda +from torch.optim.lr_scheduler import CosineAnnealingLR + +##hyper +class Config(): + model_dir_or_name="en-base-uncased" + embedding_grad= False, + train_epoch= 30 + batch_size = 100 + num_classes=5 + task= "SST" + datadir = '/remote-home/yfshao/workdir/datasets/SST' + datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} + lr=1e-3 + def __init__(self): + self.datapath = {k:os.path.join(self.datadir, v) + for k, v in self.datafile.items()} + +ops=Config() + + +##1.task相关信息：利用dataloader载入dataInfo +datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds='train') + +print(len(datainfo.datasets['train'])) +print(len(datainfo.datasets['dev'])) + + +## 2.或直接复用fastNLP的模型 +vocab = datainfo.vocabs['words'] + +# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) +embedding = StaticEmbedding(vocab) +print(len(vocab)) +print(len(datainfo.vocabs['target'])) +model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) + +## 3. 声明loss,metric,optimizer +loss=CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) +metric=AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) +optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], + lr=ops.lr, momentum=0.9, weight_decay=0) + +callbacks = [] +callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) + +device = 'cuda:0' if torch.cuda.is_available() else 'cpu' +print(device) + +for ds in datainfo.datasets.values(): + ds.apply_field(len, C.INPUT, C.INPUT_LEN) + ds.set_input(C.INPUT, C.INPUT_LEN) + ds.set_target(C.TARGET) + +## 4.定义train方法 +def train(model,datainfo,loss,metrics,optimizer,num_epochs=ops.train_epoch): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=[metrics], dev_data=datainfo.datasets['dev'], device=device, + check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, + n_epochs=num_epochs) + print(trainer.train()) + + +if __name__=="__main__": + train(model,datainfo,loss,metric,optimizer) \ No newline at end of file From 4e3fba55d884250a65463a564c4730a4275b3819 Mon Sep 17 00:00:00 2001 From: yunfan Date: Thu, 4 Jul 2019 13:56:37 +0800 Subject: [PATCH 16/27] add ID-CNN --- .../ner/model/dilated_cnn.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 reproduction/seqence_labelling/ner/model/dilated_cnn.py diff --git a/reproduction/seqence_labelling/ner/model/dilated_cnn.py b/reproduction/seqence_labelling/ner/model/dilated_cnn.py new file mode 100644 index 00000000..cd2fa64b --- /dev/null +++ b/reproduction/seqence_labelling/ner/model/dilated_cnn.py @@ -0,0 +1,111 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from fastNLP.modules.decoder import ConditionalRandomField +from fastNLP.modules.encoder import Embedding +from fastNLP.core.utils import seq_len_to_mask +from fastNLP.core.const import Const as C + + +class IDCNN(nn.Module): + def __init__(self, init_embed, char_embed, + num_cls, + repeats, num_layers, num_filters, kernel_size, + use_crf=False, use_projection=False, block_loss=False, + input_dropout=0.3, hidden_dropout=0.2, inner_dropout=0.0): + super(IDCNN, self).__init__() + self.word_embeddings = Embedding(init_embed) + self.char_embeddings = Embedding(char_embed) + embedding_size = self.word_embeddings.embedding_dim + \ + self.char_embeddings.embedding_dim + + self.conv0 = nn.Sequential( + nn.Conv1d(in_channels=embedding_size, + out_channels=num_filters, + kernel_size=kernel_size, + stride=1, dilation=1, + padding=kernel_size//2, + bias=True), + nn.ReLU(), + ) + + block = [] + for layer_i in range(num_layers): + dilated = 2 ** layer_i + block.append(nn.Conv1d( + in_channels=num_filters, + out_channels=num_filters, + kernel_size=kernel_size, + stride=1, dilation=dilated, + padding=(kernel_size//2) * dilated, + bias=True)) + block.append(nn.ReLU()) + self.block = nn.Sequential(*block) + + if use_projection: + self.projection = nn.Sequential( + nn.Conv1d( + in_channels=num_filters, + out_channels=num_filters//2, + kernel_size=1, + bias=True), + nn.ReLU(),) + encode_dim = num_filters // 2 + else: + self.projection = None + encode_dim = num_filters + + self.input_drop = nn.Dropout(input_dropout) + self.hidden_drop = nn.Dropout(hidden_dropout) + self.inner_drop = nn.Dropout(inner_dropout) + self.repeats = repeats + self.out_fc = nn.Conv1d( + in_channels=encode_dim, + out_channels=num_cls, + kernel_size=1, + bias=True) + self.crf = ConditionalRandomField( + num_tags=num_cls) if use_crf else None + self.block_loss = block_loss + + def forward(self, words, chars, seq_len, target=None): + e1 = self.word_embeddings(words) + e2 = self.char_embeddings(chars) + x = torch.cat((e1, e2), dim=-1) # b,l,h + mask = seq_len_to_mask(seq_len) + + x = x.transpose(1, 2) # b,h,l + last_output = self.conv0(x) + output = [] + for repeat in range(self.repeats): + last_output = self.block(last_output) + hidden = self.projection(last_output) if self.projection is not None else last_output + output.append(self.out_fc(hidden)) + + def compute_loss(y, t, mask): + if self.crf is not None and target is not None: + loss = self.crf(y, t, mask) + else: + t.masked_fill_(mask == 0, -100) + loss = F.cross_entropy(y, t, ignore_index=-100) + return loss + + if self.block_loss: + losses = [compute_loss(o, target, mask) for o in output] + loss = sum(losses) + else: + loss = compute_loss(output[-1], target, mask) + + scores = output[-1] + if self.crf is not None: + pred = self.crf.viterbi_decode(scores, target, mask) + else: + pred = scores.max(1)[1] * mask.long() + + return { + C.LOSS: loss, + C.OUTPUT: pred, + } + + def predict(self, words, chars, seq_len): + return self.forward(words, chars, seq_len)[C.OUTPUT] From 4b5713b5a2e2e2dcedf2b757659295016db6a9ce Mon Sep 17 00:00:00 2001 From: yunfan Date: Thu, 4 Jul 2019 14:03:53 +0800 Subject: [PATCH 17/27] update model & dataloader in text_classification --- .../text_classification/data/IMDBLoader.py | 82 +++++++++ .../text_classification/data/yelpLoader.py | 164 +++++++++++++++--- .../text_classification/train_dpcnn.py | 97 +++++++---- 3 files changed, 283 insertions(+), 60 deletions(-) create mode 100644 reproduction/text_classification/data/IMDBLoader.py diff --git a/reproduction/text_classification/data/IMDBLoader.py b/reproduction/text_classification/data/IMDBLoader.py new file mode 100644 index 00000000..2df87e26 --- /dev/null +++ b/reproduction/text_classification/data/IMDBLoader.py @@ -0,0 +1,82 @@ +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader +from fastNLP.core.vocabulary import VocabularyOption +from fastNLP.io.base_loader import DataSetLoader, DataInfo +from typing import Union, Dict, List, Iterator +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP import Vocabulary +from fastNLP import Const +# from reproduction.utils import check_dataloader_paths +from functools import partial + + +class IMDBLoader(DataSetLoader): + """ + 读取IMDB数据集，DataSet包含以下fields: + + words: list(str), 需要分类的文本 + target: str, 文本的标签 + + + """ + + def __init__(self): + super(IMDBLoader, self).__init__() + + def _load(self, path): + dataset = DataSet() + with open(path, 'r', encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split('\t') + target = parts[0] + words = parts[1].split() + dataset.append(Instance(words=words, target=target)) + if len(dataset) == 0: + raise RuntimeError(f"{path} has no valid data.") + + return dataset + + def process(self, + paths: Union[str, Dict[str, str]], + src_vocab_opt: VocabularyOption = None, + tgt_vocab_opt: VocabularyOption = None, + src_embed_opt: EmbeddingOption = None): + + # paths = check_dataloader_paths(paths) + datasets = {} + info = DataInfo() + for name, path in paths.items(): + dataset = self.load(path) + datasets[name] = dataset + + datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False) + + src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) + src_vocab.from_dataset(datasets['train'], field_name='words') + # src_vocab.from_dataset(datasets['train'], datasets["dev"], datasets["test"], field_name='words') + src_vocab.index_dataset(*datasets.values(), field_name='words') + + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) + tgt_vocab.from_dataset(datasets['train'], field_name='target') + tgt_vocab.index_dataset(*datasets.values(), field_name='target') + + info.vocabs = { + "words": src_vocab, + "target": tgt_vocab + } + + info.datasets = datasets + + if src_embed_opt is not None: + embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) + info.embeddings['words'] = embed + + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + + return info diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index c47d48fd..63605ecf 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -1,4 +1,6 @@ import ast +import csv +from typing import Iterable from fastNLP import DataSet, Instance, Vocabulary from fastNLP.core.vocabulary import VocabularyOption from fastNLP.io import JsonLoader @@ -10,11 +12,34 @@ from reproduction.Star_transformer.datasets import EmbedLoader from reproduction.utils import check_dataloader_paths +def clean_str(sentence, char_lower=False): + """ + heavily borrowed from github + https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb + :param sentence: is a str + :return: + """ + if char_lower: + sentence = sentence.lower() + import re + nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') + words = sentence.split() + words_collection = [] + for word in words: + if word in ['-lrb-', '-rrb-', '', '-r', '-l', 'b-']: + continue + tt = nonalpnum.split(word) + t = ''.join(tt) + if t != '': + words_collection.append(t) + + return words_collection + + class yelpLoader(JsonLoader): - """ 读取Yelp数据集, DataSet包含fields: - + review_id: str, 22 character unique review id user_id: str, 22 character unique user id business_id: str, 22 character business id @@ -24,23 +49,25 @@ class yelpLoader(JsonLoader): date: str, date formatted YYYY-MM-DD words: list(str), 需要分类的文本 target: str, 文本的标签 - + 数据来源: https://www.yelp.com/dataset/download - + :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` """ - - def __init__(self, fine_grained=False): + + def __init__(self, fine_grained=False, lower=False): super(yelpLoader, self).__init__() tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral', - '4.0': 'positive', '5.0': 'very positive'} + '4.0': 'positive', '5.0': 'very positive'} if not fine_grained: tag_v['1.0'] = tag_v['2.0'] tag_v['5.0'] = tag_v['4.0'] self.fine_grained = fine_grained self.tag_v = tag_v - - def _load(self, path): + self.lower = lower + + ''' + def _load_json(self, path): ds = DataSet() for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): d = ast.literal_eval(d) @@ -49,20 +76,113 @@ class yelpLoader(JsonLoader): ds.append(Instance(**d)) return ds - def process(self, paths: Union[str, Dict[str, str]], vocab_opt: VocabularyOption = None, - embed_opt: EmbeddingOption = None): + def _load_yelp2015_broken(self,path): + ds = DataSet() + with open (path,encoding='ISO 8859-1') as f: + row=f.readline() + all_count=0 + exp_count=0 + while row: + row=row.split("\t\t") + all_count+=1 + if len(row)>=3: + words=row[-1].split() + try: + target=self.tag_v[str(row[-2])+".0"] + ds.append(Instance(words=words, target=target)) + except KeyError: + exp_count+=1 + else: + exp_count+=1 + row = f.readline() + print("error sample count:",exp_count) + print("all count:",all_count) + return ds + ''' + + def _load(self, path): + ds = DataSet() + csv_reader = csv.reader(open(path, encoding='utf-8')) + all_count = 0 + real_count = 0 + for row in csv_reader: + all_count += 1 + if len(row) == 2: + target = self.tag_v[row[0] + ".0"] + words = clean_str(row[1], self.lower) + if len(words) != 0: + ds.append(Instance(words=words, target=target)) + real_count += 1 + print("all count:", all_count) + print("real count:", real_count) + return ds + + def process(self, paths: Union[str, Dict[str, str]], + train_ds: Iterable[str] = None, + src_vocab_op: VocabularyOption = None, + tgt_vocab_op: VocabularyOption = None, + embed_opt: EmbeddingOption = None, + char_level_op=False): paths = check_dataloader_paths(paths) datasets = {} - info = DataInfo() - vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) - for name, path in paths.items(): - dataset = self.load(path) - datasets[name] = dataset - vocab.from_dataset(dataset, field_name="words") - info.vocabs = vocab - info.datasets = datasets - if embed_opt is not None: - embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) - info.embeddings['words'] = embed + info = DataInfo(datasets=self.load(paths)) + src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) + _train_ds = [info.datasets[name] + for name in train_ds] if train_ds else info.datasets.values() + + # vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) + # for name, path in paths.items(): + # dataset = self.load(path) + # datasets[name] = dataset + # vocab.from_dataset(dataset, field_name="words") + # info.vocabs = vocab + # info.datasets = datasets + + def wordtochar(words): + chars = [] + for word in words: + word = word.lower() + for char in word: + chars.append(char) + return chars + + input_name, target_name = 'words', 'target' + info.vocabs = {} + # 就分隔为char形式 + if char_level_op: + for dataset in info.datasets.values(): + dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') + # if embed_opt is not None: + # embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) + # info.embeddings['words'] = embed + else: + src_vocab.from_dataset(*_train_ds, field_name=input_name) + src_vocab.index_dataset(*info.datasets.values(), field_name=input_name, new_field_name=input_name) + info.vocabs[input_name] = src_vocab + + tgt_vocab.from_dataset(*_train_ds, field_name=target_name) + tgt_vocab.index_dataset( + *info.datasets.values(), + field_name=target_name, new_field_name=target_name) + info.vocabs[target_name] = tgt_vocab + return info + +if __name__ == "__main__": + testloader = yelpLoader() + # datapath = {"train": "/remote-home/ygwang/yelp_full/train.csv", + # "test": "/remote-home/ygwang/yelp_full/test.csv"} + # datapath={"train": "/remote-home/ygwang/yelp_full/test.csv"} + datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv", + "test": "/remote-home/ygwang/yelp_polarity/test.csv"} + datainfo = testloader.process(datapath, char_level_op=True) + + len_count = 0 + for instance in datainfo.datasets["train"]: + len_count += len(instance["chars"]) + + ave_len = len_count / len(datainfo.datasets["train"]) + print(ave_len) \ No newline at end of file diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index 13ff4fc1..bf243ffb 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -1,65 +1,83 @@ # 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 + +from torch.optim.lr_scheduler import CosineAnnealingLR +import torch.cuda +from torch.optim import SGD +from fastNLP.core.trainer import Trainer +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding +from reproduction.text_classification.model.dpcnn import DPCNN +from .data.yelpLoader import yelpLoader +from fastNLP.io.dataset_loader import SSTLoader +import torch.nn as nn +from fastNLP.core import LRScheduler +from fastNLP.core.const import Const as C +import sys import os os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" -from fastNLP.core.const import Const as C -from fastNLP.core import LRScheduler -import torch.nn as nn -from fastNLP.io.dataset_loader import SSTLoader -from reproduction.text_classification.model.dpcnn import DPCNN -from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding -from fastNLP import CrossEntropyLoss, AccuracyMetric -from fastNLP.core.trainer import Trainer -from torch.optim import SGD -import torch.cuda -from torch.optim.lr_scheduler import CosineAnnealingLR +sys.path.append('../..') + + +# hyper -##hyper class Config(): - model_dir_or_name="en-base-uncased" - embedding_grad= False, - train_epoch= 30 + model_dir_or_name = "en-base-uncased" + embedding_grad = False, + train_epoch = 30 batch_size = 100 - num_classes=5 - task= "SST" - datadir = '/remote-home/yfshao/workdir/datasets/SST' - datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} - lr=1e-3 + num_classes = 2 + task = "yelp_p" + #datadir = '/remote-home/yfshao/workdir/datasets/SST' + datadir = '/remote-home/ygwang/yelp_polarity' + #datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} + datafile = {"train": "train.csv", "test": "test.csv"} + lr = 1e-3 + def __init__(self): - self.datapath = {k:os.path.join(self.datadir, v) + self.datapath = {k: os.path.join(self.datadir, v) for k, v in self.datafile.items()} -ops=Config() + +ops = Config() -##1.task相关信息：利用dataloader载入dataInfo -datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds='train') +# 1.task相关信息：利用dataloader载入dataInfo +#datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) +datainfo = yelpLoader(fine_grained=True, lower=True).process( + paths=ops.datapath, train_ds=['train']) print(len(datainfo.datasets['train'])) -print(len(datainfo.datasets['dev'])) +print(len(datainfo.datasets['test'])) -## 2.或直接复用fastNLP的模型 -vocab = datainfo.vocabs['words'] +# 2.或直接复用fastNLP的模型 +vocab = datainfo.vocabs['words'] # embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) -embedding = StaticEmbedding(vocab) +#embedding = StaticEmbedding(vocab) +embedding = StaticEmbedding( + vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) + print(len(vocab)) print(len(datainfo.vocabs['target'])) + model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) -## 3. 声明loss,metric,optimizer -loss=CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) -metric=AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) -optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], - lr=ops.lr, momentum=0.9, weight_decay=0) + +# 3. 声明loss,metric,optimizer +loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) +metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) +optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], + lr=ops.lr, momentum=0.9, weight_decay=0) callbacks = [] callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + print(device) for ds in datainfo.datasets.values(): @@ -67,14 +85,17 @@ for ds in datainfo.datasets.values(): ds.set_input(C.INPUT, C.INPUT_LEN) ds.set_target(C.TARGET) -## 4.定义train方法 -def train(model,datainfo,loss,metrics,optimizer,num_epochs=ops.train_epoch): + +# 4.定义train方法 +def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch): trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=[metrics], dev_data=datainfo.datasets['dev'], device=device, + metrics=[metrics], + dev_data=datainfo.datasets['test'], device=device, check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, n_epochs=num_epochs) + print(trainer.train()) -if __name__=="__main__": - train(model,datainfo,loss,metric,optimizer) \ No newline at end of file +if __name__ == "__main__": + train(model, datainfo, loss, metric, optimizer) From 4272778c9ae6be70204d77261e35ceebd8e8f31c Mon Sep 17 00:00:00 2001 From: yunfan Date: Sat, 6 Jul 2019 13:15:40 +0800 Subject: [PATCH 18/27] -update DPCNN & train script -use spacy tokenizer for yelp data -add set_rng_seed --- fastNLP/modules/aggregator/attention.py | 9 +- .../text_classification/data/yelpLoader.py | 18 +++- .../text_classification/model/dpcnn.py | 22 +++-- .../text_classification/train_dpcnn.py | 83 ++++++++++++------- .../text_classification/utils/util_init.py | 11 +++ 5 files changed, 94 insertions(+), 49 deletions(-) create mode 100644 reproduction/text_classification/utils/util_init.py diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index 4101b033..2bee7f2e 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -19,7 +19,7 @@ class DotAttention(nn.Module): 补上文档 """ - def __init__(self, key_size, value_size, dropout=0): + def __init__(self, key_size, value_size, dropout=0.0): super(DotAttention, self).__init__() self.key_size = key_size self.value_size = value_size @@ -37,7 +37,7 @@ class DotAttention(nn.Module): """ output = torch.matmul(Q, K.transpose(1, 2)) / self.scale if mask_out is not None: - output.masked_fill_(mask_out, -1e8) + output.masked_fill_(mask_out, -1e18) output = self.softmax(output) output = self.drop(output) return torch.matmul(output, V) @@ -67,9 +67,8 @@ class MultiHeadAttention(nn.Module): self.k_in = nn.Linear(input_size, in_size) self.v_in = nn.Linear(input_size, in_size) # follow the paper, do not apply dropout within dot-product - self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=0) + self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=dropout) self.out = nn.Linear(value_size * num_head, input_size) - self.drop = TimestepDropout(dropout) self.reset_parameters() def reset_parameters(self): @@ -105,7 +104,7 @@ class MultiHeadAttention(nn.Module): # concat all heads, do output linear atte = atte.permute(1, 2, 0, 3).contiguous().view(batch, sq, -1) - output = self.drop(self.out(atte)) + output = self.out(atte) return output diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index 63605ecf..d97f9399 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -8,11 +8,20 @@ from fastNLP.io.base_loader import DataInfo from fastNLP.io.embed_loader import EmbeddingOption from fastNLP.io.file_reader import _read_json from typing import Union, Dict -from reproduction.Star_transformer.datasets import EmbedLoader from reproduction.utils import check_dataloader_paths -def clean_str(sentence, char_lower=False): +def get_tokenizer(): + try: + import spacy + en = spacy.load('en') + print('use spacy tokenizer') + return lambda x: [w.text for w in en.tokenizer(x)] + except Exception as e: + print('use raw tokenizer') + return lambda x: x.split() + +def clean_str(sentence, tokenizer, char_lower=False): """ heavily borrowed from github https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb @@ -23,7 +32,7 @@ def clean_str(sentence, char_lower=False): sentence = sentence.lower() import re nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') - words = sentence.split() + words = tokenizer(sentence) words_collection = [] for word in words: if word in ['-lrb-', '-rrb-', '', '-r', '-l', 'b-']: @@ -65,6 +74,7 @@ class yelpLoader(JsonLoader): self.fine_grained = fine_grained self.tag_v = tag_v self.lower = lower + self.tokenizer = get_tokenizer() ''' def _load_json(self, path): @@ -109,7 +119,7 @@ class yelpLoader(JsonLoader): all_count += 1 if len(row) == 2: target = self.tag_v[row[0] + ".0"] - words = clean_str(row[1], self.lower) + words = clean_str(row[1], self.tokenizer, self.lower) if len(words) != 0: ds.append(Instance(words=words, target=target)) real_count += 1 diff --git a/reproduction/text_classification/model/dpcnn.py b/reproduction/text_classification/model/dpcnn.py index 2da7b3e5..dafe62bc 100644 --- a/reproduction/text_classification/model/dpcnn.py +++ b/reproduction/text_classification/model/dpcnn.py @@ -3,22 +3,27 @@ import torch.nn as nn from fastNLP.modules.utils import get_embeddings from fastNLP.core import Const as C + class DPCNN(nn.Module): - def __init__(self, init_embed, num_cls, n_filters=256, kernel_size=3, n_layers=7, embed_dropout=0.1, dropout=0.1): + def __init__(self, init_embed, num_cls, n_filters=256, + kernel_size=3, n_layers=7, embed_dropout=0.1, cls_dropout=0.1): super().__init__() - self.region_embed = RegionEmbedding(init_embed, out_dim=n_filters, kernel_sizes=[3, 5, 9]) + self.region_embed = RegionEmbedding( + init_embed, out_dim=n_filters, kernel_sizes=[1, 3, 5]) embed_dim = self.region_embed.embedding_dim self.conv_list = nn.ModuleList() for i in range(n_layers): self.conv_list.append(nn.Sequential( nn.ReLU(), - nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), - nn.Conv1d(n_filters, n_filters, kernel_size, padding=kernel_size//2), + nn.Conv1d(n_filters, n_filters, kernel_size, + padding=kernel_size//2), + nn.Conv1d(n_filters, n_filters, kernel_size, + padding=kernel_size//2), )) self.pool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1) self.embed_drop = nn.Dropout(embed_dropout) self.classfier = nn.Sequential( - nn.Dropout(dropout), + nn.Dropout(cls_dropout), nn.Linear(n_filters, num_cls), ) self.reset_parameters() @@ -57,7 +62,8 @@ class RegionEmbedding(nn.Module): super().__init__() if kernel_sizes is None: kernel_sizes = [5, 9] - assert isinstance(kernel_sizes, list), 'kernel_sizes should be List(int)' + assert isinstance( + kernel_sizes, list), 'kernel_sizes should be List(int)' self.embed = get_embeddings(init_embed) try: embed_dim = self.embed.embedding_dim @@ -69,14 +75,14 @@ class RegionEmbedding(nn.Module): nn.Conv1d(embed_dim, embed_dim, ksz, padding=ksz // 2), )) self.linears = nn.ModuleList([nn.Conv1d(embed_dim, out_dim, 1) - for _ in range(len(kernel_sizes) + 1)]) + for _ in range(len(kernel_sizes))]) self.embedding_dim = embed_dim def forward(self, x): x = self.embed(x) x = x.transpose(1, 2) # B, C, L - out = self.linears[0](x) + out = 0 for conv, fc in zip(self.region_embeds, self.linears[1:]): conv_i = conv(x) out = out + fc(conv_i) diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index bf243ffb..294a0742 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -1,40 +1,44 @@ # 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 -from torch.optim.lr_scheduler import CosineAnnealingLR import torch.cuda +from fastNLP.core.utils import cache_results from torch.optim import SGD +from torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR from fastNLP.core.trainer import Trainer from fastNLP import CrossEntropyLoss, AccuracyMetric from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding from reproduction.text_classification.model.dpcnn import DPCNN -from .data.yelpLoader import yelpLoader -from fastNLP.io.dataset_loader import SSTLoader +from data.yelpLoader import yelpLoader import torch.nn as nn from fastNLP.core import LRScheduler from fastNLP.core.const import Const as C -import sys +from fastNLP.core.vocabulary import VocabularyOption +from utils.util_init import set_rng_seeds import os os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" -sys.path.append('../..') - # hyper class Config(): - model_dir_or_name = "en-base-uncased" - embedding_grad = False, + seed = 12345 + model_dir_or_name = "dpcnn-yelp-p" + embedding_grad = True train_epoch = 30 batch_size = 100 num_classes = 2 task = "yelp_p" #datadir = '/remote-home/yfshao/workdir/datasets/SST' - datadir = '/remote-home/ygwang/yelp_polarity' + datadir = '/remote-home/yfshao/workdir/datasets/yelp_polarity' #datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} datafile = {"train": "train.csv", "test": "test.csv"} lr = 1e-3 + src_vocab_op = VocabularyOption() + embed_dropout = 0.3 + cls_dropout = 0.1 + weight_decay = 1e-4 def __init__(self): self.datapath = {k: os.path.join(self.datadir, v) @@ -43,15 +47,23 @@ class Config(): ops = Config() +set_rng_seeds(ops.seed) +print('RNG SEED: {}'.format(ops.seed)) # 1.task相关信息：利用dataloader载入dataInfo #datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) -datainfo = yelpLoader(fine_grained=True, lower=True).process( - paths=ops.datapath, train_ds=['train']) -print(len(datainfo.datasets['train'])) -print(len(datainfo.datasets['test'])) - +@cache_results(ops.model_dir_or_name+'-data-cache') +def load_data(): + datainfo = yelpLoader(fine_grained=True, lower=True).process( + paths=ops.datapath, train_ds=['train'], src_vocab_op=ops.src_vocab_op) + for ds in datainfo.datasets.values(): + ds.apply_field(len, C.INPUT, C.INPUT_LEN) + ds.set_input(C.INPUT, C.INPUT_LEN) + ds.set_target(C.TARGET) + return datainfo + +datainfo = load_data() # 2.或直接复用fastNLP的模型 @@ -59,43 +71,50 @@ vocab = datainfo.vocabs['words'] # embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) #embedding = StaticEmbedding(vocab) embedding = StaticEmbedding( - vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) + vocab, model_dir_or_name='en-word2vec-300', requires_grad=ops.embedding_grad, + normalize=False +) + +print(len(datainfo.datasets['train'])) +print(len(datainfo.datasets['test'])) +print(datainfo.datasets['train'][0]) print(len(vocab)) print(len(datainfo.vocabs['target'])) -model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) - +model = DPCNN(init_embed=embedding, num_cls=ops.num_classes, + embed_dropout=ops.embed_dropout, cls_dropout=ops.cls_dropout) +print(model) # 3. 声明loss,metric,optimizer loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], - lr=ops.lr, momentum=0.9, weight_decay=0) + lr=ops.lr, momentum=0.9, weight_decay=ops.weight_decay) callbacks = [] callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) +# callbacks.append +# LRScheduler(LambdaLR(optimizer, lambda epoch: ops.lr if epoch < +# ops.train_epoch * 0.8 else ops.lr * 0.1)) +# ) + +# callbacks.append( +# FitlogCallback(data=datainfo.datasets, verbose=1) +# ) device = 'cuda:0' if torch.cuda.is_available() else 'cpu' print(device) -for ds in datainfo.datasets.values(): - ds.apply_field(len, C.INPUT, C.INPUT_LEN) - ds.set_input(C.INPUT, C.INPUT_LEN) - ds.set_target(C.TARGET) - - # 4.定义train方法 -def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch): - trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=[metrics], - dev_data=datainfo.datasets['test'], device=device, - check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, - n_epochs=num_epochs) +trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=[metric], + dev_data=datainfo.datasets['test'], device=device, + check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, + n_epochs=ops.train_epoch, num_workers=4) - print(trainer.train()) if __name__ == "__main__": - train(model, datainfo, loss, metric, optimizer) + print(trainer.train()) diff --git a/reproduction/text_classification/utils/util_init.py b/reproduction/text_classification/utils/util_init.py new file mode 100644 index 00000000..fcb8fffb --- /dev/null +++ b/reproduction/text_classification/utils/util_init.py @@ -0,0 +1,11 @@ +import numpy +import torch +import random + + +def set_rng_seeds(seed): + random.seed(seed) + numpy.random.seed(seed) + torch.random.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + # print('RNG_SEED {}'.format(seed)) From 451af5318753e854deabd73ce9b6877a55480bbd Mon Sep 17 00:00:00 2001 From: yunfan Date: Sun, 7 Jul 2019 16:07:11 +0800 Subject: [PATCH 19/27] - update dpcnn - add train_idcnn - update sst loader --- fastNLP/io/data_loader/sst.py | 44 +++++---- fastNLP/io/utils.py | 69 +++++++++++++ .../ner/model/dilated_cnn.py | 65 ++++++++---- .../seqence_labelling/ner/train_idcnn.py | 99 +++++++++++++++++++ .../text_classification/data/yelpLoader.py | 13 +-- .../text_classification/train_dpcnn.py | 48 ++++----- reproduction/utils.py | 11 ++- 7 files changed, 274 insertions(+), 75 deletions(-) create mode 100644 fastNLP/io/utils.py create mode 100644 reproduction/seqence_labelling/ner/train_idcnn.py diff --git a/fastNLP/io/data_loader/sst.py b/fastNLP/io/data_loader/sst.py index 021a79b7..8d0d005f 100644 --- a/fastNLP/io/data_loader/sst.py +++ b/fastNLP/io/data_loader/sst.py @@ -5,10 +5,8 @@ from ..base_loader import DataInfo, DataSetLoader from ...core.vocabulary import VocabularyOption, Vocabulary from ...core.dataset import DataSet from ...core.instance import Instance -from ..embed_loader import EmbeddingOption, EmbedLoader +from ..utils import check_dataloader_paths, get_tokenizer -spacy.prefer_gpu() -sptk = spacy.load('en') class SSTLoader(DataSetLoader): URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' @@ -37,6 +35,7 @@ class SSTLoader(DataSetLoader): tag_v['0'] = tag_v['1'] tag_v['4'] = tag_v['3'] self.tag_v = tag_v + self.tokenizer = get_tokenizer() def _load(self, path): """ @@ -55,29 +54,37 @@ class SSTLoader(DataSetLoader): ds.append(Instance(words=words, target=tag)) return ds - @staticmethod - def _get_one(data, subtree): + def _get_one(self, data, subtree): tree = Tree.fromstring(data) if subtree: - return [([x.text for x in sptk.tokenizer(' '.join(t.leaves()))], t.label()) for t in tree.subtrees() ] - return [([x.text for x in sptk.tokenizer(' '.join(tree.leaves()))], tree.label())] + return [([x.text for x in self.tokenizer(' '.join(t.leaves()))], t.label()) for t in tree.subtrees() ] + return [([x.text for x in self.tokenizer(' '.join(tree.leaves()))], tree.label())] def process(self, - paths, - train_ds: Iterable[str] = None, + paths, train_subtree=True, src_vocab_op: VocabularyOption = None, - tgt_vocab_op: VocabularyOption = None, - src_embed_op: EmbeddingOption = None): + tgt_vocab_op: VocabularyOption = None,): + paths = check_dataloader_paths(paths) input_name, target_name = 'words', 'target' src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) tgt_vocab = Vocabulary(unknown=None, padding=None) \ if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) - info = DataInfo(datasets=self.load(paths)) - _train_ds = [info.datasets[name] - for name in train_ds] if train_ds else info.datasets.values() - src_vocab.from_dataset(*_train_ds, field_name=input_name) - tgt_vocab.from_dataset(*_train_ds, field_name=target_name) + info = DataInfo() + origin_subtree = self.subtree + self.subtree = train_subtree + info.datasets['train'] = self._load(paths['train']) + self.subtree = origin_subtree + for n, p in paths.items(): + if n != 'train': + info.datasets[n] = self._load(p) + + src_vocab.from_dataset( + info.datasets['train'], + field_name=input_name, + no_create_entry_dataset=[ds for n, ds in info.datasets.items() if n != 'train']) + tgt_vocab.from_dataset(info.datasets['train'], field_name=target_name) + src_vocab.index_dataset( *info.datasets.values(), field_name=input_name, new_field_name=input_name) @@ -89,10 +96,5 @@ class SSTLoader(DataSetLoader): target_name: tgt_vocab } - if src_embed_op is not None: - src_embed_op.vocab = src_vocab - init_emb = EmbedLoader.load_with_vocab(**src_embed_op) - info.embeddings[input_name] = init_emb - return info diff --git a/fastNLP/io/utils.py b/fastNLP/io/utils.py new file mode 100644 index 00000000..a7d2de85 --- /dev/null +++ b/fastNLP/io/utils.py @@ -0,0 +1,69 @@ +import os + +from typing import Union, Dict + + +def check_dataloader_paths(paths:Union[str, Dict[str, str]])->Dict[str, str]: + """ + 检查传入dataloader的文件的合法性。如果为合法路径，将返回至少包含'train'这个key的dict。类似于下面的结果 + { + 'train': '/some/path/to/', # 一定包含，建词表应该在这上面建立，剩下的其它文件应该只需要处理并index。 + 'test': 'xxx' # 可能有，也可能没有 + ... + } + 如果paths为不合法的，将直接进行raise相应的错误 + + :param paths: 路径. 可以为一个文件路径(则认为该文件就是train的文件); 可以为一个文件目录，将在该目录下寻找train(文件名 + 中包含train这个字段), test.txt, dev.txt; 可以为一个dict, 则key是用户自定义的某个文件的名称，value是这个文件的路径。 + :return: + """ + if isinstance(paths, str): + if os.path.isfile(paths): + return {'train': paths} + elif os.path.isdir(paths): + filenames = os.listdir(paths) + files = {} + for filename in filenames: + path_pair = None + if 'train' in filename: + path_pair = ('train', filename) + if 'dev' in filename: + if path_pair: + raise Exception("File:{} in {} contains bot `{}` and `dev`.".format(filename, paths, path_pair[0])) + path_pair = ('dev', filename) + if 'test' in filename: + if path_pair: + raise Exception("File:{} in {} contains bot `{}` and `test`.".format(filename, paths, path_pair[0])) + path_pair = ('test', filename) + if path_pair: + files[path_pair[0]] = os.path.join(paths, path_pair[1]) + return files + else: + raise FileNotFoundError(f"{paths} is not a valid file path.") + + elif isinstance(paths, dict): + if paths: + if 'train' not in paths: + raise KeyError("You have to include `train` in your dict.") + for key, value in paths.items(): + if isinstance(key, str) and isinstance(value, str): + if not os.path.isfile(value): + raise TypeError(f"{value} is not a valid file.") + else: + raise TypeError("All keys and values in paths should be str.") + return paths + else: + raise ValueError("Empty paths is not allowed.") + else: + raise TypeError(f"paths only supports str and dict. not {type(paths)}.") + +def get_tokenizer(): + try: + import spacy + spacy.prefer_gpu() + en = spacy.load('en') + print('use spacy tokenizer') + return lambda x: [w.text for w in en.tokenizer(x)] + except Exception as e: + print('use raw tokenizer') + return lambda x: x.split() diff --git a/reproduction/seqence_labelling/ner/model/dilated_cnn.py b/reproduction/seqence_labelling/ner/model/dilated_cnn.py index cd2fa64b..a4e02159 100644 --- a/reproduction/seqence_labelling/ner/model/dilated_cnn.py +++ b/reproduction/seqence_labelling/ner/model/dilated_cnn.py @@ -8,16 +8,23 @@ from fastNLP.core.const import Const as C class IDCNN(nn.Module): - def __init__(self, init_embed, char_embed, + def __init__(self, + init_embed, + char_embed, num_cls, repeats, num_layers, num_filters, kernel_size, use_crf=False, use_projection=False, block_loss=False, input_dropout=0.3, hidden_dropout=0.2, inner_dropout=0.0): super(IDCNN, self).__init__() self.word_embeddings = Embedding(init_embed) - self.char_embeddings = Embedding(char_embed) - embedding_size = self.word_embeddings.embedding_dim + \ - self.char_embeddings.embedding_dim + + if char_embed is None: + self.char_embeddings = None + embedding_size = self.word_embeddings.embedding_dim + else: + self.char_embeddings = Embedding(char_embed) + embedding_size = self.word_embeddings.embedding_dim + \ + self.char_embeddings.embedding_dim self.conv0 = nn.Sequential( nn.Conv1d(in_channels=embedding_size, @@ -31,7 +38,7 @@ class IDCNN(nn.Module): block = [] for layer_i in range(num_layers): - dilated = 2 ** layer_i + dilated = 2 ** layer_i if layer_i+1 < num_layers else 1 block.append(nn.Conv1d( in_channels=num_filters, out_channels=num_filters, @@ -67,11 +74,24 @@ class IDCNN(nn.Module): self.crf = ConditionalRandomField( num_tags=num_cls) if use_crf else None self.block_loss = block_loss + self.reset_parameters() - def forward(self, words, chars, seq_len, target=None): - e1 = self.word_embeddings(words) - e2 = self.char_embeddings(chars) - x = torch.cat((e1, e2), dim=-1) # b,l,h + def reset_parameters(self): + for m in self.modules(): + if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)): + nn.init.xavier_normal_(m.weight, gain=1) + if m.bias is not None: + nn.init.normal_(m.bias, mean=0, std=0.01) + + def forward(self, words, seq_len, target=None, chars=None): + if self.char_embeddings is None: + x = self.word_embeddings(words) + else: + if chars is None: + raise ValueError('must provide chars for model with char embedding') + e1 = self.word_embeddings(words) + e2 = self.char_embeddings(chars) + x = torch.cat((e1, e2), dim=-1) # b,l,h mask = seq_len_to_mask(seq_len) x = x.transpose(1, 2) # b,h,l @@ -84,21 +104,24 @@ class IDCNN(nn.Module): def compute_loss(y, t, mask): if self.crf is not None and target is not None: - loss = self.crf(y, t, mask) + loss = self.crf(y.transpose(1, 2), t, mask) else: t.masked_fill_(mask == 0, -100) loss = F.cross_entropy(y, t, ignore_index=-100) return loss - if self.block_loss: - losses = [compute_loss(o, target, mask) for o in output] - loss = sum(losses) + if target is not None: + if self.block_loss: + losses = [compute_loss(o, target, mask) for o in output] + loss = sum(losses) + else: + loss = compute_loss(output[-1], target, mask) else: - loss = compute_loss(output[-1], target, mask) + loss = None scores = output[-1] if self.crf is not None: - pred = self.crf.viterbi_decode(scores, target, mask) + pred, _ = self.crf.viterbi_decode(scores.transpose(1, 2), mask) else: pred = scores.max(1)[1] * mask.long() @@ -107,5 +130,13 @@ class IDCNN(nn.Module): C.OUTPUT: pred, } - def predict(self, words, chars, seq_len): - return self.forward(words, chars, seq_len)[C.OUTPUT] + def predict(self, words, seq_len, chars=None): + res = self.forward( + words=words, + seq_len=seq_len, + chars=chars, + target=None + )[C.OUTPUT] + return { + C.OUTPUT: res + } diff --git a/reproduction/seqence_labelling/ner/train_idcnn.py b/reproduction/seqence_labelling/ner/train_idcnn.py new file mode 100644 index 00000000..1781c763 --- /dev/null +++ b/reproduction/seqence_labelling/ner/train_idcnn.py @@ -0,0 +1,99 @@ +from reproduction.seqence_labelling.ner.data.OntoNoteLoader import OntoNoteNERDataLoader +from fastNLP.core.callback import FitlogCallback, LRScheduler +from fastNLP import GradientClipCallback +from torch.optim.lr_scheduler import LambdaLR, CosineAnnealingLR +from torch.optim import SGD, Adam +from fastNLP import Const +from fastNLP import RandomSampler, BucketSampler +from fastNLP import SpanFPreRecMetric +from fastNLP import Trainer +from reproduction.seqence_labelling.ner.model.dilated_cnn import IDCNN +from fastNLP.core.utils import Option +from fastNLP.modules.encoder.embedding import CNNCharEmbedding, StaticEmbedding +from fastNLP.core.utils import cache_results +import sys +import torch.cuda +import os +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +encoding_type = 'bioes' + + +def get_path(path): + return os.path.join(os.environ['HOME'], path) + +data_path = get_path('workdir/datasets/ontonotes-v4') + +ops = Option( + batch_size=128, + num_epochs=100, + lr=3e-4, + repeats=3, + num_layers=3, + num_filters=400, + use_crf=True, + gradient_clip=5, +) + +@cache_results('ontonotes-cache') +def load_data(): + + data = OntoNoteNERDataLoader(encoding_type=encoding_type).process(data_path, + lower=True) + + # char_embed = CNNCharEmbedding(vocab=data.vocabs['cap_words'], embed_size=30, char_emb_size=30, filter_nums=[30], + # kernel_sizes=[3]) + + word_embed = StaticEmbedding(vocab=data.vocabs[Const.INPUT], + model_dir_or_name='en-glove-840b-300', + requires_grad=True) + return data, [word_embed] + +data, embeds = load_data() +print(data.datasets['train'][0]) +print(list(data.vocabs.keys())) + +for ds in data.datasets.values(): + ds.rename_field('cap_words', 'chars') + ds.set_input('chars') + +word_embed = embeds[0] +char_embed = CNNCharEmbedding(data.vocabs['cap_words']) +# for ds in data.datasets: +# ds.rename_field('') + +print(data.vocabs[Const.TARGET].word2idx) + +model = IDCNN(init_embed=word_embed, + char_embed=char_embed, + num_cls=len(data.vocabs[Const.TARGET]), + repeats=ops.repeats, + num_layers=ops.num_layers, + num_filters=ops.num_filters, + kernel_size=3, + use_crf=ops.use_crf, use_projection=True, + block_loss=True, + input_dropout=0.33, hidden_dropout=0.2, inner_dropout=0.2) + +print(model) + +callbacks = [GradientClipCallback(clip_value=ops.gradient_clip, clip_type='norm'),] + +optimizer = Adam(model.parameters(), lr=ops.lr, weight_decay=0) +# scheduler = LRScheduler(LambdaLR(optimizer, lr_lambda=lambda epoch: 1 / (1 + 0.05 * epoch))) +# callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 15))) +# optimizer = SWATS(model.parameters(), verbose=True) +# optimizer = Adam(model.parameters(), lr=0.005) + +device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + +trainer = Trainer(train_data=data.datasets['train'], model=model, optimizer=optimizer, + sampler=BucketSampler(num_buckets=50, batch_size=ops.batch_size), + device=device, dev_data=data.datasets['dev'], batch_size=ops.batch_size, + metrics=SpanFPreRecMetric( + tag_vocab=data.vocabs[Const.TARGET], encoding_type=encoding_type), + check_code_level=-1, + callbacks=callbacks, num_workers=2, n_epochs=ops.num_epochs) +trainer.train() diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index d97f9399..704c29e5 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -8,18 +8,7 @@ from fastNLP.io.base_loader import DataInfo from fastNLP.io.embed_loader import EmbeddingOption from fastNLP.io.file_reader import _read_json from typing import Union, Dict -from reproduction.utils import check_dataloader_paths - - -def get_tokenizer(): - try: - import spacy - en = spacy.load('en') - print('use spacy tokenizer') - return lambda x: [w.text for w in en.tokenizer(x)] - except Exception as e: - print('use raw tokenizer') - return lambda x: x.split() +from reproduction.utils import check_dataloader_paths, get_tokenizer def clean_str(sentence, tokenizer, char_lower=False): """ diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index 294a0742..9664bf75 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -9,6 +9,7 @@ from fastNLP import CrossEntropyLoss, AccuracyMetric from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding from reproduction.text_classification.model.dpcnn import DPCNN from data.yelpLoader import yelpLoader +from fastNLP.core.sampler import BucketSampler import torch.nn as nn from fastNLP.core import LRScheduler from fastNLP.core.const import Const as C @@ -28,19 +29,20 @@ class Config(): embedding_grad = True train_epoch = 30 batch_size = 100 - num_classes = 2 task = "yelp_p" - #datadir = '/remote-home/yfshao/workdir/datasets/SST' - datadir = '/remote-home/yfshao/workdir/datasets/yelp_polarity' + #datadir = 'workdir/datasets/SST' + datadir = 'workdir/datasets/yelp_polarity' + # datadir = 'workdir/datasets/yelp_full' #datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} datafile = {"train": "train.csv", "test": "test.csv"} lr = 1e-3 - src_vocab_op = VocabularyOption() + src_vocab_op = VocabularyOption(max_size=100000) embed_dropout = 0.3 cls_dropout = 0.1 - weight_decay = 1e-4 + weight_decay = 1e-5 def __init__(self): + self.datadir = os.path.join(os.environ['HOME'], self.datadir) self.datapath = {k: os.path.join(self.datadir, v) for k, v in self.datafile.items()} @@ -53,6 +55,8 @@ print('RNG SEED: {}'.format(ops.seed)) # 1.task相关信息：利用dataloader载入dataInfo #datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) + + @cache_results(ops.model_dir_or_name+'-data-cache') def load_data(): datainfo = yelpLoader(fine_grained=True, lower=True).process( @@ -61,28 +65,23 @@ def load_data(): ds.apply_field(len, C.INPUT, C.INPUT_LEN) ds.set_input(C.INPUT, C.INPUT_LEN) ds.set_target(C.TARGET) - return datainfo + embedding = StaticEmbedding( + datainfo.vocabs['words'], model_dir_or_name='en-glove-840b-300', requires_grad=ops.embedding_grad, + normalize=False + ) + return datainfo, embedding -datainfo = load_data() + +datainfo, embedding = load_data() # 2.或直接复用fastNLP的模型 -vocab = datainfo.vocabs['words'] # embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) -#embedding = StaticEmbedding(vocab) -embedding = StaticEmbedding( - vocab, model_dir_or_name='en-word2vec-300', requires_grad=ops.embedding_grad, - normalize=False -) -print(len(datainfo.datasets['train'])) -print(len(datainfo.datasets['test'])) +print(datainfo) print(datainfo.datasets['train'][0]) -print(len(vocab)) -print(len(datainfo.vocabs['target'])) - -model = DPCNN(init_embed=embedding, num_cls=ops.num_classes, +model = DPCNN(init_embed=embedding, num_cls=len(datainfo.vocabs[C.TARGET]), embed_dropout=ops.embed_dropout, cls_dropout=ops.cls_dropout) print(model) @@ -93,11 +92,11 @@ optimizer = SGD([param for param in model.parameters() if param.requires_grad == lr=ops.lr, momentum=0.9, weight_decay=ops.weight_decay) callbacks = [] -callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) -# callbacks.append -# LRScheduler(LambdaLR(optimizer, lambda epoch: ops.lr if epoch < -# ops.train_epoch * 0.8 else ops.lr * 0.1)) -# ) +# callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) +callbacks.append( + LRScheduler(LambdaLR(optimizer, lambda epoch: ops.lr if epoch < + ops.train_epoch * 0.8 else ops.lr * 0.1)) +) # callbacks.append( # FitlogCallback(data=datainfo.datasets, verbose=1) @@ -109,6 +108,7 @@ print(device) # 4.定义train方法 trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + sampler=BucketSampler(num_buckets=50, batch_size=ops.batch_size), metrics=[metric], dev_data=datainfo.datasets['test'], device=device, check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, diff --git a/reproduction/utils.py b/reproduction/utils.py index 4f0d021e..a7d2de85 100644 --- a/reproduction/utils.py +++ b/reproduction/utils.py @@ -57,4 +57,13 @@ def check_dataloader_paths(paths:Union[str, Dict[str, str]])->Dict[str, str]: else: raise TypeError(f"paths only supports str and dict. not {type(paths)}.") - +def get_tokenizer(): + try: + import spacy + spacy.prefer_gpu() + en = spacy.load('en') + print('use spacy tokenizer') + return lambda x: [w.text for w in en.tokenizer(x)] + except Exception as e: + print('use raw tokenizer') + return lambda x: x.split() From 1d4e9968f242f2547a13ad0ab1586fc9f15a59a8 Mon Sep 17 00:00:00 2001 From: yunfan Date: Sun, 7 Jul 2019 16:29:01 +0800 Subject: [PATCH 20/27] - update star-transformer README --- reproduction/Star_transformer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reproduction/Star_transformer/README.md b/reproduction/Star_transformer/README.md index 37c5f1e9..d07d5536 100644 --- a/reproduction/Star_transformer/README.md +++ b/reproduction/Star_transformer/README.md @@ -6,7 +6,7 @@ paper: [Star-Transformer](https://arxiv.org/abs/1902.09113) |Pos Tagging|CTB 9.0|-|ACC 92.31| |Pos Tagging|CONLL 2012|-|ACC 96.51| |Named Entity Recognition|CONLL 2012|-|F1 85.66| -|Text Classification|SST|-|49.18| +|Text Classification|SST|-|51.2| |Natural Language Inference|SNLI|-|83.76| ## Usage From 5d9e064ec27595a1e09d7ddcbb27280c685b0701 Mon Sep 17 00:00:00 2001 From: lyhuang18 <42239874+lyhuang18@users.noreply.github.com> Date: Mon, 8 Jul 2019 01:11:46 +0800 Subject: [PATCH 21/27] text_classfication --- .../text_classification/data/SSTLoader.py | 90 ++++++++++++++++++- .../text_classification/train_awdlstm.py | 41 +-------- .../text_classification/train_lstm.py | 43 ++------- .../text_classification/train_lstm_att.py | 41 +-------- 4 files changed, 102 insertions(+), 113 deletions(-) diff --git a/reproduction/text_classification/data/SSTLoader.py b/reproduction/text_classification/data/SSTLoader.py index b570994e..d8403b7a 100644 --- a/reproduction/text_classification/data/SSTLoader.py +++ b/reproduction/text_classification/data/SSTLoader.py @@ -5,7 +5,8 @@ from fastNLP.core.vocabulary import VocabularyOption, Vocabulary from fastNLP import DataSet from fastNLP import Instance from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader - +import csv +from typing import Union, Dict class SSTLoader(DataSetLoader): URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' @@ -97,3 +98,90 @@ class SSTLoader(DataSetLoader): return info +class sst2Loader(DataSetLoader): + ''' + 数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', + ''' + def __init__(self): + super(sst2Loader, self).__init__() + + def _load(self, path: str) -> DataSet: + ds = DataSet() + all_count=0 + csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t') + skip_row = 0 + for idx,row in enumerate(csv_reader): + if idx<=skip_row: + continue + target = row[1] + words = row[0].split() + ds.append(Instance(words=words,target=target)) + all_count+=1 + print("all count:", all_count) + return ds + + def process(self, + paths: Union[str, Dict[str, str]], + src_vocab_opt: VocabularyOption = None, + tgt_vocab_opt: VocabularyOption = None, + src_embed_opt: EmbeddingOption = None, + char_level_op=False): + + paths = check_dataloader_paths(paths) + datasets = {} + info = DataInfo() + for name, path in paths.items(): + dataset = self.load(path) + datasets[name] = dataset + + def wordtochar(words): + chars=[] + for word in words: + word=word.lower() + for char in word: + chars.append(char) + return chars + + input_name, target_name = 'words', 'target' + info.vocabs={} + + # 就分隔为char形式 + if char_level_op: + for dataset in datasets.values(): + dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') + + src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) + src_vocab.from_dataset(datasets['train'], field_name='words') + src_vocab.index_dataset(*datasets.values(), field_name='words') + + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) + tgt_vocab.from_dataset(datasets['train'], field_name='target') + tgt_vocab.index_dataset(*datasets.values(), field_name='target') + + + info.vocabs = { + "words": src_vocab, + "target": tgt_vocab + } + + info.datasets = datasets + + + if src_embed_opt is not None: + embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) + info.embeddings['words'] = embed + + return info + +if __name__=="__main__": + datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv", + "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"} + datainfo=sst2Loader().process(datapath,char_level_op=True) + #print(datainfo.datasets["train"]) + len_count = 0 + for instance in datainfo.datasets["train"]: + len_count += len(instance["chars"]) + + ave_len = len_count / len(datainfo.datasets["train"]) + print(ave_len) \ No newline at end of file diff --git a/reproduction/text_classification/train_awdlstm.py b/reproduction/text_classification/train_awdlstm.py index ce3e52bc..e67bd25b 100644 --- a/reproduction/text_classification/train_awdlstm.py +++ b/reproduction/text_classification/train_awdlstm.py @@ -8,9 +8,7 @@ os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' import torch.nn as nn -from data.SSTLoader import SSTLoader from data.IMDBLoader import IMDBLoader -from data.yelpLoader import yelpLoader from fastNLP.modules.encoder.embedding import StaticEmbedding from model.awd_lstm import AWDLSTMSentiment @@ -41,18 +39,9 @@ opt=Config # load data -dataloaders = { - "IMDB":IMDBLoader(), - "YELP":yelpLoader(), - "SST-5":SSTLoader(subtree=True,fine_grained=True), - "SST-3":SSTLoader(subtree=True,fine_grained=False) -} - -if opt.task_name not in ["IMDB", "YELP", "SST-5", "SST-3"]: - raise ValueError("task name must in ['IMDB', 'YELP, 'SST-5', 'SST-3']") - -dataloader = dataloaders[opt.task_name] +dataloader=IMDBLoader() datainfo=dataloader.process(opt.datapath) + # print(datainfo.datasets["train"]) # print(datainfo) @@ -71,32 +60,10 @@ optimizer= Adam([param for param in model.parameters() if param.requires_grad==T def train(datainfo, model, optimizer, loss, metrics, opt): trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=metrics, dev_data=datainfo.datasets['dev'], device=0, check_code_level=-1, + metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, n_epochs=opt.train_epoch, save_path=opt.save_model_path) trainer.train() -def test(datainfo, metrics, opt): - # load model - model = ModelLoader.load_pytorch_model(opt.load_model_path) - print("model loaded!") - - # Tester - tester = Tester(datainfo.datasets['test'], model, metrics, batch_size=4, device=0) - acc = tester.test() - print("acc=",acc) - - - -parser = argparse.ArgumentParser() -parser.add_argument('--mode', required=True, dest="mode",help='set the model\'s model') - - -args = parser.parse_args() -if args.mode == 'train': +if __name__ == "__main__": train(datainfo, model, optimizer, loss, metrics, opt) -elif args.mode == 'test': - test(datainfo, metrics, opt) -else: - print('no mode specified for model!') - parser.print_help() diff --git a/reproduction/text_classification/train_lstm.py b/reproduction/text_classification/train_lstm.py index b320e79c..b89abc14 100644 --- a/reproduction/text_classification/train_lstm.py +++ b/reproduction/text_classification/train_lstm.py @@ -6,9 +6,7 @@ os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' import torch.nn as nn -from data.SSTLoader import SSTLoader from data.IMDBLoader import IMDBLoader -from data.yelpLoader import yelpLoader from fastNLP.modules.encoder.embedding import StaticEmbedding from model.lstm import BiLSTMSentiment @@ -38,18 +36,9 @@ opt=Config # load data -dataloaders = { - "IMDB":IMDBLoader(), - "YELP":yelpLoader(), - "SST-5":SSTLoader(subtree=True,fine_grained=True), - "SST-3":SSTLoader(subtree=True,fine_grained=False) -} - -if opt.task_name not in ["IMDB", "YELP", "SST-5", "SST-3"]: - raise ValueError("task name must in ['IMDB', 'YELP, 'SST-5', 'SST-3']") - -dataloader = dataloaders[opt.task_name] +dataloader=IMDBLoader() datainfo=dataloader.process(opt.datapath) + # print(datainfo.datasets["train"]) # print(datainfo) @@ -68,32 +57,10 @@ optimizer= Adam([param for param in model.parameters() if param.requires_grad==T def train(datainfo, model, optimizer, loss, metrics, opt): trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=metrics, dev_data=datainfo.datasets['dev'], device=0, check_code_level=-1, + metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, n_epochs=opt.train_epoch, save_path=opt.save_model_path) trainer.train() -def test(datainfo, metrics, opt): - # load model - model = ModelLoader.load_pytorch_model(opt.load_model_path) - print("model loaded!") - - # Tester - tester = Tester(datainfo.datasets['test'], model, metrics, batch_size=4, device=0) - acc = tester.test() - print("acc=",acc) - - - -parser = argparse.ArgumentParser() -parser.add_argument('--mode', required=True, dest="mode",help='set the model\'s model') - - -args = parser.parse_args() -if args.mode == 'train': - train(datainfo, model, optimizer, loss, metrics, opt) -elif args.mode == 'test': - test(datainfo, metrics, opt) -else: - print('no mode specified for model!') - parser.print_help() +if __name__ == "__main__": + train(datainfo, model, optimizer, loss, metrics, opt) \ No newline at end of file diff --git a/reproduction/text_classification/train_lstm_att.py b/reproduction/text_classification/train_lstm_att.py index 8db27d09..b4d37525 100644 --- a/reproduction/text_classification/train_lstm_att.py +++ b/reproduction/text_classification/train_lstm_att.py @@ -6,9 +6,7 @@ os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' import torch.nn as nn -from data.SSTLoader import SSTLoader from data.IMDBLoader import IMDBLoader -from data.yelpLoader import yelpLoader from fastNLP.modules.encoder.embedding import StaticEmbedding from model.lstm_self_attention import BiLSTM_SELF_ATTENTION @@ -40,18 +38,9 @@ opt=Config # load data -dataloaders = { - "IMDB":IMDBLoader(), - "YELP":yelpLoader(), - "SST-5":SSTLoader(subtree=True,fine_grained=True), - "SST-3":SSTLoader(subtree=True,fine_grained=False) -} - -if opt.task_name not in ["IMDB", "YELP", "SST-5", "SST-3"]: - raise ValueError("task name must in ['IMDB', 'YELP, 'SST-5', 'SST-3']") - -dataloader = dataloaders[opt.task_name] +dataloader=IMDBLoader() datainfo=dataloader.process(opt.datapath) + # print(datainfo.datasets["train"]) # print(datainfo) @@ -70,32 +59,10 @@ optimizer= Adam([param for param in model.parameters() if param.requires_grad==T def train(datainfo, model, optimizer, loss, metrics, opt): trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=metrics, dev_data=datainfo.datasets['dev'], device=0, check_code_level=-1, + metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, n_epochs=opt.train_epoch, save_path=opt.save_model_path) trainer.train() -def test(datainfo, metrics, opt): - # load model - model = ModelLoader.load_pytorch_model(opt.load_model_path) - print("model loaded!") - - # Tester - tester = Tester(datainfo.datasets['test'], model, metrics, batch_size=4, device=0) - acc = tester.test() - print("acc=",acc) - - - -parser = argparse.ArgumentParser() -parser.add_argument('--mode', required=True, dest="mode",help='set the model\'s model') - - -args = parser.parse_args() -if args.mode == 'train': +if __name__ == "__main__": train(datainfo, model, optimizer, loss, metrics, opt) -elif args.mode == 'test': - test(datainfo, metrics, opt) -else: - print('no mode specified for model!') - parser.print_help() From 46c82a7daac64c10ce425fe9c1201569dc02a75d Mon Sep 17 00:00:00 2001 From: lyhuang18 <42239874+lyhuang18@users.noreply.github.com> Date: Mon, 8 Jul 2019 01:29:42 +0800 Subject: [PATCH 22/27] text_classfication --- .../text_classification/results_LSTM.xlsx | Bin 9944 -> 0 bytes .../text_classification/train_awdlstm.py | 4 ++-- reproduction/text_classification/train_lstm.py | 4 ++-- .../text_classification/train_lstm_att.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 reproduction/text_classification/results_LSTM.xlsx diff --git a/reproduction/text_classification/results_LSTM.xlsx b/reproduction/text_classification/results_LSTM.xlsx deleted file mode 100644 index 0d7b841b12b43ee346c4d9db17032fade8c0b40a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9944 zcmeHNWmFtlw{0APHPE=by9Rd+1a}G2xVu{jmf!?;OK^8dkYEYHgIj>0!ChY`GxIXT zSV;*Y zQ45hC)h0rUXS_C5!#Y0QjUTPKibgeJC$)<0upZ5^AYS<_jgGkJsx8Zv&Qq#q$Zw*` zoeza>WwWJ%3(7R3YGYq==is64@57JRBRpxEHDR%)xTm1T;jVkcxzTx~v?X2|E+RqQ z&hV`S`f_?lXrN06T7ljT&xFtOxhgY({GwJ{n+gY~CRJs+3$X@E;&~K3s9L$X?ut}y zdvFYtV^D30&o4nLYE=gH){cV|g)q^$+P==7=ZFW(zITKjq(H%dru&($dEKO zat7PFu(JHv|2IAVgLCq4uU;0f__Ui1C3Ii*CanK_av>HduHYdi+e)tPA1F7EULTcD zMY!;emI$a$@Ek@apxyswaBe{`YP+BO>?>EL&<93FpCxe5>_EmD2`BSkZ!M{&Z5ah1=OUt z+V}F%sz%4$4_?0QnCA8Oe|(9eEV(5PcI2H)GQxcRheWt^HX>k znc2QSmP~(#b?eKll+mwB!G(FjF)Y(ZopTzXTgP?yG1IGu7pc3n|6m~aCF$EmNRRdJ zB;mXrhQ)#b05;(P0Cb3FJZ)J$9Gq=T92{(Z^kZcj1`g?LAiwOo`@kEe&XCHyB5Wkd z(g`<}R(n?2*ESi;*le|m2_@SBOWgEZ4Q;NZh_)=a(ZlWM`Yz55cGf#Vav!7wq%QiU zaBPa{yjrvJ(ua`+q?5)9nB|ytd8A^V;N03_5M|;nC^2; zOxZ=FJUAbF5`H0_!!*AcxhV<*4>Hga4rGfQc-GlVfBWO%JQmI;^%R?><^<9ZmXbSTw}hEt4w`G*JdfoV%`SrmhYGM z*wFP!2i#Ad@osrt4zAF5>#V_LIb3^f&UD16`S5tC5n4b+J0*Tdkh-zW<8EER{miqi z!Q5NdVC1D2IRsKPLFc;TOY4KuPVYQ_o!k<9LE}^xa29mv1{ad=xrwrcZ>V;%)R#_&?kIbnXhQS+ut?#FV^t`Dm%`tEvaH~hb?izQlIGI`!*`>c-O z4P3OcGci(}TF7k*-4s+zTT6p1t#E+{aP-k-nTMt0q>$|Rcm4zkCng_5T3R5T3;_TU z3X(s6^fQ0YqQCkaCkTYI%N}VAdwcVI2dVRL( z%)dfeCn{ASDd86K4Rs|8>z5%7ppSQ1&r5C9*k5rKzaFH2H+RJJ@nqk8ZJ=**3G$`? z9Wr<6;B66D0N^_s0DuR%;?I)nVhIMjy0HE{u>YvN8CrAE`Fub>BddF;%l77wd2Eg} z{Xl3$v~Y|ss+y3fF%wz*hnwgqRGtnhg)jG6l7(Q8~Fh^6_D<2cJ3-d0siYP3gF3qSfYi>@HKWOxyAN8cvh?29yIIEVV2q>jy zXiF>TftbSLp2y;AI5Ng4>Bvf-Pl|wxIUN_O6I8OW^4XGOEW^8lXl*IuU3#6tl2i)C z!I#OjhKcpMNrDu~DwqB&hL~~KWYw$mTG_AZs%y;dyg$7tXa)HOcuJ&clVbWK9BfP( zGqmB-uSS3h`RE7}i8(hqZc{m3EXU|hZ$+q9QHcjw^l1-RxG$RNz4E1P9C42WRBh+O zx7Op2IhPRDT79m)f*hU(G@ZwI?~2su?>;uuZmppbfL$x)jcJE+N_trmc=v5rx1oYr z%EN&P9AN_Np-ie6&IInKT|A0Kjt5JP#B-_KbbVn`gsI6Ajkwo?3sOK_=E9hra;vK< z*<32Dw?Ob#nk4;7!09hYk0?+lv#g#x=zHI642I@@lxr7I;D!|!jOeS2KZQ;>fp^Q6 zuL!BEU@kv=|v(d!SjJ;=0eVEj@lTkV%8!z>j;C(?^h`WivdJp!6{3tSUMXpo@g0&7Xdw zbO?1CCiHIma_YH?3ZsV-f)~yPYz&NUJmma=fhzQrvD@t?s*gD z{=Ca{)=GN;i9?(5v%g+wjk)G3rA9~2+ z@|}E9Vf(O`5WWEqksWAEW)DT|KtcqY<(Y5eKD7XPH;THR#b38jRvHd{;YMjgO`foW^M<}nO1}TRX7{OsBr7tWD(|V`sS9Bx9 z&s(sjyFA3j`gE5vmnZtjaz_xb!l~5#T*m!OxAnE*m{WXLm&o`)g|S558{68rvEs7S z^jbaqPQcgJfIHr>3g!xJ>3TC8MJLsgDtI1LWAWzn$Q@JU3IRc zL_q?9RFHuOK3fiP3JP*@*mxrM7KaHNnfK|yXM=g?=LN;Mzg}5 zf{Cl1UXpA#RRdqPTfcsqgMVma8>HsRW2zGjJ8yox%_Xt1%$qyevhiXPHg3XAH8F5b zpcX&%nFWMu*|%pYijEaj#Nqjxqe1^us7v zmc{x`C$jYx-Wf{yyuiQ~u;p&ey<*&b^nA<8;52opFI~w`^}VbMOUrQ1)ZXcH=+dAgg_ryOwM%dDJ%<8JU%K$diojHgHU|VCix5czr6A{z zg{yiBbYgsep3Zmiw%ac`5O~5x(#t(<95Dp)r)Pb^60}|Qo7Q241Ge_PICtK+@c&|W^+8@Jo0M##c1e)nno_q%;+pQ%>g zQ*i@p&VKi1VXe|rvkenJX*wYc#>ki^-Ab2~xFQKlbwZ^2^m*Gw9Cm#iJs23?;n*4{ z4w?=s3gL)RXq;2-`qPZ#uG0(wEJ*5x$k`OTh=RDoa1DRVPV|6rdqTD(0C{;;_x#YvqFGsoB^Go5b*_Lnz< zHw&F(@yz54w1+42_~T%Abg>Ayp){1`L`S}(;`PxCOup`UL>-*~T$&`oRKWw9_vSJhDNu{^@z4I_M$*7psjea&`zIcxv^%KfY-Q2nKqtCXsnf4;f|y8 z4%n0p%1u0bt#O5&N#6U z9z+V_d?3K6$N)?8?_sfSUa1I$GZ-@Bso#X|Yv_tKs%*iF1s{2bpLMcD-f7TYwR~|U*}_7xUh-F z=>Kl2q%OZoYb#8pR%BO3tW?N%o8RKmt_NWpDVpx4+c*?=9C zhiFOrGdEoHcCTSazf0>PQ56_aRoH*6!?}wf@{q%ITJ1`3{MzxgYdmM29go&3eUgQ# z)zNDA@FLye8{u43Hg9k~a*0imS@10-IwgaP23xcILHhBh)Mx616v$n#3y>Da2t=M{ z3rE=0uAkT{7K;JMcVH`{ip?wZHf$JV{R$Wuj0@FufiP3b8IEmR_8P)Dn=H~purPE+z z0^c%s?q;g*gUypgR~8wgb`)I2__Td@i<~N&#`*(Ym7Bj%MC4@lolXr;Ep_Ns#v}GN zW^z}BYVIy7veC73wLb2hQt3tv646{?K53h44Bp+!c;27e$bPm|qyE<*-`_o?&r zR#H1ptUg&;<;16ze>t;JhkK_0Mmez~zG*q9AaMr2@yB7I?Xf#ac!T9GUf6*#mj3|LB3Q3V1E$VceH}Sk!`p~sI$w` zSi9?zXNfcyeUp>Jtl7(GThwA7QCC9;1PB0ZFTahbW+~21D>T_P_<=Udf|Ja;hc5lu zJHrkcb*rKR<4+7LXI^kHO9rE_CN~`4R!4~u9@m7X!*^Kb>jaBre9BhDrPWz}(stQ{ zB*W!gH0rw|xpL@R-Ctz5OdLQY2zP$~tYgKc+X<^p>vqY#HuE{MK+p?DNQm~I!xQ!C z_ak`KjvKfCc&7h2)A0$-XEVXb45p*!ANVq|C3U!#Nra9%5A>Jzvq%(#)9`8NES%p= zD&^*4O9pR|#yfh*H-)`}+hYAK6YNOxVnKnZ5DbtZDCp0mvNU!Eo2j`vTiILuky{BW z3f*kLz9qPK68%0&6~W;J_%0H7e87zU_sJuMQ=;AxLmIn{yj^rY?p{y2?mf;|S*%aT z1PQ$n^g$U=UplHiOD?WgbOar72A-K_$h^IPjjKHQ9u*m7+2T{TU6#pqs8wA6h#mNPL-0RoLGHK<7#JPWQ*+wX2Q>GLWX>EHR^QZ{JjKr zKdAC?cIHr7VYf^m&0M)XE0>w&rXR*vB(D$iUp*}AF4R1Vaq6V!blR)fC=+R*NS^yN zxXfG2!Do0CfmneZiWoaqvrYiE-<{#7JVRs&KO$IK@V|Cd>^Mhl8g?_dN2eG!ntJzJ z?+Ms(D0D(v#33G}{KI=cl%78a?|(=?zlQKXg`bd^&d4q{kkmQMorI@P!j~5)5^B5@ zO&S|8CRelYbIGyipj)3>BJlh5b;sd#p7d+MVnTDYIHrjf=of3IxRzB1KC{g4#H#tW z=Rru!g0aTqEFEbuN=ot(eX-iu$ki`YsBZDP{d;DrqmhbZg}=?nMR>6dT)re6DJ!wI zJ2FD3V^yl(XIk^qdd@GTs&ls+?MEtTpa1T&?9p(D+~e41!g;cf`=!1g1)7NMc${U~ z_@TiT*?szWe1`P9qO7~gbaf4R^H&l80PPf>qSsmZ??nIUfh!<2ZOM z*;XG5ybnF%R(~2NSk5)BslYrjs%`V#P6NBhX+&c~Y=ND_8hh1658q9(-1E3nFX_vX zU|L2|jJEZg!#D3h?LMs6RE)K)PFXSn*}H@K&Ar1=OE0^M7J@_pP_V%NznybXa+E_v zPE!kTS`XQ8-3>5lgxm&+k$=lS3%cQxT96h~h+>EVQ4Gx-OqHD-99>vV9h|{`&K>_( zD1-!-Z@i*XHybGATy~!PNZ85W(yE;5HNXuH6H|=LJ?VB3aBS}}z!8s!=Tp|k!7PE9gsLatGrh1 zu_tp!x}#MxxStTOWuwTCm0X2#LYFo5JzySlfVlME(a3Lbf{_M^Mj(V{0skJ2#*U8v z4MvE|{@gO+JMHIw%oM_1p@r^6W!Te&ifXD9FnUi{0pz^3!2CE23c)=D16kl?V)$KD z_t`nu_aymsQAJ8~Y7SHZU#DbLzxvKim%T`NlR=AN zhZtmw5Rv(Dm9jcqSX2^*o)S6vktSs*{)XJt$XhP*nK;$UX}SU~`=&~U{FSktYj~og zaY2#QDk(?mL|S8qe)v9}m;2nz?RytvL)N&6+L14TWuuR*THxnw^uG=P`!I8cU z-b~8b&+p9q1V{SDr1B2%L>rv%M=VI9g0|NYoZf+0nHyk?UNUi<&?1}rnzmC6ItD~&&Pk?`}8~Drc$MFp$N&dEq;GyBexzt~#pCHQHgPGNb z#{Zm1`eh0Lpgj6%{C`XixxaI|7Yd?yZI-wznTA8$CVY}ATbCDCdiKl62=)n(g*NA DxVh?K diff --git a/reproduction/text_classification/train_awdlstm.py b/reproduction/text_classification/train_awdlstm.py index e67bd25b..007b2910 100644 --- a/reproduction/text_classification/train_awdlstm.py +++ b/reproduction/text_classification/train_awdlstm.py @@ -33,9 +33,9 @@ class Config(): task_name = "IMDB" datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} - load_model_path="./result_IMDB/best_BiLSTM_SELF_ATTENTION_acc_2019-07-07-04-16-51" save_model_path="./result_IMDB_test/" -opt=Config + +opt=Config() # load data diff --git a/reproduction/text_classification/train_lstm.py b/reproduction/text_classification/train_lstm.py index b89abc14..4ecc61a1 100644 --- a/reproduction/text_classification/train_lstm.py +++ b/reproduction/text_classification/train_lstm.py @@ -30,9 +30,9 @@ class Config(): task_name = "IMDB" datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} - load_model_path="./result_IMDB/best_BiLSTM_SELF_ATTENTION_acc_2019-07-07-04-16-51" save_model_path="./result_IMDB_test/" -opt=Config + +opt=Config() # load data diff --git a/reproduction/text_classification/train_lstm_att.py b/reproduction/text_classification/train_lstm_att.py index b4d37525..a6f0dd03 100644 --- a/reproduction/text_classification/train_lstm_att.py +++ b/reproduction/text_classification/train_lstm_att.py @@ -32,9 +32,9 @@ class Config(): task_name = "IMDB" datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} - load_model_path="./result_IMDB/best_BiLSTM_SELF_ATTENTION_acc_2019-07-07-04-16-51" save_model_path="./result_IMDB_test/" -opt=Config + +opt=Config() # load data From 8156f3c69e8e79eb5050f20fea046092e9d3ad4f Mon Sep 17 00:00:00 2001 From: lyhuang18 <42239874+lyhuang18@users.noreply.github.com> Date: Mon, 8 Jul 2019 05:14:36 +0800 Subject: [PATCH 23/27] =?UTF-8?q?=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- reproduction/text_classification/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/reproduction/text_classification/README.md b/reproduction/text_classification/README.md index b058fbb2..4b8f44bd 100644 --- a/reproduction/text_classification/README.md +++ b/reproduction/text_classification/README.md @@ -3,20 +3,20 @@ char_cnn :论文链接[Character-level Convolutional Networks for Text Classification](https://arxiv.org/pdf/1509.01626v3.pdf) dpcnn:论文链接[Deep Pyramid Convolutional Neural Networks for TextCategorization](https://ai.tencent.com/ailab/media/publications/ACL3-Brady.pdf) HAN:论文链接[Hierarchical Attention Networks for Document Classification](https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf) +LSTM+self_attention:论文链接[A Structured Self-attentive Sentence Embedding]() +AWD-LSTM:论文链接[Regularizing and Optimizing LSTM Language Models]() #待补充 -awd_lstm: -lstm_self_attention(BCN?): -awd-sltm: # 数据集及复现结果汇总 使用fastNLP复现的结果vs论文汇报结果(/前为fastNLP实现，后面为论文报道,-表示论文没有在该数据集上列出结果) -model name | yelp_p | sst-2|IMDB| -:---: | :---: | :---: | :---: -char_cnn | 93.80/95.12 | - |- | -dpcnn | 95.50/97.36 | - |- | -HAN |- | - |-| -BCN| - |- |-| -awd-lstm| - |- |-| +model name | yelp_p | yelp_f | sst-2|IMDB| +:---: | :---: | :---: | :---: |----- |:---: +char_cnn | 93.80/95.12 | - | - |- | +dpcnn | 95.50/97.36 | - | - |- | +HAN |- | - | - |-| +LSTM| 95.74/- |- |- |88.52/-| +AWD-LSTM| 95.96/- |- |- |88.91/-| +LSTM+self_attention| 96.34/- | - | - |89.53/-| From 8f78bf5250e7f183575a7dd6603aa1315668b217 Mon Sep 17 00:00:00 2001 From: lyhuang18 <42239874+lyhuang18@users.noreply.github.com> Date: Mon, 8 Jul 2019 05:17:50 +0800 Subject: [PATCH 24/27] =?UTF-8?q?readme=E6=A0=BC=E5=BC=8F=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- reproduction/text_classification/README.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/reproduction/text_classification/README.md b/reproduction/text_classification/README.md index 4b8f44bd..08c893b7 100644 --- a/reproduction/text_classification/README.md +++ b/reproduction/text_classification/README.md @@ -1,22 +1,28 @@ # text_classification任务模型复现 这里使用fastNLP复现以下模型： + char_cnn :论文链接[Character-level Convolutional Networks for Text Classification](https://arxiv.org/pdf/1509.01626v3.pdf) + dpcnn:论文链接[Deep Pyramid Convolutional Neural Networks for TextCategorization](https://ai.tencent.com/ailab/media/publications/ACL3-Brady.pdf) + HAN:论文链接[Hierarchical Attention Networks for Document Classification](https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf) + LSTM+self_attention:论文链接[A Structured Self-attentive Sentence Embedding]() + AWD-LSTM:论文链接[Regularizing and Optimizing LSTM Language Models]() + #待补充 # 数据集及复现结果汇总 使用fastNLP复现的结果vs论文汇报结果(/前为fastNLP实现，后面为论文报道,-表示论文没有在该数据集上列出结果) -model name | yelp_p | yelp_f | sst-2|IMDB| -:---: | :---: | :---: | :---: |----- |:---: -char_cnn | 93.80/95.12 | - | - |- | -dpcnn | 95.50/97.36 | - | - |- | -HAN |- | - | - |-| -LSTM| 95.74/- |- |- |88.52/-| -AWD-LSTM| 95.96/- |- |- |88.91/-| -LSTM+self_attention| 96.34/- | - | - |89.53/-| +model name | yelp_p | yelp_f | sst-2|IMDB +:---: | :---: | :---: | :---: |----- +char_cnn | 93.80/95.12 | - | - |- +dpcnn | 95.50/97.36 | - | - |- +HAN |- | - | - |- +LSTM| 95.74/- |- |- |88.52/- +AWD-LSTM| 95.96/- |- |- |88.91/- +LSTM+self_attention| 96.34/- | - | - |89.53/- From 2c9a6e0ba4a46a9be5302855e8007229b96c8c96 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Mon, 8 Jul 2019 09:21:12 +0800 Subject: [PATCH 25/27] =?UTF-8?q?1.=20=E4=BF=AE=E6=94=B9ELMO=E5=8A=A0?= =?UTF-8?q?=E8=BD=BDallennlp=E7=9A=84=E6=9D=83=E9=87=8D;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/vocabulary.py | 5 + fastNLP/modules/encoder/embedding.py | 119 +++++++++++--- .../ner/data/Conll2003Loader.py | 93 ----------- .../ner/data/OntoNoteLoader.py | 152 ------------------ .../seqence_labelling/ner/data/utils.py | 49 ------ reproduction/utils.py | 6 +- 6 files changed, 102 insertions(+), 322 deletions(-) delete mode 100644 reproduction/seqence_labelling/ner/data/Conll2003Loader.py delete mode 100644 reproduction/seqence_labelling/ner/data/OntoNoteLoader.py delete mode 100644 reproduction/seqence_labelling/ner/data/utils.py diff --git a/fastNLP/core/vocabulary.py b/fastNLP/core/vocabulary.py index 66aabd3d..d4fa6cc9 100644 --- a/fastNLP/core/vocabulary.py +++ b/fastNLP/core/vocabulary.py @@ -117,6 +117,8 @@ class Vocabulary(object): :param str word: 新词 """ + if word in self._no_create_word: + self._no_create_word.pop(word) self.add(word) @_check_build_status @@ -126,6 +128,9 @@ class Vocabulary(object): :param list[str] word_lst: 词的序列 """ + for word in word_lst: + if word in self._no_create_word: + self._no_create_word.pop(word) self.update(word_lst) def build_vocab(self): diff --git a/fastNLP/modules/encoder/embedding.py b/fastNLP/modules/encoder/embedding.py index 005cfe75..0d6f30e3 100644 --- a/fastNLP/modules/encoder/embedding.py +++ b/fastNLP/modules/encoder/embedding.py @@ -179,16 +179,16 @@ class StaticEmbedding(TokenEmbedding): :param model_dir_or_name: 可以有两种方式调用预训练好的static embedding：第一种是传入embedding的文件名，第二种是传入embedding 的名称。目前支持的embedding包括{`en` 或者 `en-glove-840b-300` : glove.840B.300d, `en-glove-6b-50` : glove.6B.50d, `en-word2vec-300` : GoogleNews-vectors-negative300}。第二种情况将自动查看缓存中是否存在该模型，没有的话将自动下载。 - :param requires_grad: 是否需要gradient. 默认为True - :param init_method: 如何初始化没有找到的值。可以使用torch.nn.init.*中各种方法。调用该方法时传入一个tensor对象。 - :param normailize: 是否对vector进行normalize，使得每个vector的norm为1。 + :param bool requires_grad: 是否需要gradient. 默认为True + :param callable init_method: 如何初始化没有找到的值。可以使用torch.nn.init.*中各种方法。调用该方法时传入一个tensor对象。 + :param bool normailize: 是否对vector进行normalize，使得每个vector的norm为1。 + :param bool lower: 是否将vocab中的词语小写后再和预训练的词表进行匹配。如果你的词表中包含大写的词语，或者就是需要单独 + 为大写的词语开辟一个vector表示，则将lower设置为False。 """ def __init__(self, vocab: Vocabulary, model_dir_or_name: str='en', requires_grad: bool=True, init_method=None, - normalize=False): + normalize=False, lower=False): super(StaticEmbedding, self).__init__(vocab) - # 优先定义需要下载的static embedding有哪些。这里估计需要自己搞一个server， - # 得到cache_path if model_dir_or_name.lower() in PRETRAIN_STATIC_FILES: PRETRAIN_URL = _get_base_url('static') @@ -202,8 +202,40 @@ class StaticEmbedding(TokenEmbedding): raise ValueError(f"Cannot recognize {model_dir_or_name}.") # 读取embedding - embedding = self._load_with_vocab(model_path, vocab=vocab, init_method=init_method, - normalize=normalize) + if lower: + lowered_vocab = Vocabulary(padding=vocab.padding, unknown=vocab.unknown) + for word, index in vocab: + if not vocab._is_word_no_create_entry(word): + lowered_vocab.add_word(word.lower()) # 先加入需要创建entry的 + for word in vocab._no_create_word.keys(): # 不需要创建entry的 + if word in vocab: + lowered_word = word.lower() + if lowered_word not in lowered_vocab.word_count: + lowered_vocab.add_word(lowered_word) + lowered_vocab._no_create_word[lowered_word] += 1 + print(f"All word in vocab have been lowered. There are {len(vocab)} words, {len(lowered_vocab)} unique lowered " + f"words.") + embedding = self._load_with_vocab(model_path, vocab=lowered_vocab, init_method=init_method, + normalize=normalize) + # 需要适配一下 + if not hasattr(self, 'words_to_words'): + self.words_to_words = torch.arange(len(lowered_vocab, )).long() + if lowered_vocab.unknown: + unknown_idx = lowered_vocab.unknown_idx + else: + unknown_idx = embedding.size(0) - 1 # 否则是最后一个为unknow + words_to_words = nn.Parameter(torch.full((len(vocab),), fill_value=unknown_idx).long(), + requires_grad=False) + for word, index in vocab: + if word not in lowered_vocab: + word = word.lower() + if lowered_vocab._is_word_no_create_entry(word): # 如果不需要创建entry,已经默认unknown了 + continue + words_to_words[index] = self.words_to_words[lowered_vocab.to_index(word)] + self.words_to_words = words_to_words + else: + embedding = self._load_with_vocab(model_path, vocab=vocab, init_method=init_method, + normalize=normalize) self.embedding = nn.Embedding(num_embeddings=embedding.shape[0], embedding_dim=embedding.shape[1], padding_idx=vocab.padding_idx, max_norm=None, norm_type=2, scale_grad_by_freq=False, @@ -301,7 +333,7 @@ class StaticEmbedding(TokenEmbedding): if vocab._no_create_word_length>0: if vocab.unknown is None: # 创建一个专门的unknown unknown_idx = len(matrix) - vectors = torch.cat([vectors, torch.zeros(1, dim)], dim=0).contiguous() + vectors = torch.cat((vectors, torch.zeros(1, dim)), dim=0).contiguous() else: unknown_idx = vocab.unknown_idx words_to_words = nn.Parameter(torch.full((len(vocab),), fill_value=unknown_idx).long(), @@ -438,19 +470,15 @@ class ElmoEmbedding(ContextualEmbedding): :param model_dir_or_name: 可以有两种方式调用预训练好的ELMo embedding：第一种是传入ELMo权重的文件名，第二种是传入ELMo版本的名称， 目前支持的ELMo包括{`en` : 英文版本的ELMo, `cn` : 中文版本的ELMo,}。第二种情况将自动查看缓存中是否存在该模型，没有的话将自动下载 :param layers: str, 指定返回的层数, 以,隔开不同的层。如果要返回第二层的结果'2', 返回后两层的结果'1,2'。不同的层的结果 - 按照这个顺序concat起来。默认为'2'。 - :param requires_grad: bool, 该层是否需要gradient. 默认为False + 按照这个顺序concat起来。默认为'2'。'mix'会使用可学习的权重结合不同层的表示(权重是否可训练与requires_grad保持一致， + 初始化权重对三层结果进行mean-pooling, 可以通过ElmoEmbedding.set_mix_weights_requires_grad()方法只将mix weights设置为可学习。) + :param requires_grad: bool, 该层是否需要gradient, 默认为False. :param cache_word_reprs: 可以选择对word的表示进行cache; 设置为True的话，将在初始化的时候为每个word生成对应的embedding， 并删除character encoder，之后将直接使用cache的embedding。默认为False。 """ def __init__(self, vocab: Vocabulary, model_dir_or_name: str='en', layers: str='2', requires_grad: bool=False, cache_word_reprs: bool=False): super(ElmoEmbedding, self).__init__(vocab) - layers = list(map(int, layers.split(','))) - assert len(layers) > 0, "Must choose one output" - for layer in layers: - assert 0 <= layer <= 2, "Layer index should be in range [0, 2]." - self.layers = layers # 根据model_dir_or_name检查是否存在并下载 if model_dir_or_name.lower() in PRETRAINED_ELMO_MODEL_DIR: @@ -464,8 +492,49 @@ class ElmoEmbedding(ContextualEmbedding): else: raise ValueError(f"Cannot recognize {model_dir_or_name}.") self.model = _ElmoModel(model_dir, vocab, cache_word_reprs=cache_word_reprs) + + if layers=='mix': + self.layer_weights = nn.Parameter(torch.zeros(self.model.config['encoder']['n_layers']+1), + requires_grad=requires_grad) + self.gamma = nn.Parameter(torch.ones(1), requires_grad=requires_grad) + self._get_outputs = self._get_mixed_outputs + self._embed_size = self.model.config['encoder']['projection_dim'] * 2 + else: + layers = list(map(int, layers.split(','))) + assert len(layers) > 0, "Must choose one output" + for layer in layers: + assert 0 <= layer <= 2, "Layer index should be in range [0, 2]." + self.layers = layers + self._get_outputs = self._get_layer_outputs + self._embed_size = len(self.layers) * self.model.config['encoder']['projection_dim'] * 2 + self.requires_grad = requires_grad - self._embed_size = len(self.layers) * self.model.config['encoder']['projection_dim'] * 2 + + def _get_mixed_outputs(self, outputs): + # outputs: num_layers x batch_size x max_len x hidden_size + # return: batch_size x max_len x hidden_size + weights = F.softmax(self.layer_weights+1/len(outputs), dim=0).to(outputs) + outputs = torch.einsum('l,lbij->bij', weights, outputs) + return self.gamma.to(outputs)*outputs + + def set_mix_weights_requires_grad(self, flag=True): + """ + 当初始化ElmoEmbedding时layers被设置为mix时，可以通过调用该方法设置mix weights是否可训练。如果layers不是mix，调用 + 该方法没有用。 + :param bool flag: 混合不同层表示的结果是否可以训练。 + :return: + """ + if hasattr(self, 'layer_weights'): + self.layer_weights.requires_grad = flag + self.gamma.requires_grad = flag + + def _get_layer_outputs(self, outputs): + if len(self.layers) == 1: + outputs = outputs[self.layers[0]] + else: + outputs = torch.cat(tuple([*outputs[self.layers]]), dim=-1) + + return outputs def forward(self, words: torch.LongTensor): """ @@ -480,15 +549,12 @@ class ElmoEmbedding(ContextualEmbedding): if outputs is not None: return outputs outputs = self.model(words) - if len(self.layers) == 1: - outputs = outputs[self.layers[0]] - else: - outputs = torch.cat([*outputs[self.layers]], dim=-1) - - return outputs + return self._get_outputs(outputs) def _delete_model_weights(self): - del self.layers, self.model + for name in ['layers', 'model', 'layer_weights', 'gamma']: + if hasattr(self, name): + delattr(self, name) @property def requires_grad(self): @@ -892,10 +958,11 @@ class StackEmbedding(TokenEmbedding): def __init__(self, embeds: List[TokenEmbedding]): vocabs = [] for embed in embeds: - vocabs.append(embed.get_word_vocab()) + if hasattr(embed, 'get_word_vocab'): + vocabs.append(embed.get_word_vocab()) _vocab = vocabs[0] for vocab in vocabs[1:]: - assert vocab == _vocab, "All embeddings should use the same word vocabulary." + assert vocab == _vocab, "All embeddings in StackEmbedding should use the same word vocabulary." super(StackEmbedding, self).__init__(_vocab) assert isinstance(embeds, list) diff --git a/reproduction/seqence_labelling/ner/data/Conll2003Loader.py b/reproduction/seqence_labelling/ner/data/Conll2003Loader.py deleted file mode 100644 index 577987c6..00000000 --- a/reproduction/seqence_labelling/ner/data/Conll2003Loader.py +++ /dev/null @@ -1,93 +0,0 @@ - -from fastNLP.core.vocabulary import VocabularyOption -from fastNLP.io.base_loader import DataSetLoader, DataInfo -from typing import Union, Dict -from fastNLP import Vocabulary -from fastNLP import Const -from reproduction.utils import check_dataloader_paths - -from fastNLP.io.dataset_loader import ConllLoader -from reproduction.seqence_labelling.ner.data.utils import iob2bioes, iob2 - - -class Conll2003DataLoader(DataSetLoader): - def __init__(self, task:str='ner', encoding_type:str='bioes'): - """ - 加载Conll2003格式的英语语料，该数据集的信息可以在https://www.clips.uantwerpen.be/conll2003/ner/找到。当task为pos - 时，返回的DataSet中target取值于第2列; 当task为chunk时，返回的DataSet中target取值于第3列;当task为ner时，返回 - 的DataSet中target取值于第4列。所有"-DOCSTART- -X- O O"将被忽略，这会导致数据的数量少于很多文献报道的值，但 - 鉴于"-DOCSTART- -X- O O"只是用于文档分割的符号，并不应该作为预测对象，所以我们忽略了数据中的-DOCTSTART-开头的行 - ner与chunk任务读取后的数据的target将为encoding_type类型。pos任务读取后就是pos列的数据。 - - :param task: 指定需要标注任务。可选ner, pos, chunk - """ - assert task in ('ner', 'pos', 'chunk') - index = {'ner':3, 'pos':1, 'chunk':2}[task] - self._loader = ConllLoader(headers=['raw_words', 'target'], indexes=[0, index]) - self._tag_converters = None - if task in ('ner', 'chunk'): - self._tag_converters = [iob2] - if encoding_type == 'bioes': - self._tag_converters.append(iob2bioes) - - def load(self, path: str): - dataset = self._loader.load(path) - def convert_tag_schema(tags): - for converter in self._tag_converters: - tags = converter(tags) - return tags - if self._tag_converters: - dataset.apply_field(convert_tag_schema, field_name=Const.TARGET, new_field_name=Const.TARGET) - return dataset - - def process(self, paths: Union[str, Dict[str, str]], word_vocab_opt:VocabularyOption=None, lower:bool=True): - """ - 读取并处理数据。数据中的'-DOCSTART-'开头的行会被忽略 - - :param paths: - :param word_vocab_opt: vocabulary的初始化值 - :param lower: 是否将所有字母转为小写 - :return: - """ - # 读取数据 - paths = check_dataloader_paths(paths) - data = DataInfo() - input_fields = [Const.TARGET, Const.INPUT, Const.INPUT_LEN] - target_fields = [Const.TARGET, Const.INPUT_LEN] - for name, path in paths.items(): - dataset = self.load(path) - dataset.apply_field(lambda words: words, field_name='raw_words', new_field_name=Const.INPUT) - if lower: - dataset.words.lower() - data.datasets[name] = dataset - - # 对construct vocab - word_vocab = Vocabulary(min_freq=2) if word_vocab_opt is None else Vocabulary(**word_vocab_opt) - word_vocab.from_dataset(data.datasets['train'], field_name=Const.INPUT, - no_create_entry_dataset=[dataset for name, dataset in data.datasets.items() if name!='train']) - word_vocab.index_dataset(*data.datasets.values(), field_name=Const.INPUT, new_field_name=Const.INPUT) - data.vocabs[Const.INPUT] = word_vocab - - # cap words - cap_word_vocab = Vocabulary() - cap_word_vocab.from_dataset(data.datasets['train'], field_name='raw_words', - no_create_entry_dataset=[dataset for name, dataset in data.datasets.items() if name!='train']) - cap_word_vocab.index_dataset(*data.datasets.values(), field_name='raw_words', new_field_name='cap_words') - input_fields.append('cap_words') - data.vocabs['cap_words'] = cap_word_vocab - - # 对target建vocab - target_vocab = Vocabulary(unknown=None, padding=None) - target_vocab.from_dataset(*data.datasets.values(), field_name=Const.TARGET) - target_vocab.index_dataset(*data.datasets.values(), field_name=Const.TARGET) - data.vocabs[Const.TARGET] = target_vocab - - for name, dataset in data.datasets.items(): - dataset.add_seq_len(Const.INPUT, new_field_name=Const.INPUT_LEN) - dataset.set_input(*input_fields) - dataset.set_target(*target_fields) - - return data - -if __name__ == '__main__': - pass \ No newline at end of file diff --git a/reproduction/seqence_labelling/ner/data/OntoNoteLoader.py b/reproduction/seqence_labelling/ner/data/OntoNoteLoader.py deleted file mode 100644 index 8a2c567d..00000000 --- a/reproduction/seqence_labelling/ner/data/OntoNoteLoader.py +++ /dev/null @@ -1,152 +0,0 @@ -from fastNLP.core.vocabulary import VocabularyOption -from fastNLP.io.base_loader import DataSetLoader, DataInfo -from typing import Union, Dict -from fastNLP import DataSet -from fastNLP import Vocabulary -from fastNLP import Const -from reproduction.utils import check_dataloader_paths - -from fastNLP.io.dataset_loader import ConllLoader -from reproduction.seqence_labelling.ner.data.utils import iob2bioes, iob2 - -class OntoNoteNERDataLoader(DataSetLoader): - """ - 用于读取处理为Conll格式后的OntoNote数据。将OntoNote数据处理为conll格式的过程可以参考https://github.com/yhcc/OntoNotes-5.0-NER。 - - """ - def __init__(self, encoding_type:str='bioes'): - assert encoding_type in ('bioes', 'bio') - self.encoding_type = encoding_type - if encoding_type=='bioes': - self.encoding_method = iob2bioes - else: - self.encoding_method = iob2 - - def load(self, path:str)->DataSet: - """ - 给定一个文件路径，读取数据。返回的DataSet包含以下的field - raw_words: List[str] - target: List[str] - - :param path: - :return: - """ - dataset = ConllLoader(headers=['raw_words', 'target'], indexes=[3, 10]).load(path) - def convert_to_bio(tags): - bio_tags = [] - flag = None - for tag in tags: - label = tag.strip("()*") - if '(' in tag: - bio_label = 'B-' + label - flag = label - elif flag: - bio_label = 'I-' + flag - else: - bio_label = 'O' - if ')' in tag: - flag = None - bio_tags.append(bio_label) - return self.encoding_method(bio_tags) - - def convert_word(words): - converted_words = [] - for word in words: - word = word.replace('/.', '.') # 有些结尾的.是/.形式的 - if not word.startswith('-'): - converted_words.append(word) - continue - # 以下是由于这些符号被转义了，再转回来 - tfrs = {'-LRB-':'(', - '-RRB-': ')', - '-LSB-': '[', - '-RSB-': ']', - '-LCB-': '{', - '-RCB-': '}' - } - if word in tfrs: - converted_words.append(tfrs[word]) - else: - converted_words.append(word) - return converted_words - - dataset.apply_field(convert_word, field_name='raw_words', new_field_name='raw_words') - dataset.apply_field(convert_to_bio, field_name='target', new_field_name='target') - - return dataset - - def process(self, paths: Union[str, Dict[str, str]], word_vocab_opt:VocabularyOption=None, - lower:bool=True)->DataInfo: - """ - 读取并处理数据。返回的DataInfo包含以下的内容 - vocabs: - word: Vocabulary - target: Vocabulary - datasets: - train: DataSet - words: List[int], 被设置为input - target: int. label，被同时设置为input和target - seq_len: int. 句子的长度，被同时设置为input和target - raw_words: List[str] - xxx(根据传入的paths可能有所变化) - - :param paths: - :param word_vocab_opt: vocabulary的初始化值 - :param lower: 是否使用小写 - :return: - """ - paths = check_dataloader_paths(paths) - data = DataInfo() - input_fields = [Const.TARGET, Const.INPUT, Const.INPUT_LEN] - target_fields = [Const.TARGET, Const.INPUT_LEN] - for name, path in paths.items(): - dataset = self.load(path) - dataset.apply_field(lambda words: words, field_name='raw_words', new_field_name=Const.INPUT) - if lower: - dataset.words.lower() - data.datasets[name] = dataset - - # 对construct vocab - word_vocab = Vocabulary(min_freq=2) if word_vocab_opt is None else Vocabulary(**word_vocab_opt) - word_vocab.from_dataset(data.datasets['train'], field_name=Const.INPUT, - no_create_entry_dataset=[dataset for name, dataset in data.datasets.items() if name!='train']) - word_vocab.index_dataset(*data.datasets.values(), field_name=Const.INPUT, new_field_name=Const.INPUT) - data.vocabs[Const.INPUT] = word_vocab - - # cap words - cap_word_vocab = Vocabulary() - cap_word_vocab.from_dataset(*data.datasets.values(), field_name='raw_words') - cap_word_vocab.index_dataset(*data.datasets.values(), field_name='raw_words', new_field_name='cap_words') - input_fields.append('cap_words') - data.vocabs['cap_words'] = cap_word_vocab - - # 对target建vocab - target_vocab = Vocabulary(unknown=None, padding=None) - target_vocab.from_dataset(*data.datasets.values(), field_name=Const.TARGET) - target_vocab.index_dataset(*data.datasets.values(), field_name=Const.TARGET) - data.vocabs[Const.TARGET] = target_vocab - - for name, dataset in data.datasets.items(): - dataset.add_seq_len(Const.INPUT, new_field_name=Const.INPUT_LEN) - dataset.set_input(*input_fields) - dataset.set_target(*target_fields) - - return data - - -if __name__ == '__main__': - loader = OntoNoteNERDataLoader() - dataset = loader.load('/hdd/fudanNLP/fastNLP/others/data/v4/english/test.txt') - print(dataset.target.value_count()) - print(dataset[:4]) - - -""" -train 115812 2200752 -development 15680 304684 -test 12217 230111 - -train 92403 1901772 -valid 13606 279180 -test 10258 204135 -""" \ No newline at end of file diff --git a/reproduction/seqence_labelling/ner/data/utils.py b/reproduction/seqence_labelling/ner/data/utils.py deleted file mode 100644 index 8f7af792..00000000 --- a/reproduction/seqence_labelling/ner/data/utils.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import List - -def iob2(tags:List[str])->List[str]: - """ - 检查数据是否是合法的IOB数据，如果是IOB1会被自动转换为IOB2。 - - :param tags: 需要转换的tags - """ - for i, tag in enumerate(tags): - if tag == "O": - continue - split = tag.split("-") - if len(split) != 2 or split[0] not in ["I", "B"]: - raise TypeError("The encoding schema is not a valid IOB type.") - if split[0] == "B": - continue - elif i == 0 or tags[i - 1] == "O": # conversion IOB1 to IOB2 - tags[i] = "B" + tag[1:] - elif tags[i - 1][1:] == tag[1:]: - continue - else: # conversion IOB1 to IOB2 - tags[i] = "B" + tag[1:] - return tags - -def iob2bioes(tags:List[str])->List[str]: - """ - 将iob的tag转换为bmeso编码 - :param tags: - :return: - """ - new_tags = [] - for i, tag in enumerate(tags): - if tag == 'O': - new_tags.append(tag) - else: - split = tag.split('-')[0] - if split == 'B': - if i+1!=len(tags) and tags[i+1].split('-')[0] == 'I': - new_tags.append(tag) - else: - new_tags.append(tag.replace('B-', 'S-')) - elif split == 'I': - if i + 1Dict[str, str]: path_pair = ('train', filename) if 'dev' in filename: if path_pair: - raise Exception("File:{} in {} contains bot `{}` and `dev`.".format(filename, paths, path_pair[0])) + raise Exception("File:{} in {} contains both `{}` and `dev`.".format(filename, paths, path_pair[0])) path_pair = ('dev', filename) if 'test' in filename: if path_pair: - raise Exception("File:{} in {} contains bot `{}` and `test`.".format(filename, paths, path_pair[0])) + raise Exception("File:{} in {} contains both `{}` and `test`.".format(filename, paths, path_pair[0])) path_pair = ('test', filename) if path_pair: + if path_pair[0] in files: + raise RuntimeError(f"Multiple file under {paths} have '{path_pair[0]}' in their filename.") files[path_pair[0]] = os.path.join(paths, path_pair[1]) return files else: From e867641023bc37bcffdd5af14128ed786bcc5e2b Mon Sep 17 00:00:00 2001 From: yh_cc Date: Mon, 8 Jul 2019 09:22:34 +0800 Subject: [PATCH 26/27] =?UTF-8?q?=E4=BF=AE=E6=94=B9=5Felmo.py=E7=9A=84?= =?UTF-8?q?=E6=9D=83=E9=87=8D=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/modules/encoder/_elmo.py | 458 +++++++++++++++++++------------ 1 file changed, 288 insertions(+), 170 deletions(-) diff --git a/fastNLP/modules/encoder/_elmo.py b/fastNLP/modules/encoder/_elmo.py index 4ebee819..a49634bb 100644 --- a/fastNLP/modules/encoder/_elmo.py +++ b/fastNLP/modules/encoder/_elmo.py @@ -1,12 +1,13 @@ - """ -这个页面的代码大量参考了https://github.com/HIT-SCIR/ELMoForManyLangs/tree/master/elmoformanylangs +这个页面的代码大量参考了 allenNLP """ - from typing import Optional, Tuple, List, Callable import os + +import h5py +import numpy import torch import torch.nn as nn import torch.nn.functional as F @@ -16,7 +17,6 @@ import json from ..utils import get_dropout_mask import codecs -from torch import autograd class LstmCellWithProjection(torch.nn.Module): """ @@ -58,6 +58,7 @@ class LstmCellWithProjection(torch.nn.Module): respectively. The first dimension is 1 in order to match the Pytorch API for returning stacked LSTM states. """ + def __init__(self, input_size: int, hidden_size: int, @@ -129,13 +130,13 @@ class LstmCellWithProjection(torch.nn.Module): # We have to use this '.data.new().fill_' pattern to create tensors with the correct # type - forward has no knowledge of whether these are torch.Tensors or torch.cuda.Tensors. output_accumulator = inputs.data.new(batch_size, - total_timesteps, - self.hidden_size).fill_(0) + total_timesteps, + self.hidden_size).fill_(0) if initial_state is None: full_batch_previous_memory = inputs.data.new(batch_size, - self.cell_size).fill_(0) + self.cell_size).fill_(0) full_batch_previous_state = inputs.data.new(batch_size, - self.hidden_size).fill_(0) + self.hidden_size).fill_(0) else: full_batch_previous_state = initial_state[0].squeeze(0) full_batch_previous_memory = initial_state[1].squeeze(0) @@ -169,7 +170,7 @@ class LstmCellWithProjection(torch.nn.Module): # Second conditional: Does the next shortest sequence beyond the current batch # index require computation use this timestep? while current_length_index < (len(batch_lengths) - 1) and \ - batch_lengths[current_length_index + 1] > index: + batch_lengths[current_length_index + 1] > index: current_length_index += 1 # Actually get the slices of the batch which we @@ -256,7 +257,7 @@ class LstmbiLm(nn.Module): inputs = inputs[sort_idx] inputs = nn.utils.rnn.pack_padded_sequence(inputs, sort_lens, batch_first=self.batch_first) output, hx = self.encoder(inputs, None) # -> [N,L,C] - output, _ = nn.util.rnn.pad_packed_sequence(output, batch_first=self.batch_first) + output, _ = nn.utils.rnn.pad_packed_sequence(output, batch_first=self.batch_first) _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) output = output[unsort_idx] forward, backward = output.split(self.config['encoder']['dim'], 2) @@ -316,13 +317,13 @@ class ElmobiLm(torch.nn.Module): :param seq_len: batch_size :return: torch.FloatTensor. num_layers x batch_size x max_len x hidden_size """ + max_len = inputs.size(1) sort_lens, sort_idx = torch.sort(seq_len, dim=0, descending=True) inputs = inputs[sort_idx] inputs = nn.utils.rnn.pack_padded_sequence(inputs, sort_lens, batch_first=True) output, _ = self._lstm_forward(inputs, None) _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) output = output[:, unsort_idx] - return output def _lstm_forward(self, @@ -399,7 +400,7 @@ class ElmobiLm(torch.nn.Module): torch.cat([forward_state[1], backward_state[1]], -1))) stacked_sequence_outputs: torch.FloatTensor = torch.stack(sequence_outputs) - # Stack the hidden state and memory for each layer into 2 tensors of shape + # Stack the hidden state and memory for each layer in。to 2 tensors of shape # (num_layers, batch_size, hidden_size) and (num_layers, batch_size, cell_size) # respectively. final_hidden_states, final_memory_states = zip(*final_states) @@ -408,6 +409,66 @@ class ElmobiLm(torch.nn.Module): torch.cat(final_memory_states, 0)) return stacked_sequence_outputs, final_state_tuple + def load_weights(self, weight_file: str) -> None: + """ + Load the pre-trained weights from the file. + """ + requires_grad = False + + with h5py.File(weight_file, 'r') as fin: + for i_layer, lstms in enumerate( + zip(self.forward_layers, self.backward_layers) + ): + for j_direction, lstm in enumerate(lstms): + # lstm is an instance of LSTMCellWithProjection + cell_size = lstm.cell_size + + dataset = fin['RNN_%s' % j_direction]['RNN']['MultiRNNCell']['Cell%s' % i_layer + ]['LSTMCell'] + + # tensorflow packs together both W and U matrices into one matrix, + # but pytorch maintains individual matrices. In addition, tensorflow + # packs the gates as input, memory, forget, output but pytorch + # uses input, forget, memory, output. So we need to modify the weights. + tf_weights = numpy.transpose(dataset['W_0'][...]) + torch_weights = tf_weights.copy() + + # split the W from U matrices + input_size = lstm.input_size + input_weights = torch_weights[:, :input_size] + recurrent_weights = torch_weights[:, input_size:] + tf_input_weights = tf_weights[:, :input_size] + tf_recurrent_weights = tf_weights[:, input_size:] + + # handle the different gate order convention + for torch_w, tf_w in [[input_weights, tf_input_weights], + [recurrent_weights, tf_recurrent_weights]]: + torch_w[(1 * cell_size):(2 * cell_size), :] = tf_w[(2 * cell_size):(3 * cell_size), :] + torch_w[(2 * cell_size):(3 * cell_size), :] = tf_w[(1 * cell_size):(2 * cell_size), :] + + lstm.input_linearity.weight.data.copy_(torch.FloatTensor(input_weights)) + lstm.state_linearity.weight.data.copy_(torch.FloatTensor(recurrent_weights)) + lstm.input_linearity.weight.requires_grad = requires_grad + lstm.state_linearity.weight.requires_grad = requires_grad + + # the bias weights + tf_bias = dataset['B'][...] + # tensorflow adds 1.0 to forget gate bias instead of modifying the + # parameters... + tf_bias[(2 * cell_size):(3 * cell_size)] += 1 + torch_bias = tf_bias.copy() + torch_bias[(1 * cell_size):(2 * cell_size) + ] = tf_bias[(2 * cell_size):(3 * cell_size)] + torch_bias[(2 * cell_size):(3 * cell_size) + ] = tf_bias[(1 * cell_size):(2 * cell_size)] + lstm.state_linearity.bias.data.copy_(torch.FloatTensor(torch_bias)) + lstm.state_linearity.bias.requires_grad = requires_grad + + # the projection weights + proj_weights = numpy.transpose(dataset['W_P_0'][...]) + lstm.state_projection.weight.data.copy_(torch.FloatTensor(proj_weights)) + lstm.state_projection.weight.requires_grad = requires_grad + class LstmTokenEmbedder(nn.Module): def __init__(self, config, word_emb_layer, char_emb_layer): @@ -441,7 +502,7 @@ class LstmTokenEmbedder(nn.Module): chars_emb = self.char_emb_layer(chars) # TODO 这里应该要考虑seq_len的问题 _, (chars_outputs, __) = self.char_lstm(chars_emb) - chars_outputs = chars_outputs.contiguous().view(-1, self.config['token_embedder']['char_dim'] * 2) + chars_outputs = chars_outputs.contiguous().view(-1, self.config['token_embedder']['embedding']['dim'] * 2) embs.append(chars_outputs) token_embedding = torch.cat(embs, dim=2) @@ -450,79 +511,143 @@ class LstmTokenEmbedder(nn.Module): class ConvTokenEmbedder(nn.Module): - def __init__(self, config, word_emb_layer, char_emb_layer): + def __init__(self, config, weight_file, word_emb_layer, char_emb_layer, char_vocab): super(ConvTokenEmbedder, self).__init__() - self.config = config + self.weight_file = weight_file self.word_emb_layer = word_emb_layer self.char_emb_layer = char_emb_layer self.output_dim = config['encoder']['projection_dim'] - self.emb_dim = 0 - if word_emb_layer is not None: - self.emb_dim += word_emb_layer.weight.size(1) - - if char_emb_layer is not None: - self.convolutions = [] - cnn_config = config['token_embedder'] - filters = cnn_config['filters'] - char_embed_dim = cnn_config['char_dim'] - - for i, (width, num) in enumerate(filters): - conv = torch.nn.Conv1d( - in_channels=char_embed_dim, - out_channels=num, - kernel_size=width, - bias=True - ) - self.convolutions.append(conv) - - self.convolutions = nn.ModuleList(self.convolutions) - - self.n_filters = sum(f[1] for f in filters) - self.n_highway = cnn_config['n_highway'] - - self.highways = Highway(self.n_filters, self.n_highway, activation=torch.nn.functional.relu) - self.emb_dim += self.n_filters - - self.projection = nn.Linear(self.emb_dim, self.output_dim, bias=True) + self._options = config + self.requires_grad = False + self._load_weights() + self._char_embedding_weights = char_emb_layer.weight.data + + def _load_weights(self): + self._load_cnn_weights() + self._load_highway() + self._load_projection() + + def _load_cnn_weights(self): + cnn_options = self._options['token_embedder'] + filters = cnn_options['filters'] + char_embed_dim = cnn_options['embedding']['dim'] + + convolutions = [] + for i, (width, num) in enumerate(filters): + conv = torch.nn.Conv1d( + in_channels=char_embed_dim, + out_channels=num, + kernel_size=width, + bias=True + ) + # load the weights + with h5py.File(self.weight_file, 'r') as fin: + weight = fin['CNN']['W_cnn_{}'.format(i)][...] + bias = fin['CNN']['b_cnn_{}'.format(i)][...] + + w_reshaped = numpy.transpose(weight.squeeze(axis=0), axes=(2, 1, 0)) + if w_reshaped.shape != tuple(conv.weight.data.shape): + raise ValueError("Invalid weight file") + conv.weight.data.copy_(torch.FloatTensor(w_reshaped)) + conv.bias.data.copy_(torch.FloatTensor(bias)) + + conv.weight.requires_grad = self.requires_grad + conv.bias.requires_grad = self.requires_grad + + convolutions.append(conv) + self.add_module('char_conv_{}'.format(i), conv) + + self._convolutions = convolutions + + def _load_highway(self): + # the highway layers have same dimensionality as the number of cnn filters + cnn_options = self._options['token_embedder'] + filters = cnn_options['filters'] + n_filters = sum(f[1] for f in filters) + n_highway = cnn_options['n_highway'] + + # create the layers, and load the weights + self._highways = Highway(n_filters, n_highway, activation=torch.nn.functional.relu) + for k in range(n_highway): + # The AllenNLP highway is one matrix multplication with concatenation of + # transform and carry weights. + with h5py.File(self.weight_file, 'r') as fin: + # The weights are transposed due to multiplication order assumptions in tf + # vs pytorch (tf.matmul(X, W) vs pytorch.matmul(W, X)) + w_transform = numpy.transpose(fin['CNN_high_{}'.format(k)]['W_transform'][...]) + # -1.0 since AllenNLP is g * x + (1 - g) * f(x) but tf is (1 - g) * x + g * f(x) + w_carry = -1.0 * numpy.transpose(fin['CNN_high_{}'.format(k)]['W_carry'][...]) + weight = numpy.concatenate([w_transform, w_carry], axis=0) + self._highways._layers[k].weight.data.copy_(torch.FloatTensor(weight)) + self._highways._layers[k].weight.requires_grad = self.requires_grad + + b_transform = fin['CNN_high_{}'.format(k)]['b_transform'][...] + b_carry = -1.0 * fin['CNN_high_{}'.format(k)]['b_carry'][...] + bias = numpy.concatenate([b_transform, b_carry], axis=0) + self._highways._layers[k].bias.data.copy_(torch.FloatTensor(bias)) + self._highways._layers[k].bias.requires_grad = self.requires_grad + + def _load_projection(self): + cnn_options = self._options['token_embedder'] + filters = cnn_options['filters'] + n_filters = sum(f[1] for f in filters) + + self._projection = torch.nn.Linear(n_filters, self.output_dim, bias=True) + with h5py.File(self.weight_file, 'r') as fin: + weight = fin['CNN_proj']['W_proj'][...] + bias = fin['CNN_proj']['b_proj'][...] + self._projection.weight.data.copy_(torch.FloatTensor(numpy.transpose(weight))) + self._projection.bias.data.copy_(torch.FloatTensor(bias)) + + self._projection.weight.requires_grad = self.requires_grad + self._projection.bias.requires_grad = self.requires_grad def forward(self, words, chars): - embs = [] - if self.word_emb_layer is not None: - if hasattr(self, 'words_to_words'): - words = self.words_to_words[words] - word_emb = self.word_emb_layer(words) - embs.append(word_emb) + """ + :param words: + :param chars: Tensor Shape ``(batch_size, sequence_length, 50)``: + :return Tensor Shape ``(batch_size, sequence_length + 2, embedding_dim)`` : + """ + # the character id embedding + # (batch_size * sequence_length, max_chars_per_token, embed_dim) + # character_embedding = torch.nn.functional.embedding( + # chars.view(-1, max_chars_per_token), + # self._char_embedding_weights + # ) + batch_size, sequence_length, max_char_len = chars.size() + character_embedding = self.char_emb_layer(chars).reshape(batch_size*sequence_length, max_char_len, -1) + # run convolutions + cnn_options = self._options['token_embedder'] + if cnn_options['activation'] == 'tanh': + activation = torch.tanh + elif cnn_options['activation'] == 'relu': + activation = torch.nn.functional.relu + else: + raise Exception("Unknown activation") - if self.char_emb_layer is not None: - batch_size, seq_len, _ = chars.size() - chars = chars.view(batch_size * seq_len, -1) - character_embedding = self.char_emb_layer(chars) - character_embedding = torch.transpose(character_embedding, 1, 2) - - cnn_config = self.config['token_embedder'] - if cnn_config['activation'] == 'tanh': - activation = torch.nn.functional.tanh - elif cnn_config['activation'] == 'relu': - activation = torch.nn.functional.relu - else: - raise Exception("Unknown activation") + # (batch_size * sequence_length, embed_dim, max_chars_per_token) + character_embedding = torch.transpose(character_embedding, 1, 2) + convs = [] + for i in range(len(self._convolutions)): + conv = getattr(self, 'char_conv_{}'.format(i)) + convolved = conv(character_embedding) + # (batch_size * sequence_length, n_filters for this width) + convolved, _ = torch.max(convolved, dim=-1) + convolved = activation(convolved) + convs.append(convolved) - convs = [] - for i in range(len(self.convolutions)): - convolved = self.convolutions[i](character_embedding) - # (batch_size * sequence_length, n_filters for this width) - convolved, _ = torch.max(convolved, dim=-1) - convolved = activation(convolved) - convs.append(convolved) - char_emb = torch.cat(convs, dim=-1) - char_emb = self.highways(char_emb) + # (batch_size * sequence_length, n_filters) + token_embedding = torch.cat(convs, dim=-1) - embs.append(char_emb.view(batch_size, -1, self.n_filters)) + # apply the highway layers (batch_size * sequence_length, n_filters) + token_embedding = self._highways(token_embedding) - token_embedding = torch.cat(embs, dim=2) + # final projection (batch_size * sequence_length, embedding_dim) + token_embedding = self._projection(token_embedding) - return self.projection(token_embedding) + # reshape to (batch_size, sequence_length+2, embedding_dim) + return token_embedding.view(batch_size, sequence_length, -1) class Highway(torch.nn.Module): @@ -543,6 +668,7 @@ class Highway(torch.nn.Module): activation : ``Callable[[torch.Tensor], torch.Tensor]``, optional (default=``torch.nn.functional.relu``) The non-linearity to use in the highway layers. """ + def __init__(self, input_dim: int, num_layers: int = 1, @@ -573,6 +699,7 @@ class Highway(torch.nn.Module): current_input = gate * linear_part + (1 - gate) * nonlinear_part return current_input + class _ElmoModel(nn.Module): """ 该Module是ElmoEmbedding中进行所有的heavy lifting的地方。做的工作，包括 @@ -582,11 +709,32 @@ class _ElmoModel(nn.Module): (4) 设计一个保存token的embedding，允许缓存word的表示。 """ - def __init__(self, model_dir:str, vocab:Vocabulary=None, cache_word_reprs:bool=False): + + def __init__(self, model_dir: str, vocab: Vocabulary = None, cache_word_reprs: bool = False): super(_ElmoModel, self).__init__() - config = json.load(open(os.path.join(model_dir, 'structure_config.json'), 'r')) + dir = os.walk(model_dir) + config_file = None + weight_file = None + config_count = 0 + weight_count = 0 + for path, dir_list, file_list in dir: + for file_name in file_list: + if file_name.__contains__(".json"): + config_file = file_name + config_count += 1 + elif file_name.__contains__(".hdf5"): + weight_file = file_name + weight_count += 1 + if config_count > 1 or weight_count > 1: + raise Exception(f"Multiple config files(*.json) or weight files(*.hdf5) detected in {model_dir}.") + elif config_count == 0 or weight_count == 0: + raise Exception(f"No config file or weight file found in {model_dir}") + + config = json.load(open(os.path.join(model_dir, config_file), 'r')) + self.weight_file = os.path.join(model_dir, weight_file) self.config = config + self.requires_grad = False OOV_TAG = '' PAD_TAG = '' @@ -595,48 +743,8 @@ class _ElmoModel(nn.Module): BOW_TAG = '' EOW_TAG = '' - # 将加载embedding放到这里 - token_embedder_states = torch.load(os.path.join(model_dir, 'token_embedder.pkl'), map_location='cpu') - - # For the model trained with word form word encoder. - if config['token_embedder']['word_dim'] > 0: - word_lexicon = {} - with codecs.open(os.path.join(model_dir, 'word.dic'), 'r', encoding='utf-8') as fpi: - for line in fpi: - tokens = line.strip().split('\t') - if len(tokens) == 1: - tokens.insert(0, '\u3000') - token, i = tokens - word_lexicon[token] = int(i) - # 做一些sanity check - for special_word in [PAD_TAG, OOV_TAG, BOS_TAG, EOS_TAG]: - assert special_word in word_lexicon, f"{special_word} not found in word.dic." - # 根据vocab调整word_embedding - pre_word_embedding = token_embedder_states.pop('word_emb_layer.embedding.weight') - word_emb_layer = nn.Embedding(len(vocab)+2, config['token_embedder']['word_dim']) #多增加两个是为了与 - found_word_count = 0 - for word, index in vocab: - if index == vocab.unknown_idx: # 因为fastNLP的unknow是 而在这里是所以ugly强制适配一下 - index_in_pre = word_lexicon[OOV_TAG] - found_word_count += 1 - elif index == vocab.padding_idx: # 需要pad对齐 - index_in_pre = word_lexicon[PAD_TAG] - found_word_count += 1 - elif word in word_lexicon: - index_in_pre = word_lexicon[word] - found_word_count += 1 - else: - index_in_pre = word_lexicon[OOV_TAG] - word_emb_layer.weight.data[index] = pre_word_embedding[index_in_pre] - print(f"{found_word_count} out of {len(vocab)} words were found in pretrained elmo embedding.") - word_emb_layer.weight.data[-1] = pre_word_embedding[word_lexicon[EOS_TAG]] - word_emb_layer.weight.data[-2] = pre_word_embedding[word_lexicon[BOS_TAG]] - self.word_vocab = vocab - else: - word_emb_layer = None - # For the model trained with character-based word encoder. - if config['token_embedder']['char_dim'] > 0: + if config['token_embedder']['embedding']['dim'] > 0: char_lexicon = {} with codecs.open(os.path.join(model_dir, 'char.dic'), 'r', encoding='utf-8') as fpi: for line in fpi: @@ -645,22 +753,26 @@ class _ElmoModel(nn.Module): tokens.insert(0, '\u3000') token, i = tokens char_lexicon[token] = int(i) + # 做一些sanity check for special_word in [PAD_TAG, OOV_TAG, BOW_TAG, EOW_TAG]: assert special_word in char_lexicon, f"{special_word} not found in char.dic." + # 从vocab中构建char_vocab char_vocab = Vocabulary(unknown=OOV_TAG, padding=PAD_TAG) # 需要保证与在里面 - char_vocab.add_word(BOW_TAG) - char_vocab.add_word(EOW_TAG) + char_vocab.add_word_lst([BOW_TAG, EOW_TAG, BOS_TAG, EOS_TAG]) + for word, index in vocab: char_vocab.add_word_lst(list(word)) - # 保证, 也在 - char_vocab.add_word_lst(list(BOS_TAG)) - char_vocab.add_word_lst(list(EOS_TAG)) - # 根据char_lexicon调整 - char_emb_layer = nn.Embedding(len(char_vocab), int(config['token_embedder']['char_dim'])) - pre_char_embedding = token_embedder_states.pop('char_emb_layer.embedding.weight') + + self.bos_index, self.eos_index, self._pad_index = len(vocab), len(vocab)+1, vocab.padding_idx + # 根据char_lexicon调整, 多设置一位，是预留给word padding的(该位置的char表示为全0表示) + char_emb_layer = nn.Embedding(len(char_vocab)+1, int(config['token_embedder']['embedding']['dim']), + padding_idx=len(char_vocab)) + with h5py.File(self.weight_file, 'r') as fin: + char_embed_weights = fin['char_embed'][...] + char_embed_weights = torch.from_numpy(char_embed_weights) found_char_count = 0 for char, index in char_vocab: # 调整character embedding if char in char_lexicon: @@ -668,79 +780,84 @@ class _ElmoModel(nn.Module): found_char_count += 1 else: index_in_pre = char_lexicon[OOV_TAG] - char_emb_layer.weight.data[index] = pre_char_embedding[index_in_pre] + char_emb_layer.weight.data[index] = char_embed_weights[index_in_pre] + print(f"{found_char_count} out of {len(char_vocab)} characters were found in pretrained elmo embedding.") # 生成words到chars的映射 if config['token_embedder']['name'].lower() == 'cnn': max_chars = config['token_embedder']['max_characters_per_token'] elif config['token_embedder']['name'].lower() == 'lstm': - max_chars = max(map(lambda x: len(x[0]), vocab)) + 2 # 需要补充两个与 + max_chars = max(map(lambda x: len(x[0]), vocab)) + 2 # 需要补充两个与 else: raise ValueError('Unknown token_embedder: {0}'.format(config['token_embedder']['name'])) - # 增加, 所以加2. + self.words_to_chars_embedding = nn.Parameter(torch.full((len(vocab)+2, max_chars), - fill_value=char_vocab.to_index(PAD_TAG), dtype=torch.long), + fill_value=len(char_vocab), + dtype=torch.long), requires_grad=False) - for word, index in vocab: - if len(word)+2>max_chars: - word = word[:max_chars-2] - if index==vocab.padding_idx: # 如果是pad的话，需要和给定的对齐 - word = PAD_TAG - elif index==vocab.unknown_idx: - word = OOV_TAG - char_ids = [char_vocab.to_index(BOW_TAG)] + [char_vocab.to_index(c) for c in word] + [char_vocab.to_index(EOW_TAG)] - char_ids += [char_vocab.to_index(PAD_TAG)]*(max_chars-len(char_ids)) + for word, index in list(iter(vocab)) + [(BOS_TAG, len(vocab)), (EOS_TAG, len(vocab)+1)]: + if len(word) + 2 > max_chars: + word = word[:max_chars - 2] + if index == self._pad_index: + continue + elif word == BOS_TAG or word == EOS_TAG: + char_ids = [char_vocab.to_index(BOW_TAG)] + [char_vocab.to_index(word)] + [ + char_vocab.to_index(EOW_TAG)] + char_ids += [char_vocab.to_index(PAD_TAG)] * (max_chars - len(char_ids)) + else: + char_ids = [char_vocab.to_index(BOW_TAG)] + [char_vocab.to_index(c) for c in word] + [ + char_vocab.to_index(EOW_TAG)] + char_ids += [char_vocab.to_index(PAD_TAG)] * (max_chars - len(char_ids)) self.words_to_chars_embedding[index] = torch.LongTensor(char_ids) - for index, word in enumerate([BOS_TAG, EOS_TAG]): # 加上, - if len(word)+2>max_chars: - word = word[:max_chars-2] - char_ids = [char_vocab.to_index(BOW_TAG)] + [char_vocab.to_index(c) for c in word] + [char_vocab.to_index(EOW_TAG)] - char_ids += [char_vocab.to_index(PAD_TAG)]*(max_chars-len(char_ids)) - self.words_to_chars_embedding[index+len(vocab)] = torch.LongTensor(char_ids) + self.char_vocab = char_vocab else: char_emb_layer = None if config['token_embedder']['name'].lower() == 'cnn': self.token_embedder = ConvTokenEmbedder( - config, word_emb_layer, char_emb_layer) + config, self.weight_file, None, char_emb_layer, self.char_vocab) elif config['token_embedder']['name'].lower() == 'lstm': self.token_embedder = LstmTokenEmbedder( - config, word_emb_layer, char_emb_layer) - self.token_embedder.load_state_dict(token_embedder_states, strict=False) - if config['token_embedder']['word_dim'] > 0 and vocab._no_create_word_length > 0: # 需要映射，使得来自于dev, test的idx指向unk - words_to_words = nn.Parameter(torch.arange(len(vocab)+2).long(), requires_grad=False) + config, None, char_emb_layer) + + if config['token_embedder']['word_dim'] > 0 \ + and vocab._no_create_word_length > 0: # 需要映射，使得来自于dev, test的idx指向unk + words_to_words = nn.Parameter(torch.arange(len(vocab) + 2).long(), requires_grad=False) for word, idx in vocab: if vocab._is_word_no_create_entry(word): words_to_words[idx] = vocab.unknown_idx setattr(self.token_embedder, 'words_to_words', words_to_words) self.output_dim = config['encoder']['projection_dim'] + # 暂时只考虑 elmo if config['encoder']['name'].lower() == 'elmo': self.encoder = ElmobiLm(config) elif config['encoder']['name'].lower() == 'lstm': self.encoder = LstmbiLm(config) - self.encoder.load_state_dict(torch.load(os.path.join(model_dir, 'encoder.pkl'), - map_location='cpu')) - self.bos_index = len(vocab) - self.eos_index = len(vocab) + 1 - self._pad_index = vocab.padding_idx + self.encoder.load_weights(self.weight_file) if cache_word_reprs: - if config['token_embedder']['char_dim']>0: # 只有在使用了chars的情况下有用 + if config['token_embedder']['embedding']['dim'] > 0: # 只有在使用了chars的情况下有用 print("Start to generate cache word representations.") batch_size = 320 - num_batches = self.words_to_chars_embedding.size(0)//batch_size + \ - int(self.words_to_chars_embedding.size(0)%batch_size!=0) - self.cached_word_embedding = nn.Embedding(self.words_to_chars_embedding.size(0), + # bos eos + word_size = self.words_to_chars_embedding.size(0) + num_batches = word_size // batch_size + \ + int(word_size % batch_size != 0) + + self.cached_word_embedding = nn.Embedding(word_size, config['encoder']['projection_dim']) with torch.no_grad(): for i in range(num_batches): - words = torch.arange(i*batch_size, min((i+1)*batch_size, self.words_to_chars_embedding.size(0))).long() + words = torch.arange(i * batch_size, + min((i + 1) * batch_size, word_size)).long() chars = self.words_to_chars_embedding[words].unsqueeze(1) # batch_size x 1 x max_chars - word_reprs = self.token_embedder(words.unsqueeze(1), chars).detach() # batch_size x 1 x config['encoder']['projection_dim'] + word_reprs = self.token_embedder(words.unsqueeze(1), + chars).detach() # batch_size x 1 x config['encoder']['projection_dim'] self.cached_word_embedding.weight.data[words] = word_reprs.squeeze(1) + print("Finish generating cached word representations. Going to delete the character encoder.") del self.token_embedder, self.words_to_chars_embedding else: @@ -758,7 +875,7 @@ class _ElmoModel(nn.Module): seq_len = words.ne(self._pad_index).sum(dim=-1) expanded_words[:, 1:-1] = words expanded_words[:, 0].fill_(self.bos_index) - expanded_words[torch.arange(batch_size).to(words), seq_len+1] = self.eos_index + expanded_words[torch.arange(batch_size).to(words), seq_len + 1] = self.eos_index seq_len = seq_len + 2 if hasattr(self, 'cached_word_embedding'): token_embedding = self.cached_word_embedding(expanded_words) @@ -767,16 +884,18 @@ class _ElmoModel(nn.Module): chars = self.words_to_chars_embedding[expanded_words] else: chars = None - token_embedding = self.token_embedder(expanded_words, chars) + token_embedding = self.token_embedder(expanded_words, chars) # batch_size x max_len x embed_dim + if self.config['encoder']['name'] == 'elmo': encoder_output = self.encoder(token_embedding, seq_len) - if encoder_output.size(2) < max_len+2: - dummy_tensor = encoder_output.new_zeros(encoder_output.size(0), batch_size, - max_len + 2 - encoder_output.size(2), encoder_output.size(-1)) - encoder_output = torch.cat([encoder_output, dummy_tensor], 2) - sz = encoder_output.size() # 2, batch_size, max_len, hidden_size - token_embedding = torch.cat([token_embedding, token_embedding], dim=2).view(1, sz[1], sz[2], sz[3]) - encoder_output = torch.cat([token_embedding, encoder_output], dim=0) + if encoder_output.size(2) < max_len + 2: + num_layers, _, output_len, hidden_size = encoder_output.size() + dummy_tensor = encoder_output.new_zeros(num_layers, batch_size, + max_len + 2 - output_len, hidden_size) + encoder_output = torch.cat((encoder_output, dummy_tensor), 2) + sz = encoder_output.size() # 2, batch_size, max_len, hidden_size + token_embedding = torch.cat((token_embedding, token_embedding), dim=2).view(1, sz[1], sz[2], sz[3]) + encoder_output = torch.cat((token_embedding, encoder_output), dim=0) elif self.config['encoder']['name'] == 'lstm': encoder_output = self.encoder(token_embedding, seq_len) else: @@ -784,5 +903,4 @@ class _ElmoModel(nn.Module): # 删除, . 这里没有精确地删除，但应该也不会影响最后的结果了。 encoder_output = encoder_output[:, :, 1:-1] - return encoder_output From 4687b378bb224e03317aedb08f0307af4116e1a8 Mon Sep 17 00:00:00 2001 From: wyg <1505116161@qq.com> Date: Mon, 8 Jul 2019 11:03:00 +0800 Subject: [PATCH 27/27] [verify] readme --- reproduction/text_classification/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/reproduction/text_classification/README.md b/reproduction/text_classification/README.md index 08c893b7..a318cc61 100644 --- a/reproduction/text_classification/README.md +++ b/reproduction/text_classification/README.md @@ -11,8 +11,6 @@ LSTM+self_attention:论文链接[A Structured Self-attentive Sentence Embedding] AWD-LSTM:论文链接[Regularizing and Optimizing LSTM Language Models]() -#待补充 - # 数据集及复现结果汇总 使用fastNLP复现的结果vs论文汇报结果(/前为fastNLP实现，后面为论文报道,-表示论文没有在该数据集上列出结果)