[new] text_classification:charcnn,dpcnn,HAN,awd-lstmtags/v0.4.10
@@ -5,10 +5,8 @@ from ..base_loader import DataInfo, DataSetLoader | |||
from ...core.vocabulary import VocabularyOption, Vocabulary | |||
from ...core.dataset import DataSet | |||
from ...core.instance import Instance | |||
from ..embed_loader import EmbeddingOption, EmbedLoader | |||
from ..utils import check_dataloader_paths, get_tokenizer | |||
spacy.prefer_gpu() | |||
sptk = spacy.load('en') | |||
class SSTLoader(DataSetLoader): | |||
URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' | |||
@@ -37,6 +35,7 @@ class SSTLoader(DataSetLoader): | |||
tag_v['0'] = tag_v['1'] | |||
tag_v['4'] = tag_v['3'] | |||
self.tag_v = tag_v | |||
self.tokenizer = get_tokenizer() | |||
def _load(self, path): | |||
""" | |||
@@ -55,29 +54,37 @@ class SSTLoader(DataSetLoader): | |||
ds.append(Instance(words=words, target=tag)) | |||
return ds | |||
@staticmethod | |||
def _get_one(data, subtree): | |||
def _get_one(self, data, subtree): | |||
tree = Tree.fromstring(data) | |||
if subtree: | |||
return [([x.text for x in sptk.tokenizer(' '.join(t.leaves()))], t.label()) for t in tree.subtrees() ] | |||
return [([x.text for x in sptk.tokenizer(' '.join(tree.leaves()))], tree.label())] | |||
return [([x.text for x in self.tokenizer(' '.join(t.leaves()))], t.label()) for t in tree.subtrees() ] | |||
return [([x.text for x in self.tokenizer(' '.join(tree.leaves()))], tree.label())] | |||
def process(self, | |||
paths, | |||
train_ds: Iterable[str] = None, | |||
paths, train_subtree=True, | |||
src_vocab_op: VocabularyOption = None, | |||
tgt_vocab_op: VocabularyOption = None, | |||
src_embed_op: EmbeddingOption = None): | |||
tgt_vocab_op: VocabularyOption = None,): | |||
paths = check_dataloader_paths(paths) | |||
input_name, target_name = 'words', 'target' | |||
src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) | |||
tgt_vocab = Vocabulary(unknown=None, padding=None) \ | |||
if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) | |||
info = DataInfo(datasets=self.load(paths)) | |||
_train_ds = [info.datasets[name] | |||
for name in train_ds] if train_ds else info.datasets.values() | |||
src_vocab.from_dataset(*_train_ds, field_name=input_name) | |||
tgt_vocab.from_dataset(*_train_ds, field_name=target_name) | |||
info = DataInfo() | |||
origin_subtree = self.subtree | |||
self.subtree = train_subtree | |||
info.datasets['train'] = self._load(paths['train']) | |||
self.subtree = origin_subtree | |||
for n, p in paths.items(): | |||
if n != 'train': | |||
info.datasets[n] = self._load(p) | |||
src_vocab.from_dataset( | |||
info.datasets['train'], | |||
field_name=input_name, | |||
no_create_entry_dataset=[ds for n, ds in info.datasets.items() if n != 'train']) | |||
tgt_vocab.from_dataset(info.datasets['train'], field_name=target_name) | |||
src_vocab.index_dataset( | |||
*info.datasets.values(), | |||
field_name=input_name, new_field_name=input_name) | |||
@@ -89,10 +96,5 @@ class SSTLoader(DataSetLoader): | |||
target_name: tgt_vocab | |||
} | |||
if src_embed_op is not None: | |||
src_embed_op.vocab = src_vocab | |||
init_emb = EmbedLoader.load_with_vocab(**src_embed_op) | |||
info.embeddings[input_name] = init_emb | |||
return info | |||
@@ -0,0 +1,69 @@ | |||
import os | |||
from typing import Union, Dict | |||
def check_dataloader_paths(paths:Union[str, Dict[str, str]])->Dict[str, str]: | |||
""" | |||
检查传入dataloader的文件的合法性。如果为合法路径,将返回至少包含'train'这个key的dict。类似于下面的结果 | |||
{ | |||
'train': '/some/path/to/', # 一定包含,建词表应该在这上面建立,剩下的其它文件应该只需要处理并index。 | |||
'test': 'xxx' # 可能有,也可能没有 | |||
... | |||
} | |||
如果paths为不合法的,将直接进行raise相应的错误 | |||
:param paths: 路径. 可以为一个文件路径(则认为该文件就是train的文件); 可以为一个文件目录,将在该目录下寻找train(文件名 | |||
中包含train这个字段), test.txt, dev.txt; 可以为一个dict, 则key是用户自定义的某个文件的名称,value是这个文件的路径。 | |||
:return: | |||
""" | |||
if isinstance(paths, str): | |||
if os.path.isfile(paths): | |||
return {'train': paths} | |||
elif os.path.isdir(paths): | |||
filenames = os.listdir(paths) | |||
files = {} | |||
for filename in filenames: | |||
path_pair = None | |||
if 'train' in filename: | |||
path_pair = ('train', filename) | |||
if 'dev' in filename: | |||
if path_pair: | |||
raise Exception("File:{} in {} contains bot `{}` and `dev`.".format(filename, paths, path_pair[0])) | |||
path_pair = ('dev', filename) | |||
if 'test' in filename: | |||
if path_pair: | |||
raise Exception("File:{} in {} contains bot `{}` and `test`.".format(filename, paths, path_pair[0])) | |||
path_pair = ('test', filename) | |||
if path_pair: | |||
files[path_pair[0]] = os.path.join(paths, path_pair[1]) | |||
return files | |||
else: | |||
raise FileNotFoundError(f"{paths} is not a valid file path.") | |||
elif isinstance(paths, dict): | |||
if paths: | |||
if 'train' not in paths: | |||
raise KeyError("You have to include `train` in your dict.") | |||
for key, value in paths.items(): | |||
if isinstance(key, str) and isinstance(value, str): | |||
if not os.path.isfile(value): | |||
raise TypeError(f"{value} is not a valid file.") | |||
else: | |||
raise TypeError("All keys and values in paths should be str.") | |||
return paths | |||
else: | |||
raise ValueError("Empty paths is not allowed.") | |||
else: | |||
raise TypeError(f"paths only supports str and dict. not {type(paths)}.") | |||
def get_tokenizer(): | |||
try: | |||
import spacy | |||
spacy.prefer_gpu() | |||
en = spacy.load('en') | |||
print('use spacy tokenizer') | |||
return lambda x: [w.text for w in en.tokenizer(x)] | |||
except Exception as e: | |||
print('use raw tokenizer') | |||
return lambda x: x.split() |
@@ -19,7 +19,7 @@ class DotAttention(nn.Module): | |||
补上文档 | |||
""" | |||
def __init__(self, key_size, value_size, dropout=0): | |||
def __init__(self, key_size, value_size, dropout=0.0): | |||
super(DotAttention, self).__init__() | |||
self.key_size = key_size | |||
self.value_size = value_size | |||
@@ -37,7 +37,7 @@ class DotAttention(nn.Module): | |||
""" | |||
output = torch.matmul(Q, K.transpose(1, 2)) / self.scale | |||
if mask_out is not None: | |||
output.masked_fill_(mask_out, -1e8) | |||
output.masked_fill_(mask_out, -1e18) | |||
output = self.softmax(output) | |||
output = self.drop(output) | |||
return torch.matmul(output, V) | |||
@@ -67,9 +67,8 @@ class MultiHeadAttention(nn.Module): | |||
self.k_in = nn.Linear(input_size, in_size) | |||
self.v_in = nn.Linear(input_size, in_size) | |||
# follow the paper, do not apply dropout within dot-product | |||
self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=0) | |||
self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=dropout) | |||
self.out = nn.Linear(value_size * num_head, input_size) | |||
self.drop = TimestepDropout(dropout) | |||
self.reset_parameters() | |||
def reset_parameters(self): | |||
@@ -105,7 +104,7 @@ class MultiHeadAttention(nn.Module): | |||
# concat all heads, do output linear | |||
atte = atte.permute(1, 2, 0, 3).contiguous().view(batch, sq, -1) | |||
output = self.drop(self.out(atte)) | |||
output = self.out(atte) | |||
return output | |||
@@ -6,7 +6,7 @@ paper: [Star-Transformer](https://arxiv.org/abs/1902.09113) | |||
|Pos Tagging|CTB 9.0|-|ACC 92.31| | |||
|Pos Tagging|CONLL 2012|-|ACC 96.51| | |||
|Named Entity Recognition|CONLL 2012|-|F1 85.66| | |||
|Text Classification|SST|-|49.18| | |||
|Text Classification|SST|-|51.2| | |||
|Natural Language Inference|SNLI|-|83.76| | |||
## Usage | |||
@@ -0,0 +1,142 @@ | |||
import torch | |||
import torch.nn as nn | |||
import torch.nn.functional as F | |||
from fastNLP.modules.decoder import ConditionalRandomField | |||
from fastNLP.modules.encoder import Embedding | |||
from fastNLP.core.utils import seq_len_to_mask | |||
from fastNLP.core.const import Const as C | |||
class IDCNN(nn.Module): | |||
def __init__(self, | |||
init_embed, | |||
char_embed, | |||
num_cls, | |||
repeats, num_layers, num_filters, kernel_size, | |||
use_crf=False, use_projection=False, block_loss=False, | |||
input_dropout=0.3, hidden_dropout=0.2, inner_dropout=0.0): | |||
super(IDCNN, self).__init__() | |||
self.word_embeddings = Embedding(init_embed) | |||
if char_embed is None: | |||
self.char_embeddings = None | |||
embedding_size = self.word_embeddings.embedding_dim | |||
else: | |||
self.char_embeddings = Embedding(char_embed) | |||
embedding_size = self.word_embeddings.embedding_dim + \ | |||
self.char_embeddings.embedding_dim | |||
self.conv0 = nn.Sequential( | |||
nn.Conv1d(in_channels=embedding_size, | |||
out_channels=num_filters, | |||
kernel_size=kernel_size, | |||
stride=1, dilation=1, | |||
padding=kernel_size//2, | |||
bias=True), | |||
nn.ReLU(), | |||
) | |||
block = [] | |||
for layer_i in range(num_layers): | |||
dilated = 2 ** layer_i if layer_i+1 < num_layers else 1 | |||
block.append(nn.Conv1d( | |||
in_channels=num_filters, | |||
out_channels=num_filters, | |||
kernel_size=kernel_size, | |||
stride=1, dilation=dilated, | |||
padding=(kernel_size//2) * dilated, | |||
bias=True)) | |||
block.append(nn.ReLU()) | |||
self.block = nn.Sequential(*block) | |||
if use_projection: | |||
self.projection = nn.Sequential( | |||
nn.Conv1d( | |||
in_channels=num_filters, | |||
out_channels=num_filters//2, | |||
kernel_size=1, | |||
bias=True), | |||
nn.ReLU(),) | |||
encode_dim = num_filters // 2 | |||
else: | |||
self.projection = None | |||
encode_dim = num_filters | |||
self.input_drop = nn.Dropout(input_dropout) | |||
self.hidden_drop = nn.Dropout(hidden_dropout) | |||
self.inner_drop = nn.Dropout(inner_dropout) | |||
self.repeats = repeats | |||
self.out_fc = nn.Conv1d( | |||
in_channels=encode_dim, | |||
out_channels=num_cls, | |||
kernel_size=1, | |||
bias=True) | |||
self.crf = ConditionalRandomField( | |||
num_tags=num_cls) if use_crf else None | |||
self.block_loss = block_loss | |||
self.reset_parameters() | |||
def reset_parameters(self): | |||
for m in self.modules(): | |||
if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)): | |||
nn.init.xavier_normal_(m.weight, gain=1) | |||
if m.bias is not None: | |||
nn.init.normal_(m.bias, mean=0, std=0.01) | |||
def forward(self, words, seq_len, target=None, chars=None): | |||
if self.char_embeddings is None: | |||
x = self.word_embeddings(words) | |||
else: | |||
if chars is None: | |||
raise ValueError('must provide chars for model with char embedding') | |||
e1 = self.word_embeddings(words) | |||
e2 = self.char_embeddings(chars) | |||
x = torch.cat((e1, e2), dim=-1) # b,l,h | |||
mask = seq_len_to_mask(seq_len) | |||
x = x.transpose(1, 2) # b,h,l | |||
last_output = self.conv0(x) | |||
output = [] | |||
for repeat in range(self.repeats): | |||
last_output = self.block(last_output) | |||
hidden = self.projection(last_output) if self.projection is not None else last_output | |||
output.append(self.out_fc(hidden)) | |||
def compute_loss(y, t, mask): | |||
if self.crf is not None and target is not None: | |||
loss = self.crf(y.transpose(1, 2), t, mask) | |||
else: | |||
t.masked_fill_(mask == 0, -100) | |||
loss = F.cross_entropy(y, t, ignore_index=-100) | |||
return loss | |||
if target is not None: | |||
if self.block_loss: | |||
losses = [compute_loss(o, target, mask) for o in output] | |||
loss = sum(losses) | |||
else: | |||
loss = compute_loss(output[-1], target, mask) | |||
else: | |||
loss = None | |||
scores = output[-1] | |||
if self.crf is not None: | |||
pred, _ = self.crf.viterbi_decode(scores.transpose(1, 2), mask) | |||
else: | |||
pred = scores.max(1)[1] * mask.long() | |||
return { | |||
C.LOSS: loss, | |||
C.OUTPUT: pred, | |||
} | |||
def predict(self, words, seq_len, chars=None): | |||
res = self.forward( | |||
words=words, | |||
seq_len=seq_len, | |||
chars=chars, | |||
target=None | |||
)[C.OUTPUT] | |||
return { | |||
C.OUTPUT: res | |||
} |
@@ -0,0 +1,99 @@ | |||
from reproduction.seqence_labelling.ner.data.OntoNoteLoader import OntoNoteNERDataLoader | |||
from fastNLP.core.callback import FitlogCallback, LRScheduler | |||
from fastNLP import GradientClipCallback | |||
from torch.optim.lr_scheduler import LambdaLR, CosineAnnealingLR | |||
from torch.optim import SGD, Adam | |||
from fastNLP import Const | |||
from fastNLP import RandomSampler, BucketSampler | |||
from fastNLP import SpanFPreRecMetric | |||
from fastNLP import Trainer | |||
from reproduction.seqence_labelling.ner.model.dilated_cnn import IDCNN | |||
from fastNLP.core.utils import Option | |||
from fastNLP.modules.encoder.embedding import CNNCharEmbedding, StaticEmbedding | |||
from fastNLP.core.utils import cache_results | |||
import sys | |||
import torch.cuda | |||
import os | |||
os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' | |||
os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' | |||
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" | |||
encoding_type = 'bioes' | |||
def get_path(path): | |||
return os.path.join(os.environ['HOME'], path) | |||
data_path = get_path('workdir/datasets/ontonotes-v4') | |||
ops = Option( | |||
batch_size=128, | |||
num_epochs=100, | |||
lr=3e-4, | |||
repeats=3, | |||
num_layers=3, | |||
num_filters=400, | |||
use_crf=True, | |||
gradient_clip=5, | |||
) | |||
@cache_results('ontonotes-cache') | |||
def load_data(): | |||
data = OntoNoteNERDataLoader(encoding_type=encoding_type).process(data_path, | |||
lower=True) | |||
# char_embed = CNNCharEmbedding(vocab=data.vocabs['cap_words'], embed_size=30, char_emb_size=30, filter_nums=[30], | |||
# kernel_sizes=[3]) | |||
word_embed = StaticEmbedding(vocab=data.vocabs[Const.INPUT], | |||
model_dir_or_name='en-glove-840b-300', | |||
requires_grad=True) | |||
return data, [word_embed] | |||
data, embeds = load_data() | |||
print(data.datasets['train'][0]) | |||
print(list(data.vocabs.keys())) | |||
for ds in data.datasets.values(): | |||
ds.rename_field('cap_words', 'chars') | |||
ds.set_input('chars') | |||
word_embed = embeds[0] | |||
char_embed = CNNCharEmbedding(data.vocabs['cap_words']) | |||
# for ds in data.datasets: | |||
# ds.rename_field('') | |||
print(data.vocabs[Const.TARGET].word2idx) | |||
model = IDCNN(init_embed=word_embed, | |||
char_embed=char_embed, | |||
num_cls=len(data.vocabs[Const.TARGET]), | |||
repeats=ops.repeats, | |||
num_layers=ops.num_layers, | |||
num_filters=ops.num_filters, | |||
kernel_size=3, | |||
use_crf=ops.use_crf, use_projection=True, | |||
block_loss=True, | |||
input_dropout=0.33, hidden_dropout=0.2, inner_dropout=0.2) | |||
print(model) | |||
callbacks = [GradientClipCallback(clip_value=ops.gradient_clip, clip_type='norm'),] | |||
optimizer = Adam(model.parameters(), lr=ops.lr, weight_decay=0) | |||
# scheduler = LRScheduler(LambdaLR(optimizer, lr_lambda=lambda epoch: 1 / (1 + 0.05 * epoch))) | |||
# callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 15))) | |||
# optimizer = SWATS(model.parameters(), verbose=True) | |||
# optimizer = Adam(model.parameters(), lr=0.005) | |||
device = 'cuda:0' if torch.cuda.is_available() else 'cpu' | |||
trainer = Trainer(train_data=data.datasets['train'], model=model, optimizer=optimizer, | |||
sampler=BucketSampler(num_buckets=50, batch_size=ops.batch_size), | |||
device=device, dev_data=data.datasets['dev'], batch_size=ops.batch_size, | |||
metrics=SpanFPreRecMetric( | |||
tag_vocab=data.vocabs[Const.TARGET], encoding_type=encoding_type), | |||
check_code_level=-1, | |||
callbacks=callbacks, num_workers=2, n_epochs=ops.num_epochs) | |||
trainer.train() |
@@ -0,0 +1,26 @@ | |||
# text_classification任务模型复现 | |||
这里使用fastNLP复现以下模型: | |||
char_cnn :论文链接[Character-level Convolutional Networks for Text Classification](https://arxiv.org/pdf/1509.01626v3.pdf) | |||
dpcnn:论文链接[Deep Pyramid Convolutional Neural Networks for TextCategorization](https://ai.tencent.com/ailab/media/publications/ACL3-Brady.pdf) | |||
HAN:论文链接[Hierarchical Attention Networks for Document Classification](https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf) | |||
LSTM+self_attention:论文链接[A Structured Self-attentive Sentence Embedding](<https://arxiv.org/pdf/1703.03130.pdf>) | |||
AWD-LSTM:论文链接[Regularizing and Optimizing LSTM Language Models](<https://arxiv.org/pdf/1708.02182.pdf>) | |||
# 数据集及复现结果汇总 | |||
使用fastNLP复现的结果vs论文汇报结果(/前为fastNLP实现,后面为论文报道,-表示论文没有在该数据集上列出结果) | |||
model name | yelp_p | yelp_f | sst-2|IMDB | |||
:---: | :---: | :---: | :---: |----- | |||
char_cnn | 93.80/95.12 | - | - |- | |||
dpcnn | 95.50/97.36 | - | - |- | |||
HAN |- | - | - |- | |||
LSTM| 95.74/- |- |- |88.52/- | |||
AWD-LSTM| 95.96/- |- |- |88.91/- | |||
LSTM+self_attention| 96.34/- | - | - |89.53/- | |||
@@ -0,0 +1,110 @@ | |||
from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader | |||
from fastNLP.core.vocabulary import VocabularyOption | |||
from fastNLP.io.base_loader import DataSetLoader, DataInfo | |||
from typing import Union, Dict, List, Iterator | |||
from fastNLP import DataSet | |||
from fastNLP import Instance | |||
from fastNLP import Vocabulary | |||
from fastNLP import Const | |||
# from reproduction.utils import check_dataloader_paths | |||
from functools import partial | |||
class IMDBLoader(DataSetLoader): | |||
""" | |||
读取IMDB数据集,DataSet包含以下fields: | |||
words: list(str), 需要分类的文本 | |||
target: str, 文本的标签 | |||
""" | |||
def __init__(self): | |||
super(IMDBLoader, self).__init__() | |||
def _load(self, path): | |||
dataset = DataSet() | |||
with open(path, 'r', encoding="utf-8") as f: | |||
for line in f: | |||
line = line.strip() | |||
if not line: | |||
continue | |||
parts = line.split('\t') | |||
target = parts[0] | |||
words = parts[1].lower().split() | |||
dataset.append(Instance(words=words, target=target)) | |||
if len(dataset)==0: | |||
raise RuntimeError(f"{path} has no valid data.") | |||
return dataset | |||
def process(self, | |||
paths: Union[str, Dict[str, str]], | |||
src_vocab_opt: VocabularyOption = None, | |||
tgt_vocab_opt: VocabularyOption = None, | |||
src_embed_opt: EmbeddingOption = None, | |||
char_level_op=False): | |||
datasets = {} | |||
info = DataInfo() | |||
for name, path in paths.items(): | |||
dataset = self.load(path) | |||
datasets[name] = dataset | |||
def wordtochar(words): | |||
chars = [] | |||
for word in words: | |||
word = word.lower() | |||
for char in word: | |||
chars.append(char) | |||
return chars | |||
if char_level_op: | |||
for dataset in datasets.values(): | |||
dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') | |||
datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False) | |||
src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) | |||
src_vocab.from_dataset(datasets['train'], field_name='words') | |||
src_vocab.index_dataset(*datasets.values(), field_name='words') | |||
tgt_vocab = Vocabulary(unknown=None, padding=None) \ | |||
if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) | |||
tgt_vocab.from_dataset(datasets['train'], field_name='target') | |||
tgt_vocab.index_dataset(*datasets.values(), field_name='target') | |||
info.vocabs = { | |||
"words": src_vocab, | |||
"target": tgt_vocab | |||
} | |||
info.datasets = datasets | |||
if src_embed_opt is not None: | |||
embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) | |||
info.embeddings['words'] = embed | |||
for name, dataset in info.datasets.items(): | |||
dataset.set_input("words") | |||
dataset.set_target("target") | |||
return info | |||
if __name__=="__main__": | |||
datapath = {"train": "/remote-home/ygwang/IMDB_data/train.csv", | |||
"test": "/remote-home/ygwang/IMDB_data/test.csv"} | |||
datainfo=IMDBLoader().process(datapath,char_level_op=True) | |||
#print(datainfo.datasets["train"]) | |||
len_count = 0 | |||
for instance in datainfo.datasets["train"]: | |||
len_count += len(instance["chars"]) | |||
ave_len = len_count / len(datainfo.datasets["train"]) | |||
print(ave_len) | |||
@@ -32,7 +32,7 @@ class MTL16Loader(DataSetLoader): | |||
continue | |||
parts = line.split('\t') | |||
target = parts[0] | |||
words = parts[1].split() | |||
words = parts[1].lower().split() | |||
dataset.append(Instance(words=words, target=target)) | |||
if len(dataset)==0: | |||
raise RuntimeError(f"{path} has no valid data.") | |||
@@ -72,4 +72,8 @@ class MTL16Loader(DataSetLoader): | |||
embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) | |||
info.embeddings['words'] = embed | |||
for name, dataset in info.datasets.items(): | |||
dataset.set_input("words") | |||
dataset.set_target("target") | |||
return info |
@@ -0,0 +1,187 @@ | |||
from typing import Iterable | |||
from nltk import Tree | |||
from fastNLP.io.base_loader import DataInfo, DataSetLoader | |||
from fastNLP.core.vocabulary import VocabularyOption, Vocabulary | |||
from fastNLP import DataSet | |||
from fastNLP import Instance | |||
from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader | |||
import csv | |||
from typing import Union, Dict | |||
class SSTLoader(DataSetLoader): | |||
URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' | |||
DATA_DIR = 'sst/' | |||
""" | |||
别名::class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader` | |||
读取SST数据集, DataSet包含fields:: | |||
words: list(str) 需要分类的文本 | |||
target: str 文本的标签 | |||
数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip | |||
:param subtree: 是否将数据展开为子树,扩充数据量. Default: ``False`` | |||
:param fine_grained: 是否使用SST-5标准,若 ``False`` , 使用SST-2。Default: ``False`` | |||
""" | |||
def __init__(self, subtree=False, fine_grained=False): | |||
self.subtree = subtree | |||
tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral', | |||
'3': 'positive', '4': 'very positive'} | |||
if not fine_grained: | |||
tag_v['0'] = tag_v['1'] | |||
tag_v['4'] = tag_v['3'] | |||
self.tag_v = tag_v | |||
def _load(self, path): | |||
""" | |||
:param str path: 存储数据的路径 | |||
:return: 一个 :class:`~fastNLP.DataSet` 类型的对象 | |||
""" | |||
datalist = [] | |||
with open(path, 'r', encoding='utf-8') as f: | |||
datas = [] | |||
for l in f: | |||
datas.extend([(s, self.tag_v[t]) | |||
for s, t in self._get_one(l, self.subtree)]) | |||
ds = DataSet() | |||
for words, tag in datas: | |||
ds.append(Instance(words=words, target=tag)) | |||
return ds | |||
@staticmethod | |||
def _get_one(data, subtree): | |||
tree = Tree.fromstring(data) | |||
if subtree: | |||
return [(t.leaves(), t.label()) for t in tree.subtrees()] | |||
return [(tree.leaves(), tree.label())] | |||
def process(self, | |||
paths, | |||
train_ds: Iterable[str] = None, | |||
src_vocab_op: VocabularyOption = None, | |||
tgt_vocab_op: VocabularyOption = None, | |||
src_embed_op: EmbeddingOption = None): | |||
input_name, target_name = 'words', 'target' | |||
src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) | |||
tgt_vocab = Vocabulary(unknown=None, padding=None) \ | |||
if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) | |||
info = DataInfo(datasets=self.load(paths)) | |||
_train_ds = [info.datasets[name] | |||
for name in train_ds] if train_ds else info.datasets.values() | |||
src_vocab.from_dataset(*_train_ds, field_name=input_name) | |||
tgt_vocab.from_dataset(*_train_ds, field_name=target_name) | |||
src_vocab.index_dataset( | |||
*info.datasets.values(), | |||
field_name=input_name, new_field_name=input_name) | |||
tgt_vocab.index_dataset( | |||
*info.datasets.values(), | |||
field_name=target_name, new_field_name=target_name) | |||
info.vocabs = { | |||
input_name: src_vocab, | |||
target_name: tgt_vocab | |||
} | |||
if src_embed_op is not None: | |||
src_embed_op.vocab = src_vocab | |||
init_emb = EmbedLoader.load_with_vocab(**src_embed_op) | |||
info.embeddings[input_name] = init_emb | |||
for name, dataset in info.datasets.items(): | |||
dataset.set_input(input_name) | |||
dataset.set_target(target_name) | |||
return info | |||
class sst2Loader(DataSetLoader): | |||
''' | |||
数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', | |||
''' | |||
def __init__(self): | |||
super(sst2Loader, self).__init__() | |||
def _load(self, path: str) -> DataSet: | |||
ds = DataSet() | |||
all_count=0 | |||
csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t') | |||
skip_row = 0 | |||
for idx,row in enumerate(csv_reader): | |||
if idx<=skip_row: | |||
continue | |||
target = row[1] | |||
words = row[0].split() | |||
ds.append(Instance(words=words,target=target)) | |||
all_count+=1 | |||
print("all count:", all_count) | |||
return ds | |||
def process(self, | |||
paths: Union[str, Dict[str, str]], | |||
src_vocab_opt: VocabularyOption = None, | |||
tgt_vocab_opt: VocabularyOption = None, | |||
src_embed_opt: EmbeddingOption = None, | |||
char_level_op=False): | |||
paths = check_dataloader_paths(paths) | |||
datasets = {} | |||
info = DataInfo() | |||
for name, path in paths.items(): | |||
dataset = self.load(path) | |||
datasets[name] = dataset | |||
def wordtochar(words): | |||
chars=[] | |||
for word in words: | |||
word=word.lower() | |||
for char in word: | |||
chars.append(char) | |||
return chars | |||
input_name, target_name = 'words', 'target' | |||
info.vocabs={} | |||
# 就分隔为char形式 | |||
if char_level_op: | |||
for dataset in datasets.values(): | |||
dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') | |||
src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) | |||
src_vocab.from_dataset(datasets['train'], field_name='words') | |||
src_vocab.index_dataset(*datasets.values(), field_name='words') | |||
tgt_vocab = Vocabulary(unknown=None, padding=None) \ | |||
if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) | |||
tgt_vocab.from_dataset(datasets['train'], field_name='target') | |||
tgt_vocab.index_dataset(*datasets.values(), field_name='target') | |||
info.vocabs = { | |||
"words": src_vocab, | |||
"target": tgt_vocab | |||
} | |||
info.datasets = datasets | |||
if src_embed_opt is not None: | |||
embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) | |||
info.embeddings['words'] = embed | |||
return info | |||
if __name__=="__main__": | |||
datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv", | |||
"dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"} | |||
datainfo=sst2Loader().process(datapath,char_level_op=True) | |||
#print(datainfo.datasets["train"]) | |||
len_count = 0 | |||
for instance in datainfo.datasets["train"]: | |||
len_count += len(instance["chars"]) | |||
ave_len = len_count / len(datainfo.datasets["train"]) | |||
print(ave_len) |
@@ -0,0 +1,187 @@ | |||
from typing import Iterable | |||
from nltk import Tree | |||
from fastNLP.io.base_loader import DataInfo, DataSetLoader | |||
from fastNLP.core.vocabulary import VocabularyOption, Vocabulary | |||
from fastNLP import DataSet | |||
from fastNLP import Instance | |||
from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader | |||
import csv | |||
from typing import Union, Dict | |||
class SSTLoader(DataSetLoader): | |||
URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' | |||
DATA_DIR = 'sst/' | |||
""" | |||
别名::class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader` | |||
读取SST数据集, DataSet包含fields:: | |||
words: list(str) 需要分类的文本 | |||
target: str 文本的标签 | |||
数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip | |||
:param subtree: 是否将数据展开为子树,扩充数据量. Default: ``False`` | |||
:param fine_grained: 是否使用SST-5标准,若 ``False`` , 使用SST-2。Default: ``False`` | |||
""" | |||
def __init__(self, subtree=False, fine_grained=False): | |||
self.subtree = subtree | |||
tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral', | |||
'3': 'positive', '4': 'very positive'} | |||
if not fine_grained: | |||
tag_v['0'] = tag_v['1'] | |||
tag_v['4'] = tag_v['3'] | |||
self.tag_v = tag_v | |||
def _load(self, path): | |||
""" | |||
:param str path: 存储数据的路径 | |||
:return: 一个 :class:`~fastNLP.DataSet` 类型的对象 | |||
""" | |||
datalist = [] | |||
with open(path, 'r', encoding='utf-8') as f: | |||
datas = [] | |||
for l in f: | |||
datas.extend([(s, self.tag_v[t]) | |||
for s, t in self._get_one(l, self.subtree)]) | |||
ds = DataSet() | |||
for words, tag in datas: | |||
ds.append(Instance(words=words, target=tag)) | |||
return ds | |||
@staticmethod | |||
def _get_one(data, subtree): | |||
tree = Tree.fromstring(data) | |||
if subtree: | |||
return [(t.leaves(), t.label()) for t in tree.subtrees()] | |||
return [(tree.leaves(), tree.label())] | |||
def process(self, | |||
paths, | |||
train_ds: Iterable[str] = None, | |||
src_vocab_op: VocabularyOption = None, | |||
tgt_vocab_op: VocabularyOption = None, | |||
src_embed_op: EmbeddingOption = None): | |||
input_name, target_name = 'words', 'target' | |||
src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) | |||
tgt_vocab = Vocabulary(unknown=None, padding=None) \ | |||
if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) | |||
info = DataInfo(datasets=self.load(paths)) | |||
_train_ds = [info.datasets[name] | |||
for name in train_ds] if train_ds else info.datasets.values() | |||
src_vocab.from_dataset(*_train_ds, field_name=input_name) | |||
tgt_vocab.from_dataset(*_train_ds, field_name=target_name) | |||
src_vocab.index_dataset( | |||
*info.datasets.values(), | |||
field_name=input_name, new_field_name=input_name) | |||
tgt_vocab.index_dataset( | |||
*info.datasets.values(), | |||
field_name=target_name, new_field_name=target_name) | |||
info.vocabs = { | |||
input_name: src_vocab, | |||
target_name: tgt_vocab | |||
} | |||
if src_embed_op is not None: | |||
src_embed_op.vocab = src_vocab | |||
init_emb = EmbedLoader.load_with_vocab(**src_embed_op) | |||
info.embeddings[input_name] = init_emb | |||
for name, dataset in info.datasets.items(): | |||
dataset.set_input(input_name) | |||
dataset.set_target(target_name) | |||
return info | |||
class sst2Loader(DataSetLoader): | |||
''' | |||
数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', | |||
''' | |||
def __init__(self): | |||
super(sst2Loader, self).__init__() | |||
def _load(self, path: str) -> DataSet: | |||
ds = DataSet() | |||
all_count=0 | |||
csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t') | |||
skip_row = 0 | |||
for idx,row in enumerate(csv_reader): | |||
if idx<=skip_row: | |||
continue | |||
target = row[1] | |||
words = row[0].split() | |||
ds.append(Instance(words=words,target=target)) | |||
all_count+=1 | |||
print("all count:", all_count) | |||
return ds | |||
def process(self, | |||
paths: Union[str, Dict[str, str]], | |||
src_vocab_opt: VocabularyOption = None, | |||
tgt_vocab_opt: VocabularyOption = None, | |||
src_embed_opt: EmbeddingOption = None, | |||
char_level_op=False): | |||
paths = check_dataloader_paths(paths) | |||
datasets = {} | |||
info = DataInfo() | |||
for name, path in paths.items(): | |||
dataset = self.load(path) | |||
datasets[name] = dataset | |||
def wordtochar(words): | |||
chars=[] | |||
for word in words: | |||
word=word.lower() | |||
for char in word: | |||
chars.append(char) | |||
return chars | |||
input_name, target_name = 'words', 'target' | |||
info.vocabs={} | |||
# 就分隔为char形式 | |||
if char_level_op: | |||
for dataset in datasets.values(): | |||
dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') | |||
src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) | |||
src_vocab.from_dataset(datasets['train'], field_name='words') | |||
src_vocab.index_dataset(*datasets.values(), field_name='words') | |||
tgt_vocab = Vocabulary(unknown=None, padding=None) \ | |||
if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) | |||
tgt_vocab.from_dataset(datasets['train'], field_name='target') | |||
tgt_vocab.index_dataset(*datasets.values(), field_name='target') | |||
info.vocabs = { | |||
"words": src_vocab, | |||
"target": tgt_vocab | |||
} | |||
info.datasets = datasets | |||
if src_embed_opt is not None: | |||
embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab) | |||
info.embeddings['words'] = embed | |||
return info | |||
if __name__=="__main__": | |||
datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv", | |||
"dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"} | |||
datainfo=sst2Loader().process(datapath,char_level_op=True) | |||
#print(datainfo.datasets["train"]) | |||
len_count = 0 | |||
for instance in datainfo.datasets["train"]: | |||
len_count += len(instance["chars"]) | |||
ave_len = len_count / len(datainfo.datasets["train"]) | |||
print(ave_len) |
@@ -1,18 +1,64 @@ | |||
import ast | |||
import csv | |||
from typing import Iterable | |||
from fastNLP import DataSet, Instance, Vocabulary | |||
from fastNLP.core.vocabulary import VocabularyOption | |||
from fastNLP.io import JsonLoader | |||
from fastNLP.io.base_loader import DataInfo | |||
from fastNLP.io.base_loader import DataInfo,DataSetLoader | |||
from fastNLP.io.embed_loader import EmbeddingOption | |||
from fastNLP.io.file_reader import _read_json | |||
from typing import Union, Dict | |||
from reproduction.Star_transformer.datasets import EmbedLoader | |||
from reproduction.utils import check_dataloader_paths | |||
from reproduction.utils import check_dataloader_paths, get_tokenizer | |||
def clean_str(sentence, tokenizer, char_lower=False): | |||
""" | |||
heavily borrowed from github | |||
https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb | |||
:param sentence: is a str | |||
:return: | |||
""" | |||
if char_lower: | |||
sentence = sentence.lower() | |||
import re | |||
nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') | |||
words = tokenizer(sentence) | |||
words_collection = [] | |||
for word in words: | |||
if word in ['-lrb-', '-rrb-', '<sssss>', '-r', '-l', 'b-']: | |||
continue | |||
tt = nonalpnum.split(word) | |||
t = ''.join(tt) | |||
if t != '': | |||
words_collection.append(t) | |||
return words_collection | |||
class yelpLoader(JsonLoader): | |||
class yelpLoader(DataSetLoader): | |||
""" | |||
读取Yelp_full/Yelp_polarity数据集, DataSet包含fields: | |||
words: list(str), 需要分类的文本 | |||
target: str, 文本的标签 | |||
chars:list(str),未index的字符列表 | |||
数据集:yelp_full/yelp_polarity | |||
:param fine_grained: 是否使用SST-5标准,若 ``False`` , 使用SST-2。Default: ``False`` | |||
""" | |||
def __init__(self, fine_grained=False,lower=False): | |||
super(yelpLoader, self).__init__() | |||
tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral', | |||
'4.0': 'positive', '5.0': 'very positive'} | |||
if not fine_grained: | |||
tag_v['1.0'] = tag_v['2.0'] | |||
tag_v['5.0'] = tag_v['4.0'] | |||
self.fine_grained = fine_grained | |||
self.tag_v = tag_v | |||
self.lower = lower | |||
self.tokenizer = get_tokenizer() | |||
''' | |||
读取Yelp数据集, DataSet包含fields: | |||
review_id: str, 22 character unique review id | |||
@@ -27,20 +73,8 @@ class yelpLoader(JsonLoader): | |||
数据来源: https://www.yelp.com/dataset/download | |||
:param fine_grained: 是否使用SST-5标准,若 ``False`` , 使用SST-2。Default: ``False`` | |||
""" | |||
def __init__(self, fine_grained=False): | |||
super(yelpLoader, self).__init__() | |||
tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral', | |||
'4.0': 'positive', '5.0': 'very positive'} | |||
if not fine_grained: | |||
tag_v['1.0'] = tag_v['2.0'] | |||
tag_v['5.0'] = tag_v['4.0'] | |||
self.fine_grained = fine_grained | |||
self.tag_v = tag_v | |||
def _load(self, path): | |||
def _load_json(self, path): | |||
ds = DataSet() | |||
for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): | |||
d = ast.literal_eval(d) | |||
@@ -48,21 +82,116 @@ class yelpLoader(JsonLoader): | |||
d["target"] = self.tag_v[str(d.pop("stars"))] | |||
ds.append(Instance(**d)) | |||
return ds | |||
def _load_yelp2015_broken(self,path): | |||
ds = DataSet() | |||
with open (path,encoding='ISO 8859-1') as f: | |||
row=f.readline() | |||
all_count=0 | |||
exp_count=0 | |||
while row: | |||
row=row.split("\t\t") | |||
all_count+=1 | |||
if len(row)>=3: | |||
words=row[-1].split() | |||
try: | |||
target=self.tag_v[str(row[-2])+".0"] | |||
ds.append(Instance(words=words, target=target)) | |||
except KeyError: | |||
exp_count+=1 | |||
else: | |||
exp_count+=1 | |||
row = f.readline() | |||
print("error sample count:",exp_count) | |||
print("all count:",all_count) | |||
return ds | |||
''' | |||
def _load(self, path): | |||
ds = DataSet() | |||
csv_reader=csv.reader(open(path,encoding='utf-8')) | |||
all_count=0 | |||
real_count=0 | |||
for row in csv_reader: | |||
all_count+=1 | |||
if len(row)==2: | |||
target=self.tag_v[row[0]+".0"] | |||
words = clean_str(row[1], self.tokenizer, self.lower) | |||
if len(words)!=0: | |||
ds.append(Instance(words=words,target=target)) | |||
real_count += 1 | |||
print("all count:", all_count) | |||
print("real count:", real_count) | |||
return ds | |||
def process(self, paths: Union[str, Dict[str, str]], vocab_opt: VocabularyOption = None, | |||
embed_opt: EmbeddingOption = None): | |||
def process(self, paths: Union[str, Dict[str, str]], | |||
train_ds: Iterable[str] = None, | |||
src_vocab_op: VocabularyOption = None, | |||
tgt_vocab_op: VocabularyOption = None, | |||
embed_opt: EmbeddingOption = None, | |||
char_level_op=False): | |||
paths = check_dataloader_paths(paths) | |||
datasets = {} | |||
info = DataInfo() | |||
vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt) | |||
for name, path in paths.items(): | |||
dataset = self.load(path) | |||
datasets[name] = dataset | |||
vocab.from_dataset(dataset, field_name="words") | |||
info.vocabs = vocab | |||
info.datasets = datasets | |||
if embed_opt is not None: | |||
embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) | |||
info.embeddings['words'] = embed | |||
info = DataInfo(datasets=self.load(paths)) | |||
src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) | |||
tgt_vocab = Vocabulary(unknown=None, padding=None) \ | |||
if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) | |||
_train_ds = [info.datasets[name] | |||
for name in train_ds] if train_ds else info.datasets.values() | |||
def wordtochar(words): | |||
chars=[] | |||
for word in words: | |||
word=word.lower() | |||
for char in word: | |||
chars.append(char) | |||
return chars | |||
input_name, target_name = 'words', 'target' | |||
info.vocabs={} | |||
#就分隔为char形式 | |||
if char_level_op: | |||
for dataset in info.datasets.values(): | |||
dataset.apply_field(wordtochar, field_name="words",new_field_name='chars') | |||
# if embed_opt is not None: | |||
# embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab) | |||
# info.embeddings['words'] = embed | |||
else: | |||
src_vocab.from_dataset(*_train_ds, field_name=input_name) | |||
src_vocab.index_dataset(*info.datasets.values(),field_name=input_name, new_field_name=input_name) | |||
info.vocabs[input_name]=src_vocab | |||
tgt_vocab.from_dataset(*_train_ds, field_name=target_name) | |||
tgt_vocab.index_dataset( | |||
*info.datasets.values(), | |||
field_name=target_name, new_field_name=target_name) | |||
info.vocabs[target_name]=tgt_vocab | |||
info.datasets['train'],info.datasets['dev']=info.datasets['train'].split(0.1, shuffle=False) | |||
for name, dataset in info.datasets.items(): | |||
dataset.set_input("words") | |||
dataset.set_target("target") | |||
return info | |||
if __name__=="__main__": | |||
testloader=yelpLoader() | |||
# datapath = {"train": "/remote-home/ygwang/yelp_full/train.csv", | |||
# "test": "/remote-home/ygwang/yelp_full/test.csv"} | |||
#datapath={"train": "/remote-home/ygwang/yelp_full/test.csv"} | |||
datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv", | |||
"test": "/remote-home/ygwang/yelp_polarity/test.csv"} | |||
datainfo=testloader.process(datapath,char_level_op=True) | |||
len_count=0 | |||
for instance in datainfo.datasets["train"]: | |||
len_count+=len(instance["chars"]) | |||
ave_len=len_count/len(datainfo.datasets["train"]) | |||
print(ave_len) |
@@ -0,0 +1,109 @@ | |||
import torch | |||
import torch.nn as nn | |||
from torch.autograd import Variable | |||
from fastNLP.modules.utils import get_embeddings | |||
from fastNLP.core import Const as C | |||
def pack_sequence(tensor_seq, padding_value=0.0): | |||
if len(tensor_seq) <= 0: | |||
return | |||
length = [v.size(0) for v in tensor_seq] | |||
max_len = max(length) | |||
size = [len(tensor_seq), max_len] | |||
size.extend(list(tensor_seq[0].size()[1:])) | |||
ans = torch.Tensor(*size).fill_(padding_value) | |||
if tensor_seq[0].data.is_cuda: | |||
ans = ans.cuda() | |||
ans = Variable(ans) | |||
for i, v in enumerate(tensor_seq): | |||
ans[i, :length[i], :] = v | |||
return ans | |||
class HANCLS(nn.Module): | |||
def __init__(self, init_embed, num_cls): | |||
super(HANCLS, self).__init__() | |||
self.embed = get_embeddings(init_embed) | |||
self.han = HAN(input_size=300, | |||
output_size=num_cls, | |||
word_hidden_size=50, word_num_layers=1, word_context_size=100, | |||
sent_hidden_size=50, sent_num_layers=1, sent_context_size=100 | |||
) | |||
def forward(self, input_sents): | |||
# input_sents [B, num_sents, seq-len] dtype long | |||
# target | |||
B, num_sents, seq_len = input_sents.size() | |||
input_sents = input_sents.view(-1, seq_len) # flat | |||
words_embed = self.embed(input_sents) # should be [B*num-sent, seqlen , word-dim] | |||
words_embed = words_embed.view(B, num_sents, seq_len, -1) # recover # [B, num-sent, seqlen , word-dim] | |||
out = self.han(words_embed) | |||
return {C.OUTPUT: out} | |||
def predict(self, input_sents): | |||
x = self.forward(input_sents)[C.OUTPUT] | |||
return {C.OUTPUT: torch.argmax(x, 1)} | |||
class HAN(nn.Module): | |||
def __init__(self, input_size, output_size, | |||
word_hidden_size, word_num_layers, word_context_size, | |||
sent_hidden_size, sent_num_layers, sent_context_size): | |||
super(HAN, self).__init__() | |||
self.word_layer = AttentionNet(input_size, | |||
word_hidden_size, | |||
word_num_layers, | |||
word_context_size) | |||
self.sent_layer = AttentionNet(2 * word_hidden_size, | |||
sent_hidden_size, | |||
sent_num_layers, | |||
sent_context_size) | |||
self.output_layer = nn.Linear(2 * sent_hidden_size, output_size) | |||
self.softmax = nn.LogSoftmax(dim=1) | |||
def forward(self, batch_doc): | |||
# input is a sequence of matrix | |||
doc_vec_list = [] | |||
for doc in batch_doc: | |||
sent_mat = self.word_layer(doc) # doc's dim (num_sent, seq_len, word_dim) | |||
doc_vec_list.append(sent_mat) # sent_mat's dim (num_sent, vec_dim) | |||
doc_vec = self.sent_layer(pack_sequence(doc_vec_list)) | |||
output = self.softmax(self.output_layer(doc_vec)) | |||
return output | |||
class AttentionNet(nn.Module): | |||
def __init__(self, input_size, gru_hidden_size, gru_num_layers, context_vec_size): | |||
super(AttentionNet, self).__init__() | |||
self.input_size = input_size | |||
self.gru_hidden_size = gru_hidden_size | |||
self.gru_num_layers = gru_num_layers | |||
self.context_vec_size = context_vec_size | |||
# Encoder | |||
self.gru = nn.GRU(input_size=input_size, | |||
hidden_size=gru_hidden_size, | |||
num_layers=gru_num_layers, | |||
batch_first=True, | |||
bidirectional=True) | |||
# Attention | |||
self.fc = nn.Linear(2 * gru_hidden_size, context_vec_size) | |||
self.tanh = nn.Tanh() | |||
self.softmax = nn.Softmax(dim=1) | |||
# context vector | |||
self.context_vec = nn.Parameter(torch.Tensor(context_vec_size, 1)) | |||
self.context_vec.data.uniform_(-0.1, 0.1) | |||
def forward(self, inputs): | |||
# GRU part | |||
h_t, hidden = self.gru(inputs) # inputs's dim (batch_size, seq_len, word_dim) | |||
u = self.tanh(self.fc(h_t)) | |||
# Attention part | |||
alpha = self.softmax(torch.matmul(u, self.context_vec)) # u's dim (batch_size, seq_len, context_vec_size) | |||
output = torch.bmm(torch.transpose(h_t, 1, 2), alpha) # alpha's dim (batch_size, seq_len, 1) | |||
return torch.squeeze(output, dim=2) # output's dim (batch_size, 2*hidden_size, 1) |
@@ -0,0 +1,31 @@ | |||
import torch | |||
import torch.nn as nn | |||
from fastNLP.core.const import Const as C | |||
from .awdlstm_module import LSTM | |||
from fastNLP.modules import encoder | |||
from fastNLP.modules.decoder.mlp import MLP | |||
class AWDLSTMSentiment(nn.Module): | |||
def __init__(self, init_embed, | |||
num_classes, | |||
hidden_dim=256, | |||
num_layers=1, | |||
nfc=128, | |||
wdrop=0.5): | |||
super(AWDLSTMSentiment,self).__init__() | |||
self.embed = encoder.Embedding(init_embed) | |||
self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True, wdrop=wdrop) | |||
self.mlp = MLP(size_layer=[hidden_dim* 2, nfc, num_classes]) | |||
def forward(self, words): | |||
x_emb = self.embed(words) | |||
output, _ = self.lstm(x_emb) | |||
output = self.mlp(output[:,-1,:]) | |||
return {C.OUTPUT: output} | |||
def predict(self, words): | |||
output = self(words) | |||
_, predict = output[C.OUTPUT].max(dim=1) | |||
return {C.OUTPUT: predict} | |||
@@ -0,0 +1,86 @@ | |||
""" | |||
轻量封装的 Pytorch LSTM 模块. | |||
可在 forward 时传入序列的长度, 自动对padding做合适的处理. | |||
""" | |||
__all__ = [ | |||
"LSTM" | |||
] | |||
import torch | |||
import torch.nn as nn | |||
import torch.nn.utils.rnn as rnn | |||
from fastNLP.modules.utils import initial_parameter | |||
from torch import autograd | |||
from .weight_drop import WeightDrop | |||
class LSTM(nn.Module): | |||
""" | |||
别名::class:`fastNLP.modules.LSTM` :class:`fastNLP.modules.encoder.lstm.LSTM` | |||
LSTM 模块, 轻量封装的Pytorch LSTM. 在提供seq_len的情况下,将自动使用pack_padded_sequence; 同时默认将forget gate的bias初始化 | |||
为1; 且可以应对DataParallel中LSTM的使用问题。 | |||
:param input_size: 输入 `x` 的特征维度 | |||
:param hidden_size: 隐状态 `h` 的特征维度. | |||
:param num_layers: rnn的层数. Default: 1 | |||
:param dropout: 层间dropout概率. Default: 0 | |||
:param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` | |||
:param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 | |||
:(batch, seq, feature). Default: ``False`` | |||
:param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` | |||
""" | |||
def __init__(self, input_size, hidden_size=100, num_layers=1, dropout=0.0, batch_first=True, | |||
bidirectional=False, bias=True, wdrop=0.5): | |||
super(LSTM, self).__init__() | |||
self.batch_first = batch_first | |||
self.lstm = nn.LSTM(input_size, hidden_size, num_layers, bias=bias, batch_first=batch_first, | |||
dropout=dropout, bidirectional=bidirectional) | |||
self.lstm = WeightDrop(self.lstm, ['weight_hh_l0'], dropout=wdrop) | |||
self.init_param() | |||
def init_param(self): | |||
for name, param in self.named_parameters(): | |||
if 'bias' in name: | |||
# based on https://github.com/pytorch/pytorch/issues/750#issuecomment-280671871 | |||
param.data.fill_(0) | |||
n = param.size(0) | |||
start, end = n // 4, n // 2 | |||
param.data[start:end].fill_(1) | |||
else: | |||
nn.init.xavier_uniform_(param) | |||
def forward(self, x, seq_len=None, h0=None, c0=None): | |||
""" | |||
:param x: [batch, seq_len, input_size] 输入序列 | |||
:param seq_len: [batch, ] 序列长度, 若为 ``None``, 所有输入看做一样长. Default: ``None`` | |||
:param h0: [batch, hidden_size] 初始隐状态, 若为 ``None`` , 设为全0向量. Default: ``None`` | |||
:param c0: [batch, hidden_size] 初始Cell状态, 若为 ``None`` , 设为全0向量. Default: ``None`` | |||
:return (output, ht) 或 output: 若 ``get_hidden=True`` [batch, seq_len, hidden_size*num_direction] 输出序列 | |||
和 [batch, hidden_size*num_direction] 最后时刻隐状态. | |||
""" | |||
batch_size, max_len, _ = x.size() | |||
if h0 is not None and c0 is not None: | |||
hx = (h0, c0) | |||
else: | |||
hx = None | |||
if seq_len is not None and not isinstance(x, rnn.PackedSequence): | |||
sort_lens, sort_idx = torch.sort(seq_len, dim=0, descending=True) | |||
if self.batch_first: | |||
x = x[sort_idx] | |||
else: | |||
x = x[:, sort_idx] | |||
x = rnn.pack_padded_sequence(x, sort_lens, batch_first=self.batch_first) | |||
output, hx = self.lstm(x, hx) # -> [N,L,C] | |||
output, _ = rnn.pad_packed_sequence(output, batch_first=self.batch_first, total_length=max_len) | |||
_, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) | |||
if self.batch_first: | |||
output = output[unsort_idx] | |||
else: | |||
output = output[:, unsort_idx] | |||
else: | |||
output, hx = self.lstm(x, hx) | |||
return output, hx |
@@ -1 +1,90 @@ | |||
# TODO | |||
''' | |||
@author: https://github.com/ahmedbesbes/character-based-cnn | |||
这里借鉴了上述链接中char-cnn model的代码,改动主要为将其改动为符合fastnlp的pipline | |||
''' | |||
import torch | |||
import torch.nn as nn | |||
from fastNLP.core.const import Const as C | |||
class CharacterLevelCNN(nn.Module): | |||
def __init__(self, args,embedding): | |||
super(CharacterLevelCNN, self).__init__() | |||
self.config=args.char_cnn_config | |||
self.embedding=embedding | |||
conv_layers = [] | |||
for i, conv_layer_parameter in enumerate(self.config['model_parameters'][args.model_size]['conv']): | |||
if i == 0: | |||
#in_channels = args.number_of_characters + len(args.extra_characters) | |||
in_channels = args.embedding_dim | |||
out_channels = conv_layer_parameter[0] | |||
else: | |||
in_channels, out_channels = conv_layer_parameter[0], conv_layer_parameter[0] | |||
if conv_layer_parameter[2] != -1: | |||
conv_layer = nn.Sequential(nn.Conv1d(in_channels, | |||
out_channels, | |||
kernel_size=conv_layer_parameter[1], padding=0), | |||
nn.ReLU(), | |||
nn.MaxPool1d(conv_layer_parameter[2])) | |||
else: | |||
conv_layer = nn.Sequential(nn.Conv1d(in_channels, | |||
out_channels, | |||
kernel_size=conv_layer_parameter[1], padding=0), | |||
nn.ReLU()) | |||
conv_layers.append(conv_layer) | |||
self.conv_layers = nn.ModuleList(conv_layers) | |||
input_shape = (args.batch_size, args.max_length, | |||
args.number_of_characters + len(args.extra_characters)) | |||
dimension = self._get_conv_output(input_shape) | |||
print('dimension :', dimension) | |||
fc_layer_parameter = self.config['model_parameters'][args.model_size]['fc'][0] | |||
fc_layers = nn.ModuleList([ | |||
nn.Sequential( | |||
nn.Linear(dimension, fc_layer_parameter), nn.Dropout(0.5)), | |||
nn.Sequential(nn.Linear(fc_layer_parameter, | |||
fc_layer_parameter), nn.Dropout(0.5)), | |||
nn.Linear(fc_layer_parameter, args.num_classes), | |||
]) | |||
self.fc_layers = fc_layers | |||
if args.model_size == 'small': | |||
self._create_weights(mean=0.0, std=0.05) | |||
elif args.model_size == 'large': | |||
self._create_weights(mean=0.0, std=0.02) | |||
def _create_weights(self, mean=0.0, std=0.05): | |||
for module in self.modules(): | |||
if isinstance(module, nn.Conv1d) or isinstance(module, nn.Linear): | |||
module.weight.data.normal_(mean, std) | |||
def _get_conv_output(self, shape): | |||
input = torch.rand(shape) | |||
output = input.transpose(1, 2) | |||
# forward pass through conv layers | |||
for i in range(len(self.conv_layers)): | |||
output = self.conv_layers[i](output) | |||
output = output.view(output.size(0), -1) | |||
n_size = output.size(1) | |||
return n_size | |||
def forward(self, chars): | |||
input=self.embedding(chars) | |||
output = input.transpose(1, 2) | |||
# forward pass through conv layers | |||
for i in range(len(self.conv_layers)): | |||
output = self.conv_layers[i](output) | |||
output = output.view(output.size(0), -1) | |||
# forward pass through fc layers | |||
for i in range(len(self.fc_layers)): | |||
output = self.fc_layers[i](output) | |||
return {C.OUTPUT: output} |
@@ -1 +1,97 @@ | |||
# TODO | |||
import torch | |||
import torch.nn as nn | |||
from fastNLP.modules.utils import get_embeddings | |||
from fastNLP.core import Const as C | |||
class DPCNN(nn.Module): | |||
def __init__(self, init_embed, num_cls, n_filters=256, | |||
kernel_size=3, n_layers=7, embed_dropout=0.1, cls_dropout=0.1): | |||
super().__init__() | |||
self.region_embed = RegionEmbedding( | |||
init_embed, out_dim=n_filters, kernel_sizes=[1, 3, 5]) | |||
embed_dim = self.region_embed.embedding_dim | |||
self.conv_list = nn.ModuleList() | |||
for i in range(n_layers): | |||
self.conv_list.append(nn.Sequential( | |||
nn.ReLU(), | |||
nn.Conv1d(n_filters, n_filters, kernel_size, | |||
padding=kernel_size//2), | |||
nn.Conv1d(n_filters, n_filters, kernel_size, | |||
padding=kernel_size//2), | |||
)) | |||
self.pool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1) | |||
self.embed_drop = nn.Dropout(embed_dropout) | |||
self.classfier = nn.Sequential( | |||
nn.Dropout(cls_dropout), | |||
nn.Linear(n_filters, num_cls), | |||
) | |||
self.reset_parameters() | |||
def reset_parameters(self): | |||
for m in self.modules(): | |||
if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)): | |||
nn.init.normal_(m.weight, mean=0, std=0.01) | |||
if m.bias is not None: | |||
nn.init.normal_(m.bias, mean=0, std=0.01) | |||
def forward(self, words, seq_len=None): | |||
words = words.long() | |||
# get region embeddings | |||
x = self.region_embed(words) | |||
x = self.embed_drop(x) | |||
# not pooling on first conv | |||
x = self.conv_list[0](x) + x | |||
for conv in self.conv_list[1:]: | |||
x = self.pool(x) | |||
x = conv(x) + x | |||
# B, C, L => B, C | |||
x, _ = torch.max(x, dim=2) | |||
x = self.classfier(x) | |||
return {C.OUTPUT: x} | |||
def predict(self, words, seq_len=None): | |||
x = self.forward(words, seq_len)[C.OUTPUT] | |||
return {C.OUTPUT: torch.argmax(x, 1)} | |||
class RegionEmbedding(nn.Module): | |||
def __init__(self, init_embed, out_dim=300, kernel_sizes=None): | |||
super().__init__() | |||
if kernel_sizes is None: | |||
kernel_sizes = [5, 9] | |||
assert isinstance( | |||
kernel_sizes, list), 'kernel_sizes should be List(int)' | |||
self.embed = get_embeddings(init_embed) | |||
try: | |||
embed_dim = self.embed.embedding_dim | |||
except Exception: | |||
embed_dim = self.embed.embed_size | |||
self.region_embeds = nn.ModuleList() | |||
for ksz in kernel_sizes: | |||
self.region_embeds.append(nn.Sequential( | |||
nn.Conv1d(embed_dim, embed_dim, ksz, padding=ksz // 2), | |||
)) | |||
self.linears = nn.ModuleList([nn.Conv1d(embed_dim, out_dim, 1) | |||
for _ in range(len(kernel_sizes))]) | |||
self.embedding_dim = embed_dim | |||
def forward(self, x): | |||
x = self.embed(x) | |||
x = x.transpose(1, 2) | |||
# B, C, L | |||
out = 0 | |||
for conv, fc in zip(self.region_embeds, self.linears[1:]): | |||
conv_i = conv(x) | |||
out = out + fc(conv_i) | |||
# B, C, L | |||
return out | |||
if __name__ == '__main__': | |||
x = torch.randint(0, 10000, size=(5, 15), dtype=torch.long) | |||
model = DPCNN((10000, 300), 20) | |||
y = model(x) | |||
print(y.size(), y.mean(1), y.std(1)) |
@@ -0,0 +1,30 @@ | |||
import torch | |||
import torch.nn as nn | |||
from fastNLP.core.const import Const as C | |||
from fastNLP.modules.encoder.lstm import LSTM | |||
from fastNLP.modules import encoder | |||
from fastNLP.modules.decoder.mlp import MLP | |||
class BiLSTMSentiment(nn.Module): | |||
def __init__(self, init_embed, | |||
num_classes, | |||
hidden_dim=256, | |||
num_layers=1, | |||
nfc=128): | |||
super(BiLSTMSentiment,self).__init__() | |||
self.embed = encoder.Embedding(init_embed) | |||
self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True) | |||
self.mlp = MLP(size_layer=[hidden_dim* 2, nfc, num_classes]) | |||
def forward(self, words): | |||
x_emb = self.embed(words) | |||
output, _ = self.lstm(x_emb) | |||
output = self.mlp(output[:,-1,:]) | |||
return {C.OUTPUT: output} | |||
def predict(self, words): | |||
output = self(words) | |||
_, predict = output[C.OUTPUT].max(dim=1) | |||
return {C.OUTPUT: predict} | |||
@@ -0,0 +1,35 @@ | |||
import torch | |||
import torch.nn as nn | |||
from fastNLP.core.const import Const as C | |||
from fastNLP.modules.encoder.lstm import LSTM | |||
from fastNLP.modules import encoder | |||
from fastNLP.modules.aggregator.attention import SelfAttention | |||
from fastNLP.modules.decoder.mlp import MLP | |||
class BiLSTM_SELF_ATTENTION(nn.Module): | |||
def __init__(self, init_embed, | |||
num_classes, | |||
hidden_dim=256, | |||
num_layers=1, | |||
attention_unit=256, | |||
attention_hops=1, | |||
nfc=128): | |||
super(BiLSTM_SELF_ATTENTION,self).__init__() | |||
self.embed = encoder.Embedding(init_embed) | |||
self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True) | |||
self.attention = SelfAttention(input_size=hidden_dim * 2 , attention_unit=attention_unit, attention_hops=attention_hops) | |||
self.mlp = MLP(size_layer=[hidden_dim* 2*attention_hops, nfc, num_classes]) | |||
def forward(self, words): | |||
x_emb = self.embed(words) | |||
output, _ = self.lstm(x_emb) | |||
after_attention, penalty = self.attention(output,words) | |||
after_attention =after_attention.view(after_attention.size(0),-1) | |||
output = self.mlp(after_attention) | |||
return {C.OUTPUT: output} | |||
def predict(self, words): | |||
output = self(words) | |||
_, predict = output[C.OUTPUT].max(dim=1) | |||
return {C.OUTPUT: predict} |
@@ -0,0 +1,99 @@ | |||
import torch | |||
from torch.nn import Parameter | |||
from functools import wraps | |||
class WeightDrop(torch.nn.Module): | |||
def __init__(self, module, weights, dropout=0, variational=False): | |||
super(WeightDrop, self).__init__() | |||
self.module = module | |||
self.weights = weights | |||
self.dropout = dropout | |||
self.variational = variational | |||
self._setup() | |||
def widget_demagnetizer_y2k_edition(*args, **kwargs): | |||
# We need to replace flatten_parameters with a nothing function | |||
# It must be a function rather than a lambda as otherwise pickling explodes | |||
# We can't write boring code though, so ... WIDGET DEMAGNETIZER Y2K EDITION! | |||
# (╯°□°)╯︵ ┻━┻ | |||
return | |||
def _setup(self): | |||
# Terrible temporary solution to an issue regarding compacting weights re: CUDNN RNN | |||
if issubclass(type(self.module), torch.nn.RNNBase): | |||
self.module.flatten_parameters = self.widget_demagnetizer_y2k_edition | |||
for name_w in self.weights: | |||
print('Applying weight drop of {} to {}'.format(self.dropout, name_w)) | |||
w = getattr(self.module, name_w) | |||
del self.module._parameters[name_w] | |||
self.module.register_parameter(name_w + '_raw', Parameter(w.data)) | |||
def _setweights(self): | |||
for name_w in self.weights: | |||
raw_w = getattr(self.module, name_w + '_raw') | |||
w = None | |||
if self.variational: | |||
mask = torch.autograd.Variable(torch.ones(raw_w.size(0), 1)) | |||
if raw_w.is_cuda: mask = mask.cuda() | |||
mask = torch.nn.functional.dropout(mask, p=self.dropout, training=True) | |||
w = mask.expand_as(raw_w) * raw_w | |||
else: | |||
w = torch.nn.functional.dropout(raw_w, p=self.dropout, training=self.training) | |||
setattr(self.module, name_w, w) | |||
def forward(self, *args): | |||
self._setweights() | |||
return self.module.forward(*args) | |||
if __name__ == '__main__': | |||
import torch | |||
from weight_drop import WeightDrop | |||
# Input is (seq, batch, input) | |||
x = torch.autograd.Variable(torch.randn(2, 1, 10)).cuda() | |||
h0 = None | |||
### | |||
print('Testing WeightDrop') | |||
print('=-=-=-=-=-=-=-=-=-=') | |||
### | |||
print('Testing WeightDrop with Linear') | |||
lin = WeightDrop(torch.nn.Linear(10, 10), ['weight'], dropout=0.9) | |||
lin.cuda() | |||
run1 = [x.sum() for x in lin(x).data] | |||
run2 = [x.sum() for x in lin(x).data] | |||
print('All items should be different') | |||
print('Run 1:', run1) | |||
print('Run 2:', run2) | |||
assert run1[0] != run2[0] | |||
assert run1[1] != run2[1] | |||
print('---') | |||
### | |||
print('Testing WeightDrop with LSTM') | |||
wdrnn = WeightDrop(torch.nn.LSTM(10, 10), ['weight_hh_l0'], dropout=0.9) | |||
wdrnn.cuda() | |||
run1 = [x.sum() for x in wdrnn(x, h0)[0].data] | |||
run2 = [x.sum() for x in wdrnn(x, h0)[0].data] | |||
print('First timesteps should be equal, all others should differ') | |||
print('Run 1:', run1) | |||
print('Run 2:', run2) | |||
# First time step, not influenced by hidden to hidden weights, should be equal | |||
assert run1[0] == run2[0] | |||
# Second step should not | |||
assert run1[1] != run2[1] | |||
print('---') |
@@ -0,0 +1,109 @@ | |||
# 首先需要加入以下的路径到环境变量,因为当前只对内部测试开放,所以需要手动申明一下路径 | |||
import os | |||
import sys | |||
sys.path.append('../../') | |||
os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' | |||
os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' | |||
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" | |||
from fastNLP.core.const import Const as C | |||
from fastNLP.core import LRScheduler | |||
import torch.nn as nn | |||
from fastNLP.io.dataset_loader import SSTLoader | |||
from reproduction.text_classification.data.yelpLoader import yelpLoader | |||
from reproduction.text_classification.model.HAN import HANCLS | |||
from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding | |||
from fastNLP import CrossEntropyLoss, AccuracyMetric | |||
from fastNLP.core.trainer import Trainer | |||
from torch.optim import SGD | |||
import torch.cuda | |||
from torch.optim.lr_scheduler import CosineAnnealingLR | |||
##hyper | |||
class Config(): | |||
model_dir_or_name = "en-base-uncased" | |||
embedding_grad = False, | |||
train_epoch = 30 | |||
batch_size = 100 | |||
num_classes = 5 | |||
task = "yelp" | |||
#datadir = '/remote-home/lyli/fastNLP/yelp_polarity/' | |||
datadir = '/remote-home/ygwang/yelp_polarity/' | |||
datafile = {"train": "train.csv", "test": "test.csv"} | |||
lr = 1e-3 | |||
def __init__(self): | |||
self.datapath = {k: os.path.join(self.datadir, v) | |||
for k, v in self.datafile.items()} | |||
ops = Config() | |||
##1.task相关信息:利用dataloader载入dataInfo | |||
datainfo = yelpLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) | |||
print(len(datainfo.datasets['train'])) | |||
print(len(datainfo.datasets['test'])) | |||
# post process | |||
def make_sents(words): | |||
sents = [words] | |||
return sents | |||
for dataset in datainfo.datasets.values(): | |||
dataset.apply_field(make_sents, field_name='words', new_field_name='input_sents') | |||
datainfo = datainfo | |||
datainfo.datasets['train'].set_input('input_sents') | |||
datainfo.datasets['test'].set_input('input_sents') | |||
datainfo.datasets['train'].set_target('target') | |||
datainfo.datasets['test'].set_target('target') | |||
## 2.或直接复用fastNLP的模型 | |||
vocab = datainfo.vocabs['words'] | |||
# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) | |||
embedding = StaticEmbedding(vocab) | |||
print(len(vocab)) | |||
print(len(datainfo.vocabs['target'])) | |||
# model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) | |||
model = HANCLS(init_embed=embedding, num_cls=ops.num_classes) | |||
## 3. 声明loss,metric,optimizer | |||
loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) | |||
metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) | |||
optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], | |||
lr=ops.lr, momentum=0.9, weight_decay=0) | |||
callbacks = [] | |||
callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) | |||
device = 'cuda:0' if torch.cuda.is_available() else 'cpu' | |||
print(device) | |||
for ds in datainfo.datasets.values(): | |||
ds.apply_field(len, C.INPUT, C.INPUT_LEN) | |||
ds.set_input(C.INPUT, C.INPUT_LEN) | |||
ds.set_target(C.TARGET) | |||
## 4.定义train方法 | |||
def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch): | |||
trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, | |||
metrics=[metrics], dev_data=datainfo.datasets['test'], device=device, | |||
check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, | |||
n_epochs=num_epochs) | |||
print(trainer.train()) | |||
if __name__ == "__main__": | |||
train(model, datainfo, loss, metric, optimizer) |
@@ -0,0 +1,69 @@ | |||
# 这个模型需要在pytorch=0.4下运行,weight_drop不支持1.0 | |||
# 首先需要加入以下的路径到环境变量,因为当前只对内部测试开放,所以需要手动申明一下路径 | |||
import os | |||
os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' | |||
os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' | |||
import torch.nn as nn | |||
from data.IMDBLoader import IMDBLoader | |||
from fastNLP.modules.encoder.embedding import StaticEmbedding | |||
from model.awd_lstm import AWDLSTMSentiment | |||
from fastNLP.core.const import Const as C | |||
from fastNLP import CrossEntropyLoss, AccuracyMetric | |||
from fastNLP import Trainer, Tester | |||
from torch.optim import Adam | |||
from fastNLP.io.model_io import ModelLoader, ModelSaver | |||
import argparse | |||
class Config(): | |||
train_epoch= 10 | |||
lr=0.001 | |||
num_classes=2 | |||
hidden_dim=256 | |||
num_layers=1 | |||
nfc=128 | |||
wdrop=0.5 | |||
task_name = "IMDB" | |||
datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} | |||
save_model_path="./result_IMDB_test/" | |||
opt=Config() | |||
# load data | |||
dataloader=IMDBLoader() | |||
datainfo=dataloader.process(opt.datapath) | |||
# print(datainfo.datasets["train"]) | |||
# print(datainfo) | |||
# define model | |||
vocab=datainfo.vocabs['words'] | |||
embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) | |||
model=AWDLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc, wdrop=opt.wdrop) | |||
# define loss_function and metrics | |||
loss=CrossEntropyLoss() | |||
metrics=AccuracyMetric() | |||
optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) | |||
def train(datainfo, model, optimizer, loss, metrics, opt): | |||
trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, | |||
metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, | |||
n_epochs=opt.train_epoch, save_path=opt.save_model_path) | |||
trainer.train() | |||
if __name__ == "__main__": | |||
train(datainfo, model, optimizer, loss, metrics, opt) |
@@ -0,0 +1,205 @@ | |||
# 首先需要加入以下的路径到环境变量,因为当前只对内部测试开放,所以需要手动申明一下路径 | |||
import os | |||
os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' | |||
os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' | |||
import sys | |||
sys.path.append('../..') | |||
from fastNLP.core.const import Const as C | |||
import torch.nn as nn | |||
from data.yelpLoader import yelpLoader | |||
from data.sstLoader import sst2Loader | |||
from data.IMDBLoader import IMDBLoader | |||
from model.char_cnn import CharacterLevelCNN | |||
from fastNLP.core.vocabulary import Vocabulary | |||
from fastNLP.models.cnn_text_classification import CNNText | |||
from fastNLP.modules.encoder.embedding import CNNCharEmbedding,StaticEmbedding,StackEmbedding,LSTMCharEmbedding | |||
from fastNLP import CrossEntropyLoss, AccuracyMetric | |||
from fastNLP.core.trainer import Trainer | |||
from torch.optim import SGD | |||
from torch.autograd import Variable | |||
import torch | |||
from fastNLP import BucketSampler | |||
##hyper | |||
#todo 这里加入fastnlp的记录 | |||
class Config(): | |||
model_dir_or_name="en-base-uncased" | |||
embedding_grad= False, | |||
bert_embedding_larers= '4,-2,-1' | |||
train_epoch= 50 | |||
num_classes=2 | |||
task= "IMDB" | |||
#yelp_p | |||
datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv", | |||
"test": "/remote-home/ygwang/yelp_polarity/test.csv"} | |||
#IMDB | |||
#datapath = {"train": "/remote-home/ygwang/IMDB_data/train.csv", | |||
# "test": "/remote-home/ygwang/IMDB_data/test.csv"} | |||
# sst | |||
# datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv", | |||
# "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"} | |||
lr=0.01 | |||
batch_size=128 | |||
model_size="large" | |||
number_of_characters=69 | |||
extra_characters='' | |||
max_length=1014 | |||
char_cnn_config={ | |||
"alphabet": { | |||
"en": { | |||
"lower": { | |||
"alphabet": "abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}", | |||
"number_of_characters": 69 | |||
}, | |||
"both": { | |||
"alphabet": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}", | |||
"number_of_characters": 95 | |||
} | |||
} | |||
}, | |||
"model_parameters": { | |||
"small": { | |||
"conv": [ | |||
#依次是channel,kennnel_size,maxpooling_size | |||
[256,7,3], | |||
[256,7,3], | |||
[256,3,-1], | |||
[256,3,-1], | |||
[256,3,-1], | |||
[256,3,3] | |||
], | |||
"fc": [1024,1024] | |||
}, | |||
"large":{ | |||
"conv":[ | |||
[1024, 7, 3], | |||
[1024, 7, 3], | |||
[1024, 3, -1], | |||
[1024, 3, -1], | |||
[1024, 3, -1], | |||
[1024, 3, 3] | |||
], | |||
"fc": [2048,2048] | |||
} | |||
}, | |||
"data": { | |||
"text_column": "SentimentText", | |||
"label_column": "Sentiment", | |||
"max_length": 1014, | |||
"num_of_classes": 2, | |||
"encoding": None, | |||
"chunksize": 50000, | |||
"max_rows": 100000, | |||
"preprocessing_steps": ["lower", "remove_hashtags", "remove_urls", "remove_user_mentions"] | |||
}, | |||
"training": { | |||
"batch_size": 128, | |||
"learning_rate": 0.01, | |||
"epochs": 10, | |||
"optimizer": "sgd" | |||
} | |||
} | |||
ops=Config | |||
##1.task相关信息:利用dataloader载入dataInfo | |||
#dataloader=sst2Loader() | |||
#dataloader=IMDBLoader() | |||
dataloader=yelpLoader(fine_grained=True) | |||
datainfo=dataloader.process(ops.datapath,char_level_op=True) | |||
char_vocab=ops.char_cnn_config["alphabet"]["en"]["lower"]["alphabet"] | |||
ops.number_of_characters=len(char_vocab) | |||
ops.embedding_dim=ops.number_of_characters | |||
#chartoindex | |||
def chartoindex(chars): | |||
max_seq_len=ops.max_length | |||
zero_index=len(char_vocab) | |||
char_index_list=[] | |||
for char in chars: | |||
if char in char_vocab: | |||
char_index_list.append(char_vocab.index(char)) | |||
else: | |||
#<unk>和<pad>均使用最后一个作为embbeding | |||
char_index_list.append(zero_index) | |||
if len(char_index_list) > max_seq_len: | |||
char_index_list = char_index_list[:max_seq_len] | |||
elif 0 < len(char_index_list) < max_seq_len: | |||
char_index_list = char_index_list+[zero_index]*(max_seq_len-len(char_index_list)) | |||
elif len(char_index_list) == 0: | |||
char_index_list=[zero_index]*max_seq_len | |||
return char_index_list | |||
for dataset in datainfo.datasets.values(): | |||
dataset.apply_field(chartoindex,field_name='chars',new_field_name='chars') | |||
datainfo.datasets['train'].set_input('chars') | |||
datainfo.datasets['test'].set_input('chars') | |||
datainfo.datasets['train'].set_target('target') | |||
datainfo.datasets['test'].set_target('target') | |||
##2. 定义/组装模型,这里可以随意,就如果是fastNLP封装好的,类似CNNText就直接用初始化调用就好了,这里只是给出一个伪框架表示占位,在这里建立符合fastNLP输入输出规范的model | |||
class ModelFactory(nn.Module): | |||
""" | |||
用于拼装embedding,encoder,decoder 以及设计forward过程 | |||
:param embedding: embbeding model | |||
:param encoder: encoder model | |||
:param decoder: decoder model | |||
""" | |||
def __int__(self,embedding,encoder,decoder,**kwargs): | |||
super(ModelFactory,self).__init__() | |||
self.embedding=embedding | |||
self.encoder=encoder | |||
self.decoder=decoder | |||
def forward(self,x): | |||
return {C.OUTPUT:None} | |||
## 2.或直接复用fastNLP的模型 | |||
#vocab=datainfo.vocabs['words'] | |||
vocab_label=datainfo.vocabs['target'] | |||
''' | |||
# emded_char=CNNCharEmbedding(vocab) | |||
# embed_word = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50', requires_grad=True) | |||
# embedding=StackEmbedding([emded_char, embed_word]) | |||
# cnn_char_embed = CNNCharEmbedding(vocab) | |||
# lstm_char_embed = LSTMCharEmbedding(vocab) | |||
# embedding = StackEmbedding([cnn_char_embed, lstm_char_embed]) | |||
''' | |||
#one-hot embedding | |||
embedding_weight= Variable(torch.zeros(len(char_vocab)+1, len(char_vocab))) | |||
for i in range(len(char_vocab)): | |||
embedding_weight[i][i]=1 | |||
embedding=nn.Embedding(num_embeddings=len(char_vocab)+1,embedding_dim=len(char_vocab),padding_idx=len(char_vocab),_weight=embedding_weight) | |||
for para in embedding.parameters(): | |||
para.requires_grad=False | |||
#CNNText太过于简单 | |||
#model=CNNText(init_embed=embedding, num_classes=ops.num_classes) | |||
model=CharacterLevelCNN(ops,embedding) | |||
## 3. 声明loss,metric,optimizer | |||
loss=CrossEntropyLoss | |||
metric=AccuracyMetric | |||
optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], lr=ops.lr) | |||
## 4.定义train方法 | |||
def train(model,datainfo,loss,metrics,optimizer,num_epochs=100): | |||
trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss(target='target'), | |||
metrics=[metrics(target='target')], dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, | |||
n_epochs=num_epochs) | |||
print(trainer.train()) | |||
if __name__=="__main__": | |||
#print(vocab_label) | |||
#print(datainfo.datasets["train"]) | |||
train(model,datainfo,loss,metric,optimizer,num_epochs=ops.train_epoch) | |||
@@ -0,0 +1,120 @@ | |||
# 首先需要加入以下的路径到环境变量,因为当前只对内部测试开放,所以需要手动申明一下路径 | |||
import torch.cuda | |||
from fastNLP.core.utils import cache_results | |||
from torch.optim import SGD | |||
from torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR | |||
from fastNLP.core.trainer import Trainer | |||
from fastNLP import CrossEntropyLoss, AccuracyMetric | |||
from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding | |||
from reproduction.text_classification.model.dpcnn import DPCNN | |||
from data.yelpLoader import yelpLoader | |||
from fastNLP.core.sampler import BucketSampler | |||
import torch.nn as nn | |||
from fastNLP.core import LRScheduler | |||
from fastNLP.core.const import Const as C | |||
from fastNLP.core.vocabulary import VocabularyOption | |||
from utils.util_init import set_rng_seeds | |||
import os | |||
os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' | |||
os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' | |||
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" | |||
# hyper | |||
class Config(): | |||
seed = 12345 | |||
model_dir_or_name = "dpcnn-yelp-p" | |||
embedding_grad = True | |||
train_epoch = 30 | |||
batch_size = 100 | |||
task = "yelp_p" | |||
#datadir = 'workdir/datasets/SST' | |||
datadir = 'workdir/datasets/yelp_polarity' | |||
# datadir = 'workdir/datasets/yelp_full' | |||
#datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"} | |||
datafile = {"train": "train.csv", "test": "test.csv"} | |||
lr = 1e-3 | |||
src_vocab_op = VocabularyOption(max_size=100000) | |||
embed_dropout = 0.3 | |||
cls_dropout = 0.1 | |||
weight_decay = 1e-5 | |||
def __init__(self): | |||
self.datadir = os.path.join(os.environ['HOME'], self.datadir) | |||
self.datapath = {k: os.path.join(self.datadir, v) | |||
for k, v in self.datafile.items()} | |||
ops = Config() | |||
set_rng_seeds(ops.seed) | |||
print('RNG SEED: {}'.format(ops.seed)) | |||
# 1.task相关信息:利用dataloader载入dataInfo | |||
#datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) | |||
@cache_results(ops.model_dir_or_name+'-data-cache') | |||
def load_data(): | |||
datainfo = yelpLoader(fine_grained=True, lower=True).process( | |||
paths=ops.datapath, train_ds=['train'], src_vocab_op=ops.src_vocab_op) | |||
for ds in datainfo.datasets.values(): | |||
ds.apply_field(len, C.INPUT, C.INPUT_LEN) | |||
ds.set_input(C.INPUT, C.INPUT_LEN) | |||
ds.set_target(C.TARGET) | |||
embedding = StaticEmbedding( | |||
datainfo.vocabs['words'], model_dir_or_name='en-glove-840b-300', requires_grad=ops.embedding_grad, | |||
normalize=False | |||
) | |||
return datainfo, embedding | |||
datainfo, embedding = load_data() | |||
# 2.或直接复用fastNLP的模型 | |||
# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) | |||
print(datainfo) | |||
print(datainfo.datasets['train'][0]) | |||
model = DPCNN(init_embed=embedding, num_cls=len(datainfo.vocabs[C.TARGET]), | |||
embed_dropout=ops.embed_dropout, cls_dropout=ops.cls_dropout) | |||
print(model) | |||
# 3. 声明loss,metric,optimizer | |||
loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) | |||
metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) | |||
optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], | |||
lr=ops.lr, momentum=0.9, weight_decay=ops.weight_decay) | |||
callbacks = [] | |||
# callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) | |||
callbacks.append( | |||
LRScheduler(LambdaLR(optimizer, lambda epoch: ops.lr if epoch < | |||
ops.train_epoch * 0.8 else ops.lr * 0.1)) | |||
) | |||
# callbacks.append( | |||
# FitlogCallback(data=datainfo.datasets, verbose=1) | |||
# ) | |||
device = 'cuda:0' if torch.cuda.is_available() else 'cpu' | |||
print(device) | |||
# 4.定义train方法 | |||
trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, | |||
sampler=BucketSampler(num_buckets=50, batch_size=ops.batch_size), | |||
metrics=[metric], | |||
dev_data=datainfo.datasets['test'], device=device, | |||
check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, | |||
n_epochs=ops.train_epoch, num_workers=4) | |||
if __name__ == "__main__": | |||
print(trainer.train()) |
@@ -0,0 +1,66 @@ | |||
# 首先需要加入以下的路径到环境变量,因为当前只对内部测试开放,所以需要手动申明一下路径 | |||
import os | |||
os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' | |||
os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' | |||
import torch.nn as nn | |||
from data.IMDBLoader import IMDBLoader | |||
from fastNLP.modules.encoder.embedding import StaticEmbedding | |||
from model.lstm import BiLSTMSentiment | |||
from fastNLP.core.const import Const as C | |||
from fastNLP import CrossEntropyLoss, AccuracyMetric | |||
from fastNLP import Trainer, Tester | |||
from torch.optim import Adam | |||
from fastNLP.io.model_io import ModelLoader, ModelSaver | |||
import argparse | |||
class Config(): | |||
train_epoch= 10 | |||
lr=0.001 | |||
num_classes=2 | |||
hidden_dim=256 | |||
num_layers=1 | |||
nfc=128 | |||
task_name = "IMDB" | |||
datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} | |||
save_model_path="./result_IMDB_test/" | |||
opt=Config() | |||
# load data | |||
dataloader=IMDBLoader() | |||
datainfo=dataloader.process(opt.datapath) | |||
# print(datainfo.datasets["train"]) | |||
# print(datainfo) | |||
# define model | |||
vocab=datainfo.vocabs['words'] | |||
embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) | |||
model=BiLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc) | |||
# define loss_function and metrics | |||
loss=CrossEntropyLoss() | |||
metrics=AccuracyMetric() | |||
optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) | |||
def train(datainfo, model, optimizer, loss, metrics, opt): | |||
trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, | |||
metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, | |||
n_epochs=opt.train_epoch, save_path=opt.save_model_path) | |||
trainer.train() | |||
if __name__ == "__main__": | |||
train(datainfo, model, optimizer, loss, metrics, opt) |
@@ -0,0 +1,68 @@ | |||
# 首先需要加入以下的路径到环境变量,因为当前只对内部测试开放,所以需要手动申明一下路径 | |||
import os | |||
os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' | |||
os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' | |||
import torch.nn as nn | |||
from data.IMDBLoader import IMDBLoader | |||
from fastNLP.modules.encoder.embedding import StaticEmbedding | |||
from model.lstm_self_attention import BiLSTM_SELF_ATTENTION | |||
from fastNLP.core.const import Const as C | |||
from fastNLP import CrossEntropyLoss, AccuracyMetric | |||
from fastNLP import Trainer, Tester | |||
from torch.optim import Adam | |||
from fastNLP.io.model_io import ModelLoader, ModelSaver | |||
import argparse | |||
class Config(): | |||
train_epoch= 10 | |||
lr=0.001 | |||
num_classes=2 | |||
hidden_dim=256 | |||
num_layers=1 | |||
attention_unit=256 | |||
attention_hops=1 | |||
nfc=128 | |||
task_name = "IMDB" | |||
datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"} | |||
save_model_path="./result_IMDB_test/" | |||
opt=Config() | |||
# load data | |||
dataloader=IMDBLoader() | |||
datainfo=dataloader.process(opt.datapath) | |||
# print(datainfo.datasets["train"]) | |||
# print(datainfo) | |||
# define model | |||
vocab=datainfo.vocabs['words'] | |||
embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) | |||
model=BiLSTM_SELF_ATTENTION(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, attention_unit=opt.attention_unit, attention_hops=opt.attention_hops, nfc=opt.nfc) | |||
# define loss_function and metrics | |||
loss=CrossEntropyLoss() | |||
metrics=AccuracyMetric() | |||
optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) | |||
def train(datainfo, model, optimizer, loss, metrics, opt): | |||
trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, | |||
metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, | |||
n_epochs=opt.train_epoch, save_path=opt.save_model_path) | |||
trainer.train() | |||
if __name__ == "__main__": | |||
train(datainfo, model, optimizer, loss, metrics, opt) |
@@ -0,0 +1,11 @@ | |||
import numpy | |||
import torch | |||
import random | |||
def set_rng_seeds(seed): | |||
random.seed(seed) | |||
numpy.random.seed(seed) | |||
torch.random.manual_seed(seed) | |||
torch.cuda.manual_seed_all(seed) | |||
# print('RNG_SEED {}'.format(seed)) |
@@ -59,4 +59,13 @@ def check_dataloader_paths(paths:Union[str, Dict[str, str]])->Dict[str, str]: | |||
else: | |||
raise TypeError(f"paths only supports str and dict. not {type(paths)}.") | |||
def get_tokenizer(): | |||
try: | |||
import spacy | |||
spacy.prefer_gpu() | |||
en = spacy.load('en') | |||
print('use spacy tokenizer') | |||
return lambda x: [w.text for w in en.tokenizer(x)] | |||
except Exception as e: | |||
print('use raw tokenizer') | |||
return lambda x: x.split() |