diff --git a/reproduction/text_classification/data/sstLoader.py b/reproduction/text_classification/data/sstLoader.py index bffb67fd..0d1b647c 100644 --- a/reproduction/text_classification/data/sstLoader.py +++ b/reproduction/text_classification/data/sstLoader.py @@ -1,13 +1,101 @@ -import csv from typing import Iterable -from fastNLP import DataSet, Instance, Vocabulary -from fastNLP.core.vocabulary import VocabularyOption -from fastNLP.io.base_loader import DataInfo,DataSetLoader -from fastNLP.io.embed_loader import EmbeddingOption -from fastNLP.io.file_reader import _read_json -from typing import Union, Dict -from reproduction.Star_transformer.datasets import EmbedLoader -from reproduction.utils import check_dataloader_paths +from nltk import Tree +from fastNLP.io.base_loader import DataInfo, DataSetLoader +from fastNLP.core.vocabulary import VocabularyOption, Vocabulary +from fastNLP import DataSet +from fastNLP import Instance +from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader + + +class SSTLoader(DataSetLoader): + URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' + DATA_DIR = 'sst/' + + """ + 别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader` + + 读取SST数据集, DataSet包含fields:: + + words: list(str) 需要分类的文本 + target: str 文本的标签 + + 数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip + + :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False`` + :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` + """ + + def __init__(self, subtree=False, fine_grained=False): + self.subtree = subtree + + tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral', + '3': 'positive', '4': 'very positive'} + if not fine_grained: + tag_v['0'] = tag_v['1'] + tag_v['4'] = tag_v['3'] + self.tag_v = tag_v + + def _load(self, path): + """ + + :param str path: 存储数据的路径 + :return: 一个 :class:`~fastNLP.DataSet` 类型的对象 + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + datas = [] + for l in f: + datas.extend([(s, self.tag_v[t]) + for s, t in self._get_one(l, self.subtree)]) + ds = DataSet() + for words, tag in datas: + ds.append(Instance(words=words, target=tag)) + return ds + + @staticmethod + def _get_one(data, subtree): + tree = Tree.fromstring(data) + if subtree: + return [(t.leaves(), t.label()) for t in tree.subtrees()] + return [(tree.leaves(), tree.label())] + + def process(self, + paths, + train_ds: Iterable[str] = None, + src_vocab_op: VocabularyOption = None, + tgt_vocab_op: VocabularyOption = None, + src_embed_op: EmbeddingOption = None): + input_name, target_name = 'words', 'target' + src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) + tgt_vocab = Vocabulary(unknown=None, padding=None) \ + if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) + + info = DataInfo(datasets=self.load(paths)) + _train_ds = [info.datasets[name] + for name in train_ds] if train_ds else info.datasets.values() + src_vocab.from_dataset(*_train_ds, field_name=input_name) + tgt_vocab.from_dataset(*_train_ds, field_name=target_name) + src_vocab.index_dataset( + *info.datasets.values(), + field_name=input_name, new_field_name=input_name) + tgt_vocab.index_dataset( + *info.datasets.values(), + field_name=target_name, new_field_name=target_name) + info.vocabs = { + input_name: src_vocab, + target_name: tgt_vocab + } + + if src_embed_op is not None: + src_embed_op.vocab = src_vocab + init_emb = EmbedLoader.load_with_vocab(**src_embed_op) + info.embeddings[input_name] = init_emb + + for name, dataset in info.datasets.items(): + dataset.set_input(input_name) + dataset.set_target(target_name) + + return info class sst2Loader(DataSetLoader): ''' diff --git a/reproduction/text_classification/data/yelpLoader.py b/reproduction/text_classification/data/yelpLoader.py index 0e65fb20..280e8be0 100644 --- a/reproduction/text_classification/data/yelpLoader.py +++ b/reproduction/text_classification/data/yelpLoader.py @@ -184,6 +184,12 @@ class yelpLoader(DataSetLoader): info.vocabs[target_name]=tgt_vocab + info.datasets['train'],info.datasets['dev']=info.datasets['train'].split(0.1, shuffle=False) + + for name, dataset in info.datasets.items(): + dataset.set_input("words") + dataset.set_target("target") + return info if __name__=="__main__": diff --git a/reproduction/text_classification/model/HAN.py b/reproduction/text_classification/model/HAN.py new file mode 100644 index 00000000..0902d1e4 --- /dev/null +++ b/reproduction/text_classification/model/HAN.py @@ -0,0 +1,109 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable +from fastNLP.modules.utils import get_embeddings +from fastNLP.core import Const as C + + +def pack_sequence(tensor_seq, padding_value=0.0): + if len(tensor_seq) <= 0: + return + length = [v.size(0) for v in tensor_seq] + max_len = max(length) + size = [len(tensor_seq), max_len] + size.extend(list(tensor_seq[0].size()[1:])) + ans = torch.Tensor(*size).fill_(padding_value) + if tensor_seq[0].data.is_cuda: + ans = ans.cuda() + ans = Variable(ans) + for i, v in enumerate(tensor_seq): + ans[i, :length[i], :] = v + return ans + + +class HANCLS(nn.Module): + def __init__(self, init_embed, num_cls): + super(HANCLS, self).__init__() + + self.embed = get_embeddings(init_embed) + self.han = HAN(input_size=300, + output_size=num_cls, + word_hidden_size=50, word_num_layers=1, word_context_size=100, + sent_hidden_size=50, sent_num_layers=1, sent_context_size=100 + ) + + def forward(self, input_sents): + # input_sents [B, num_sents, seq-len] dtype long + # target + B, num_sents, seq_len = input_sents.size() + input_sents = input_sents.view(-1, seq_len) # flat + words_embed = self.embed(input_sents) # should be [B*num-sent, seqlen , word-dim] + words_embed = words_embed.view(B, num_sents, seq_len, -1) # recover # [B, num-sent, seqlen , word-dim] + out = self.han(words_embed) + + return {C.OUTPUT: out} + + def predict(self, input_sents): + x = self.forward(input_sents)[C.OUTPUT] + return {C.OUTPUT: torch.argmax(x, 1)} + + +class HAN(nn.Module): + def __init__(self, input_size, output_size, + word_hidden_size, word_num_layers, word_context_size, + sent_hidden_size, sent_num_layers, sent_context_size): + super(HAN, self).__init__() + + self.word_layer = AttentionNet(input_size, + word_hidden_size, + word_num_layers, + word_context_size) + self.sent_layer = AttentionNet(2 * word_hidden_size, + sent_hidden_size, + sent_num_layers, + sent_context_size) + self.output_layer = nn.Linear(2 * sent_hidden_size, output_size) + self.softmax = nn.LogSoftmax(dim=1) + + def forward(self, batch_doc): + # input is a sequence of matrix + doc_vec_list = [] + for doc in batch_doc: + sent_mat = self.word_layer(doc) # doc's dim (num_sent, seq_len, word_dim) + doc_vec_list.append(sent_mat) # sent_mat's dim (num_sent, vec_dim) + doc_vec = self.sent_layer(pack_sequence(doc_vec_list)) + output = self.softmax(self.output_layer(doc_vec)) + return output + + +class AttentionNet(nn.Module): + def __init__(self, input_size, gru_hidden_size, gru_num_layers, context_vec_size): + super(AttentionNet, self).__init__() + + self.input_size = input_size + self.gru_hidden_size = gru_hidden_size + self.gru_num_layers = gru_num_layers + self.context_vec_size = context_vec_size + + # Encoder + self.gru = nn.GRU(input_size=input_size, + hidden_size=gru_hidden_size, + num_layers=gru_num_layers, + batch_first=True, + bidirectional=True) + # Attention + self.fc = nn.Linear(2 * gru_hidden_size, context_vec_size) + self.tanh = nn.Tanh() + self.softmax = nn.Softmax(dim=1) + # context vector + self.context_vec = nn.Parameter(torch.Tensor(context_vec_size, 1)) + self.context_vec.data.uniform_(-0.1, 0.1) + + def forward(self, inputs): + # GRU part + h_t, hidden = self.gru(inputs) # inputs's dim (batch_size, seq_len, word_dim) + u = self.tanh(self.fc(h_t)) + # Attention part + alpha = self.softmax(torch.matmul(u, self.context_vec)) # u's dim (batch_size, seq_len, context_vec_size) + output = torch.bmm(torch.transpose(h_t, 1, 2), alpha) # alpha's dim (batch_size, seq_len, 1) + return torch.squeeze(output, dim=2) # output's dim (batch_size, 2*hidden_size, 1) diff --git a/reproduction/text_classification/train_HAN.py b/reproduction/text_classification/train_HAN.py new file mode 100644 index 00000000..b1135342 --- /dev/null +++ b/reproduction/text_classification/train_HAN.py @@ -0,0 +1,109 @@ +# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 + +import os +import sys +sys.path.append('../../') +os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' +os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +from fastNLP.core.const import Const as C +from fastNLP.core import LRScheduler +import torch.nn as nn +from fastNLP.io.dataset_loader import SSTLoader +from reproduction.text_classification.data.yelpLoader import yelpLoader +from reproduction.text_classification.model.HAN import HANCLS +from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP.core.trainer import Trainer +from torch.optim import SGD +import torch.cuda +from torch.optim.lr_scheduler import CosineAnnealingLR + + +##hyper + +class Config(): + model_dir_or_name = "en-base-uncased" + embedding_grad = False, + train_epoch = 30 + batch_size = 100 + num_classes = 5 + task = "yelp" + #datadir = '/remote-home/lyli/fastNLP/yelp_polarity/' + datadir = '/remote-home/ygwang/yelp_polarity/' + datafile = {"train": "train.csv", "test": "test.csv"} + lr = 1e-3 + + def __init__(self): + self.datapath = {k: os.path.join(self.datadir, v) + for k, v in self.datafile.items()} + + +ops = Config() + +##1.task相关信息：利用dataloader载入dataInfo + +datainfo = yelpLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) +print(len(datainfo.datasets['train'])) +print(len(datainfo.datasets['test'])) + + +# post process +def make_sents(words): + sents = [words] + return sents + + +for dataset in datainfo.datasets.values(): + dataset.apply_field(make_sents, field_name='words', new_field_name='input_sents') + +datainfo = datainfo +datainfo.datasets['train'].set_input('input_sents') +datainfo.datasets['test'].set_input('input_sents') +datainfo.datasets['train'].set_target('target') +datainfo.datasets['test'].set_target('target') + +## 2.或直接复用fastNLP的模型 + +vocab = datainfo.vocabs['words'] +# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) +embedding = StaticEmbedding(vocab) + +print(len(vocab)) +print(len(datainfo.vocabs['target'])) + +# model = DPCNN(init_embed=embedding, num_cls=ops.num_classes) +model = HANCLS(init_embed=embedding, num_cls=ops.num_classes) + +## 3. 声明loss,metric,optimizer +loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET) +metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET) +optimizer = SGD([param for param in model.parameters() if param.requires_grad == True], + lr=ops.lr, momentum=0.9, weight_decay=0) + +callbacks = [] +callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) + +device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + +print(device) + +for ds in datainfo.datasets.values(): + ds.apply_field(len, C.INPUT, C.INPUT_LEN) + ds.set_input(C.INPUT, C.INPUT_LEN) + ds.set_target(C.TARGET) + + +## 4.定义train方法 +def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch): + trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=[metrics], dev_data=datainfo.datasets['test'], device=device, + check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, + n_epochs=num_epochs) + + print(trainer.train()) + + +if __name__ == "__main__": + train(model, datainfo, loss, metric, optimizer) diff --git a/reproduction/text_classification/train_char_cnn.py b/reproduction/text_classification/train_char_cnn.py index c2c983a4..050527fe 100644 --- a/reproduction/text_classification/train_char_cnn.py +++ b/reproduction/text_classification/train_char_cnn.py @@ -7,7 +7,6 @@ import sys sys.path.append('../..') from fastNLP.core.const import Const as C import torch.nn as nn -from fastNLP.io.dataset_loader import SSTLoader from data.yelpLoader import yelpLoader from data.sstLoader import sst2Loader from data.IMDBLoader import IMDBLoader @@ -107,9 +106,9 @@ ops=Config ##1.task相关信息：利用dataloader载入dataInfo -dataloader=sst2Loader() -dataloader=IMDBLoader() -#dataloader=yelpLoader(fine_grained=True) +#dataloader=sst2Loader() +#dataloader=IMDBLoader() +dataloader=yelpLoader(fine_grained=True) datainfo=dataloader.process(ops.datapath,char_level_op=True) char_vocab=ops.char_cnn_config["alphabet"]["en"]["lower"]["alphabet"] ops.number_of_characters=len(char_vocab)