Merge pull request #177 from SrWYG/dev0.5.0

[new] text_classification:charcnn,dpcnn,HAN,awd-lstm

fastNLP/io/data_loader/sst.py

@@ -5,10 +5,8 @@ from ..base_loader import DataInfo, DataSetLoader
 from ...core.vocabulary import VocabularyOption, Vocabulary
 from ...core.dataset import DataSet
 from ...core.instance import Instance
-from ..embed_loader import EmbeddingOption, EmbedLoader
+from ..utils import check_dataloader_paths, get_tokenizer
 
-spacy.prefer_gpu()
-sptk = spacy.load('en')
 
 class SSTLoader(DataSetLoader):
     URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip'
@@ -37,6 +35,7 @@ class SSTLoader(DataSetLoader):
             tag_v['0'] = tag_v['1']
             tag_v['4'] = tag_v['3']
         self.tag_v = tag_v
+        self.tokenizer = get_tokenizer()
 
     def _load(self, path):
         """
@@ -55,29 +54,37 @@ class SSTLoader(DataSetLoader):
             ds.append(Instance(words=words, target=tag))
         return ds
 
-    @staticmethod
-    def _get_one(data, subtree):
+    def _get_one(self, data, subtree):
         tree = Tree.fromstring(data)
         if subtree:
-            return [([x.text for x in sptk.tokenizer(' '.join(t.leaves()))], t.label()) for t in tree.subtrees() ]
-        return [([x.text for x in sptk.tokenizer(' '.join(tree.leaves()))], tree.label())]
+            return [([x.text for x in self.tokenizer(' '.join(t.leaves()))], t.label()) for t in tree.subtrees() ]
+        return [([x.text for x in self.tokenizer(' '.join(tree.leaves()))], tree.label())]
 
     def process(self,
-                paths,
-                train_ds: Iterable[str] = None,
+                paths, train_subtree=True,
                 src_vocab_op: VocabularyOption = None,
-                tgt_vocab_op: VocabularyOption = None,
-                src_embed_op: EmbeddingOption = None):
+                tgt_vocab_op: VocabularyOption = None,):
+        paths = check_dataloader_paths(paths)
         input_name, target_name = 'words', 'target'
         src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op)
         tgt_vocab = Vocabulary(unknown=None, padding=None) \
             if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op)
 
-        info = DataInfo(datasets=self.load(paths))
-        _train_ds = [info.datasets[name]
-                     for name in train_ds] if train_ds else info.datasets.values()
-        src_vocab.from_dataset(*_train_ds, field_name=input_name)
-        tgt_vocab.from_dataset(*_train_ds, field_name=target_name)
+        info = DataInfo()
+        origin_subtree = self.subtree
+        self.subtree = train_subtree
+        info.datasets['train'] = self._load(paths['train'])
+        self.subtree = origin_subtree
+        for n, p in paths.items():
+            if n != 'train':
+                info.datasets[n] = self._load(p)
+
+        src_vocab.from_dataset(
+            info.datasets['train'],
+            field_name=input_name,
+            no_create_entry_dataset=[ds for n, ds in info.datasets.items() if n != 'train'])
+        tgt_vocab.from_dataset(info.datasets['train'], field_name=target_name)
+
         src_vocab.index_dataset(
             *info.datasets.values(),
             field_name=input_name, new_field_name=input_name)
@@ -89,10 +96,5 @@ class SSTLoader(DataSetLoader):
             target_name: tgt_vocab
         }
 
-        if src_embed_op is not None:
-            src_embed_op.vocab = src_vocab
-            init_emb = EmbedLoader.load_with_vocab(**src_embed_op)
-            info.embeddings[input_name] = init_emb
-
         return info
 

fastNLP/io/utils.py

@@ -0,0 +1,69 @@
+import os
+
+from typing import Union, Dict
+
+
+def check_dataloader_paths(paths:Union[str, Dict[str, str]])->Dict[str, str]:
+    """
+    检查传入dataloader的文件的合法性。如果为合法路径，将返回至少包含'train'这个key的dict。类似于下面的结果
+    {
+        'train': '/some/path/to/', # 一定包含，建词表应该在这上面建立，剩下的其它文件应该只需要处理并index。
+        'test': 'xxx' # 可能有，也可能没有
+        ...
+    }
+    如果paths为不合法的，将直接进行raise相应的错误
+
+    :param paths: 路径. 可以为一个文件路径(则认为该文件就是train的文件); 可以为一个文件目录，将在该目录下寻找train(文件名
+        中包含train这个字段), test.txt, dev.txt; 可以为一个dict, 则key是用户自定义的某个文件的名称，value是这个文件的路径。
+    :return:
+    """
+    if isinstance(paths, str):
+        if os.path.isfile(paths):
+            return {'train': paths}
+        elif os.path.isdir(paths):
+            filenames = os.listdir(paths)
+            files = {}
+            for filename in filenames:
+                path_pair = None
+                if 'train' in filename:
+                    path_pair = ('train', filename)
+                if 'dev' in filename:
+                    if path_pair:
+                        raise Exception("File:{} in {} contains bot `{}` and `dev`.".format(filename, paths, path_pair[0]))
+                    path_pair = ('dev', filename)
+                if 'test' in filename:
+                    if path_pair:
+                        raise Exception("File:{} in {} contains bot `{}` and `test`.".format(filename, paths, path_pair[0]))
+                    path_pair = ('test', filename)
+                if path_pair:
+                    files[path_pair[0]] = os.path.join(paths, path_pair[1])
+            return files
+        else:
+            raise FileNotFoundError(f"{paths} is not a valid file path.")
+
+    elif isinstance(paths, dict):
+        if paths:
+            if 'train' not in paths:
+                raise KeyError("You have to include `train` in your dict.")
+            for key, value in paths.items():
+                if isinstance(key, str) and isinstance(value, str):
+                    if not os.path.isfile(value):
+                        raise TypeError(f"{value} is not a valid file.")
+                else:
+                    raise TypeError("All keys and values in paths should be str.")
+            return paths
+        else:
+            raise ValueError("Empty paths is not allowed.")
+    else:
+        raise TypeError(f"paths only supports str and dict. not {type(paths)}.")
+
+def get_tokenizer():
+    try:
+        import spacy
+        spacy.prefer_gpu()
+        en = spacy.load('en')
+        print('use spacy tokenizer')
+        return lambda x: [w.text for w in en.tokenizer(x)]
+    except Exception as e:
+        print('use raw tokenizer')
+        return lambda x: x.split()

fastNLP/modules/aggregator/attention.py

@@ -19,7 +19,7 @@ class DotAttention(nn.Module):
         补上文档
     """
     
-    def __init__(self, key_size, value_size, dropout=0):
+    def __init__(self, key_size, value_size, dropout=0.0):
         super(DotAttention, self).__init__()
         self.key_size = key_size
         self.value_size = value_size
@@ -37,7 +37,7 @@ class DotAttention(nn.Module):
         """
         output = torch.matmul(Q, K.transpose(1, 2)) / self.scale
         if mask_out is not None:
-            output.masked_fill_(mask_out, -1e8)
+            output.masked_fill_(mask_out, -1e18)
         output = self.softmax(output)
         output = self.drop(output)
         return torch.matmul(output, V)
@@ -67,9 +67,8 @@ class MultiHeadAttention(nn.Module):
         self.k_in = nn.Linear(input_size, in_size)
         self.v_in = nn.Linear(input_size, in_size)
         # follow the paper, do not apply dropout within dot-product
-        self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=0)
+        self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=dropout)
         self.out = nn.Linear(value_size * num_head, input_size)
-        self.drop = TimestepDropout(dropout)
         self.reset_parameters()
     
     def reset_parameters(self):
@@ -105,7 +104,7 @@ class MultiHeadAttention(nn.Module):
         
         # concat all heads, do output linear
         atte = atte.permute(1, 2, 0, 3).contiguous().view(batch, sq, -1)
-        output = self.drop(self.out(atte))
+        output = self.out(atte)
         return output
 
 

reproduction/Star_transformer/README.md

@@ -6,7 +6,7 @@ paper: [Star-Transformer](https://arxiv.org/abs/1902.09113)
 |Pos Tagging|CTB 9.0|-|ACC 92.31|
 |Pos Tagging|CONLL 2012|-|ACC 96.51|
 |Named Entity Recognition|CONLL 2012|-|F1 85.66|
-|Text Classification|SST|-|49.18|
+|Text Classification|SST|-|51.2|
 |Natural Language Inference|SNLI|-|83.76|
 
 ## Usage

reproduction/seqence_labelling/ner/model/dilated_cnn.py

@@ -0,0 +1,142 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from fastNLP.modules.decoder import ConditionalRandomField
+from fastNLP.modules.encoder import Embedding
+from fastNLP.core.utils import seq_len_to_mask
+from fastNLP.core.const import Const as C
+
+
+class IDCNN(nn.Module):
+    def __init__(self,
+                 init_embed,
+                 char_embed,
+                 num_cls,
+                 repeats, num_layers, num_filters, kernel_size,
+                 use_crf=False, use_projection=False, block_loss=False,
+                 input_dropout=0.3, hidden_dropout=0.2, inner_dropout=0.0):
+        super(IDCNN, self).__init__()
+        self.word_embeddings = Embedding(init_embed)
+
+        if char_embed is None:
+            self.char_embeddings = None
+            embedding_size = self.word_embeddings.embedding_dim
+        else:
+            self.char_embeddings = Embedding(char_embed)
+            embedding_size = self.word_embeddings.embedding_dim + \
+                self.char_embeddings.embedding_dim
+
+        self.conv0 = nn.Sequential(
+            nn.Conv1d(in_channels=embedding_size,
+                      out_channels=num_filters,
+                      kernel_size=kernel_size,
+                      stride=1, dilation=1,
+                      padding=kernel_size//2,
+                      bias=True),
+            nn.ReLU(),
+        )
+
+        block = []
+        for layer_i in range(num_layers):
+            dilated = 2 ** layer_i if layer_i+1 < num_layers else 1
+            block.append(nn.Conv1d(
+                in_channels=num_filters,
+                out_channels=num_filters,
+                kernel_size=kernel_size,
+                stride=1, dilation=dilated,
+                padding=(kernel_size//2) * dilated,
+                bias=True))
+            block.append(nn.ReLU())
+        self.block = nn.Sequential(*block)
+
+        if use_projection:
+            self.projection = nn.Sequential(
+                nn.Conv1d(
+                    in_channels=num_filters,
+                    out_channels=num_filters//2,
+                    kernel_size=1,
+                    bias=True),
+                nn.ReLU(),)
+            encode_dim = num_filters // 2
+        else:
+            self.projection = None
+            encode_dim = num_filters
+
+        self.input_drop = nn.Dropout(input_dropout)
+        self.hidden_drop = nn.Dropout(hidden_dropout)
+        self.inner_drop = nn.Dropout(inner_dropout)
+        self.repeats = repeats
+        self.out_fc = nn.Conv1d(
+            in_channels=encode_dim,
+            out_channels=num_cls,
+            kernel_size=1,
+            bias=True)
+        self.crf = ConditionalRandomField(
+            num_tags=num_cls) if use_crf else None
+        self.block_loss = block_loss
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        for m in self.modules():
+            if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)):
+                nn.init.xavier_normal_(m.weight, gain=1)
+                if m.bias is not None:
+                    nn.init.normal_(m.bias, mean=0, std=0.01)
+
+    def forward(self, words, seq_len, target=None, chars=None):
+        if self.char_embeddings is None:
+            x = self.word_embeddings(words)
+        else:
+            if chars is None:
+                raise ValueError('must provide chars for model with char embedding')
+            e1 = self.word_embeddings(words)
+            e2 = self.char_embeddings(chars)
+            x = torch.cat((e1, e2), dim=-1) # b,l,h
+        mask = seq_len_to_mask(seq_len)
+
+        x = x.transpose(1, 2) # b,h,l
+        last_output = self.conv0(x)
+        output = []
+        for repeat in range(self.repeats):
+            last_output = self.block(last_output)
+            hidden = self.projection(last_output) if self.projection is not None else last_output
+            output.append(self.out_fc(hidden))
+
+        def compute_loss(y, t, mask):
+            if self.crf is not None and target is not None:
+                loss = self.crf(y.transpose(1, 2), t, mask)
+            else:
+                t.masked_fill_(mask == 0, -100)
+                loss = F.cross_entropy(y, t, ignore_index=-100)
+            return loss
+
+        if target is not None:
+            if self.block_loss:
+                losses = [compute_loss(o, target, mask) for o in output]
+                loss = sum(losses)
+            else:
+                loss = compute_loss(output[-1], target, mask)
+        else:
+            loss = None
+
+        scores = output[-1]
+        if self.crf is not None:
+            pred, _ = self.crf.viterbi_decode(scores.transpose(1, 2), mask)
+        else:
+            pred = scores.max(1)[1] * mask.long()
+
+        return {
+            C.LOSS: loss,
+            C.OUTPUT: pred,
+        }
+
+    def predict(self, words, seq_len, chars=None):
+        res = self.forward(
+            words=words,
+            seq_len=seq_len,
+            chars=chars,
+            target=None
+        )[C.OUTPUT]
+        return {
+            C.OUTPUT: res
+        }

reproduction/seqence_labelling/ner/train_idcnn.py

@@ -0,0 +1,99 @@
+from reproduction.seqence_labelling.ner.data.OntoNoteLoader import OntoNoteNERDataLoader
+from fastNLP.core.callback import FitlogCallback, LRScheduler
+from fastNLP import GradientClipCallback
+from torch.optim.lr_scheduler import LambdaLR, CosineAnnealingLR
+from torch.optim import SGD, Adam
+from fastNLP import Const
+from fastNLP import RandomSampler, BucketSampler
+from fastNLP import SpanFPreRecMetric
+from fastNLP import Trainer
+from reproduction.seqence_labelling.ner.model.dilated_cnn import IDCNN
+from fastNLP.core.utils import Option
+from fastNLP.modules.encoder.embedding import CNNCharEmbedding, StaticEmbedding
+from fastNLP.core.utils import cache_results
+import sys
+import torch.cuda
+import os
+os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/'
+os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches'
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+
+encoding_type = 'bioes'
+
+
+def get_path(path):
+    return os.path.join(os.environ['HOME'], path)
+
+data_path = get_path('workdir/datasets/ontonotes-v4')
+
+ops = Option(
+    batch_size=128,
+    num_epochs=100,
+    lr=3e-4,
+    repeats=3,
+    num_layers=3,
+    num_filters=400,
+    use_crf=True,
+    gradient_clip=5,
+)
+
+@cache_results('ontonotes-cache')
+def load_data():
+
+    data = OntoNoteNERDataLoader(encoding_type=encoding_type).process(data_path,
+                                                                  lower=True)
+
+    # char_embed = CNNCharEmbedding(vocab=data.vocabs['cap_words'], embed_size=30, char_emb_size=30, filter_nums=[30],
+    #                               kernel_sizes=[3])
+
+    word_embed = StaticEmbedding(vocab=data.vocabs[Const.INPUT],
+                                 model_dir_or_name='en-glove-840b-300',
+                                 requires_grad=True)
+    return data, [word_embed]
+
+data, embeds = load_data()
+print(data.datasets['train'][0])
+print(list(data.vocabs.keys()))
+
+for ds in data.datasets.values():
+    ds.rename_field('cap_words', 'chars')
+    ds.set_input('chars')
+
+word_embed = embeds[0]
+char_embed = CNNCharEmbedding(data.vocabs['cap_words'])
+# for ds in data.datasets:
+#     ds.rename_field('')
+
+print(data.vocabs[Const.TARGET].word2idx)
+
+model = IDCNN(init_embed=word_embed,
+              char_embed=char_embed,
+              num_cls=len(data.vocabs[Const.TARGET]),
+              repeats=ops.repeats,
+              num_layers=ops.num_layers,
+              num_filters=ops.num_filters,
+              kernel_size=3,
+              use_crf=ops.use_crf, use_projection=True,
+              block_loss=True,
+              input_dropout=0.33, hidden_dropout=0.2, inner_dropout=0.2)
+
+print(model)
+
+callbacks = [GradientClipCallback(clip_value=ops.gradient_clip, clip_type='norm'),]
+
+optimizer = Adam(model.parameters(), lr=ops.lr, weight_decay=0)
+# scheduler = LRScheduler(LambdaLR(optimizer, lr_lambda=lambda epoch: 1 / (1 + 0.05 * epoch)))
+# callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 15)))
+# optimizer = SWATS(model.parameters(), verbose=True)
+# optimizer = Adam(model.parameters(), lr=0.005)
+
+device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+
+trainer = Trainer(train_data=data.datasets['train'], model=model, optimizer=optimizer,
+                  sampler=BucketSampler(num_buckets=50, batch_size=ops.batch_size),
+                  device=device, dev_data=data.datasets['dev'], batch_size=ops.batch_size,
+                  metrics=SpanFPreRecMetric(
+                      tag_vocab=data.vocabs[Const.TARGET], encoding_type=encoding_type),
+                  check_code_level=-1,
+                  callbacks=callbacks, num_workers=2, n_epochs=ops.num_epochs)
+trainer.train()

reproduction/text_classification/README.md

@@ -0,0 +1,26 @@
+# text_classification任务模型复现
+这里使用fastNLP复现以下模型：
+
+char_cnn :论文链接[Character-level Convolutional Networks for Text Classification](https://arxiv.org/pdf/1509.01626v3.pdf)
+
+dpcnn:论文链接[Deep Pyramid Convolutional Neural Networks for TextCategorization](https://ai.tencent.com/ailab/media/publications/ACL3-Brady.pdf)
+
+HAN:论文链接[Hierarchical Attention Networks for Document Classification](https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf)
+
+LSTM+self_attention:论文链接[A Structured Self-attentive Sentence Embedding](<https://arxiv.org/pdf/1703.03130.pdf>)
+
+AWD-LSTM:论文链接[Regularizing and Optimizing LSTM Language Models](<https://arxiv.org/pdf/1708.02182.pdf>)
+
+# 数据集及复现结果汇总
+
+使用fastNLP复现的结果vs论文汇报结果(/前为fastNLP实现，后面为论文报道,-表示论文没有在该数据集上列出结果)
+
+model name | yelp_p | yelp_f | sst-2|IMDB
+:---: | :---: | :---: | :---: |----- 
+char_cnn | 93.80/95.12 | - | - |- 
+dpcnn | 95.50/97.36 | - | - |- 
+HAN |- | - | - |-
+LSTM| 95.74/- |- |- |88.52/-
+AWD-LSTM| 95.96/- |- |- |88.91/-
+LSTM+self_attention| 96.34/- | - | - |89.53/-
+

reproduction/text_classification/data/IMDBLoader.py

@@ -0,0 +1,110 @@
+from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader
+from fastNLP.core.vocabulary import VocabularyOption
+from fastNLP.io.base_loader import DataSetLoader, DataInfo
+from typing import Union, Dict, List, Iterator
+from fastNLP import DataSet
+from fastNLP import Instance
+from fastNLP import Vocabulary
+from fastNLP import Const
+# from reproduction.utils import check_dataloader_paths
+from functools import partial
+
+
+class IMDBLoader(DataSetLoader):
+    """
+    读取IMDB数据集，DataSet包含以下fields:
+
+        words: list(str), 需要分类的文本
+        target: str, 文本的标签
+
+
+    """
+
+    def __init__(self):
+        super(IMDBLoader, self).__init__()
+
+    def _load(self, path):
+        dataset = DataSet()
+        with open(path, 'r', encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                parts = line.split('\t')
+                target = parts[0]
+                words = parts[1].lower().split()
+                dataset.append(Instance(words=words, target=target))
+
+        if len(dataset)==0:
+            raise RuntimeError(f"{path} has no valid data.")
+
+        return dataset
+    
+    def process(self,
+                paths: Union[str, Dict[str, str]],
+                src_vocab_opt: VocabularyOption = None,
+                tgt_vocab_opt: VocabularyOption = None,
+                src_embed_opt: EmbeddingOption = None,
+                char_level_op=False):
+      
+        datasets = {}
+        info = DataInfo()
+        for name, path in paths.items():
+            dataset = self.load(path)
+            datasets[name] = dataset
+            
+        def wordtochar(words):
+            chars = []
+            for word in words:
+                word = word.lower()
+                for char in word:
+                    chars.append(char)
+            return chars
+
+        if char_level_op:
+            for dataset in datasets.values():
+                dataset.apply_field(wordtochar, field_name="words", new_field_name='chars')
+
+        datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False)
+
+        src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt)
+        src_vocab.from_dataset(datasets['train'], field_name='words')
+
+        src_vocab.index_dataset(*datasets.values(), field_name='words')
+
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt)
+        tgt_vocab.from_dataset(datasets['train'], field_name='target')
+        tgt_vocab.index_dataset(*datasets.values(), field_name='target')
+
+        info.vocabs = {
+            "words": src_vocab,
+            "target": tgt_vocab
+        }
+
+        info.datasets = datasets
+
+        if src_embed_opt is not None:
+            embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab)
+            info.embeddings['words'] = embed
+
+        for name, dataset in info.datasets.items():
+            dataset.set_input("words")
+            dataset.set_target("target")
+
+        return info
+
+
+
+if __name__=="__main__":
+    datapath = {"train": "/remote-home/ygwang/IMDB_data/train.csv",
+                "test": "/remote-home/ygwang/IMDB_data/test.csv"}
+    datainfo=IMDBLoader().process(datapath,char_level_op=True)
+    #print(datainfo.datasets["train"])
+    len_count = 0
+    for instance in datainfo.datasets["train"]:
+        len_count += len(instance["chars"])
+
+    ave_len = len_count / len(datainfo.datasets["train"])
+    print(ave_len)
+

reproduction/text_classification/data/MTL16Loader.py

@@ -32,7 +32,7 @@ class MTL16Loader(DataSetLoader):
                     continue
                 parts = line.split('\t')
                 target = parts[0]
-                words = parts[1].split()
+                words = parts[1].lower().split()
                 dataset.append(Instance(words=words, target=target))
         if len(dataset)==0:
             raise RuntimeError(f"{path} has no valid data.")
@@ -72,4 +72,8 @@ class MTL16Loader(DataSetLoader):
             embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab)
             info.embeddings['words'] = embed
 
+        for name, dataset in info.datasets.items():
+            dataset.set_input("words")
+            dataset.set_target("target")
+
         return info

reproduction/text_classification/data/SSTLoader.py

@@ -0,0 +1,187 @@
+from typing import Iterable
+from nltk import Tree
+from fastNLP.io.base_loader import DataInfo, DataSetLoader
+from fastNLP.core.vocabulary import VocabularyOption, Vocabulary
+from fastNLP import DataSet
+from fastNLP import Instance
+from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader
+import csv
+from typing import Union, Dict
+
+class SSTLoader(DataSetLoader):
+    URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip'
+    DATA_DIR = 'sst/'
+
+    """
+    别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader`
+
+    读取SST数据集, DataSet包含fields::
+
+        words: list(str) 需要分类的文本
+        target: str 文本的标签
+
+    数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip
+
+    :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False``
+    :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False``
+    """
+
+    def __init__(self, subtree=False, fine_grained=False):
+        self.subtree = subtree
+
+        tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral',
+                 '3': 'positive', '4': 'very positive'}
+        if not fine_grained:
+            tag_v['0'] = tag_v['1']
+            tag_v['4'] = tag_v['3']
+        self.tag_v = tag_v
+
+    def _load(self, path):
+        """
+
+        :param str path: 存储数据的路径
+        :return: 一个 :class:`~fastNLP.DataSet` 类型的对象
+        """
+        datalist = []
+        with open(path, 'r', encoding='utf-8') as f:
+            datas = []
+            for l in f:
+                datas.extend([(s, self.tag_v[t])
+                              for s, t in self._get_one(l, self.subtree)])
+        ds = DataSet()
+        for words, tag in datas:
+            ds.append(Instance(words=words, target=tag))
+        return ds
+
+    @staticmethod
+    def _get_one(data, subtree):
+        tree = Tree.fromstring(data)
+        if subtree:
+            return [(t.leaves(), t.label()) for t in tree.subtrees()]
+        return [(tree.leaves(), tree.label())]
+
+    def process(self,
+                paths,
+                train_ds: Iterable[str] = None,
+                src_vocab_op: VocabularyOption = None,
+                tgt_vocab_op: VocabularyOption = None,
+                src_embed_op: EmbeddingOption = None):
+        input_name, target_name = 'words', 'target'
+        src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op)
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op)
+
+        info = DataInfo(datasets=self.load(paths))
+        _train_ds = [info.datasets[name]
+                     for name in train_ds] if train_ds else info.datasets.values()
+        src_vocab.from_dataset(*_train_ds, field_name=input_name)
+        tgt_vocab.from_dataset(*_train_ds, field_name=target_name)
+        src_vocab.index_dataset(
+            *info.datasets.values(),
+            field_name=input_name, new_field_name=input_name)
+        tgt_vocab.index_dataset(
+            *info.datasets.values(),
+            field_name=target_name, new_field_name=target_name)
+        info.vocabs = {
+            input_name: src_vocab,
+            target_name: tgt_vocab
+        }
+
+        if src_embed_op is not None:
+            src_embed_op.vocab = src_vocab
+            init_emb = EmbedLoader.load_with_vocab(**src_embed_op)
+            info.embeddings[input_name] = init_emb
+
+        for name, dataset in info.datasets.items():
+            dataset.set_input(input_name)
+            dataset.set_target(target_name)
+
+        return info
+
+class sst2Loader(DataSetLoader):
+    '''
+    数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',
+    '''
+    def __init__(self):
+        super(sst2Loader, self).__init__()
+
+    def _load(self, path: str) -> DataSet:
+        ds = DataSet()
+        all_count=0
+        csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t')
+        skip_row = 0
+        for idx,row in enumerate(csv_reader):
+            if idx<=skip_row:
+                continue
+            target = row[1]
+            words = row[0].split()
+            ds.append(Instance(words=words,target=target))
+            all_count+=1
+        print("all count:", all_count)
+        return ds
+
+    def process(self,
+                paths: Union[str, Dict[str, str]],
+                src_vocab_opt: VocabularyOption = None,
+                tgt_vocab_opt: VocabularyOption = None,
+                src_embed_opt: EmbeddingOption = None,
+                char_level_op=False):
+
+        paths = check_dataloader_paths(paths)
+        datasets = {}
+        info = DataInfo()
+        for name, path in paths.items():
+            dataset = self.load(path)
+            datasets[name] = dataset
+
+        def wordtochar(words):
+            chars=[]
+            for word in words:
+                word=word.lower()
+                for char in word:
+                    chars.append(char)
+            return chars
+
+        input_name, target_name = 'words', 'target'
+        info.vocabs={}
+
+        # 就分隔为char形式
+        if char_level_op:
+            for dataset in datasets.values():
+                dataset.apply_field(wordtochar, field_name="words", new_field_name='chars')
+
+        src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt)
+        src_vocab.from_dataset(datasets['train'], field_name='words')
+        src_vocab.index_dataset(*datasets.values(), field_name='words')
+
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt)
+        tgt_vocab.from_dataset(datasets['train'], field_name='target')
+        tgt_vocab.index_dataset(*datasets.values(), field_name='target')
+
+
+        info.vocabs = {
+            "words": src_vocab,
+            "target": tgt_vocab
+        }
+
+        info.datasets = datasets
+
+
+        if src_embed_opt is not None:
+            embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab)
+            info.embeddings['words'] = embed
+
+        return info
+
+if __name__=="__main__":
+    datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv",
+                "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"}
+    datainfo=sst2Loader().process(datapath,char_level_op=True)
+    #print(datainfo.datasets["train"])
+    len_count = 0
+    for instance in datainfo.datasets["train"]:
+        len_count += len(instance["chars"])
+
+    ave_len = len_count / len(datainfo.datasets["train"])
+    print(ave_len)

reproduction/text_classification/data/sstLoader.py

@@ -0,0 +1,187 @@
+from typing import Iterable
+from nltk import Tree
+from fastNLP.io.base_loader import DataInfo, DataSetLoader
+from fastNLP.core.vocabulary import VocabularyOption, Vocabulary
+from fastNLP import DataSet
+from fastNLP import Instance
+from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader
+import csv
+from typing import Union, Dict
+
+class SSTLoader(DataSetLoader):
+    URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip'
+    DATA_DIR = 'sst/'
+
+    """
+    别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader`
+
+    读取SST数据集, DataSet包含fields::
+
+        words: list(str) 需要分类的文本
+        target: str 文本的标签
+
+    数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip
+
+    :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False``
+    :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False``
+    """
+
+    def __init__(self, subtree=False, fine_grained=False):
+        self.subtree = subtree
+
+        tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral',
+                 '3': 'positive', '4': 'very positive'}
+        if not fine_grained:
+            tag_v['0'] = tag_v['1']
+            tag_v['4'] = tag_v['3']
+        self.tag_v = tag_v
+
+    def _load(self, path):
+        """
+
+        :param str path: 存储数据的路径
+        :return: 一个 :class:`~fastNLP.DataSet` 类型的对象
+        """
+        datalist = []
+        with open(path, 'r', encoding='utf-8') as f:
+            datas = []
+            for l in f:
+                datas.extend([(s, self.tag_v[t])
+                              for s, t in self._get_one(l, self.subtree)])
+        ds = DataSet()
+        for words, tag in datas:
+            ds.append(Instance(words=words, target=tag))
+        return ds
+
+    @staticmethod
+    def _get_one(data, subtree):
+        tree = Tree.fromstring(data)
+        if subtree:
+            return [(t.leaves(), t.label()) for t in tree.subtrees()]
+        return [(tree.leaves(), tree.label())]
+
+    def process(self,
+                paths,
+                train_ds: Iterable[str] = None,
+                src_vocab_op: VocabularyOption = None,
+                tgt_vocab_op: VocabularyOption = None,
+                src_embed_op: EmbeddingOption = None):
+        input_name, target_name = 'words', 'target'
+        src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op)
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op)
+
+        info = DataInfo(datasets=self.load(paths))
+        _train_ds = [info.datasets[name]
+                     for name in train_ds] if train_ds else info.datasets.values()
+        src_vocab.from_dataset(*_train_ds, field_name=input_name)
+        tgt_vocab.from_dataset(*_train_ds, field_name=target_name)
+        src_vocab.index_dataset(
+            *info.datasets.values(),
+            field_name=input_name, new_field_name=input_name)
+        tgt_vocab.index_dataset(
+            *info.datasets.values(),
+            field_name=target_name, new_field_name=target_name)
+        info.vocabs = {
+            input_name: src_vocab,
+            target_name: tgt_vocab
+        }
+
+        if src_embed_op is not None:
+            src_embed_op.vocab = src_vocab
+            init_emb = EmbedLoader.load_with_vocab(**src_embed_op)
+            info.embeddings[input_name] = init_emb
+
+        for name, dataset in info.datasets.items():
+            dataset.set_input(input_name)
+            dataset.set_target(target_name)
+
+        return info
+
+class sst2Loader(DataSetLoader):
+    '''
+    数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',
+    '''
+    def __init__(self):
+        super(sst2Loader, self).__init__()
+
+    def _load(self, path: str) -> DataSet:
+        ds = DataSet()
+        all_count=0
+        csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t')
+        skip_row = 0
+        for idx,row in enumerate(csv_reader):
+            if idx<=skip_row:
+                continue
+            target = row[1]
+            words = row[0].split()
+            ds.append(Instance(words=words,target=target))
+            all_count+=1
+        print("all count:", all_count)
+        return ds
+
+    def process(self,
+                paths: Union[str, Dict[str, str]],
+                src_vocab_opt: VocabularyOption = None,
+                tgt_vocab_opt: VocabularyOption = None,
+                src_embed_opt: EmbeddingOption = None,
+                char_level_op=False):
+
+        paths = check_dataloader_paths(paths)
+        datasets = {}
+        info = DataInfo()
+        for name, path in paths.items():
+            dataset = self.load(path)
+            datasets[name] = dataset
+
+        def wordtochar(words):
+            chars=[]
+            for word in words:
+                word=word.lower()
+                for char in word:
+                    chars.append(char)
+            return chars
+
+        input_name, target_name = 'words', 'target'
+        info.vocabs={}
+
+        # 就分隔为char形式
+        if char_level_op:
+            for dataset in datasets.values():
+                dataset.apply_field(wordtochar, field_name="words", new_field_name='chars')
+
+        src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt)
+        src_vocab.from_dataset(datasets['train'], field_name='words')
+        src_vocab.index_dataset(*datasets.values(), field_name='words')
+
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt)
+        tgt_vocab.from_dataset(datasets['train'], field_name='target')
+        tgt_vocab.index_dataset(*datasets.values(), field_name='target')
+
+
+        info.vocabs = {
+            "words": src_vocab,
+            "target": tgt_vocab
+        }
+
+        info.datasets = datasets
+
+
+        if src_embed_opt is not None:
+            embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab)
+            info.embeddings['words'] = embed
+
+        return info
+
+if __name__=="__main__":
+    datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv",
+                "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"}
+    datainfo=sst2Loader().process(datapath,char_level_op=True)
+    #print(datainfo.datasets["train"])
+    len_count = 0
+    for instance in datainfo.datasets["train"]:
+        len_count += len(instance["chars"])
+
+    ave_len = len_count / len(datainfo.datasets["train"])
+    print(ave_len)

reproduction/text_classification/data/yelpLoader.py

@@ -1,18 +1,64 @@
 import ast
+import csv
+from typing import Iterable
 from fastNLP import DataSet, Instance, Vocabulary
 from fastNLP.core.vocabulary import VocabularyOption
 from fastNLP.io import JsonLoader
-from fastNLP.io.base_loader import DataInfo
+from fastNLP.io.base_loader import DataInfo,DataSetLoader
 from fastNLP.io.embed_loader import EmbeddingOption
 from fastNLP.io.file_reader import _read_json
 from typing import Union, Dict
-from reproduction.Star_transformer.datasets import EmbedLoader
-from reproduction.utils import check_dataloader_paths
+from reproduction.utils import check_dataloader_paths, get_tokenizer
+
+def clean_str(sentence, tokenizer, char_lower=False):
+    """
+    heavily borrowed from github
+    https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb
+    :param sentence:  is a str
+    :return:
+    """
+    if char_lower:
+        sentence = sentence.lower()
+    import re
+    nonalpnum = re.compile('[^0-9a-zA-Z?!\']+')
+    words = tokenizer(sentence)
+    words_collection = []
+    for word in words:
+        if word in ['-lrb-', '-rrb-', '<sssss>', '-r', '-l', 'b-']:
+            continue
+        tt = nonalpnum.split(word)
+        t = ''.join(tt)
+        if t != '':
+            words_collection.append(t)
+
+    return words_collection
 
 
-class yelpLoader(JsonLoader):
+class yelpLoader(DataSetLoader):
     
     """
+    读取Yelp_full/Yelp_polarity数据集, DataSet包含fields:
+        words: list(str), 需要分类的文本
+        target: str, 文本的标签
+        chars:list(str),未index的字符列表
+
+    数据集：yelp_full/yelp_polarity
+    :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False``
+    """
+    
+    def __init__(self, fine_grained=False,lower=False):
+        super(yelpLoader, self).__init__()
+        tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral',
+                 '4.0': 'positive', '5.0': 'very positive'}
+        if not fine_grained:
+            tag_v['1.0'] = tag_v['2.0']
+            tag_v['5.0'] = tag_v['4.0']
+        self.fine_grained = fine_grained
+        self.tag_v = tag_v
+        self.lower = lower
+        self.tokenizer = get_tokenizer()
+
+    '''
     读取Yelp数据集, DataSet包含fields:
     
         review_id: str, 22 character unique review id
@@ -27,20 +73,8 @@ class yelpLoader(JsonLoader):
     
     数据来源: https://www.yelp.com/dataset/download
     
-    :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False``
-    """
-    
-    def __init__(self, fine_grained=False):
-        super(yelpLoader, self).__init__()
-        tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral',
-            '4.0': 'positive', '5.0': 'very positive'}
-        if not fine_grained:
-            tag_v['1.0'] = tag_v['2.0']
-            tag_v['5.0'] = tag_v['4.0']
-        self.fine_grained = fine_grained
-        self.tag_v = tag_v
-    
-    def _load(self, path):
+
+    def _load_json(self, path):
         ds = DataSet()
         for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna):
             d = ast.literal_eval(d)
@@ -48,21 +82,116 @@ class yelpLoader(JsonLoader):
             d["target"] = self.tag_v[str(d.pop("stars"))]
             ds.append(Instance(**d))
         return ds
+    
+    def _load_yelp2015_broken(self,path):
+        ds = DataSet()
+        with open (path,encoding='ISO 8859-1') as f:
+            row=f.readline()
+            all_count=0
+            exp_count=0
+            while row:
+                row=row.split("\t\t")
+                all_count+=1
+                if len(row)>=3:
+                    words=row[-1].split()
+                    try:
+                        target=self.tag_v[str(row[-2])+".0"]
+                        ds.append(Instance(words=words, target=target))
+                    except KeyError:
+                        exp_count+=1
+                else:
+                    exp_count+=1
+                row = f.readline()
+            print("error sample count:",exp_count)
+            print("all count:",all_count)
+        return ds
+    '''
+
+    def _load(self, path):
+        ds = DataSet()
+        csv_reader=csv.reader(open(path,encoding='utf-8'))
+        all_count=0
+        real_count=0
+        for row in csv_reader:
+            all_count+=1
+            if len(row)==2:
+                target=self.tag_v[row[0]+".0"]
+                words = clean_str(row[1], self.tokenizer, self.lower)
+                if len(words)!=0:
+                    ds.append(Instance(words=words,target=target))
+                    real_count += 1
+        print("all count:", all_count)
+        print("real count:", real_count)
+        return ds
+
 
-    def process(self, paths: Union[str, Dict[str, str]], vocab_opt: VocabularyOption = None,
-                embed_opt: EmbeddingOption = None):
+
+    def process(self, paths: Union[str, Dict[str, str]],
+                train_ds: Iterable[str] = None,
+                src_vocab_op: VocabularyOption = None,
+                tgt_vocab_op: VocabularyOption = None,
+                embed_opt: EmbeddingOption = None,
+                char_level_op=False):
         paths = check_dataloader_paths(paths)
         datasets = {}
-        info = DataInfo()
-        vocab = Vocabulary(min_freq=2) if vocab_opt is None else Vocabulary(**vocab_opt)
-        for name, path in paths.items():
-            dataset = self.load(path)
-            datasets[name] = dataset
-            vocab.from_dataset(dataset, field_name="words")
-        info.vocabs = vocab
-        info.datasets = datasets
-        if embed_opt is not None:
-            embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab)
-            info.embeddings['words'] = embed
+        info = DataInfo(datasets=self.load(paths))
+        src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op)
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op)
+        _train_ds = [info.datasets[name]
+                     for name in train_ds] if train_ds else info.datasets.values()
+
+
+        def wordtochar(words):
+
+            chars=[]
+            for word in words:
+                word=word.lower()
+                for char in word:
+                    chars.append(char)
+            return chars
+
+        input_name, target_name = 'words', 'target'
+        info.vocabs={}
+        #就分隔为char形式
+        if char_level_op:
+            for dataset in info.datasets.values():
+                dataset.apply_field(wordtochar, field_name="words",new_field_name='chars')
+        # if embed_opt is not None:
+        #     embed = EmbedLoader.load_with_vocab(**embed_opt, vocab=vocab)
+        #     info.embeddings['words'] = embed
+        else:
+            src_vocab.from_dataset(*_train_ds, field_name=input_name)
+            src_vocab.index_dataset(*info.datasets.values(),field_name=input_name, new_field_name=input_name)
+            info.vocabs[input_name]=src_vocab
+
+        tgt_vocab.from_dataset(*_train_ds, field_name=target_name)
+        tgt_vocab.index_dataset(
+            *info.datasets.values(),
+            field_name=target_name, new_field_name=target_name)
+
+        info.vocabs[target_name]=tgt_vocab
+
+        info.datasets['train'],info.datasets['dev']=info.datasets['train'].split(0.1, shuffle=False)
+
+        for name, dataset in info.datasets.items():
+            dataset.set_input("words")
+            dataset.set_target("target")
+
         return info
 
+if __name__=="__main__":
+    testloader=yelpLoader()
+    # datapath = {"train": "/remote-home/ygwang/yelp_full/train.csv",
+    #             "test": "/remote-home/ygwang/yelp_full/test.csv"}
+    #datapath={"train": "/remote-home/ygwang/yelp_full/test.csv"}
+    datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv",
+                "test": "/remote-home/ygwang/yelp_polarity/test.csv"}
+    datainfo=testloader.process(datapath,char_level_op=True)
+
+    len_count=0
+    for instance in datainfo.datasets["train"]:
+        len_count+=len(instance["chars"])
+
+    ave_len=len_count/len(datainfo.datasets["train"])
+    print(ave_len)

reproduction/text_classification/model/HAN.py

@@ -0,0 +1,109 @@
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from fastNLP.modules.utils import get_embeddings
+from fastNLP.core import Const as C
+
+
+def pack_sequence(tensor_seq, padding_value=0.0):
+    if len(tensor_seq) <= 0:
+        return
+    length = [v.size(0) for v in tensor_seq]
+    max_len = max(length)
+    size = [len(tensor_seq), max_len]
+    size.extend(list(tensor_seq[0].size()[1:]))
+    ans = torch.Tensor(*size).fill_(padding_value)
+    if tensor_seq[0].data.is_cuda:
+        ans = ans.cuda()
+    ans = Variable(ans)
+    for i, v in enumerate(tensor_seq):
+        ans[i, :length[i], :] = v
+    return ans
+
+
+class HANCLS(nn.Module):
+    def __init__(self, init_embed, num_cls):
+        super(HANCLS, self).__init__()
+
+        self.embed = get_embeddings(init_embed)
+        self.han = HAN(input_size=300,
+                       output_size=num_cls,
+                       word_hidden_size=50, word_num_layers=1, word_context_size=100,
+                       sent_hidden_size=50, sent_num_layers=1, sent_context_size=100
+                       )
+
+    def forward(self, input_sents):
+        # input_sents [B, num_sents, seq-len] dtype long
+        # target
+        B, num_sents, seq_len = input_sents.size()
+        input_sents = input_sents.view(-1, seq_len)  # flat
+        words_embed = self.embed(input_sents)  # should be [B*num-sent, seqlen , word-dim]
+        words_embed = words_embed.view(B, num_sents, seq_len, -1)  # recover # [B, num-sent, seqlen , word-dim]
+        out = self.han(words_embed)
+
+        return {C.OUTPUT: out}
+
+    def predict(self, input_sents):
+        x = self.forward(input_sents)[C.OUTPUT]
+        return {C.OUTPUT: torch.argmax(x, 1)}
+
+
+class HAN(nn.Module):
+    def __init__(self, input_size, output_size,
+                 word_hidden_size, word_num_layers, word_context_size,
+                 sent_hidden_size, sent_num_layers, sent_context_size):
+        super(HAN, self).__init__()
+
+        self.word_layer = AttentionNet(input_size,
+                                       word_hidden_size,
+                                       word_num_layers,
+                                       word_context_size)
+        self.sent_layer = AttentionNet(2 * word_hidden_size,
+                                       sent_hidden_size,
+                                       sent_num_layers,
+                                       sent_context_size)
+        self.output_layer = nn.Linear(2 * sent_hidden_size, output_size)
+        self.softmax = nn.LogSoftmax(dim=1)
+
+    def forward(self, batch_doc):
+        # input is a sequence of matrix
+        doc_vec_list = []
+        for doc in batch_doc:
+            sent_mat = self.word_layer(doc)  # doc's dim (num_sent, seq_len, word_dim)
+            doc_vec_list.append(sent_mat)  # sent_mat's dim (num_sent, vec_dim)
+        doc_vec = self.sent_layer(pack_sequence(doc_vec_list))
+        output = self.softmax(self.output_layer(doc_vec))
+        return output
+
+
+class AttentionNet(nn.Module):
+    def __init__(self, input_size, gru_hidden_size, gru_num_layers, context_vec_size):
+        super(AttentionNet, self).__init__()
+
+        self.input_size = input_size
+        self.gru_hidden_size = gru_hidden_size
+        self.gru_num_layers = gru_num_layers
+        self.context_vec_size = context_vec_size
+
+        # Encoder
+        self.gru = nn.GRU(input_size=input_size,
+                          hidden_size=gru_hidden_size,
+                          num_layers=gru_num_layers,
+                          batch_first=True,
+                          bidirectional=True)
+        # Attention
+        self.fc = nn.Linear(2 * gru_hidden_size, context_vec_size)
+        self.tanh = nn.Tanh()
+        self.softmax = nn.Softmax(dim=1)
+        # context vector
+        self.context_vec = nn.Parameter(torch.Tensor(context_vec_size, 1))
+        self.context_vec.data.uniform_(-0.1, 0.1)
+
+    def forward(self, inputs):
+        # GRU part
+        h_t, hidden = self.gru(inputs)  # inputs's dim (batch_size, seq_len,  word_dim)
+        u = self.tanh(self.fc(h_t))
+        # Attention part
+        alpha = self.softmax(torch.matmul(u, self.context_vec))  # u's dim (batch_size, seq_len, context_vec_size)
+        output = torch.bmm(torch.transpose(h_t, 1, 2), alpha)  # alpha's dim (batch_size, seq_len, 1)
+        return torch.squeeze(output, dim=2)  # output's dim (batch_size, 2*hidden_size, 1)

reproduction/text_classification/model/awd_lstm.py

@@ -0,0 +1,31 @@
+import torch
+import torch.nn as nn
+from fastNLP.core.const import Const as C
+from .awdlstm_module import LSTM
+from fastNLP.modules import encoder
+from fastNLP.modules.decoder.mlp import MLP
+
+
+class AWDLSTMSentiment(nn.Module):
+    def __init__(self, init_embed,
+                 num_classes,
+                 hidden_dim=256,
+                 num_layers=1,
+                 nfc=128,
+                 wdrop=0.5):
+        super(AWDLSTMSentiment,self).__init__()
+        self.embed = encoder.Embedding(init_embed)
+        self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True, wdrop=wdrop)
+        self.mlp = MLP(size_layer=[hidden_dim* 2, nfc, num_classes])
+
+    def forward(self, words):
+        x_emb = self.embed(words)
+        output, _ = self.lstm(x_emb)
+        output = self.mlp(output[:,-1,:])
+        return {C.OUTPUT: output}
+
+    def predict(self, words):
+        output = self(words)
+        _, predict = output[C.OUTPUT].max(dim=1)
+        return {C.OUTPUT: predict}
+

reproduction/text_classification/model/awdlstm_module.py

@@ -0,0 +1,86 @@
+"""
+轻量封装的 Pytorch LSTM 模块.
+可在 forward 时传入序列的长度, 自动对padding做合适的处理.
+"""
+__all__ = [
+    "LSTM"
+]
+
+import torch
+import torch.nn as nn
+import torch.nn.utils.rnn as rnn
+
+from fastNLP.modules.utils import initial_parameter
+from torch import autograd
+from .weight_drop import WeightDrop
+
+
+class LSTM(nn.Module):
+    """
+    别名：:class:`fastNLP.modules.LSTM`  :class:`fastNLP.modules.encoder.lstm.LSTM`
+
+    LSTM 模块, 轻量封装的Pytorch LSTM. 在提供seq_len的情况下，将自动使用pack_padded_sequence; 同时默认将forget gate的bias初始化
+        为1; 且可以应对DataParallel中LSTM的使用问题。
+
+    :param input_size:  输入 `x` 的特征维度
+    :param hidden_size: 隐状态 `h` 的特征维度.
+    :param num_layers: rnn的层数. Default: 1
+    :param dropout: 层间dropout概率. Default: 0
+    :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False``
+    :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为
+        :(batch, seq, feature). Default: ``False``
+    :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True``
+    """
+    
+    def __init__(self, input_size, hidden_size=100, num_layers=1, dropout=0.0, batch_first=True,
+                 bidirectional=False, bias=True, wdrop=0.5):
+        super(LSTM, self).__init__()
+        self.batch_first = batch_first
+        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, bias=bias, batch_first=batch_first,
+                            dropout=dropout, bidirectional=bidirectional)
+        self.lstm = WeightDrop(self.lstm, ['weight_hh_l0'], dropout=wdrop)
+        self.init_param()
+
+    def init_param(self):
+        for name, param in self.named_parameters():
+            if 'bias' in name:
+                # based on https://github.com/pytorch/pytorch/issues/750#issuecomment-280671871
+                param.data.fill_(0)
+                n = param.size(0)
+                start, end = n // 4, n // 2
+                param.data[start:end].fill_(1)
+            else:
+                nn.init.xavier_uniform_(param)
+
+    def forward(self, x, seq_len=None, h0=None, c0=None):
+        """
+
+        :param x: [batch, seq_len, input_size] 输入序列
+        :param seq_len: [batch, ] 序列长度, 若为 ``None``, 所有输入看做一样长. Default: ``None``
+        :param h0: [batch, hidden_size] 初始隐状态, 若为 ``None`` , 设为全0向量. Default: ``None``
+        :param c0: [batch, hidden_size] 初始Cell状态, 若为 ``None`` , 设为全0向量. Default: ``None``
+        :return (output, ht) 或 output: 若 ``get_hidden=True`` [batch, seq_len, hidden_size*num_direction] 输出序列
+            和 [batch, hidden_size*num_direction] 最后时刻隐状态.
+        """
+        batch_size, max_len, _ = x.size()
+        if h0 is not None and c0 is not None:
+            hx = (h0, c0)
+        else:
+            hx = None
+        if seq_len is not None and not isinstance(x, rnn.PackedSequence):
+            sort_lens, sort_idx = torch.sort(seq_len, dim=0, descending=True)
+            if self.batch_first:
+                x = x[sort_idx]
+            else:
+                x = x[:, sort_idx]
+            x = rnn.pack_padded_sequence(x, sort_lens, batch_first=self.batch_first)
+            output, hx = self.lstm(x, hx)  # -> [N,L,C]
+            output, _ = rnn.pad_packed_sequence(output, batch_first=self.batch_first, total_length=max_len)
+            _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False)
+            if self.batch_first:
+                output = output[unsort_idx]
+            else:
+                output = output[:, unsort_idx]
+        else:
+            output, hx = self.lstm(x, hx)
+        return output, hx

reproduction/text_classification/model/char_cnn.py

@@ -1 +1,90 @@
-# TODO
+'''
+@author: https://github.com/ahmedbesbes/character-based-cnn
+这里借鉴了上述链接中char-cnn model的代码，改动主要为将其改动为符合fastnlp的pipline
+'''
+import torch
+import torch.nn as nn
+from fastNLP.core.const import Const as C
+
+class CharacterLevelCNN(nn.Module):
+    def __init__(self, args,embedding):
+        super(CharacterLevelCNN, self).__init__()
+
+        self.config=args.char_cnn_config
+        self.embedding=embedding
+
+        conv_layers = []
+        for i, conv_layer_parameter in enumerate(self.config['model_parameters'][args.model_size]['conv']):
+            if i == 0:
+                #in_channels = args.number_of_characters + len(args.extra_characters)
+                in_channels = args.embedding_dim
+                out_channels = conv_layer_parameter[0]
+            else:
+                in_channels, out_channels = conv_layer_parameter[0], conv_layer_parameter[0]
+
+            if conv_layer_parameter[2] != -1:
+                conv_layer = nn.Sequential(nn.Conv1d(in_channels,
+                                                     out_channels,
+                                                     kernel_size=conv_layer_parameter[1], padding=0),
+                                           nn.ReLU(),
+                                           nn.MaxPool1d(conv_layer_parameter[2]))
+            else:
+                conv_layer = nn.Sequential(nn.Conv1d(in_channels,
+                                                     out_channels,
+                                                     kernel_size=conv_layer_parameter[1], padding=0),
+                                           nn.ReLU())
+            conv_layers.append(conv_layer)
+        self.conv_layers = nn.ModuleList(conv_layers)
+
+        input_shape = (args.batch_size, args.max_length,
+                       args.number_of_characters + len(args.extra_characters))
+        dimension = self._get_conv_output(input_shape)
+
+        print('dimension :', dimension)
+
+        fc_layer_parameter = self.config['model_parameters'][args.model_size]['fc'][0]
+        fc_layers = nn.ModuleList([
+            nn.Sequential(
+                nn.Linear(dimension, fc_layer_parameter), nn.Dropout(0.5)),
+            nn.Sequential(nn.Linear(fc_layer_parameter,
+                                    fc_layer_parameter), nn.Dropout(0.5)),
+            nn.Linear(fc_layer_parameter, args.num_classes),
+        ])
+
+        self.fc_layers = fc_layers
+
+        if args.model_size == 'small':
+            self._create_weights(mean=0.0, std=0.05)
+        elif args.model_size == 'large':
+            self._create_weights(mean=0.0, std=0.02)
+
+    def _create_weights(self, mean=0.0, std=0.05):
+        for module in self.modules():
+            if isinstance(module, nn.Conv1d) or isinstance(module, nn.Linear):
+                module.weight.data.normal_(mean, std)
+
+    def _get_conv_output(self, shape):
+        input = torch.rand(shape)
+        output = input.transpose(1, 2)
+        # forward pass through conv layers
+        for i in range(len(self.conv_layers)):
+            output = self.conv_layers[i](output)
+
+        output = output.view(output.size(0), -1)
+        n_size = output.size(1)
+        return n_size
+
+    def forward(self, chars):
+        input=self.embedding(chars)
+        output = input.transpose(1, 2)
+        # forward pass through conv layers
+        for i in range(len(self.conv_layers)):
+            output = self.conv_layers[i](output)
+
+        output = output.view(output.size(0), -1)
+
+        # forward pass through fc layers
+        for i in range(len(self.fc_layers)):
+            output = self.fc_layers[i](output)
+
+        return {C.OUTPUT: output}

reproduction/text_classification/model/dpcnn.py

@@ -1 +1,97 @@
-# TODO
+import torch
+import torch.nn as nn
+from fastNLP.modules.utils import get_embeddings
+from fastNLP.core import Const as C
+
+
+class DPCNN(nn.Module):
+    def __init__(self, init_embed, num_cls, n_filters=256,
+                 kernel_size=3, n_layers=7, embed_dropout=0.1, cls_dropout=0.1):
+        super().__init__()
+        self.region_embed = RegionEmbedding(
+            init_embed, out_dim=n_filters, kernel_sizes=[1, 3, 5])
+        embed_dim = self.region_embed.embedding_dim
+        self.conv_list = nn.ModuleList()
+        for i in range(n_layers):
+            self.conv_list.append(nn.Sequential(
+                nn.ReLU(),
+                nn.Conv1d(n_filters, n_filters, kernel_size,
+                          padding=kernel_size//2),
+                nn.Conv1d(n_filters, n_filters, kernel_size,
+                          padding=kernel_size//2),
+            ))
+        self.pool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)
+        self.embed_drop = nn.Dropout(embed_dropout)
+        self.classfier = nn.Sequential(
+            nn.Dropout(cls_dropout),
+            nn.Linear(n_filters, num_cls),
+        )
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        for m in self.modules():
+            if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Linear)):
+                nn.init.normal_(m.weight, mean=0, std=0.01)
+                if m.bias is not None:
+                    nn.init.normal_(m.bias, mean=0, std=0.01)
+
+    def forward(self, words, seq_len=None):
+        words = words.long()
+        # get region embeddings
+        x = self.region_embed(words)
+        x = self.embed_drop(x)
+
+        # not pooling on first conv
+        x = self.conv_list[0](x) + x
+        for conv in self.conv_list[1:]:
+            x = self.pool(x)
+            x = conv(x) + x
+
+        # B, C, L => B, C
+        x, _ = torch.max(x, dim=2)
+        x = self.classfier(x)
+        return {C.OUTPUT: x}
+
+    def predict(self, words, seq_len=None):
+        x = self.forward(words, seq_len)[C.OUTPUT]
+        return {C.OUTPUT: torch.argmax(x, 1)}
+
+
+class RegionEmbedding(nn.Module):
+    def __init__(self, init_embed, out_dim=300, kernel_sizes=None):
+        super().__init__()
+        if kernel_sizes is None:
+            kernel_sizes = [5, 9]
+        assert isinstance(
+            kernel_sizes, list), 'kernel_sizes should be List(int)'
+        self.embed = get_embeddings(init_embed)
+        try:
+            embed_dim = self.embed.embedding_dim
+        except Exception:
+            embed_dim = self.embed.embed_size
+        self.region_embeds = nn.ModuleList()
+        for ksz in kernel_sizes:
+            self.region_embeds.append(nn.Sequential(
+                nn.Conv1d(embed_dim, embed_dim, ksz, padding=ksz // 2),
+            ))
+        self.linears = nn.ModuleList([nn.Conv1d(embed_dim, out_dim, 1)
+                                      for _ in range(len(kernel_sizes))])
+        self.embedding_dim = embed_dim
+
+    def forward(self, x):
+        x = self.embed(x)
+        x = x.transpose(1, 2)
+        # B, C, L
+        out = 0
+        for conv, fc in zip(self.region_embeds, self.linears[1:]):
+            conv_i = conv(x)
+            out = out + fc(conv_i)
+        # B, C, L
+        return out
+
+
+if __name__ == '__main__':
+    x = torch.randint(0, 10000, size=(5, 15), dtype=torch.long)
+    model = DPCNN((10000, 300), 20)
+    y = model(x)
+    print(y.size(), y.mean(1), y.std(1))

reproduction/text_classification/model/lstm.py

@@ -0,0 +1,30 @@
+import torch
+import torch.nn as nn
+from fastNLP.core.const import Const as C
+from fastNLP.modules.encoder.lstm import LSTM
+from fastNLP.modules import encoder
+from fastNLP.modules.decoder.mlp import MLP
+
+
+class BiLSTMSentiment(nn.Module):
+    def __init__(self, init_embed,
+                 num_classes,
+                 hidden_dim=256,
+                 num_layers=1,
+                 nfc=128):
+        super(BiLSTMSentiment,self).__init__()
+        self.embed = encoder.Embedding(init_embed)
+        self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True)
+        self.mlp = MLP(size_layer=[hidden_dim* 2, nfc, num_classes])
+
+    def forward(self, words):
+        x_emb = self.embed(words)
+        output, _ = self.lstm(x_emb)
+        output = self.mlp(output[:,-1,:])
+        return {C.OUTPUT: output}
+
+    def predict(self, words):
+        output = self(words)
+        _, predict = output[C.OUTPUT].max(dim=1)
+        return {C.OUTPUT: predict}
+

reproduction/text_classification/model/lstm_self_attention.py

@@ -0,0 +1,35 @@
+import torch
+import torch.nn as nn
+from fastNLP.core.const import Const as C
+from fastNLP.modules.encoder.lstm import LSTM
+from fastNLP.modules import encoder
+from fastNLP.modules.aggregator.attention import SelfAttention
+from fastNLP.modules.decoder.mlp import MLP
+
+
+class BiLSTM_SELF_ATTENTION(nn.Module):
+    def __init__(self, init_embed,
+                 num_classes,
+                 hidden_dim=256,
+                 num_layers=1,
+                 attention_unit=256,
+                 attention_hops=1,
+                 nfc=128):
+        super(BiLSTM_SELF_ATTENTION,self).__init__()
+        self.embed = encoder.Embedding(init_embed)
+        self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True)
+        self.attention = SelfAttention(input_size=hidden_dim * 2 , attention_unit=attention_unit, attention_hops=attention_hops)
+        self.mlp = MLP(size_layer=[hidden_dim* 2*attention_hops, nfc, num_classes])
+
+    def forward(self, words):
+        x_emb = self.embed(words)
+        output, _ = self.lstm(x_emb)
+        after_attention, penalty = self.attention(output,words)
+        after_attention =after_attention.view(after_attention.size(0),-1)
+        output = self.mlp(after_attention)
+        return {C.OUTPUT: output}
+
+    def predict(self, words):
+        output = self(words)
+        _, predict = output[C.OUTPUT].max(dim=1)
+        return {C.OUTPUT: predict}

reproduction/text_classification/model/weight_drop.py

@@ -0,0 +1,99 @@
+import torch
+from torch.nn import Parameter
+from functools import wraps
+
+class WeightDrop(torch.nn.Module):
+    def __init__(self, module, weights, dropout=0, variational=False):
+        super(WeightDrop, self).__init__()
+        self.module = module
+        self.weights = weights
+        self.dropout = dropout
+        self.variational = variational
+        self._setup()
+
+    def widget_demagnetizer_y2k_edition(*args, **kwargs):
+        # We need to replace flatten_parameters with a nothing function
+        # It must be a function rather than a lambda as otherwise pickling explodes
+        # We can't write boring code though, so ... WIDGET DEMAGNETIZER Y2K EDITION!
+        # (╯°□°）╯︵ ┻━┻
+        return
+
+    def _setup(self):
+        # Terrible temporary solution to an issue regarding compacting weights re: CUDNN RNN
+        if issubclass(type(self.module), torch.nn.RNNBase):
+            self.module.flatten_parameters = self.widget_demagnetizer_y2k_edition
+
+        for name_w in self.weights:
+            print('Applying weight drop of {} to {}'.format(self.dropout, name_w))
+            w = getattr(self.module, name_w)
+            del self.module._parameters[name_w]
+            self.module.register_parameter(name_w + '_raw', Parameter(w.data))
+
+    def _setweights(self):
+        for name_w in self.weights:
+            raw_w = getattr(self.module, name_w + '_raw')
+            w = None
+            if self.variational:
+                mask = torch.autograd.Variable(torch.ones(raw_w.size(0), 1))
+                if raw_w.is_cuda: mask = mask.cuda()
+                mask = torch.nn.functional.dropout(mask, p=self.dropout, training=True)
+                w = mask.expand_as(raw_w) * raw_w
+            else:
+                w = torch.nn.functional.dropout(raw_w, p=self.dropout, training=self.training)
+            setattr(self.module, name_w, w)
+
+    def forward(self, *args):
+        self._setweights()
+        return self.module.forward(*args)
+
+if __name__ == '__main__':
+    import torch
+    from weight_drop import WeightDrop
+
+    # Input is (seq, batch, input)
+    x = torch.autograd.Variable(torch.randn(2, 1, 10)).cuda()
+    h0 = None
+
+    ###
+
+    print('Testing WeightDrop')
+    print('=-=-=-=-=-=-=-=-=-=')
+
+    ###
+
+    print('Testing WeightDrop with Linear')
+
+    lin = WeightDrop(torch.nn.Linear(10, 10), ['weight'], dropout=0.9)
+    lin.cuda()
+    run1 = [x.sum() for x in lin(x).data]
+    run2 = [x.sum() for x in lin(x).data]
+
+    print('All items should be different')
+    print('Run 1:', run1)
+    print('Run 2:', run2)
+
+    assert run1[0] != run2[0]
+    assert run1[1] != run2[1]
+
+    print('---')
+
+    ###
+
+    print('Testing WeightDrop with LSTM')
+
+    wdrnn = WeightDrop(torch.nn.LSTM(10, 10), ['weight_hh_l0'], dropout=0.9)
+    wdrnn.cuda()
+
+    run1 = [x.sum() for x in wdrnn(x, h0)[0].data]
+    run2 = [x.sum() for x in wdrnn(x, h0)[0].data]
+
+    print('First timesteps should be equal, all others should differ')
+    print('Run 1:', run1)
+    print('Run 2:', run2)
+
+    # First time step, not influenced by hidden to hidden weights, should be equal
+    assert run1[0] == run2[0]
+    # Second step should not
+    assert run1[1] != run2[1]
+
+    print('---')

reproduction/text_classification/train_HAN.py

@@ -0,0 +1,109 @@
+# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径
+
+import os
+import sys
+sys.path.append('../../')
+os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/'
+os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches'
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+
+from fastNLP.core.const import Const as C
+from fastNLP.core import LRScheduler
+import torch.nn as nn
+from fastNLP.io.dataset_loader import SSTLoader
+from reproduction.text_classification.data.yelpLoader import yelpLoader
+from reproduction.text_classification.model.HAN import HANCLS
+from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding
+from fastNLP import CrossEntropyLoss, AccuracyMetric
+from fastNLP.core.trainer import Trainer
+from torch.optim import SGD
+import torch.cuda
+from torch.optim.lr_scheduler import CosineAnnealingLR
+
+
+##hyper
+
+class Config():
+    model_dir_or_name = "en-base-uncased"
+    embedding_grad = False,
+    train_epoch = 30
+    batch_size = 100
+    num_classes = 5
+    task = "yelp"
+    #datadir = '/remote-home/lyli/fastNLP/yelp_polarity/'
+    datadir = '/remote-home/ygwang/yelp_polarity/'
+    datafile = {"train": "train.csv", "test": "test.csv"}
+    lr = 1e-3
+
+    def __init__(self):
+        self.datapath = {k: os.path.join(self.datadir, v)
+                         for k, v in self.datafile.items()}
+
+
+ops = Config()
+
+##1.task相关信息：利用dataloader载入dataInfo
+
+datainfo = yelpLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train'])
+print(len(datainfo.datasets['train']))
+print(len(datainfo.datasets['test']))
+
+
+# post process
+def make_sents(words):
+    sents = [words]
+    return sents
+
+
+for dataset in datainfo.datasets.values():
+    dataset.apply_field(make_sents, field_name='words', new_field_name='input_sents')
+
+datainfo = datainfo
+datainfo.datasets['train'].set_input('input_sents')
+datainfo.datasets['test'].set_input('input_sents')
+datainfo.datasets['train'].set_target('target')
+datainfo.datasets['test'].set_target('target')
+
+## 2.或直接复用fastNLP的模型
+
+vocab = datainfo.vocabs['words']
+# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)])
+embedding = StaticEmbedding(vocab)
+
+print(len(vocab))
+print(len(datainfo.vocabs['target']))
+
+# model = DPCNN(init_embed=embedding, num_cls=ops.num_classes)
+model = HANCLS(init_embed=embedding, num_cls=ops.num_classes)
+
+## 3. 声明loss,metric,optimizer
+loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET)
+metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET)
+optimizer = SGD([param for param in model.parameters() if param.requires_grad == True],
+                lr=ops.lr, momentum=0.9, weight_decay=0)
+
+callbacks = []
+callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5)))
+
+device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+
+print(device)
+
+for ds in datainfo.datasets.values():
+    ds.apply_field(len, C.INPUT, C.INPUT_LEN)
+    ds.set_input(C.INPUT, C.INPUT_LEN)
+    ds.set_target(C.TARGET)
+
+
+## 4.定义train方法
+def train(model, datainfo, loss, metrics, optimizer, num_epochs=ops.train_epoch):
+    trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss,
+                      metrics=[metrics], dev_data=datainfo.datasets['test'], device=device,
+                      check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks,
+                      n_epochs=num_epochs)
+
+    print(trainer.train())
+
+
+if __name__ == "__main__":
+    train(model, datainfo, loss, metric, optimizer)

reproduction/text_classification/train_awdlstm.py

@@ -0,0 +1,69 @@
+# 这个模型需要在pytorch=0.4下运行，weight_drop不支持1.0
+
+# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径
+import os
+os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/'
+os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches'
+
+
+import torch.nn as nn
+
+from data.IMDBLoader import IMDBLoader
+from fastNLP.modules.encoder.embedding import StaticEmbedding
+from model.awd_lstm import AWDLSTMSentiment
+
+from fastNLP.core.const import Const as C
+from fastNLP import CrossEntropyLoss, AccuracyMetric
+from fastNLP import Trainer, Tester
+from torch.optim import Adam
+from fastNLP.io.model_io import ModelLoader, ModelSaver
+
+import argparse
+
+
+class Config():
+    train_epoch= 10
+    lr=0.001
+
+    num_classes=2
+    hidden_dim=256
+    num_layers=1
+    nfc=128
+    wdrop=0.5
+
+    task_name = "IMDB"
+    datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"}
+    save_model_path="./result_IMDB_test/"
+
+opt=Config()
+
+
+# load data
+dataloader=IMDBLoader()
+datainfo=dataloader.process(opt.datapath)
+
+# print(datainfo.datasets["train"])
+# print(datainfo)
+
+
+# define model
+vocab=datainfo.vocabs['words']
+embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True)
+model=AWDLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc, wdrop=opt.wdrop)
+
+
+# define loss_function and metrics
+loss=CrossEntropyLoss()
+metrics=AccuracyMetric()
+optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr)
+
+
+def train(datainfo, model, optimizer, loss, metrics, opt):
+    trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss,
+                        metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1,
+                        n_epochs=opt.train_epoch, save_path=opt.save_model_path)
+    trainer.train()
+
+
+if __name__ == "__main__":
+    train(datainfo, model, optimizer, loss, metrics, opt)

reproduction/text_classification/train_char_cnn.py

@@ -0,0 +1,205 @@
+# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径
+import os
+os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/'
+os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches'
+
+import sys
+sys.path.append('../..')
+from fastNLP.core.const import Const as C
+import torch.nn as nn
+from data.yelpLoader import yelpLoader
+from data.sstLoader import sst2Loader
+from data.IMDBLoader import IMDBLoader
+from model.char_cnn import CharacterLevelCNN
+from fastNLP.core.vocabulary import Vocabulary
+from fastNLP.models.cnn_text_classification import CNNText
+from fastNLP.modules.encoder.embedding import CNNCharEmbedding,StaticEmbedding,StackEmbedding,LSTMCharEmbedding
+from fastNLP import CrossEntropyLoss, AccuracyMetric
+from fastNLP.core.trainer import Trainer
+from torch.optim import SGD
+from torch.autograd import Variable
+import torch
+from fastNLP import BucketSampler
+
+##hyper
+#todo 这里加入fastnlp的记录
+class Config():
+    model_dir_or_name="en-base-uncased"
+    embedding_grad= False,
+    bert_embedding_larers= '4,-2,-1'
+    train_epoch= 50
+    num_classes=2
+    task= "IMDB"
+    #yelp_p
+    datapath = {"train": "/remote-home/ygwang/yelp_polarity/train.csv",
+               "test": "/remote-home/ygwang/yelp_polarity/test.csv"}
+    #IMDB
+    #datapath = {"train": "/remote-home/ygwang/IMDB_data/train.csv",
+    #           "test": "/remote-home/ygwang/IMDB_data/test.csv"}
+    # sst
+    # datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv",
+    #           "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"}
+
+    lr=0.01
+    batch_size=128
+    model_size="large"
+    number_of_characters=69
+    extra_characters=''
+    max_length=1014
+
+    char_cnn_config={
+        "alphabet": {
+            "en": {
+                "lower": {
+                    "alphabet": "abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}",
+                    "number_of_characters": 69
+                },
+                "both": {
+                    "alphabet": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}",
+                    "number_of_characters": 95
+                }
+            }
+        },
+        "model_parameters": {
+            "small": {
+                "conv": [
+                    #依次是channel，kennnel_size，maxpooling_size
+                    [256,7,3],
+                    [256,7,3],
+                    [256,3,-1],
+                    [256,3,-1],
+                    [256,3,-1],
+                    [256,3,3]
+                ],
+                "fc": [1024,1024]
+            },
+            "large":{
+                "conv":[
+                    [1024, 7, 3],
+                    [1024, 7, 3],
+                    [1024, 3, -1],
+                    [1024, 3, -1],
+                    [1024, 3, -1],
+                    [1024, 3, 3]
+                ],
+                "fc": [2048,2048]
+            }
+        },
+        "data": {
+            "text_column": "SentimentText",
+            "label_column": "Sentiment",
+            "max_length": 1014,
+            "num_of_classes": 2,
+            "encoding": None,
+            "chunksize": 50000,
+            "max_rows": 100000,
+            "preprocessing_steps": ["lower", "remove_hashtags", "remove_urls", "remove_user_mentions"]
+        },
+        "training": {
+            "batch_size": 128,
+            "learning_rate": 0.01,
+            "epochs": 10,
+            "optimizer": "sgd"
+        }
+    }
+ops=Config
+
+
+##1.task相关信息：利用dataloader载入dataInfo
+#dataloader=sst2Loader()
+#dataloader=IMDBLoader()
+dataloader=yelpLoader(fine_grained=True)
+datainfo=dataloader.process(ops.datapath,char_level_op=True)
+char_vocab=ops.char_cnn_config["alphabet"]["en"]["lower"]["alphabet"]
+ops.number_of_characters=len(char_vocab)
+ops.embedding_dim=ops.number_of_characters
+
+#chartoindex
+def chartoindex(chars):
+    max_seq_len=ops.max_length
+    zero_index=len(char_vocab)
+    char_index_list=[]
+    for char in chars:
+        if char in char_vocab:
+             char_index_list.append(char_vocab.index(char))
+        else:
+            #<unk>和<pad>均使用最后一个作为embbeding
+            char_index_list.append(zero_index)
+    if len(char_index_list) > max_seq_len:
+        char_index_list = char_index_list[:max_seq_len]
+    elif 0 < len(char_index_list) < max_seq_len:
+        char_index_list = char_index_list+[zero_index]*(max_seq_len-len(char_index_list))
+    elif len(char_index_list) == 0:
+        char_index_list=[zero_index]*max_seq_len
+    return char_index_list
+
+for dataset in datainfo.datasets.values():
+    dataset.apply_field(chartoindex,field_name='chars',new_field_name='chars')
+
+datainfo.datasets['train'].set_input('chars')
+datainfo.datasets['test'].set_input('chars')
+datainfo.datasets['train'].set_target('target')
+datainfo.datasets['test'].set_target('target')
+
+##2. 定义/组装模型，这里可以随意，就如果是fastNLP封装好的，类似CNNText就直接用初始化调用就好了，这里只是给出一个伪框架表示占位，在这里建立符合fastNLP输入输出规范的model
+class ModelFactory(nn.Module):
+    """
+        用于拼装embedding，encoder，decoder 以及设计forward过程
+
+        :param embedding:  embbeding model
+        :param encoder: encoder model
+        :param decoder: decoder model
+
+        """
+    def __int__(self,embedding,encoder,decoder,**kwargs):
+        super(ModelFactory,self).__init__()
+        self.embedding=embedding
+        self.encoder=encoder
+        self.decoder=decoder
+
+    def forward(self,x):
+        return {C.OUTPUT:None}
+
+## 2.或直接复用fastNLP的模型
+#vocab=datainfo.vocabs['words']
+vocab_label=datainfo.vocabs['target']
+'''
+# emded_char=CNNCharEmbedding(vocab)
+# embed_word = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50', requires_grad=True)
+# embedding=StackEmbedding([emded_char, embed_word])
+# cnn_char_embed = CNNCharEmbedding(vocab)
+# lstm_char_embed = LSTMCharEmbedding(vocab)
+# embedding = StackEmbedding([cnn_char_embed, lstm_char_embed])
+'''
+#one-hot embedding
+embedding_weight= Variable(torch.zeros(len(char_vocab)+1, len(char_vocab)))
+
+for i in range(len(char_vocab)):
+    embedding_weight[i][i]=1
+embedding=nn.Embedding(num_embeddings=len(char_vocab)+1,embedding_dim=len(char_vocab),padding_idx=len(char_vocab),_weight=embedding_weight)
+for para in embedding.parameters():
+    para.requires_grad=False
+#CNNText太过于简单
+#model=CNNText(init_embed=embedding, num_classes=ops.num_classes)
+model=CharacterLevelCNN(ops,embedding)
+
+## 3. 声明loss,metric,optimizer
+loss=CrossEntropyLoss
+metric=AccuracyMetric
+optimizer= SGD([param for param in model.parameters() if param.requires_grad==True], lr=ops.lr)
+
+## 4.定义train方法
+def train(model,datainfo,loss,metrics,optimizer,num_epochs=100):
+    trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss(target='target'),
+                      metrics=[metrics(target='target')], dev_data=datainfo.datasets['test'], device=0, check_code_level=-1,
+                      n_epochs=num_epochs)
+    print(trainer.train())
+
+
+
+if __name__=="__main__":
+    #print(vocab_label)
+
+    #print(datainfo.datasets["train"])
+    train(model,datainfo,loss,metric,optimizer,num_epochs=ops.train_epoch)
+    

reproduction/text_classification/train_dpcnn.py

@@ -0,0 +1,120 @@
+# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径
+
+import torch.cuda
+from fastNLP.core.utils import cache_results
+from torch.optim import SGD
+from torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR
+from fastNLP.core.trainer import Trainer
+from fastNLP import CrossEntropyLoss, AccuracyMetric
+from fastNLP.modules.encoder.embedding import StaticEmbedding, CNNCharEmbedding, StackEmbedding
+from reproduction.text_classification.model.dpcnn import DPCNN
+from data.yelpLoader import yelpLoader
+from fastNLP.core.sampler import BucketSampler
+import torch.nn as nn
+from fastNLP.core import LRScheduler
+from fastNLP.core.const import Const as C
+from fastNLP.core.vocabulary import VocabularyOption
+from utils.util_init import set_rng_seeds
+import os
+os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/'
+os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches'
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+
+
+# hyper
+
+class Config():
+    seed = 12345
+    model_dir_or_name = "dpcnn-yelp-p"
+    embedding_grad = True
+    train_epoch = 30
+    batch_size = 100
+    task = "yelp_p"
+    #datadir = 'workdir/datasets/SST'
+    datadir = 'workdir/datasets/yelp_polarity'
+    # datadir = 'workdir/datasets/yelp_full'
+    #datafile = {"train": "train.txt", "dev": "dev.txt", "test": "test.txt"}
+    datafile = {"train": "train.csv",  "test": "test.csv"}
+    lr = 1e-3
+    src_vocab_op = VocabularyOption(max_size=100000)
+    embed_dropout = 0.3
+    cls_dropout = 0.1
+    weight_decay = 1e-5
+
+    def __init__(self):
+        self.datadir = os.path.join(os.environ['HOME'], self.datadir)
+        self.datapath = {k: os.path.join(self.datadir, v)
+                         for k, v in self.datafile.items()}
+
+
+ops = Config()
+
+set_rng_seeds(ops.seed)
+print('RNG SEED: {}'.format(ops.seed))
+
+# 1.task相关信息：利用dataloader载入dataInfo
+
+#datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train'])
+
+
+@cache_results(ops.model_dir_or_name+'-data-cache')
+def load_data():
+    datainfo = yelpLoader(fine_grained=True, lower=True).process(
+        paths=ops.datapath, train_ds=['train'], src_vocab_op=ops.src_vocab_op)
+    for ds in datainfo.datasets.values():
+        ds.apply_field(len, C.INPUT, C.INPUT_LEN)
+        ds.set_input(C.INPUT, C.INPUT_LEN)
+        ds.set_target(C.TARGET)
+    embedding = StaticEmbedding(
+        datainfo.vocabs['words'], model_dir_or_name='en-glove-840b-300', requires_grad=ops.embedding_grad,
+        normalize=False
+    )
+    return datainfo, embedding
+
+
+datainfo, embedding = load_data()
+
+# 2.或直接复用fastNLP的模型
+
+# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)])
+
+print(datainfo)
+print(datainfo.datasets['train'][0])
+
+model = DPCNN(init_embed=embedding, num_cls=len(datainfo.vocabs[C.TARGET]),
+              embed_dropout=ops.embed_dropout, cls_dropout=ops.cls_dropout)
+print(model)
+
+# 3. 声明loss,metric,optimizer
+loss = CrossEntropyLoss(pred=C.OUTPUT, target=C.TARGET)
+metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET)
+optimizer = SGD([param for param in model.parameters() if param.requires_grad == True],
+                lr=ops.lr, momentum=0.9, weight_decay=ops.weight_decay)
+
+callbacks = []
+# callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5)))
+callbacks.append(
+    LRScheduler(LambdaLR(optimizer, lambda epoch: ops.lr if epoch <
+                         ops.train_epoch * 0.8 else ops.lr * 0.1))
+)
+
+# callbacks.append(
+#     FitlogCallback(data=datainfo.datasets, verbose=1)
+# )
+
+device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+
+print(device)
+
+# 4.定义train方法
+trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss,
+                  sampler=BucketSampler(num_buckets=50, batch_size=ops.batch_size),
+                  metrics=[metric],
+                  dev_data=datainfo.datasets['test'], device=device,
+                  check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks,
+                  n_epochs=ops.train_epoch, num_workers=4)
+
+
+
+if __name__ == "__main__":
+    print(trainer.train())

reproduction/text_classification/train_lstm.py

@@ -0,0 +1,66 @@
+# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径
+import os
+os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/'
+os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches'
+
+
+import torch.nn as nn
+
+from data.IMDBLoader import IMDBLoader
+from fastNLP.modules.encoder.embedding import StaticEmbedding
+from model.lstm import BiLSTMSentiment
+
+from fastNLP.core.const import Const as C
+from fastNLP import CrossEntropyLoss, AccuracyMetric
+from fastNLP import Trainer, Tester
+from torch.optim import Adam
+from fastNLP.io.model_io import ModelLoader, ModelSaver
+
+import argparse
+
+
+class Config():
+    train_epoch= 10
+    lr=0.001
+
+    num_classes=2
+    hidden_dim=256
+    num_layers=1
+    nfc=128
+
+    task_name = "IMDB"
+    datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"}
+    save_model_path="./result_IMDB_test/"
+
+opt=Config()
+
+
+# load data
+dataloader=IMDBLoader()
+datainfo=dataloader.process(opt.datapath)
+
+# print(datainfo.datasets["train"])
+# print(datainfo)
+
+
+# define model
+vocab=datainfo.vocabs['words']
+embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True)
+model=BiLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc)
+
+
+# define loss_function and metrics
+loss=CrossEntropyLoss()
+metrics=AccuracyMetric()
+optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr)
+
+
+def train(datainfo, model, optimizer, loss, metrics, opt):
+    trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss,
+                        metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1,
+                        n_epochs=opt.train_epoch, save_path=opt.save_model_path)
+    trainer.train()
+
+
+if __name__ == "__main__":
+    train(datainfo, model, optimizer, loss, metrics, opt)

reproduction/text_classification/train_lstm_att.py

@@ -0,0 +1,68 @@
+# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径
+import os
+os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/'
+os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches'
+
+
+import torch.nn as nn
+
+from data.IMDBLoader import IMDBLoader
+from fastNLP.modules.encoder.embedding import StaticEmbedding
+from model.lstm_self_attention import BiLSTM_SELF_ATTENTION
+
+from fastNLP.core.const import Const as C
+from fastNLP import CrossEntropyLoss, AccuracyMetric
+from fastNLP import Trainer, Tester
+from torch.optim import Adam
+from fastNLP.io.model_io import ModelLoader, ModelSaver
+
+import argparse
+
+
+class Config():
+    train_epoch= 10
+    lr=0.001
+
+    num_classes=2
+    hidden_dim=256
+    num_layers=1
+    attention_unit=256
+    attention_hops=1
+    nfc=128
+
+    task_name = "IMDB"
+    datapath={"train":"IMDB_data/train.csv", "test":"IMDB_data/test.csv"}
+    save_model_path="./result_IMDB_test/"
+
+opt=Config()
+
+
+# load data
+dataloader=IMDBLoader()
+datainfo=dataloader.process(opt.datapath)
+
+# print(datainfo.datasets["train"])
+# print(datainfo)
+
+
+# define model
+vocab=datainfo.vocabs['words']
+embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True)
+model=BiLSTM_SELF_ATTENTION(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, attention_unit=opt.attention_unit, attention_hops=opt.attention_hops, nfc=opt.nfc)
+
+
+# define loss_function and metrics
+loss=CrossEntropyLoss()
+metrics=AccuracyMetric()
+optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr)
+
+
+def train(datainfo, model, optimizer, loss, metrics, opt):
+    trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss,
+                        metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1,
+                        n_epochs=opt.train_epoch, save_path=opt.save_model_path)
+    trainer.train()
+
+
+if __name__ == "__main__":
+    train(datainfo, model, optimizer, loss, metrics, opt)

reproduction/text_classification/utils/util_init.py

@@ -0,0 +1,11 @@
+import numpy
+import torch
+import random
+
+
+def set_rng_seeds(seed):
+    random.seed(seed)
+    numpy.random.seed(seed)
+    torch.random.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    # print('RNG_SEED {}'.format(seed))

reproduction/utils.py

@@ -59,4 +59,13 @@ def check_dataloader_paths(paths:Union[str, Dict[str, str]])->Dict[str, str]:
     else:
         raise TypeError(f"paths only supports str and dict. not {type(paths)}.")
 
-
+def get_tokenizer():
+    try:
+        import spacy
+        spacy.prefer_gpu()
+        en = spacy.load('en')
+        print('use spacy tokenizer')
+        return lambda x: [w.text for w in en.tokenizer(x)]
+    except Exception as e:
+        print('use raw tokenizer')
+        return lambda x: x.split()