Merge pull request #181 from SrWYG/dev0.5.0

[fix] 移除废弃的SSTloader

reproduction/text_classification/README.md

@@ -20,7 +20,7 @@ model name | yelp_p | yelp_f | sst-2|IMDB
 char_cnn | 93.80/95.12 | - | - |- 
 dpcnn | 95.50/97.36 | - | - |- 
 HAN |- | - | - |-
-LSTM| 95.74/- |- |- |88.52/-
-AWD-LSTM| 95.96/- |- |- |88.91/-
-LSTM+self_attention| 96.34/- | - | - |89.53/-
+LSTM| 95.74/- |64.16/- |- |88.52/-
+AWD-LSTM| 95.96/- |64.74/- |- |88.91/-
+LSTM+self_attention| 96.34/- | 65.78/- | - |89.53/-
 

reproduction/text_classification/data/SSTLoader.py

@@ -1,187 +0,0 @@
-from typing import Iterable
-from nltk import Tree
-from fastNLP.io.base_loader import DataInfo, DataSetLoader
-from fastNLP.core.vocabulary import VocabularyOption, Vocabulary
-from fastNLP import DataSet
-from fastNLP import Instance
-from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader
-import csv
-from typing import Union, Dict
-
-class SSTLoader(DataSetLoader):
-    URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip'
-    DATA_DIR = 'sst/'
-
-    """
-    别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader`
-
-    读取SST数据集, DataSet包含fields::
-
-        words: list(str) 需要分类的文本
-        target: str 文本的标签
-
-    数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip
-
-    :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False``
-    :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False``
-    """
-
-    def __init__(self, subtree=False, fine_grained=False):
-        self.subtree = subtree
-
-        tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral',
-                 '3': 'positive', '4': 'very positive'}
-        if not fine_grained:
-            tag_v['0'] = tag_v['1']
-            tag_v['4'] = tag_v['3']
-        self.tag_v = tag_v
-
-    def _load(self, path):
-        """
-
-        :param str path: 存储数据的路径
-        :return: 一个 :class:`~fastNLP.DataSet` 类型的对象
-        """
-        datalist = []
-        with open(path, 'r', encoding='utf-8') as f:
-            datas = []
-            for l in f:
-                datas.extend([(s, self.tag_v[t])
-                              for s, t in self._get_one(l, self.subtree)])
-        ds = DataSet()
-        for words, tag in datas:
-            ds.append(Instance(words=words, target=tag))
-        return ds
-
-    @staticmethod
-    def _get_one(data, subtree):
-        tree = Tree.fromstring(data)
-        if subtree:
-            return [(t.leaves(), t.label()) for t in tree.subtrees()]
-        return [(tree.leaves(), tree.label())]
-
-    def process(self,
-                paths,
-                train_ds: Iterable[str] = None,
-                src_vocab_op: VocabularyOption = None,
-                tgt_vocab_op: VocabularyOption = None,
-                src_embed_op: EmbeddingOption = None):
-        input_name, target_name = 'words', 'target'
-        src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op)
-        tgt_vocab = Vocabulary(unknown=None, padding=None) \
-            if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op)
-
-        info = DataInfo(datasets=self.load(paths))
-        _train_ds = [info.datasets[name]
-                     for name in train_ds] if train_ds else info.datasets.values()
-        src_vocab.from_dataset(*_train_ds, field_name=input_name)
-        tgt_vocab.from_dataset(*_train_ds, field_name=target_name)
-        src_vocab.index_dataset(
-            *info.datasets.values(),
-            field_name=input_name, new_field_name=input_name)
-        tgt_vocab.index_dataset(
-            *info.datasets.values(),
-            field_name=target_name, new_field_name=target_name)
-        info.vocabs = {
-            input_name: src_vocab,
-            target_name: tgt_vocab
-        }
-
-        if src_embed_op is not None:
-            src_embed_op.vocab = src_vocab
-            init_emb = EmbedLoader.load_with_vocab(**src_embed_op)
-            info.embeddings[input_name] = init_emb
-
-        for name, dataset in info.datasets.items():
-            dataset.set_input(input_name)
-            dataset.set_target(target_name)
-
-        return info
-
-class sst2Loader(DataSetLoader):
-    '''
-    数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',
-    '''
-    def __init__(self):
-        super(sst2Loader, self).__init__()
-
-    def _load(self, path: str) -> DataSet:
-        ds = DataSet()
-        all_count=0
-        csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t')
-        skip_row = 0
-        for idx,row in enumerate(csv_reader):
-            if idx<=skip_row:
-                continue
-            target = row[1]
-            words = row[0].split()
-            ds.append(Instance(words=words,target=target))
-            all_count+=1
-        print("all count:", all_count)
-        return ds
-
-    def process(self,
-                paths: Union[str, Dict[str, str]],
-                src_vocab_opt: VocabularyOption = None,
-                tgt_vocab_opt: VocabularyOption = None,
-                src_embed_opt: EmbeddingOption = None,
-                char_level_op=False):
-
-        paths = check_dataloader_paths(paths)
-        datasets = {}
-        info = DataInfo()
-        for name, path in paths.items():
-            dataset = self.load(path)
-            datasets[name] = dataset
-
-        def wordtochar(words):
-            chars=[]
-            for word in words:
-                word=word.lower()
-                for char in word:
-                    chars.append(char)
-            return chars
-
-        input_name, target_name = 'words', 'target'
-        info.vocabs={}
-
-        # 就分隔为char形式
-        if char_level_op:
-            for dataset in datasets.values():
-                dataset.apply_field(wordtochar, field_name="words", new_field_name='chars')
-
-        src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt)
-        src_vocab.from_dataset(datasets['train'], field_name='words')
-        src_vocab.index_dataset(*datasets.values(), field_name='words')
-
-        tgt_vocab = Vocabulary(unknown=None, padding=None) \
-            if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt)
-        tgt_vocab.from_dataset(datasets['train'], field_name='target')
-        tgt_vocab.index_dataset(*datasets.values(), field_name='target')
-
-
-        info.vocabs = {
-            "words": src_vocab,
-            "target": tgt_vocab
-        }
-
-        info.datasets = datasets
-
-
-        if src_embed_opt is not None:
-            embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab)
-            info.embeddings['words'] = embed
-
-        return info
-
-if __name__=="__main__":
-    datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv",
-                "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"}
-    datainfo=sst2Loader().process(datapath,char_level_op=True)
-    #print(datainfo.datasets["train"])
-    len_count = 0
-    for instance in datainfo.datasets["train"]:
-        len_count += len(instance["chars"])
-
-    ave_len = len_count / len(datainfo.datasets["train"])
-    print(ave_len)

reproduction/text_classification/data/sstLoader.py → reproduction/text_classification/data/sstloader.py

@@ -9,27 +9,22 @@ import csv
 from typing import Union, Dict
 from reproduction.utils import check_dataloader_paths, get_tokenizer
 
+
 class SSTLoader(DataSetLoader):
     URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip'
     DATA_DIR = 'sst/'
 
     """
     别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader`
-
     读取SST数据集, DataSet包含fields::
-
         words: list(str) 需要分类的文本
         target: str 文本的标签
-
     数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip
-
     :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False``
     :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False``
     """
-
     def __init__(self, subtree=False, fine_grained=False):
         self.subtree = subtree
-
         tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral',
                  '3': 'positive', '4': 'very positive'}
         if not fine_grained:
@@ -39,7 +34,6 @@ class SSTLoader(DataSetLoader):
 
     def _load(self, path):
         """
-
         :param str path: 存储数据的路径
         :return: 一个 :class:`~fastNLP.DataSet` 类型的对象
         """
@@ -54,6 +48,7 @@ class SSTLoader(DataSetLoader):
             ds.append(Instance(words=words, target=tag))
         return ds
 
+
     @staticmethod
     def _get_one(data, subtree):
         tree = Tree.fromstring(data)
@@ -61,6 +56,7 @@ class SSTLoader(DataSetLoader):
             return [(t.leaves(), t.label()) for t in tree.subtrees()]
         return [(tree.leaves(), tree.label())]
 
+
     def process(self,
                 paths,
                 train_ds: Iterable[str] = None,
@@ -88,25 +84,30 @@ class SSTLoader(DataSetLoader):
             target_name: tgt_vocab
         }
 
+
         if src_embed_op is not None:
             src_embed_op.vocab = src_vocab
             init_emb = EmbedLoader.load_with_vocab(**src_embed_op)
             info.embeddings[input_name] = init_emb
 
+
         for name, dataset in info.datasets.items():
             dataset.set_input(input_name)
             dataset.set_target(target_name)
-
         return info
 
+
+
 class sst2Loader(DataSetLoader):
     '''
     数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',
     '''
+
     def __init__(self):
         super(sst2Loader, self).__init__()
         self.tokenizer = get_tokenizer()
 
+
     def _load(self, path: str) -> DataSet:
         ds = DataSet()
         all_count=0
@@ -122,6 +123,8 @@ class sst2Loader(DataSetLoader):
         print("all count:", all_count)
         return ds
 
+
+
     def process(self,
                 paths: Union[str, Dict[str, str]],
                 src_vocab_opt: VocabularyOption = None,
@@ -153,7 +156,6 @@ class sst2Loader(DataSetLoader):
         if char_level_op:
             for dataset in datasets.values():
                 dataset.apply_field(wordtochar, field_name="words", new_field_name='chars')
-
         src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt)
         src_vocab.from_dataset(datasets['train'], field_name='words')
         src_vocab.index_dataset(*datasets.values(), field_name='words')
@@ -171,21 +173,26 @@ class sst2Loader(DataSetLoader):
 
         info.datasets = datasets
 
-
         if src_embed_opt is not None:
             embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab)
             info.embeddings['words'] = embed
 
+        for name, dataset in info.datasets.items():
+            dataset.set_input("words")
+            dataset.set_target("target")
+
         return info
 
+
+
 if __name__=="__main__":
     datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv",
                 "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"}
     datainfo=sst2Loader().process(datapath,char_level_op=True)
     #print(datainfo.datasets["train"])
+
     len_count = 0
     for instance in datainfo.datasets["train"]:
         len_count += len(instance["chars"])
-
     ave_len = len_count / len(datainfo.datasets["train"])
     print(ave_len)