From 945b30bb6174751130744231aa26119bf9bb2601 Mon Sep 17 00:00:00 2001 From: Xipeng Qiu Date: Sat, 17 Aug 2019 19:57:17 +0800 Subject: [PATCH 01/92] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b35776dc..476c129f 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ fastNLP 在 embeddings 模块中内置了几种不同的embedding：静态embedd ## 项目结构 -![](./docs/source/figures/workflow.png) + fastNLP的大致工作流程如上图所示，而项目结构如下： From 78af3491a432cb10b36d9cf17b75c12e40146026 Mon Sep 17 00:00:00 2001 From: zide05 <845465009@qq.com> Date: Mon, 26 Aug 2019 14:03:40 +0800 Subject: [PATCH 02/92] =?UTF-8?q?=E4=BF=AE=E6=94=B9tutorial?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/tutorials/tutorial_4_loss_optimizer.rst | 7 +++++-- docs/source/tutorials/tutorial_5_datasetiter.rst | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/source/tutorials/tutorial_4_loss_optimizer.rst b/docs/source/tutorials/tutorial_4_loss_optimizer.rst index f863a7a8..a53ef89b 100644 --- a/docs/source/tutorials/tutorial_4_loss_optimizer.rst +++ b/docs/source/tutorials/tutorial_4_loss_optimizer.rst @@ -1,4 +1,4 @@ -============================================================================== +============================================================================== 动手实现一个文本分类器I-使用Trainer和Tester快速训练和测试 ============================================================================== @@ -19,7 +19,9 @@ loader = SSTLoader() #这里的all.txt是下载好数据后train.txt、dev.txt、test.txt的组合 - dataset = loader.load("./trainDevTestTrees_PTB/trees/all.txt") + #loader.load(path)会首先判断path是否为none，若是则自动从网站下载数据，若不是则读入数据并返回databundle + databundle_ = loader.load("./trainDevTestTrees_PTB/trees/all.txt") + dataset = databundle_.datasets['train'] print(dataset[0]) 输出数据如下:: @@ -31,6 +33,7 @@ 数据处理 + 可以使用事先定义的 :class:`~fastNLP.io.SSTPipe` 类对数据进行基本预处理，这里我们手动进行处理。 我们使用 :class:`~fastNLP.DataSet` 类的 :meth:`~fastNLP.DataSet.apply` 方法将 ``target`` :mod:`~fastNLP.core.field` 转化为整数。 .. code-block:: python diff --git a/docs/source/tutorials/tutorial_5_datasetiter.rst b/docs/source/tutorials/tutorial_5_datasetiter.rst index e81b18dd..2ec753c3 100644 --- a/docs/source/tutorials/tutorial_5_datasetiter.rst +++ b/docs/source/tutorials/tutorial_5_datasetiter.rst @@ -20,7 +20,9 @@ loader = SSTLoader() #这里的all.txt是下载好数据后train.txt、dev.txt、test.txt的组合 - dataset = loader.load("./trainDevTestTrees_PTB/trees/all.txt") + #loader.load(path)会首先判断path是否为none，若是则自动从网站下载数据，若不是则读入数据并返回databundle + databundle_ = loader.load("./trainDevTestTrees_PTB/trees/all.txt") + dataset = databundle_.datasets['train'] print(dataset[0]) 输出数据如下:: @@ -32,6 +34,7 @@ 数据处理 + 可以使用事先定义的 :class:`~fastNLP.io.SSTPipe` 类对数据进行基本预处理，这里我们手动进行处理。 我们使用 :class:`~fastNLP.DataSet` 类的 :meth:`~fastNLP.DataSet.apply` 方法将 ``target`` :mod:`~fastNLP.core.field` 转化为整数。 .. code-block:: python From 53975c045a6841e38d4a7cfcc23abea6de0fe3f3 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Mon, 26 Aug 2019 14:58:36 +0800 Subject: [PATCH 03/92] update the doc-tool & fix an importing bug --- docs/count.py | 42 ++++++++++++++++++++++++++++++++++ fastNLP/modules/decoder/crf.py | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/count.py b/docs/count.py index e1aad115..72868403 100644 --- a/docs/count.py +++ b/docs/count.py @@ -1,7 +1,28 @@ +import inspect import os import sys +def _colored_string(string: str, color: str or int) -> str: + """在终端中显示一串有颜色的文字 + :param string: 在终端中显示的文字 + :param color: 文字的颜色 + :return: + """ + if isinstance(color, str): + color = { + "black": 30, "Black": 30, "BLACK": 30, + "red": 31, "Red": 31, "RED": 31, + "green": 32, "Green": 32, "GREEN": 32, + "yellow": 33, "Yellow": 33, "YELLOW": 33, + "blue": 34, "Blue": 34, "BLUE": 34, + "purple": 35, "Purple": 35, "PURPLE": 35, + "cyan": 36, "Cyan": 36, "CYAN": 36, + "white": 37, "White": 37, "WHITE": 37 + }[color] + return "\033[%dm%s\033[0m" % (color, string) + + def find_all_modules(): modules = {} children = {} @@ -55,10 +76,31 @@ def create_rst_file(modules, name, children): fout.write(" " + module + "\n") +def check_file(m, name): + for item, obj in inspect.getmembers(m): + if inspect.isclass(obj) and obj.__module__ == name: + print(obj) + if inspect.isfunction(obj) and obj.__module__ == name: + print("FUNC", obj) + + +def check_files(modules): + for name in sorted(modules.keys()): + if name == 'fastNLP.core.utils': + check_file(modules[name], name) + + def main(): + print(_colored_string('Getting modules...', "Blue")) modules, to_doc, children = find_all_modules() + print(_colored_string('Done!', "Green")) + print(_colored_string('Creating rst files...', "Blue")) for name in to_doc: create_rst_file(modules, name, children) + print(_colored_string('Done!', "Green")) + print(_colored_string('Checking all files...', "Blue")) + check_files(modules) + print(_colored_string('Done!', "Green")) if __name__ == "__main__": diff --git a/fastNLP/modules/decoder/crf.py b/fastNLP/modules/decoder/crf.py index b47d0162..f63d46e3 100644 --- a/fastNLP/modules/decoder/crf.py +++ b/fastNLP/modules/decoder/crf.py @@ -9,7 +9,7 @@ import torch from torch import nn from ..utils import initial_parameter -from ...core import Vocabulary +from ...core.vocabulary import Vocabulary def allowed_transitions(id2target, encoding_type='bio', include_start_end=False): From b4e542095d34e3831a7f98b3d4e9e0a41e6e3f77 Mon Sep 17 00:00:00 2001 From: xxliu Date: Mon, 26 Aug 2019 19:21:35 +0800 Subject: [PATCH 04/92] pipe --- fastNLP/io/loader/__init__.py | 5 +- fastNLP/io/loader/coreference.py | 24 ++++ fastNLP/io/pipe/__init__.py | 3 + fastNLP/io/pipe/coreference.py | 115 ++++++++++++++++++ reproduction/coreference_resolution/README.md | 2 +- .../data_load/__init__.py | 0 .../data_load/cr_loader.py | 68 ----------- .../test/test_dataloader.py | 20 +-- reproduction/coreference_resolution/train.py | 10 +- reproduction/coreference_resolution/valid.py | 10 +- 10 files changed, 166 insertions(+), 91 deletions(-) create mode 100644 fastNLP/io/loader/coreference.py create mode 100644 fastNLP/io/pipe/coreference.py delete mode 100644 reproduction/coreference_resolution/data_load/__init__.py delete mode 100644 reproduction/coreference_resolution/data_load/cr_loader.py diff --git a/fastNLP/io/loader/__init__.py b/fastNLP/io/loader/__init__.py index 6c23f213..aae3171a 100644 --- a/fastNLP/io/loader/__init__.py +++ b/fastNLP/io/loader/__init__.py @@ -71,7 +71,9 @@ __all__ = [ "QuoraLoader", "SNLILoader", "QNLILoader", - "RTELoader" + "RTELoader", + + "CRLoader" ] from .classification import YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader from .conll import ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader @@ -81,3 +83,4 @@ from .json import JsonLoader from .loader import Loader from .matching import MNLILoader, QuoraLoader, SNLILoader, QNLILoader, RTELoader from .conll import MsraNERLoader, PeopleDailyNERLoader, WeiboNERLoader +from .coreference import CRLoader \ No newline at end of file diff --git a/fastNLP/io/loader/coreference.py b/fastNLP/io/loader/coreference.py new file mode 100644 index 00000000..c8d9bbf5 --- /dev/null +++ b/fastNLP/io/loader/coreference.py @@ -0,0 +1,24 @@ +from ...core.dataset import DataSet +from ..file_reader import _read_json +from ...core.instance import Instance +from .json import JsonLoader + + +class CRLoader(JsonLoader): + def __init__(self, fields=None, dropna=False): + super().__init__(fields, dropna) + + def _load(self, path): + """ + 加载数据 + :param path: + :return: + """ + dataset = DataSet() + for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): + if self.fields: + ins = {self.fields[k]: v for k, v in d.items()} + else: + ins = d + dataset.append(Instance(**ins)) + return dataset \ No newline at end of file diff --git a/fastNLP/io/pipe/__init__.py b/fastNLP/io/pipe/__init__.py index 048e4cfe..d99b68c4 100644 --- a/fastNLP/io/pipe/__init__.py +++ b/fastNLP/io/pipe/__init__.py @@ -37,6 +37,8 @@ __all__ = [ "QuoraPipe", "QNLIPipe", "MNLIPipe", + + "CoreferencePipe" ] from .classification import YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe @@ -46,3 +48,4 @@ from .matching import MatchingBertPipe, RTEBertPipe, SNLIBertPipe, QuoraBertPipe from .pipe import Pipe from .conll import Conll2003Pipe from .cws import CWSPipe +from .coreference import CoreferencePipe diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py new file mode 100644 index 00000000..bdf6a132 --- /dev/null +++ b/fastNLP/io/pipe/coreference.py @@ -0,0 +1,115 @@ +__all__ = [ + "CoreferencePipe" + +] + +from .pipe import Pipe +from ..data_bundle import DataBundle +from ..loader.coreference import CRLoader +from fastNLP.core.vocabulary import Vocabulary +import numpy as np +import collections + + +class CoreferencePipe(Pipe): + + def __init__(self,config): + super().__init__() + self.config = config + + def process(self, data_bundle: DataBundle): + genres = {g: i for i, g in enumerate(["bc", "bn", "mz", "nw", "pt", "tc", "wb"])} + vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name='sentences') + vocab.build_vocab() + word2id = vocab.word2idx + char_dict = get_char_dict(self.config.char_path) + for name, ds in data_bundle.datasets.items(): + ds.apply(lambda x: doc2numpy(x['sentences'], word2id, char_dict, max(self.config.filter), + self.config.max_sentences, is_train=name == 'train')[0], + new_field_name='doc_np') + ds.apply(lambda x: doc2numpy(x['sentences'], word2id, char_dict, max(self.config.filter), + self.config.max_sentences, is_train=name == 'train')[1], + new_field_name='char_index') + ds.apply(lambda x: doc2numpy(x['sentences'], word2id, char_dict, max(self.config.filter), + self.config.max_sentences, is_train=name == 'train')[2], + new_field_name='seq_len') + ds.apply(lambda x: speaker2numpy(x["speakers"], self.config.max_sentences, is_train=name == 'train'), + new_field_name='speaker_ids_np') + ds.apply(lambda x: genres[x["doc_key"][:2]], new_field_name='genre') + + ds.set_ignore_type('clusters') + ds.set_padder('clusters', None) + ds.set_input("sentences", "doc_np", "speaker_ids_np", "genre", "char_index", "seq_len") + ds.set_target("clusters") + return data_bundle + + def process_from_file(self, paths): + bundle = CRLoader().load(paths) + return self.process(bundle) + + +# helper + +def doc2numpy(doc, word2id, chardict, max_filter, max_sentences, is_train): + docvec, char_index, length, max_len = _doc2vec(doc, word2id, chardict, max_filter, max_sentences, is_train) + assert max(length) == max_len + assert char_index.shape[0] == len(length) + assert char_index.shape[1] == max_len + doc_np = np.zeros((len(docvec), max_len), int) + for i in range(len(docvec)): + for j in range(len(docvec[i])): + doc_np[i][j] = docvec[i][j] + return doc_np, char_index, length + +def _doc2vec(doc,word2id,char_dict,max_filter,max_sentences,is_train): + max_len = 0 + max_word_length = 0 + docvex = [] + length = [] + if is_train: + sent_num = min(max_sentences,len(doc)) + else: + sent_num = len(doc) + + for i in range(sent_num): + sent = doc[i] + length.append(len(sent)) + if (len(sent) > max_len): + max_len = len(sent) + sent_vec =[] + for j,word in enumerate(sent): + if len(word)>max_word_length: + max_word_length = len(word) + if word in word2id: + sent_vec.append(word2id[word]) + else: + sent_vec.append(word2id["UNK"]) + docvex.append(sent_vec) + + char_index = np.zeros((sent_num, max_len, max_word_length),dtype=int) + for i in range(sent_num): + sent = doc[i] + for j,word in enumerate(sent): + char_index[i, j, :len(word)] = [char_dict[c] for c in word] + + return docvex,char_index,length,max_len + +def speaker2numpy(speakers_raw,max_sentences,is_train): + if is_train and len(speakers_raw)> max_sentences: + speakers_raw = speakers_raw[0:max_sentences] + speakers = flatten(speakers_raw) + speaker_dict = {s: i for i, s in enumerate(set(speakers))} + speaker_ids = np.array([speaker_dict[s] for s in speakers]) + return speaker_ids + +# 展平 +def flatten(l): + return [item for sublist in l for item in sublist] + +def get_char_dict(path): + vocab = [""] + with open(path) as f: + vocab.extend(c.strip() for c in f.readlines()) + char_dict = collections.defaultdict(int) + char_dict.update({c: i for i, c in enumerate(vocab)}) + return char_dict \ No newline at end of file diff --git a/reproduction/coreference_resolution/README.md b/reproduction/coreference_resolution/README.md index 7cbcd052..c1a286e5 100644 --- a/reproduction/coreference_resolution/README.md +++ b/reproduction/coreference_resolution/README.md @@ -1,4 +1,4 @@ -# 共指消解复现 +# 指代消解复现 ## 介绍 Coreference resolution是查找文本中指向同一现实实体的所有表达式的任务。 对于涉及自然语言理解的许多更高级别的NLP任务来说， diff --git a/reproduction/coreference_resolution/data_load/__init__.py b/reproduction/coreference_resolution/data_load/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/reproduction/coreference_resolution/data_load/cr_loader.py b/reproduction/coreference_resolution/data_load/cr_loader.py deleted file mode 100644 index 5ed73473..00000000 --- a/reproduction/coreference_resolution/data_load/cr_loader.py +++ /dev/null @@ -1,68 +0,0 @@ -from fastNLP.io.dataset_loader import JsonLoader,DataSet,Instance -from fastNLP.io.file_reader import _read_json -from fastNLP.core.vocabulary import Vocabulary -from fastNLP.io.data_bundle import DataBundle -from reproduction.coreference_resolution.model.config import Config -import reproduction.coreference_resolution.model.preprocess as preprocess - - -class CRLoader(JsonLoader): - def __init__(self, fields=None, dropna=False): - super().__init__(fields, dropna) - - def _load(self, path): - """ - 加载数据 - :param path: - :return: - """ - dataset = DataSet() - for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): - if self.fields: - ins = {self.fields[k]: v for k, v in d.items()} - else: - ins = d - dataset.append(Instance(**ins)) - return dataset - - def process(self, paths, **kwargs): - data_info = DataBundle() - for name in ['train', 'test', 'dev']: - data_info.datasets[name] = self.load(paths[name]) - - config = Config() - vocab = Vocabulary().from_dataset(*data_info.datasets.values(), field_name='sentences') - vocab.build_vocab() - word2id = vocab.word2idx - - char_dict = preprocess.get_char_dict(config.char_path) - data_info.vocabs = vocab - - genres = {g: i for i, g in enumerate(["bc", "bn", "mz", "nw", "pt", "tc", "wb"])} - - for name, ds in data_info.datasets.items(): - ds.apply(lambda x: preprocess.doc2numpy(x['sentences'], word2id, char_dict, max(config.filter), - config.max_sentences, is_train=name=='train')[0], - new_field_name='doc_np') - ds.apply(lambda x: preprocess.doc2numpy(x['sentences'], word2id, char_dict, max(config.filter), - config.max_sentences, is_train=name=='train')[1], - new_field_name='char_index') - ds.apply(lambda x: preprocess.doc2numpy(x['sentences'], word2id, char_dict, max(config.filter), - config.max_sentences, is_train=name=='train')[2], - new_field_name='seq_len') - ds.apply(lambda x: preprocess.speaker2numpy(x["speakers"], config.max_sentences, is_train=name=='train'), - new_field_name='speaker_ids_np') - ds.apply(lambda x: genres[x["doc_key"][:2]], new_field_name='genre') - - ds.set_ignore_type('clusters') - ds.set_padder('clusters', None) - ds.set_input("sentences", "doc_np", "speaker_ids_np", "genre", "char_index", "seq_len") - ds.set_target("clusters") - - # train_dev, test = self.ds.split(348 / (2802 + 343 + 348), shuffle=False) - # train, dev = train_dev.split(343 / (2802 + 343), shuffle=False) - - return data_info - - - diff --git a/reproduction/coreference_resolution/test/test_dataloader.py b/reproduction/coreference_resolution/test/test_dataloader.py index 0d9dae52..6a3be520 100644 --- a/reproduction/coreference_resolution/test/test_dataloader.py +++ b/reproduction/coreference_resolution/test/test_dataloader.py @@ -1,14 +1,14 @@ + + import unittest -from ..data_load.cr_loader import CRLoader +from fastNLP.io.pipe.coreference import CoreferencePipe +from reproduction.coreference_resolution.model.config import Config class Test_CRLoader(unittest.TestCase): def test_cr_loader(self): - train_path = 'data/train.english.jsonlines.mini' - dev_path = 'data/dev.english.jsonlines.minid' - test_path = 'data/test.english.jsonlines' - cr = CRLoader() - data_info = cr.process({'train':train_path,'dev':dev_path,'test':test_path}) - - print(data_info.datasets['train'][0]) - print(data_info.datasets['dev'][0]) - print(data_info.datasets['test'][0]) + config = Config() + bundle = CoreferencePipe(config).process_from_file({'train': config.train_path, 'dev': config.dev_path,'test': config.test_path}) + + print(bundle.datasets['train'][0]) + print(bundle.datasets['dev'][0]) + print(bundle.datasets['test'][0]) diff --git a/reproduction/coreference_resolution/train.py b/reproduction/coreference_resolution/train.py index a231a575..6c26cf4c 100644 --- a/reproduction/coreference_resolution/train.py +++ b/reproduction/coreference_resolution/train.py @@ -7,7 +7,8 @@ from torch.optim import Adam from fastNLP.core.callback import Callback, GradientClipCallback from fastNLP.core.trainer import Trainer -from reproduction.coreference_resolution.data_load.cr_loader import CRLoader +from fastNLP.io.pipe.coreference import CoreferencePipe + from reproduction.coreference_resolution.model.config import Config from reproduction.coreference_resolution.model.model_re import Model from reproduction.coreference_resolution.model.softmax_loss import SoftmaxLoss @@ -38,11 +39,8 @@ if __name__ == "__main__": @cache_results('cache.pkl') def cache(): - cr_train_dev_test = CRLoader() - - data_info = cr_train_dev_test.process({'train': config.train_path, 'dev': config.dev_path, - 'test': config.test_path}) - return data_info + bundle = CoreferencePipe(Config()).process_from_file({'train': config.train_path, 'dev': config.dev_path,'test': config.test_path}) + return bundle data_info = cache() print("数据集划分：\ntrain:", str(len(data_info.datasets["train"])), "\ndev:" + str(len(data_info.datasets["dev"])) + "\ntest:" + str(len(data_info.datasets["test"]))) diff --git a/reproduction/coreference_resolution/valid.py b/reproduction/coreference_resolution/valid.py index 826332c6..454629e1 100644 --- a/reproduction/coreference_resolution/valid.py +++ b/reproduction/coreference_resolution/valid.py @@ -1,7 +1,8 @@ import torch from reproduction.coreference_resolution.model.config import Config from reproduction.coreference_resolution.model.metric import CRMetric -from reproduction.coreference_resolution.data_load.cr_loader import CRLoader +from fastNLP.io.pipe.coreference import CoreferencePipe + from fastNLP import Tester import argparse @@ -11,13 +12,12 @@ if __name__=='__main__': parser.add_argument('--path') args = parser.parse_args() - cr_loader = CRLoader() config = Config() - data_info = cr_loader.process({'train': config.train_path, 'dev': config.dev_path, - 'test': config.test_path}) + bundle = CoreferencePipe(Config()).process_from_file( + {'train': config.train_path, 'dev': config.dev_path, 'test': config.test_path}) metirc = CRMetric() model = torch.load(args.path) - tester = Tester(data_info.datasets['test'],model,metirc,batch_size=1,device="cuda:0") + tester = Tester(bundle.datasets['test'],model,metirc,batch_size=1,device="cuda:0") tester.test() print('test over') From 19bbaf11b6989a1a29384d5b1516bf934ccac296 Mon Sep 17 00:00:00 2001 From: yh Date: Tue, 27 Aug 2019 01:54:15 +0800 Subject: [PATCH 05/92] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=9B=B4pytorch?= =?UTF-8?q?=E7=9A=84=E6=96=B9=E5=BC=8F=E5=A4=84=E7=90=86embedding=E4=B8=AD?= =?UTF-8?q?=E7=9A=84parameter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/embeddings/bert_embedding.py | 2 +- fastNLP/embeddings/char_embedding.py | 14 ++++++-------- fastNLP/embeddings/elmo_embedding.py | 5 ++--- fastNLP/embeddings/static_embedding.py | 16 +++++++--------- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index 6a10c489..f3ef69dd 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -345,7 +345,7 @@ class _WordBertModel(nn.Module): self._wordpiece_pad_index = self.tokenzier.vocab['[PAD]'] # 需要用于生成word_piece print("Found(Or segment into word pieces) {} words out of {}.".format(found_count, len(vocab))) self.word_to_wordpieces = np.array(word_to_wordpieces) - self.word_pieces_lengths = nn.Parameter(torch.LongTensor(word_pieces_lengths), requires_grad=False) + self.register_buffer('word_pieces_lengths', torch.LongTensor(word_pieces_lengths)) print("Successfully generate word pieces.") def forward(self, words): diff --git a/fastNLP/embeddings/char_embedding.py b/fastNLP/embeddings/char_embedding.py index 520e85e6..ea0d4e93 100644 --- a/fastNLP/embeddings/char_embedding.py +++ b/fastNLP/embeddings/char_embedding.py @@ -82,10 +82,9 @@ class CNNCharEmbedding(TokenEmbedding): print(f"In total, there are {len(self.char_vocab)} distinct characters.") # 对vocab进行index max_word_len = max(map(lambda x: len(x[0]), vocab)) - self.words_to_chars_embedding = nn.Parameter(torch.full((len(vocab), max_word_len), - fill_value=self.char_pad_index, dtype=torch.long), - requires_grad=False) - self.word_lengths = nn.Parameter(torch.zeros(len(vocab)).long(), requires_grad=False) + self.register_buffer('words_to_chars_embedding', torch.full((len(vocab), max_word_len), + fill_value=self.char_pad_index, dtype=torch.long)) + self.register_buffer('word_lengths', torch.zeros(len(vocab)).long()) for word, index in vocab: # if index!=vocab.padding_idx: # 如果是pad的话，直接就为pad_value了。修改为不区分pad, 这样所有的也是同一个embed self.words_to_chars_embedding[index, :len(word)] = \ @@ -235,10 +234,9 @@ class LSTMCharEmbedding(TokenEmbedding): print(f"In total, there are {len(self.char_vocab)} distinct characters.") # 对vocab进行index self.max_word_len = max(map(lambda x: len(x[0]), vocab)) - self.words_to_chars_embedding = nn.Parameter(torch.full((len(vocab), self.max_word_len), - fill_value=self.char_pad_index, dtype=torch.long), - requires_grad=False) - self.word_lengths = nn.Parameter(torch.zeros(len(vocab)).long(), requires_grad=False) + self.register_buffer('words_to_chars_embedding', torch.full((len(vocab), self.max_word_len), + fill_value=self.char_pad_index, dtype=torch.long)) + self.register_buffer('word_lengths', torch.zeros(len(vocab)).long()) for word, index in vocab: # if index!=vocab.padding_idx: # 如果是pad的话，直接就为pad_value了. 修改为不区分pad与否 self.words_to_chars_embedding[index, :len(word)] = \ diff --git a/fastNLP/embeddings/elmo_embedding.py b/fastNLP/embeddings/elmo_embedding.py index 24cd052e..80178d21 100644 --- a/fastNLP/embeddings/elmo_embedding.py +++ b/fastNLP/embeddings/elmo_embedding.py @@ -240,10 +240,9 @@ class _ElmoModel(nn.Module): # 生成words到chars的映射 max_chars = config['char_cnn']['max_characters_per_token'] - self.words_to_chars_embedding = nn.Parameter(torch.full((len(vocab) + 2, max_chars), + self.register_buffer('words_to_chars_embedding', torch.full((len(vocab) + 2, max_chars), fill_value=len(char_vocab), - dtype=torch.long), - requires_grad=False) + dtype=torch.long)) for word, index in list(iter(vocab)) + [(BOS_TAG, len(vocab)), (EOS_TAG, len(vocab) + 1)]: if len(word) + 2 > max_chars: word = word[:max_chars - 2] diff --git a/fastNLP/embeddings/static_embedding.py b/fastNLP/embeddings/static_embedding.py index a75ad18f..b0141682 100644 --- a/fastNLP/embeddings/static_embedding.py +++ b/fastNLP/embeddings/static_embedding.py @@ -121,28 +121,27 @@ class StaticEmbedding(TokenEmbedding): embedding = self._load_with_vocab(model_path, vocab=lowered_vocab, init_method=init_method) else: embedding = self._randomly_init_embed(len(vocab), embedding_dim, init_method) - self.words_to_words = nn.Parameter(torch.arange(len(vocab)).long(), requires_grad=False) + self.register_buffer('words_to_words', torch.arange(len(vocab)).long()) if lowered_vocab.unknown: unknown_idx = lowered_vocab.unknown_idx else: unknown_idx = embedding.size(0) - 1 # 否则是最后一个为unknow - self.words_to_words = nn.Parameter(torch.arange(len(vocab)).long(), requires_grad=False) - words_to_words = nn.Parameter(torch.full((len(vocab),), fill_value=unknown_idx).long(), - requires_grad=False) + self.register_buffer('words_to_words', torch.arange(len(vocab)).long()) + words_to_words = torch.full((len(vocab),), fill_value=unknown_idx).long() for word, index in vocab: if word not in lowered_vocab: word = word.lower() if word not in lowered_vocab and lowered_vocab._is_word_no_create_entry(word): continue # 如果不需要创建entry,已经默认unknown了 words_to_words[index] = self.words_to_words[lowered_vocab.to_index(word)] - self.words_to_words = words_to_words + self.register_buffer('words_to_words', words_to_words) self._word_unk_index = lowered_vocab.unknown_idx # 替换一下unknown的index else: if model_path: embedding = self._load_with_vocab(model_path, vocab=vocab, init_method=init_method) else: embedding = self._randomly_init_embed(len(vocab), embedding_dim, init_method) - self.words_to_words = nn.Parameter(torch.arange(len(vocab)).long(), requires_grad=False) + self.register_buffer('words_to_words', torch.arange(len(vocab)).long()) if not self.only_norm_found_vector and normalize: embedding /= (torch.norm(embedding, dim=1, keepdim=True) + 1e-12) @@ -151,7 +150,7 @@ class StaticEmbedding(TokenEmbedding): index_in_truncated_vocab = truncated_words_to_words[i] truncated_words_to_words[i] = self.words_to_words[index_in_truncated_vocab] del self.words_to_words - self.words_to_words = nn.Parameter(truncated_words_to_words, requires_grad=False) + self.register_buffer('words_to_words', truncated_words_to_words) self.embedding = nn.Embedding(num_embeddings=embedding.shape[0], embedding_dim=embedding.shape[1], padding_idx=vocab.padding_idx, @@ -273,8 +272,7 @@ class StaticEmbedding(TokenEmbedding): vectors = torch.cat((vectors, torch.zeros(1, dim)), dim=0).contiguous() else: unknown_idx = vocab.unknown_idx - self.words_to_words = nn.Parameter(torch.full((len(vocab), ), fill_value=unknown_idx).long(), - requires_grad=False) + self.register_buffer('words_to_words', torch.full((len(vocab), ), fill_value=unknown_idx).long()) for index, (index_in_vocab, vec) in enumerate(matrix.items()): if vec is not None: From 04737a105d1d57c334ebb664cac64d4331a8593a Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 27 Aug 2019 20:46:05 +0800 Subject: [PATCH 06/92] update the doc-tool to show __init__ and class doc separately --- docs/count.py | 7 ++++--- docs/source/conf.py | 6 ++++-- docs/source/fastNLP.core.rst | 3 +-- docs/source/fastNLP.embeddings.rst | 1 + docs/source/fastNLP.io.rst | 1 + docs/source/fastNLP.models.biaffine_parser.rst | 1 - docs/source/fastNLP.models.cnn_text_classification.rst | 1 - docs/source/fastNLP.models.rst | 2 +- docs/source/fastNLP.models.sequence_labeling.rst | 1 - docs/source/fastNLP.models.snli.rst | 1 - docs/source/fastNLP.models.star_transformer.rst | 1 - docs/source/fastNLP.modules.decoder.rst | 1 - docs/source/fastNLP.modules.encoder.rst | 1 - docs/source/fastNLP.modules.rst | 2 +- docs/source/fastNLP.modules.utils.rst | 1 - docs/source/fastNLP.rst | 1 + 16 files changed, 14 insertions(+), 17 deletions(-) diff --git a/docs/count.py b/docs/count.py index 72868403..c75173ef 100644 --- a/docs/count.py +++ b/docs/count.py @@ -66,12 +66,13 @@ def create_rst_file(modules, name, children): fout.write(t + "\n") fout.write("\n") fout.write(".. automodule:: " + name + "\n") - if len(m.__all__) > 0: + if name != "fastNLP.core" and len(m.__all__) > 0: fout.write(" :members: " + ", ".join(m.__all__) + "\n") - fout.write(" :inherited-members:\n") + if not (name.startswith('fastNLP.models') or name.startswith('fastNLP.modules')): + fout.write(" :inherited-members:\n") fout.write("\n") if name in children: - fout.write("子模块\n------\n\n.. toctree::\n\n") + fout.write("子模块\n------\n\n.. toctree::\n :maxdepth: 1\n\n") for module in children[name]: fout.write(" " + module + "\n") diff --git a/docs/source/conf.py b/docs/source/conf.py index 83cb7185..7536ee32 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -168,10 +168,12 @@ texinfo_documents = [ # -- Extension configuration ------------------------------------------------- def maybe_skip_member(app, what, name, obj, skip, options): - if name.startswith("_"): - return True if obj.__doc__ is None: return True + if name == "__init__": + return False + if name.startswith("_"): + return True return False diff --git a/docs/source/fastNLP.core.rst b/docs/source/fastNLP.core.rst index 56de46e9..15fe29d5 100644 --- a/docs/source/fastNLP.core.rst +++ b/docs/source/fastNLP.core.rst @@ -2,13 +2,12 @@ fastNLP.core ============ .. automodule:: fastNLP.core - :members: DataSet, Instance, FieldArray, Padder, AutoPadder, EngChar2DPadder, Vocabulary, DataSetIter, BatchIter, TorchLoaderIter, Const, Tester, Trainer, cache_results, seq_len_to_mask, get_seq_len, logger, Callback, GradientClipCallback, EarlyStopCallback, FitlogCallback, EvaluateCallback, LRScheduler, ControlC, LRFinder, TensorboardCallback, WarmupCallback, SaveModelCallback, EchoCallback, TesterCallback, CallbackException, EarlyStopError, LossFunc, CrossEntropyLoss, L1Loss, BCELoss, NLLLoss, LossInForward, AccuracyMetric, SpanFPreRecMetric, ExtractiveQAMetric, Optimizer, SGD, Adam, AdamW, SequentialSampler, BucketSampler, RandomSampler, Sampler - :inherited-members: 子模块 ------ .. toctree:: + :maxdepth: 1 fastNLP.core.batch fastNLP.core.callback diff --git a/docs/source/fastNLP.embeddings.rst b/docs/source/fastNLP.embeddings.rst index 8376408c..b9e6a853 100644 --- a/docs/source/fastNLP.embeddings.rst +++ b/docs/source/fastNLP.embeddings.rst @@ -9,6 +9,7 @@ fastNLP.embeddings ------ .. toctree:: + :maxdepth: 1 fastNLP.embeddings.bert_embedding fastNLP.embeddings.char_embedding diff --git a/docs/source/fastNLP.io.rst b/docs/source/fastNLP.io.rst index 2aacb883..96df9d6c 100644 --- a/docs/source/fastNLP.io.rst +++ b/docs/source/fastNLP.io.rst @@ -9,6 +9,7 @@ fastNLP.io ------ .. toctree:: + :maxdepth: 1 fastNLP.io.data_bundle fastNLP.io.embed_loader diff --git a/docs/source/fastNLP.models.biaffine_parser.rst b/docs/source/fastNLP.models.biaffine_parser.rst index c3dbb0a5..395638fe 100644 --- a/docs/source/fastNLP.models.biaffine_parser.rst +++ b/docs/source/fastNLP.models.biaffine_parser.rst @@ -3,5 +3,4 @@ fastNLP.models.biaffine_parser .. automodule:: fastNLP.models.biaffine_parser :members: BiaffineParser, GraphParser - :inherited-members: diff --git a/docs/source/fastNLP.models.cnn_text_classification.rst b/docs/source/fastNLP.models.cnn_text_classification.rst index fe4bb157..e9ed7ee1 100644 --- a/docs/source/fastNLP.models.cnn_text_classification.rst +++ b/docs/source/fastNLP.models.cnn_text_classification.rst @@ -3,5 +3,4 @@ fastNLP.models.cnn_text_classification .. automodule:: fastNLP.models.cnn_text_classification :members: CNNText - :inherited-members: diff --git a/docs/source/fastNLP.models.rst b/docs/source/fastNLP.models.rst index 88854a79..fb782de1 100644 --- a/docs/source/fastNLP.models.rst +++ b/docs/source/fastNLP.models.rst @@ -3,12 +3,12 @@ fastNLP.models .. automodule:: fastNLP.models :members: CNNText, SeqLabeling, AdvSeqLabel, ESIM, StarTransEnc, STSeqLabel, STNLICls, STSeqCls, BiaffineParser, GraphParser - :inherited-members: 子模块 ------ .. toctree:: + :maxdepth: 1 fastNLP.models.biaffine_parser fastNLP.models.cnn_text_classification diff --git a/docs/source/fastNLP.models.sequence_labeling.rst b/docs/source/fastNLP.models.sequence_labeling.rst index b66e637e..f6551f8b 100644 --- a/docs/source/fastNLP.models.sequence_labeling.rst +++ b/docs/source/fastNLP.models.sequence_labeling.rst @@ -3,5 +3,4 @@ fastNLP.models.sequence_labeling .. automodule:: fastNLP.models.sequence_labeling :members: SeqLabeling, AdvSeqLabel - :inherited-members: diff --git a/docs/source/fastNLP.models.snli.rst b/docs/source/fastNLP.models.snli.rst index 8551051a..eed02139 100644 --- a/docs/source/fastNLP.models.snli.rst +++ b/docs/source/fastNLP.models.snli.rst @@ -3,5 +3,4 @@ fastNLP.models.snli .. automodule:: fastNLP.models.snli :members: ESIM - :inherited-members: diff --git a/docs/source/fastNLP.models.star_transformer.rst b/docs/source/fastNLP.models.star_transformer.rst index f4b5989e..80ab5b33 100644 --- a/docs/source/fastNLP.models.star_transformer.rst +++ b/docs/source/fastNLP.models.star_transformer.rst @@ -3,5 +3,4 @@ fastNLP.models.star_transformer .. automodule:: fastNLP.models.star_transformer :members: StarTransEnc, STNLICls, STSeqCls, STSeqLabel - :inherited-members: diff --git a/docs/source/fastNLP.modules.decoder.rst b/docs/source/fastNLP.modules.decoder.rst index b121f9e9..de6e0d9d 100644 --- a/docs/source/fastNLP.modules.decoder.rst +++ b/docs/source/fastNLP.modules.decoder.rst @@ -3,5 +3,4 @@ fastNLP.modules.decoder .. automodule:: fastNLP.modules.decoder :members: MLP, ConditionalRandomField, viterbi_decode, allowed_transitions - :inherited-members: diff --git a/docs/source/fastNLP.modules.encoder.rst b/docs/source/fastNLP.modules.encoder.rst index 6b44a192..fceabbdb 100644 --- a/docs/source/fastNLP.modules.encoder.rst +++ b/docs/source/fastNLP.modules.encoder.rst @@ -3,5 +3,4 @@ fastNLP.modules.encoder .. automodule:: fastNLP.modules.encoder :members: ConvolutionCharEncoder, LSTMCharEncoder, ConvMaxpool, LSTM, StarTransformer, TransformerEncoder, VarRNN, VarLSTM, VarGRU, MaxPool, MaxPoolWithMask, AvgPool, AvgPoolWithMask, MultiHeadAttention - :inherited-members: diff --git a/docs/source/fastNLP.modules.rst b/docs/source/fastNLP.modules.rst index 6134d0dd..b7c259ed 100644 --- a/docs/source/fastNLP.modules.rst +++ b/docs/source/fastNLP.modules.rst @@ -3,12 +3,12 @@ fastNLP.modules .. automodule:: fastNLP.modules :members: ConvolutionCharEncoder, LSTMCharEncoder, ConvMaxpool, LSTM, StarTransformer, TransformerEncoder, VarRNN, VarLSTM, VarGRU, MaxPool, MaxPoolWithMask, AvgPool, AvgPoolWithMask, MultiHeadAttention, MLP, ConditionalRandomField, viterbi_decode, allowed_transitions, TimestepDropout - :inherited-members: 子模块 ------ .. toctree:: + :maxdepth: 1 fastNLP.modules.decoder fastNLP.modules.encoder diff --git a/docs/source/fastNLP.modules.utils.rst b/docs/source/fastNLP.modules.utils.rst index e28ca35a..101a0f45 100644 --- a/docs/source/fastNLP.modules.utils.rst +++ b/docs/source/fastNLP.modules.utils.rst @@ -3,5 +3,4 @@ fastNLP.modules.utils .. automodule:: fastNLP.modules.utils :members: initial_parameter, summary - :inherited-members: diff --git a/docs/source/fastNLP.rst b/docs/source/fastNLP.rst index f22ea936..e01817f7 100644 --- a/docs/source/fastNLP.rst +++ b/docs/source/fastNLP.rst @@ -9,6 +9,7 @@ fastNLP ------ .. toctree:: + :maxdepth: 1 fastNLP.core fastNLP.embeddings From 169f519ffb0133b5f553d04c17c9f2cac0edebcb Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 27 Aug 2019 21:07:22 +0800 Subject: [PATCH 07/92] ignore the methods inherited from torch.nn.Embedding --- docs/count.py | 3 ++- docs/source/fastNLP.embeddings.bert_embedding.rst | 1 - docs/source/fastNLP.embeddings.char_embedding.rst | 1 - docs/source/fastNLP.embeddings.contextual_embedding.rst | 1 - docs/source/fastNLP.embeddings.elmo_embedding.rst | 1 - docs/source/fastNLP.embeddings.embedding.rst | 1 - docs/source/fastNLP.embeddings.rst | 1 - docs/source/fastNLP.embeddings.stack_embedding.rst | 1 - docs/source/fastNLP.embeddings.static_embedding.rst | 1 - docs/source/fastNLP.embeddings.utils.rst | 1 - docs/source/fastNLP.io.dataset_loader.rst | 6 ------ 11 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 docs/source/fastNLP.io.dataset_loader.rst diff --git a/docs/count.py b/docs/count.py index c75173ef..6a5d256b 100644 --- a/docs/count.py +++ b/docs/count.py @@ -68,7 +68,8 @@ def create_rst_file(modules, name, children): fout.write(".. automodule:: " + name + "\n") if name != "fastNLP.core" and len(m.__all__) > 0: fout.write(" :members: " + ", ".join(m.__all__) + "\n") - if not (name.startswith('fastNLP.models') or name.startswith('fastNLP.modules')): + short = name[len("fastNLP."):] + if not (short.startswith('models') or short.startswith('modules') or short.startswith('embeddings')): fout.write(" :inherited-members:\n") fout.write("\n") if name in children: diff --git a/docs/source/fastNLP.embeddings.bert_embedding.rst b/docs/source/fastNLP.embeddings.bert_embedding.rst index 51828cb0..1b59dc35 100644 --- a/docs/source/fastNLP.embeddings.bert_embedding.rst +++ b/docs/source/fastNLP.embeddings.bert_embedding.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.bert_embedding .. automodule:: fastNLP.embeddings.bert_embedding :members: BertEmbedding, BertWordPieceEncoder - :inherited-members: diff --git a/docs/source/fastNLP.embeddings.char_embedding.rst b/docs/source/fastNLP.embeddings.char_embedding.rst index a9b129d8..bc8d64f9 100644 --- a/docs/source/fastNLP.embeddings.char_embedding.rst +++ b/docs/source/fastNLP.embeddings.char_embedding.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.char_embedding .. automodule:: fastNLP.embeddings.char_embedding :members: CNNCharEmbedding, LSTMCharEmbedding - :inherited-members: diff --git a/docs/source/fastNLP.embeddings.contextual_embedding.rst b/docs/source/fastNLP.embeddings.contextual_embedding.rst index ee64c7a0..74e5f5be 100644 --- a/docs/source/fastNLP.embeddings.contextual_embedding.rst +++ b/docs/source/fastNLP.embeddings.contextual_embedding.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.contextual_embedding .. automodule:: fastNLP.embeddings.contextual_embedding :members: ContextualEmbedding - :inherited-members: diff --git a/docs/source/fastNLP.embeddings.elmo_embedding.rst b/docs/source/fastNLP.embeddings.elmo_embedding.rst index 06cc13af..b8c6d41c 100644 --- a/docs/source/fastNLP.embeddings.elmo_embedding.rst +++ b/docs/source/fastNLP.embeddings.elmo_embedding.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.elmo_embedding .. automodule:: fastNLP.embeddings.elmo_embedding :members: ElmoEmbedding - :inherited-members: diff --git a/docs/source/fastNLP.embeddings.embedding.rst b/docs/source/fastNLP.embeddings.embedding.rst index 4d5fcf46..6793446b 100644 --- a/docs/source/fastNLP.embeddings.embedding.rst +++ b/docs/source/fastNLP.embeddings.embedding.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.embedding .. automodule:: fastNLP.embeddings.embedding :members: Embedding, TokenEmbedding - :inherited-members: diff --git a/docs/source/fastNLP.embeddings.rst b/docs/source/fastNLP.embeddings.rst index b9e6a853..f4f4a3e0 100644 --- a/docs/source/fastNLP.embeddings.rst +++ b/docs/source/fastNLP.embeddings.rst @@ -3,7 +3,6 @@ fastNLP.embeddings .. automodule:: fastNLP.embeddings :members: Embedding, TokenEmbedding, StaticEmbedding, ElmoEmbedding, BertEmbedding, BertWordPieceEncoder, StackEmbedding, LSTMCharEmbedding, CNNCharEmbedding, get_embeddings - :inherited-members: 子模块 ------ diff --git a/docs/source/fastNLP.embeddings.stack_embedding.rst b/docs/source/fastNLP.embeddings.stack_embedding.rst index 6af91623..a07d1ef5 100644 --- a/docs/source/fastNLP.embeddings.stack_embedding.rst +++ b/docs/source/fastNLP.embeddings.stack_embedding.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.stack_embedding .. automodule:: fastNLP.embeddings.stack_embedding :members: StackEmbedding - :inherited-members: diff --git a/docs/source/fastNLP.embeddings.static_embedding.rst b/docs/source/fastNLP.embeddings.static_embedding.rst index 2df1c329..219ce0e5 100644 --- a/docs/source/fastNLP.embeddings.static_embedding.rst +++ b/docs/source/fastNLP.embeddings.static_embedding.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.static_embedding .. automodule:: fastNLP.embeddings.static_embedding :members: StaticEmbedding - :inherited-members: diff --git a/docs/source/fastNLP.embeddings.utils.rst b/docs/source/fastNLP.embeddings.utils.rst index 13e5936b..077487c1 100644 --- a/docs/source/fastNLP.embeddings.utils.rst +++ b/docs/source/fastNLP.embeddings.utils.rst @@ -3,5 +3,4 @@ fastNLP.embeddings.utils .. automodule:: fastNLP.embeddings.utils :members: get_embeddings - :inherited-members: diff --git a/docs/source/fastNLP.io.dataset_loader.rst b/docs/source/fastNLP.io.dataset_loader.rst deleted file mode 100644 index c211ecf9..00000000 --- a/docs/source/fastNLP.io.dataset_loader.rst +++ /dev/null @@ -1,6 +0,0 @@ -fastNLP.io.dataset_loader -========================= - -.. automodule:: fastNLP.io.dataset_loader - :members: CSVLoader, JsonLoader - From fbbb2fcd8e6526143cd789f9bb7e370d966ac4c4 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 27 Aug 2019 21:33:18 +0800 Subject: [PATCH 08/92] fix some bugs in docs --- fastNLP/core/callback.py | 21 ++++++++++++--------- fastNLP/io/data_bundle.py | 4 ++-- fastNLP/io/pipe/conll.py | 4 ++-- fastNLP/io/pipe/matching.py | 4 ++-- fastNLP/io/pipe/pipe.py | 2 +- fastNLP/io/pipe/utils.py | 4 ++-- 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index 2c130061..dde9a31a 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -70,10 +70,11 @@ __all__ = [ ] import os +import sys +from copy import deepcopy import torch -from copy import deepcopy -import sys + from .utils import _save_model try: @@ -928,13 +929,15 @@ class WarmupCallback(Callback): class SaveModelCallback(Callback): """ 由于Trainer在训练过程中只会保存最佳的模型， 该callback可实现多种方式的结果存储。 - 会根据训练开始的时间戳在save_dir下建立文件夹，再在文件夹下存放多个模型 - -save_dir - -2019-07-03-15-06-36 - -epoch:0_step:20_{metric_key}:{evaluate_performance}.pt # metric是给定的metric_key, evaluate_performance是性能 - -epoch:1_step:40_{metric_key}:{evaluate_performance}.pt - -2019-07-03-15-10-00 - -epoch:0_step:20_{metric_key}:{evaluate_performance}.pt # metric是给定的metric_key, evaluate_perfomance是性能 + 会根据训练开始的时间戳在save_dir下建立文件夹，再在文件夹下存放多个模型:: + + -save_dir + -2019-07-03-15-06-36 + -epoch:0_step:20_{metric_key}:{evaluate_performance}.pt # metric是给定的metric_key, evaluate_performance是性能 + -epoch:1_step:40_{metric_key}:{evaluate_performance}.pt + -2019-07-03-15-10-00 + -epoch:0_step:20_{metric_key}:{evaluate_performance}.pt # metric是给定的metric_key, evaluate_perfomance是性能 + :param str save_dir: 将模型存放在哪个目录下，会在该目录下创建以时间戳命名的目录，并存放模型 :param int top: 保存dev表现top多少模型。-1为保存所有模型。 :param bool only_param: 是否只保存模型d饿权重。 diff --git a/fastNLP/io/data_bundle.py b/fastNLP/io/data_bundle.py index db60a86f..10f924f0 100644 --- a/fastNLP/io/data_bundle.py +++ b/fastNLP/io/data_bundle.py @@ -204,7 +204,7 @@ class DataBundle: 行的数据进行类型和维度推断本列的数据的类型和维度。 :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; 如果为False，则报错 - :return self + :return: self """ for field_name in field_names: for name, dataset in self.datasets.items(): @@ -229,7 +229,7 @@ class DataBundle: 行的数据进行类型和维度推断本列的数据的类型和维度。 :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; 如果为False，则报错 - :return self + :return: self """ for field_name in field_names: for name, dataset in self.datasets.items(): diff --git a/fastNLP/io/pipe/conll.py b/fastNLP/io/pipe/conll.py index 2efec8e0..eb7d4909 100644 --- a/fastNLP/io/pipe/conll.py +++ b/fastNLP/io/pipe/conll.py @@ -51,7 +51,7 @@ class _NERPipe(Pipe): "[AL-AIN, United, Arab, ...]", "[B-LOC, B-LOC, I-LOC, ...]" "[...]", "[...]" - :param DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]。 + :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]。 在传入DataBundle基础上原位修改。 :return: DataBundle """ @@ -244,7 +244,7 @@ class _CNNERPipe(Pipe): raw_chars列为List[str], 是未转换的原始数据; chars列为List[int]，是转换为index的输入数据; target列是List[int]，是转换为index的 target。返回的DataSet中被设置为input有chars, target, seq_len; 设置为target有target。 - :param DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]。 + :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]。 在传入DataBundle基础上原位修改。 :return: DataBundle """ diff --git a/fastNLP/io/pipe/matching.py b/fastNLP/io/pipe/matching.py index 699438c8..747e7b44 100644 --- a/fastNLP/io/pipe/matching.py +++ b/fastNLP/io/pipe/matching.py @@ -177,7 +177,7 @@ class MatchingPipe(Pipe): def _tokenize(self, data_bundle, field_names, new_field_names): """ - :param DataBundle data_bundle: DataBundle. + :param ~fastNLP.DataBundle data_bundle: DataBundle. :param list field_names: List[str], 需要tokenize的field名称 :param list new_field_names: List[str], tokenize之后field的名称，与field_names一一对应。 :return: 输入的DataBundle对象 @@ -199,7 +199,7 @@ class MatchingPipe(Pipe): "This site includes a...", "The Government Executive...", "not_entailment" "...", "..." - :param data_bundle: 通过loader读取得到的data_bundle，里面包含了数据集的原始数据内容 + :param ~fastNLP.DataBundle data_bundle: 通过loader读取得到的data_bundle，里面包含了数据集的原始数据内容 :return: data_bundle """ data_bundle = self._tokenize(data_bundle, [Const.RAW_WORDS(0), Const.RAW_WORDS(1)], diff --git a/fastNLP/io/pipe/pipe.py b/fastNLP/io/pipe/pipe.py index a1435fd3..12d9c1cb 100644 --- a/fastNLP/io/pipe/pipe.py +++ b/fastNLP/io/pipe/pipe.py @@ -15,7 +15,7 @@ class Pipe: """ 对输入的DataBundle进行处理，然后返回该DataBundle。 - :param data_bundle: 需要处理的DataBundle对象 + :param ~fastNLP.DataBundle data_bundle: 需要处理的DataBundle对象 :return: """ raise NotImplementedError diff --git a/fastNLP/io/pipe/utils.py b/fastNLP/io/pipe/utils.py index f32f58b7..ea7e0aa8 100644 --- a/fastNLP/io/pipe/utils.py +++ b/fastNLP/io/pipe/utils.py @@ -92,7 +92,7 @@ def _indexize(data_bundle, input_field_names=Const.INPUT, target_field_names=Con """ 在dataset中的field_name列建立词表，Const.TARGET列建立词表，并把词表加入到data_bundle中。 - :param data_bundle: + :param ~fastNLP.DataBundle data_bundle: :param: str,list input_field_names: :param: str,list target_field_names: 这一列的vocabulary没有unknown和padding :return: @@ -154,7 +154,7 @@ def _drop_empty_instance(data_bundle, field_name): """ 删除data_bundle的DataSet中存在的某个field为空的情况 - :param data_bundle: DataBundle + :param ~fastNLP.DataBundle data_bundle: :param str field_name: 对哪个field进行检查，如果为None，则任意field为空都会删掉 :return: 传入的DataBundle """ From 6201f661789e36c4e1e116846cc84d586aca2abd Mon Sep 17 00:00:00 2001 From: yh_cc Date: Wed, 28 Aug 2019 22:56:02 +0800 Subject: [PATCH 09/92] =?UTF-8?q?Trainer=E4=B8=AD=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E6=9C=80=E4=BD=B3=E6=A8=A1=E5=9E=8B=E5=AD=98=E5=9C=A8bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 290a89c1..61969c2e 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -718,7 +718,7 @@ class Trainer(object): self._save_model(self.model, "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time])) elif self._load_best_model: - self._best_model_states = {name: param.cpu().clone() for name, param in self.model.named_parameters()} + self._best_model_states = {name: param.cpu().clone() for name, param in self.model.state_dict()} self.best_dev_perf = res self.best_dev_epoch = epoch self.best_dev_step = step From a46b8f129b88ef5b53692f18cf609ceeb31e48c0 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Wed, 28 Aug 2019 23:06:13 +0800 Subject: [PATCH 10/92] =?UTF-8?q?Trainer=E4=B8=AD=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E6=9C=80=E4=BD=B3=E6=A8=A1=E5=9E=8B=E5=AD=98=E5=9C=A8bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 61969c2e..a47f108b 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -718,7 +718,7 @@ class Trainer(object): self._save_model(self.model, "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time])) elif self._load_best_model: - self._best_model_states = {name: param.cpu().clone() for name, param in self.model.state_dict()} + self._best_model_states = {name: param.cpu().clone() for name, param in self.model.state_dict().items()} self.best_dev_perf = res self.best_dev_epoch = epoch self.best_dev_step = step From 55e736bf4c9020ce404400b605d1c2febd8d0766 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Wed, 28 Aug 2019 23:53:20 +0800 Subject: [PATCH 11/92] =?UTF-8?q?SpanFMetric=E5=A2=9E=E5=8A=A0=E5=AF=B9enc?= =?UTF-8?q?oding=5Ftype=E5=92=8Ctag=5Fvocab=E7=9A=84=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/metrics.py | 26 ++++++++++++++++++++++++++ test/core/test_metrics.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index 1d1e3819..28d88fbc 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -23,6 +23,7 @@ from .utils import _get_func_signature from .utils import seq_len_to_mask from .vocabulary import Vocabulary from abc import abstractmethod +import warnings class MetricBase(object): @@ -492,6 +493,30 @@ def _bio_tag_to_spans(tags, ignore_labels=None): return [(span[0], (span[1][0], span[1][1] + 1)) for span in spans if span[0] not in ignore_labels] +def _check_tag_vocab_and_encoding_type(vocab:Vocabulary, encoding_type:str): + """ + 检查vocab中的tag是否与encoding_type是匹配的 + + :param vocab: target的Vocabulary + :param encoding_type: bio, bmes, bioes, bmeso + :return: + """ + tag_set = set() + for tag, idx in vocab: + if idx in (vocab.unknown_idx, vocab.padding_idx): + continue + tag = tag[:1] + tag_set.add(tag) + tags = encoding_type + for tag in tag_set: + assert tag in tags, f"{tag} is not a valid tag in encoding type:{encoding_type}. Please check your " \ + f"encoding_type." + tags = tags.replace(tag, '') # 删除该值 + if tags: # 如果不为空，说明出现了未使用的tag + warnings.warn(f"Tag:{tags} in encoding type:{encoding_type} is not presented in your Vocabulary. Check your " + "encoding_type.") + + class SpanFPreRecMetric(MetricBase): r""" 别名：:class:`fastNLP.SpanFPreRecMetric` :class:`fastNLP.core.metrics.SpanFPreRecMetric` @@ -546,6 +571,7 @@ class SpanFPreRecMetric(MetricBase): raise ValueError("f_type only supports `micro` or `macro`', got {}.".format(f_type)) self.encoding_type = encoding_type + _check_tag_vocab_and_encoding_type(tag_vocab, encoding_type) if self.encoding_type == 'bmes': self.tag_to_span_func = _bmes_tag_to_spans elif self.encoding_type == 'bio': diff --git a/test/core/test_metrics.py b/test/core/test_metrics.py index 236066d6..5a7c55cf 100644 --- a/test/core/test_metrics.py +++ b/test/core/test_metrics.py @@ -338,6 +338,41 @@ class SpanF1PreRecMetric(unittest.TestCase): for key, value in expected_metric.items(): self.assertAlmostEqual(value, metric_value[key], places=5) + def test_encoding_type(self): + # 检查传入的tag_vocab与encoding_type不符合时，是否会报错 + vocabs = {} + import random + from itertools import product + for encoding_type in ['bio', 'bioes', 'bmeso']: + vocab = Vocabulary(unknown=None, padding=None) + for i in range(random.randint(10, 100)): + label = str(random.randint(1, 10)) + for tag in encoding_type: + if tag!='o': + vocab.add_word(f'{tag}-{label}') + else: + vocab.add_word('o') + vocabs[encoding_type] = vocab + for e1, e2 in product(['bio', 'bioes', 'bmeso'], ['bio', 'bioes', 'bmeso']): + with self.subTest(e1=e1, e2=e2): + if e1==e2: + metric = SpanFPreRecMetric(vocabs[e1], encoding_type=e2) + else: + s2 = set(e2) + s2.update(set(e1)) + if s2==set(e2): + continue + with self.assertRaises(AssertionError): + metric = SpanFPreRecMetric(vocabs[e1], encoding_type=e2) + for encoding_type in ['bio', 'bioes', 'bmeso']: + with self.assertRaises(AssertionError): + metric = SpanFPreRecMetric(vocabs[encoding_type], encoding_type='bmes') + + with self.assertWarns(Warning): + vocab = Vocabulary(unknown=None, padding=None).add_word_lst(list('bmes')) + metric = SpanFPreRecMetric(vocab, encoding_type='bmeso') + vocab = Vocabulary().add_word_lst(list('bmes')) + metric = SpanFPreRecMetric(vocab, encoding_type='bmeso') class TestUsefulFunctions(unittest.TestCase): # 测试metrics.py中一些看上去挺有用的函数 From cbe5b347e54ce5181887743c62b06aabcd00b778 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Wed, 28 Aug 2019 23:53:53 +0800 Subject: [PATCH 12/92] =?UTF-8?q?SpanFMetric=E5=A2=9E=E5=8A=A0=E5=AF=B9enc?= =?UTF-8?q?oding=5Ftype=E5=92=8Ctag=5Fvocab=E7=9A=84=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index 28d88fbc..0dc601a3 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -505,7 +505,7 @@ def _check_tag_vocab_and_encoding_type(vocab:Vocabulary, encoding_type:str): for tag, idx in vocab: if idx in (vocab.unknown_idx, vocab.padding_idx): continue - tag = tag[:1] + tag = tag[:1].lower() tag_set.add(tag) tags = encoding_type for tag in tag_set: From 5d8a8c98c6997fda7afa236de8523c0c1916201d Mon Sep 17 00:00:00 2001 From: xuyige Date: Thu, 29 Aug 2019 01:43:04 +0800 Subject: [PATCH 13/92] 1. delete io/data_loader dir; 2. delete model/enas*; 3. delete legacy dir; 4. delete DateSetLoader and relevant codes; 5. fix a test code error in core/test_dataset.py; 6. delete io.BaseLoader and relevant code. --- fastNLP/io/__init__.py | 1 - fastNLP/io/data_bundle.py | 197 +---------- fastNLP/io/data_loader/__init__.py | 39 --- fastNLP/io/data_loader/conll.py | 109 ------ fastNLP/io/data_loader/imdb.py | 99 ------ fastNLP/io/data_loader/matching.py | 248 ------------- fastNLP/io/data_loader/mnli.py | 62 ---- fastNLP/io/data_loader/mtl.py | 68 ---- fastNLP/io/data_loader/people_daily.py | 85 ----- fastNLP/io/data_loader/qnli.py | 47 --- fastNLP/io/data_loader/quora.py | 34 -- fastNLP/io/data_loader/rte.py | 47 --- fastNLP/io/data_loader/snli.py | 46 --- fastNLP/io/data_loader/sst.py | 180 ---------- fastNLP/io/data_loader/yelp.py | 132 ------- fastNLP/io/dataset_loader.py | 121 ------- fastNLP/io/embed_loader.py | 13 +- fastNLP/io/model_io.py | 4 +- fastNLP/models/enas_controller.py | 228 ------------ fastNLP/models/enas_model.py | 393 --------------------- fastNLP/models/enas_trainer.py | 384 -------------------- fastNLP/models/enas_utils.py | 58 ---- legacy/api/README.md | 44 --- legacy/api/__init__.py | 2 - legacy/api/api.py | 463 ------------------------- legacy/api/converter.py | 181 ---------- legacy/api/examples.py | 56 --- legacy/api/pipeline.py | 33 -- legacy/api/processor.py | 428 ----------------------- legacy/api/utils.py | 134 ------- legacy/automl/__init__.py | 0 legacy/automl/enas_controller.py | 223 ------------ legacy/automl/enas_model.py | 388 --------------------- legacy/automl/enas_trainer.py | 383 -------------------- legacy/automl/enas_utils.py | 53 --- legacy/component/__init__.py | 1 - legacy/component/bert_tokenizer.py | 378 -------------------- test/core/test_dataset.py | 5 +- test/io/test_data_loader.py | 15 - test/io/test_dataset_loader.py | 77 ---- 40 files changed, 14 insertions(+), 5445 deletions(-) delete mode 100644 fastNLP/io/data_loader/__init__.py delete mode 100644 fastNLP/io/data_loader/conll.py delete mode 100644 fastNLP/io/data_loader/imdb.py delete mode 100644 fastNLP/io/data_loader/matching.py delete mode 100644 fastNLP/io/data_loader/mnli.py delete mode 100644 fastNLP/io/data_loader/mtl.py delete mode 100644 fastNLP/io/data_loader/people_daily.py delete mode 100644 fastNLP/io/data_loader/qnli.py delete mode 100644 fastNLP/io/data_loader/quora.py delete mode 100644 fastNLP/io/data_loader/rte.py delete mode 100644 fastNLP/io/data_loader/snli.py delete mode 100644 fastNLP/io/data_loader/sst.py delete mode 100644 fastNLP/io/data_loader/yelp.py delete mode 100644 fastNLP/io/dataset_loader.py delete mode 100644 fastNLP/models/enas_controller.py delete mode 100644 fastNLP/models/enas_model.py delete mode 100644 fastNLP/models/enas_trainer.py delete mode 100644 fastNLP/models/enas_utils.py delete mode 100644 legacy/api/README.md delete mode 100644 legacy/api/__init__.py delete mode 100644 legacy/api/api.py delete mode 100644 legacy/api/converter.py delete mode 100644 legacy/api/examples.py delete mode 100644 legacy/api/pipeline.py delete mode 100644 legacy/api/processor.py delete mode 100644 legacy/api/utils.py delete mode 100644 legacy/automl/__init__.py delete mode 100644 legacy/automl/enas_controller.py delete mode 100644 legacy/automl/enas_model.py delete mode 100644 legacy/automl/enas_trainer.py delete mode 100644 legacy/automl/enas_utils.py delete mode 100644 legacy/component/__init__.py delete mode 100644 legacy/component/bert_tokenizer.py delete mode 100644 test/io/test_data_loader.py delete mode 100644 test/io/test_dataset_loader.py diff --git a/fastNLP/io/__init__.py b/fastNLP/io/__init__.py index 8ed1956a..251b7292 100644 --- a/fastNLP/io/__init__.py +++ b/fastNLP/io/__init__.py @@ -82,7 +82,6 @@ __all__ = [ from .embed_loader import EmbedLoader from .data_bundle import DataBundle -from .dataset_loader import CSVLoader, JsonLoader from .model_io import ModelLoader, ModelSaver from .loader import * diff --git a/fastNLP/io/data_bundle.py b/fastNLP/io/data_bundle.py index 10f924f0..969730a3 100644 --- a/fastNLP/io/data_bundle.py +++ b/fastNLP/io/data_bundle.py @@ -6,112 +6,10 @@ __all__ = [ 'DataBundle', ] -import _pickle as pickle -import os -from typing import Union, Dict - from ..core.dataset import DataSet from ..core.vocabulary import Vocabulary -class BaseLoader(object): - """ - 各个 Loader 的基类，提供了 API 的参考。 - - """ - - def __init__(self): - super(BaseLoader, self).__init__() - - @staticmethod - def load_lines(data_path): - """ - 按行读取，舍弃每行两侧空白字符，返回list of str - - :param data_path: 读取数据的路径 - """ - with open(data_path, "r", encoding="utf=8") as f: - text = f.readlines() - return [line.strip() for line in text] - - @classmethod - def load(cls, data_path): - """ - 先按行读取，去除一行两侧空白，再提取每行的字符。返回list of list of str - - :param data_path: - """ - with open(data_path, "r", encoding="utf-8") as f: - text = f.readlines() - return [[word for word in sent.strip()] for sent in text] - - @classmethod - def load_with_cache(cls, data_path, cache_path): - """缓存版的load - """ - if os.path.isfile(cache_path) and os.path.getmtime(data_path) < os.path.getmtime(cache_path): - with open(cache_path, 'rb') as f: - return pickle.load(f) - else: - obj = cls.load(data_path) - with open(cache_path, 'wb') as f: - pickle.dump(obj, f) - return obj - - -def _download_from_url(url, path): - try: - from tqdm.auto import tqdm - except: - from ..core.utils import _pseudo_tqdm as tqdm - import requests - - """Download file""" - r = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, stream=True) - chunk_size = 16 * 1024 - total_size = int(r.headers.get('Content-length', 0)) - with open(path, "wb") as file, \ - tqdm(total=total_size, unit='B', unit_scale=1, desc=path.split('/')[-1]) as t: - for chunk in r.iter_content(chunk_size): - if chunk: - file.write(chunk) - t.update(len(chunk)) - - -def _uncompress(src, dst): - import zipfile - import gzip - import tarfile - import os - - def unzip(src, dst): - with zipfile.ZipFile(src, 'r') as f: - f.extractall(dst) - - def ungz(src, dst): - with gzip.open(src, 'rb') as f, open(dst, 'wb') as uf: - length = 16 * 1024 # 16KB - buf = f.read(length) - while buf: - uf.write(buf) - buf = f.read(length) - - def untar(src, dst): - with tarfile.open(src, 'r:gz') as f: - f.extractall(dst) - - fn, ext = os.path.splitext(src) - _, ext_2 = os.path.splitext(fn) - if ext == '.zip': - unzip(src, dst) - elif ext == '.gz' and ext_2 != '.tar': - ungz(src, dst) - elif (ext == '.gz' and ext_2 == '.tar') or ext_2 == '.tgz': - untar(src, dst) - else: - raise ValueError('unsupported file {}'.format(src)) - - class DataBundle: """ 经过处理的数据信息，包括一系列数据集（比如：分开的训练集、验证集和测试集）以及各个field对应的vocabulary。该对象一般由fastNLP中各种 @@ -154,7 +52,7 @@ class DataBundle: self.datasets[name] = dataset return self - def get_dataset(self, name:str)->DataSet: + def get_dataset(self, name: str) -> DataSet: """ 获取名为name的dataset @@ -163,7 +61,7 @@ class DataBundle: """ return self.datasets[name] - def delete_dataset(self, name:str): + def delete_dataset(self, name: str): """ 删除名为name的DataSet @@ -173,7 +71,7 @@ class DataBundle: self.datasets.pop(name, None) return self - def get_vocab(self, field_name:str)->Vocabulary: + def get_vocab(self, field_name: str) -> Vocabulary: """ 获取field名为field_name对应的vocab @@ -182,7 +80,7 @@ class DataBundle: """ return self.vocabs[field_name] - def delete_vocab(self, field_name:str): + def delete_vocab(self, field_name: str): """ 删除vocab :param str field_name: @@ -312,90 +210,3 @@ class DataBundle: return _str -class DataSetLoader: - """ - 别名：:class:`fastNLP.io.DataSetLoader` :class:`fastNLP.io.dataset_loader.DataSetLoader` - - 定义了各种 DataSetLoader 所需的API 接口，开发者应该继承它实现各种的 DataSetLoader。 - - 开发者至少应该编写如下内容: - - - _load 函数：从一个数据文件中读取数据到一个 :class:`~fastNLP.DataSet` - - load 函数（可以使用基类的方法）：从一个或多个数据文件中读取数据到一个或多个 :class:`~fastNLP.DataSet` - - process 函数：一个或多个从数据文件中读取数据，并处理成可以训练的一个或多个 :class:`~fastNLP.DataSet` - - **process 函数中可以 调用load 函数或 _load 函数** - - """ - URL = '' - DATA_DIR = '' - - ROOT_DIR = '.fastnlp/datasets/' - UNCOMPRESS = True - - def _download(self, url: str, pdir: str, uncompress=True) -> str: - """ - - 从 ``url`` 下载数据到 ``path``， 如果 ``uncompress`` 为 ``True`` ，自动解压。 - - :param url: 下载的网站 - :param pdir: 下载到的目录 - :param uncompress: 是否自动解压缩 - :return: 数据的存放路径 - """ - fn = os.path.basename(url) - path = os.path.join(pdir, fn) - """check data exists""" - if not os.path.exists(path): - os.makedirs(pdir, exist_ok=True) - _download_from_url(url, path) - if uncompress: - dst = os.path.join(pdir, 'data') - if not os.path.exists(dst): - _uncompress(path, dst) - return dst - return path - - def download(self): - return self._download( - self.URL, - os.path.join(self.ROOT_DIR, self.DATA_DIR), - uncompress=self.UNCOMPRESS) - - def load(self, paths: Union[str, Dict[str, str]]) -> Union[DataSet, Dict[str, DataSet]]: - """ - 从指定一个或多个路径中的文件中读取数据，返回一个或多个数据集 :class:`~fastNLP.DataSet` 。 - 如果处理多个路径，传入的 dict 中的 key 与返回的 dict 中的 key 保存一致。 - - :param Union[str, Dict[str, str]] paths: 文件路径 - :return: :class:`~fastNLP.DataSet` 类的对象或存储多个 :class:`~fastNLP.DataSet` 的字典 - """ - if isinstance(paths, str): - return self._load(paths) - return {name: self._load(path) for name, path in paths.items()} - - def _load(self, path: str) -> DataSet: - """从指定路径的文件中读取数据,返回 :class:`~fastNLP.DataSet` 类型的对象 - - :param str path: 文件路径 - :return: 一个 :class:`~fastNLP.DataSet` 类型的对象 - """ - raise NotImplementedError - - def process(self, paths: Union[str, Dict[str, str]], **options) -> DataBundle: - """ - 对于特定的任务和数据集，读取并处理数据，返回处理DataInfo类对象或字典。 - - 从指定一个或多个路径中的文件中读取数据，DataInfo对象中可以包含一个或多个数据集 。 - 如果处理多个路径，传入的 dict 的 key 与返回DataInfo中的 dict 中的 key 保存一致。 - - 返回的 :class:`DataBundle` 对象有如下属性： - - - vocabs: 由从数据集中获取的词表组成的字典，每个词表 - - datasets: 一个dict，包含一系列 :class:`~fastNLP.DataSet` 类型的对象。其中 field 的命名参考 :mod:`~fastNLP.core.const` - - :param paths: 原始数据读取的路径 - :param options: 根据不同的任务和数据集，设计自己的参数 - :return: 返回一个 DataBundle - """ - raise NotImplementedError diff --git a/fastNLP/io/data_loader/__init__.py b/fastNLP/io/data_loader/__init__.py deleted file mode 100644 index 8a9dd60b..00000000 --- a/fastNLP/io/data_loader/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -"""undocumented -.. warning:: - - 本模块在 `0.5.0版本` 中被废弃，由 :mod:`~fastNLP.io.loader` 和 :mod:`~fastNLP.io.pipe` 模块替代。 - -用于读数据集的模块, 可以读取文本分类、序列标注、Matching任务的数据集 - -这些模块的具体介绍如下，您可以通过阅读 :doc:`教程` 来进行了解。 -""" -__all__ = [ - 'ConllLoader', - 'Conll2003Loader', - 'IMDBLoader', - 'MatchingLoader', - 'SNLILoader', - 'MNLILoader', - 'MTL16Loader', - 'PeopleDailyCorpusLoader', - 'QNLILoader', - 'QuoraLoader', - 'RTELoader', - 'SSTLoader', - 'SST2Loader', - 'YelpLoader', -] - - -from .conll import ConllLoader, Conll2003Loader -from .imdb import IMDBLoader -from .matching import MatchingLoader -from .mnli import MNLILoader -from .mtl import MTL16Loader -from .people_daily import PeopleDailyCorpusLoader -from .qnli import QNLILoader -from .quora import QuoraLoader -from .rte import RTELoader -from .snli import SNLILoader -from .sst import SSTLoader, SST2Loader -from .yelp import YelpLoader diff --git a/fastNLP/io/data_loader/conll.py b/fastNLP/io/data_loader/conll.py deleted file mode 100644 index 31a90881..00000000 --- a/fastNLP/io/data_loader/conll.py +++ /dev/null @@ -1,109 +0,0 @@ - -from ...core.dataset import DataSet -from ...core.instance import Instance -from ..data_bundle import DataSetLoader -from ..file_reader import _read_conll -from typing import Union, Dict -from ..utils import check_loader_paths -from ..data_bundle import DataBundle - -class ConllLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.ConllLoader` :class:`fastNLP.io.data_loader.ConllLoader` - - 该ConllLoader支持读取的数据格式: 以空行隔开两个sample，除了分割行，每一行用空格或者制表符隔开不同的元素。如下例所示: - - Example:: - - # 文件中的内容 - Nadim NNP B-NP B-PER - Ladki NNP I-NP I-PER - - AL-AIN NNP B-NP B-LOC - United NNP B-NP B-LOC - Arab NNP I-NP I-LOC - Emirates NNPS I-NP I-LOC - 1996-12-06 CD I-NP O - ... - - # 如果用以下的参数读取，返回的DataSet将包含raw_words和pos两个field, 这两个field的值分别取自于第0列与第1列 - dataset = ConllLoader(headers=['raw_words', 'pos'], indexes=[0, 1])._load('/path/to/train.conll') - # 如果用以下的参数读取，返回的DataSet将包含raw_words和ner两个field, 这两个field的值分别取自于第0列与第2列 - dataset = ConllLoader(headers=['raw_words', 'ner'], indexes=[0, 3])._load('/path/to/train.conll') - # 如果用以下的参数读取，返回的DataSet将包含raw_words, pos和ner三个field - dataset = ConllLoader(headers=['raw_words', 'pos', 'ner'], indexes=[0, 1, 3])._load('/path/to/train.conll') - - dataset = ConllLoader(headers=['raw_words', 'pos'], indexes=[0, 1])._load('/path/to/train.conll')中DataSet的raw_words - 列与pos列的内容都是List[str] - - 数据中以"-DOCSTART-"开头的行将被忽略，因为该符号在conll 2003中被用为文档分割符。 - - :param list headers: 每一列数据的名称，需为List or Tuple of str。``header`` 与 ``indexes`` 一一对应 - :param list indexes: 需要保留的数据列下标，从0开始。若为 ``None`` ，则所有列都保留。Default: ``None`` - :param bool dropna: 是否忽略非法数据，若 ``False`` ，遇到非法数据时抛出 ``ValueError`` 。Default: ``True`` - """ - - def __init__(self, headers, indexes=None, dropna=True): - super(ConllLoader, self).__init__() - if not isinstance(headers, (list, tuple)): - raise TypeError( - 'invalid headers: {}, should be list of strings'.format(headers)) - self.headers = headers - self.dropna = dropna - if indexes is None: - self.indexes = list(range(len(self.headers))) - else: - if len(indexes) != len(headers): - raise ValueError - self.indexes = indexes - - def _load(self, path): - """ - 传入的一个文件路径，将该文件读入DataSet中，field由Loader初始化时指定的headers决定。 - - :param str path: 文件的路径 - :return: DataSet - """ - ds = DataSet() - for idx, data in _read_conll(path, indexes=self.indexes, dropna=self.dropna): - ins = {h: data[i] for i, h in enumerate(self.headers)} - ds.append(Instance(**ins)) - return ds - - def load(self, paths: Union[str, Dict[str, str]]) -> DataBundle: - """ - 从指定一个或多个路径中的文件中读取数据，返回:class:`~fastNLP.io.DataBundle` 。 - - 读取的field根据ConllLoader初始化时传入的headers决定。 - - :param Union[str, Dict[str, str]] paths: - :return: :class:`~fastNLP.DataSet` 类的对象或 :class:`~fastNLP.io.DataBundle` 的字典 - """ - paths = check_loader_paths(paths) - datasets = {name: self._load(path) for name, path in paths.items()} - data_bundle = DataBundle(datasets=datasets) - return data_bundle - - -class Conll2003Loader(ConllLoader): - """ - 别名：:class:`fastNLP.io.Conll2003Loader` :class:`fastNLP.io.data_loader.Conll2003Loader` - - 该Loader用以读取Conll2003数据，conll2003的数据可以在https://github.com/davidsbatista/NER-datasets/tree/master/CONLL2003 - 找到。数据中以"-DOCSTART-"开头的行将被忽略，因为该符号在conll 2003中被用为文档分割符。 - - 返回的DataSet将具有以下['raw_words', 'pos', 'chunks', 'ner']四个field, 每个field中的内容都是List[str]。 - - .. csv-table:: Conll2003Loader处理之 :header: "raw_words", "words", "target", "seq_len" - - "[Nadim, Ladki]", "[1, 2]", "[1, 2]", 2 - "[AL-AIN, United, Arab, ...]", "[3, 4, 5,...]", "[3, 4]", 5 - "[...]", "[...]", "[...]", . - - """ - - def __init__(self): - headers = [ - 'raw_words', 'pos', 'chunks', 'ner', - ] - super(Conll2003Loader, self).__init__(headers=headers) diff --git a/fastNLP/io/data_loader/imdb.py b/fastNLP/io/data_loader/imdb.py deleted file mode 100644 index c9dda76e..00000000 --- a/fastNLP/io/data_loader/imdb.py +++ /dev/null @@ -1,99 +0,0 @@ - -from typing import Union, Dict - -from ..embed_loader import EmbeddingOption, EmbedLoader -from ..data_bundle import DataSetLoader, DataBundle -from ...core.vocabulary import VocabularyOption, Vocabulary -from ...core.dataset import DataSet -from ...core.instance import Instance -from ...core.const import Const - -from ..utils import get_tokenizer - - -class IMDBLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.IMDBLoader` :class:`fastNLP.io.data_loader.IMDBLoader` - - 读取IMDB数据集，DataSet包含以下fields: - - words: list(str), 需要分类的文本 - - target: str, 文本的标签 - - """ - - def __init__(self): - super(IMDBLoader, self).__init__() - self.tokenizer = get_tokenizer() - - def _load(self, path): - dataset = DataSet() - with open(path, 'r', encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - parts = line.split('\t') - target = parts[0] - words = self.tokenizer(parts[1].lower()) - dataset.append(Instance(words=words, target=target)) - - if len(dataset) == 0: - raise RuntimeError(f"{path} has no valid data.") - - return dataset - - def process(self, - paths: Union[str, Dict[str, str]], - src_vocab_opt: VocabularyOption = None, - tgt_vocab_opt: VocabularyOption = None, - char_level_op=False): - - datasets = {} - info = DataBundle() - for name, path in paths.items(): - dataset = self.load(path) - datasets[name] = dataset - - def wordtochar(words): - chars = [] - for word in words: - word = word.lower() - for char in word: - chars.append(char) - chars.append('') - chars.pop() - return chars - - if char_level_op: - for dataset in datasets.values(): - dataset.apply_field(wordtochar, field_name="words", new_field_name='chars') - - datasets["train"], datasets["dev"] = datasets["train"].split(0.1, shuffle=False) - - src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) - src_vocab.from_dataset(datasets['train'], field_name='words') - - src_vocab.index_dataset(*datasets.values(), field_name='words') - - tgt_vocab = Vocabulary(unknown=None, padding=None) \ - if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) - tgt_vocab.from_dataset(datasets['train'], field_name='target') - tgt_vocab.index_dataset(*datasets.values(), field_name='target') - - info.vocabs = { - Const.INPUT: src_vocab, - Const.TARGET: tgt_vocab - } - - info.datasets = datasets - - for name, dataset in info.datasets.items(): - dataset.set_input(Const.INPUT) - dataset.set_target(Const.TARGET) - - return info - - - diff --git a/fastNLP/io/data_loader/matching.py b/fastNLP/io/data_loader/matching.py deleted file mode 100644 index 41c9a98d..00000000 --- a/fastNLP/io/data_loader/matching.py +++ /dev/null @@ -1,248 +0,0 @@ -import os - -from typing import Union, Dict, List - -from ...core.const import Const -from ...core.vocabulary import Vocabulary -from ..data_bundle import DataBundle, DataSetLoader -from ..file_utils import _get_base_url, cached_path, PRETRAINED_BERT_MODEL_DIR -from ...modules.encoder.bert import BertTokenizer - - -class MatchingLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.MatchingLoader` :class:`fastNLP.io.data_loader.MatchingLoader` - - 读取Matching任务的数据集 - - :param dict paths: key是数据集名称（如train、dev、test），value是对应的文件名 - """ - - def __init__(self, paths: dict=None): - self.paths = paths - - def _load(self, path): - """ - :param str path: 待读取数据集的路径名 - :return: fastNLP.DataSet ds: 返回一个DataSet对象，里面必须包含3个field：其中两个分别为两个句子 - 的原始字符串文本，第三个为标签 - """ - raise NotImplementedError - - def process(self, paths: Union[str, Dict[str, str]], dataset_name: str=None, - to_lower=False, seq_len_type: str=None, bert_tokenizer: str=None, - cut_text: int = None, get_index=True, auto_pad_length: int=None, - auto_pad_token: str='', set_input: Union[list, str, bool]=True, - set_target: Union[list, str, bool]=True, concat: Union[str, list, bool]=None, - extra_split: List[str]=None, ) -> DataBundle: - """ - :param paths: str或者Dict[str, str]。如果是str，则为数据集所在的文件夹或者是全路径文件名：如果是文件夹， - 则会从self.paths里面找对应的数据集名称与文件名。如果是Dict，则为数据集名称（如train、dev、test）和 - 对应的全路径文件名。 - :param str dataset_name: 如果在paths里传入的是一个数据集的全路径文件名，那么可以用dataset_name来定义 - 这个数据集的名字，如果不定义则默认为train。 - :param bool to_lower: 是否将文本自动转为小写。默认值为False。 - :param str seq_len_type: 提供的seq_len类型，支持 ``seq_len`` ：提供一个数字作为句子长度； ``mask`` : - 提供一个0/1的mask矩阵作为句子长度； ``bert`` ：提供segment_type_id（第一个句子为0，第二个句子为1）和 - attention mask矩阵（0/1的mask矩阵）。默认值为None，即不提供seq_len - :param str bert_tokenizer: bert tokenizer所使用的词表所在的文件夹路径 - :param int cut_text: 将长于cut_text的内容截掉。默认为None，即不截。 - :param bool get_index: 是否需要根据词表将文本转为index - :param int auto_pad_length: 是否需要将文本自动pad到一定长度（超过这个长度的文本将会被截掉），默认为不会自动pad - :param str auto_pad_token: 自动pad的内容 - :param set_input: 如果为True，则会自动将相关的field（名字里含有Const.INPUT的）设置为input，如果为False - 则不会将任何field设置为input。如果传入str或者List[str]，则会根据传入的内容将相对应的field设置为input， - 于此同时其他field不会被设置为input。默认值为True。 - :param set_target: set_target将控制哪些field可以被设置为target，用法与set_input一致。默认值为True。 - :param concat: 是否需要将两个句子拼接起来。如果为False则不会拼接。如果为True则会在两个句子之间插入一个。 - 如果传入一个长度为4的list，则分别表示插在第一句开始前、第一句结束后、第二句开始前、第二句结束后的标识符。如果 - 传入字符串 ``bert`` ，则会采用bert的拼接方式，等价于['[CLS]', '[SEP]', '', '[SEP]']. - :param extra_split: 额外的分隔符，即除了空格之外的用于分词的字符。 - :return: - """ - if isinstance(set_input, str): - set_input = [set_input] - if isinstance(set_target, str): - set_target = [set_target] - if isinstance(set_input, bool): - auto_set_input = set_input - else: - auto_set_input = False - if isinstance(set_target, bool): - auto_set_target = set_target - else: - auto_set_target = False - if isinstance(paths, str): - if os.path.isdir(paths): - path = {n: os.path.join(paths, self.paths[n]) for n in self.paths.keys()} - else: - path = {dataset_name if dataset_name is not None else 'train': paths} - else: - path = paths - - data_info = DataBundle() - for data_name in path.keys(): - data_info.datasets[data_name] = self._load(path[data_name]) - - for data_name, data_set in data_info.datasets.items(): - if auto_set_input: - data_set.set_input(Const.INPUTS(0), Const.INPUTS(1)) - if auto_set_target: - if Const.TARGET in data_set.get_field_names(): - data_set.set_target(Const.TARGET) - - if extra_split is not None: - for data_name, data_set in data_info.datasets.items(): - data_set.apply(lambda x: ' '.join(x[Const.INPUTS(0)]), new_field_name=Const.INPUTS(0)) - data_set.apply(lambda x: ' '.join(x[Const.INPUTS(1)]), new_field_name=Const.INPUTS(1)) - - for s in extra_split: - data_set.apply(lambda x: x[Const.INPUTS(0)].replace(s, ' ' + s + ' '), - new_field_name=Const.INPUTS(0)) - data_set.apply(lambda x: x[Const.INPUTS(0)].replace(s, ' ' + s + ' '), - new_field_name=Const.INPUTS(0)) - - _filt = lambda x: x - data_set.apply(lambda x: list(filter(_filt, x[Const.INPUTS(0)].split(' '))), - new_field_name=Const.INPUTS(0), is_input=auto_set_input) - data_set.apply(lambda x: list(filter(_filt, x[Const.INPUTS(1)].split(' '))), - new_field_name=Const.INPUTS(1), is_input=auto_set_input) - _filt = None - - if to_lower: - for data_name, data_set in data_info.datasets.items(): - data_set.apply(lambda x: [w.lower() for w in x[Const.INPUTS(0)]], new_field_name=Const.INPUTS(0), - is_input=auto_set_input) - data_set.apply(lambda x: [w.lower() for w in x[Const.INPUTS(1)]], new_field_name=Const.INPUTS(1), - is_input=auto_set_input) - - if bert_tokenizer is not None: - if bert_tokenizer.lower() in PRETRAINED_BERT_MODEL_DIR: - PRETRAIN_URL = _get_base_url('bert') - model_name = PRETRAINED_BERT_MODEL_DIR[bert_tokenizer] - model_url = PRETRAIN_URL + model_name - model_dir = cached_path(model_url, name='embedding') - # 检查是否存在 - elif os.path.isdir(bert_tokenizer): - model_dir = bert_tokenizer - else: - raise ValueError(f"Cannot recognize BERT tokenizer from {bert_tokenizer}.") - - words_vocab = Vocabulary(padding='[PAD]', unknown='[UNK]') - with open(os.path.join(model_dir, 'vocab.txt'), 'r') as f: - lines = f.readlines() - lines = [line.strip() for line in lines] - words_vocab.add_word_lst(lines) - words_vocab.build_vocab() - - tokenizer = BertTokenizer.from_pretrained(model_dir) - - for data_name, data_set in data_info.datasets.items(): - for fields in data_set.get_field_names(): - if Const.INPUT in fields: - data_set.apply(lambda x: tokenizer.tokenize(' '.join(x[fields])), new_field_name=fields, - is_input=auto_set_input) - - if isinstance(concat, bool): - concat = 'default' if concat else None - if concat is not None: - if isinstance(concat, str): - CONCAT_MAP = {'bert': ['[CLS]', '[SEP]', '', '[SEP]'], - 'default': ['', '', '', '']} - if concat.lower() in CONCAT_MAP: - concat = CONCAT_MAP[concat] - else: - concat = 4 * [concat] - assert len(concat) == 4, \ - f'Please choose a list with 4 symbols which at the beginning of first sentence ' \ - f'the end of first sentence, the begin of second sentence, and the end of second' \ - f'sentence. Your input is {concat}' - - for data_name, data_set in data_info.datasets.items(): - data_set.apply(lambda x: [concat[0]] + x[Const.INPUTS(0)] + [concat[1]] + [concat[2]] + - x[Const.INPUTS(1)] + [concat[3]], new_field_name=Const.INPUT) - data_set.apply(lambda x: [w for w in x[Const.INPUT] if len(w) > 0], new_field_name=Const.INPUT, - is_input=auto_set_input) - - if seq_len_type is not None: - if seq_len_type == 'seq_len': # - for data_name, data_set in data_info.datasets.items(): - for fields in data_set.get_field_names(): - if Const.INPUT in fields: - data_set.apply(lambda x: len(x[fields]), - new_field_name=fields.replace(Const.INPUT, Const.INPUT_LEN), - is_input=auto_set_input) - elif seq_len_type == 'mask': - for data_name, data_set in data_info.datasets.items(): - for fields in data_set.get_field_names(): - if Const.INPUT in fields: - data_set.apply(lambda x: [1] * len(x[fields]), - new_field_name=fields.replace(Const.INPUT, Const.INPUT_LEN), - is_input=auto_set_input) - elif seq_len_type == 'bert': - for data_name, data_set in data_info.datasets.items(): - if Const.INPUT not in data_set.get_field_names(): - raise KeyError(f'Field ``{Const.INPUT}`` not in {data_name} data set: ' - f'got {data_set.get_field_names()}') - data_set.apply(lambda x: [0] * (len(x[Const.INPUTS(0)]) + 2) + [1] * (len(x[Const.INPUTS(1)]) + 1), - new_field_name=Const.INPUT_LENS(0), is_input=auto_set_input) - data_set.apply(lambda x: [1] * len(x[Const.INPUT_LENS(0)]), - new_field_name=Const.INPUT_LENS(1), is_input=auto_set_input) - - if auto_pad_length is not None: - cut_text = min(auto_pad_length, cut_text if cut_text is not None else auto_pad_length) - - if cut_text is not None: - for data_name, data_set in data_info.datasets.items(): - for fields in data_set.get_field_names(): - if (Const.INPUT in fields) or ((Const.INPUT_LEN in fields) and (seq_len_type != 'seq_len')): - data_set.apply(lambda x: x[fields][: cut_text], new_field_name=fields, - is_input=auto_set_input) - - data_set_list = [d for n, d in data_info.datasets.items()] - assert len(data_set_list) > 0, f'There are NO data sets in data info!' - - if bert_tokenizer is None: - words_vocab = Vocabulary(padding=auto_pad_token) - words_vocab = words_vocab.from_dataset(*[d for n, d in data_info.datasets.items() if 'train' in n], - field_name=[n for n in data_set_list[0].get_field_names() - if (Const.INPUT in n)], - no_create_entry_dataset=[d for n, d in data_info.datasets.items() - if 'train' not in n]) - target_vocab = Vocabulary(padding=None, unknown=None) - target_vocab = target_vocab.from_dataset(*[d for n, d in data_info.datasets.items() if 'train' in n], - field_name=Const.TARGET) - data_info.vocabs = {Const.INPUT: words_vocab, Const.TARGET: target_vocab} - - if get_index: - for data_name, data_set in data_info.datasets.items(): - for fields in data_set.get_field_names(): - if Const.INPUT in fields: - data_set.apply(lambda x: [words_vocab.to_index(w) for w in x[fields]], new_field_name=fields, - is_input=auto_set_input) - - if Const.TARGET in data_set.get_field_names(): - data_set.apply(lambda x: target_vocab.to_index(x[Const.TARGET]), new_field_name=Const.TARGET, - is_input=auto_set_input, is_target=auto_set_target) - - if auto_pad_length is not None: - if seq_len_type == 'seq_len': - raise RuntimeError(f'the sequence will be padded with the length {auto_pad_length}, ' - f'so the seq_len_type cannot be `{seq_len_type}`!') - for data_name, data_set in data_info.datasets.items(): - for fields in data_set.get_field_names(): - if Const.INPUT in fields: - data_set.apply(lambda x: x[fields] + [words_vocab.to_index(words_vocab.padding)] * - (auto_pad_length - len(x[fields])), new_field_name=fields, - is_input=auto_set_input) - elif (Const.INPUT_LEN in fields) and (seq_len_type != 'seq_len'): - data_set.apply(lambda x: x[fields] + [0] * (auto_pad_length - len(x[fields])), - new_field_name=fields, is_input=auto_set_input) - - for data_name, data_set in data_info.datasets.items(): - if isinstance(set_input, list): - data_set.set_input(*[inputs for inputs in set_input if inputs in data_set.get_field_names()]) - if isinstance(set_target, list): - data_set.set_target(*[target for target in set_target if target in data_set.get_field_names()]) - - return data_info diff --git a/fastNLP/io/data_loader/mnli.py b/fastNLP/io/data_loader/mnli.py deleted file mode 100644 index 65863f3d..00000000 --- a/fastNLP/io/data_loader/mnli.py +++ /dev/null @@ -1,62 +0,0 @@ - -from ...core.const import Const - -from .matching import MatchingLoader -from ..dataset_loader import CSVLoader - - -class MNLILoader(MatchingLoader, CSVLoader): - """ - 别名：:class:`fastNLP.io.MNLILoader` :class:`fastNLP.io.data_loader.MNLILoader` - - 读取MNLI数据集，读取的DataSet包含fields:: - - words1: list(str)，第一句文本, premise - - words2: list(str), 第二句文本, hypothesis - - target: str, 真实标签 - - 数据来源: - """ - - def __init__(self, paths: dict=None): - paths = paths if paths is not None else { - 'train': 'train.tsv', - 'dev_matched': 'dev_matched.tsv', - 'dev_mismatched': 'dev_mismatched.tsv', - 'test_matched': 'test_matched.tsv', - 'test_mismatched': 'test_mismatched.tsv', - # 'test_0.9_matched': 'multinli_0.9_test_matched_unlabeled.txt', - # 'test_0.9_mismatched': 'multinli_0.9_test_mismatched_unlabeled.txt', - - # test_0.9_mathed与mismatched是MNLI0.9版本的（数据来源：kaggle） - } - MatchingLoader.__init__(self, paths=paths) - CSVLoader.__init__(self, sep='\t') - self.fields = { - 'sentence1_binary_parse': Const.INPUTS(0), - 'sentence2_binary_parse': Const.INPUTS(1), - 'gold_label': Const.TARGET, - } - - def _load(self, path): - ds = CSVLoader._load(self, path) - - for k, v in self.fields.items(): - if k in ds.get_field_names(): - ds.rename_field(k, v) - - if Const.TARGET in ds.get_field_names(): - if ds[0][Const.TARGET] == 'hidden': - ds.delete_field(Const.TARGET) - - parentheses_table = str.maketrans({'(': None, ')': None}) - - ds.apply(lambda ins: ins[Const.INPUTS(0)].translate(parentheses_table).strip().split(), - new_field_name=Const.INPUTS(0)) - ds.apply(lambda ins: ins[Const.INPUTS(1)].translate(parentheses_table).strip().split(), - new_field_name=Const.INPUTS(1)) - if Const.TARGET in ds.get_field_names(): - ds.drop(lambda x: x[Const.TARGET] == '-') - return ds diff --git a/fastNLP/io/data_loader/mtl.py b/fastNLP/io/data_loader/mtl.py deleted file mode 100644 index 923aadfb..00000000 --- a/fastNLP/io/data_loader/mtl.py +++ /dev/null @@ -1,68 +0,0 @@ - -from typing import Union, Dict - -from ..data_bundle import DataBundle -from ..dataset_loader import CSVLoader -from ...core.vocabulary import Vocabulary, VocabularyOption -from ...core.const import Const -from ..utils import check_loader_paths - - -class MTL16Loader(CSVLoader): - """ - 别名：:class:`fastNLP.io.MTL16Loader` :class:`fastNLP.io.data_loader.MTL16Loader` - - 读取MTL16数据集，DataSet包含以下fields: - - words: list(str), 需要分类的文本 - - target: str, 文本的标签 - - 数据来源：https://pan.baidu.com/s/1c2L6vdA - - """ - - def __init__(self): - super(MTL16Loader, self).__init__(headers=(Const.TARGET, Const.INPUT), sep='\t') - - def _load(self, path): - dataset = super(MTL16Loader, self)._load(path) - dataset.apply(lambda x: x[Const.INPUT].lower().split(), new_field_name=Const.INPUT) - if len(dataset) == 0: - raise RuntimeError(f"{path} has no valid data.") - - return dataset - - def process(self, - paths: Union[str, Dict[str, str]], - src_vocab_opt: VocabularyOption = None, - tgt_vocab_opt: VocabularyOption = None,): - - paths = check_loader_paths(paths) - datasets = {} - info = DataBundle() - for name, path in paths.items(): - dataset = self.load(path) - datasets[name] = dataset - - src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) - src_vocab.from_dataset(datasets['train'], field_name=Const.INPUT) - src_vocab.index_dataset(*datasets.values(), field_name=Const.INPUT) - - tgt_vocab = Vocabulary(unknown=None, padding=None) \ - if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) - tgt_vocab.from_dataset(datasets['train'], field_name=Const.TARGET) - tgt_vocab.index_dataset(*datasets.values(), field_name=Const.TARGET) - - info.vocabs = { - Const.INPUT: src_vocab, - Const.TARGET: tgt_vocab - } - - info.datasets = datasets - - for name, dataset in info.datasets.items(): - dataset.set_input(Const.INPUT) - dataset.set_target(Const.TARGET) - - return info diff --git a/fastNLP/io/data_loader/people_daily.py b/fastNLP/io/data_loader/people_daily.py deleted file mode 100644 index afd66744..00000000 --- a/fastNLP/io/data_loader/people_daily.py +++ /dev/null @@ -1,85 +0,0 @@ - -from ..data_bundle import DataSetLoader -from ...core.dataset import DataSet -from ...core.instance import Instance -from ...core.const import Const - - -class PeopleDailyCorpusLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.PeopleDailyCorpusLoader` :class:`fastNLP.io.data_loader.PeopleDailyCorpusLoader` - - 读取人民日报数据集 - """ - - def __init__(self, pos=True, ner=True): - super(PeopleDailyCorpusLoader, self).__init__() - self.pos = pos - self.ner = ner - - def _load(self, data_path): - with open(data_path, "r", encoding="utf-8") as f: - sents = f.readlines() - examples = [] - for sent in sents: - if len(sent) <= 2: - continue - inside_ne = False - sent_pos_tag = [] - sent_words = [] - sent_ner = [] - words = sent.strip().split()[1:] - for word in words: - if "[" in word and "]" in word: - ner_tag = "U" - print(word) - elif "[" in word: - inside_ne = True - ner_tag = "B" - word = word[1:] - elif "]" in word: - ner_tag = "L" - word = word[:word.index("]")] - if inside_ne is True: - inside_ne = False - else: - raise RuntimeError("only ] appears!") - else: - if inside_ne is True: - ner_tag = "I" - else: - ner_tag = "O" - tmp = word.split("/") - token, pos = tmp[0], tmp[1] - sent_ner.append(ner_tag) - sent_pos_tag.append(pos) - sent_words.append(token) - example = [sent_words] - if self.pos is True: - example.append(sent_pos_tag) - if self.ner is True: - example.append(sent_ner) - examples.append(example) - return self.convert(examples) - - def convert(self, data): - """ - - :param data: python 内置对象 - :return: 一个 :class:`~fastNLP.DataSet` 类型的对象 - """ - data_set = DataSet() - for item in data: - sent_words = item[0] - if self.pos is True and self.ner is True: - instance = Instance( - words=sent_words, pos_tags=item[1], ner=item[2]) - elif self.pos is True: - instance = Instance(words=sent_words, pos_tags=item[1]) - elif self.ner is True: - instance = Instance(words=sent_words, ner=item[1]) - else: - instance = Instance(words=sent_words) - data_set.append(instance) - data_set.apply(lambda ins: len(ins[Const.INPUT]), new_field_name=Const.INPUT_LEN) - return data_set diff --git a/fastNLP/io/data_loader/qnli.py b/fastNLP/io/data_loader/qnli.py deleted file mode 100644 index 84b0f3d6..00000000 --- a/fastNLP/io/data_loader/qnli.py +++ /dev/null @@ -1,47 +0,0 @@ - -from ...core.const import Const - -from .matching import MatchingLoader -from ..dataset_loader import CSVLoader - - -class QNLILoader(MatchingLoader, CSVLoader): - """ - 别名：:class:`fastNLP.io.QNLILoader` :class:`fastNLP.io.data_loader.QNLILoader` - - 读取QNLI数据集，读取的DataSet包含fields:: - - words1: list(str)，第一句文本, premise - - words2: list(str), 第二句文本, hypothesis - - target: str, 真实标签 - - 数据来源: - """ - - def __init__(self, paths: dict=None): - paths = paths if paths is not None else { - 'train': 'train.tsv', - 'dev': 'dev.tsv', - 'test': 'test.tsv' # test set has not label - } - MatchingLoader.__init__(self, paths=paths) - self.fields = { - 'question': Const.INPUTS(0), - 'sentence': Const.INPUTS(1), - 'label': Const.TARGET, - } - CSVLoader.__init__(self, sep='\t') - - def _load(self, path): - ds = CSVLoader._load(self, path) - - for k, v in self.fields.items(): - if k in ds.get_field_names(): - ds.rename_field(k, v) - for fields in ds.get_all_fields(): - if Const.INPUT in fields: - ds.apply(lambda x: x[fields].strip().split(), new_field_name=fields) - - return ds diff --git a/fastNLP/io/data_loader/quora.py b/fastNLP/io/data_loader/quora.py deleted file mode 100644 index d0ee41ec..00000000 --- a/fastNLP/io/data_loader/quora.py +++ /dev/null @@ -1,34 +0,0 @@ - -from ...core.const import Const - -from .matching import MatchingLoader -from ..dataset_loader import CSVLoader - - -class QuoraLoader(MatchingLoader, CSVLoader): - """ - 别名：:class:`fastNLP.io.QuoraLoader` :class:`fastNLP.io.data_loader.QuoraLoader` - - 读取MNLI数据集，读取的DataSet包含fields:: - - words1: list(str)，第一句文本, premise - - words2: list(str), 第二句文本, hypothesis - - target: str, 真实标签 - - 数据来源: - """ - - def __init__(self, paths: dict=None): - paths = paths if paths is not None else { - 'train': 'train.tsv', - 'dev': 'dev.tsv', - 'test': 'test.tsv', - } - MatchingLoader.__init__(self, paths=paths) - CSVLoader.__init__(self, sep='\t', headers=(Const.TARGET, Const.INPUTS(0), Const.INPUTS(1), 'pairID')) - - def _load(self, path): - ds = CSVLoader._load(self, path) - return ds diff --git a/fastNLP/io/data_loader/rte.py b/fastNLP/io/data_loader/rte.py deleted file mode 100644 index f8c5e2fc..00000000 --- a/fastNLP/io/data_loader/rte.py +++ /dev/null @@ -1,47 +0,0 @@ - -from ...core.const import Const - -from .matching import MatchingLoader -from ..dataset_loader import CSVLoader - - -class RTELoader(MatchingLoader, CSVLoader): - """ - 别名：:class:`fastNLP.io.RTELoader` :class:`fastNLP.io.data_loader.RTELoader` - - 读取RTE数据集，读取的DataSet包含fields:: - - words1: list(str)，第一句文本, premise - - words2: list(str), 第二句文本, hypothesis - - target: str, 真实标签 - - 数据来源: - """ - - def __init__(self, paths: dict=None): - paths = paths if paths is not None else { - 'train': 'train.tsv', - 'dev': 'dev.tsv', - 'test': 'test.tsv' # test set has not label - } - MatchingLoader.__init__(self, paths=paths) - self.fields = { - 'sentence1': Const.INPUTS(0), - 'sentence2': Const.INPUTS(1), - 'label': Const.TARGET, - } - CSVLoader.__init__(self, sep='\t') - - def _load(self, path): - ds = CSVLoader._load(self, path) - - for k, v in self.fields.items(): - if k in ds.get_field_names(): - ds.rename_field(k, v) - for fields in ds.get_all_fields(): - if Const.INPUT in fields: - ds.apply(lambda x: x[fields].strip().split(), new_field_name=fields) - - return ds diff --git a/fastNLP/io/data_loader/snli.py b/fastNLP/io/data_loader/snli.py deleted file mode 100644 index 1db0ac5b..00000000 --- a/fastNLP/io/data_loader/snli.py +++ /dev/null @@ -1,46 +0,0 @@ - -from ...core.const import Const - -from .matching import MatchingLoader -from ..dataset_loader import JsonLoader - - -class SNLILoader(MatchingLoader, JsonLoader): - """ - 别名：:class:`fastNLP.io.SNLILoader` :class:`fastNLP.io.data_loader.SNLILoader` - - 读取SNLI数据集，读取的DataSet包含fields:: - - words1: list(str)，第一句文本, premise - - words2: list(str), 第二句文本, hypothesis - - target: str, 真实标签 - - 数据来源: https://nlp.stanford.edu/projects/snli/snli_1.0.zip - """ - - def __init__(self, paths: dict=None): - fields = { - 'sentence1_binary_parse': Const.INPUTS(0), - 'sentence2_binary_parse': Const.INPUTS(1), - 'gold_label': Const.TARGET, - } - paths = paths if paths is not None else { - 'train': 'snli_1.0_train.jsonl', - 'dev': 'snli_1.0_dev.jsonl', - 'test': 'snli_1.0_test.jsonl'} - MatchingLoader.__init__(self, paths=paths) - JsonLoader.__init__(self, fields=fields) - - def _load(self, path): - ds = JsonLoader._load(self, path) - - parentheses_table = str.maketrans({'(': None, ')': None}) - - ds.apply(lambda ins: ins[Const.INPUTS(0)].translate(parentheses_table).strip().split(), - new_field_name=Const.INPUTS(0)) - ds.apply(lambda ins: ins[Const.INPUTS(1)].translate(parentheses_table).strip().split(), - new_field_name=Const.INPUTS(1)) - ds.drop(lambda x: x[Const.TARGET] == '-') - return ds diff --git a/fastNLP/io/data_loader/sst.py b/fastNLP/io/data_loader/sst.py deleted file mode 100644 index 2034fc2b..00000000 --- a/fastNLP/io/data_loader/sst.py +++ /dev/null @@ -1,180 +0,0 @@ - -from typing import Union, Dict -from nltk import Tree - -from ..data_bundle import DataBundle, DataSetLoader -from ..dataset_loader import CSVLoader -from ...core.vocabulary import VocabularyOption, Vocabulary -from ...core.dataset import DataSet -from ...core.const import Const -from ...core.instance import Instance -from ..utils import check_loader_paths, get_tokenizer - - -class SSTLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.data_loader.SSTLoader` - - 读取SST数据集, DataSet包含fields:: - - words: list(str) 需要分类的文本 - target: str 文本的标签 - - 数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip - - :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False`` - :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` - """ - - URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' - DATA_DIR = 'sst/' - - def __init__(self, subtree=False, fine_grained=False): - self.subtree = subtree - - tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral', - '3': 'positive', '4': 'very positive'} - if not fine_grained: - tag_v['0'] = tag_v['1'] - tag_v['4'] = tag_v['3'] - self.tag_v = tag_v - self.tokenizer = get_tokenizer() - - def _load(self, path): - """ - - :param str path: 存储数据的路径 - :return: 一个 :class:`~fastNLP.DataSet` 类型的对象 - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - datas = [] - for l in f: - datas.extend([(s, self.tag_v[t]) - for s, t in self._get_one(l, self.subtree)]) - ds = DataSet() - for words, tag in datas: - ds.append(Instance(words=words, target=tag)) - return ds - - def _get_one(self, data, subtree): - tree = Tree.fromstring(data) - if subtree: - return [(self.tokenizer(' '.join(t.leaves())), t.label()) for t in tree.subtrees() ] - return [(self.tokenizer(' '.join(tree.leaves())), tree.label())] - - def process(self, - paths, train_subtree=True, - src_vocab_op: VocabularyOption = None, - tgt_vocab_op: VocabularyOption = None,): - paths = check_loader_paths(paths) - input_name, target_name = 'words', 'target' - src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) - tgt_vocab = Vocabulary(unknown=None, padding=None) \ - if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) - - info = DataBundle() - origin_subtree = self.subtree - self.subtree = train_subtree - info.datasets['train'] = self._load(paths['train']) - self.subtree = origin_subtree - for n, p in paths.items(): - if n != 'train': - info.datasets[n] = self._load(p) - - src_vocab.from_dataset( - info.datasets['train'], - field_name=input_name, - no_create_entry_dataset=[ds for n, ds in info.datasets.items() if n != 'train']) - tgt_vocab.from_dataset(info.datasets['train'], field_name=target_name) - - src_vocab.index_dataset( - *info.datasets.values(), - field_name=input_name, new_field_name=input_name) - tgt_vocab.index_dataset( - *info.datasets.values(), - field_name=target_name, new_field_name=target_name) - info.vocabs = { - input_name: src_vocab, - target_name: tgt_vocab - } - - return info - - -class SST2Loader(CSVLoader): - """ - 别名：:class:`fastNLP.io.SST2Loader` :class:`fastNLP.io.data_loader.SST2Loader` - - 数据来源 SST: https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8 - """ - - def __init__(self): - super(SST2Loader, self).__init__(sep='\t') - self.tokenizer = get_tokenizer() - self.field = {'sentence': Const.INPUT, 'label': Const.TARGET} - - def _load(self, path: str) -> DataSet: - ds = super(SST2Loader, self)._load(path) - for k, v in self.field.items(): - if k in ds.get_field_names(): - ds.rename_field(k, v) - ds.apply(lambda x: self.tokenizer(x[Const.INPUT]), new_field_name=Const.INPUT) - print("all count:", len(ds)) - return ds - - def process(self, - paths: Union[str, Dict[str, str]], - src_vocab_opt: VocabularyOption = None, - tgt_vocab_opt: VocabularyOption = None, - char_level_op=False): - - paths = check_loader_paths(paths) - datasets = {} - info = DataBundle() - for name, path in paths.items(): - dataset = self.load(path) - dataset.apply_field(lambda words:words.copy(), field_name='words', new_field_name='raw_words') - datasets[name] = dataset - - def wordtochar(words): - chars = [] - for word in words: - word = word.lower() - for char in word: - chars.append(char) - chars.append('') - chars.pop() - return chars - - input_name, target_name = Const.INPUT, Const.TARGET - info.vocabs={} - - # 就分隔为char形式 - if char_level_op: - for dataset in datasets.values(): - dataset.apply_field(wordtochar, field_name=Const.INPUT, new_field_name=Const.CHAR_INPUT) - src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt) - src_vocab.from_dataset(datasets['train'], field_name=Const.INPUT, no_create_entry_dataset=[ - dataset for name, dataset in datasets.items() if name!='train' - ]) - src_vocab.index_dataset(*datasets.values(), field_name=Const.INPUT) - - tgt_vocab = Vocabulary(unknown=None, padding=None) \ - if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt) - tgt_vocab.from_dataset(datasets['train'], field_name=Const.TARGET) - tgt_vocab.index_dataset(*datasets.values(), field_name=Const.TARGET) - - info.vocabs = { - Const.INPUT: src_vocab, - Const.TARGET: tgt_vocab - } - - info.datasets = datasets - - for name, dataset in info.datasets.items(): - dataset.set_input(Const.INPUT) - dataset.set_target(Const.TARGET) - - return info - diff --git a/fastNLP/io/data_loader/yelp.py b/fastNLP/io/data_loader/yelp.py deleted file mode 100644 index f2bc60c8..00000000 --- a/fastNLP/io/data_loader/yelp.py +++ /dev/null @@ -1,132 +0,0 @@ - -import csv -from typing import Iterable - -from ...core.const import Const -from ...core.dataset import DataSet -from ...core.instance import Instance -from ...core.vocabulary import VocabularyOption, Vocabulary -from ..data_bundle import DataBundle, DataSetLoader -from typing import Union, Dict -from ..utils import check_loader_paths, get_tokenizer - - -class YelpLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.YelpLoader` :class:`fastNLP.io.data_loader.YelpLoader` - 读取Yelp_full/Yelp_polarity数据集, DataSet包含fields: - - words: list(str), 需要分类的文本 - - target: str, 文本的标签 - - chars:list(str),未index的字符列表 - - 数据集：yelp_full/yelp_polarity - - :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` - :param lower: 是否需要自动转小写，默认为False。 - """ - - def __init__(self, fine_grained=False, lower=False): - super(YelpLoader, self).__init__() - tag_v = {'1.0': 'very negative', '2.0': 'negative', '3.0': 'neutral', - '4.0': 'positive', '5.0': 'very positive'} - if not fine_grained: - tag_v['1.0'] = tag_v['2.0'] - tag_v['5.0'] = tag_v['4.0'] - self.fine_grained = fine_grained - self.tag_v = tag_v - self.lower = lower - self.tokenizer = get_tokenizer() - - def _load(self, path): - ds = DataSet() - csv_reader = csv.reader(open(path, encoding='utf-8')) - all_count = 0 - real_count = 0 - for row in csv_reader: - all_count += 1 - if len(row) == 2: - target = self.tag_v[row[0] + ".0"] - words = clean_str(row[1], self.tokenizer, self.lower) - if len(words) != 0: - ds.append(Instance(words=words, target=target)) - real_count += 1 - print("all count:", all_count) - print("real count:", real_count) - return ds - - def process(self, paths: Union[str, Dict[str, str]], - train_ds: Iterable[str] = None, - src_vocab_op: VocabularyOption = None, - tgt_vocab_op: VocabularyOption = None, - char_level_op=False): - paths = check_loader_paths(paths) - info = DataBundle(datasets=self.load(paths)) - src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op) - tgt_vocab = Vocabulary(unknown=None, padding=None) \ - if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op) - _train_ds = [info.datasets[name] - for name in train_ds] if train_ds else info.datasets.values() - - def wordtochar(words): - chars = [] - for word in words: - word = word.lower() - for char in word: - chars.append(char) - chars.append('') - chars.pop() - return chars - - input_name, target_name = Const.INPUT, Const.TARGET - info.vocabs = {} - # 就分隔为char形式 - if char_level_op: - for dataset in info.datasets.values(): - dataset.apply_field(wordtochar, field_name=Const.INPUT, new_field_name=Const.CHAR_INPUT) - else: - src_vocab.from_dataset(*_train_ds, field_name=input_name) - src_vocab.index_dataset(*info.datasets.values(), field_name=input_name, new_field_name=input_name) - info.vocabs[input_name] = src_vocab - - tgt_vocab.from_dataset(*_train_ds, field_name=target_name) - tgt_vocab.index_dataset( - *info.datasets.values(), - field_name=target_name, new_field_name=target_name) - - info.vocabs[target_name] = tgt_vocab - - info.datasets['train'], info.datasets['dev'] = info.datasets['train'].split(0.1, shuffle=False) - - for name, dataset in info.datasets.items(): - dataset.set_input(Const.INPUT) - dataset.set_target(Const.TARGET) - - return info - - -def clean_str(sentence, tokenizer, char_lower=False): - """ - heavily borrowed from github - https://github.com/LukeZhuang/Hierarchical-Attention-Network/blob/master/yelp-preprocess.ipynb - :param sentence: is a str - :return: - """ - if char_lower: - sentence = sentence.lower() - import re - nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') - words = tokenizer(sentence) - words_collection = [] - for word in words: - if word in ['-lrb-', '-rrb-', '', '-r', '-l', 'b-']: - continue - tt = nonalpnum.split(word) - t = ''.join(tt) - if t != '': - words_collection.append(t) - - return words_collection - diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py deleted file mode 100644 index fca0de69..00000000 --- a/fastNLP/io/dataset_loader.py +++ /dev/null @@ -1,121 +0,0 @@ -"""undocumented -.. warning:: - - 本模块将在 `0.5.0版本` 中被废弃，由 :mod:`~fastNLP.io.loader` 和 :mod:`~fastNLP.io.pipe` 模块替代。 - -dataset_loader模块实现了许多 DataSetLoader, 用于读取不同格式的数据, 并返回 `DataSet` , -得到的 :class:`~fastNLP.DataSet` 对象可以直接传入 :class:`~fastNLP.Trainer` 和 :class:`~fastNLP.Tester`, 用于模型的训练和测试。 -以SNLI数据集为例:: - - loader = SNLILoader() - train_ds = loader.load('path/to/train') - dev_ds = loader.load('path/to/dev') - test_ds = loader.load('path/to/test') - - # ... do stuff - -为 fastNLP 提供 DataSetLoader 的开发者请参考 :class:`~fastNLP.io.DataSetLoader` 的介绍。 - -""" -__all__ = [ - 'CSVLoader', - 'JsonLoader', -] - - -from .data_bundle import DataSetLoader -from .file_reader import _read_csv, _read_json -from ..core.dataset import DataSet -from ..core.instance import Instance - - -class JsonLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.JsonLoader` :class:`fastNLP.io.dataset_loader.JsonLoader` - - 读取json格式数据.数据必须按行存储,每行是一个包含各类属性的json对象 - - :param dict fields: 需要读入的json属性名称, 和读入后在DataSet中存储的field_name - ``fields`` 的 `key` 必须是json对象的属性名. ``fields`` 的 `value` 为读入后在DataSet存储的 `field_name` , - `value` 也可为 ``None`` , 这时读入后的 `field_name` 与json对象对应属性同名 - ``fields`` 可为 ``None`` , 这时,json对象所有属性都保存在DataSet中. Default: ``None`` - :param bool dropna: 是否忽略非法数据,若 ``True`` 则忽略,若 ``False`` ,在遇到非法数据时,抛出 ``ValueError`` . - Default: ``False`` - """ - - def __init__(self, fields=None, dropna=False): - super(JsonLoader, self).__init__() - self.dropna = dropna - self.fields = None - self.fields_list = None - if fields: - self.fields = {} - for k, v in fields.items(): - self.fields[k] = k if v is None else v - self.fields_list = list(self.fields.keys()) - - def _load(self, path): - ds = DataSet() - for idx, d in _read_json(path, fields=self.fields_list, dropna=self.dropna): - if self.fields: - ins = {self.fields[k]: v for k, v in d.items()} - else: - ins = d - ds.append(Instance(**ins)) - return ds - - -class CSVLoader(DataSetLoader): - """ - 别名：:class:`fastNLP.io.CSVLoader` :class:`fastNLP.io.dataset_loader.CSVLoader` - - 读取CSV格式的数据集。返回 ``DataSet`` - - :param List[str] headers: CSV文件的文件头.定义每一列的属性名称,即返回的DataSet中`field`的名称 - 若为 ``None`` ,则将读入文件的第一行视作 ``headers`` . Default: ``None`` - :param str sep: CSV文件中列与列之间的分隔符. Default: "," - :param bool dropna: 是否忽略非法数据,若 ``True`` 则忽略,若 ``False`` ,在遇到非法数据时,抛出 ``ValueError`` . - Default: ``False`` - """ - - def __init__(self, headers=None, sep=",", dropna=False): - self.headers = headers - self.sep = sep - self.dropna = dropna - - def _load(self, path): - ds = DataSet() - for idx, data in _read_csv(path, headers=self.headers, - sep=self.sep, dropna=self.dropna): - ds.append(Instance(**data)) - return ds - - -def _cut_long_sentence(sent, max_sample_length=200): - """ - 将长于max_sample_length的sentence截成多段，只会在有空格的地方发生截断。 - 所以截取的句子可能长于或者短于max_sample_length - - :param sent: str. - :param max_sample_length: int. - :return: list of str. - """ - sent_no_space = sent.replace(' ', '') - cutted_sentence = [] - if len(sent_no_space) > max_sample_length: - parts = sent.strip().split() - new_line = '' - length = 0 - for part in parts: - length += len(part) - new_line += part + ' ' - if length > max_sample_length: - new_line = new_line[:-1] - cutted_sentence.append(new_line) - length = 0 - new_line = '' - if new_line != '': - cutted_sentence.append(new_line[:-1]) - else: - cutted_sentence.append(sent) - return cutted_sentence diff --git a/fastNLP/io/embed_loader.py b/fastNLP/io/embed_loader.py index 780d91e4..a157901f 100644 --- a/fastNLP/io/embed_loader.py +++ b/fastNLP/io/embed_loader.py @@ -13,7 +13,6 @@ import warnings import numpy as np -from .data_bundle import BaseLoader from ..core.utils import Option from ..core.vocabulary import Vocabulary @@ -32,7 +31,7 @@ class EmbeddingOption(Option): ) -class EmbedLoader(BaseLoader): +class EmbedLoader: """ 别名：:class:`fastNLP.io.EmbedLoader` :class:`fastNLP.io.embed_loader.EmbedLoader` @@ -84,9 +83,9 @@ class EmbedLoader(BaseLoader): word = ''.join(parts[:-dim]) nums = parts[-dim:] # 对齐unk与pad - if word==padding and vocab.padding is not None: + if word == padding and vocab.padding is not None: word = vocab.padding - elif word==unknown and vocab.unknown is not None: + elif word == unknown and vocab.unknown is not None: word = vocab.unknown if word in vocab: index = vocab.to_index(word) @@ -171,7 +170,7 @@ class EmbedLoader(BaseLoader): index = vocab.to_index(key) matrix[index] = vec - if (unknown is not None and not found_unknown) or (padding is not None and not found_pad): + if ((unknown is not None) and (not found_unknown)) or ((padding is not None) and (not found_pad)): start_idx = 0 if padding is not None: start_idx += 1 @@ -180,9 +179,9 @@ class EmbedLoader(BaseLoader): mean = np.mean(matrix[start_idx:], axis=0, keepdims=True) std = np.std(matrix[start_idx:], axis=0, keepdims=True) - if (unknown is not None and not found_unknown): + if (unknown is not None) and (not found_unknown): matrix[start_idx - 1] = np.random.randn(1, dim).astype(dtype) * std + mean - if (padding is not None and not found_pad): + if (padding is not None) and (not found_pad): matrix[0] = np.random.randn(1, dim).astype(dtype) * std + mean if normalize: diff --git a/fastNLP/io/model_io.py b/fastNLP/io/model_io.py index 22ced1ce..a1899f51 100644 --- a/fastNLP/io/model_io.py +++ b/fastNLP/io/model_io.py @@ -8,10 +8,8 @@ __all__ = [ import torch -from .data_bundle import BaseLoader - -class ModelLoader(BaseLoader): +class ModelLoader: """ 别名：:class:`fastNLP.io.ModelLoader` :class:`fastNLP.io.model_io.ModelLoader` diff --git a/fastNLP/models/enas_controller.py b/fastNLP/models/enas_controller.py deleted file mode 100644 index eec820e4..00000000 --- a/fastNLP/models/enas_controller.py +++ /dev/null @@ -1,228 +0,0 @@ -"""undocumented -Code Modified from https://github.com/carpedm20/ENAS-pytorch -A module with NAS controller-related code. -""" - -__all__ = [] - -import collections -import os - -import torch -import torch.nn.functional as F - -from . import enas_utils as utils -from .enas_utils import Node - - -def _construct_dags(prev_nodes, activations, func_names, num_blocks): - """Constructs a set of DAGs based on the actions, i.e., previous nodes and - activation functions, sampled from the controller/policy pi. - - Args: - prev_nodes: Previous node actions from the policy. - activations: Activations sampled from the policy. - func_names: Mapping from activation function names to functions. - num_blocks: Number of blocks in the target RNN cell. - - Returns: - A list of DAGs defined by the inputs. - - RNN cell DAGs are represented in the following way: - - 1. Each element (node) in a DAG is a list of `Node`s. - - 2. The `Node`s in the list dag[i] correspond to the subsequent nodes - that take the output from node i as their own input. - - 3. dag[-1] is the node that takes input from x^{(t)} and h^{(t - 1)}. - dag[-1] always feeds dag[0]. - dag[-1] acts as if `w_xc`, `w_hc`, `w_xh` and `w_hh` are its - weights. - - 4. dag[N - 1] is the node that produces the hidden state passed to - the next timestep. dag[N - 1] is also always a leaf node, and therefore - is always averaged with the other leaf nodes and fed to the output - decoder. - """ - dags = [] - for nodes, func_ids in zip(prev_nodes, activations): - dag = collections.defaultdict(list) - - # add first node - dag[-1] = [Node(0, func_names[func_ids[0]])] - dag[-2] = [Node(0, func_names[func_ids[0]])] - - # add following nodes - for jdx, (idx, func_id) in enumerate(zip(nodes, func_ids[1:])): - dag[utils.to_item(idx)].append(Node(jdx + 1, func_names[func_id])) - - leaf_nodes = set(range(num_blocks)) - dag.keys() - - # merge with avg - for idx in leaf_nodes: - dag[idx] = [Node(num_blocks, 'avg')] - - # This is actually y^{(t)}. h^{(t)} is node N - 1 in - # the graph, where N Is the number of nodes. I.e., h^{(t)} takes - # only one other node as its input. - # last h[t] node - last_node = Node(num_blocks + 1, 'h[t]') - dag[num_blocks] = [last_node] - dags.append(dag) - - return dags - - -class Controller(torch.nn.Module): - """Based on - https://github.com/pytorch/examples/blob/master/word_language_model/model.py - - RL controllers do not necessarily have much to do with - language models. - - Base the controller RNN on the GRU from: - https://github.com/ikostrikov/pytorch-a2c-ppo-acktr/blob/master/model.py - """ - def __init__(self, num_blocks=4, controller_hid=100, cuda=False): - torch.nn.Module.__init__(self) - - # `num_tokens` here is just the activation function - # for every even step, - self.shared_rnn_activations = ['tanh', 'ReLU', 'identity', 'sigmoid'] - self.num_tokens = [len(self.shared_rnn_activations)] - self.controller_hid = controller_hid - self.use_cuda = cuda - self.num_blocks = num_blocks - for idx in range(num_blocks): - self.num_tokens += [idx + 1, len(self.shared_rnn_activations)] - self.func_names = self.shared_rnn_activations - - num_total_tokens = sum(self.num_tokens) - - self.encoder = torch.nn.Embedding(num_total_tokens, - controller_hid) - self.lstm = torch.nn.LSTMCell(controller_hid, controller_hid) - - # Perhaps these weights in the decoder should be - # shared? At least for the activation functions, which all have the - # same size. - self.decoders = [] - for idx, size in enumerate(self.num_tokens): - decoder = torch.nn.Linear(controller_hid, size) - self.decoders.append(decoder) - - self._decoders = torch.nn.ModuleList(self.decoders) - - self.reset_parameters() - self.static_init_hidden = utils.keydefaultdict(self.init_hidden) - - def _get_default_hidden(key): - return utils.get_variable( - torch.zeros(key, self.controller_hid), - self.use_cuda, - requires_grad=False) - - self.static_inputs = utils.keydefaultdict(_get_default_hidden) - - def reset_parameters(self): - init_range = 0.1 - for param in self.parameters(): - param.data.uniform_(-init_range, init_range) - for decoder in self.decoders: - decoder.bias.data.fill_(0) - - def forward(self, # pylint:disable=arguments-differ - inputs, - hidden, - block_idx, - is_embed): - if not is_embed: - embed = self.encoder(inputs) - else: - embed = inputs - - hx, cx = self.lstm(embed, hidden) - logits = self.decoders[block_idx](hx) - - logits /= 5.0 - - # # exploration - # if self.args.mode == 'train': - # logits = (2.5 * F.tanh(logits)) - - return logits, (hx, cx) - - def sample(self, batch_size=1, with_details=False, save_dir=None): - """Samples a set of `args.num_blocks` many computational nodes from the - controller, where each node is made up of an activation function, and - each node except the last also includes a previous node. - """ - if batch_size < 1: - raise Exception(f'Wrong batch_size: {batch_size} < 1') - - # [B, L, H] - inputs = self.static_inputs[batch_size] - hidden = self.static_init_hidden[batch_size] - - activations = [] - entropies = [] - log_probs = [] - prev_nodes = [] - # The RNN controller alternately outputs an activation, - # followed by a previous node, for each block except the last one, - # which only gets an activation function. The last node is the output - # node, and its previous node is the average of all leaf nodes. - for block_idx in range(2*(self.num_blocks - 1) + 1): - logits, hidden = self.forward(inputs, - hidden, - block_idx, - is_embed=(block_idx == 0)) - - probs = F.softmax(logits, dim=-1) - log_prob = F.log_softmax(logits, dim=-1) - # .mean() for entropy? - entropy = -(log_prob * probs).sum(1, keepdim=False) - - action = probs.multinomial(num_samples=1).data - selected_log_prob = log_prob.gather( - 1, utils.get_variable(action, requires_grad=False)) - - # why the [:, 0] here? Should it be .squeeze(), or - # .view()? Same below with `action`. - entropies.append(entropy) - log_probs.append(selected_log_prob[:, 0]) - - # 0: function, 1: previous node - mode = block_idx % 2 - inputs = utils.get_variable( - action[:, 0] + sum(self.num_tokens[:mode]), - requires_grad=False) - - if mode == 0: - activations.append(action[:, 0]) - elif mode == 1: - prev_nodes.append(action[:, 0]) - - prev_nodes = torch.stack(prev_nodes).transpose(0, 1) - activations = torch.stack(activations).transpose(0, 1) - - dags = _construct_dags(prev_nodes, - activations, - self.func_names, - self.num_blocks) - - if save_dir is not None: - for idx, dag in enumerate(dags): - utils.draw_network(dag, - os.path.join(save_dir, f'graph{idx}.png')) - - if with_details: - return dags, torch.cat(log_probs), torch.cat(entropies) - - return dags - - def init_hidden(self, batch_size): - zeros = torch.zeros(batch_size, self.controller_hid) - return (utils.get_variable(zeros, self.use_cuda, requires_grad=False), - utils.get_variable(zeros.clone(), self.use_cuda, requires_grad=False)) diff --git a/fastNLP/models/enas_model.py b/fastNLP/models/enas_model.py deleted file mode 100644 index 2e8ca713..00000000 --- a/fastNLP/models/enas_model.py +++ /dev/null @@ -1,393 +0,0 @@ -"""undocumented -Module containing the shared RNN model. -Code Modified from https://github.com/carpedm20/ENAS-pytorch -""" - -__all__ = [] - -import collections - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Variable - -from . import enas_utils as utils -from .base_model import BaseModel - - -def _get_dropped_weights(w_raw, dropout_p, is_training): - """Drops out weights to implement DropConnect. - - Args: - w_raw: Full, pre-dropout, weights to be dropped out. - dropout_p: Proportion of weights to drop out. - is_training: True iff _shared_ model is training. - - Returns: - The dropped weights. - - Why does torch.nn.functional.dropout() return: - 1. `torch.autograd.Variable()` on the training loop - 2. `torch.nn.Parameter()` on the controller or eval loop, when - training = False... - - Even though the call to `_setweights` in the Smerity repo's - `weight_drop.py` does not have this behaviour, and `F.dropout` always - returns `torch.autograd.Variable` there, even when `training=False`? - - The above TODO is the reason for the hacky check for `torch.nn.Parameter`. - """ - dropped_w = F.dropout(w_raw, p=dropout_p, training=is_training) - - if isinstance(dropped_w, torch.nn.Parameter): - dropped_w = dropped_w.clone() - - return dropped_w - - -class EmbeddingDropout(torch.nn.Embedding): - """Class for dropping out embeddings by zero'ing out parameters in the - embedding matrix. - - This is equivalent to dropping out particular words, e.g., in the sentence - 'the quick brown fox jumps over the lazy dog', dropping out 'the' would - lead to the sentence '### quick brown fox jumps over ### lazy dog' (in the - embedding vector space). - - See 'A Theoretically Grounded Application of Dropout in Recurrent Neural - Networks', (Gal and Ghahramani, 2016). - """ - - def __init__(self, - num_embeddings, - embedding_dim, - max_norm=None, - norm_type=2, - scale_grad_by_freq=False, - sparse=False, - dropout=0.1, - scale=None): - """Embedding constructor. - - Args: - dropout: Dropout probability. - scale: Used to scale parameters of embedding weight matrix that are - not dropped out. Note that this is _in addition_ to the - `1/(1 - dropout)` scaling. - - See `torch.nn.Embedding` for remaining arguments. - """ - torch.nn.Embedding.__init__(self, - num_embeddings=num_embeddings, - embedding_dim=embedding_dim, - max_norm=max_norm, - norm_type=norm_type, - scale_grad_by_freq=scale_grad_by_freq, - sparse=sparse) - self.dropout = dropout - assert (dropout >= 0.0) and (dropout < 1.0), ('Dropout must be >= 0.0 ' - 'and < 1.0') - self.scale = scale - - def forward(self, inputs): # pylint:disable=arguments-differ - """Embeds `inputs` with the dropped out embedding weight matrix.""" - if self.training: - dropout = self.dropout - else: - dropout = 0 - - if dropout: - mask = self.weight.data.new(self.weight.size(0), 1) - mask.bernoulli_(1 - dropout) - mask = mask.expand_as(self.weight) - mask = mask / (1 - dropout) - masked_weight = self.weight * Variable(mask) - else: - masked_weight = self.weight - if self.scale and self.scale != 1: - masked_weight = masked_weight * self.scale - - return F.embedding(inputs, - masked_weight, - max_norm=self.max_norm, - norm_type=self.norm_type, - scale_grad_by_freq=self.scale_grad_by_freq, - sparse=self.sparse) - - -class LockedDropout(nn.Module): - # code from https://github.com/salesforce/awd-lstm-lm/blob/master/locked_dropout.py - def __init__(self): - super().__init__() - - def forward(self, x, dropout=0.5): - if not self.training or not dropout: - return x - m = x.data.new(1, x.size(1), x.size(2)).bernoulli_(1 - dropout) - mask = Variable(m, requires_grad=False) / (1 - dropout) - mask = mask.expand_as(x) - return mask * x - - -class ENASModel(BaseModel): - """Shared RNN model.""" - - def __init__(self, embed_num, num_classes, num_blocks=4, cuda=False, shared_hid=1000, shared_embed=1000): - super(ENASModel, self).__init__() - - self.use_cuda = cuda - - self.shared_hid = shared_hid - self.num_blocks = num_blocks - self.decoder = nn.Linear(self.shared_hid, num_classes) - self.encoder = EmbeddingDropout(embed_num, - shared_embed, - dropout=0.1) - self.lockdrop = LockedDropout() - self.dag = None - - # Tie weights - # self.decoder.weight = self.encoder.weight - - # Since W^{x, c} and W^{h, c} are always summed, there - # is no point duplicating their bias offset parameter. Likewise for - # W^{x, h} and W^{h, h}. - self.w_xc = nn.Linear(shared_embed, self.shared_hid) - self.w_xh = nn.Linear(shared_embed, self.shared_hid) - - # The raw weights are stored here because the hidden-to-hidden weights - # are weight dropped on the forward pass. - self.w_hc_raw = torch.nn.Parameter( - torch.Tensor(self.shared_hid, self.shared_hid)) - self.w_hh_raw = torch.nn.Parameter( - torch.Tensor(self.shared_hid, self.shared_hid)) - self.w_hc = None - self.w_hh = None - - self.w_h = collections.defaultdict(dict) - self.w_c = collections.defaultdict(dict) - - for idx in range(self.num_blocks): - for jdx in range(idx + 1, self.num_blocks): - self.w_h[idx][jdx] = nn.Linear(self.shared_hid, - self.shared_hid, - bias=False) - self.w_c[idx][jdx] = nn.Linear(self.shared_hid, - self.shared_hid, - bias=False) - - self._w_h = nn.ModuleList([self.w_h[idx][jdx] - for idx in self.w_h - for jdx in self.w_h[idx]]) - self._w_c = nn.ModuleList([self.w_c[idx][jdx] - for idx in self.w_c - for jdx in self.w_c[idx]]) - - self.batch_norm = None - # if args.mode == 'train': - # self.batch_norm = nn.BatchNorm1d(self.shared_hid) - # else: - # self.batch_norm = None - - self.reset_parameters() - self.static_init_hidden = utils.keydefaultdict(self.init_hidden) - - def setDAG(self, dag): - if self.dag is None: - self.dag = dag - - def forward(self, word_seq, hidden=None): - inputs = torch.transpose(word_seq, 0, 1) - - time_steps = inputs.size(0) - batch_size = inputs.size(1) - - self.w_hh = _get_dropped_weights(self.w_hh_raw, - 0.5, - self.training) - self.w_hc = _get_dropped_weights(self.w_hc_raw, - 0.5, - self.training) - - # hidden = self.static_init_hidden[batch_size] if hidden is None else hidden - hidden = self.static_init_hidden[batch_size] - - embed = self.encoder(inputs) - - embed = self.lockdrop(embed, 0.65 if self.training else 0) - - # The norm of hidden states are clipped here because - # otherwise ENAS is especially prone to exploding activations on the - # forward pass. This could probably be fixed in a more elegant way, but - # it might be exposing a weakness in the ENAS algorithm as currently - # proposed. - # - # For more details, see - # https://github.com/carpedm20/ENAS-pytorch/issues/6 - clipped_num = 0 - max_clipped_norm = 0 - h1tohT = [] - logits = [] - for step in range(time_steps): - x_t = embed[step] - logit, hidden = self.cell(x_t, hidden, self.dag) - - hidden_norms = hidden.norm(dim=-1) - max_norm = 25.0 - if hidden_norms.data.max() > max_norm: - # Just directly use the torch slice operations - # in PyTorch v0.4. - # - # This workaround for PyTorch v0.3.1 does everything in numpy, - # because the PyTorch slicing and slice assignment is too - # flaky. - hidden_norms = hidden_norms.data.cpu().numpy() - - clipped_num += 1 - if hidden_norms.max() > max_clipped_norm: - max_clipped_norm = hidden_norms.max() - - clip_select = hidden_norms > max_norm - clip_norms = hidden_norms[clip_select] - - mask = np.ones(hidden.size()) - normalizer = max_norm / clip_norms - normalizer = normalizer[:, np.newaxis] - - mask[clip_select] = normalizer - - if self.use_cuda: - hidden *= torch.autograd.Variable( - torch.FloatTensor(mask).cuda(), requires_grad=False) - else: - hidden *= torch.autograd.Variable( - torch.FloatTensor(mask), requires_grad=False) - logits.append(logit) - h1tohT.append(hidden) - - h1tohT = torch.stack(h1tohT) - output = torch.stack(logits) - raw_output = output - - output = self.lockdrop(output, 0.4 if self.training else 0) - - # Pooling - output = torch.mean(output, 0) - - decoded = self.decoder(output) - - extra_out = {'dropped': decoded, - 'hiddens': h1tohT, - 'raw': raw_output} - return {'pred': decoded, 'hidden': hidden, 'extra_out': extra_out} - - def cell(self, x, h_prev, dag): - """Computes a single pass through the discovered RNN cell.""" - c = {} - h = {} - f = {} - - f[0] = self.get_f(dag[-1][0].name) - c[0] = torch.sigmoid(self.w_xc(x) + F.linear(h_prev, self.w_hc, None)) - h[0] = (c[0] * f[0](self.w_xh(x) + F.linear(h_prev, self.w_hh, None)) + - (1 - c[0]) * h_prev) - - leaf_node_ids = [] - q = collections.deque() - q.append(0) - - # Computes connections from the parent nodes `node_id` - # to their child nodes `next_id` recursively, skipping leaf nodes. A - # leaf node is a node whose id == `self.num_blocks`. - # - # Connections between parent i and child j should be computed as - # h_j = c_j*f_{ij}{(W^h_{ij}*h_i)} + (1 - c_j)*h_i, - # where c_j = \sigmoid{(W^c_{ij}*h_i)} - # - # See Training details from Section 3.1 of the paper. - # - # The following algorithm does a breadth-first (since `q.popleft()` is - # used) search over the nodes and computes all the hidden states. - while True: - if len(q) == 0: - break - - node_id = q.popleft() - nodes = dag[node_id] - - for next_node in nodes: - next_id = next_node.id - if next_id == self.num_blocks: - leaf_node_ids.append(node_id) - assert len(nodes) == 1, ('parent of leaf node should have ' - 'only one child') - continue - - w_h = self.w_h[node_id][next_id] - w_c = self.w_c[node_id][next_id] - - f[next_id] = self.get_f(next_node.name) - c[next_id] = torch.sigmoid(w_c(h[node_id])) - h[next_id] = (c[next_id] * f[next_id](w_h(h[node_id])) + - (1 - c[next_id]) * h[node_id]) - - q.append(next_id) - - # Instead of averaging loose ends, perhaps there should - # be a set of separate unshared weights for each "loose" connection - # between each node in a cell and the output. - # - # As it stands, all weights W^h_{ij} are doing double duty by - # connecting both from i to j, as well as from i to the output. - - # average all the loose ends - leaf_nodes = [h[node_id] for node_id in leaf_node_ids] - output = torch.mean(torch.stack(leaf_nodes, 2), -1) - - # stabilizing the Updates of omega - if self.batch_norm is not None: - output = self.batch_norm(output) - - return output, h[self.num_blocks - 1] - - def init_hidden(self, batch_size): - zeros = torch.zeros(batch_size, self.shared_hid) - return utils.get_variable(zeros, self.use_cuda, requires_grad=False) - - def get_f(self, name): - name = name.lower() - if name == 'relu': - f = torch.relu - elif name == 'tanh': - f = torch.tanh - elif name == 'identity': - f = lambda x: x - elif name == 'sigmoid': - f = torch.sigmoid - return f - - @property - def num_parameters(self): - def size(p): - return np.prod(p.size()) - - return sum([size(param) for param in self.parameters()]) - - def reset_parameters(self): - init_range = 0.025 - # init_range = 0.025 if self.args.mode == 'train' else 0.04 - for param in self.parameters(): - param.data.uniform_(-init_range, init_range) - self.decoder.bias.data.fill_(0) - - def predict(self, word_seq): - """ - - :param word_seq: torch.LongTensor, [batch_size, seq_len] - :return predict: dict of torch.LongTensor, [batch_size, seq_len] - """ - output = self(word_seq) - _, predict = output['pred'].max(dim=1) - return {'pred': predict} diff --git a/fastNLP/models/enas_trainer.py b/fastNLP/models/enas_trainer.py deleted file mode 100644 index 98d778cd..00000000 --- a/fastNLP/models/enas_trainer.py +++ /dev/null @@ -1,384 +0,0 @@ -"""undocumented -Code Modified from https://github.com/carpedm20/ENAS-pytorch -""" - -__all__ = [] - -import math -import time -from datetime import datetime, timedelta - -import numpy as np -import torch -from torch.optim import Adam - -try: - from tqdm.auto import tqdm -except: - from ..core.utils import _pseudo_tqdm as tqdm - -from ..core.trainer import Trainer -from ..core.batch import DataSetIter -from ..core.callback import CallbackException -from ..core.dataset import DataSet -from ..core.utils import _move_dict_value_to_device -from . import enas_utils as utils -from ..core.utils import _build_args - - -def _get_no_grad_ctx_mgr(): - """Returns a the `torch.no_grad` context manager for PyTorch version >= - 0.4, or a no-op context manager otherwise. - """ - return torch.no_grad() - - -class ENASTrainer(Trainer): - """A class to wrap training code.""" - - def __init__(self, train_data, model, controller, **kwargs): - """Constructor for training algorithm. - :param DataSet train_data: the training data - :param torch.nn.modules.module model: a PyTorch model - :param torch.nn.modules.module controller: a PyTorch model - """ - self.final_epochs = kwargs['final_epochs'] - kwargs.pop('final_epochs') - super(ENASTrainer, self).__init__(train_data, model, **kwargs) - self.controller_step = 0 - self.shared_step = 0 - self.max_length = 35 - - self.shared = model - self.controller = controller - - self.shared_optim = Adam( - self.shared.parameters(), - lr=20.0, - weight_decay=1e-7) - - self.controller_optim = Adam( - self.controller.parameters(), - lr=3.5e-4) - - def train(self, load_best_model=True): - """ - :param bool load_best_model: 该参数只有在初始化提供了dev_data的情况下有效，如果True, trainer将在返回之前重新加载dev表现 - 最好的模型参数。 - :return results: 返回一个字典类型的数据, - 内含以下内容:: - - seconds: float, 表示训练时长 - 以下三个内容只有在提供了dev_data的情况下会有。 - best_eval: Dict of Dict, 表示evaluation的结果 - best_epoch: int，在第几个epoch取得的最佳值 - best_step: int, 在第几个step(batch)更新取得的最佳值 - - """ - results = {} - if self.n_epochs <= 0: - print(f"training epoch is {self.n_epochs}, nothing was done.") - results['seconds'] = 0. - return results - try: - if torch.cuda.is_available() and "cuda" in self.device: - self.model = self.model.cuda() - self._model_device = self.model.parameters().__next__().device - self._mode(self.model, is_test=False) - - self.start_time = str(datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) - start_time = time.time() - print("training epochs started " + self.start_time, flush=True) - - try: - self.callback_manager.on_train_begin() - self._train() - self.callback_manager.on_train_end() - except (CallbackException, KeyboardInterrupt) as e: - self.callback_manager.on_exception(e) - - if self.dev_data is not None: - print( - "\nIn Epoch:{}/Step:{}, got best dev performance:".format(self.best_dev_epoch, self.best_dev_step) + - self.tester._format_eval_results(self.best_dev_perf), ) - results['best_eval'] = self.best_dev_perf - results['best_epoch'] = self.best_dev_epoch - results['best_step'] = self.best_dev_step - if load_best_model: - model_name = "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time]) - load_succeed = self._load_model(self.model, model_name) - if load_succeed: - print("Reloaded the best model.") - else: - print("Fail to reload best model.") - finally: - pass - results['seconds'] = round(time.time() - start_time, 2) - - return results - - def _train(self): - if not self.use_tqdm: - from fastNLP.core.utils import _pseudo_tqdm as inner_tqdm - else: - inner_tqdm = tqdm - self.step = 0 - start = time.time() - total_steps = (len(self.train_data) // self.batch_size + int( - len(self.train_data) % self.batch_size != 0)) * self.n_epochs - with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: - avg_loss = 0 - data_iterator = DataSetIter(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - prefetch=self.prefetch) - for epoch in range(1, self.n_epochs + 1): - pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) - last_stage = (epoch > self.n_epochs + 1 - self.final_epochs) - if epoch == self.n_epochs + 1 - self.final_epochs: - print('Entering the final stage. (Only train the selected structure)') - # early stopping - self.callback_manager.on_epoch_begin() - - # 1. Training the shared parameters omega of the child models - self.train_shared(pbar) - - # 2. Training the controller parameters theta - if not last_stage: - self.train_controller() - - if ((self.validate_every > 0 and self.step % self.validate_every == 0) or - (self.validate_every < 0 and self.step % len(data_iterator) == 0)) \ - and self.dev_data is not None: - if not last_stage: - self.derive() - eval_res = self._do_validation(epoch=epoch, step=self.step) - eval_str = "Evaluation at Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, - total_steps) + \ - self.tester._format_eval_results(eval_res) - pbar.write(eval_str) - - # lr decay; early stopping - self.callback_manager.on_epoch_end() - # =============== epochs end =================== # - pbar.close() - # ============ tqdm end ============== # - - def get_loss(self, inputs, targets, hidden, dags): - """Computes the loss for the same batch for M models. - - This amounts to an estimate of the loss, which is turned into an - estimate for the gradients of the shared model. - """ - if not isinstance(dags, list): - dags = [dags] - - loss = 0 - for dag in dags: - self.shared.setDAG(dag) - inputs = _build_args(self.shared.forward, **inputs) - inputs['hidden'] = hidden - result = self.shared(**inputs) - output, hidden, extra_out = result['pred'], result['hidden'], result['extra_out'] - - self.callback_manager.on_loss_begin(targets, result) - sample_loss = self._compute_loss(result, targets) - loss += sample_loss - - assert len(dags) == 1, 'there are multiple `hidden` for multple `dags`' - return loss, hidden, extra_out - - def train_shared(self, pbar=None, max_step=None, dag=None): - """Train the language model for 400 steps of minibatches of 64 - examples. - - Args: - max_step: Used to run extra training steps as a warm-up. - dag: If not None, is used instead of calling sample(). - - BPTT is truncated at 35 timesteps. - - For each weight update, gradients are estimated by sampling M models - from the fixed controller policy, and averaging their gradients - computed on a batch of training data. - """ - model = self.shared - model.train() - self.controller.eval() - - hidden = self.shared.init_hidden(self.batch_size) - - abs_max_grad = 0 - abs_max_hidden_norm = 0 - step = 0 - raw_total_loss = 0 - total_loss = 0 - train_idx = 0 - avg_loss = 0 - data_iterator = DataSetIter(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - prefetch=self.prefetch) - - for batch_x, batch_y in data_iterator: - _move_dict_value_to_device(batch_x, batch_y, device=self._model_device) - indices = data_iterator.get_batch_indices() - # negative sampling; replace unknown; re-weight batch_y - self.callback_manager.on_batch_begin(batch_x, batch_y, indices) - # prediction = self._data_forward(self.model, batch_x) - - dags = self.controller.sample(1) - inputs, targets = batch_x, batch_y - # self.callback_manager.on_loss_begin(batch_y, prediction) - loss, hidden, extra_out = self.get_loss(inputs, - targets, - hidden, - dags) - hidden.detach_() - - avg_loss += loss.item() - - # Is loss NaN or inf? requires_grad = False - self.callback_manager.on_backward_begin(loss) - self._grad_backward(loss) - self.callback_manager.on_backward_end() - - self._update() - self.callback_manager.on_step_end() - - if (self.step + 1) % self.print_every == 0: - if self.use_tqdm: - print_output = "loss:{0:<6.5f}".format(avg_loss / self.print_every) - pbar.update(self.print_every) - else: - end = time.time() - diff = timedelta(seconds=round(end - start)) - print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format( - epoch, self.step, avg_loss, diff) - pbar.set_postfix_str(print_output) - avg_loss = 0 - self.step += 1 - step += 1 - self.shared_step += 1 - self.callback_manager.on_batch_end() - # ================= mini-batch end ==================== # - - def get_reward(self, dag, entropies, hidden, valid_idx=0): - """Computes the perplexity of a single sampled model on a minibatch of - validation data. - """ - if not isinstance(entropies, np.ndarray): - entropies = entropies.data.cpu().numpy() - - data_iterator = DataSetIter(self.dev_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - prefetch=self.prefetch) - - for inputs, targets in data_iterator: - valid_loss, hidden, _ = self.get_loss(inputs, targets, hidden, dag) - valid_loss = utils.to_item(valid_loss.data) - - valid_ppl = math.exp(valid_loss) - - R = 80 / valid_ppl - - rewards = R + 1e-4 * entropies - - return rewards, hidden - - def train_controller(self): - """Fixes the shared parameters and updates the controller parameters. - - The controller is updated with a score function gradient estimator - (i.e., REINFORCE), with the reward being c/valid_ppl, where valid_ppl - is computed on a minibatch of validation data. - - A moving average baseline is used. - - The controller is trained for 2000 steps per epoch (i.e., - first (Train Shared) phase -> second (Train Controller) phase). - """ - model = self.controller - model.train() - # Why can't we call shared.eval() here? Leads to loss - # being uniformly zero for the controller. - # self.shared.eval() - - avg_reward_base = None - baseline = None - adv_history = [] - entropy_history = [] - reward_history = [] - - hidden = self.shared.init_hidden(self.batch_size) - total_loss = 0 - valid_idx = 0 - for step in range(20): - # sample models - dags, log_probs, entropies = self.controller.sample( - with_details=True) - - # calculate reward - np_entropies = entropies.data.cpu().numpy() - # No gradients should be backpropagated to the - # shared model during controller training, obviously. - with _get_no_grad_ctx_mgr(): - rewards, hidden = self.get_reward(dags, - np_entropies, - hidden, - valid_idx) - - reward_history.extend(rewards) - entropy_history.extend(np_entropies) - - # moving average baseline - if baseline is None: - baseline = rewards - else: - decay = 0.95 - baseline = decay * baseline + (1 - decay) * rewards - - adv = rewards - baseline - adv_history.extend(adv) - - # policy loss - loss = -log_probs * utils.get_variable(adv, - 'cuda' in self.device, - requires_grad=False) - - loss = loss.sum() # or loss.mean() - - # update - self.controller_optim.zero_grad() - loss.backward() - - self.controller_optim.step() - - total_loss += utils.to_item(loss.data) - - if ((step % 50) == 0) and (step > 0): - reward_history, adv_history, entropy_history = [], [], [] - total_loss = 0 - - self.controller_step += 1 - # prev_valid_idx = valid_idx - # valid_idx = ((valid_idx + self.max_length) % - # (self.valid_data.size(0) - 1)) - # # Whenever we wrap around to the beginning of the - # # validation data, we reset the hidden states. - # if prev_valid_idx > valid_idx: - # hidden = self.shared.init_hidden(self.batch_size) - - def derive(self, sample_num=10, valid_idx=0): - """We are always deriving based on the very first batch - of validation data? This seems wrong... - """ - hidden = self.shared.init_hidden(self.batch_size) - - dags, _, entropies = self.controller.sample(sample_num, - with_details=True) - - max_R = 0 - best_dag = None - for dag in dags: - R, _ = self.get_reward(dag, entropies, hidden, valid_idx) - if R.max() > max_R: - max_R = R.max() - best_dag = dag - - self.model.setDAG(best_dag) diff --git a/fastNLP/models/enas_utils.py b/fastNLP/models/enas_utils.py deleted file mode 100644 index cd6c2503..00000000 --- a/fastNLP/models/enas_utils.py +++ /dev/null @@ -1,58 +0,0 @@ -"""undocumented -Code Modified from https://github.com/carpedm20/ENAS-pytorch -""" - -__all__ = [] - -import collections -from collections import defaultdict - -import numpy as np -import torch -from torch.autograd import Variable - - -def detach(h): - if type(h) == Variable: - return Variable(h.data) - else: - return tuple(detach(v) for v in h) - - -def get_variable(inputs, cuda=False, **kwargs): - if type(inputs) in [list, np.ndarray]: - inputs = torch.Tensor(inputs) - if cuda: - out = Variable(inputs.cuda(), **kwargs) - else: - out = Variable(inputs, **kwargs) - return out - - -def update_lr(optimizer, lr): - for param_group in optimizer.param_groups: - param_group['lr'] = lr - - -Node = collections.namedtuple('Node', ['id', 'name']) - - -class keydefaultdict(defaultdict): - def __missing__(self, key): - if self.default_factory is None: - raise KeyError(key) - else: - ret = self[key] = self.default_factory(key) - return ret - - -def to_item(x): - """Converts x, possibly scalar and possibly tensor, to a Python scalar.""" - if isinstance(x, (float, int)): - return x - - if float(torch.__version__[0:3]) < 0.4: - assert (x.dim() == 1) and (len(x) == 1) - return x[0] - - return x.item() diff --git a/legacy/api/README.md b/legacy/api/README.md deleted file mode 100644 index 73560f9f..00000000 --- a/legacy/api/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# fastNLP 高级接口 - -### 环境与配置 -1. 系统环境：linux/ubuntu(推荐) -2. 编程语言：Python>=3.6 -3. Python包依赖 - - **torch==1.0** - - numpy>=1.14.2 - -### 中文分词 -```python -text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', - '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', - '那么这款无人机到底有多厉害？'] -from fastNLP.api import CWS -cws = CWS(device='cpu') -print(cws.predict(text)) -# ['编者 按 : 7月 12日 , 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一 款 高 科技 隐形 无人 机雷电 之 神 。', '这 款 飞行 从 外型 上 来 看 酷似 电影 中 的 太空 飞行器 , 据 英国 方面 介绍 , 可以 实现 洲际 远程 打击 。', '那么 这 款 无人 机 到底 有 多 厉害 ?'] -``` - -### 词性标注 -```python -# 输入已分词序列 -text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', - '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], - ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] -from fastNLP.api import POS -pos = POS(device='cpu') -print(pos.predict(text)) -# [['编者/NN', '按：/NN', '7月/NT', '12日/NT', '，/PU', '英国/NR', '航空/NN', '航天/NN', '系统/NN', '公司/NN', '公布/VV', '了/AS', '该/DT', '公司/NN', '研制/VV', '的/DEC', '第一款/NN', '高科技/NN', '隐形/AD', '无人机/VV', '雷电之神/NN', '。/PU'], ['那么/AD', '这/DT', '款/NN', '无人机/VV', '到底/AD', '有/VE', '多/AD', '厉害/VA', '？/PU']] -``` - -### 句法分析 -```python -text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', - '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], - ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] -from fastNLP.api import Parser -parser = Parser(device='cpu') -print(parser.predict(text)) -# [['2/nn', '4/nn', '4/nn', '20/tmod', '11/punct', '10/nn', '10/nn', '10/nn', '10/nn', '11/nsubj', '20/dep', '11/asp', '14/det', '15/nsubj', '18/rcmod', '15/cpm', '18/nn', '11/dobj', '20/advmod', '0/root', '20/dobj', '20/punct'], ['4/advmod', '3/det', '8/xsubj', '8/dep', '8/advmod', '8/dep', '8/advmod', '0/root', '8/punct']] -``` - -完整样例见`examples.py` \ No newline at end of file diff --git a/legacy/api/__init__.py b/legacy/api/__init__.py deleted file mode 100644 index 5171d8c2..00000000 --- a/legacy/api/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__all__ = ["CWS", "POS", "Parser"] -from .api import CWS, POS, Parser diff --git a/legacy/api/api.py b/legacy/api/api.py deleted file mode 100644 index 1408731f..00000000 --- a/legacy/api/api.py +++ /dev/null @@ -1,463 +0,0 @@ -import warnings - -import torch - -warnings.filterwarnings('ignore') -import os - -from fastNLP.core.dataset import DataSet -from .utils import load_url -from .processor import ModelProcessor -from fastNLP.io.dataset_loader import _cut_long_sentence -from fastNLP.io.data_loader import ConllLoader -from fastNLP.core.instance import Instance -from ..api.pipeline import Pipeline -from fastNLP.core.metrics import SpanFPreRecMetric -from .processor import IndexerProcessor - -# TODO add pretrain urls -model_urls = { - "cws": "http://123.206.98.91:8888/download/cws_lstm_ctb9_1_20-09908656.pkl", - "pos": "http://123.206.98.91:8888/download/pos_tag_model_20190119-43f8b435.pkl", - "parser": "http://123.206.98.91:8888/download/parser_20190204-c72ca5c0.pkl" -} - - -class ConllCWSReader(object): - """Deprecated. Use ConllLoader for all types of conll-format files.""" - - def __init__(self): - pass - - def load(self, path, cut_long_sent=False): - """ - 返回的DataSet只包含raw_sentence这个field，内容为str。 - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - :: - - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.strip().split()) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_char_lst(sample) - if res is None: - continue - line = ' '.join(res) - if cut_long_sent: - sents = _cut_long_sentence(line) - else: - sents = [line] - for raw_sentence in sents: - ds.append(Instance(raw_sentence=raw_sentence)) - return ds - - def get_char_lst(self, sample): - if len(sample) == 0: - return None - text = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - return text - - -class ConllxDataLoader(ConllLoader): - """返回“词级别”的标签信息，包括词、词性、（句法）头依赖、（句法）边标签。跟``ZhConllPOSReader``完全不同。 - - Deprecated. Use ConllLoader for all types of conll-format files. - """ - - def __init__(self): - headers = [ - 'words', 'pos_tags', 'heads', 'labels', - ] - indexs = [ - 1, 3, 6, 7, - ] - super(ConllxDataLoader, self).__init__(headers=headers, indexes=indexs) - - -class API: - def __init__(self): - self.pipeline = None - self._dict = None - - def predict(self, *args, **kwargs): - """Do prediction for the given input. - """ - raise NotImplementedError - - def test(self, file_path): - """Test performance over the given data set. - - :param str file_path: - :return: a dictionary of metric values - """ - raise NotImplementedError - - def load(self, path, device): - if os.path.exists(os.path.expanduser(path)): - _dict = torch.load(path, map_location='cpu') - else: - _dict = load_url(path, map_location='cpu') - self._dict = _dict - self.pipeline = _dict['pipeline'] - for processor in self.pipeline.pipeline: - if isinstance(processor, ModelProcessor): - processor.set_model_device(device) - - -class POS(API): - """FastNLP API for Part-Of-Speech tagging. - - :param str model_path: the path to the model. - :param str device: device name such as "cpu" or "cuda:0". Use the same notation as PyTorch. - - """ - - def __init__(self, model_path=None, device='cpu'): - super(POS, self).__init__() - if model_path is None: - model_path = model_urls['pos'] - - self.load(model_path, device) - - def predict(self, content): - """predict函数的介绍， - 函数介绍的第二句，这句话不会换行 - - :param content: list of list of str. Each string is a token(word). - :return answer: list of list of str. Each string is a tag. - """ - if not hasattr(self, "pipeline"): - raise ValueError("You have to load model first.") - - sentence_list = content - # 1. 检查sentence的类型 - for sentence in sentence_list: - if not all((type(obj) == str for obj in sentence)): - raise ValueError("Input must be list of list of string.") - - # 2. 组建dataset - dataset = DataSet() - dataset.add_field("words", sentence_list) - - # 3. 使用pipeline - self.pipeline(dataset) - - def merge_tag(words_list, tags_list): - rtn = [] - for words, tags in zip(words_list, tags_list): - rtn.append([w + "/" + t for w, t in zip(words, tags)]) - return rtn - - output = dataset.field_arrays["tag"].content - if isinstance(content, str): - return output[0] - elif isinstance(content, list): - return merge_tag(content, output) - - def test(self, file_path): - test_data = ConllxDataLoader().load(file_path) - - save_dict = self._dict - tag_vocab = save_dict["tag_vocab"] - pipeline = save_dict["pipeline"] - index_tag = IndexerProcessor(vocab=tag_vocab, field_name="tag", new_added_field_name="truth", is_input=False) - pipeline.pipeline = [index_tag] + pipeline.pipeline - - test_data.rename_field("pos_tags", "tag") - pipeline(test_data) - test_data.set_target("truth") - prediction = test_data.field_arrays["predict"].content - truth = test_data.field_arrays["truth"].content - seq_len = test_data.field_arrays["word_seq_origin_len"].content - - # padding by hand - max_length = max([len(seq) for seq in prediction]) - for idx in range(len(prediction)): - prediction[idx] = list(prediction[idx]) + ([0] * (max_length - len(prediction[idx]))) - truth[idx] = list(truth[idx]) + ([0] * (max_length - len(truth[idx]))) - evaluator = SpanFPreRecMetric(tag_vocab=tag_vocab, pred="predict", target="truth", - seq_len="word_seq_origin_len") - evaluator({"predict": torch.Tensor(prediction), "word_seq_origin_len": torch.Tensor(seq_len)}, - {"truth": torch.Tensor(truth)}) - test_result = evaluator.get_metric() - f1 = round(test_result['f'] * 100, 2) - pre = round(test_result['pre'] * 100, 2) - rec = round(test_result['rec'] * 100, 2) - - return {"F1": f1, "precision": pre, "recall": rec} - - -class CWS(API): - """ - 中文分词高级接口。 - - :param model_path: 当model_path为None，使用默认位置的model。如果默认位置不存在，则自动下载模型 - :param device: str，可以为'cpu', 'cuda'或'cuda:0'等。会将模型load到相应device进行推断。 - """ - - def __init__(self, model_path=None, device='cpu'): - - super(CWS, self).__init__() - if model_path is None: - model_path = model_urls['cws'] - - self.load(model_path, device) - - def predict(self, content): - """ - 分词接口。 - - :param content: str或List[str], 例如: "中文分词很重要！"， 返回的结果是"中文 分词 很 重要 !"。 如果传入的为List[str]，比如 - [ "中文分词很重要！", ...], 返回的结果["中文 分词 很 重要 !", ...]。 - :return: str或List[str], 根据输入的的类型决定。 - """ - if not hasattr(self, 'pipeline'): - raise ValueError("You have to load model first.") - - sentence_list = [] - # 1. 检查sentence的类型 - if isinstance(content, str): - sentence_list.append(content) - elif isinstance(content, list): - sentence_list = content - - # 2. 组建dataset - dataset = DataSet() - dataset.add_field('raw_sentence', sentence_list) - - # 3. 使用pipeline - self.pipeline(dataset) - - output = dataset.get_field('output').content - if isinstance(content, str): - return output[0] - elif isinstance(content, list): - return output - - def test(self, filepath): - """ - 传入一个分词文件路径，返回该数据集上分词f1, precision, recall。 - 分词文件应该为:: - - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - - 以空行分割两个句子，有内容的每行有7列。 - - :param filepath: str, 文件路径路径。 - :return: float, float, float. 分别f1, precision, recall. - """ - tag_proc = self._dict['tag_proc'] - cws_model = self.pipeline.pipeline[-2].model - pipeline = self.pipeline.pipeline[:-2] - - pipeline.insert(1, tag_proc) - pp = Pipeline(pipeline) - - reader = ConllCWSReader() - - # te_filename = '/home/hyan/ctb3/test.conllx' - te_dataset = reader.load(filepath) - pp(te_dataset) - - from ..core.tester import Tester - from ..core.metrics import SpanFPreRecMetric - - tester = Tester(data=te_dataset, model=cws_model, metrics=SpanFPreRecMetric(tag_proc.get_vocab()), batch_size=64, - verbose=0) - eval_res = tester.test() - - f1 = eval_res['SpanFPreRecMetric']['f'] - pre = eval_res['SpanFPreRecMetric']['pre'] - rec = eval_res['SpanFPreRecMetric']['rec'] - # print("f1:{:.2f}, pre:{:.2f}, rec:{:.2f}".format(f1, pre, rec)) - - return {"F1": f1, "precision": pre, "recall": rec} - - -class Parser(API): - def __init__(self, model_path=None, device='cpu'): - super(Parser, self).__init__() - if model_path is None: - model_path = model_urls['parser'] - - self.pos_tagger = POS(device=device) - self.load(model_path, device) - - def predict(self, content): - if not hasattr(self, 'pipeline'): - raise ValueError("You have to load model first.") - - # 1. 利用POS得到分词和pos tagging结果 - pos_out = self.pos_tagger.predict(content) - # pos_out = ['这里/NN 是/VB 分词/NN 结果/NN'.split()] - - # 2. 组建dataset - dataset = DataSet() - dataset.add_field('wp', pos_out) - dataset.apply(lambda x: [''] + [w.split('/')[0] for w in x['wp']], new_field_name='words') - dataset.apply(lambda x: [''] + [w.split('/')[1] for w in x['wp']], new_field_name='pos') - dataset.rename_field("words", "raw_words") - - # 3. 使用pipeline - self.pipeline(dataset) - dataset.apply(lambda x: [str(arc) for arc in x['arc_pred']], new_field_name='arc_pred') - dataset.apply(lambda x: [arc + '/' + label for arc, label in - zip(x['arc_pred'], x['label_pred_seq'])][1:], new_field_name='output') - # output like: [['2/top', '0/root', '4/nn', '2/dep']] - return dataset.field_arrays['output'].content - - def load_test_file(self, path): - def get_one(sample): - sample = list(map(list, zip(*sample))) - if len(sample) == 0: - return None - for w in sample[7]: - if w == '_': - print('Error Sample {}'.format(sample)) - return None - # return word_seq, pos_seq, head_seq, head_tag_seq - return sample[1], sample[3], list(map(int, sample[6])), sample[7] - - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - data = [get_one(sample) for sample in datalist] - data_list = list(filter(lambda x: x is not None, data)) - return data_list - - def test(self, filepath): - data = self.load_test_file(filepath) - - def convert(data): - BOS = '' - dataset = DataSet() - for sample in data: - word_seq = [BOS] + sample[0] - pos_seq = [BOS] + sample[1] - heads = [0] + sample[2] - head_tags = [BOS] + sample[3] - dataset.append(Instance(raw_words=word_seq, - pos=pos_seq, - gold_heads=heads, - arc_true=heads, - tags=head_tags)) - return dataset - - ds = convert(data) - pp = self.pipeline - for p in pp: - if p.field_name == 'word_list': - p.field_name = 'gold_words' - elif p.field_name == 'pos_list': - p.field_name = 'gold_pos' - # ds.rename_field("words", "raw_words") - # ds.rename_field("tag", "pos") - pp(ds) - head_cor, label_cor, total = 0, 0, 0 - for ins in ds: - head_gold = ins['gold_heads'] - head_pred = ins['arc_pred'] - length = len(head_gold) - total += length - for i in range(length): - head_cor += 1 if head_pred[i] == head_gold[i] else 0 - uas = head_cor / total - # print('uas:{:.2f}'.format(uas)) - - for p in pp: - if p.field_name == 'gold_words': - p.field_name = 'word_list' - elif p.field_name == 'gold_pos': - p.field_name = 'pos_list' - - return {"USA": round(uas, 5)} - - -class Analyzer: - def __init__(self, device='cpu'): - - self.cws = CWS(device=device) - self.pos = POS(device=device) - self.parser = Parser(device=device) - - def predict(self, content, seg=False, pos=False, parser=False): - if seg is False and pos is False and parser is False: - seg = True - output_dict = {} - if seg: - seg_output = self.cws.predict(content) - output_dict['seg'] = seg_output - if pos: - pos_output = self.pos.predict(content) - output_dict['pos'] = pos_output - if parser: - parser_output = self.parser.predict(content) - output_dict['parser'] = parser_output - - return output_dict - - def test(self, filepath): - output_dict = {} - if self.cws: - seg_output = self.cws.test(filepath) - output_dict['seg'] = seg_output - if self.pos: - pos_output = self.pos.test(filepath) - output_dict['pos'] = pos_output - if self.parser: - parser_output = self.parser.test(filepath) - output_dict['parser'] = parser_output - - return output_dict diff --git a/legacy/api/converter.py b/legacy/api/converter.py deleted file mode 100644 index 4e03e465..00000000 --- a/legacy/api/converter.py +++ /dev/null @@ -1,181 +0,0 @@ -import re - - -class SpanConverter: - def __init__(self, replace_tag, pattern): - super(SpanConverter, self).__init__() - - self.replace_tag = replace_tag - self.pattern = pattern - - def find_certain_span_and_replace(self, sentence): - replaced_sentence = '' - prev_end = 0 - for match in re.finditer(self.pattern, sentence): - start, end = match.span() - span = sentence[start:end] - replaced_sentence += sentence[prev_end:start] + self.span_to_special_tag(span) - prev_end = end - replaced_sentence += sentence[prev_end:] - - return replaced_sentence - - def span_to_special_tag(self, span): - - return self.replace_tag - - def find_certain_span(self, sentence): - spans = [] - for match in re.finditer(self.pattern, sentence): - spans.append(match.span()) - return spans - - -class AlphaSpanConverter(SpanConverter): - def __init__(self): - replace_tag = '' - # 理想状态下仅处理纯为字母的情况, 但不处理<[a-zA-Z]+>(因为这应该是特殊的tag). - pattern = '[a-zA-Z]+(?=[\u4e00-\u9fff ,%.!<\\-"])' - - super(AlphaSpanConverter, self).__init__(replace_tag, pattern) - - -class DigitSpanConverter(SpanConverter): - def __init__(self): - replace_tag = '' - pattern = '\d[\d\\.]*(?=[\u4e00-\u9fff ,%.!<-])' - - super(DigitSpanConverter, self).__init__(replace_tag, pattern) - - def span_to_special_tag(self, span): - # return self.special_tag - if span[0] == '0' and len(span) > 2: - return '' - decimal_point_count = 0 # one might have more than one decimal pointers - for idx, char in enumerate(span): - if char == '.' or char == '﹒' or char == '·': - decimal_point_count += 1 - if span[-1] == '.' or span[-1] == '﹒' or span[-1] == '·': - # last digit being decimal point means this is not a number - if decimal_point_count == 1: - return span - else: - return '' - if decimal_point_count == 1: - return '' - elif decimal_point_count > 1: - return '' - else: - return '' - - -class TimeConverter(SpanConverter): - def __init__(self): - replace_tag = '' - pattern = '\d+[:：∶][\d:：∶]+(?=[\u4e00-\u9fff ,%.!<-])' - - super().__init__(replace_tag, pattern) - - -class MixNumAlphaConverter(SpanConverter): - def __init__(self): - replace_tag = '' - pattern = None - - super().__init__(replace_tag, pattern) - - def find_certain_span_and_replace(self, sentence): - replaced_sentence = '' - start = 0 - matching_flag = False - number_flag = False - alpha_flag = False - link_flag = False - slash_flag = False - bracket_flag = False - for idx in range(len(sentence)): - if re.match('[0-9a-zA-Z/\\(\\)\'′&\\-]', sentence[idx]): - if not matching_flag: - replaced_sentence += sentence[start:idx] - start = idx - if re.match('[0-9]', sentence[idx]): - number_flag = True - elif re.match('[\'′&\\-]', sentence[idx]): - link_flag = True - elif re.match('/', sentence[idx]): - slash_flag = True - elif re.match('[\\(\\)]', sentence[idx]): - bracket_flag = True - else: - alpha_flag = True - matching_flag = True - elif re.match('[\\.]', sentence[idx]): - pass - else: - if matching_flag: - if (number_flag and alpha_flag) or (link_flag and alpha_flag) \ - or (slash_flag and alpha_flag) or (link_flag and number_flag) \ - or (number_flag and bracket_flag) or (bracket_flag and alpha_flag): - span = sentence[start:idx] - start = idx - replaced_sentence += self.span_to_special_tag(span) - matching_flag = False - number_flag = False - alpha_flag = False - link_flag = False - slash_flag = False - bracket_flag = False - - replaced_sentence += sentence[start:] - return replaced_sentence - - def find_certain_span(self, sentence): - spans = [] - start = 0 - matching_flag = False - number_flag = False - alpha_flag = False - link_flag = False - slash_flag = False - bracket_flag = False - for idx in range(len(sentence)): - if re.match('[0-9a-zA-Z/\\(\\)\'′&\\-]', sentence[idx]): - if not matching_flag: - start = idx - if re.match('[0-9]', sentence[idx]): - number_flag = True - elif re.match('[\'′&\\-]', sentence[idx]): - link_flag = True - elif re.match('/', sentence[idx]): - slash_flag = True - elif re.match('[\\(\\)]', sentence[idx]): - bracket_flag = True - else: - alpha_flag = True - matching_flag = True - elif re.match('[\\.]', sentence[idx]): - pass - else: - if matching_flag: - if (number_flag and alpha_flag) or (link_flag and alpha_flag) \ - or (slash_flag and alpha_flag) or (link_flag and number_flag) \ - or (number_flag and bracket_flag) or (bracket_flag and alpha_flag): - spans.append((start, idx)) - start = idx - - matching_flag = False - number_flag = False - alpha_flag = False - link_flag = False - slash_flag = False - bracket_flag = False - - return spans - - -class EmailConverter(SpanConverter): - def __init__(self): - replaced_tag = "" - pattern = '[0-9a-zA-Z]+[@][.﹒0-9a-zA-Z@]+(?=[\u4e00-\u9fff ,%.!<\\-"$])' - - super(EmailConverter, self).__init__(replaced_tag, pattern) diff --git a/legacy/api/examples.py b/legacy/api/examples.py deleted file mode 100644 index c1b2e155..00000000 --- a/legacy/api/examples.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -api/example.py contains all API examples provided by fastNLP. -It is used as a tutorial for API or a test script since it is difficult to test APIs in travis. - -""" -from . import CWS, POS, Parser - -text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', - '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', - '那么这款无人机到底有多厉害？'] - - -def chinese_word_segmentation(): - cws = CWS(device='cpu') - print(cws.predict(text)) - - -def chinese_word_segmentation_test(): - cws = CWS(device='cpu') - print(cws.test("../../test/data_for_tests/zh_sample.conllx")) - - -def pos_tagging(): - # 输入已分词序列 - text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', - '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], - ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] - pos = POS(device='cpu') - print(pos.predict(text)) - - -def pos_tagging_test(): - pos = POS(device='cpu') - print(pos.test("../../test/data_for_tests/zh_sample.conllx")) - - -def syntactic_parsing(): - text = [['编者', '按：', '7月', '12日', '，', '英国', '航空', '航天', '系统', '公司', '公布', '了', '该', '公司', - '研制', '的', '第一款', '高科技', '隐形', '无人机', '雷电之神', '。'], - ['那么', '这', '款', '无人机', '到底', '有', '多', '厉害', '？']] - parser = Parser(device='cpu') - print(parser.predict(text)) - - -def syntactic_parsing_test(): - parser = Parser(device='cpu') - print(parser.test("../../test/data_for_tests/zh_sample.conllx")) - - -if __name__ == "__main__": - # chinese_word_segmentation() - # chinese_word_segmentation_test() - # pos_tagging() - # pos_tagging_test() - syntactic_parsing() - # syntactic_parsing_test() diff --git a/legacy/api/pipeline.py b/legacy/api/pipeline.py deleted file mode 100644 index 2cec16b3..00000000 --- a/legacy/api/pipeline.py +++ /dev/null @@ -1,33 +0,0 @@ -from ..api.processor import Processor - - -class Pipeline: - """ - Pipeline takes a DataSet object as input, runs multiple processors sequentially, and - outputs a DataSet object. - """ - - def __init__(self, processors=None): - self.pipeline = [] - if isinstance(processors, list): - for proc in processors: - assert isinstance(proc, Processor), "Must be a Processor, not {}.".format(type(proc)) - self.pipeline = processors - - def add_processor(self, processor): - assert isinstance(processor, Processor), "Must be a Processor, not {}.".format(type(processor)) - self.pipeline.append(processor) - - def process(self, dataset): - assert len(self.pipeline) != 0, "You need to add some processor first." - - for proc in self.pipeline: - dataset = proc(dataset) - - return dataset - - def __call__(self, *args, **kwargs): - return self.process(*args, **kwargs) - - def __getitem__(self, item): - return self.pipeline[item] diff --git a/legacy/api/processor.py b/legacy/api/processor.py deleted file mode 100644 index 4c442ed2..00000000 --- a/legacy/api/processor.py +++ /dev/null @@ -1,428 +0,0 @@ -import re -from collections import defaultdict - -import torch - -from fastNLP.core.batch import Batch -from fastNLP.core.dataset import DataSet -from fastNLP.core.sampler import SequentialSampler -from fastNLP.core.vocabulary import Vocabulary - - -class Processor(object): - def __init__(self, field_name, new_added_field_name): - """ - - :param field_name: 处理哪个field - :param new_added_field_name: 如果为None，则认为是field_name，即覆盖原有的field - """ - self.field_name = field_name - if new_added_field_name is None: - self.new_added_field_name = field_name - else: - self.new_added_field_name = new_added_field_name - - def process(self, *args, **kwargs): - raise NotImplementedError - - def __call__(self, *args, **kwargs): - return self.process(*args, **kwargs) - - -class FullSpaceToHalfSpaceProcessor(Processor): - """全角转半角，以字符为处理单元 - - """ - - def __init__(self, field_name, change_alpha=True, change_digit=True, change_punctuation=True, - change_space=True): - super(FullSpaceToHalfSpaceProcessor, self).__init__(field_name, None) - - self.change_alpha = change_alpha - self.change_digit = change_digit - self.change_punctuation = change_punctuation - self.change_space = change_space - - FH_SPACE = [(u"　", u" ")] - FH_NUM = [ - (u"０", u"0"), (u"１", u"1"), (u"２", u"2"), (u"３", u"3"), (u"４", u"4"), - (u"５", u"5"), (u"６", u"6"), (u"７", u"7"), (u"８", u"8"), (u"９", u"9")] - FH_ALPHA = [ - (u"ａ", u"a"), (u"ｂ", u"b"), (u"ｃ", u"c"), (u"ｄ", u"d"), (u"ｅ", u"e"), - (u"ｆ", u"f"), (u"ｇ", u"g"), (u"ｈ", u"h"), (u"ｉ", u"i"), (u"ｊ", u"j"), - (u"ｋ", u"k"), (u"ｌ", u"l"), (u"ｍ", u"m"), (u"ｎ", u"n"), (u"ｏ", u"o"), - (u"ｐ", u"p"), (u"ｑ", u"q"), (u"ｒ", u"r"), (u"ｓ", u"s"), (u"ｔ", u"t"), - (u"ｕ", u"u"), (u"ｖ", u"v"), (u"ｗ", u"w"), (u"ｘ", u"x"), (u"ｙ", u"y"), - (u"ｚ", u"z"), - (u"Ａ", u"A"), (u"Ｂ", u"B"), (u"Ｃ", u"C"), (u"Ｄ", u"D"), (u"Ｅ", u"E"), - (u"Ｆ", u"F"), (u"Ｇ", u"G"), (u"Ｈ", u"H"), (u"Ｉ", u"I"), (u"Ｊ", u"J"), - (u"Ｋ", u"K"), (u"Ｌ", u"L"), (u"Ｍ", u"M"), (u"Ｎ", u"N"), (u"Ｏ", u"O"), - (u"Ｐ", u"P"), (u"Ｑ", u"Q"), (u"Ｒ", u"R"), (u"Ｓ", u"S"), (u"Ｔ", u"T"), - (u"Ｕ", u"U"), (u"Ｖ", u"V"), (u"Ｗ", u"W"), (u"Ｘ", u"X"), (u"Ｙ", u"Y"), - (u"Ｚ", u"Z")] - # 谨慎使用标点符号转换, 因为"5．12特大地震"转换后可能就成了"5.12特大地震" - FH_PUNCTUATION = [ - (u'％', u'%'), (u'！', u'!'), (u'＂', u'\"'), (u'＇', u'\''), (u'＃', u'#'), - (u'￥', u'$'), (u'＆', u'&'), (u'（', u'('), (u'）', u')'), (u'＊', u'*'), - (u'＋', u'+'), (u'，', u','), (u'－', u'-'), (u'．', u'.'), (u'／', u'/'), - (u'：', u':'), (u'；', u';'), (u'＜', u'<'), (u'＝', u'='), (u'＞', u'>'), - (u'？', u'?'), (u'＠', u'@'), (u'［', u'['), (u'］', u']'), (u'＼', u'\\'), - (u'＾', u'^'), (u'＿', u'_'), (u'｀', u'`'), (u'～', u'~'), (u'｛', u'{'), - (u'｝', u'}'), (u'｜', u'|')] - FHs = [] - if self.change_alpha: - FHs = FH_ALPHA - if self.change_digit: - FHs += FH_NUM - if self.change_punctuation: - FHs += FH_PUNCTUATION - if self.change_space: - FHs += FH_SPACE - self.convert_map = {k: v for k, v in FHs} - - def process(self, dataset): - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - - def inner_proc(ins): - sentence = ins[self.field_name] - new_sentence = [""] * len(sentence) - for idx, char in enumerate(sentence): - if char in self.convert_map: - char = self.convert_map[char] - new_sentence[idx] = char - return "".join(new_sentence) - - dataset.apply(inner_proc, new_field_name=self.field_name) - return dataset - - -class PreAppendProcessor(Processor): - """ - 向某个field的起始增加data(应该为str类型)。该field需要为list类型。即新增的field为 - [data] + instance[field_name] - - """ - - def __init__(self, data, field_name, new_added_field_name=None): - super(PreAppendProcessor, self).__init__(field_name, new_added_field_name) - self.data = data - - def process(self, dataset): - dataset.apply(lambda ins: [self.data] + ins[self.field_name], new_field_name=self.new_added_field_name) - return dataset - - -class SliceProcessor(Processor): - """ - 从某个field中只取部分内容。等价于instance[field_name][start:end:step] - - """ - - def __init__(self, start, end, step, field_name, new_added_field_name=None): - super(SliceProcessor, self).__init__(field_name, new_added_field_name) - for o in (start, end, step): - assert isinstance(o, int) or o is None - self.slice = slice(start, end, step) - - def process(self, dataset): - dataset.apply(lambda ins: ins[self.field_name][self.slice], new_field_name=self.new_added_field_name) - return dataset - - -class Num2TagProcessor(Processor): - """ - 将一句话中的数字转换为某个tag。 - - """ - - def __init__(self, tag, field_name, new_added_field_name=None): - """ - - :param tag: str, 将数字转换为该tag - :param field_name: - :param new_added_field_name: - """ - super(Num2TagProcessor, self).__init__(field_name, new_added_field_name) - self.tag = tag - self.pattern = r'[-+]?([0-9]+[.]?[0-9]*)+[/eE]?[-+]?([0-9]+[.]?[0-9]*)' - - def process(self, dataset): - - def inner_proc(ins): - s = ins[self.field_name] - new_s = [None] * len(s) - for i, w in enumerate(s): - if re.search(self.pattern, w) is not None: - w = self.tag - new_s[i] = w - return new_s - - dataset.apply(inner_proc, new_field_name=self.new_added_field_name) - return dataset - - -class IndexerProcessor(Processor): - """ - 给定一个vocabulary , 将指定field转换为index形式。指定field应该是一维的list，比如 - ['我', '是', xxx] - """ - - def __init__(self, vocab, field_name, new_added_field_name, delete_old_field=False, is_input=True): - - assert isinstance(vocab, Vocabulary), "Only Vocabulary class is allowed, not {}.".format(type(vocab)) - - super(IndexerProcessor, self).__init__(field_name, new_added_field_name) - self.vocab = vocab - self.delete_old_field = delete_old_field - self.is_input = is_input - - def set_vocab(self, vocab): - assert isinstance(vocab, Vocabulary), "Only Vocabulary class is allowed, not {}.".format(type(vocab)) - - self.vocab = vocab - - def process(self, dataset): - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: [self.vocab.to_index(token) for token in ins[self.field_name]], - new_field_name=self.new_added_field_name) - if self.is_input: - dataset.set_input(self.new_added_field_name) - - if self.delete_old_field: - dataset.delete_field(self.field_name) - - return dataset - - -class VocabProcessor(Processor): - """ - 传入若干个DataSet以建立vocabulary。 - - """ - - def __init__(self, field_name, min_freq=1, max_size=None): - super(VocabProcessor, self).__init__(field_name, None) - self.vocab = Vocabulary(min_freq=min_freq, max_size=max_size) - - def process(self, *datasets): - for dataset in datasets: - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) - - def get_vocab(self): - self.vocab.build_vocab() - return self.vocab - - -class SeqLenProcessor(Processor): - """ - 根据某个field新增一个sequence length的field。取该field的第一维 - - """ - - def __init__(self, field_name, new_added_field_name='seq_lens', is_input=True): - super(SeqLenProcessor, self).__init__(field_name, new_added_field_name) - self.is_input = is_input - - def process(self, dataset): - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: len(ins[self.field_name]), new_field_name=self.new_added_field_name) - if self.is_input: - dataset.set_input(self.new_added_field_name) - return dataset - - -from fastNLP.core.utils import _build_args - - -class ModelProcessor(Processor): - def __init__(self, model, seq_len_field_name='seq_lens', batch_size=32): - """ - 传入一个model，在process()时传入一个dataset，该processor会通过Batch将DataSet的内容输出给model.predict或者model.forward. - model输出的内容会被增加到dataset中，field_name由model输出决定。如果生成的内容维度不是(Batch_size, )与 - (Batch_size, 1)，则使用seqence length这个field进行unpad - TODO 这个类需要删除对seq_lens的依赖。 - - :param seq_len_field_name: - :param batch_size: - """ - super(ModelProcessor, self).__init__(None, None) - self.batch_size = batch_size - self.seq_len_field_name = seq_len_field_name - self.model = model - - def process(self, dataset): - self.model.eval() - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - data_iterator = Batch(dataset, batch_size=self.batch_size, sampler=SequentialSampler()) - - batch_output = defaultdict(list) - predict_func = self.model.forward - with torch.no_grad(): - for batch_x, _ in data_iterator: - refined_batch_x = _build_args(predict_func, **batch_x) - prediction = predict_func(**refined_batch_x) - seq_lens = batch_x[self.seq_len_field_name].tolist() - - for key, value in prediction.items(): - tmp_batch = [] - value = value.cpu().numpy() - if len(value.shape) == 1 or (len(value.shape) == 2 and value.shape[1] == 1): - batch_output[key].extend(value.tolist()) - else: - for idx, seq_len in enumerate(seq_lens): - tmp_batch.append(value[idx, :seq_len]) - batch_output[key].extend(tmp_batch) - if not self.seq_len_field_name in prediction: - batch_output[self.seq_len_field_name].extend(seq_lens) - - # TODO 当前的实现会导致之后的processor需要知道model输出的output的key是什么 - for field_name, fields in batch_output.items(): - dataset.add_field(field_name, fields, is_input=True, is_target=False) - - return dataset - - def set_model(self, model): - self.model = model - - def set_model_device(self, device): - device = torch.device(device) - self.model.to(device) - - -class Index2WordProcessor(Processor): - """ - 将DataSet中某个为index的field根据vocab转换为str - - """ - - def __init__(self, vocab, field_name, new_added_field_name): - super(Index2WordProcessor, self).__init__(field_name, new_added_field_name) - self.vocab = vocab - - def process(self, dataset): - dataset.apply(lambda ins: [self.vocab.to_word(w) for w in ins[self.field_name]], - new_field_name=self.new_added_field_name) - return dataset - - -class SetTargetProcessor(Processor): - def __init__(self, *fields, flag=True): - super(SetTargetProcessor, self).__init__(None, None) - self.fields = fields - self.flag = flag - - def process(self, dataset): - dataset.set_target(*self.fields, flag=self.flag) - return dataset - - -class SetInputProcessor(Processor): - def __init__(self, *fields, flag=True): - super(SetInputProcessor, self).__init__(None, None) - self.fields = fields - self.flag = flag - - def process(self, dataset): - dataset.set_input(*self.fields, flag=self.flag) - return dataset - - -class VocabIndexerProcessor(Processor): - """ - 根据DataSet创建Vocabulary，并将其用数字index。新生成的index的field会被放在new_added_filed_name, 如果没有提供 - new_added_field_name, 则覆盖原有的field_name. - - """ - - def __init__(self, field_name, new_added_filed_name=None, min_freq=1, max_size=None, - verbose=0, is_input=True): - """ - - :param field_name: 从哪个field_name创建词表，以及对哪个field_name进行index操作 - :param new_added_filed_name: index时，生成的index field的名称，如果不传入，则覆盖field_name. - :param min_freq: 创建的Vocabulary允许的单词最少出现次数. - :param max_size: 创建的Vocabulary允许的最大的单词数量 - :param verbose: 0, 不输出任何信息；1，输出信息 - :param bool is_input: - """ - super(VocabIndexerProcessor, self).__init__(field_name, new_added_filed_name) - self.min_freq = min_freq - self.max_size = max_size - - self.verbose = verbose - self.is_input = is_input - - def construct_vocab(self, *datasets): - """ - 使用传入的DataSet创建vocabulary - - :param datasets: DataSet类型的数据，用于构建vocabulary - :return: - """ - self.vocab = Vocabulary(min_freq=self.min_freq, max_size=self.max_size) - for dataset in datasets: - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) - self.vocab.build_vocab() - if self.verbose: - print("Vocabulary Constructed, has {} items.".format(len(self.vocab))) - - def process(self, *datasets, only_index_dataset=None): - """ - 若还未建立Vocabulary，则使用dataset中的DataSet建立vocabulary；若已经有了vocabulary则使用已有的vocabulary。得到vocabulary - 后，则会index datasets与only_index_dataset。 - - :param datasets: DataSet类型的数据 - :param only_index_dataset: DataSet, or list of DataSet. 该参数中的内容只会被用于index，不会被用于生成vocabulary。 - :return: - """ - if len(datasets) == 0 and not hasattr(self, 'vocab'): - raise RuntimeError("You have to construct vocabulary first. Or you have to pass datasets to construct it.") - if not hasattr(self, 'vocab'): - self.construct_vocab(*datasets) - else: - if self.verbose: - print("Using constructed vocabulary with {} items.".format(len(self.vocab))) - to_index_datasets = [] - if len(datasets) != 0: - for dataset in datasets: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - to_index_datasets.append(dataset) - - if not (only_index_dataset is None): - if isinstance(only_index_dataset, list): - for dataset in only_index_dataset: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - to_index_datasets.append(dataset) - elif isinstance(only_index_dataset, DataSet): - to_index_datasets.append(only_index_dataset) - else: - raise TypeError('Only DataSet or list of DataSet is allowed, not {}.'.format(type(only_index_dataset))) - - for dataset in to_index_datasets: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: [self.vocab.to_index(token) for token in ins[self.field_name]], - new_field_name=self.new_added_field_name, is_input=self.is_input) - # 只返回一个，infer时为了跟其他processor保持一致 - if len(to_index_datasets) == 1: - return to_index_datasets[0] - - def set_vocab(self, vocab): - assert isinstance(vocab, Vocabulary), "Only fastNLP.core.Vocabulary is allowed, not {}.".format(type(vocab)) - self.vocab = vocab - - def delete_vocab(self): - del self.vocab - - def get_vocab_size(self): - return len(self.vocab) - - def set_verbose(self, verbose): - """ - 设置processor verbose状态。 - - :param verbose: int, 0，不输出任何信息；1，输出vocab 信息。 - :return: - """ - self.verbose = verbose diff --git a/legacy/api/utils.py b/legacy/api/utils.py deleted file mode 100644 index 184e5fe6..00000000 --- a/legacy/api/utils.py +++ /dev/null @@ -1,134 +0,0 @@ -import hashlib -import os -import re -import shutil -import sys -import tempfile - -import torch - -try: - from requests.utils import urlparse - from requests import get as urlopen - requests_available = True -except ImportError: - requests_available = False - if sys.version_info[0] == 2: - from urlparse import urlparse # noqa f811 - from urllib2 import urlopen # noqa f811 - else: - from urllib.request import urlopen - from urllib.parse import urlparse -try: - from tqdm.auto import tqdm -except: - from fastNLP.core.utils import _pseudo_tqdm as tqdm - -# matches bfd8deac from resnet18-bfd8deac.pth -HASH_REGEX = re.compile(r'-([a-f0-9]*)\.') - - -def load_url(url, model_dir=None, map_location=None, progress=True): - r"""Loads the Torch serialized object at the given URL. - - If the object is already present in `model_dir`, it's deserialized and - returned. The filename part of the URL should follow the naming convention - ``filename-.ext`` where ```` is the first eight or more - digits of the SHA256 hash of the contents of the file. The hash is used to - ensure unique names and to verify the contents of the file. - - The default value of `model_dir` is ``$TORCH_HOME/models`` where - ``$TORCH_HOME`` defaults to ``~/.torch``. The default directory can be - overridden with the ``$TORCH_MODEL_ZOO`` environment variable. - - Args: - url (string): URL of the object to download - model_dir (string, optional): directory in which to save the object - map_location (optional): a function or a dict specifying how to remap storage locations (see torch.load) - progress (bool, optional): whether or not to display a progress bar to stderr - - Example: - # >>> state_dict = model_zoo.load_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth') - - """ - if model_dir is None: - torch_home = os.path.expanduser(os.getenv('fastNLP_HOME', '~/.fastNLP')) - model_dir = os.getenv('fastNLP_MODEL_ZOO', os.path.join(torch_home, 'models')) - if not os.path.exists(model_dir): - os.makedirs(model_dir) - parts = urlparse(url) - filename = os.path.basename(parts.path) - cached_file = os.path.join(model_dir, filename) - if not os.path.exists(cached_file): - sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file)) - # hash_prefix = HASH_REGEX.search(filename).group(1) - _download_url_to_file(url, cached_file, hash_prefix=None, progress=progress) - return torch.load(cached_file, map_location=map_location) - - -def _download_url_to_file(url, dst, hash_prefix, progress): - if requests_available: - u = urlopen(url, stream=True) - file_size = int(u.headers["Content-Length"]) - u = u.raw - else: - u = urlopen(url) - meta = u.info() - if hasattr(meta, 'getheaders'): - file_size = int(meta.getheaders("Content-Length")[0]) - else: - file_size = int(meta.get_all("Content-Length")[0]) - - f = tempfile.NamedTemporaryFile(delete=False) - try: - if hash_prefix is not None: - sha256 = hashlib.sha256() - with tqdm(total=file_size, disable=not progress) as pbar: - while True: - buffer = u.read(8192) - if len(buffer) == 0: - break - f.write(buffer) - if hash_prefix is not None: - sha256.update(buffer) - pbar.update(len(buffer)) - - f.close() - if hash_prefix is not None: - digest = sha256.hexdigest() - if digest[:len(hash_prefix)] != hash_prefix: - raise RuntimeError('invalid hash value (expected "{}", got "{}")' - .format(hash_prefix, digest)) - shutil.move(f.name, dst) - finally: - f.close() - if os.path.exists(f.name): - os.remove(f.name) - - -if tqdm is None: - # fake tqdm if it's not installed - class tqdm(object): - - def __init__(self, total, disable=False): - self.total = total - self.disable = disable - self.n = 0 - - def update(self, n): - if self.disable: - return - - self.n += n - sys.stderr.write("\r{0:.1f}%".format(100 * self.n / float(self.total))) - sys.stderr.flush() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.disable: - return - - sys.stderr.write('\n') - diff --git a/legacy/automl/__init__.py b/legacy/automl/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/legacy/automl/enas_controller.py b/legacy/automl/enas_controller.py deleted file mode 100644 index 6ddbb211..00000000 --- a/legacy/automl/enas_controller.py +++ /dev/null @@ -1,223 +0,0 @@ -# Code Modified from https://github.com/carpedm20/ENAS-pytorch -"""A module with NAS controller-related code.""" -import collections -import os - -import torch -import torch.nn.functional as F - -import fastNLP.automl.enas_utils as utils -from fastNLP.automl.enas_utils import Node - - -def _construct_dags(prev_nodes, activations, func_names, num_blocks): - """Constructs a set of DAGs based on the actions, i.e., previous nodes and - activation functions, sampled from the controller/policy pi. - - Args: - prev_nodes: Previous node actions from the policy. - activations: Activations sampled from the policy. - func_names: Mapping from activation function names to functions. - num_blocks: Number of blocks in the target RNN cell. - - Returns: - A list of DAGs defined by the inputs. - - RNN cell DAGs are represented in the following way: - - 1. Each element (node) in a DAG is a list of `Node`s. - - 2. The `Node`s in the list dag[i] correspond to the subsequent nodes - that take the output from node i as their own input. - - 3. dag[-1] is the node that takes input from x^{(t)} and h^{(t - 1)}. - dag[-1] always feeds dag[0]. - dag[-1] acts as if `w_xc`, `w_hc`, `w_xh` and `w_hh` are its - weights. - - 4. dag[N - 1] is the node that produces the hidden state passed to - the next timestep. dag[N - 1] is also always a leaf node, and therefore - is always averaged with the other leaf nodes and fed to the output - decoder. - """ - dags = [] - for nodes, func_ids in zip(prev_nodes, activations): - dag = collections.defaultdict(list) - - # add first node - dag[-1] = [Node(0, func_names[func_ids[0]])] - dag[-2] = [Node(0, func_names[func_ids[0]])] - - # add following nodes - for jdx, (idx, func_id) in enumerate(zip(nodes, func_ids[1:])): - dag[utils.to_item(idx)].append(Node(jdx + 1, func_names[func_id])) - - leaf_nodes = set(range(num_blocks)) - dag.keys() - - # merge with avg - for idx in leaf_nodes: - dag[idx] = [Node(num_blocks, 'avg')] - - # This is actually y^{(t)}. h^{(t)} is node N - 1 in - # the graph, where N Is the number of nodes. I.e., h^{(t)} takes - # only one other node as its input. - # last h[t] node - last_node = Node(num_blocks + 1, 'h[t]') - dag[num_blocks] = [last_node] - dags.append(dag) - - return dags - - -class Controller(torch.nn.Module): - """Based on - https://github.com/pytorch/examples/blob/master/word_language_model/model.py - - RL controllers do not necessarily have much to do with - language models. - - Base the controller RNN on the GRU from: - https://github.com/ikostrikov/pytorch-a2c-ppo-acktr/blob/master/model.py - """ - def __init__(self, num_blocks=4, controller_hid=100, cuda=False): - torch.nn.Module.__init__(self) - - # `num_tokens` here is just the activation function - # for every even step, - self.shared_rnn_activations = ['tanh', 'ReLU', 'identity', 'sigmoid'] - self.num_tokens = [len(self.shared_rnn_activations)] - self.controller_hid = controller_hid - self.use_cuda = cuda - self.num_blocks = num_blocks - for idx in range(num_blocks): - self.num_tokens += [idx + 1, len(self.shared_rnn_activations)] - self.func_names = self.shared_rnn_activations - - num_total_tokens = sum(self.num_tokens) - - self.encoder = torch.nn.Embedding(num_total_tokens, - controller_hid) - self.lstm = torch.nn.LSTMCell(controller_hid, controller_hid) - - # Perhaps these weights in the decoder should be - # shared? At least for the activation functions, which all have the - # same size. - self.decoders = [] - for idx, size in enumerate(self.num_tokens): - decoder = torch.nn.Linear(controller_hid, size) - self.decoders.append(decoder) - - self._decoders = torch.nn.ModuleList(self.decoders) - - self.reset_parameters() - self.static_init_hidden = utils.keydefaultdict(self.init_hidden) - - def _get_default_hidden(key): - return utils.get_variable( - torch.zeros(key, self.controller_hid), - self.use_cuda, - requires_grad=False) - - self.static_inputs = utils.keydefaultdict(_get_default_hidden) - - def reset_parameters(self): - init_range = 0.1 - for param in self.parameters(): - param.data.uniform_(-init_range, init_range) - for decoder in self.decoders: - decoder.bias.data.fill_(0) - - def forward(self, # pylint:disable=arguments-differ - inputs, - hidden, - block_idx, - is_embed): - if not is_embed: - embed = self.encoder(inputs) - else: - embed = inputs - - hx, cx = self.lstm(embed, hidden) - logits = self.decoders[block_idx](hx) - - logits /= 5.0 - - # # exploration - # if self.args.mode == 'train': - # logits = (2.5 * F.tanh(logits)) - - return logits, (hx, cx) - - def sample(self, batch_size=1, with_details=False, save_dir=None): - """Samples a set of `args.num_blocks` many computational nodes from the - controller, where each node is made up of an activation function, and - each node except the last also includes a previous node. - """ - if batch_size < 1: - raise Exception(f'Wrong batch_size: {batch_size} < 1') - - # [B, L, H] - inputs = self.static_inputs[batch_size] - hidden = self.static_init_hidden[batch_size] - - activations = [] - entropies = [] - log_probs = [] - prev_nodes = [] - # The RNN controller alternately outputs an activation, - # followed by a previous node, for each block except the last one, - # which only gets an activation function. The last node is the output - # node, and its previous node is the average of all leaf nodes. - for block_idx in range(2*(self.num_blocks - 1) + 1): - logits, hidden = self.forward(inputs, - hidden, - block_idx, - is_embed=(block_idx == 0)) - - probs = F.softmax(logits, dim=-1) - log_prob = F.log_softmax(logits, dim=-1) - # .mean() for entropy? - entropy = -(log_prob * probs).sum(1, keepdim=False) - - action = probs.multinomial(num_samples=1).data - selected_log_prob = log_prob.gather( - 1, utils.get_variable(action, requires_grad=False)) - - # why the [:, 0] here? Should it be .squeeze(), or - # .view()? Same below with `action`. - entropies.append(entropy) - log_probs.append(selected_log_prob[:, 0]) - - # 0: function, 1: previous node - mode = block_idx % 2 - inputs = utils.get_variable( - action[:, 0] + sum(self.num_tokens[:mode]), - requires_grad=False) - - if mode == 0: - activations.append(action[:, 0]) - elif mode == 1: - prev_nodes.append(action[:, 0]) - - prev_nodes = torch.stack(prev_nodes).transpose(0, 1) - activations = torch.stack(activations).transpose(0, 1) - - dags = _construct_dags(prev_nodes, - activations, - self.func_names, - self.num_blocks) - - if save_dir is not None: - for idx, dag in enumerate(dags): - utils.draw_network(dag, - os.path.join(save_dir, f'graph{idx}.png')) - - if with_details: - return dags, torch.cat(log_probs), torch.cat(entropies) - - return dags - - def init_hidden(self, batch_size): - zeros = torch.zeros(batch_size, self.controller_hid) - return (utils.get_variable(zeros, self.use_cuda, requires_grad=False), - utils.get_variable(zeros.clone(), self.use_cuda, requires_grad=False)) diff --git a/legacy/automl/enas_model.py b/legacy/automl/enas_model.py deleted file mode 100644 index 4f9fb449..00000000 --- a/legacy/automl/enas_model.py +++ /dev/null @@ -1,388 +0,0 @@ -# Code Modified from https://github.com/carpedm20/ENAS-pytorch - -"""Module containing the shared RNN model.""" -import collections - -import numpy as np -import torch -import torch.nn.functional as F -from torch import nn -from torch.autograd import Variable - -import fastNLP.automl.enas_utils as utils -from fastNLP.models.base_model import BaseModel - - -def _get_dropped_weights(w_raw, dropout_p, is_training): - """Drops out weights to implement DropConnect. - - Args: - w_raw: Full, pre-dropout, weights to be dropped out. - dropout_p: Proportion of weights to drop out. - is_training: True iff _shared_ model is training. - - Returns: - The dropped weights. - - Why does torch.nn.functional.dropout() return: - 1. `torch.autograd.Variable()` on the training loop - 2. `torch.nn.Parameter()` on the controller or eval loop, when - training = False... - - Even though the call to `_setweights` in the Smerity repo's - `weight_drop.py` does not have this behaviour, and `F.dropout` always - returns `torch.autograd.Variable` there, even when `training=False`? - - The above TODO is the reason for the hacky check for `torch.nn.Parameter`. - """ - dropped_w = F.dropout(w_raw, p=dropout_p, training=is_training) - - if isinstance(dropped_w, torch.nn.Parameter): - dropped_w = dropped_w.clone() - - return dropped_w - -class EmbeddingDropout(torch.nn.Embedding): - """Class for dropping out embeddings by zero'ing out parameters in the - embedding matrix. - - This is equivalent to dropping out particular words, e.g., in the sentence - 'the quick brown fox jumps over the lazy dog', dropping out 'the' would - lead to the sentence '### quick brown fox jumps over ### lazy dog' (in the - embedding vector space). - - See 'A Theoretically Grounded Application of Dropout in Recurrent Neural - Networks', (Gal and Ghahramani, 2016). - """ - def __init__(self, - num_embeddings, - embedding_dim, - max_norm=None, - norm_type=2, - scale_grad_by_freq=False, - sparse=False, - dropout=0.1, - scale=None): - """Embedding constructor. - - Args: - dropout: Dropout probability. - scale: Used to scale parameters of embedding weight matrix that are - not dropped out. Note that this is _in addition_ to the - `1/(1 - dropout)` scaling. - - See `torch.nn.Embedding` for remaining arguments. - """ - torch.nn.Embedding.__init__(self, - num_embeddings=num_embeddings, - embedding_dim=embedding_dim, - max_norm=max_norm, - norm_type=norm_type, - scale_grad_by_freq=scale_grad_by_freq, - sparse=sparse) - self.dropout = dropout - assert (dropout >= 0.0) and (dropout < 1.0), ('Dropout must be >= 0.0 ' - 'and < 1.0') - self.scale = scale - - def forward(self, inputs): # pylint:disable=arguments-differ - """Embeds `inputs` with the dropped out embedding weight matrix.""" - if self.training: - dropout = self.dropout - else: - dropout = 0 - - if dropout: - mask = self.weight.data.new(self.weight.size(0), 1) - mask.bernoulli_(1 - dropout) - mask = mask.expand_as(self.weight) - mask = mask / (1 - dropout) - masked_weight = self.weight * Variable(mask) - else: - masked_weight = self.weight - if self.scale and self.scale != 1: - masked_weight = masked_weight * self.scale - - return F.embedding(inputs, - masked_weight, - max_norm=self.max_norm, - norm_type=self.norm_type, - scale_grad_by_freq=self.scale_grad_by_freq, - sparse=self.sparse) - - -class LockedDropout(nn.Module): - # code from https://github.com/salesforce/awd-lstm-lm/blob/master/locked_dropout.py - def __init__(self): - super().__init__() - - def forward(self, x, dropout=0.5): - if not self.training or not dropout: - return x - m = x.data.new(1, x.size(1), x.size(2)).bernoulli_(1 - dropout) - mask = Variable(m, requires_grad=False) / (1 - dropout) - mask = mask.expand_as(x) - return mask * x - - -class ENASModel(BaseModel): - """Shared RNN model.""" - def __init__(self, embed_num, num_classes, num_blocks=4, cuda=False, shared_hid=1000, shared_embed=1000): - super(ENASModel, self).__init__() - - self.use_cuda = cuda - - self.shared_hid = shared_hid - self.num_blocks = num_blocks - self.decoder = nn.Linear(self.shared_hid, num_classes) - self.encoder = EmbeddingDropout(embed_num, - shared_embed, - dropout=0.1) - self.lockdrop = LockedDropout() - self.dag = None - - # Tie weights - # self.decoder.weight = self.encoder.weight - - # Since W^{x, c} and W^{h, c} are always summed, there - # is no point duplicating their bias offset parameter. Likewise for - # W^{x, h} and W^{h, h}. - self.w_xc = nn.Linear(shared_embed, self.shared_hid) - self.w_xh = nn.Linear(shared_embed, self.shared_hid) - - # The raw weights are stored here because the hidden-to-hidden weights - # are weight dropped on the forward pass. - self.w_hc_raw = torch.nn.Parameter( - torch.Tensor(self.shared_hid, self.shared_hid)) - self.w_hh_raw = torch.nn.Parameter( - torch.Tensor(self.shared_hid, self.shared_hid)) - self.w_hc = None - self.w_hh = None - - self.w_h = collections.defaultdict(dict) - self.w_c = collections.defaultdict(dict) - - for idx in range(self.num_blocks): - for jdx in range(idx + 1, self.num_blocks): - self.w_h[idx][jdx] = nn.Linear(self.shared_hid, - self.shared_hid, - bias=False) - self.w_c[idx][jdx] = nn.Linear(self.shared_hid, - self.shared_hid, - bias=False) - - self._w_h = nn.ModuleList([self.w_h[idx][jdx] - for idx in self.w_h - for jdx in self.w_h[idx]]) - self._w_c = nn.ModuleList([self.w_c[idx][jdx] - for idx in self.w_c - for jdx in self.w_c[idx]]) - - self.batch_norm = None - # if args.mode == 'train': - # self.batch_norm = nn.BatchNorm1d(self.shared_hid) - # else: - # self.batch_norm = None - - self.reset_parameters() - self.static_init_hidden = utils.keydefaultdict(self.init_hidden) - - def setDAG(self, dag): - if self.dag is None: - self.dag = dag - - def forward(self, word_seq, hidden=None): - inputs = torch.transpose(word_seq, 0, 1) - - time_steps = inputs.size(0) - batch_size = inputs.size(1) - - - self.w_hh = _get_dropped_weights(self.w_hh_raw, - 0.5, - self.training) - self.w_hc = _get_dropped_weights(self.w_hc_raw, - 0.5, - self.training) - - # hidden = self.static_init_hidden[batch_size] if hidden is None else hidden - hidden = self.static_init_hidden[batch_size] - - embed = self.encoder(inputs) - - embed = self.lockdrop(embed, 0.65 if self.training else 0) - - # The norm of hidden states are clipped here because - # otherwise ENAS is especially prone to exploding activations on the - # forward pass. This could probably be fixed in a more elegant way, but - # it might be exposing a weakness in the ENAS algorithm as currently - # proposed. - # - # For more details, see - # https://github.com/carpedm20/ENAS-pytorch/issues/6 - clipped_num = 0 - max_clipped_norm = 0 - h1tohT = [] - logits = [] - for step in range(time_steps): - x_t = embed[step] - logit, hidden = self.cell(x_t, hidden, self.dag) - - hidden_norms = hidden.norm(dim=-1) - max_norm = 25.0 - if hidden_norms.data.max() > max_norm: - # Just directly use the torch slice operations - # in PyTorch v0.4. - # - # This workaround for PyTorch v0.3.1 does everything in numpy, - # because the PyTorch slicing and slice assignment is too - # flaky. - hidden_norms = hidden_norms.data.cpu().numpy() - - clipped_num += 1 - if hidden_norms.max() > max_clipped_norm: - max_clipped_norm = hidden_norms.max() - - clip_select = hidden_norms > max_norm - clip_norms = hidden_norms[clip_select] - - mask = np.ones(hidden.size()) - normalizer = max_norm/clip_norms - normalizer = normalizer[:, np.newaxis] - - mask[clip_select] = normalizer - - if self.use_cuda: - hidden *= torch.autograd.Variable( - torch.FloatTensor(mask).cuda(), requires_grad=False) - else: - hidden *= torch.autograd.Variable( - torch.FloatTensor(mask), requires_grad=False) - logits.append(logit) - h1tohT.append(hidden) - - h1tohT = torch.stack(h1tohT) - output = torch.stack(logits) - raw_output = output - - output = self.lockdrop(output, 0.4 if self.training else 0) - - #Pooling - output = torch.mean(output, 0) - - decoded = self.decoder(output) - - extra_out = {'dropped': decoded, - 'hiddens': h1tohT, - 'raw': raw_output} - return {'pred': decoded, 'hidden': hidden, 'extra_out': extra_out} - - def cell(self, x, h_prev, dag): - """Computes a single pass through the discovered RNN cell.""" - c = {} - h = {} - f = {} - - f[0] = self.get_f(dag[-1][0].name) - c[0] = torch.sigmoid(self.w_xc(x) + F.linear(h_prev, self.w_hc, None)) - h[0] = (c[0]*f[0](self.w_xh(x) + F.linear(h_prev, self.w_hh, None)) + - (1 - c[0])*h_prev) - - leaf_node_ids = [] - q = collections.deque() - q.append(0) - - # Computes connections from the parent nodes `node_id` - # to their child nodes `next_id` recursively, skipping leaf nodes. A - # leaf node is a node whose id == `self.num_blocks`. - # - # Connections between parent i and child j should be computed as - # h_j = c_j*f_{ij}{(W^h_{ij}*h_i)} + (1 - c_j)*h_i, - # where c_j = \sigmoid{(W^c_{ij}*h_i)} - # - # See Training details from Section 3.1 of the paper. - # - # The following algorithm does a breadth-first (since `q.popleft()` is - # used) search over the nodes and computes all the hidden states. - while True: - if len(q) == 0: - break - - node_id = q.popleft() - nodes = dag[node_id] - - for next_node in nodes: - next_id = next_node.id - if next_id == self.num_blocks: - leaf_node_ids.append(node_id) - assert len(nodes) == 1, ('parent of leaf node should have ' - 'only one child') - continue - - w_h = self.w_h[node_id][next_id] - w_c = self.w_c[node_id][next_id] - - f[next_id] = self.get_f(next_node.name) - c[next_id] = torch.sigmoid(w_c(h[node_id])) - h[next_id] = (c[next_id]*f[next_id](w_h(h[node_id])) + - (1 - c[next_id])*h[node_id]) - - q.append(next_id) - - # Instead of averaging loose ends, perhaps there should - # be a set of separate unshared weights for each "loose" connection - # between each node in a cell and the output. - # - # As it stands, all weights W^h_{ij} are doing double duty by - # connecting both from i to j, as well as from i to the output. - - # average all the loose ends - leaf_nodes = [h[node_id] for node_id in leaf_node_ids] - output = torch.mean(torch.stack(leaf_nodes, 2), -1) - - # stabilizing the Updates of omega - if self.batch_norm is not None: - output = self.batch_norm(output) - - return output, h[self.num_blocks - 1] - - def init_hidden(self, batch_size): - zeros = torch.zeros(batch_size, self.shared_hid) - return utils.get_variable(zeros, self.use_cuda, requires_grad=False) - - def get_f(self, name): - name = name.lower() - if name == 'relu': - f = torch.relu - elif name == 'tanh': - f = torch.tanh - elif name == 'identity': - f = lambda x: x - elif name == 'sigmoid': - f = torch.sigmoid - return f - - - @property - def num_parameters(self): - def size(p): - return np.prod(p.size()) - return sum([size(param) for param in self.parameters()]) - - - def reset_parameters(self): - init_range = 0.025 - # init_range = 0.025 if self.args.mode == 'train' else 0.04 - for param in self.parameters(): - param.data.uniform_(-init_range, init_range) - self.decoder.bias.data.fill_(0) - - def predict(self, word_seq): - """ - - :param word_seq: torch.LongTensor, [batch_size, seq_len] - :return predict: dict of torch.LongTensor, [batch_size, seq_len] - """ - output = self(word_seq) - _, predict = output['pred'].max(dim=1) - return {'pred': predict} diff --git a/legacy/automl/enas_trainer.py b/legacy/automl/enas_trainer.py deleted file mode 100644 index e3524aa9..00000000 --- a/legacy/automl/enas_trainer.py +++ /dev/null @@ -1,383 +0,0 @@ -# Code Modified from https://github.com/carpedm20/ENAS-pytorch - -import math -import time -from datetime import datetime -from datetime import timedelta - -import numpy as np -import torch - -try: - from tqdm.auto import tqdm -except: - from fastNLP.core.utils import _pseudo_tqdm as tqdm - -from fastNLP.core.batch import Batch -from fastNLP.core.callback import CallbackException -from fastNLP.core.dataset import DataSet -from fastNLP.core.utils import _move_dict_value_to_device -import fastNLP -from . import enas_utils as utils -from fastNLP.core.utils import _build_args - -from torch.optim import Adam - - -def _get_no_grad_ctx_mgr(): - """Returns a the `torch.no_grad` context manager for PyTorch version >= - 0.4, or a no-op context manager otherwise. - """ - return torch.no_grad() - - -class ENASTrainer(fastNLP.Trainer): - """A class to wrap training code.""" - def __init__(self, train_data, model, controller, **kwargs): - """Constructor for training algorithm. - :param DataSet train_data: the training data - :param torch.nn.modules.module model: a PyTorch model - :param torch.nn.modules.module controller: a PyTorch model - """ - self.final_epochs = kwargs['final_epochs'] - kwargs.pop('final_epochs') - super(ENASTrainer, self).__init__(train_data, model, **kwargs) - self.controller_step = 0 - self.shared_step = 0 - self.max_length = 35 - - self.shared = model - self.controller = controller - - self.shared_optim = Adam( - self.shared.parameters(), - lr=20.0, - weight_decay=1e-7) - - self.controller_optim = Adam( - self.controller.parameters(), - lr=3.5e-4) - - def train(self, load_best_model=True): - """ - :param bool load_best_model: 该参数只有在初始化提供了dev_data的情况下有效，如果True, trainer将在返回之前重新加载dev表现 - 最好的模型参数。 - :return results: 返回一个字典类型的数据, - 内含以下内容:: - - seconds: float, 表示训练时长 - 以下三个内容只有在提供了dev_data的情况下会有。 - best_eval: Dict of Dict, 表示evaluation的结果 - best_epoch: int，在第几个epoch取得的最佳值 - best_step: int, 在第几个step(batch)更新取得的最佳值 - - """ - results = {} - if self.n_epochs <= 0: - print(f"training epoch is {self.n_epochs}, nothing was done.") - results['seconds'] = 0. - return results - try: - if torch.cuda.is_available() and self.use_cuda: - self.model = self.model.cuda() - self._model_device = self.model.parameters().__next__().device - self._mode(self.model, is_test=False) - - self.start_time = str(datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) - start_time = time.time() - print("training epochs started " + self.start_time, flush=True) - - try: - self.callback_manager.on_train_begin() - self._train() - self.callback_manager.on_train_end(self.model) - except (CallbackException, KeyboardInterrupt) as e: - self.callback_manager.on_exception(e, self.model) - - if self.dev_data is not None: - print("\nIn Epoch:{}/Step:{}, got best dev performance:".format(self.best_dev_epoch, self.best_dev_step) + - self.tester._format_eval_results(self.best_dev_perf),) - results['best_eval'] = self.best_dev_perf - results['best_epoch'] = self.best_dev_epoch - results['best_step'] = self.best_dev_step - if load_best_model: - model_name = "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time]) - load_succeed = self._load_model(self.model, model_name) - if load_succeed: - print("Reloaded the best model.") - else: - print("Fail to reload best model.") - finally: - pass - results['seconds'] = round(time.time() - start_time, 2) - - return results - - def _train(self): - if not self.use_tqdm: - from fastNLP.core.utils import _pseudo_tqdm as inner_tqdm - else: - inner_tqdm = tqdm - self.step = 0 - start = time.time() - total_steps = (len(self.train_data) // self.batch_size + int( - len(self.train_data) % self.batch_size != 0)) * self.n_epochs - with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: - avg_loss = 0 - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - prefetch=self.prefetch) - for epoch in range(1, self.n_epochs+1): - pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) - last_stage = (epoch > self.n_epochs + 1 - self.final_epochs) - if epoch == self.n_epochs + 1 - self.final_epochs: - print('Entering the final stage. (Only train the selected structure)') - # early stopping - self.callback_manager.on_epoch_begin(epoch, self.n_epochs) - - # 1. Training the shared parameters omega of the child models - self.train_shared(pbar) - - # 2. Training the controller parameters theta - if not last_stage: - self.train_controller() - - if ((self.validate_every > 0 and self.step % self.validate_every == 0) or - (self.validate_every < 0 and self.step % len(data_iterator) == 0)) \ - and self.dev_data is not None: - if not last_stage: - self.derive() - eval_res = self._do_validation(epoch=epoch, step=self.step) - eval_str = "Evaluation at Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, - total_steps) + \ - self.tester._format_eval_results(eval_res) - pbar.write(eval_str) - - # lr decay; early stopping - self.callback_manager.on_epoch_end(epoch, self.n_epochs, self.optimizer) - # =============== epochs end =================== # - pbar.close() - # ============ tqdm end ============== # - - - def get_loss(self, inputs, targets, hidden, dags): - """Computes the loss for the same batch for M models. - - This amounts to an estimate of the loss, which is turned into an - estimate for the gradients of the shared model. - """ - if not isinstance(dags, list): - dags = [dags] - - loss = 0 - for dag in dags: - self.shared.setDAG(dag) - inputs = _build_args(self.shared.forward, **inputs) - inputs['hidden'] = hidden - result = self.shared(**inputs) - output, hidden, extra_out = result['pred'], result['hidden'], result['extra_out'] - - self.callback_manager.on_loss_begin(targets, result) - sample_loss = self._compute_loss(result, targets) - loss += sample_loss - - assert len(dags) == 1, 'there are multiple `hidden` for multple `dags`' - return loss, hidden, extra_out - - def train_shared(self, pbar=None, max_step=None, dag=None): - """Train the language model for 400 steps of minibatches of 64 - examples. - - Args: - max_step: Used to run extra training steps as a warm-up. - dag: If not None, is used instead of calling sample(). - - BPTT is truncated at 35 timesteps. - - For each weight update, gradients are estimated by sampling M models - from the fixed controller policy, and averaging their gradients - computed on a batch of training data. - """ - model = self.shared - model.train() - self.controller.eval() - - hidden = self.shared.init_hidden(self.batch_size) - - abs_max_grad = 0 - abs_max_hidden_norm = 0 - step = 0 - raw_total_loss = 0 - total_loss = 0 - train_idx = 0 - avg_loss = 0 - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - prefetch=self.prefetch) - - for batch_x, batch_y in data_iterator: - _move_dict_value_to_device(batch_x, batch_y, device=self._model_device) - indices = data_iterator.get_batch_indices() - # negative sampling; replace unknown; re-weight batch_y - self.callback_manager.on_batch_begin(batch_x, batch_y, indices) - # prediction = self._data_forward(self.model, batch_x) - - dags = self.controller.sample(1) - inputs, targets = batch_x, batch_y - # self.callback_manager.on_loss_begin(batch_y, prediction) - loss, hidden, extra_out = self.get_loss(inputs, - targets, - hidden, - dags) - hidden.detach_() - - avg_loss += loss.item() - - # Is loss NaN or inf? requires_grad = False - self.callback_manager.on_backward_begin(loss, self.model) - self._grad_backward(loss) - self.callback_manager.on_backward_end(self.model) - - self._update() - self.callback_manager.on_step_end(self.optimizer) - - if (self.step+1) % self.print_every == 0: - if self.use_tqdm: - print_output = "loss:{0:<6.5f}".format(avg_loss / self.print_every) - pbar.update(self.print_every) - else: - end = time.time() - diff = timedelta(seconds=round(end - start)) - print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format( - epoch, self.step, avg_loss, diff) - pbar.set_postfix_str(print_output) - avg_loss = 0 - self.step += 1 - step += 1 - self.shared_step += 1 - self.callback_manager.on_batch_end() - # ================= mini-batch end ==================== # - - - def get_reward(self, dag, entropies, hidden, valid_idx=0): - """Computes the perplexity of a single sampled model on a minibatch of - validation data. - """ - if not isinstance(entropies, np.ndarray): - entropies = entropies.data.cpu().numpy() - - data_iterator = Batch(self.dev_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, - prefetch=self.prefetch) - - for inputs, targets in data_iterator: - valid_loss, hidden, _ = self.get_loss(inputs, targets, hidden, dag) - valid_loss = utils.to_item(valid_loss.data) - - valid_ppl = math.exp(valid_loss) - - R = 80 / valid_ppl - - rewards = R + 1e-4 * entropies - - return rewards, hidden - - def train_controller(self): - """Fixes the shared parameters and updates the controller parameters. - - The controller is updated with a score function gradient estimator - (i.e., REINFORCE), with the reward being c/valid_ppl, where valid_ppl - is computed on a minibatch of validation data. - - A moving average baseline is used. - - The controller is trained for 2000 steps per epoch (i.e., - first (Train Shared) phase -> second (Train Controller) phase). - """ - model = self.controller - model.train() - # Why can't we call shared.eval() here? Leads to loss - # being uniformly zero for the controller. - # self.shared.eval() - - avg_reward_base = None - baseline = None - adv_history = [] - entropy_history = [] - reward_history = [] - - hidden = self.shared.init_hidden(self.batch_size) - total_loss = 0 - valid_idx = 0 - for step in range(20): - # sample models - dags, log_probs, entropies = self.controller.sample( - with_details=True) - - # calculate reward - np_entropies = entropies.data.cpu().numpy() - # No gradients should be backpropagated to the - # shared model during controller training, obviously. - with _get_no_grad_ctx_mgr(): - rewards, hidden = self.get_reward(dags, - np_entropies, - hidden, - valid_idx) - - - reward_history.extend(rewards) - entropy_history.extend(np_entropies) - - # moving average baseline - if baseline is None: - baseline = rewards - else: - decay = 0.95 - baseline = decay * baseline + (1 - decay) * rewards - - adv = rewards - baseline - adv_history.extend(adv) - - # policy loss - loss = -log_probs*utils.get_variable(adv, - self.use_cuda, - requires_grad=False) - - loss = loss.sum() # or loss.mean() - - # update - self.controller_optim.zero_grad() - loss.backward() - - self.controller_optim.step() - - total_loss += utils.to_item(loss.data) - - if ((step % 50) == 0) and (step > 0): - reward_history, adv_history, entropy_history = [], [], [] - total_loss = 0 - - self.controller_step += 1 - # prev_valid_idx = valid_idx - # valid_idx = ((valid_idx + self.max_length) % - # (self.valid_data.size(0) - 1)) - # # Whenever we wrap around to the beginning of the - # # validation data, we reset the hidden states. - # if prev_valid_idx > valid_idx: - # hidden = self.shared.init_hidden(self.batch_size) - - def derive(self, sample_num=10, valid_idx=0): - """We are always deriving based on the very first batch - of validation data? This seems wrong... - """ - hidden = self.shared.init_hidden(self.batch_size) - - dags, _, entropies = self.controller.sample(sample_num, - with_details=True) - - max_R = 0 - best_dag = None - for dag in dags: - R, _ = self.get_reward(dag, entropies, hidden, valid_idx) - if R.max() > max_R: - max_R = R.max() - best_dag = dag - - self.model.setDAG(best_dag) diff --git a/legacy/automl/enas_utils.py b/legacy/automl/enas_utils.py deleted file mode 100644 index 7a53dd12..00000000 --- a/legacy/automl/enas_utils.py +++ /dev/null @@ -1,53 +0,0 @@ -# Code Modified from https://github.com/carpedm20/ENAS-pytorch - -from __future__ import print_function - -import collections -from collections import defaultdict - -import numpy as np -import torch -from torch.autograd import Variable - - -def detach(h): - if type(h) == Variable: - return Variable(h.data) - else: - return tuple(detach(v) for v in h) - -def get_variable(inputs, cuda=False, **kwargs): - if type(inputs) in [list, np.ndarray]: - inputs = torch.Tensor(inputs) - if cuda: - out = Variable(inputs.cuda(), **kwargs) - else: - out = Variable(inputs, **kwargs) - return out - -def update_lr(optimizer, lr): - for param_group in optimizer.param_groups: - param_group['lr'] = lr - -Node = collections.namedtuple('Node', ['id', 'name']) - - -class keydefaultdict(defaultdict): - def __missing__(self, key): - if self.default_factory is None: - raise KeyError(key) - else: - ret = self[key] = self.default_factory(key) - return ret - - -def to_item(x): - """Converts x, possibly scalar and possibly tensor, to a Python scalar.""" - if isinstance(x, (float, int)): - return x - - if float(torch.__version__[0:3]) < 0.4: - assert (x.dim() == 1) and (len(x) == 1) - return x[0] - - return x.item() diff --git a/legacy/component/__init__.py b/legacy/component/__init__.py deleted file mode 100644 index c6784aef..00000000 --- a/legacy/component/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .bert_tokenizer import BertTokenizer diff --git a/legacy/component/bert_tokenizer.py b/legacy/component/bert_tokenizer.py deleted file mode 100644 index 6354076d..00000000 --- a/legacy/component/bert_tokenizer.py +++ /dev/null @@ -1,378 +0,0 @@ -""" -bert_tokenizer.py is modified from huggingface/pytorch-pretrained-BERT, which is licensed under the Apache License 2.0. -""" -import collections -import os -import unicodedata -from io import open - - -PRETRAINED_VOCAB_ARCHIVE_MAP = { - 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", - 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", - 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt", - 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt", - 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-vocab.txt", - 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt", - 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt", -} -PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP = { - 'bert-base-uncased': 512, - 'bert-large-uncased': 512, - 'bert-base-cased': 512, - 'bert-large-cased': 512, - 'bert-base-multilingual-uncased': 512, - 'bert-base-multilingual-cased': 512, - 'bert-base-chinese': 512, -} -VOCAB_NAME = 'vocab.txt' - - -def load_vocab(vocab_file): - """Loads a vocabulary file into a dictionary.""" - vocab = collections.OrderedDict() - index = 0 - with open(vocab_file, "r", encoding="utf-8") as reader: - while True: - token = reader.readline() - if not token: - break - token = token.strip() - vocab[token] = index - index += 1 - return vocab - - -def whitespace_tokenize(text): - """Runs basic whitespace cleaning and splitting on a piece of text.""" - text = text.strip() - if not text: - return [] - tokens = text.split() - return tokens - - -class BertTokenizer(object): - """Runs end-to-end tokenization: punctuation splitting + wordpiece""" - - def __init__(self, vocab_file, do_lower_case=True, max_len=None, do_basic_tokenize=True, - never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")): - """Constructs a BertTokenizer. - Args: - vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input - Only has an effect when do_wordpiece_only=False - do_basic_tokenize: Whether to do basic tokenization before wordpiece. - max_len: An artificial maximum length to truncate tokenized sequences to; - Effective maximum length is always the minimum of this - value (if specified) and the underlying BERT model's - sequence length. - never_split: List of tokens which will never be split during tokenization. - Only has an effect when do_wordpiece_only=False - """ - if not os.path.isfile(vocab_file): - raise ValueError( - "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " - "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) - self.vocab = load_vocab(vocab_file) - self.ids_to_tokens = collections.OrderedDict( - [(ids, tok) for tok, ids in self.vocab.items()]) - self.do_basic_tokenize = do_basic_tokenize - if do_basic_tokenize: - self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, - never_split=never_split) - self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) - self.max_len = max_len if max_len is not None else int(1e12) - - def tokenize(self, text): - split_tokens = [] - if self.do_basic_tokenize: - for token in self.basic_tokenizer.tokenize(text): - for sub_token in self.wordpiece_tokenizer.tokenize(token): - split_tokens.append(sub_token) - else: - split_tokens = self.wordpiece_tokenizer.tokenize(text) - return split_tokens - - def convert_tokens_to_ids(self, tokens): - """Converts a sequence of tokens into ids using the vocab.""" - ids = [] - for token in tokens: - ids.append(self.vocab[token]) - if len(ids) > self.max_len: - print( - "WARNING!\n\"" - "Token indices sequence length is longer than the specified maximum " - "sequence length for this BERT model ({} > {}). Running this" - " sequence through BERT will result in indexing errors".format(len(ids), self.max_len) - ) - return ids - - def convert_ids_to_tokens(self, ids): - """Converts a sequence of ids in wordpiece tokens using the vocab.""" - tokens = [] - for i in ids: - tokens.append(self.ids_to_tokens[i]) - return tokens - - def save_vocabulary(self, vocab_path): - """Save the tokenizer vocabulary to a directory or file.""" - index = 0 - if os.path.isdir(vocab_path): - vocab_file = os.path.join(vocab_path, VOCAB_NAME) - with open(vocab_file, "w", encoding="utf-8") as writer: - for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): - if index != token_index: - print("Saving vocabulary to {}: vocabulary indices are not consecutive." - " Please check that the vocabulary is not corrupted!".format(vocab_file)) - index = token_index - writer.write(token + u'\n') - index += 1 - return vocab_file - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, cache_dir=None, *inputs, **kwargs): - """ - Instantiate a PreTrainedBertModel from a pre-trained model file. - Download and cache the pre-trained model file if needed. - """ - if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP: - vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name_or_path] - if '-cased' in pretrained_model_name_or_path and kwargs.get('do_lower_case', True): - print("The pre-trained model you are loading is a cased model but you have not set " - "`do_lower_case` to False. We are setting `do_lower_case=False` for you but " - "you may want to check this behavior.") - kwargs['do_lower_case'] = False - elif '-cased' not in pretrained_model_name_or_path and not kwargs.get('do_lower_case', True): - print("The pre-trained model you are loading is an uncased model but you have set " - "`do_lower_case` to False. We are setting `do_lower_case=True` for you " - "but you may want to check this behavior.") - kwargs['do_lower_case'] = True - else: - vocab_file = pretrained_model_name_or_path - if os.path.isdir(vocab_file): - vocab_file = os.path.join(vocab_file, VOCAB_NAME) - # redirect to the cache, if necessary - resolved_vocab_file = vocab_file - print("loading vocabulary file {}".format(vocab_file)) - if pretrained_model_name_or_path in PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP: - # if we're using a pretrained model, ensure the tokenizer wont index sequences longer - # than the number of positional embeddings - max_len = PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP[pretrained_model_name_or_path] - kwargs['max_len'] = min(kwargs.get('max_len', int(1e12)), max_len) - # Instantiate tokenizer. - tokenizer = cls(resolved_vocab_file, *inputs, **kwargs) - return tokenizer - - -class BasicTokenizer(object): - """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" - - def __init__(self, - do_lower_case=True, - never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")): - """Constructs a BasicTokenizer. - Args: - do_lower_case: Whether to lower case the input. - """ - self.do_lower_case = do_lower_case - self.never_split = never_split - - def tokenize(self, text): - """Tokenizes a piece of text.""" - text = self._clean_text(text) - # This was added on November 1st, 2018 for the multilingual and Chinese - # models. This is also applied to the English models now, but it doesn't - # matter since the English models were not trained on any Chinese data - # and generally don't have any Chinese data in them (there are Chinese - # characters in the vocabulary because Wikipedia does have some Chinese - # words in the English Wikipedia.). - text = self._tokenize_chinese_chars(text) - orig_tokens = whitespace_tokenize(text) - split_tokens = [] - for token in orig_tokens: - if self.do_lower_case and token not in self.never_split: - token = token.lower() - token = self._run_strip_accents(token) - split_tokens.extend(self._run_split_on_punc(token)) - - output_tokens = whitespace_tokenize(" ".join(split_tokens)) - return output_tokens - - def _run_strip_accents(self, text): - """Strips accents from a piece of text.""" - text = unicodedata.normalize("NFD", text) - output = [] - for char in text: - cat = unicodedata.category(char) - if cat == "Mn": - continue - output.append(char) - return "".join(output) - - def _run_split_on_punc(self, text): - """Splits punctuation on a piece of text.""" - if text in self.never_split: - return [text] - chars = list(text) - i = 0 - start_new_word = True - output = [] - while i < len(chars): - char = chars[i] - if _is_punctuation(char): - output.append([char]) - start_new_word = True - else: - if start_new_word: - output.append([]) - start_new_word = False - output[-1].append(char) - i += 1 - - return ["".join(x) for x in output] - - def _tokenize_chinese_chars(self, text): - """Adds whitespace around any CJK character.""" - output = [] - for char in text: - cp = ord(char) - if self._is_chinese_char(cp): - output.append(" ") - output.append(char) - output.append(" ") - else: - output.append(char) - return "".join(output) - - def _is_chinese_char(self, cp): - """Checks whether CP is the codepoint of a CJK character.""" - # This defines a "chinese character" as anything in the CJK Unicode block: - # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) - # - # Note that the CJK Unicode block is NOT all Japanese and Korean characters, - # despite its name. The modern Korean Hangul alphabet is a different block, - # as is Japanese Hiragana and Katakana. Those alphabets are used to write - # space-separated words, so they are not treated specially and handled - # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # - return True - - return False - - def _clean_text(self, text): - """Performs invalid character removal and whitespace cleanup on text.""" - output = [] - for char in text: - cp = ord(char) - if cp == 0 or cp == 0xfffd or _is_control(char): - continue - if _is_whitespace(char): - output.append(" ") - else: - output.append(char) - return "".join(output) - - -class WordpieceTokenizer(object): - """Runs WordPiece tokenization.""" - - def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100): - self.vocab = vocab - self.unk_token = unk_token - self.max_input_chars_per_word = max_input_chars_per_word - - def tokenize(self, text): - """Tokenizes a piece of text into its word pieces. - This uses a greedy longest-match-first algorithm to perform tokenization - using the given vocabulary. - For example: - input = "unaffable" - output = ["un", "##aff", "##able"] - Args: - text: A single token or whitespace separated tokens. This should have - already been passed through `BasicTokenizer`. - Returns: - A list of wordpiece tokens. - """ - - output_tokens = [] - for token in whitespace_tokenize(text): - chars = list(token) - if len(chars) > self.max_input_chars_per_word: - output_tokens.append(self.unk_token) - continue - - is_bad = False - start = 0 - sub_tokens = [] - while start < len(chars): - end = len(chars) - cur_substr = None - while start < end: - substr = "".join(chars[start:end]) - if start > 0: - substr = "##" + substr - if substr in self.vocab: - cur_substr = substr - break - end -= 1 - if cur_substr is None: - is_bad = True - break - sub_tokens.append(cur_substr) - start = end - - if is_bad: - output_tokens.append(self.unk_token) - else: - output_tokens.extend(sub_tokens) - return output_tokens - - -def _is_whitespace(char): - """Checks whether `chars` is a whitespace character.""" - # \t, \n, and \r are technically contorl characters but we treat them - # as whitespace since they are generally considered as such. - if char == " " or char == "\t" or char == "\n" or char == "\r": - return True - cat = unicodedata.category(char) - if cat == "Zs": - return True - return False - - -def _is_control(char): - """Checks whether `chars` is a control character.""" - # These are technically control characters but we count them as whitespace - # characters. - if char == "\t" or char == "\n" or char == "\r": - return False - cat = unicodedata.category(char) - if cat.startswith("C"): - return True - return False - - -def _is_punctuation(char): - """Checks whether `chars` is a punctuation character.""" - cp = ord(char) - # We treat all non-letter/number ASCII as punctuation. - # Characters such as "^", "$", and "`" are not in the Unicode - # Punctuation class but we treat them as punctuation anyways, for - # consistency. - if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or - (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): - return True - cat = unicodedata.category(char) - if cat.startswith("P"): - return True - return False - diff --git a/test/core/test_dataset.py b/test/core/test_dataset.py index 9c05c334..059d52d2 100644 --- a/test/core/test_dataset.py +++ b/test/core/test_dataset.py @@ -182,8 +182,9 @@ class TestDataSetMethods(unittest.TestCase): def test_apply2(self): def split_sent(ins): return ins['raw_sentence'].split() - csv_loader = CSVLoader(headers=['raw_sentence', 'label'],sep='\t') - dataset = csv_loader.load('test/data_for_tests/tutorial_sample_dataset.csv') + csv_loader = CSVLoader(headers=['raw_sentence', 'label'], sep='\t') + data_bundle = csv_loader.load('test/data_for_tests/tutorial_sample_dataset.csv') + dataset = data_bundle.datasets['train'] dataset.drop(lambda x: len(x['raw_sentence'].split()) == 0, inplace=True) dataset.apply(split_sent, new_field_name='words', is_input=True) # print(dataset) diff --git a/test/io/test_data_loader.py b/test/io/test_data_loader.py deleted file mode 100644 index 5b1bb749..00000000 --- a/test/io/test_data_loader.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from fastNLP.core.const import Const -from fastNLP.io.data_loader import MNLILoader - - -class TestDataLoader(unittest.TestCase): - - def test_mnli_loader(self): - ds = MNLILoader().process('test/data_for_tests/sample_mnli.tsv', - to_lower=True, get_index=True, seq_len_type='mask') - self.assertTrue('train' in ds.datasets) - self.assertTrue(len(ds.datasets) == 1) - self.assertTrue(len(ds.datasets['train']) == 11) - self.assertTrue(isinstance(ds.datasets['train'][0][Const.INPUT_LENS(0)], list)) diff --git a/test/io/test_dataset_loader.py b/test/io/test_dataset_loader.py deleted file mode 100644 index 6fb8e4f7..00000000 --- a/test/io/test_dataset_loader.py +++ /dev/null @@ -1,77 +0,0 @@ -import unittest -import os -from fastNLP.io import CSVLoader, JsonLoader -from fastNLP.io.data_loader import SSTLoader, SNLILoader, Conll2003Loader, PeopleDailyCorpusLoader - - -class TestDatasetLoader(unittest.TestCase): - - def test_Conll2003Loader(self): - """ - Test the the loader of Conll2003 dataset - """ - dataset_path = "test/data_for_tests/conll_2003_example.txt" - loader = Conll2003Loader() - dataset_2003 = loader.load(dataset_path) - - def test_PeopleDailyCorpusLoader(self): - data_set = PeopleDailyCorpusLoader().load("test/data_for_tests/people_daily_raw.txt") - - def test_CSVLoader(self): - ds = CSVLoader(sep='\t', headers=['words', 'label']) \ - .load('test/data_for_tests/tutorial_sample_dataset.csv') - assert len(ds) > 0 - - def test_SNLILoader(self): - ds = SNLILoader().load('test/data_for_tests/sample_snli.jsonl') - assert len(ds) == 3 - - def test_JsonLoader(self): - ds = JsonLoader().load('test/data_for_tests/sample_snli.jsonl') - assert len(ds) == 3 - - def no_test_SST(self): - train_data = """(3 (2 (2 The) (2 Rock)) (4 (3 (2 is) (4 (2 destined) (2 (2 (2 (2 (2 to) (2 (2 be) (2 (2 the) (2 (2 21st) (2 (2 (2 Century) (2 's)) (2 (3 new) (2 (2 ``) (2 Conan)))))))) (2 '')) (2 and)) (3 (2 that) (3 (2 he) (3 (2 's) (3 (2 going) (3 (2 to) (4 (3 (2 make) (3 (3 (2 a) (3 splash)) (2 (2 even) (3 greater)))) (2 (2 than) (2 (2 (2 (2 (1 (2 Arnold) (2 Schwarzenegger)) (2 ,)) (2 (2 Jean-Claud) (2 (2 Van) (2 Damme)))) (2 or)) (2 (2 Steven) (2 Segal))))))))))))) (2 .))) -(4 (4 (4 (2 The) (4 (3 gorgeously) (3 (2 elaborate) (2 continuation)))) (2 (2 (2 of) (2 ``)) (2 (2 The) (2 (2 (2 Lord) (2 (2 of) (2 (2 the) (2 Rings)))) (2 (2 '') (2 trilogy)))))) (2 (3 (2 (2 is) (2 (2 so) (2 huge))) (2 (2 that) (3 (2 (2 (2 a) (2 column)) (2 (2 of) (2 words))) (2 (2 (2 (2 can) (1 not)) (3 adequately)) (2 (2 describe) (2 (3 (2 (2 co-writer\/director) (2 (2 Peter) (3 (2 Jackson) (2 's)))) (3 (2 expanded) (2 vision))) (2 (2 of) (2 (2 (2 J.R.R.) (2 (2 Tolkien) (2 's))) (2 Middle-earth))))))))) (2 .))) -(3 (3 (2 (2 (2 (2 (2 Singer\/composer) (2 (2 Bryan) (2 Adams))) (2 (2 contributes) (2 (2 (2 a) (2 slew)) (2 (2 of) (2 songs))))) (2 (2 --) (2 (2 (2 (2 a) (2 (2 few) (3 potential))) (2 (2 (2 hits) (2 ,)) (2 (2 (2 a) (2 few)) (1 (1 (2 more) (1 (2 simply) (2 intrusive))) (2 (2 to) (2 (2 the) (2 story))))))) (2 --)))) (2 but)) (3 (4 (2 the) (3 (2 whole) (2 package))) (2 (3 certainly) (3 (2 captures) (2 (1 (2 the) (2 (2 (2 intended) (2 (2 ,) (2 (2 er) (2 ,)))) (3 spirit))) (2 (2 of) (2 (2 the) (2 piece)))))))) (2 .)) -(2 (2 (2 You) (2 (2 'd) (2 (2 think) (2 (2 by) (2 now))))) (2 (2 America) (2 (2 (2 would) (1 (2 have) (2 (2 (2 had) (1 (2 enough) (2 (2 of) (2 (2 plucky) (2 (2 British) (1 eccentrics)))))) (4 (2 with) (4 (3 hearts) (3 (2 of) (3 gold))))))) (2 .)))) -""" - test_data = """(3 (2 Yet) (3 (2 (2 the) (2 act)) (3 (4 (3 (2 is) (3 (2 still) (4 charming))) (2 here)) (2 .)))) -(4 (2 (2 Whether) (2 (2 (2 (2 or) (1 not)) (3 (2 you) (2 (2 're) (3 (3 enlightened) (2 (2 by) (2 (2 any) (2 (2 of) (2 (2 Derrida) (2 's))))))))) (2 (2 lectures) (2 (2 on) (2 (2 ``) (2 (2 (2 (2 (2 (2 the) (2 other)) (2 '')) (2 and)) (2 ``)) (2 (2 the) (2 self)))))))) (3 (2 ,) (3 (2 '') (3 (2 Derrida) (3 (3 (2 is) (4 (2 an) (4 (4 (2 undeniably) (3 (4 (3 fascinating) (2 and)) (4 playful))) (2 fellow)))) (2 .)))))) -(4 (3 (2 (2 Just) (2 (2 the) (2 labour))) (3 (2 involved) (3 (2 in) (4 (2 creating) (3 (3 (2 the) (3 (3 layered) (2 richness))) (3 (2 of) (3 (2 (2 the) (2 imagery)) (2 (2 in) (3 (2 (2 this) (2 chiaroscuro)) (2 (2 of) (2 (2 (2 madness) (2 and)) (2 light)))))))))))) (3 (3 (2 is) (4 astonishing)) (2 .))) -(3 (3 (2 Part) (3 (2 of) (4 (2 (2 the) (3 charm)) (2 (2 of) (2 (2 Satin) (2 Rouge)))))) (3 (3 (2 is) (3 (2 that) (3 (2 it) (2 (1 (2 avoids) (2 (2 the) (1 obvious))) (3 (2 with) (3 (3 (3 humour) (2 and)) (2 lightness))))))) (2 .))) -(4 (2 (2 a) (2 (2 screenplay) (2 more))) (3 (4 ingeniously) (2 (2 constructed) (2 (2 (2 (2 than) (2 ``)) (2 Memento)) (2 ''))))) -(3 (2 ``) (3 (2 (2 Extreme) (2 Ops)) (3 (2 '') (4 (4 (3 exceeds) (2 expectations)) (2 .))))) -""" - train, test = 'train--', 'test--' - with open(train, 'w', encoding='utf-8') as f: - f.write(train_data) - with open(test, 'w', encoding='utf-8') as f: - f.write(test_data) - - loader = SSTLoader() - info = loader.process( - {train: train, test: test}, - train_ds=[train], - src_vocab_op=dict(min_freq=2) - ) - assert len(list(info.vocabs.items())) == 2 - assert len(list(info.datasets.items())) == 2 - print(info.vocabs) - print(info.datasets) - os.remove(train), os.remove(test) - - # def test_import(self): - # import fastNLP - # from fastNLP.io import SNLILoader - # ds = SNLILoader().process('test/data_for_tests/sample_snli.jsonl', to_lower=True, - # get_index=True, seq_len_type='seq_len', extra_split=['-']) - # assert 'train' in ds.datasets - # assert len(ds.datasets) == 1 - # assert len(ds.datasets['train']) == 3 - # - # ds = SNLILoader().process('test/data_for_tests/sample_snli.jsonl', to_lower=True, - # get_index=True, seq_len_type='seq_len') - # assert 'train' in ds.datasets - # assert len(ds.datasets) == 1 - # assert len(ds.datasets['train']) == 3 From 39de27f472fab631b97b47d4934b05f10019b081 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Thu, 29 Aug 2019 08:19:36 +0800 Subject: [PATCH 14/92] Update BertModel.from_pretrained function. Now can pass a model_dir_or_name instead of model_dir. --- fastNLP/embeddings/bert_embedding.py | 29 ++++------ fastNLP/modules/encoder/bert.py | 81 ++++++++++++++-------------- 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index b1b1a200..e15c15f5 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -18,7 +18,7 @@ from itertools import chain from ..core.vocabulary import Vocabulary from ..io.file_utils import _get_embedding_url, cached_path, PRETRAINED_BERT_MODEL_DIR -from ..modules.encoder.bert import _WordPieceBertModel, BertModel, BertTokenizer +from ..modules.encoder.bert import _WordPieceBertModel, BertModel, BertTokenizer, _get_bert_dir from .contextual_embedding import ContextualEmbedding import warnings from ..core import logger @@ -70,19 +70,16 @@ class BertEmbedding(ContextualEmbedding): pool_method: str = 'first', word_dropout=0, dropout=0, include_cls_sep: bool = False, pooled_cls=True, requires_grad: bool = False, auto_truncate: bool = False): super(BertEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) - - # 根据model_dir_or_name检查是否存在并下载 + if model_dir_or_name.lower() in PRETRAINED_BERT_MODEL_DIR: if 'cn' in model_dir_or_name.lower() and pool_method not in ('first', 'last'): + logger.warn("For Chinese bert, pooled_method should choose from 'first', 'last' in order to achieve" + " faster speed.") warnings.warn("For Chinese bert, pooled_method should choose from 'first', 'last' in order to achieve" " faster speed.") - model_url = _get_embedding_url('bert', model_dir_or_name.lower()) - model_dir = cached_path(model_url, name='embedding') - # 检查是否存在 - elif os.path.isdir(os.path.abspath(os.path.expanduser(model_dir_or_name))): - model_dir = os.path.abspath(os.path.expanduser(model_dir_or_name)) - else: - raise ValueError(f"Cannot recognize {model_dir_or_name}.") + + # 根据model_dir_or_name检查是否存在并下载 + model_dir = _get_bert_dir(model_dir_or_name) self._word_sep_index = None if '[SEP]' in vocab: @@ -173,15 +170,9 @@ class BertWordPieceEncoder(nn.Module): def __init__(self, model_dir_or_name: str = 'en-base-uncased', layers: str = '-1', pooled_cls: bool = False, word_dropout=0, dropout=0, requires_grad: bool = False): super().__init__() - - if model_dir_or_name.lower() in PRETRAINED_BERT_MODEL_DIR: - model_url = _get_embedding_url('bert', model_dir_or_name.lower()) - model_dir = cached_path(model_url, name='embedding') - # 检查是否存在 - elif os.path.isdir(os.path.expanduser(os.path.abspath(model_dir_or_name))): - model_dir = model_dir_or_name - else: - raise ValueError(f"Cannot recognize {model_dir_or_name}.") + + # 根据model_dir_or_name检查是否存在并下载 + model_dir = _get_bert_dir(model_dir_or_name) self.model = _WordPieceBertModel(model_dir=model_dir, layers=layers, pooled_cls=pooled_cls) self._sep_index = self.model._sep_index diff --git a/fastNLP/modules/encoder/bert.py b/fastNLP/modules/encoder/bert.py index 5026f48a..89a1b09d 100644 --- a/fastNLP/modules/encoder/bert.py +++ b/fastNLP/modules/encoder/bert.py @@ -18,13 +18,13 @@ import torch from torch import nn from ..utils import _get_file_name_base_on_postfix +from ...io.file_utils import _get_embedding_url, cached_path, PRETRAINED_BERT_MODEL_DIR from ...core import logger CONFIG_FILE = 'bert_config.json' VOCAB_NAME = 'vocab.txt' - class BertConfig(object): """Configuration class to store the configuration of a `BertModel`. """ @@ -133,6 +133,19 @@ def swish(x): ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish} +def _get_bert_dir(model_dir_or_name: str = 'en-base-uncased'): + if model_dir_or_name.lower() in PRETRAINED_BERT_MODEL_DIR: + model_url = _get_embedding_url('bert', model_dir_or_name.lower()) + model_dir = cached_path(model_url, name='embedding') + # 检查是否存在 + elif os.path.isdir(os.path.abspath(os.path.expanduser(model_dir_or_name))): + model_dir = os.path.abspath(os.path.expanduser(model_dir_or_name)) + else: + logger.error(f"Cannot recognize BERT dir or name ``{model_dir_or_name}``.") + raise ValueError(f"Cannot recognize BERT dir or name ``{model_dir_or_name}``.") + return model_dir + + class BertLayerNorm(nn.Module): def __init__(self, hidden_size, eps=1e-12): """Construct a layernorm module in the TF style (epsilon inside the square root). @@ -339,27 +352,9 @@ class BertModel(nn.Module): BERT(Bidirectional Embedding Representations from Transformers). - 如果你想使用预训练好的权重矩阵，请在以下网址下载. - sources:: - - 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin", - 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-pytorch_model.bin", - 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-pytorch_model.bin", - 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-pytorch_model.bin", - 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-pytorch_model.bin", - 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-pytorch_model.bin", - 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-pytorch_model.bin", - 'bert-base-german-cased': "https://int-deepset-models-bert.s3.eu-central-1.amazonaws.com/pytorch/bert-base-german-cased-pytorch_model.bin", - 'bert-large-uncased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-pytorch_model.bin", - 'bert-large-cased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-pytorch_model.bin", - 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-pytorch_model.bin", - 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-pytorch_model.bin", - 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-pytorch_model.bin" - - 用预训练权重矩阵来建立BERT模型:: - model = BertModel.from_pretrained("path/to/weights/directory") + model = BertModel.from_pretrained(model_dir_or_name) 用随机初始化权重矩阵来建立BERT模型:: @@ -440,11 +435,15 @@ class BertModel(nn.Module): return encoded_layers, pooled_output @classmethod - def from_pretrained(cls, pretrained_model_dir, *inputs, **kwargs): + def from_pretrained(cls, pretrained_model_dir_or_name, *inputs, **kwargs): state_dict = kwargs.get('state_dict', None) kwargs.pop('state_dict', None) kwargs.pop('cache_dir', None) kwargs.pop('from_tf', None) + + # get model dir from name or dir + pretrained_model_dir = _get_bert_dir(pretrained_model_dir_or_name) + # Load config config_file = _get_file_name_base_on_postfix(pretrained_model_dir, '.json') config = BertConfig.from_json_file(config_file) @@ -493,6 +492,8 @@ class BertModel(nn.Module): if len(unexpected_keys) > 0: logger.warn("Weights from pretrained model not used in {}: {}".format( model.__class__.__name__, unexpected_keys)) + + logger.info(f"Load pre-trained BERT parameters from dir {pretrained_model_dir}.") return model @@ -562,7 +563,7 @@ class WordpieceTokenizer(object): output_tokens.append(self.unk_token) else: output_tokens.extend(sub_tokens) - if len(output_tokens)==0: #防止里面全是空格或者回车符号 + if len(output_tokens) == 0: # 防止里面全是空格或者回车符号 return [self.unk_token] return output_tokens @@ -673,14 +674,14 @@ class BasicTokenizer(object): # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # + if (((cp >= 0x4E00) and (cp <= 0x9FFF)) or # + ((cp >= 0x3400) and (cp <= 0x4DBF)) or # + ((cp >= 0x20000) and (cp <= 0x2A6DF)) or # + ((cp >= 0x2A700) and (cp <= 0x2B73F)) or # + ((cp >= 0x2B740) and (cp <= 0x2B81F)) or # + ((cp >= 0x2B820) and (cp <= 0x2CEAF)) or + ((cp >= 0xF900) and (cp <= 0xFAFF)) or # + ((cp >= 0x2F800) and (cp <= 0x2FA1F))): # return True return False @@ -730,8 +731,8 @@ def _is_punctuation(char): # Characters such as "^", "$", and "`" are not in the Unicode # Punctuation class but we treat them as punctuation anyways, for # consistency. - if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or - (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + if (((cp >= 33) and (cp <= 47)) or ((cp >= 58) and (cp <= 64)) or + ((cp >= 91) and (cp <= 96)) or ((cp >= 123) and (cp <= 126))): return True cat = unicodedata.category(char) if cat.startswith("P"): @@ -830,11 +831,11 @@ class BertTokenizer(object): return vocab_file @classmethod - def from_pretrained(cls, model_dir, *inputs, **kwargs): + def from_pretrained(cls, model_dir_or_name, *inputs, **kwargs): """ - 给定path，直接读取vocab. - + 给定模型的名字或者路径，直接读取vocab. """ + model_dir = _get_bert_dir(model_dir_or_name) pretrained_model_name_or_path = _get_file_name_base_on_postfix(model_dir, '.txt') logger.info("loading vocabulary file {}".format(pretrained_model_name_or_path)) max_len = 512 @@ -843,17 +844,19 @@ class BertTokenizer(object): tokenizer = cls(pretrained_model_name_or_path, *inputs, **kwargs) return tokenizer + class _WordPieceBertModel(nn.Module): """ 这个模块用于直接计算word_piece的结果. """ - def __init__(self, model_dir: str, layers: str = '-1', pooled_cls:bool=False): + def __init__(self, model_dir_or_name: str, layers: str = '-1', pooled_cls: bool=False): super().__init__() - self.tokenzier = BertTokenizer.from_pretrained(model_dir) - self.encoder = BertModel.from_pretrained(model_dir) + self.model_dir = _get_bert_dir(model_dir_or_name) + self.tokenzier = BertTokenizer.from_pretrained(self.model_dir) + self.encoder = BertModel.from_pretrained(self.model_dir) # 检查encoder_layer_number是否合理 encoder_layer_number = len(self.encoder.encoder.layer) self.layers = list(map(int, layers.split(','))) @@ -914,7 +917,7 @@ class _WordPieceBertModel(nn.Module): attn_masks = word_pieces.ne(self._wordpiece_pad_index) bert_outputs, pooled_cls = self.encoder(word_pieces, token_type_ids=token_type_ids, attention_mask=attn_masks, - output_all_encoded_layers=True) + output_all_encoded_layers=True) # output_layers = [self.layers] # len(self.layers) x batch_size x max_word_piece_length x hidden_size outputs = bert_outputs[0].new_zeros((len(self.layers), batch_size, max_len, bert_outputs[0].size(-1))) for l_index, l in enumerate(self.layers): From 09d0b74595c8273b8bcb3af48a84cdcd5e6c982e Mon Sep 17 00:00:00 2001 From: yhcc Date: Thu, 29 Aug 2019 09:56:36 +0800 Subject: [PATCH 15/92] Update .travis.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TRAVIS默认已经加入了 --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0d63417a..210d158a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,6 @@ language: python python: - "3.6" - -env: - - TRAVIS=1 # command to install dependencies install: - pip install --quiet -r requirements.txt From 146a004deee58f139ba7317e7b66740a709947ba Mon Sep 17 00:00:00 2001 From: yh_cc Date: Thu, 29 Aug 2019 10:12:30 +0800 Subject: [PATCH 16/92] =?UTF-8?q?=E4=BF=AE=E6=94=B9travis=20converage?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .coverage | 1 + .travis.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .coverage diff --git a/.coverage b/.coverage new file mode 100644 index 00000000..a6d89bc8 --- /dev/null +++ b/.coverage @@ -0,0 +1 @@ +!coverage.py: This is a private format, don't read it directly!{"lines":{"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/__init__.py":[12,14,15,18,19,20,22,23,24,26,27,29,30,31,32,33,34,35,37,38,39,41,42,43,45,46,47,48,50,51,52,53,55,56,57,58,59,60,62,64,66,68,69,70,71,72],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/__init__.py":[6,9,10,11,12,13,14,15,16,17,18,21,22,23,24,25,26,27],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/embedding.py":[128,129,130,131,4,133,7,8,11,12,13,140,15,141,142,18,146,148,143,144,145,155,157,39,41,169,43,45,174,47,48,177,178,49,51,181,182,52,55,185,186,179,60,61,63,193,68,199,72,201,73,75,76,205,82,85,86,87,89,90,91,93,104,111,119,120,121,122,123,124,125,126,127],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/utils.py":[4,5,6,7,9,42,43,12,44,45,46,16,24,57,26,27,28,25,31],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/__init__.py":[13,15,17,19,20,21,22,24,26,27,28,30,32,33,35,36,37,38,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,56,57,58,59,60,61,63,64,65,67,68,69,70,72,73,74,75,78,79,80,83,84,85,86,87,88,89,90,91,92,93,94],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/_logger.py":[1,130,131,4,132,134,7,8,9,10,11,137,13,140,15,16,143,19,20,24,25,26,155,27,29,30,31,32,33,45,46,47,49,50,51,52,53,56,78,79,80,83,84,88,92,94,95,99,100,101,102,103,106,107,108,110,114,119,125,127],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/batch.py":[4,6,7,8,11,13,14,15,16,18,19,20,21,24,29,32,33,34,35,36,37,38,40,42,43,44,45,47,50,57,58,59,60,61,62,63,64,65,67,68,69,70,73,74,75,76,80,81,83,84,85,87,92,99,100,101,102,103,105,106,108,109,112,113,114,115,116,117,119,120,122,124,125,126,127,129,130,131,132,133,135,136,138,139,141,146,171,174,175,176,177,178,181,182,183,184,185,186,187,189,190,193,194,202,204,207,211,215,223,224,225,226,227,228,229,230,233],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/sampler.py":[3,5,6,7,8,134,135,11,140,13,137,16,149,150,151,24,153,26,155,156,158,160,34,162,163,164,166,165,40,167,42,170,43,46,52,54,55,58,186,187,188,190,191,192,193,68,70,71,72,73,75,83,84,86,87,89,90,91,92,93,94,96,97,98,100,102,103,104,105,106,107,108,109,110,112,113,114,115,117,120],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/dataset.py":[515,516,518,532,543,550,552,554,560,562,570,578,585,586,587,589,590,592,606,607,608,609,610,611,617,619,631,632,633,634,635,640,641,643,660,676,688,694,696,702,704,722,723,725,726,727,728,729,734,737,738,859,740,742,751,752,753,754,755,756,757,758,862,760,761,762,763,764,765,766,767,768,770,771,772,774,791,792,793,794,795,796,285,287,290,291,803,293,294,806,296,297,298,299,300,301,302,303,811,305,809,824,314,316,317,318,319,320,321,834,835,836,837,838,322,323,324,325,326,327,328,334,329,332,337,849,338,339,340,342,335,344,857,858,347,348,861,345,350,346,865,864,863,866,860,351,868,354,353,356,871,867,875,869,870,360,363,364,877,365,367,369,883,884,886,376,377,378,379,380,381,382,383,384,385,386,387,388,894,895,896,897,402,409,410,412,413,415,420,421,422,423,425,426,427,431,432,434,872,441,443,445,447,451,452,453,454,459,873,474,807,486,487,488,490,491,493,499,500,502,503,505,506,507,509],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/field.py":[4,7,8,9,12,13,14,15,16,18,19,21,22,535,25,26,27,28,29,30,33,34,35,36,37,38,41,43,44,557,46,559,560,47,562,48,52,53,54,563,56,57,58,59,60,564,62,63,64,568,570,67,68,572,70,71,72,65,74,578,76,580,78,590,80,591,585,83,592,85,593,87,594,89,595,596,597,598,599,95,96,97,98,99,100,613,101,614,102,615,609,616,617,618,104,106,108,622,624,113,114,115,116,629,117,118,120,119,122,130,131,132,133,134,135,136,137,138,651,139,653,140,141,142,146,147,148,149,150,663,152,659,661,157,158,159,160,162,165,677,167,169,681,682,683,685,686,175,687,177,178,688,180,181,182,183,184,690,691,187,692,693,190,694,192,697,200,201,202,205,206,207,209,211,212,214,220,221,222,226,45,236,242,244,252,254,255,256,257,259,261,278,565,566,567,298,569,571,318,573,574,575,339,576,577,579,359,581,582,379,584,586,626,398,419,695,428,429,430,431,432,433,434,435,436,437,438,439,441,443,444,445,446,447,448,450,451,452,453,454,455,456,458,459,460,465,482,484,485,487,490,491,610],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/utils.py":[3,5,517,6,7,518,10,11,12,13,14,521,16,17,18,19,20,524,22,23,527,528,26,27,535,29,536,540,541,542,543,35,544,545,547,546,40,548,551,552,553,554,46,550,49,563,564,53,565,567,568,569,59,60,574,62,63,64,67,522,592,599,609,615,530,118,119,120,121,122,124,125,126,127,641,129,130,132,131,134,647,648,649,650,651,652,135,139,140,656,142,144,147,659,145,146,662,663,664,151,666,667,152,669,670,153,672,673,674,163,676,165,678,679,168,681,682,643,685,644,645,192,709,217,218,219,220,222,736,738,227,739,740,226,229,232,233,230,148,231,745,234,235,149,236,237,238,239,240,244,245,241,242,243,246,247,248,249,250,251,252,253,254,255,256,259,260,263,154,271,273,274,156,277,157,280,158,642,288,289,159,291,292,293,294,295,296,297,298,161,301,316,333,334,335,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,364,388,389,390,391,392,393,396,397,405,411,413,416,417,421,430,433,436,437,438,439,440,290,445,449,451,452,454,456,457,458,460,463,465,466,469,470,471,475,476,477,478,479,480,485,496,497,498,499,500,501,502,503,505,506,507,508,509,510,511],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/instance.py":[58,5,37,39,7,11,46,47,48,52,53,55,56,24,26,59,28,30],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/const.py":[4,7,11,29,30,31,32,33,34,35,36,37,39,42,43,45,51,56,61,64,65,67,70,71,73,76,77,79],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/callback.py":[1024,513,1026,1030,1031,1036,529,531,1043,1044,1060,1071,562,51,53,1077,55,56,57,58,59,60,61,62,63,64,65,66,576,68,69,1092,583,72,73,74,76,78,80,81,84,85,87,88,89,91,92,603,606,97,612,106,108,621,109,623,110,113,111,118,120,123,125,128,130,133,135,648,138,140,143,145,660,148,150,153,155,159,161,674,164,166,681,169,171,683,685,686,175,687,177,688,179,692,693,183,696,701,189,191,703,705,706,708,197,710,199,721,722,210,212,723,726,724,728,729,730,220,733,222,741,229,231,743,745,746,748,237,749,239,750,751,752,753,756,245,754,247,758,761,759,252,765,254,766,767,768,770,771,260,773,262,774,775,776,778,779,780,781,782,783,777,785,786,275,787,788,789,791,790,273,794,283,795,796,797,287,799,289,800,801,802,805,293,295,303,818,820,821,310,311,312,313,822,315,316,823,318,830,824,321,322,826,836,828,827,831,832,833,841,329,331,332,333,334,839,336,337,842,851,339,340,341,852,855,348,349,350,863,351,353,864,357,870,871,361,875,365,369,881,373,377,889,890,381,385,389,902,393,907,397,912,401,405,410,411,922,929,420,428,945,946,437,961,964,455,968,457,459,461,462,463,468,469,471,472,473,987,479,482,504,489,491,1003,492,493,494,496,1009,497,1014,1016,506,1020],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/tester.py":[34,35,37,38,40,41,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,62,66,92,94,95,97,100,102,103,104,105,106,107,109,110,111,118,119,121,127,131,132,134,138,139,141,148,149,150,151,152,153,154,155,156,158,159,160,162,164,165,166,167,170,171,173,174,176,177,178,181,182,183,184,185,187,188,189,190,191,192,194,195,196,197,199,206,207,209,211,213,214,215,217,223,224,225,226,227,228],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/metrics.py":[4,6,7,8,9,12,13,15,16,18,19,20,21,22,23,24,25,26,29,117,119,120,121,122,124,133,137,138,141,151,156,158,165,166,179,180,181,182,183,185,186,187,188,192,193,194,195,200,208,209,213,215,230,231,235,236,239,240,241,242,246,247,249,250,253,254,255,256,257,258,259,262,263,264,265,267,270,271,272,274,277,278,279,280,281,282,284,285,286,287,288,290,292,295,305,307,309,311,313,314,316,329,330,332,336,340,341,343,345,346,347,348,350,354,355,356,357,359,360,362,369,370,371,372,373,376,386,388,389,390,391,392,393,394,395,396,398,399,400,401,402,406,437,468,477,479,480,481,482,483,484,485,486,488,489,491,492,493,496,504,505,506,507,508,509,510,511,512,514,515,516,520,561,564,566,568,570,573,574,575,576,577,578,579,580,581,582,586,587,588,589,590,592,593,595,597,598,599,601,609,612,616,620,622,623,624,625,633,634,635,636,637,638,640,641,643,644,646,647,648,649,651,652,653,655,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,681,686,687,688,689,690,691,692,694,695,696,697,699,700,702,704,712,713,714,716,719,726,727,728,729,730,732,733,734,736,738,742,743,747,750,759,760,761,762,763,766,776,777,778,779,780,781,784,799,802,804,806,808,810,811,813,814,816,818,819,820,821,823,825,827,836,837,838,839,841,842,845,846,850,851,852,853,855,856,857,858,859,862,863,864,865,867,868,870,873,875,876,878,879,880,881,883,884,885,887,888,891,893,895,897,900,901,903,905,907,909,910,911,912,914,916,918,919,920,921,923,929,932,933,935,936,937,939,940,942,944,945,946,947,949],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/vocabulary.py":[4,7,8,11,12,13,15,16,17,18,21,26,35,40,42,43,44,46,49,54,56,57,58,59,61,62,64,67,90,92,93,94,95,96,97,98,99,100,102,104,105,116,117,118,120,121,133,134,135,137,145,146,147,148,149,150,151,153,154,166,168,169,181,182,184,190,191,192,193,194,195,197,198,199,200,201,202,203,204,205,206,207,209,214,215,217,219,221,229,231,242,244,251,252,253,254,258,259,273,279,280,282,283,285,287,289,291,292,295,296,297,301,302,303,304,305,311,313,317,337,338,342,343,344,345,346,348,349,350,352,354,355,356,358,359,360,361,368,369,370,371,377,379,385,387,398,400,401,406,407,408,410,411,416,417,418,420,428,430,443,447,448,450,451,453,457,458,460,463,465,466],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/_parallel_utils.py":[1,97,3,5,7,8,9,10,11,76,104,14,105,107],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/losses.py":[4,6,8,9,11,12,13,14,17,18,20,21,23,24,25,26,27,28,29,30,31,34,37,39,40,41,43,52,55,62,63,76,77,78,79,80,82,83,84,85,89,90,91,92,102,110,112,113,114,115,119,120,122,123,125,126,127,128,129,130,131,134,135,136,137,139,141,142,143,145,148,149,150,151,152,153,155,156,157,158,160,162,163,165,168,188,190,192,193,194,195,198,201,222,224,225,226,227,228,229,230,232,233,234,235,236,239,240,241,242,243,245,246,249,259,261,262,263,264,265,267,268,271,280,282,283,284,285,286,288,289,292,303,305,306,307,308,309,310,312,313,316,323,325,326,327,329,331,332,333,334,335,336,337,338,339,340,341,343,345,347,353,356,357,358,359,360,361,366,374,377,386,387,395,410,432],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/optimizer.py":[4,6,7,8,9,135,138,12,13,14,15,18,151,24,26,27,156,29,30,32,35,41,43,47,48,51,54,61,68,70,71,72,73,75,76,78,80,83,90,92,93,95,96,98,99,101,103,106],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/trainer.py":[517,518,519,521,522,523,524,525,526,527,528,529,530,531,532,533,534,536,537,538,539,540,541,545,547,548,551,552,553,554,555,556,557,558,559,560,561,562,564,565,567,570,571,573,593,594,598,599,600,601,602,603,604,606,607,608,609,619,620,621,622,623,624,625,626,627,628,629,630,634,635,637,639,640,641,643,644,645,646,647,648,649,650,651,652,653,654,656,657,658,659,660,662,663,666,667,668,669,672,673,674,676,677,679,680,681,682,683,685,686,687,688,689,690,691,693,694,695,696,697,698,700,701,705,707,708,711,712,713,715,716,717,720,721,722,723,724,725,727,728,857,730,737,740,742,746,747,352,749,750,751,752,755,757,764,765,766,768,775,777,800,802,812,813,816,818,823,824,825,826,827,829,319,831,321,832,835,324,325,326,833,328,329,330,843,332,333,841,847,336,848,338,851,340,341,342,343,344,339,853,854,855,349,350,351,856,345,346,858,347,348,864,865,868,869,353,354,355,356,358,872,873,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,895,896,898,899,900,901,902,903,904,905,907,908,909,910,911,913,914,915,916,917,918,919,920,924,925,927,928,418,932,936,425,426,427,937,941,942,939,940,431,943,433,944,945,947,437,438,948,949,441,954,950,444,951,958,449,450,961,962,964,454,965,456,968,458,970,971,974,466,482,484,485,489,490,491,498,499,502,503,506,507,510,511],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/static_embedding.py":[4,7,9,11,12,13,14,16,17,18,19,20,21,22,25,66,69,70,71,72,75,76,77,78,79,83,84,91,92,119,121,122,123,124,127,128,130,133,134,135,136,140,141,142,143,144,146,147,148,150,151,153,154,155,156,158,164,165,166,167,168,169,171,179,181,182,186,188,202,204,205,207,209,210,226,227,229,230,231,232,233,237,238,239,240,241,242,243,244,245,246,247,248,249,250,252,254,257,258,259,260,261,262,269,270,271,272,275,277,279,283,284,285,286,287,288,290,292,299,300,301,302,303,304],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/__init__.py":[13,15,17,19,21,22,23,24,25,26,28,29,30,31,32,33,34,35,37,38,40,42,43,44,45,46,48,50,51,52,53,54,55,57,58,59,60,61,63,65,66,67,68,69,70,71,72,73,74,75,76,78,79,83,84,85,87,88],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/embed_loader.py":[4,6,7,10,11,12,14,16,17,20,22,23,24,25,34,39,41,44,45,46,63,64,66,67,68,69,70,71,72,73,75,76,77,78,80,81,82,83,84,86,88,90,91,92,93,100,101,102,103,104,105,106,107,108,109,111,112,114,116,117,118,133,134,135,136,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,157,166,168,169,170,171,173,174,175,176,177,178,180,181,182,183,184,185,187,188,190],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/data_bundle.py":[4,6,9,10,13,142,27,29,30,31,159,33,45,55,184,64,74,203,83,92,117],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/model_io.py":[32,3,5,6,9,42,12,17,19,53,22,55,62],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/__init__.py":[44,47,49,50,51,52,53,54,56,57,58,59,60,61,62,63,65,66,68,70,71,72,73,74,76,77,78,79,80,81,82,83],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/classification.py":[1,259,4,5,6,7,8,9,261,264,12,13,14,15,16,17,19,20,21,279,24,291,164,45,47,304,50,178,180,306,309,183,72,73,201,339,244,119,120],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/loader.py":[65,66,1,4,33,70,7,67,9,10,11,12,68,78,15,19,21,22,24,63],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/file_utils.py":[4,7,8,9,10,11,14,15,16,17,18,19,21,22,23,25,28,29,30,32,33,35,36,38,40,41,43,44,45,46,50,51,52,53,54,58,60,61,62,63,64,65,66,67,68,69,71,73,74,76,77,78,79,83,84,85,86,87,88,89,90,91,92,93,94,96,97,98,99,102,103,104,107,108,109,110,114,159,186,202,228,252,273,293,306,418,427,434,443],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/utils.py":[33,34,35,4,36,7,10,11,12,14,17,81],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/conll.py":[1,4,5,6,7,8,9,10,11,12,15,16,17,18,19,146,21,22,23,24,25,150,278,28,279,282,286,287,408,421,175,177,183,62,446,64,448,451,325,204,78,208,92,349,222,273,224,351,354,227,404,117,405,119,125],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/file_reader.py":[33,34,3,35,5,7,9,41,42,12,43,78,47,44,24,25,26,30],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/csv.py":[32,1,34,33,4,35,36,7,8,9,10,37,13,24,26,27,28,29,30],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/cws.py":[1,4,38,7,8,9,10,11,39,13,14,15,47,18,56],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/json.py":[1,4,38,7,8,9,10,13,25,27],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/loader/matching.py":[1,129,4,5,6,7,8,11,12,13,15,16,17,18,19,20,273,277,23,159,35,37,40,170,298,300,303,184,186,189,318,66,216,98,228,109,241,243,246,120,122],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/pipe/__init__.py":[9,11,13,15,16,17,18,19,21,22,23,24,25,26,28,29,30,31,32,33,34,35,36,37,38,39,42,43,44,46,47,48],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/pipe/classification.py":[1,4,5,6,7,8,134,264,11,392,13,15,16,17,18,19,20,21,22,408,24,410,28,414,32,34,37,172,52,182,315,320,449,195,197,70,201,333,335,339,89,218,247,228,104,106,119,249,382],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/pipe/pipe.py":[1,4,7,10,13,14,23],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/pipe/utils.py":[1,66,153,4,5,6,39,9,137,11,12,15,87,121,91],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/pipe/conll.py":[1,4,5,6,7,8,9,12,13,14,15,16,17,18,19,20,141,272,23,286,288,34,36,293,43,306,308,182,313,192,328,330,79,208,210,215,225,98,227,100,233,113,114],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/pipe/matching.py":[128,1,129,259,4,5,6,7,8,9,10,11,12,13,14,15,135,140,18,19,20,21,22,146,147,25,152,260,134,169,42,171,44,177,50,265,266,191,64,141,271,272,247,248,122,123,253,254],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/io/pipe/cws.py":[1,4,7,8,136,10,11,12,13,14,17,155,157,34,168,50,65,202,84,110,254],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/__init__.py":[18,22,23,25,27,29,31,33,34,35,37,38,39,40,42,44,45,46,47,49,52,53,54,55,56],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/decoder/__init__.py":[4,6,7,8,9,12,13,14,15],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/decoder/crf.py":[1,4,5,8,9,11,12,15,29,31,32,33,34,35,36,37,38,40,41,42,43,44,46,47,48,50,51,52,53,54,55,56,57,58,59,60,63,73,74,75,76,93,94,95,96,97,98,102,121,122,123,124,125,126,127,128,157,170,173,175,177,178,181,182,183,184,186,187,192,194,196,204,205,206,207,209,211,212,213,214,215,216,218,219,221,223,231,232,233,236,237,238,240,242,243,244,245,246,247,248,250,252,261,262,263,264,265,267,269,282,283,284,287,288,289,290,291,295,296,297,298,299,300,301,302,303,304,306,310,311,312,314,316,317,318,319,320,321,322,323,328,329],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/utils.py":[4,134,7,8,11,12,14,15,16,19,35,37,39,41,43,45,47,49,52,54,56,57,60,61,62,63,64,65,67,68,69,70,72,73,74,75,77,80,83,120],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/decoder/mlp.py":[1,4,7,8,10,13,44,46,47,48,49,50,51,52,53,55,57,60,61,62,64,65,71,72,73,75,76,79,86,88,93,94,95,96,98,99],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/decoder/utils.py":[1,4,6,9],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/__init__.py":[4,9,10,12,14,16,18,20,21,22,24,25,26,27,29,32,33,34,35,36,37,38,39,40],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/attention.py":[128,1,4,132,7,9,10,11,13,16,20,22,23,24,25,26,27,28,30,38,39,40,41,42,43,46,175,55,184,57,186,58,59,60,61,62,64,65,66,67,69,198,70,71,73,74,75,76,77,78,80,212,88,89,90,92,93,94,97,98,99,100,101,102,105,106,107,110,126],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/bert.py":[512,4,517,7,10,11,12,13,14,15,17,18,20,21,22,24,25,28,30,44,571,70,586,587,75,76,77,591,78,79,80,81,82,83,84,85,600,86,87,92,100,107,621,110,115,119,632,125,126,129,133,136,654,149,150,153,154,667,155,156,158,159,160,161,162,165,167,169,170,171,172,173,689,177,178,180,181,182,183,184,187,188,189,703,191,192,193,194,197,198,199,200,715,204,205,206,208,209,210,212,214,727,215,216,217,219,220,221,222,224,225,226,229,230,743,744,232,747,235,239,241,242,243,244,245,248,249,250,251,252,253,255,256,257,258,259,262,263,776,264,265,266,268,269,270,271,274,275,786,276,277,278,279,283,796,284,285,286,289,290,291,292,293,294,296,809,297,298,299,300,303,304,816,305,306,307,308,310,311,312,313,314,317,318,319,320,833,321,323,324,325,326,327,328,329,330,331,334,335,848,336,337,338,340,852,854,343,344,345,346,349,877,374,376,377,378,385,386,387,388,389,390,391,393,396,909,399,400,401,402,403,404,406,407,409,410,417,424,425,427,428,429,430,431,432,433,434,435,437,500,509,510],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/char_encoder.py":[1,4,5,7,8,10,14,25,27,28,29,30,32,34,36,41,43,45,47,48,49,50,52,54,55,57,58,61,68,70,77,78,80,81,82,83,84,85,87,92,93,94,95,96,98,99],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/conv_maxpool.py":[1,4,6,7,8,11,23,25,26,28,29,32,33,36,37,38,43,52,59,60,69,77,79,80,81,82,84,85,86],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/lstm.py":[4,7,10,11,12,15,30,33,34,35,36,37,38,40,41,42,44,45,46,47,49,51,61,62,65,66,67,68,69,72,73,74,75,76,77,82],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/pooling.py":[1,129,4,5,6,7,135,9,10,137,13,141,25,27,38,62,67,69,73,85,86,88,92,102,107,109,114],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/star_transformer.py":[3,6,9,10,11,12,15,32,34,35,36,38,40,41,42,43,44,45,46,48,49,53,63,65,67,68,69,71,72,76,77,78,79,80,81,82,83,85,87,89,91,94,95,96,99,100,101,102,104,107,109,111,112,114,116,117,118,119,120,121,122,123,124,125,126,127,129,130,132,134,137,138,140,141,142,143,144,146,149,151,153,154,156,158,159,160,161,162,163,164,165,166],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/transformer.py":[1,4,6,8,9,12,26,28,29,30,31,32,33,34,35,36,37,39,46,47,48,49,50,51,52,54,55,56,58,65,66,69,70,71,72,73],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/dropout.py":[1,4,7,10,14,16,17,18,19,20,24],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/variational_rnn.py":[3,6,7,8,11,12,13,15,16,25,28,31,33,34,35,36,37,38,40,52,53,54,55,56,58,59,60,61,62,63,64,66,67,69,70,73,74,75,76,77,79,80,81,82,83,84,85,86,87,88,89,96,97,98,99,102,120,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,146,147,148,149,150,151,152,153,155,163,164,165,166,167,168,169,170,172,173,175,176,177,178,179,181,182,183,184,185,186,187,188,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,210,212,213,215,216,218,219,221,224,239,241,242,243,245,246,249,264,266,270,274,289,291,295],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/elmo_embedding.py":[4,7,136,10,11,12,13,14,15,141,17,18,19,20,21,23,155,163,171,173,305,58,61,92,99,111,119],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/modules/encoder/_elmo.py":[514,3,515,5,7,263,9,10,11,12,264,14,528,17,409,410,309,56,65,453,327,328,85,98,493,239,240,251,510],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/contextual_embedding.py":[99,4,7,104,10,12,76,14,15,16,17,18,19,20,23,24,27],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/bert_embedding.py":[4,7,8,135,11,12,14,15,16,17,271,19,20,21,22,23,24,149,273,27,157,168,171,186,67,198,71,203,207,211,215,95,98,227,361,115,250],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/char_embedding.py":[4,7,8,11,12,13,14,16,17,18,19,20,21,22,25,57,61,62,64,65,67,68,70,71,72,85,87,88,89,91,92,93,94,95,98,99,101,104,106,108,109,110,111,113,120,121,122,123,124,125,127,128,129,130,131,132,133,134,135,136,137,138,142,143,145,161,168,169,170,172,173,174,175,177,180,211,216,217,219,221,222,224,225,226,239,241,242,243,245,246,247,248,249,252,253,255,258,260,261,263,264,265,267,274,275,276,277,278,279,281,282,283,284,285,286,289,290,291,292,297,299,301,318],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/embeddings/stack_embedding.py":[4,7,10,12,13,15,18,37,39,40,41,42,43,44,45,46,48,49,50,51,52,53,55,64,71,75,87,92,99,100,101,102,103,104],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/__init__.py":[32,33,34,9,11,13,14,16,18,19,20,21,23,24,27,28,30,31],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/base_model.py":[32,1,33,3,5,7,10,12,14,15,17,20,24,25,26,27,29,30],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/bert.py":[4,6,8,10,11,13,14,15,16,17,20,57,58,59,60,61,65,67,68,69,71,77,78,80,81,82,83,84,86,91,93,98,135,136,137,138,139,142,144,145,146,148,154,155,156,157,158,159,160,161,162,164,169,171,176,215,216,217,218,219,222,224,225,226,228,234,235,236,237,239,251,253,258,300,301,302,303,306,308,311,313,319,320,321,322,323,324,326,343,345],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/biaffine_parser.py":[3,5,517,6,520,9,10,11,12,522,14,523,16,17,18,19,20,21,22,23,24,25,530,536,28,539,542,534,544,33,34,35,36,37,38,39,40,41,545,546,547,548,46,47,48,49,50,51,52,53,45,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,73,74,75,76,524,77,78,79,80,525,81,84,82,87,526,527,92,93,94,95,528,96,97,99,101,102,103,104,105,107,108,109,110,111,112,531,114,115,116,117,118,119,120,121,122,532,124,125,126,533,128,131,136,138,139,141,142,151,152,153,154,155,156,157,158,160,161,170,171,172,173,174,175,176,177,178,179,182,188,190,191,192,193,194,195,198,200,549,207,208,209,210,211,42,214,43,222,44,224,225,226,227,229,236,237,238,241,262,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,305,306,307,308,310,311,312,313,314,315,316,317,318,322,323,324,325,326,327,328,329,330,331,333,334,335,336,337,338,339,341,342,344,362,366,368,369,371,372,373,376,377,378,379,380,381,382,383,385,386,387,391,392,393,394,397,400,402,403,405,406,416,417,418,419,420,421,422,424,437,438,439,440,441,442,443,444,445,446,447,449,450,451,452,453,454,456,469,470,471,472,473,474,477,489,493,494,495,496,497,498,499,502],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/cnn_text_classification.py":[4,7,10,11,13,14,15,16,19,32,38,39,42,43,44,45,46,47,48,50,57,58,59,60,62,63,64,65,67,74,75,76],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/sequence_labeling.py":[3,5,6,10,11,12,14,15,16,17,18,19,20,21,22,25,39,41,61,75,78,82,93,95,96,98,99,100,101,102,104,112,113,114,116,118,120,122,124,132,134,136,138,140,141,143,151,152,153,154,155,156,158,159,160,161,162,163,165,170,171,174,189,191,193,195,196,197,198,199,200,201,202,203,204,206,207,213,218,219,221,229,230,231,232,233,234,236,237,238,239,240,241,243,252,253,254,257,259,263,264,267,269,270,271,272,273,274,275,277,279,287,289,296],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/snli.py":[4,6,9,10,11,12,14,15,16,17,20,32,35,36,38,41,42,43,44,45,48,49,50,51,52,54,57,58,59,60,61,63,65,66,68,77,78,79,80,81,82,83,87,89,90,91,92,94,95,99,100,101,102,104,105,107,113,115,116,117,121,122,123,124,126,127,128,129,130,131,134,136,137,138,139,142,143,144,145,146,147,148,149,151,153,154,155,156,158,160,162,165,167,168,169,174,177,178,179,182,183,184,185,186,187,189,190,193,194,195,196,197,198,199,202,204,205,208,209,211,213,214,215,216,217,218,220],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/models/star_transformer.py":[3,5,6,7,8,11,12,14,15,16,17,20,36,38,46,47,48,49,51,52,53,54,55,56,58,67,68,69,70,73,74,75,76,77,78,79,80,83,84,85,88,89,90,91,92,93,94,95,96,99,100,101,102,105,123,133,134,135,136,137,138,139,140,141,142,143,145,152,153,154,155,156,158,165,166,167,170,188,198,199,200,201,202,203,204,205,206,207,208,210,217,218,219,220,221,223,230,231,232,235,253,263,264,265,266,267,268,269,270,271,272,273,275,284,285,287,288,289,291,292,293,294,296,305,306,307],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/dist_trainer.py":[3,4,5,6,7,9,10,11,12,13,14,15,16,18,19,20,21,22,23,24,25,26,152,29,30,157,34,169,47,304,50,179,183,312,58,320,332,343,355,229],"/hdd/fudanNLP/fastNLP/fastNLP/fastNLP/core/predictor.py":[1,4,7,9,11,12,13,14,17,25,27,28,31,32,33,35,42,44,47,48,49,50,51,53,56,58,59,60,61,62,64,67,68,69,70,80,81]}} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 210d158a..bd7a34f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ install: - pip install pytest-cov # command to run tests script: - - pytest --cov=./ test/ + - pytest --cov=fastNLP test/ after_success: - bash <(curl -s https://codecov.io/bash) From 1756e3ffdf1ffa7ac4d296883fc5ebf4e3ad38c9 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Thu, 29 Aug 2019 11:16:59 +0800 Subject: [PATCH 17/92] =?UTF-8?q?1.=E4=BF=AE=E5=A4=8DMNLILoader=E4=B8=AD?= =?UTF-8?q?=E7=9A=84bug;=202.=E4=BF=AE=E5=A4=8Dfield=E4=B8=AD=E7=9A=84tens?= =?UTF-8?q?or=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/field.py | 6 +++--- fastNLP/core/vocabulary.py | 4 ++-- fastNLP/io/loader/matching.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fastNLP/core/field.py b/fastNLP/core/field.py index 05f987c2..859dfb1f 100644 --- a/fastNLP/core/field.py +++ b/fastNLP/core/field.py @@ -595,7 +595,7 @@ class AutoPadder(Padder): max_len = max(map(len, contents)) tensor = torch.full((len(contents), max_len), fill_value=self.pad_val, dtype=field_ele_dtype) for i, content_i in enumerate(contents): - tensor[i, :len(content_i)] = torch.tensor(content_i) + tensor[i, :len(content_i)] = content_i.clone().detach() elif dim == 2: max_len = max(map(len, contents)) max_word_len = max([max([len(content_ii) for content_ii in content_i]) for @@ -604,7 +604,7 @@ class AutoPadder(Padder): dtype=field_ele_dtype) for i, content_i in enumerate(contents): for j, content_ii in enumerate(content_i): - tensor[i, j, :len(content_ii)] = torch.tensor(content_ii) + tensor[i, j, :len(content_ii)] = content_ii.clone().detach() else: shapes = set([np.shape(content_i) for content_i in contents]) if len(shapes) > 1: @@ -615,7 +615,7 @@ class AutoPadder(Padder): tensor = torch.full([len(contents)] + list(shape), fill_value=self.pad_val, dtype=field_ele_dtype) for i, content_i in enumerate(contents): - tensor[i] = torch.tensor(content_i, dtype=field_ele_dtype) + tensor[i] = content_i.clone().detach().to(field_ele_dtype) else: raise RuntimeError( f"Field:{field_name} has 3 dimensions, every sample should have the same shape.") diff --git a/fastNLP/core/vocabulary.py b/fastNLP/core/vocabulary.py index 52d33a5a..cd4f2c0f 100644 --- a/fastNLP/core/vocabulary.py +++ b/fastNLP/core/vocabulary.py @@ -253,7 +253,7 @@ class Vocabulary(object): if self.unknown is not None: return self.word2idx[self.unknown] else: - raise ValueError("word {} not in vocabulary".format(w)) + raise ValueError("word `{}` not in vocabulary".format(w)) @_check_build_vocab def index_dataset(self, *datasets, field_name, new_field_name=None): @@ -360,7 +360,7 @@ class Vocabulary(object): try: dataset.apply(construct_vocab) except BaseException as e: - log("When processing the `{}` dataset, the following error occurred:".format(idx)) + logger.error("When processing the `{}` dataset, the following error occurred:".format(idx)) raise e else: raise TypeError("Only DataSet type is allowed.") diff --git a/fastNLP/io/loader/matching.py b/fastNLP/io/loader/matching.py index 7f03ca3e..a21d0845 100644 --- a/fastNLP/io/loader/matching.py +++ b/fastNLP/io/loader/matching.py @@ -41,7 +41,7 @@ class MNLILoader(Loader): ds = DataSet() with open(path, 'r', encoding='utf-8') as f: f.readline() # 跳过header - if path.endswith("test.tsv"): + if path.endswith("test_matched.tsv") or path.endswith('test_mismatched.tsv'): warnings.warn("RTE's test file has no target.") for line in f: line = line.strip() From 0908c736ebc1a2afb9c36c908391943b08a45e95 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Thu, 29 Aug 2019 16:38:17 +0800 Subject: [PATCH 18/92] fix code in BertModel.from_pretrained and BertEmbedding --- fastNLP/embeddings/bert_embedding.py | 20 +++++++------------- fastNLP/modules/encoder/bert.py | 12 +++++++----- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index e15c15f5..d1a5514a 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -17,8 +17,8 @@ import numpy as np from itertools import chain from ..core.vocabulary import Vocabulary -from ..io.file_utils import _get_embedding_url, cached_path, PRETRAINED_BERT_MODEL_DIR -from ..modules.encoder.bert import _WordPieceBertModel, BertModel, BertTokenizer, _get_bert_dir +from ..io.file_utils import PRETRAINED_BERT_MODEL_DIR +from ..modules.encoder.bert import _WordPieceBertModel, BertModel, BertTokenizer from .contextual_embedding import ContextualEmbedding import warnings from ..core import logger @@ -77,15 +77,12 @@ class BertEmbedding(ContextualEmbedding): " faster speed.") warnings.warn("For Chinese bert, pooled_method should choose from 'first', 'last' in order to achieve" " faster speed.") - - # 根据model_dir_or_name检查是否存在并下载 - model_dir = _get_bert_dir(model_dir_or_name) self._word_sep_index = None if '[SEP]' in vocab: self._word_sep_index = vocab['[SEP]'] - self.model = _WordBertModel(model_dir=model_dir, vocab=vocab, layers=layers, + self.model = _WordBertModel(model_dir_or_name=model_dir_or_name, vocab=vocab, layers=layers, pool_method=pool_method, include_cls_sep=include_cls_sep, pooled_cls=pooled_cls, auto_truncate=auto_truncate, min_freq=2) @@ -170,11 +167,8 @@ class BertWordPieceEncoder(nn.Module): def __init__(self, model_dir_or_name: str = 'en-base-uncased', layers: str = '-1', pooled_cls: bool = False, word_dropout=0, dropout=0, requires_grad: bool = False): super().__init__() - - # 根据model_dir_or_name检查是否存在并下载 - model_dir = _get_bert_dir(model_dir_or_name) - self.model = _WordPieceBertModel(model_dir=model_dir, layers=layers, pooled_cls=pooled_cls) + self.model = _WordPieceBertModel(model_dir_or_name=model_dir_or_name, layers=layers, pooled_cls=pooled_cls) self._sep_index = self.model._sep_index self._wordpiece_pad_index = self.model._wordpiece_pad_index self._wordpiece_unk_index = self.model._wordpiece_unknown_index @@ -269,12 +263,12 @@ class BertWordPieceEncoder(nn.Module): class _WordBertModel(nn.Module): - def __init__(self, model_dir: str, vocab: Vocabulary, layers: str = '-1', pool_method: str = 'first', + def __init__(self, model_dir_or_name: str, vocab: Vocabulary, layers: str = '-1', pool_method: str = 'first', include_cls_sep: bool = False, pooled_cls: bool = False, auto_truncate: bool = False, min_freq=2): super().__init__() - self.tokenzier = BertTokenizer.from_pretrained(model_dir) - self.encoder = BertModel.from_pretrained(model_dir) + self.tokenzier = BertTokenizer.from_pretrained(model_dir_or_name) + self.encoder = BertModel.from_pretrained(model_dir_or_name) self._max_position_embeddings = self.encoder.config.max_position_embeddings # 检查encoder_layer_number是否合理 encoder_layer_number = len(self.encoder.encoder.layer) diff --git a/fastNLP/modules/encoder/bert.py b/fastNLP/modules/encoder/bert.py index 89a1b09d..e73a8172 100644 --- a/fastNLP/modules/encoder/bert.py +++ b/fastNLP/modules/encoder/bert.py @@ -143,7 +143,7 @@ def _get_bert_dir(model_dir_or_name: str = 'en-base-uncased'): else: logger.error(f"Cannot recognize BERT dir or name ``{model_dir_or_name}``.") raise ValueError(f"Cannot recognize BERT dir or name ``{model_dir_or_name}``.") - return model_dir + return str(model_dir) class BertLayerNorm(nn.Module): @@ -453,6 +453,9 @@ class BertModel(nn.Module): if state_dict is None: weights_path = _get_file_name_base_on_postfix(pretrained_model_dir, '.bin') state_dict = torch.load(weights_path, map_location='cpu') + else: + logger.error(f'Cannot load parameters through `state_dict` variable.') + raise RuntimeError(f'Cannot load parameters through `state_dict` variable.') old_keys = [] new_keys = [] @@ -493,7 +496,7 @@ class BertModel(nn.Module): logger.warn("Weights from pretrained model not used in {}: {}".format( model.__class__.__name__, unexpected_keys)) - logger.info(f"Load pre-trained BERT parameters from dir {pretrained_model_dir}.") + logger.info(f"Load pre-trained BERT parameters from file {weights_path}.") return model @@ -854,9 +857,8 @@ class _WordPieceBertModel(nn.Module): def __init__(self, model_dir_or_name: str, layers: str = '-1', pooled_cls: bool=False): super().__init__() - self.model_dir = _get_bert_dir(model_dir_or_name) - self.tokenzier = BertTokenizer.from_pretrained(self.model_dir) - self.encoder = BertModel.from_pretrained(self.model_dir) + self.tokenzier = BertTokenizer.from_pretrained(model_dir_or_name) + self.encoder = BertModel.from_pretrained(model_dir_or_name) # 检查encoder_layer_number是否合理 encoder_layer_number = len(self.encoder.encoder.layer) self.layers = list(map(int, layers.split(','))) From 9e6f4ffb8bf29020e7871f06eef4f8e0d32e3774 Mon Sep 17 00:00:00 2001 From: lyhuang18 <42239874+lyhuang18@users.noreply.github.com> Date: Fri, 30 Aug 2019 01:21:59 +0800 Subject: [PATCH 19/92] =?UTF-8?q?datasetloader=E6=94=B9=E6=88=90pipe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../text_classification/train_awdlstm.py | 23 ++++++++--------- .../text_classification/train_lstm.py | 25 ++++++++----------- .../text_classification/train_lstm_att.py | 25 ++++++++----------- 3 files changed, 32 insertions(+), 41 deletions(-) diff --git a/reproduction/text_classification/train_awdlstm.py b/reproduction/text_classification/train_awdlstm.py index b2a67fdb..7537e6f7 100644 --- a/reproduction/text_classification/train_awdlstm.py +++ b/reproduction/text_classification/train_awdlstm.py @@ -1,11 +1,9 @@ # 这个模型需要在pytorch=0.4下运行，weight_drop不支持1.0 -# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 -import os -os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' -os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +import sys +sys.path.append('../..') -from fastNLP.io.data_loader import IMDBLoader +from fastNLP.io.pipe.classification import IMDBPipe from fastNLP.embeddings import StaticEmbedding from model.awd_lstm import AWDLSTMSentiment @@ -32,15 +30,14 @@ opt=Config() # load data -dataloader=IMDBLoader() -datainfo=dataloader.process(opt.datapath) +data_bundle=IMDBPipe.process_from_file(opt.datapath) -# print(datainfo.datasets["train"]) -# print(datainfo) +# print(data_bundle.datasets["train"]) +# print(data_bundle) # define model -vocab=datainfo.vocabs['words'] +vocab=data_bundle.vocabs['words'] embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) model=AWDLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc, wdrop=opt.wdrop) @@ -52,11 +49,11 @@ optimizer= Adam([param for param in model.parameters() if param.requires_grad==T def train(datainfo, model, optimizer, loss, metrics, opt): - trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, + trainer = Trainer(data_bundle.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=metrics, dev_data=data_bundle.datasets['test'], device=0, check_code_level=-1, n_epochs=opt.train_epoch, save_path=opt.save_model_path) trainer.train() if __name__ == "__main__": - train(datainfo, model, optimizer, loss, metrics, opt) + train(data_bundle, model, optimizer, loss, metrics, opt) diff --git a/reproduction/text_classification/train_lstm.py b/reproduction/text_classification/train_lstm.py index 40f77061..a23be0cb 100644 --- a/reproduction/text_classification/train_lstm.py +++ b/reproduction/text_classification/train_lstm.py @@ -1,9 +1,7 @@ -# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 -import os -os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' -os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +import sys +sys.path.append('../..') -from fastNLP.io.data_loader import IMDBLoader +from fastNLP.io.pipe.classification import IMDBPipe from fastNLP.embeddings import StaticEmbedding from model.lstm import BiLSTMSentiment @@ -29,15 +27,14 @@ opt=Config() # load data -dataloader=IMDBLoader() -datainfo=dataloader.process(opt.datapath) +data_bundle=IMDBPipe.process_from_file(opt.datapath) -# print(datainfo.datasets["train"]) -# print(datainfo) +# print(data_bundle.datasets["train"]) +# print(data_bundle) # define model -vocab=datainfo.vocabs['words'] +vocab=data_bundle.vocabs['words'] embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) model=BiLSTMSentiment(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, nfc=opt.nfc) @@ -48,12 +45,12 @@ metrics=AccuracyMetric() optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) -def train(datainfo, model, optimizer, loss, metrics, opt): - trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, +def train(data_bundle, model, optimizer, loss, metrics, opt): + trainer = Trainer(data_bundle.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=metrics, dev_data=data_bundle.datasets['test'], device=0, check_code_level=-1, n_epochs=opt.train_epoch, save_path=opt.save_model_path) trainer.train() if __name__ == "__main__": - train(datainfo, model, optimizer, loss, metrics, opt) \ No newline at end of file + train(data_bundle, model, optimizer, loss, metrics, opt) \ No newline at end of file diff --git a/reproduction/text_classification/train_lstm_att.py b/reproduction/text_classification/train_lstm_att.py index 1052f606..a2b8612d 100644 --- a/reproduction/text_classification/train_lstm_att.py +++ b/reproduction/text_classification/train_lstm_att.py @@ -1,9 +1,7 @@ -# 首先需要加入以下的路径到环境变量，因为当前只对内部测试开放，所以需要手动申明一下路径 -import os -os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' -os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +import sys +sys.path.append('../..') -from fastNLP.io.data_loader import IMDBLoader +from fastNLP.io.pipe.classification import IMDBPipe from fastNLP.embeddings import StaticEmbedding from model.lstm_self_attention import BiLSTM_SELF_ATTENTION @@ -31,15 +29,14 @@ opt=Config() # load data -dataloader=IMDBLoader() -datainfo=dataloader.process(opt.datapath) +data_bundle=IMDBPipe.process_from_file(opt.datapath) -# print(datainfo.datasets["train"]) -# print(datainfo) +# print(data_bundle.datasets["train"]) +# print(data_bundle) # define model -vocab=datainfo.vocabs['words'] +vocab=data_bundle.vocabs['words'] embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-840b-300', requires_grad=True) model=BiLSTM_SELF_ATTENTION(init_embed=embed, num_classes=opt.num_classes, hidden_dim=opt.hidden_dim, num_layers=opt.num_layers, attention_unit=opt.attention_unit, attention_hops=opt.attention_hops, nfc=opt.nfc) @@ -50,12 +47,12 @@ metrics=AccuracyMetric() optimizer= Adam([param for param in model.parameters() if param.requires_grad==True], lr=opt.lr) -def train(datainfo, model, optimizer, loss, metrics, opt): - trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, - metrics=metrics, dev_data=datainfo.datasets['test'], device=0, check_code_level=-1, +def train(data_bundle, model, optimizer, loss, metrics, opt): + trainer = Trainer(data_bundle.datasets['train'], model, optimizer=optimizer, loss=loss, + metrics=metrics, dev_data=data_bundle.datasets['test'], device=0, check_code_level=-1, n_epochs=opt.train_epoch, save_path=opt.save_model_path) trainer.train() if __name__ == "__main__": - train(datainfo, model, optimizer, loss, metrics, opt) + train(data_bundle, model, optimizer, loss, metrics, opt) From 9529f89abd41ee7ef0d9e2e32596ef9ee1aedb1e Mon Sep 17 00:00:00 2001 From: yh Date: Fri, 30 Aug 2019 19:54:28 +0800 Subject: [PATCH 20/92] =?UTF-8?q?=E5=A2=9E=E5=8A=A0DataBundle=E7=9A=84?= =?UTF-8?q?=E6=96=B9=E6=B3=95=EF=BC=9B=E5=A2=9E=E5=8A=A0BilSTMCRF=E7=9A=84?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/dataset.py | 14 ++-- fastNLP/io/data_bundle.py | 72 +++++++++++++++- fastNLP/models/sequence_labeling.py | 83 ++++++++----------- .../seqence_labelling/ner/train_ontonote.py | 4 +- 4 files changed, 112 insertions(+), 61 deletions(-) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 51bcef43..551cf1f8 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -575,18 +575,18 @@ class DataSet(object): """ return len(self) - def rename_field(self, old_name, new_name): + def rename_field(self, field_name, new_field_name): """ 将某个field重新命名. - :param str old_name: 原来的field名称。 - :param str new_name: 修改为new_name。 + :param str field_name: 原来的field名称。 + :param str new_field_name: 修改为new_name。 """ - if old_name in self.field_arrays: - self.field_arrays[new_name] = self.field_arrays.pop(old_name) - self.field_arrays[new_name].name = new_name + if field_name in self.field_arrays: + self.field_arrays[new_field_name] = self.field_arrays.pop(field_name) + self.field_arrays[new_field_name].name = new_field_name else: - raise KeyError("DataSet has no field named {}.".format(old_name)) + raise KeyError("DataSet has no field named {}.".format(field_name)) return self def set_target(self, *field_names, flag=True, use_1st_ins_infer_dim_type=True): diff --git a/fastNLP/io/data_bundle.py b/fastNLP/io/data_bundle.py index 969730a3..f30add34 100644 --- a/fastNLP/io/data_bundle.py +++ b/fastNLP/io/data_bundle.py @@ -139,9 +139,44 @@ class DataBundle: dataset.set_target(field_name, flag=flag, use_1st_ins_infer_dim_type=use_1st_ins_infer_dim_type) return self + def set_pad_val(self, field_name, pad_val, ignore_miss_dataset=True): + """ + 将DataBundle中所有的DataSet中名为field_name的Field的padding值设置为pad_val. + + :param str field_name: + :param int pad_val: + :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; + 如果为False，则报错 + :return: self + """ + for name, dataset in self.datasets.items(): + if dataset.has_field(field_name=field_name): + dataset.set_pad_val(field_name=field_name, pad_val=pad_val) + elif not ignore_miss_dataset: + raise KeyError(f"{field_name} not found DataSet:{name}.") + return self + + def set_ignore_type(self, *field_names, flag=True, ignore_miss_dataset=True): + """ + 将DataBundle中所有的DataSet中名为*field_names的Field的ignore_type设置为flag状态 + + :param str field_names: + :param bool flag: + :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; + 如果为False，则报错 + :return: self + """ + for name, dataset in self.datasets.items(): + for field_name in field_names: + if dataset.has_field(field_name=field_name): + dataset.set_ignore_type(field_name, flag=flag) + elif not ignore_miss_dataset: + raise KeyError(f"{field_name} not found DataSet:{name}.") + return self + def copy_field(self, field_name, new_field_name, ignore_miss_dataset=True): """ - 将DataBundle中所有的field_name复制一份叫new_field_name. + 将DataBundle中所有的DataSet中名为field_name的Field复制一份并命名为叫new_field_name. :param str field_name: :param str new_field_name: @@ -156,9 +191,42 @@ class DataBundle: raise KeyError(f"{field_name} not found DataSet:{name}.") return self + def rename_field(self, field_name, new_field_name, ignore_miss_dataset=True): + """ + 将DataBundle中所有DataSet中名为field_name的field重命名为new_field_name. + + :param str field_name: + :param str new_field_name: + :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; + 如果为False，则报错 + :return: self + """ + for name, dataset in self.datasets.items(): + if dataset.has_field(field_name=field_name): + dataset.rename_field(field_name=field_name, new_field_name=new_field_name) + elif not ignore_miss_dataset: + raise KeyError(f"{field_name} not found DataSet:{name}.") + return self + + def delete_field(self, field_name, ignore_miss_dataset=True): + """ + 将DataBundle中所有DataSet中名为field_name的field删除掉. + + :param str field_name: + :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; + 如果为False，则报错 + :return: self + """ + for name, dataset in self.datasets.items(): + if dataset.has_field(field_name=field_name): + dataset.delete_field(field_name=field_name) + elif not ignore_miss_dataset: + raise KeyError(f"{field_name} not found DataSet:{name}.") + return self + def apply_field(self, func, field_name:str, new_field_name:str, ignore_miss_dataset=True, **kwargs): """ - 对DataBundle中所有的dataset使用apply方法 + 对DataBundle中所有的dataset使用apply_field方法 :param callable func: input是instance中名为 `field_name` 的field的内容。 :param str field_name: 传入func的是哪个field。 diff --git a/fastNLP/models/sequence_labeling.py b/fastNLP/models/sequence_labeling.py index 0dff21f0..0c573a90 100644 --- a/fastNLP/models/sequence_labeling.py +++ b/fastNLP/models/sequence_labeling.py @@ -4,7 +4,7 @@ __all__ = [ "SeqLabeling", "AdvSeqLabel", - # "BiLSTMCRF" + "BiLSTMCRF" ] import torch @@ -14,7 +14,6 @@ import torch.nn.functional as F from .base_model import BaseModel from ..core.const import Const as C from ..core.utils import seq_len_to_mask -from ..embeddings import embedding from ..embeddings import get_embeddings from ..modules import ConditionalRandomField from ..modules import LSTM @@ -24,18 +23,15 @@ from ..modules.decoder.crf import allowed_transitions class BiLSTMCRF(BaseModel): """ - 结构为BiLSTM + FC + Dropout + CRF. + 结构为embedding + BiLSTM + FC + Dropout + CRF. - .. todo:: - 继续补充文档 - - :param embed: tuple: - :param num_classes: - :param num_layers: - :param hidden_size: - :param dropout: - :param target_vocab: - :param encoding_type: + :param embed: 支持(1)fastNLP的各种Embedding, (2) tuple, 指明num_embedding, dimension, 如(1000, 100) + :param num_classes: 一共多少个类 + :param num_layers: BiLSTM的层数 + :param hidden_size: BiLSTM的hidden_size，实际hidden size为该值的两倍(前向、后向) + :param dropout: dropout的概率，0为不dropout + :param target_vocab: Vocabulary对象，target与index的对应关系 + :param encoding_type: encoding的类型，支持'bioes', 'bmes', 'bio', 'bmeso'等 """ def __init__(self, embed, num_classes, num_layers=1, hidden_size=100, dropout=0.5, target_vocab=None, encoding_type=None): @@ -86,21 +82,20 @@ class SeqLabeling(BaseModel): 一个基础的Sequence labeling的模型。 用于做sequence labeling的基础类。结构包含一层Embedding，一层LSTM(单向，一层)，一层FC，以及一层CRF。 - :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray init_embed: Embedding的大小(传入tuple(int, int), - 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, Embedding, ndarray等则直接使用该值初始化Embedding + :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), + 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, embedding, ndarray等则直接使用该值初始化Embedding :param int hidden_size: LSTM隐藏层的大小 :param int num_classes: 一共有多少类 """ - def __init__(self, init_embed, hidden_size, num_classes): + def __init__(self, embed, hidden_size, num_classes): super(SeqLabeling, self).__init__() - self.Embedding = embedding.Embedding(init_embed) - self.Rnn = encoder.LSTM(self.Embedding.embedding_dim, hidden_size) - self.Linear = nn.Linear(hidden_size, num_classes) - self.Crf = decoder.ConditionalRandomField(num_classes) - self.mask = None - + self.embedding = get_embeddings(embed) + self.rnn = encoder.LSTM(self.embedding.embedding_dim, hidden_size) + self.fc = nn.Linear(hidden_size, num_classes) + self.crf = decoder.ConditionalRandomField(num_classes) + def forward(self, words, seq_len, target): """ :param torch.LongTensor words: [batch_size, max_len]，序列的index @@ -109,17 +104,14 @@ class SeqLabeling(BaseModel): :return y: If truth is None, return list of [decode path(list)]. Used in testing and predicting. If truth is not None, return loss, a scalar. Used in training. """ - assert words.shape[0] == seq_len.shape[0] - assert target.shape == words.shape - self.mask = self._make_mask(words, seq_len) - - x = self.Embedding(words) + mask = seq_len_to_mask(seq_len, max_len=words.size(1)) + x = self.embedding(words) # [batch_size, max_len, word_emb_dim] - x, _ = self.Rnn(x, seq_len) + x, _ = self.rnn(x, seq_len) # [batch_size, max_len, hidden_size * direction] - x = self.Linear(x) + x = self.fc(x) # [batch_size, max_len, num_classes] - return {C.LOSS: self._internal_loss(x, target)} + return {C.LOSS: self._internal_loss(x, target, mask)} def predict(self, words, seq_len): """ @@ -129,18 +121,18 @@ class SeqLabeling(BaseModel): :param torch.LongTensor seq_len: [batch_size,] :return: {'pred': xx}, [batch_size, max_len] """ - self.mask = self._make_mask(words, seq_len) + mask = seq_len_to_mask(seq_len, max_len=words.size(1)) - x = self.Embedding(words) + x = self.embedding(words) # [batch_size, max_len, word_emb_dim] - x, _ = self.Rnn(x, seq_len) + x, _ = self.rnn(x, seq_len) # [batch_size, max_len, hidden_size * direction] - x = self.Linear(x) + x = self.fc(x) # [batch_size, max_len, num_classes] - pred = self._decode(x) + pred = self._decode(x, mask) return {C.OUTPUT: pred} - def _internal_loss(self, x, y): + def _internal_loss(self, x, y, mask): """ Negative log likelihood loss. :param x: Tensor, [batch_size, max_len, tag_size] @@ -152,22 +144,15 @@ class SeqLabeling(BaseModel): y = y.long() assert x.shape[:2] == y.shape assert y.shape == self.mask.shape - total_loss = self.Crf(x, y, self.mask) + total_loss = self.crf(x, y, mask) return torch.mean(total_loss) - def _make_mask(self, x, seq_len): - batch_size, max_len = x.size(0), x.size(1) - mask = seq_len_to_mask(seq_len) - mask = mask.view(batch_size, max_len) - mask = mask.to(x).float() - return mask - - def _decode(self, x): + def _decode(self, x, mask): """ :param torch.FloatTensor x: [batch_size, max_len, tag_size] :return prediction: [batch_size, max_len] """ - tag_seq, _ = self.Crf.viterbi_decode(x, self.mask) + tag_seq, _ = self.crf.viterbi_decode(x, mask) return tag_seq @@ -177,7 +162,7 @@ class AdvSeqLabel(nn.Module): 更复杂的Sequence Labelling模型。结构为Embedding, LayerNorm, 双向LSTM(两层)，FC，LayerNorm，DropOut，FC，CRF。 - :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray init_embed: Embedding的大小(传入tuple(int, int), + :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, Embedding, ndarray等则直接使用该值初始化Embedding :param int hidden_size: LSTM的隐层大小 :param int num_classes: 有多少个类 @@ -188,11 +173,11 @@ class AdvSeqLabel(nn.Module): :param str encoding_type: 支持"BIO", "BMES", "BEMSO", 只有在id2words不为None的情况有用。 """ - def __init__(self, init_embed, hidden_size, num_classes, dropout=0.3, id2words=None, encoding_type='bmes'): + def __init__(self, embed, hidden_size, num_classes, dropout=0.3, id2words=None, encoding_type='bmes'): super().__init__() - self.Embedding = embedding.Embedding(init_embed) + self.Embedding = get_embeddings(embed) self.norm1 = torch.nn.LayerNorm(self.Embedding.embedding_dim) self.Rnn = encoder.LSTM(input_size=self.Embedding.embedding_dim, hidden_size=hidden_size, num_layers=2, dropout=dropout, diff --git a/reproduction/seqence_labelling/ner/train_ontonote.py b/reproduction/seqence_labelling/ner/train_ontonote.py index ee80b6f7..9fd13100 100644 --- a/reproduction/seqence_labelling/ner/train_ontonote.py +++ b/reproduction/seqence_labelling/ner/train_ontonote.py @@ -18,11 +18,9 @@ from fastNLP.io.pipe.conll import OntoNotesNERPipe #######hyper normalize = False -lower = False lr = 0.01 dropout = 0.5 batch_size = 32 -job_embed = False data_name = 'ontonote' #######hyper @@ -41,7 +39,7 @@ def cache(): word_dropout=0.01, dropout=dropout, lower=True, - min_freq=2) + min_freq=1) return data, char_embed, word_embed data, char_embed, word_embed = cache() From 82b5726686dcbac9f9a2032537f53c3eb77f7698 Mon Sep 17 00:00:00 2001 From: yunfan Date: Sat, 24 Aug 2019 13:59:30 +0800 Subject: [PATCH 21/92] update transformer --- fastNLP/modules/encoder/transformer.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index ce9172d5..70b82bde 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -32,9 +32,10 @@ class TransformerEncoder(nn.Module): self.norm1 = nn.LayerNorm(model_size) self.ffn = nn.Sequential(nn.Linear(model_size, inner_size), nn.ReLU(), - nn.Linear(inner_size, model_size), - TimestepDropout(dropout), ) + nn.Dropout(dropout), + nn.Linear(inner_size, model_size)) self.norm2 = nn.LayerNorm(model_size) + self.dropout = nn.Dropout(dropout) def forward(self, input, seq_mask=None, atte_mask_out=None): """ @@ -43,17 +44,20 @@ class TransformerEncoder(nn.Module): :param seq_mask: [batch, seq_len] :return: [batch, seq_len, model_size] """ + input = self.norm1(input) attention = self.atte(input, input, input, atte_mask_out) - norm_atte = self.norm1(attention + input) - attention *= seq_mask - output = self.ffn(norm_atte) - output = self.norm2(output + norm_atte) - output *= seq_mask + input = input + self.dropout(attention) + # attention *= seq_mask + input = self.norm2(input) + output = self.ffn(input) + input = input + self.dropout(output) + # output *= seq_mask return output def __init__(self, num_layers, **kargs): super(TransformerEncoder, self).__init__() self.layers = nn.ModuleList([self.SubLayer(**kargs) for _ in range(num_layers)]) + self.norm = nn.LayerNorm(kargs['model_size']) def forward(self, x, seq_mask=None): """ @@ -70,4 +74,4 @@ class TransformerEncoder(nn.Module): seq_mask = seq_mask[:, :, None] for layer in self.layers: output = layer(output, seq_mask, atte_mask_out) - return output + return self.norm(output) From 44af647839fe99f69b9364457ff3636df6367204 Mon Sep 17 00:00:00 2001 From: yunfan Date: Thu, 29 Aug 2019 20:19:13 +0800 Subject: [PATCH 22/92] [update] change data-loader to pipe --- .../text_classification/train_dpcnn.py | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/reproduction/text_classification/train_dpcnn.py b/reproduction/text_classification/train_dpcnn.py index f3f4e231..c7f5751c 100644 --- a/reproduction/text_classification/train_dpcnn.py +++ b/reproduction/text_classification/train_dpcnn.py @@ -8,21 +8,18 @@ from fastNLP.core.trainer import Trainer from fastNLP import CrossEntropyLoss, AccuracyMetric from fastNLP.embeddings import StaticEmbedding from reproduction.text_classification.model.dpcnn import DPCNN -from fastNLP.io.data_loader import YelpLoader from fastNLP.core.sampler import BucketSampler from fastNLP.core import LRScheduler from fastNLP.core.const import Const as C from fastNLP.core.vocabulary import VocabularyOption -from fastNLP.core.dist_trainer import DistTrainer from utils.util_init import set_rng_seeds from fastNLP import logger import os -# os.environ['FASTNLP_BASE_URL'] = 'http://10.141.222.118:8888/file/download/' -# os.environ['FASTNLP_CACHE_DIR'] = '/remote-home/hyan01/fastnlp_caches' +from fastNLP.io import YelpFullPipe, YelpPolarityPipe + os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # hyper logger.add_file('log', 'INFO') -print(logger.handlers) class Config(): seed = 12345 @@ -50,18 +47,14 @@ class Config(): ops = Config() set_rng_seeds(ops.seed) -# print('RNG SEED: {}'.format(ops.seed)) logger.info('RNG SEED %d'%ops.seed) # 1.task相关信息：利用dataloader载入dataInfo -#datainfo=SSTLoader(fine_grained=True).process(paths=ops.datapath, train_ds=['train']) - @cache_results(ops.model_dir_or_name+'-data-cache') def load_data(): - datainfo = YelpLoader(fine_grained=True, lower=True).process( - paths=ops.datapath, train_ds=['train'], src_vocab_op=ops.src_vocab_op) + datainfo = YelpFullPipe(lower=True, tokenizer='raw').process_from_file(ops.datapath) for ds in datainfo.datasets.values(): ds.apply_field(len, C.INPUT, C.INPUT_LEN) ds.set_input(C.INPUT, C.INPUT_LEN) @@ -79,11 +72,8 @@ print(embedding.embedding.weight.data.mean(), embedding.embedding.weight.data.st # 2.或直接复用fastNLP的模型 -# embedding = StackEmbedding([StaticEmbedding(vocab), CNNCharEmbedding(vocab, 100)]) -datainfo.datasets['train'] = datainfo.datasets['train'][:1000] -datainfo.datasets['test'] = datainfo.datasets['test'][:1000] -# print(datainfo) -# print(datainfo.datasets['train'][0]) +# datainfo.datasets['train'] = datainfo.datasets['train'][:1000] # for debug purpose +# datainfo.datasets['test'] = datainfo.datasets['test'][:1000] logger.info(datainfo) model = DPCNN(init_embed=embedding, num_cls=len(datainfo.vocabs[C.TARGET]), @@ -99,14 +89,7 @@ optimizer = SGD([param for param in model.parameters() if param.requires_grad == callbacks = [] callbacks.append(LRScheduler(CosineAnnealingLR(optimizer, 5))) -# callbacks.append( -# LRScheduler(LambdaLR(optimizer, lambda epoch: ops.lr if epoch < -# ops.train_epoch * 0.8 else ops.lr * 0.1)) -# ) -# callbacks.append( -# FitlogCallback(data=datainfo.datasets, verbose=1) -# ) device = 'cuda:0' if torch.cuda.is_available() else 'cpu' @@ -114,12 +97,15 @@ device = 'cuda:0' if torch.cuda.is_available() else 'cpu' logger.info(device) # 4.定义train方法 +# normal trainer trainer = Trainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, sampler=BucketSampler(num_buckets=50, batch_size=ops.batch_size), metrics=[metric], use_tqdm=False, save_path='save', dev_data=datainfo.datasets['test'], device=device, check_code_level=-1, batch_size=ops.batch_size, callbacks=callbacks, n_epochs=ops.train_epoch, num_workers=4) + +# distributed trainer # trainer = DistTrainer(datainfo.datasets['train'], model, optimizer=optimizer, loss=loss, # metrics=[metric], # dev_data=datainfo.datasets['test'], device='cuda', From bbda73c14f2352583f1a89bafdd1ff7471543cc4 Mon Sep 17 00:00:00 2001 From: yunfan Date: Fri, 30 Aug 2019 21:48:00 +0800 Subject: [PATCH 23/92] [update] transformer --- fastNLP/modules/encoder/attention.py | 39 +++++++++++--------------- fastNLP/modules/encoder/transformer.py | 17 ++++++----- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/fastNLP/modules/encoder/attention.py b/fastNLP/modules/encoder/attention.py index 02bd078a..6a973864 100644 --- a/fastNLP/modules/encoder/attention.py +++ b/fastNLP/modules/encoder/attention.py @@ -30,14 +30,14 @@ class DotAttention(nn.Module): def forward(self, Q, K, V, mask_out=None): """ - :param Q: [batch, seq_len_q, key_size] - :param K: [batch, seq_len_k, key_size] - :param V: [batch, seq_len_k, value_size] - :param mask_out: [batch, 1, seq_len] or [batch, seq_len_q, seq_len_k] + :param Q: [..., seq_len_q, key_size] + :param K: [..., seq_len_k, key_size] + :param V: [..., seq_len_k, value_size] + :param mask_out: [..., 1, seq_len] or [..., seq_len_q, seq_len_k] """ - output = torch.matmul(Q, K.transpose(1, 2)) / self.scale + output = torch.matmul(Q, K.transpose(-1, -2)) / self.scale if mask_out is not None: - output.masked_fill_(mask_out, -1e18) + output.masked_fill_(mask_out, -1e9) output = self.softmax(output) output = self.drop(output) return torch.matmul(output, V) @@ -65,17 +65,16 @@ class MultiHeadAttention(nn.Module): self.q_in = nn.Linear(input_size, in_size) self.k_in = nn.Linear(input_size, in_size) self.v_in = nn.Linear(input_size, in_size) - # follow the paper, do not apply dropout within dot-product self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=dropout) self.out = nn.Linear(value_size * num_head, input_size) self.reset_parameters() def reset_parameters(self): sqrt = math.sqrt - nn.init.normal_(self.q_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.key_size))) - nn.init.normal_(self.k_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.key_size))) - nn.init.normal_(self.v_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.value_size))) - nn.init.xavier_normal_(self.out.weight) + nn.init.normal_(self.q_in.weight, mean=0, std=sqrt(1.0 / self.input_size)) + nn.init.normal_(self.k_in.weight, mean=0, std=sqrt(1.0 / self.input_size)) + nn.init.normal_(self.v_in.weight, mean=0, std=sqrt(1.0 / self.input_size)) + nn.init.normal_(self.out.weight, mean=0, std=sqrt(1.0 / self.input_size)) def forward(self, Q, K, V, atte_mask_out=None): """ @@ -89,20 +88,16 @@ class MultiHeadAttention(nn.Module): sk = K.size(1) d_k, d_v, n_head = self.key_size, self.value_size, self.num_head # input linear - q = self.q_in(Q).view(batch, sq, n_head, d_k) - k = self.k_in(K).view(batch, sk, n_head, d_k) - v = self.v_in(V).view(batch, sk, n_head, d_v) - - # transpose q, k and v to do batch attention - q = q.permute(2, 0, 1, 3).contiguous().view(-1, sq, d_k) - k = k.permute(2, 0, 1, 3).contiguous().view(-1, sk, d_k) - v = v.permute(2, 0, 1, 3).contiguous().view(-1, sk, d_v) + q = self.q_in(Q).view(batch, sq, n_head, d_k).transpose(1, 2) + k = self.k_in(K).view(batch, sk, n_head, d_k).transpose(1, 2) + v = self.v_in(V).view(batch, sk, n_head, d_v).transpose(1, 2) + if atte_mask_out is not None: - atte_mask_out = atte_mask_out.repeat(n_head, 1, 1) - atte = self.attention(q, k, v, atte_mask_out).view(n_head, batch, sq, d_v) + atte_mask_out = atte_mask_out[:,None,:,:] # [bsz,1,1,len] + atte = self.attention(q, k, v, atte_mask_out).view(batch, n_head, sq, d_v) # concat all heads, do output linear - atte = atte.permute(1, 2, 0, 3).contiguous().view(batch, sq, -1) + atte = atte.transpose(1, 2).contiguous().view(batch, sq, -1) output = self.out(atte) return output diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index 70b82bde..d8a612a0 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -5,8 +5,7 @@ __all__ = [ ] from torch import nn -from fastNLP.modules.encoder.attention import MultiHeadAttention -from ..dropout import TimestepDropout +from .attention import MultiHeadAttention class TransformerEncoder(nn.Module): @@ -29,12 +28,12 @@ class TransformerEncoder(nn.Module): def __init__(self, model_size, inner_size, key_size, value_size, num_head, dropout=0.1): super(TransformerEncoder.SubLayer, self).__init__() self.atte = MultiHeadAttention(model_size, key_size, value_size, num_head, dropout) - self.norm1 = nn.LayerNorm(model_size) + self.norm1 = nn.LayerNorm(model_size, eps=1e-6) self.ffn = nn.Sequential(nn.Linear(model_size, inner_size), nn.ReLU(), nn.Dropout(dropout), nn.Linear(inner_size, model_size)) - self.norm2 = nn.LayerNorm(model_size) + self.norm2 = nn.LayerNorm(model_size, eps=1e-6) self.dropout = nn.Dropout(dropout) def forward(self, input, seq_mask=None, atte_mask_out=None): @@ -47,17 +46,17 @@ class TransformerEncoder(nn.Module): input = self.norm1(input) attention = self.atte(input, input, input, atte_mask_out) input = input + self.dropout(attention) - # attention *= seq_mask + attention *= seq_mask input = self.norm2(input) output = self.ffn(input) input = input + self.dropout(output) - # output *= seq_mask - return output + input *= seq_mask + return input def __init__(self, num_layers, **kargs): super(TransformerEncoder, self).__init__() self.layers = nn.ModuleList([self.SubLayer(**kargs) for _ in range(num_layers)]) - self.norm = nn.LayerNorm(kargs['model_size']) + self.norm = nn.LayerNorm(kargs['model_size'], eps=1e-6) def forward(self, x, seq_mask=None): """ @@ -70,7 +69,7 @@ class TransformerEncoder(nn.Module): if seq_mask is None: atte_mask_out = None else: - atte_mask_out = (seq_mask < 1)[:, None, :] + atte_mask_out = (seq_mask == 0)[:, None, :] seq_mask = seq_mask[:, :, None] for layer in self.layers: output = layer(output, seq_mask, atte_mask_out) From 4440801dbfc9bea20a86be6ceeb1431f5d020681 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Sun, 1 Sep 2019 01:19:10 +0800 Subject: [PATCH 24/92] 1. update bert.py and fix a bug in bert_embedding to adapt torch 1.2.0; 2. update models/bert.py and add BertForSentenceMatching model, now a BertEmbedding param should be passed to these five models; 3. create a small bert version for testing and modify test/models/test_bert.py; 4. move small glove and word2vec files to data_for_tests/embedding/small_static_embedding dir and fix relevant test codes; 5. delete some __init__.py files in test dir. --- fastNLP/embeddings/bert_embedding.py | 2 +- fastNLP/models/bert.py | 373 ++++++------------ fastNLP/modules/encoder/bert.py | 4 +- test/__init__.py | 3 - .../embedding/small_bert/config.json | 13 + .../small_bert/small_pytorch_model.bin | Bin 0 -> 37965 bytes .../embedding/small_bert/vocab.txt | 20 + .../glove.6B.50d_test.txt | 0 .../small_static_embedding}/word2vec_test.txt | 0 test/embeddings/__init__.py | 0 test/embeddings/test_bert_embedding.py | 11 +- test/embeddings/test_static_embedding.py | 6 +- test/io/test_embed_loader.py | 8 +- test/models/__init__.py | 0 test/models/test_bert.py | 86 ++-- test/modules/__init__.py | 0 test/modules/decoder/__init__.py | 0 17 files changed, 225 insertions(+), 301 deletions(-) delete mode 100644 test/__init__.py create mode 100644 test/data_for_tests/embedding/small_bert/config.json create mode 100644 test/data_for_tests/embedding/small_bert/small_pytorch_model.bin create mode 100644 test/data_for_tests/embedding/small_bert/vocab.txt rename test/data_for_tests/{ => embedding/small_static_embedding}/glove.6B.50d_test.txt (100%) rename test/data_for_tests/{ => embedding/small_static_embedding}/word2vec_test.txt (100%) delete mode 100644 test/embeddings/__init__.py delete mode 100644 test/models/__init__.py delete mode 100644 test/modules/__init__.py delete mode 100644 test/modules/decoder/__init__.py diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index d1a5514a..f6c36623 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -393,7 +393,7 @@ class _WordBertModel(nn.Module): batch_indexes = torch.arange(batch_size).to(words) word_pieces[batch_indexes, word_pieces_lengths + 1] = self._sep_index if self._has_sep_in_vocab: # 但[SEP]在vocab中出现应该才会需要token_ids - sep_mask = word_pieces.eq(self._sep_index) # batch_size x max_len + sep_mask = word_pieces.eq(self._sep_index).long() # batch_size x max_len sep_mask_cumsum = sep_mask.flip(dims=[-1]).cumsum(dim=-1).flip(dims=[-1]) token_type_ids = sep_mask_cumsum.fmod(2) if token_type_ids[0, 0].item(): # 如果开头是奇数，则需要flip一下结果，因为需要保证开头为0 diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py index 0a89b765..08f16db2 100644 --- a/fastNLP/models/bert.py +++ b/fastNLP/models/bert.py @@ -5,253 +5,145 @@ bert.py is modified from huggingface/pytorch-pretrained-BERT, which is licensed __all__ = [] -import os +import warnings import torch from torch import nn from .base_model import BaseModel from ..core.const import Const -from ..core.utils import seq_len_to_mask +from ..core._logger import logger from ..modules.encoder import BertModel from ..modules.encoder.bert import BertConfig, CONFIG_FILE +from ..embeddings.bert_embedding import BertEmbedding class BertForSequenceClassification(BaseModel): """BERT model for classification. - This module is composed of the BERT model with a linear layer on top of - the pooled output. - Params: - `config`: a BertConfig class instance with the configuration to build a new model. - `num_labels`: the number of classes for the classifier. Default = 2. - Inputs: - `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] - with the word token indices in the vocabulary. Items in the batch should begin with the special "CLS" token. (see the tokens preprocessing logic in the scripts - `extract_features.py`, `run_classifier.py` and `run_squad.py`) - `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token - types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to - a `sentence B` token (see BERT paper for more details). - `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices - selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max - input sequence length in the current batch. It's the mask that we typically use for attention when - a batch has varying length sentences. - `labels`: labels for the classification output: torch.LongTensor of shape [batch_size] - with indices selected in [0, ..., num_labels]. - Outputs: - if `labels` is not `None`: - Outputs the CrossEntropy classification loss of the output with the labels. - if `labels` is `None`: - Outputs the classification logits of shape [batch_size, num_labels]. - Example usage: - ```python - # Already been converted into WordPiece token ids - input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) - input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) - token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) - config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, - num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) - num_labels = 2 - model = BertForSequenceClassification(num_labels, config) - logits = model(input_ids, token_type_ids, input_mask) - ``` """ - def __init__(self, num_labels, config=None, bert_dir=None): + def __init__(self, init_embed: BertEmbedding, num_labels: int=2): super(BertForSequenceClassification, self).__init__() + self.num_labels = num_labels - if bert_dir is not None: - self.bert = BertModel.from_pretrained(bert_dir) - config = BertConfig(os.path.join(bert_dir, CONFIG_FILE)) - else: - if config is None: - config = BertConfig(30522) - self.bert = BertModel(config) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - self.classifier = nn.Linear(config.hidden_size, num_labels) - - @classmethod - def from_pretrained(cls, num_labels, pretrained_model_dir): - config = BertConfig(pretrained_model_dir) - model = cls(num_labels=num_labels, config=config, bert_dir=pretrained_model_dir) - return model - - def forward(self, words, seq_len=None, target=None): - if seq_len is None: - seq_len = torch.ones_like(words, dtype=words.dtype, device=words.device) - if len(seq_len.size()) + 1 == len(words.size()): - seq_len = seq_len_to_mask(seq_len, max_len=words.size(-1)) - _, pooled_output = self.bert(words, attention_mask=seq_len, output_all_encoded_layers=False) - pooled_output = self.dropout(pooled_output) - logits = self.classifier(pooled_output) + self.bert = init_embed + self.dropout = nn.Dropout(0.1) + self.classifier = nn.Linear(self.bert.embedding_dim, num_labels) + + if not self.bert.model.include_cls_sep: + warn_msg = "Bert for sequence classification excepts BertEmbedding `include_cls_sep` True, but got False." + logger.warn(warn_msg) + warnings.warn(warn_msg) + + def forward(self, words): + hidden = self.dropout(self.bert(words)) + cls_hidden = hidden[:, 0] + logits = self.classifier(cls_hidden) + + return {Const.OUTPUT: logits} + + def predict(self, words): + logits = self.forward(words)[Const.OUTPUT] + return {Const.OUTPUT: torch.argmax(logits, dim=-1)} + + +class BertForSentenceMatching(BaseModel): + + """BERT model for matching. + """ + def __init__(self, init_embed: BertEmbedding, num_labels: int=2): + super(BertForSentenceMatching, self).__init__() + self.num_labels = num_labels + self.bert = init_embed + self.dropout = nn.Dropout(0.1) + self.classifier = nn.Linear(self.bert.embedding_dim, num_labels) + + if not self.bert.model.include_cls_sep: + error_msg = "Bert for sentence matching excepts BertEmbedding `include_cls_sep` True, but got False." + logger.error(error_msg) + raise RuntimeError(error_msg) - if target is not None: - loss_fct = nn.CrossEntropyLoss() - loss = loss_fct(logits, target) - return {Const.OUTPUT: logits, Const.LOSS: loss} - else: - return {Const.OUTPUT: logits} + def forward(self, words): + hidden = self.dropout(self.bert(words)) + cls_hidden = hidden[:, 0] + logits = self.classifier(cls_hidden) - def predict(self, words, seq_len=None): - logits = self.forward(words, seq_len=seq_len)[Const.OUTPUT] + return {Const.OUTPUT: logits} + + def predict(self, words): + logits = self.forward(words)[Const.OUTPUT] return {Const.OUTPUT: torch.argmax(logits, dim=-1)} class BertForMultipleChoice(BaseModel): """BERT model for multiple choice tasks. - This module is composed of the BERT model with a linear layer on top of - the pooled output. - Params: - `config`: a BertConfig class instance with the configuration to build a new model. - `num_choices`: the number of classes for the classifier. Default = 2. - Inputs: - `input_ids`: a torch.LongTensor of shape [batch_size, num_choices, sequence_length] - with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts - `extract_features.py`, `run_classifier.py` and `run_squad.py`) - `token_type_ids`: an optional torch.LongTensor of shape [batch_size, num_choices, sequence_length] - with the token types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` - and type 1 corresponds to a `sentence B` token (see BERT paper for more details). - `attention_mask`: an optional torch.LongTensor of shape [batch_size, num_choices, sequence_length] with indices - selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max - input sequence length in the current batch. It's the mask that we typically use for attention when - a batch has varying length sentences. - `labels`: labels for the classification output: torch.LongTensor of shape [batch_size] - with indices selected in [0, ..., num_choices]. - Outputs: - if `labels` is not `None`: - Outputs the CrossEntropy classification loss of the output with the labels. - if `labels` is `None`: - Outputs the classification logits of shape [batch_size, num_labels]. - Example usage: - ```python - # Already been converted into WordPiece token ids - input_ids = torch.LongTensor([[[31, 51, 99], [15, 5, 0]], [[12, 16, 42], [14, 28, 57]]]) - input_mask = torch.LongTensor([[[1, 1, 1], [1, 1, 0]],[[1,1,0], [1, 0, 0]]]) - token_type_ids = torch.LongTensor([[[0, 0, 1], [0, 1, 0]],[[0, 1, 1], [0, 0, 1]]]) - config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, - num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) - num_choices = 2 - model = BertForMultipleChoice(num_choices, config, bert_dir) - logits = model(input_ids, token_type_ids, input_mask) - ``` """ - def __init__(self, num_choices, config=None, bert_dir=None): + def __init__(self, init_embed: BertEmbedding, num_choices=2): super(BertForMultipleChoice, self).__init__() + self.num_choices = num_choices - if bert_dir is not None: - self.bert = BertModel.from_pretrained(bert_dir) - else: - if config is None: - config = BertConfig(30522) - self.bert = BertModel(config) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - self.classifier = nn.Linear(config.hidden_size, 1) - - @classmethod - def from_pretrained(cls, num_choices, pretrained_model_dir): - config = BertConfig(pretrained_model_dir) - model = cls(num_choices=num_choices, config=config, bert_dir=pretrained_model_dir) - return model - - def forward(self, words, seq_len1=None, seq_len2=None, target=None): - input_ids, token_type_ids, attention_mask = words, seq_len1, seq_len2 - flat_input_ids = input_ids.view(-1, input_ids.size(-1)) - flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) - flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) - _, pooled_output = self.bert(flat_input_ids, flat_token_type_ids, flat_attention_mask, output_all_encoded_layers=False) + self.bert = init_embed + self.dropout = nn.Dropout(0.1) + self.classifier = nn.Linear(self.bert.embedding_dim, 1) + self.include_cls_sep = init_embed.model.include_cls_sep + + if not self.bert.model.include_cls_sep: + error_msg = "Bert for multiple choice excepts BertEmbedding `include_cls_sep` True, but got False." + logger.error(error_msg) + raise RuntimeError(error_msg) + + def forward(self, words): + """ + :param torch.Tensor words: [batch_size, num_choices, seq_len] + :return: [batch_size, num_labels] + """ + batch_size, num_choices, seq_len = words.size() + + input_ids = words.view(batch_size * num_choices, seq_len) + hidden = self.bert(input_ids) + pooled_output = hidden[:, 0] pooled_output = self.dropout(pooled_output) logits = self.classifier(pooled_output) reshaped_logits = logits.view(-1, self.num_choices) - if target is not None: - loss_fct = nn.CrossEntropyLoss() - loss = loss_fct(reshaped_logits, target) - return {Const.OUTPUT: reshaped_logits, Const.LOSS: loss} - else: - return {Const.OUTPUT: reshaped_logits} + return {Const.OUTPUT: reshaped_logits} - def predict(self, words, seq_len1=None, seq_len2=None,): - logits = self.forward(words, seq_len1=seq_len1, seq_len2=seq_len2)[Const.OUTPUT] + def predict(self, words): + logits = self.forward(words)[Const.OUTPUT] return {Const.OUTPUT: torch.argmax(logits, dim=-1)} class BertForTokenClassification(BaseModel): """BERT model for token-level classification. - This module is composed of the BERT model with a linear layer on top of - the full hidden state of the last layer. - Params: - `config`: a BertConfig class instance with the configuration to build a new model. - `num_labels`: the number of classes for the classifier. Default = 2. - `bert_dir`: a dir which contains the bert parameters within file `pytorch_model.bin` - Inputs: - `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] - with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts - `extract_features.py`, `run_classifier.py` and `run_squad.py`) - `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token - types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to - a `sentence B` token (see BERT paper for more details). - `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices - selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max - input sequence length in the current batch. It's the mask that we typically use for attention when - a batch has varying length sentences. - `labels`: labels for the classification output: torch.LongTensor of shape [batch_size, sequence_length] - with indices selected in [0, ..., num_labels]. - Outputs: - if `labels` is not `None`: - Outputs the CrossEntropy classification loss of the output with the labels. - if `labels` is `None`: - Outputs the classification logits of shape [batch_size, sequence_length, num_labels]. - Example usage: - ```python - # Already been converted into WordPiece token ids - input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) - input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) - token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) - config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, - num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) - num_labels = 2 - bert_dir = 'your-bert-file-dir' - model = BertForTokenClassification(num_labels, config, bert_dir) - logits = model(input_ids, token_type_ids, input_mask) - ``` """ - def __init__(self, num_labels, config=None, bert_dir=None): + def __init__(self, init_embed: BertEmbedding, num_labels): super(BertForTokenClassification, self).__init__() + self.num_labels = num_labels - if bert_dir is not None: - self.bert = BertModel.from_pretrained(bert_dir) - else: - if config is None: - config = BertConfig(30522) - self.bert = BertModel(config) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - self.classifier = nn.Linear(config.hidden_size, num_labels) - - @classmethod - def from_pretrained(cls, num_labels, pretrained_model_dir): - config = BertConfig(pretrained_model_dir) - model = cls(num_labels=num_labels, config=config, bert_dir=pretrained_model_dir) - return model - - def forward(self, words, seq_len1=None, seq_len2=None, target=None): - sequence_output, _ = self.bert(words, seq_len1, seq_len2, output_all_encoded_layers=False) + self.bert = init_embed + self.dropout = nn.Dropout(0.1) + self.classifier = nn.Linear(self.bert.embedding_dim, num_labels) + self.include_cls_sep = init_embed.model.include_cls_sep + + if self.include_cls_sep: + warn_msg = "Bert for token classification excepts BertEmbedding `include_cls_sep` False, but got True." + warnings.warn(warn_msg) + logger.warn(warn_msg) + + def forward(self, words): + """ + :param torch.Tensor words: [batch_size, seq_len] + :return: [batch_size, seq_len, num_labels] + """ + sequence_output = self.bert(words) + if self.include_cls_sep: + sequence_output = sequence_output[:, 1: -1] # [batch_size, seq_len, embed_dim] sequence_output = self.dropout(sequence_output) logits = self.classifier(sequence_output) - if target is not None: - loss_fct = nn.CrossEntropyLoss() - # Only keep active parts of the loss - if seq_len2 is not None: - active_loss = seq_len2.view(-1) == 1 - active_logits = logits.view(-1, self.num_labels)[active_loss] - active_labels = target.view(-1)[active_loss] - loss = loss_fct(active_logits, active_labels) - else: - loss = loss_fct(logits.view(-1, self.num_labels), target.view(-1)) - return {Const.OUTPUT: logits, Const.LOSS: loss} - else: - return {Const.OUTPUT: logits} - - def predict(self, words, seq_len1=None, seq_len2=None): - logits = self.forward(words, seq_len1, seq_len2)[Const.OUTPUT] + return {Const.OUTPUT: logits} + + def predict(self, words): + logits = self.forward(words)[Const.OUTPUT] return {Const.OUTPUT: torch.argmax(logits, dim=-1)} @@ -298,53 +190,24 @@ class BertForQuestionAnswering(BaseModel): start_logits, end_logits = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config=None, bert_dir=None): + def __init__(self, init_embed: BertEmbedding, num_labels=2): super(BertForQuestionAnswering, self).__init__() - if bert_dir is not None: - self.bert = BertModel.from_pretrained(bert_dir) - else: - if config is None: - config = BertConfig(30522) - self.bert = BertModel(config) - # TODO check with Google if it's normal there is no dropout on the token classifier of SQuAD in the TF version - # self.dropout = nn.Dropout(config.hidden_dropout_prob) - self.qa_outputs = nn.Linear(config.hidden_size, 2) - - @classmethod - def from_pretrained(cls, pretrained_model_dir): - config = BertConfig(pretrained_model_dir) - model = cls(config=config, bert_dir=pretrained_model_dir) - return model - - def forward(self, words, seq_len1=None, seq_len2=None, target1=None, target2=None): - sequence_output, _ = self.bert(words, seq_len1, seq_len2, output_all_encoded_layers=False) - logits = self.qa_outputs(sequence_output) - start_logits, end_logits = logits.split(1, dim=-1) - start_logits = start_logits.squeeze(-1) - end_logits = end_logits.squeeze(-1) - - if target1 is not None and target2 is not None: - # If we are on multi-GPU, split add a dimension - if len(target1.size()) > 1: - target1 = target1.squeeze(-1) - if len(target2.size()) > 1: - target2 = target2.squeeze(-1) - # sometimes the start/end positions are outside our model inputs, we ignore these terms - ignored_index = start_logits.size(1) - target1.clamp_(0, ignored_index) - target2.clamp_(0, ignored_index) - - loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index) - start_loss = loss_fct(start_logits, target1) - end_loss = loss_fct(end_logits, target2) - total_loss = (start_loss + end_loss) / 2 - return {Const.OUTPUTS(0): start_logits, Const.OUTPUTS(1): end_logits, Const.LOSS: total_loss} - else: - return {Const.OUTPUTS(0): start_logits, Const.OUTPUTS(1): end_logits} - - def predict(self, words, seq_len1=None, seq_len2=None): - logits = self.forward(words, seq_len1, seq_len2) - start_logits = logits[Const.OUTPUTS(0)] - end_logits = logits[Const.OUTPUTS(1)] - return {Const.OUTPUTS(0): torch.argmax(start_logits, dim=-1), - Const.OUTPUTS(1): torch.argmax(end_logits, dim=-1)} + + self.bert = init_embed + self.num_labels = num_labels + self.qa_outputs = nn.Linear(self.bert.embedding_dim, self.num_labels) + + if not self.bert.model.include_cls_sep: + error_msg = "Bert for multiple choice excepts BertEmbedding `include_cls_sep` True, but got False." + logger.error(error_msg) + raise RuntimeError(error_msg) + + def forward(self, words): + sequence_output = self.bert(words) + logits = self.qa_outputs(sequence_output) # [batch_size, seq_len, num_labels] + + return {Const.OUTPUTS(i): logits[:, :, i] for i in range(self.num_labels)} + + def predict(self, words): + logits = self.forward(words) + return {Const.OUTPUTS(i): torch.argmax(logits[Const.OUTPUTS(i)], dim=-1) for i in range(self.num_labels)} diff --git a/fastNLP/modules/encoder/bert.py b/fastNLP/modules/encoder/bert.py index e73a8172..6f6c4291 100644 --- a/fastNLP/modules/encoder/bert.py +++ b/fastNLP/modules/encoder/bert.py @@ -435,14 +435,14 @@ class BertModel(nn.Module): return encoded_layers, pooled_output @classmethod - def from_pretrained(cls, pretrained_model_dir_or_name, *inputs, **kwargs): + def from_pretrained(cls, model_dir_or_name, *inputs, **kwargs): state_dict = kwargs.get('state_dict', None) kwargs.pop('state_dict', None) kwargs.pop('cache_dir', None) kwargs.pop('from_tf', None) # get model dir from name or dir - pretrained_model_dir = _get_bert_dir(pretrained_model_dir_or_name) + pretrained_model_dir = _get_bert_dir(model_dir_or_name) # Load config config_file = _get_file_name_base_on_postfix(pretrained_model_dir, '.json') diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index c7a5f082..00000000 --- a/test/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import fastNLP - -__all__ = ["fastNLP"] diff --git a/test/data_for_tests/embedding/small_bert/config.json b/test/data_for_tests/embedding/small_bert/config.json new file mode 100644 index 00000000..3e516872 --- /dev/null +++ b/test/data_for_tests/embedding/small_bert/config.json @@ -0,0 +1,13 @@ +{ + "attention_probs_dropout_prob": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 16, + "initializer_range": 0.02, + "intermediate_size": 64, + "max_position_embeddings": 32, + "num_attention_heads": 4, + "num_hidden_layers": 2, + "type_vocab_size": 2, + "vocab_size": 20 +} \ No newline at end of file diff --git a/test/data_for_tests/embedding/small_bert/small_pytorch_model.bin b/test/data_for_tests/embedding/small_bert/small_pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..fe968fb5d64a87b224d0ed9d793e6bf3aeb70971 GIT binary patch literal 37965 zcmd43c|29^_y3PjhL9o3kR*-f@tkXIMMWj0)1*N~R8ly{ku=?foKO)`5}_hVNebs$ zyFo>BZYpV>=TsUs{x+xk{eItlZlB-#_kDc-_<20;wbx!<`&`fS+Sk7Jy4K$3UZs$z zy71dJk5>#NEI#nItGvm%0*21N)vtq>~q_f_J85%a@DVxf|kf^R2o zFE~^zRQ6KzRc!l=u+VwJ&R&X`5i-N>KB2}S62TjC5p%G!CS&Qvd zg`N8M_wC#U$$gmryda^X>=0Ftg<<~U39_K7P{~(=Ys0g5apJkS*g4xdI5|2AmD|71 zobX6tXYZl%W5vQQxaerInv1rAQP4bHR;PEQ8cJ)yl@eX-C$AnYMy=qV8z3WP?!ga3g+92OE3 zI;;H*|Id?Q{ErkajhpG&W4M$SO)U3i@~0v9mMJPB?(;Sc~g;U|YvpYlMTm>nYt@32cpp1qUM;a8WAzq)j4>(W`)rHfd|6A1aT zE{Q}40wLpT^RF&L=Y(-1#A;#Nm$b66@)vUhwe7T9MFcIJYbA^f5=H;@y|a9o{}LjgO^7)% zfIzV@NFbam1DGcf&KC%S|Lak)*neSU&|jlsL5C2I_D(j!kiVo@_%A6I$)pICk%Wnb z;R2yhMk103BLqV6e|<`1B<*5EcF1AxXyYJU{7a0eUt%n26C+xNuv9EuCJ-){A*_%H zR|G5{0{S%L6~4B)6lcuXKX?%ThkGxE1_`RfK+-BIVZ zE{@y{^u%8hoNPZ0UPI%Br#L0JTX&6&pjIqAEfAiO5uB9>&k2O*egB=t{Wg34jiCL@ zU+A!ZCui<%dhypNxb$lj)U|#2KV@Cli-ipW;bmFZS0uu#0%2pv>i_Ece_RjOIs|ZV zaj_9L{Q|iD3*bf@z)cyzEwS*nKzK(6a91L{ClKEMkNoYQzx%%d+Q0vU4!gIvv2zeU z{MG%VU)?`$>;8$X`=?^zGlB5Atos)d;Y)$=RY%j4?f;rhG^f+Xk-QTN-wT8vWF#LY!cPL>=Z>z&4oKR?XzhT5o38D-tmexvF~0s1<6D~; z-(?6t#KNBfk(|7oTswtGp7SIk1YOJLb#0GoaJHin{6BR2EA1^-JjUtg&TZ{HgSk#SkW%7x%IhRvVq{D4xgu0w37U^+TAnMLp8KFMsNkj(i(H@-r zugBPb<|d+^+>sqn+H#8o2N#iHTZ>;}8gc%=#5Cr1WMZ0do(#1Y=ZQt8oE3KW1tRv@%MNXU*h@3eqgLUCNiHO%8<#YDGIro2qZM&{TggdfBRA+9A5kXrE znWPNo{*u&{bK4{x!fj=gLpe_@a^tK(nG(^CY5i?NJ}jc9cN< zudMB=j^~d4M^c`xoxQz{gN;bg)4YLuT_!%^BQr|H#-@8?fPSup+;<7P34`G?Vk&Alp(c(JXF9Hb`f4 zo($BV^TeV6&I&|xI4c7Udn* z@y~RQwoZ;*okJAS*5a3lV$N?r(_FG9isW|M^EJ_8&XXZVah_PTgtG!sG-qXqOF2&> zTGk$2&e@KVxZe=lam5w7Pv zv1kKl1)^lm$_P_9Pa@jb9!=$JM`_@{)cg0m)2?6|cT7io&bHj*U6kI|LdLg=bARz= zaBdslW^OCv+roKbQ6^^vqOF{j@nvzIM6|6vn$1~XgTEFD;bCD5xpLEgE}TUTA&X)~1?yAk>c4Sv=KIh4hc5t3pw3D*}(Js!)kP0|YA}VZ;?&hp- z_kTocm#2vP_YQ#^xiXJvPg@I_JjI;*OP;-)+a^y5x0SK%<2LiOK* zZMSJ**WZF|_d}t^--GSbx={1CV0&#;>$hNA+Q*f=|MoR`?S)(I--7LBWSzeS?QGi@ zC%V4{+m}sxzXjWua@~Imwl6UCe+#y+dJX;_v}<37_xLT?ULxrETd=)!Vfb6Hy|7~R zTd=+GWBm7^eS2NX2JaIQe*Gmg6&mMv%d!&+N-mDehapjj?MoT zlr12Heg8A~>y{|A_>bUUDTlCMkd>SQcOi3shN>)Wj%Yyt4eE48y$#W1TOl=Z3dv;> zFvKwvYgSg$ye@J~e&s!AKC~QDHr(LPaLvcG^OwQAfPh-07Q5`!7W{nRI*i*H2X=6j zGzB=|CgnHOa^M(dM#2)ZZ0}fV`b>d+V*3b>8(T6OPNwiIWHVMyeU8JwAHbRK@=*U| z6R4yn!>;=eNyG|Im~K=GDnrZYN%w=Ol^KF>($?TwuWeAS_k*n7^OUCS8-)+FTczfX zrF7@9qvXWeL^$_aBHgt^dPN!Y9;KKwxA$~r^1zBMjJr;XuK8kW zvkHV5AHlmFUUxKO3`Rf2?n#QwM>{k8VEYP& zD93^Kd3D@W)d};gc=%vdGG6&^3Rjo+!jRiWU^JtGJ{g$^l8AvYeMSYztr$vDCp7b? z_|AZVUuKeT*RAO)%M9Xv+LGKEU`Be*G9{<<8HjR}1J`&t*41db)F;IrN4vZvmtTD& zfo5A^z=kl=l5iKbf{){=bCYqY{TjIW?KIh;vH?fU1E@Z6U#j{un1lv|psMK>yypO* z`NRm$e_2))uel9ouC2iHGWaL&IACguCNt)eFH9V`m5!}DKqQ{_Y->|C>{XZx+x!O6N$E3* z;*5Qur|$&UVx8DnL2uHpY$3^RDuu$pKQW?u39$2{VBQcD=Fd;@`1FnrCd`S%`xkXF z#VwpJOFDoCo+rqtL`A0TNdxto?Zon)s<0;_6p;@4B6Yjd8*Z8$qd9RV*bD|Z*Fc-y z^5=aTjdh^-&5=1AyO*EuZq6njwuDmQSNbDUPTHB7j}PA+#f0&v@mBE*v{5%hkD6LY zU%L@1(q7Qe=3d~~@(1Rxn2iJKccUc61>bALqF85nA+tGt#bYZ zpDy2qiUf035StB>LJcfm+bmr*h!2~VmXirl=SYv@MEErNGTxr(3Q_yKA&^X9dWTy> zdggWf@VpWJNKS_V(^F~n?HuY@l8-<7|AE2VHp7atG=9%wb*PBhK_ZN-nR%A|Fi5Es zqe4&MChZ~Yqgi`gddw}o`=pyDxUW#e)r-#K$~nhK$cyPPV|6#|=Ujk+amsM`aX$D& z?4%0U(_G4Sq)AoZWx@1Kfta~{IgKkALY<`D;ZDjr6m4^$I%B#pIgbZJX?QTR;pSDm ztjmJ;lQ_C+jth1=p-vL}z9UH|l=(jUdeW$$8DwLm8hjYyL9clKB=19&Q7zUSbShsH zof~UOVf8cA_Bcsz=jNh@#ZgE#Ghr;>C6e5iy3FHPR~oaeKiq6u30`$$A@$~M>8Rzt z7{4YMom#fjsq5UBWcNgVvf(tGGfkCUvGxNkad(8U5jxm=V}vx`UyUh>yiX?CzsJa4 zNAb~9BaA<3k85t;fyfhEA*jNv>J8hKJ-sG|%0K@~?6%2EulMQ1-fCpw(T0!Mcin7~ zxyKZF6Sq-YZa$IsxIiy=vLHvR6xo&D`_V+)nf3g-Roa+lgZ!;3P^HSl{N^+8;z%T( zT{9Xm&k#P5By?`N1kWZav!A|nhOh=ztXA8KNy9_=i*C8$^OZj6yh{a5wpv5PfNl&t zZp1WZFJ8KB#@xS~1Id0KSnHEWFNF*Tj~kv4IyW2YCM03wOyv1(WnlT-4RG+&Zs?so z2qTwCvH13Vq-LQQes~YbI%>nujy@0ZKE9}R?-@qaoyJ8OtKmaQJ>1WDf6?;Z?8|ZPJ^U7GV~s#mC&4o=!3bLB(p<0Y^j*VTEcqA<2OE{4>DmDtb>$na%~xf6 z6up2LgPU;0BLS0c%_b3EzUUr35+C1uMY`7PBVh;D!Ttwbq4m`*D6R5>&8_2Mrt>No zc2}7_xi0|%HfCV8$ppONHG-E$OKGR8FYt1oi>SZlBYEAlliD_SMhvXu&+n8EqMn2{ zjW`2&kwb9GRXr5JKHR3Yi)beEadDX*tMOKu?aCN2Auk``vy3>Hm$wZ++8ANATm)Tn zZ3#vz$b->@tK?lb7bsPm5B-+bpr+Sz_<7$H4Rf5a>fQ<%^7&wpvZwUwLpE_&M*Qy-z&=qn;M{vt=u2&(-7oXq|^WE@`q+A5|b|%{~5X z!BsM0@iypvH60keV`$T?z}$}*#P{9zg8J02M}<52xbNu;qQ5O1f=jy6AZLV@-rX=` zS}z({UI^o?v%zj_3Zc6aXxWB8c@mTH(7HJVzh5F@Kd-&h{PtYX=bD;1+Iz&YuCKUs%VU|v3c93l$zFDNr zl-?iCpPPLRK3_NwO;f`#Y?d;kvCo=cv3C;&e$xQYL_?K`yO=BIX;8VnifNi3WG@?s^Dt7{q*i9ZRQcN1>+4CV3p&C_*4b7>}L_?paWYh zw}SCAOP1PfI!goXH_|Zb0~a*~WWZ24cKM;hWZd~elFFJu)iXtQg|r2BrNofL6UDIX zb{@6-S`PImMASHHHoM0!g9tj+6Q+0(*}3-|&g>h6GrGKo&I30=QbKQ>t5r%{*K}r0 zw;q7-^q17VF%I)4S;Ek<*NB0K7mV<@hZ6sEDFGukufGG~nH~n+eWn<$bO}vo+{fhe z4dii{G8-N9hHMyBgYWy8Q|*u)q-ExII6r(o88^Bwn`E>dJSXnPyu}UV{d0FXzCDXr zq-K-XdL>}oFOAgC90SkdQc23iHRMdc!R!W?L@fQd9<5EiX-e=ZsImKj4w1)*$@v4g zUuze%ZtV=aQa$+x-+qMi-^Rd#@ZEy^BP149)Kr6!yDn4wZ9N>(Go*t2r&6WuM`+cJljvcR3AIE1 zAibh@Pz|l=U?TsF?({KbzxXA!dgNFgB7p4n9Uy{Q;CORnkC>bQ%=bLh{pR z!blS>M)ycB2)^P&@4X)ga=+%mu5XptXtRls{iquq)F{uk6um;LN6{py8n;9 z=EtIF%-WZZk2mX4^Zr+*mrh4RL3a^qsl6bz_sy7z?xvWd;DQ!e!(i7Hdv?;n8TdW+ z3p}hSpcNl?L6t5y|FGvta*_}~x4J>x(^81)cMrpK7UQ*C9uB%{gA)Un;9=!m=rHOf zju|Sz+l!j$*!przRa3|NE5}Kh5&BrRbT~}-^qy9P%!Y(h+Qih>ib=7Eqr*~?pahj60-PAt%=aBqMQnB{gQ=ww)n2t3Y|p3> zyB{l|PJ0qvPhJ>M?y6uNrE1G%RH zj0-~z9wfkoBzwG^SV637r8vo=3^i>+@l(Jbuy5!U z{to>kRtG(soWQ^IC_IYZUpdqv0+~Eci+P&NDuY+_w!bBGUgCo5Z13Z>>U!ch`a4ZK zG#@vgNJHhf^HK4^AUqyWLrK3CZ1VscxN-3*p0zXsdFvfenAHR=pH2arRtMRguELDF z1*FA#E#7jimFkzi!PqC6c%*nA4%v`SeU{Iqxi7!ca{V)qw580Y^R>OywXy=jtXc?T zElFBe{=;7t>^<=jvm0vNfW`%b}@M!cp4N=7_zU%^oG5zdiX@!mc4X;B{c56g?sfh zaCL1FR=6CZW;>_SBcT@vyFr&S8xP|5zTQxBVI9PDIpQpT)_^>q+BD!13qPl3VNd_poJrs{^9LX*_TZBQ2hcZnwyW*4e zLXsMO1)fx%fCQCH)DJ4ei7RUG{>=vZaM4`ask1hXjy8tx>$))0zc(YxzY3#&?#4q& z4zyP3C%M>n2#Lzrj^2y);Qp{gtQ%fVbj=LWrMCiAd;N%4BDt8 z!Rj+C$>qZZ=WeXqV@2#5B2QF~b%KnDK2UaIJZW&;i?F>v^Jc9+7;h7UNnSLZG4jUg z^DVKwb}P=CUsrgyrr$|Z-Wr`0%k((kJDhjtX|aCMqb_ng7TYb#he2czj*0sEq5Ee<*==4)l8 zV5sFTIA}3~KTP@*w{+Wqv3=9fbB7$L7ofm{1LCV3>-NL#ujFdIjeWMSEuAk5huOyc8Jm_wBbSnBJC27eA@L(OZz z0nBpnhNTGW!ZeCOZ_l|^+&)~+E z3{_-HZMsu7C>sLyJs~?o5>U8sJ-8njh>A*+Y5w$0u%+P)%Gr3qdGod8j%E_|Y0yME z>piO9y@Rgj*JT(}C7aq_HZK*L&A@nEJSxo+`GY zBN8vcn4&mTd1i#o;g@jIrAl0;UQgsFG@_x$TzbdX1XII}nHsSko7M9xJ+8kFd#p?Y z;Rs!F!)pPQEB0l6ET2JFkT-a4QvurLSfbW6UG|7c88#2wjB4AOfO@u^WgNlqJ)>Dgt7|0fK`psetj65j@(nDb2ay#&tsy&F9tTX6(tCz1UK<|_i@OeH z%m*9cT_Za*NeQF5JFe5oD~j+$uN~Cq=?8S{q{m3s=acLW2SB@O9USi-M0wV3cwgR# z?EkJrLzi!c+}r2z?yTYbvqK+2>(PU-xpE6Us?Nfj107+*xN~5({s}hM+Ek6LKPFWl z^^{)p&ZMfZt;pQgXVClRC+K@_Imoxx+7a4+;Q_;+`}YYP{{ci&fJ%2R#(cyC3CUP%+)Rgm)qv1X)p*UM4=7mbKuD_`I zRP2F~!OCp9TO91#UrCA?hPj#h2x|M@gAnys==H9K>Q||-CdI?i*}9r%`o}Af?9C#l zCaqy^gJq@6f`Dbx@eG22IU8Akw;wo`B_;HESR%U+0TQex%{zF6&5L zx0R5;v@`s8^AV?fSp)ttm8d*qEP1%`7`ZiQEm<<*91bwQOeap*3DxWGNhkN~4%#_g znM1krLC-7r`S&eyV(UWx~v$j`$%a^T?ANHp8?&kw)AWz1;4F}fE-pQ54{xF z&3RkklzTU}x@Qw!wDDjabD#39_bW-Aymr6@&;YaQHKcW-78@nXMoSHOlA+lj?A%*0 zc)tV8O{(bS)(j(>-I;NY zi%C-BXv!+*K>gg)aOsu_)5SUlHYdlT96p6DyoY$c`6DC`KS!fpZ>ME?RebMv@tA0Q zhz9>DfoG}((Br2%eye^;PL&!!fJOrZEX#xXd$rO>n&Ux3?gM$8rp0(GSmV2jVRVpH zD9}8F%>BDDx2GIi7QGl=|BzAx@i>fm8jELkkH#^*^ZCon5E4h;C7T!I;EHwcaFAQP zG=G>1>(!?Sv|?ki+Rg+so~?oLLzLlipB`w|`x0GreH1IXR8H<^#liQ=;cUu96|6Y( zj+8VRpmFg==v1W-A-BeZd$ucVdTWno=k&u9=@QKJK7uu=nes4tB1*fod~0 zg5XjlcCLGlQ4_l0wcX`VXs*q;pXvwo+0wKAX!i&+?ke6Cm@?O7K;S zguBlINzNi~xXV>R3PxM7&+JCBInhlp@9IlB^!8=SY)%55_vs)~wqV|mX0h8?IksiS zX6U!TjP>t?IN`%6uKuUPM6FJwmX&l5#cF<9N@=0S%K^ zQ=MKJbY*chvbFM5Ghh+=Hy?nhgN&J7Ne^Hh*#_C(qsZZ(1|ZH!=H{~Q_|T}H-l$Cj z%RCnt^;rWQ7AZ>uxqCu+UMo%GR%I?thj3ow5!iJp6rv{Yql+U;_%ZXJ2blSck5z%wA%4_H_wk|`P)jI55n{@>F_mSyZL-s$8B?c$FNcUNCv?j$E!nU6v zGe0lG&rZ7uQ&r5(O()2psk^Y}md!N3nK4h6`QZsZ_cOJ0lF+d2C7XSq)LA> z|M{PW>_kf)n&jw5mF}!(Q`7T^TElVdH6|IsJP~<{hoRL@4a6_6N>h?PRMiy4(aB2s zOnqd3W}M+dx@FFLnCo&A@29PRODbtFCf@{yt?L5cSAE27uXp0LedqCgLnpRq?G@Tx zzmyyt?gIf^>T&ahrLgD1F=#dTM2{CeA^U3TaO&pUh=(tePm0ShCdQ9yzC1{F3Le9% zx;)C2$}yzk3*JdrU}nE~EZzK}D>Yb`1UI`z!|d1FG2u`>e#j6&F4khUzaDq5c}~J4 z(@13CL#*&hz|&!AWWODfHB%l)YwJ8ff2BGbU@L*zuHI-d+Z=jyd4kKG-eQT*S}cs* zj$=y*sec?u`eUY7V@94~uSMbQNPpP*#D~V5(Wf^b+8llv)|EAM3_}$Bq>juz z+H&(dd0ylKPhK4&?=4A>6Q9`QrIZvaL2&U;jcX zR+M2~BE$c5Z9VnfWkVyZr$SQJcI;M`L$$Ng;bWZvW4!+k8u*WdOD`4C@PZZ-61x;u zKdqq2`@Z4lwO4rRjo!Ft^<9`kwm`N26ZjULC^cyP0N;nmv#J$AG z9VR>?m@fZ3j=xzl5^HN-lN&o-q1IcUeYM*bw1#h@cf<+|6LOj_zt{;p>xyt(lL}5# z(&cLUAILe9K`Vw%f#kiN=%E|)S;3boJokJ)Y>QRFhR=mCy{j^-@W2|so?i^p=S;-6 zCwoHIqQNBT`WRG-xkvP_*kWf}4WiQ6orFX`#4|4ym`Q&ws7hHh1>(MsL-Q+Kp1N)c zeOW4p)#sj*#4p29EVu}8JrhGsGjPqseb9I%QyNmBOtrF9D;KZKfX}AR7#&#;E5vqi zRig)8;GoW)94W=7jJKGSV}K(LZNyhPMo{FW3Z>lhi5E+@(4MMm;NbqvWN3CWhQ*IS zRl^8!RXqb3`C(Y~Qwj+aEg?E}JU(UAh-bY8W4Z1uY52-w>A4X&D5nl9w|~Nu<>W+O1F*!l3-Dw_tUOu$IPJw9r2QiVx|q2iNFBYLXJ;y~=_$2u{G0J(6kHPljMz zz6MVf7tmYrmT+0aoQ~?@k7{@WuN=5S{1?t6mupr-)*snWZMIx$xXd2nI;F!WoiRAE z@*5eLI)ux6IziyK9iTgY9G(#=;Ux7x!1LyFJknfDSI>52izQCbs*yv2RA1qWk^7~l zv;v0b?7|yx4i)vT;A^)*xFXR59gY*W=Kc$)9lHivP0LA$StnLBcME)^8({mL9D08} zqGb@0AL;TAS$pz3~{ru7YSyCAg7P2&k0{c-}b@ z!v_zA$0cUW=J#A~={uLFwQL|-j@m;HR&U1o>R4D3p^9gM4QN^VRJi0Hg=cM?z~j_7 z+V7hK`#skZR7dW?f+U5iEyjVE^hN`|{^*A4lMUd_Z4LA{UJ0pr>tLR1G^}TfahvBi zl6}%2nw(!rhx#1?@L342Oh&OygU$i3Odrk(D!^)w787sdfL~wiMdH+o1K9^?nGp}+ zYb{`0#TfSJb3AAv4Zq4LnJ8MVn*cP{fZ`Bk2{W+_K$jSYm|M&0Dk<};ukL|B|>66Vj$Uf#SJ(rk|??-UYc@j5BE1ouD zdXfrd(wAZLJqL{Py9tU>NAST~9d=RQEYyuRr{nJR1lMa*QPVyE=XuS6IRRVIYgP`D z#tLF=8Ut$8<+SjNE_k}wP>T<-5F@$YjiD?pFfMM&kcYv z)BGSLQjc0b+zv+JZdg#F#Q&pDZ?=2+I`VpSPx>t50KG9Om(W#fXifj_jJ4(ya^#*q z^iUs(g_BQEuSpHOZ5Ey+XZ%~zRc}9t2A;yKA$H``ATwrhpb6IPkY{U#MUZ1Dr{Qwh z2HJagC7EJ!0VBTcpkCwMA-typIab2p%@tC*=IRAFtf|PTU=6nJ&7^V3F_>SLMl3Ox z=Dywvu%&7oB)zSKbk_`WBiaJyIT}Lg zB~|>fQVF$>oPn*0>dXqy3p8=wPdvUt0$!e{KyyqU-4?kSeXmRDi`yQ!>uouCx;G!R z_imDY&U}U?UVA`!es3rosLyyAt^p-=ZjDm<96zlurgyb3;KK9>EI6r!?;T^%kFUax zv#`dLgKKa@VJ)Pl@t8Ns+E6q{k*|zqpnRwc(^%Av8G3F%@m%3TQ@XZ-jpcC~k}(+i z9_FE$B#Yd8@|qSzWl*2isbHkD1Wed+$k55-r<&Wsi0v~`cbqYEyw4QYVm5(uH@m_? z{{%=GSw`&E-ip1=enb2HXLEh`^p@+`8(S=tRNqIg-u&J8Mqr{fr z7qA|}Btc%Y>HKt|90Vw!rkBn#1Gu9!F(;WM5Bg%z>Dp}B<_Ag zoV*gaXOgR7(Y(V%`>i74>H?fPD3;_;9Zd&Bt)dDFCQQG3;V|jY4p?Y=0oN#up%R}m z`XOIHM%QIyP&m?8M|whd#c;A=^(*dO967G`Ru3B%C_s2s3hr!PM~*L>#>*e5&t5w0 zCB6EKUSB!RhT zvl5^CyoV6m0?eIug~&fmh8Ix-S*z&|T#fcDe(L!EV%7&jrZ^XylsmEVM+)irS&Ojz zkrG?JssZGCx}wEmA$0GSfp4C>krCe3Fp+yE+~^;nW|a!C`t6$`2dN>MNVj1x3q%uGVF0*VfaugEDd9+Btkr#a4Lz6;Lnr zCeD7Sfk!fIaO9QkQq6gqU{n1Bw6=`F0of_EXO1$X$a4qp#$!;w^*lVTJ4EUpmJ{~R zayXh3D?KLm0gc23;JSM*4stmQhNVfcbnOoCof}H+M;YSeO>(HN_zgd(SCbhxE)mtF zD#S&v&~jBFOk7~hdUhH|%rCDaCrLKn++-F89ahG(c9$_weFjeG*GLr28j$^50#iy7 za6#Z<_-vAjD-~a1{GKd+_fj1!SL?w{>^cWMy;kEuxhpWWuRq4s{)ueOu`1;{55Cc? zPc++cKiXf&;cB}lX^*|*py}o{0HX#_?f#r^!aXBT>ADjS>l{FT?`vSRcs*CE7n4*A zceoL@1+Fa7VQYF@;_c^K!6f_^hGq1C^5@&=lm}b5{Qoo*U;jqkwg!-&7gFF$-5PN6 zNg)1fzvHTFW5}at<*4900!=Giak=|(Y;ks?nn9n@+rWl3^mvA~CeKkZYADLZpQ9^# zQB1H)!8dji`o=X0D|nMpGhicn`aJ#rUEU@4>v@n(-x8qHkTk`#V2a0U(3EgBkD{~S zyZa)ndU1%WVg5mk48NfAz3oKbZy|hOn;=@U5v?!zk=AE=G(FuJT)!!>7sutojYt8g zG#`Yo8y;ihkkiC`pdapx>5S=%i%^`wV;^L1L?erPbos=-utKWLTzl>WuExD!y3$$b z-A9or5Cq}(KeK51j=QA#T6e1J@r8bN_Qav{%u&uY0jO0R>YH=d%jEtTF>W2Nd2#~A zw|eoHB@RM|qN!x&`!cw<_A~@F_&`z^AJM!FXZK8_FY@J>tCC)DN;H99@Yw@ZOUAJQ z6(^us`6@gi5{&;`2Dz_NuveJ|t}4}I?@pOPn7E&Kna<~*OP)pwZYeTxFE`MMR<(5S zqxTSJz`|S2wWL@@8H0*X&=KkfK&N&fs~ahz+Go#`2g8=)t4W8UN?(quKMBL&BNb$S zSqxZrvH{g-cRI7!j^5aJ0gD!UlccEqSgv2pcL`W7-R7;pB)SIRip=4#zkW4Y!QC?! z9a}G5zV#?HobStA$yvx4%(}r>JU*TA#Zla|_3kv=y$Gk!!SvLa9~9Sf&zz0+;C}8s zk-BBRWW=^EFgrtwd2nefUMm^MSo7*3!t4i``h5@G6PyoM4xE9SGulvAWd@gC&4k)> zrN~oNW6Lsb!PWc9XuAG0e5g!B;m}x+`y+&%ccN4ApRt;Om zm&1#BsVLWIfhtqAG4;qjaDJOdHh(h#g%3{La|BEHHt;&UPw#=w-3BsaGxbPX-XJzo z*8`Sh4Z?l{VsO9wMl=qosk(GD1ZVQxVA4o+?!CqSY}(r0^p@oXW|=i>e0U zh|a69ziKY;p~Q&2@;#V->+XjpU9+IzUOBE`SI)nk$r4S03M&rN!?8p3sN1pQSh99C z$j>|}?RQik{+QmK^yvQpR(te-NiE)J)EbKAr{;j$*eu+3%%A%A_(nH3>$1Cg)}fM~ z1}(C!#4_%k>JaW7?(kC&AzJSg`Rrql0+mnHS(FT;lPsCInp4nY!90F=x3!qNN`Wah z%&9DJSVDKXR*@LbB{XMn0Zg%q1;6<#Nt2r^uK&qEu9XV&#>y1qrgp*YC$w1i%u2GW z@HuJu^AS06rz;e-Dl_k<)uW30CaBmb!RGWPl0NVf&#_AjS8IvIqF7bzY8+2yf92lw z&N+#VAro$#vro1-z>4{n5EscX z*4M-?$10SVTRu`cWjLbg{9SPNvjcpVe@TCQ@`P)nDk1h*9zSd$%l+>I3gDdH7~bY? z*DxWr29rHH<5itRG~T5ONyJ-fJ}w&uWGp9p?=*1lf-ivhtn;+HS^?40&4bfZRGAHT zAJbLS))T|21$4S(H=LjRnjRc`86$JYB0&lGs`bTrr3C%i6jV?f216f>M1|SqkefUM z!yLv#bA2w9%U`2&`+Xy4E_Y+J&Mm@Yf(KA`R}WgY6u{+Wd(il<4Ky7uhxfx&+0Wg- z@|}-eg9*ts;1X~YPl-;$g{SfSPQe>+gv7e4j8uZA$vMz_uEB&Xy2M{S>;Nf>w1BLK zx~TGTA{+9(M0!sA2ov))+3yNzn6K_gQxi1NQAY!Mcn3hv>8n^1o`Nqlv!svSOb2nw zFxcsEo9vr77YS8jx8K#OG`Rek=9Cmv(cyg|JK`*EJ*o~5CNHP^eyXyieY()7&iCQT z`m6lb@M>JftHRGAU*PiYJnR~ff`aj%Nm<=^B9{{&9p`4k_=FLn|F|pb$KM2Nk{07v z%VPMP-2kr*rZ8(PT$q5z;WWqX2%hWZLXWxsz%@_j!;y;VaQvMGZmNw#)iXB%lix_Q zr`^JN!%bmNqY|d=I*0-HS7QX>-l22*0KDTL5DI6)>xg)oe8L*{-E4v75r4uZQ8P7G z-3%ux-{7neW4JMPYgJy|HeCL60_cuC01Gk)@-!aWlCF2o;ZO}H*g4n(YBq#|cf=qV zb*c~37&?Vo=yaAEb7OBoYk%T;&`+9Ont-Z$VW9WO4!?W7gt2kG$#iZE`0Ra)^W^)X z^U)d9eBfOW1o(sNp6%4GDuo@**8m&IS$;KBi>~1X;NjaHmUvsxjZ4=;gVABUq)a`((c`wMz zadT+GH(T@^9|OsztH|&%t02?XjZVD!4EOc4WyhJVp?oE3HaU)6JTB-Mp4~^pa;%-m8P*(J1QjSqbijav#(8BqVg&0Wo%p%qQ;I zd5_Rue4q1X*l%Z(G)hi{!53nr+r@Gq7M_y&1d8dZAv<}FgQb``+l6X)24QxfgnLKo zEacWya<#S{7}9$w`D2R`GrVC3-G5;fweHlDt$nUYroO4d%boVY%Q zj5`(Z?VJw%F<}%;?@<6jRga}(N}tkM&N__h-H+f<;SJvwE5Ontsa4VsKqJ9eorxAmy2X619f+QRWc0rw^0{bw#&@Cn@)nK6X566Xm zNthnO{RZRD_4sbk8058dW|)c>BvOsR&mT79_7nx?A*zHdw-R3lK?$6SA!1UluwuyE8*FDH`uAT z2LhZ+(5&k_xY+y?6rAi---i2~6iXb0)vQ3?$QNiUXUVWkgUFV-h??DP| z*$+K7oK)aF&DZ?c`rhojg>%5{`D&c-a6go7T7wn0VzK#9v~<~z96TgHp4@et3O9Q^ z#-=HHth(GxxE8tqjhDKj-kt8`n89#Jh1q2D$z_!T$7i5tAMX7sqb=Nff~VobtyVB> zErq_`AIVhrNVvE}ld5_RB|6+Qxy-pHptWB_Y_|LG$j z=M zR^V~TfIoV(J(Jbbis20gR2%;Y=lL7Jx!ngz**?Glp3Dkn|g1>fc$$HowZoH$>|AY)d)Cvp2QTh641HuiriY4 zM24?_K(>neFc;UX$EG9JwAL$+Kh1bM&NC{dBaU$O`JRjM{qkyZJJSK{x*Q=lo_#{q zG99MnO(XU`D}s8jA^5=vd@0G_ zevT^J-XsHpRMBw4eY$_d4D=k^gAU!}P-T^Vjaa`6Be~NeV4h+yZJnJ21=A1HmJKJs zDJGF#_&k(7anJ<%N~U=&f^Q$Ff?Ngi zbgiWCr`|<_$YN^tbsDItQ*5n$OII29XWi!Cp)Q6S;O$rqtR8E^%+_y&yGc_pmbVTj ztgr|1!Fp)=(Vf*DH3bYz zDJHrTy-4$=omHn!twx9b2IR%N5b0>M4VY=>Pb_&oVfNR3&@k{EMEk^`=%F0(sM(Ga z3;WSqj(nV6wFwsXt)~U+RzdYn33`4t0lzKYB;@;RC}DdsrV3l3!gGgoh(|2FaodVn zzGfXir)~;+s*4xhJYf^idmAz2O)hOre1t{d2Ms&wh@-k1bNrhHsIPk~<+VH}GtQ~P zGUF^#dQl7asDH+c$34)~J`n;gRN}s+E|}%&${3HDjkoeL( z1-IT$V0P^5M~A(4FfqBK`2MGKj8@8ry&gfZdIHzCIUC9C6_!lV&(~J}X`S;S{~S+H za4syrWeb*W3$bt;pBfflpwBs9AnHIaypMTJ?$5hR=bTAEt?L2kJgSHtH=|%t=TahA zUqlaoPQqsVs&t5LJ$E18xrP5_!3^4B zbCKrlIE#-1rb6U{do<#;I8lk129f5Q@L6bRD@4cGDj6rd6mg??XoU50~V^ zg}cM}G5S3zdm+ji1tmaB&^pkuc!ZAcCo{J!jKOe^323>v!a&y{c=1V;(bY0y6XeEt zA7e|wFkT$a=*~y)l8+D)y^hi0-J&5o(@5+jIY{{VrxGZ7w1?||MPBWcpx5S z;`@MU4a01WH2if?53T0~g7O3%tVxtaR^lwaQu8MQS`0*n9zlyKBH-p5M4g4lQ}${K z4Z3Uy&lNa-UFj967=29r?_UM2^EaRo{=y#LwUCzn5-aAtB046MnfH>2H5X#w;aVj& zx*>?FH{RlP-%UVY6(!L1Da1PGL-2uHgMW96v(86sF@BaQNqM*e@UJaZsx)V-N;b2M z^gZZXW=x%28?0hdD|jWD#t?Km7m7`~;I!8z-u~9RaMRL&wyNZV`w6b+`85tV>Ab)k zeG~TPi~V>egv&-J8Zz!>JMjCTW^@_TvAUTuo4MsJ%<3*$j?=HIQ42*L*}riQN1LYa9Q=2(Bk z;m<=DjA!ZjyJAd2n*_H`bfe*XTR8LSCiXUavBA}W5FO#hiP`hO>b(}RczTj2o=pgbMe3r7PbEOp%`r5#*{s|tuYX}mjYtZm}Fbdk0ljNuZByNIC zozqJ=ly!hqZPN#BhDS{nj$_|!G=fl=OXnr|@ORq9z}lZD;Vyd?Oh&iEqV_3x!3)7n zF$6x$kiLlG)qBVEVsGUTgOsx}9$XpM1sH=Z-Px;C~(zHo3sP zf<2hMqz2D6)nUp%9k7^J14pWBae?C<=((x`KC$01cJ5*DKA_7Kl|IGb35j6;B?ruJ zltavtDD=7#2RRe1n52~t;DXsf80`s!RflJjqbFopgO}&QIrlf%ahl%8=qQ}aAIC5T zkN+3{<$vE3{x3gS^S_PY&o6s%_oqB$t~c@&KkA_4zIk}&UpRi$&;zT38(}0c1{40S z!NoPVQ88)~-(cZu%l_aG)H*JgA6B@Jx8K!`HMPIY+nbXOM#X1f;rJuy7>-ypbvcrP zL3sN;2JW2HrIo>PbTqXNjl^#fW^y*~4VSCVoj-v8uYdF2?GkpJ_zq*+rI~8`ouF}f zgqH36!M8s75Z0GA!_7t&JY{%^SdEe%p1gxX|P>&6<|7d zm_O|+T1k{@AbxYkncB%HlF>=t?_@yp*KW|ASC5U$v{C+JB4o9E;A<@U1`>Yp#NcWG zT%Y=bXWz}8x1=l}sAxR%CjC88m>7*?)w`kTo;UrZqQus`nTAU}pVFP%C*av>!yuPO z!K1ka&S2xG^q)-cVDb5(le^_LT*YXRU;4 z3+v(cf>^4p!)1YL9I)ocPa^zlJoB(1lQ;4#93Rw&J z7q*biI2RYGhhjqeVeFo1$&Q7{u|{q0AbFr0WZ z4=|@Sm?sUtfjP-_Nrba$czqRYUD1z<_Ssx_-jLbb!_tdVJ^ZBFE|)nzY*O|mqM3%Hbi#c!8b1X zAd)?uXHs#OXzp`^*LnUe;uTJ(R}7Ecx^bnbc?u;G2_omKh*)b|W$d5g7CPG8o56q1<6b@~`}~<$vm&^grL{ z|7?BypYQX(otXI_FZ^#G=*;P6o=$WaZ=B#i$g5ZenjQuCU2Q49{Z9a7&8eZC^Jnpo zg$Xji<&3>v{v+$>a4~BxHx8{1cu+c%4CmkG9d{5!;a(o+@4GVfmnOq+@hNb7_6K74 zb|rj#atb1|x9~K&-qIDC&1BE**Ko&aGAp*g8k~pxAp15S;`$!Jv0Zn`q|2W{)^j2# zsh`JF>m8wO;2KTSols>8y?|{s-8D#N?A8_)|DDItd zihok*3|jorbmY}oCB?|iFinb9lkS{MR@Pg5bhG@W+r@ z`ahpxQf&y1`#G7;|Mmp~N^hcz^F!F3ri4O4zrk|~!R)s?FmROt4wV2&D~TlEdn6gd zX+`*C>MOp*oNw5dya_&5uOpUrr_q`=lDlOxZ2XRM*xhsq7j()HU;XE>`1TTbBc{Re zho-ReXfzZiuA(!(-k^^C`x#YNT_P4@3l&jMv2V{hQ1kc%jZ$gwSNtVOU*SxQU*4fH z8HyOwo(2{(o#@TU1JL?A7jBC7gO)6(8$Y;0d|M_%ZEGRd;cut^g3gj^uN)dzEr1q$ z1(w}bjgRXdkb4{M;V#$7$UF%{Kj(b(a*9LSh34$1*ccq=JH*5BT9B;!8pDO__%S!S zp`fLkJf3$QCf>M0I!zm4WVlSt1qV5{%fdYz|0o_^=6&F@HJWU3 z{U&_8vkmIz66)KqlRn?BMW;&cXKG4+W4D?(tNr^arHgB5@B=s8t+sFA) znFDYtR*O|%ejdfnnK30=jqr3l*LCcCMQY;O!I0}g_OWx&(z_B9v-!{>a}@(dhiE5m zMtmZL%<{_+{@_I6U#{!qJ@6Q%Ud>?C)=y=8rK;fvUlW9q&(PNeDxkY{9;uHi#OkVa zq8iyoP6P&l`J-$iY+ebArZw>PLWpGiSManwIRcNEQGwr%`~p=0`2u&M=shcF?NrJdBzf*xczA+ znoc9|^mi1;MOZ*aS`YEWJcA+D>$Grin-bAlm<~I({(@GI3|gyak9sHLVD0K#Fe~{SI`ZUDQC@|8mVXH)xAEcj)|D`I zvlvVaNd>h}*Xb5m#7-FGzMs<_E4>tbcGa?S`bO#w92$thT(1JyQ?Q2B%aLMhLMo}Y zLoW9943e^E*CAuzXtD2!c-{bZKcI#{^EnvhjPMPh=xCQ?A1|FVZ0E=ma*%vR|MbPu5?_!YkLf zz2e0veCkpL=JHMA?LQGye`JBja0n!=Y@q35q3AgH0mgfeVdB^sbndT&g3G3yC$|kG z9=rul`zi>vxB$VNcA~_0qCE}bT+Tm_H0*ywYl@Pf@8A?1x}nJV1q*Sviz8?aZ{aj- zdz^Vvl7$v47$__iHKt%PL|G)Wwe6l=lo+i#@Jgkg?9P6Klp4PujUnz|qP zLdj1{7#w@X^X2#`Zkr8g_vu=g!QKV6QLfK^LW7Z)tHF*oL%PP?f~d(>^F|^&VDVfz zNY-fpqci?c$Z1o{F0|3yrbx8mdORg3PNCqp3y}4(6mk^xc}R!-jt~$QJlBp4t=~H3yK4Q|0x(ImL_AYeBsRCA>aO7YzIvto${Oaa>~p zR;$9G(MN{SQu&RKHH)dAg)7~0C60XA83+FM^TF3A5`0%Evls0wn8|aeFxonw>6fBG z4Co!^m`6e6)xi*OY@5iQnbrgq$CcQ<_F43%RRt+qqQKjqxgB=BJPsy@^%)o(BBxKN zgV_EY_$`SP9?I&8#22^kqnx{{(n;U6HZhv=UYeR^Y>UDW+jBmkW+t z0WyBl;Iilr-k73Fx>HZl{hP|cK|`C#_jy;G$DP3v+h4=}kY+0L&I&?AV&UrQiTFu= zGqvD9hR(ce9KXmNMinL4?#DBsWTPuh?kpl!UvEMGmED;7P9KQ7Qn9A;ef)i@7!syM zVr2OjcxKXr(?i~X)cF-?A1{C_L~arDr&`4Cpb@wCav6rBNvNcp3w1|rK{`3OxMl8D z{H>W!>azsEWZpVt2Lte#-yi;hC<*4t+i~pBD-SRUoyoq6i6dLyDx&SuJX$i?LH#01 zVgFpDNfDRf#gsVCE4+jP_5;N~OWvcg$us^zya4v6l$my`SFl)))5(vYgKJ~8*bs&& znNk7jBl4tqcNr+0D(0)zUj&c6U*UyiBz7MeAp2WH*p82{X|nAla`2EjgnXEfNzT)8 z!N3!~^U(%kXNY|NQv<*ToTqUCE70WM7KoX32ZOmRh-q#n%$*Sh&CaeQB4`^_Ww{c= z2QIj2#W19^W8BYgMf#<_6XCo>I_>I8IM}rw4>=$mTd2(H9Oh#!^NR+)ILnVv4r7Hk za9JHu26&v;>VBaFi`^Nls1wAA-!zEf`cFiAvkvTiBMzEMvq-G3BK6MJz>%B?T$t@e z{S8Xc$zBq(nYr|u**pB&APXC&p|uoxlpGngr!T3Eom*9YfD0^0>h7LUNB42)0g7+>vey)EsHSOkux~Bs4zB7gL zGqq%P-!T-tUkIMd=VA7KQ?@bUeDRK*3QXXynK-319KUDVW41vfT;;Urym&!&{7+xd zdQd@}A52At6<~7rUqC18E3kN$KlaGlkyU&-UigAOh;aJCyV)^|rm3gFrePs8YxKrs z!8Le$-x=DIqR2en>&)1tRgrC9=D?|GC!xMKlE%COe5J1o@psO{g(@|?_HZ*VebHvL zm7W7LMgbMPhe?Mf_c_QeW_HP{!mXTzw0~qTt@J6RtzXunWW-*o$N2`a6=l@eE|BO= z5hjoFy zPNIq{!?}BO9y%&LBITJn*fnPkctne^>hulSHbsDOk18e4_rxM8?}0snlh9g{%Ub;( z$_W3TdHH|qAxzFoo}=kEShVaax;?c;Jiq(>T-PAAo3wYiZA)%Gc>1NphM@SNUz9qDp$7}A|pAU4jd6gwZZUPAgD$BpzF8y zL7Gb;=Dm_6e;bUzXM+lAd}<*F6ys=#`ag8vFUi}c_L47JyaG!%)Pmk5eH`07O5Y4- zg442lFl0A?$A77_8~?72ILlPgzk|%bhqfE-b*FPkheXCYR`g(RBsrR*#O%XULxI( z1JF_KHuP>5hht9DLG{-SdN#v>K7J92T8<(lA!;){0T=0sGcA`szy}XCp4CcOSZ+9vxA$T)nNgv`M8Eh;hDGJz%<=2=T4OUL zItFnJyk%e!-CZ13dWD{nd<6ne&x5;-9oD$ngJqC8JO6nkQFlqgzN6{5N{-9EDm_4( z-dmuWm5n1eUy%R{SKjWcYf;Emj8X4XWTa-S#WU+CFhXgAeD}m0o@e_NQvdoFWVoD# zOfy@cXJ>%!r9$32<30H4ZXyf|2%=kQ7;b*M0oT0LWMi8e2}hNHRjVmVwgvI9;|Wda zT?M7QE>Iq4BD@I0Sp3H(^mzvuyU^-$#Y7;eEn5ST>2V_ADGB?Zm1@k_KLGn zvX!JEn-rT(;P`frWWhQ588xsqW;cocAQ7#puxrarTmzRNy6p<@jLIzb;m`3rwe!Oe ztbLbXX#Wik&zQv2ZqJ4VE~@xcQIWW;u0gx|bNI5me-e=$72tEc0JXl}KFb!y-F3N2FJr3gD zCc{{k3!-xms5lJcfSDhiDJRV|ANWK1^tx%xiuE*Iw20TR&I2|HorTEn4mfLp7h3x1 zF#n37Pl>=A|jU9maGj<<Ai9Dnh$4_IcHu%5PV#6ZFw zg54a@^zlWw{7(fOYbgwU4ube++TbNtiSYgc82-&gp|f%1mm{Z*mpsQbUvoBL@>(>T zAj-zuCKQiyeA|e{LC}EBeAjn^OzxH6F#9vYj-yKCx3MG2H0Bro=D0QQWdd=~?lY}E zIRWn#iLhN`rab>oGPEV)2{D%3PM)phczms&iq(HigIO?+Cp-HzR8I7xGSyt~mM;aS z9~*e_`|gq@ha!mmF$uV4ehqQ@O7It)#0Z$nGnWm{k`uQCK&FPvv|8*0vxk0Iaxw_E z#wsxjB|G7I@jWP5ugj?a)CQZ6FL_V=R)NmRczQ?T2|2R+GkiK?0uf&xqsiynAYdDY z29JGE;qN8<&Yk%!QYJAEtGNHepYLeFv;|PQtOlc1>uJ9{*V*@5LGMg0whCFN1J`~? z!>2RBa3U}cO+p{TS${cJOYi}&`P?6p^+bZ1C*i~OPQxkLW6B!a8o@U8*LdtkF-}_x zD2f3oY+6upu+ldoCif-e!Pv9+`cY(MVi znB4?b%G02B_d`0SZN*%fa{!`qG}--m&Gc&UoZ_<6jaV4C1IA4rLywE^@yN^Nbkoo0 zc;KZtk6PCQ<6PitR4)x48} zUtaVUiLTbhu8d~TUl58vb&f-^`8%Ta`3uIVNHM2m!_cW#iOrdEmpq>>4BbbiK}SM` z5&gOpI~6u!1(`}+xIUAVq%yc2i^aJW5^QBoIvG7#OEqrKrKghLK~cFqGq*#S`l-3G z-!6`l&ZsBg>~{;axw}R5bup;?hq&>|MR2vhOU>3z!FOp9co>wK;r*T5IczFB2jk%Q z!rS}}-t);OL(UKH8$*-Lk9f7`OmV%LCOE_-@achQ%-;5|SW9L;WIy?dr=+UT$Y&XT z+86|u58NpGat@JDO-DA-5W6>g72 zD2PF^*)OF58{#Ol`8lain9aDZ$;4w_ z;c)7FC13d7Bxco@0*n_A2Zfq=vT&LrI}&mUZh!s@w}ee_%N}7SCfx&Oos~w78+EiY z>?U+R@}gwLZn{xdo|x6v)7QUOkti8DFDjr@WO>EwjG zB71w6Dts4+hcMYiaB%!o^6|6_6mEHeUd`hef6HMK*zSb4zwHIBKkHFjCCH8d(4?bK+f?;ojyKX$2X;Tlf%MBItj{n*E*D-v=RN0$NP88}_k|o&mN|@ft+m-c!F71Zw2FlL zzXJn~9U6z8h2h6ia8XAV_8n1W6O=|!Xjc#r-U%>YeGiA%4D$Pxr7$8xk=OeE1S>R9 zfZ5Mv=m{@bp0ryin0%kg`ITa@JL3v=-p&Il`$7CU^%YS%@tRicp1@dz0`y4TBVNUY zSSsU)9^u*GH+UJ7k~7g~ycsUvQc0}e{=+XpQfw~%gBXsH<+$M(ir&@cuW-{xEISN#p|By9`ny}#hFFaFY!gJTN;Th^LA(dIuZ1iUt zX5i5pP?FV!R$dxOtPdkHS!Tr5*n-BGi?hcoRGDU1Q+Cp32RJC;#fzGwipy`<;R=pP zP+R>Ne0IoVNBto*Dx3(J3F~n5!3m7EeFwcOH{i#R{qWf_h+Nmt#}&NG;4$$zwwsUg z51qIS`Eq$yPn^5?O}WwNt5Ze2m+Qf@1A8Hmc?kV-VZ0IJa6H3X!s}zgsP4}F)FS=? z71>>g#^&5J8Mq1?!d+qe8&ec=Xyes8#G>X+3)ZpWGX1<{5A$iwM>61c%_>DdlIFL` z^(6`RIJah$mbBB><3j9?nM1H|a{^BP`;nIK6+;?m02ae6 zozNIYDnmn{C3PFA^v$4}XC`sE%xYLQB7#n82GCKvmHn&!kn8UY@$wvFQQayEcYF%Q z0n_IYbL%zje0iC=PjKPOT>D68+s$U5m8O%T)O#4+EX`b*tO$**;aGfBhTR~;!jwpH z*p_t;gTsE3Sq5P+5@ifay&r+Dcql3_ctXlwD>Ft>-yk>m87=J^APQW^W6UIoX7?mv z&Gl1s4$86#p*=WrFqP0E9i8sphy6){MB zvyu!b1L}McM2#bt$g4pmOkL1~S{fa6-ZoFN-n|EX)@GyR=lgt7RgT@gr5!x?`hc8f zI`!R_hD)2R@LSS)@VzI1LK8GWyH<&Wl6;uIXCk(3l3@pAOh}NxLNuKihOz!E*sdR- zP0@<1X6I4fl4docwPqN@p9I3O{-xNaD2#r(dFbMy2MJ@HWNJz`#&{1>QSV@o`*0i2 zADPO$PdCASsdoOV6$e*zEYGTk<}hO% zb14M9v)+srb`r^CdEPwx9lUlyF+48$ge)mdfgKmj*;#`L=>9H;V+2bx8B>dKN47U{ zt@>gm9V|n|YPS*1PA=Cks>BEd--A)rmn7$AHe7va&ClC4go!PE0j}gB7bwK75g+$`Gy_$E@9p zOUf0QvJHNyd|ij9qa2RDh7wG&VnrZp)+;`_;9`r|O1Jnjv@1I}^%4LO>ul|v_RtT%}(T^K1iDSCfp{Zq#p=3p#BnpxPnguyU;|2A(_s!;?Z$Xn7Kb@p7SBC>i4F z#felYLq-mdLt%UfD;_yAVv}FO>AK0qS=)56_LU4bq{fzP|I)Eo1hALS+3k% zgh_0tRvMmd=_M-yZh`lzNZOFPgK^q;9q&Xv#OB^SvhCY*;v#klWN-N~_YUXtZ0<%7 zlV{aXZdHvAdw5uJDu~~{k0sA^{(*JODBSXCAce9qeA%sKyzk>K(As?lL_YmGEt;pz znlcvT_?Ar=?i>ITx z{*&h&M5F!}IP57v`Qcc;)y7Mxw$_@J&lRAN?u&@ciwo%E=>*&QN}+7pN8BrNhe~F6 z!{6g926)wx#6SJKpsq{A_n`%Pi$sy8ltSwJ+YUC1?trGz#c)X53HIemv6c5Gu^PJ8 z=*?yB4A$<370Tkw_G`*))Y}_4(ig~c93KZ}&cd`&{2K1NQHDYZ;kd|$yEk|nTwmku0}J4Bxw-h#^GQ{b0u649G5ot^O`l%$^O zBAeMqtVnyXZCzC4sWOoz>OlTb8mBD-z-X1FpW z1_f#*)Z)-)a8NpK#ilLA4F@Xuy`DVA#xRh`-IRf8Dc5-0d-Rxyy%%{Iy5VR(yc46$ zI4z=39ka(P3kXCVL~Q8j($X=>s}E5r>i07>3!}FH4TIx&u5>_ zX@-SS0#LQq8(tF;HY(DHy!23ovcQEx z=$~{AC8TpXJ?l6$D-?shx(PR{WsLaC>%#R%%jgWo8-U9{b^a3@E*SPp$~cQ_;bL z@aAzU@i0CI*2Uj(EK2|rgyV3-d|4<{KFwdX!IlS-dAyE&qPWQF7Tl^fX50VO!-gX+ z%s$W6d|kUGFtUy7B7{x=o^=RcJS++)U@Je-T93;NUq$29eO3}1qez#VIul^6#FT8= zzz6g9D5RnQ(jo+J{|kV;5N9mIW;*&{3A?@TEE>zcgO;I(q#@H0Mpx@&Qg^XVDV(2!a-ogou$H7 zOlB=V{(_rB=XpgXk=XxTmdIc6$3<;N@z&f_?6uV)3I?2qHGLdg(piELqD3gHHXU!7 zsgQv@K0o}_S-LJtnlWCGgQ+s#Y2alA2wM;X1DXB2?vh?&zIGy=uWLzj7g@3MAz48N&lpEXTQ~@C{C;qDSa4`n-tKw6x1Wpt2LQMwR%w+xk&buZTvt%);8z zl~6ca99DQ8N1p%(d>tgq)-E??6$}w=pFf~5U5{~UeuFtKQN?2nA9Y*$No_yp1Mg_# zy08|ErAiFUzCTRv884tNfvMQg_!tDY=fizDS-i*fh5YBO1^2FE3}12vB91&I3RZ%Q z>NG{%|KuI)x304CsTz-&wi_v`&cWZ#pZM8i~7j5!{ zo?5~4|MD9rJrE^+c9QHD(TNykJU~nzO+x$gwHQ#FLbrL#(ufQdIj6riV^BHV1{0It^m= z=g?AUgnzLp5?9z3;XU#1MCe5kJ(ib6tF82za?32lTd`p9c@_q4%%^uQ6k(unHT>>q zr%KvdtmLfcWQXZxY%05I^?QLB)#Pcy(2r>BZJ)+wa$V5>^!emM%33<>rV;iFS6Q0b zBtw%T$F14N(9OO6Xg7BT{W;ZQ?YH zV_4$Z1oN)ffl}i{h)8@-j_KxL%%W|~;@@JdvQ8XNz1M&W{I8Q_ng2ygka!@xleS428Eg!}b1s5Dj^??Q6`#8Q^IcdMMo{Y%kqq(9F zYwfH-$_{VFfDuz}o>4rQxmDBlqGxd5TAuOQ%FU)at%~!sEI?Ad4JP%hMCaa6{6=4J zjMw+YzBMZBbX9j8@@l6e+C?<@#%#!X){GU|;!Nk@OdQ)lVV|!k?(Qgqp@|dl&<`Qp zcl#{99-jd|4psOt`3;x}JU}VO8oF$u3#xM++rGATVk#%V_~_pv%5S^in0p1rm!!gt z8);PHg)x-tsI!{4G>DE_EtS?@hSzNyFgwDqSS#Zp72R1%zx;Sfqf*b~rmXj5@LdW? z+qsLq_F4oI&*@S7kPme2V{xX5ca5f;l7^wc0npLj&T)e?K=yt(Z7l794*xl1F+B|l zUw#mYd~VKIf*vDxSQ!VWD>G@al_W1tm`#zZ1Do+p)IGrkGV3*9vlgdeMyHU2Lr36H z#3fi}lmfLv)5x6fad0fT2OriK;6YBWXdAtRQtOIwSWp$o%6k6uFFY`4+71U_sF85< z^C)+~4I?L%@YfFB0&A^w?r(M;-ygIFw>xKW(}g*VP0S3;7rBmB%kM!&!dx1YpUAI! zeiZDQC&TxDv)Q7G_2}6=5ob&1k+A+OrdGF*Unjp08|sTNi<$FQ!ZzM^cpR_;U%CCp8GQ=$V*5L*O4&qQ7Ip;JyG8O!?+oCm zegs&if92=h84rimjp!=Jop9@PXYsyEy72yh07i{V!o8QuNY8@XXw;H~asCv1paQg1 z>cFFG7xl`!2tErvAY`5iUcGY%rC*2gH~aGF(4Z!4G!B6sFF#VcqXU-f3cv|}DJJWO zE0LRUllP}ojTv<6#AD4jawX{HbjfDkNQjm_5ESYQy#T7E$Ls) zt@OM28wfLb&+!?7S=il$)vnHTi`Q<9KCzC}=yJUmxko&~h1n>k@|1ilk0!~1GH|SL zDx-d}6Arm^TxAn!Fy{DZ^OrcIos>3P@>?Ihx$M%1PZ3mGr~was*$-D8oWWt&CeX9_ z3{%hcQZ;b}3^9ttzL`O|DgH1v%@<`xYmY%y{c3Q{t3`Kndm7@N50YO;fZ4Jdm{3j^ zy5x&>JCC8ukv(9&Ukf|BOt2#&AKgxwu{LeXu}N4MHY^+gov(3ZweB1yVWK#QIjS%(u)E$|$=J*$mg>6Vp zEW*hG<27W{+Hq{9b{gOH*$w)pXcn8}vL33en=tnNJ6ba*h(_AoMwPdgIM!ti9l0NH zNA)-8%{omjFAL$@6T*{&(lT0D`BvO5n0 zBa1*ip5smah@j?=r!Y=-F~|=W!IdtuXp?;%kIdkcNM-}b{Ss%$;3O{Cqyk&YWm%cR zZYU~igKrZX=uz89@?~u$@4KEZma_Jgc{mZbIv&C;WujbvrWWuVibFjRl*z%dVZxU-3e25G0D+DMK5$ch0s z0XJ~Ce3cjW=r+oji_%HY2qpSv=&j!a20cCGlj=26Q-l!x#1Gk>N_eO)0y?eou;1(< zy`Q4NtNu8i70h`@9HWD9&73gk7U1$XLYj Date: Sun, 1 Sep 2019 02:00:03 +0800 Subject: [PATCH 25/92] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dsequence=20labeling=20?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/models/sequence_labeling.py | 41 ++++++++++----------------- test/models/test_sequence_labeling.py | 17 ++++++++++- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/fastNLP/models/sequence_labeling.py b/fastNLP/models/sequence_labeling.py index 0c573a90..6e839bea 100644 --- a/fastNLP/models/sequence_labeling.py +++ b/fastNLP/models/sequence_labeling.py @@ -39,14 +39,14 @@ class BiLSTMCRF(BaseModel): self.embed = get_embeddings(embed) if num_layers>1: - self.lstm = LSTM(embed.embedding_dim, num_layers=num_layers, hidden_size=hidden_size, bidirectional=True, + self.lstm = LSTM(self.embed.embedding_dim, num_layers=num_layers, hidden_size=hidden_size, bidirectional=True, batch_first=True, dropout=dropout) else: - self.lstm = LSTM(embed.embedding_dim, num_layers=num_layers, hidden_size=hidden_size, bidirectional=True, + self.lstm = LSTM(self.embed.embedding_dim, num_layers=num_layers, hidden_size=hidden_size, bidirectional=True, batch_first=True) self.dropout = nn.Dropout(dropout) - self.fc = nn.Linear(hidden_size, num_classes) + self.fc = nn.Linear(hidden_size*2, num_classes) trans = None if target_vocab is not None and encoding_type is not None: @@ -56,7 +56,7 @@ class BiLSTMCRF(BaseModel): def _forward(self, words, seq_len=None, target=None): words = self.embed(words) - feats = self.lstm(words, seq_len=seq_len) + feats, _ = self.lstm(words, seq_len=seq_len) feats = self.fc(feats) feats = self.dropout(feats) logits = F.log_softmax(feats, dim=-1) @@ -142,8 +142,6 @@ class SeqLabeling(BaseModel): """ x = x.float() y = y.long() - assert x.shape[:2] == y.shape - assert y.shape == self.mask.shape total_loss = self.crf(x, y, mask) return torch.mean(total_loss) @@ -195,36 +193,29 @@ class AdvSeqLabel(nn.Module): allowed_transitions=allowed_transitions(id2words, encoding_type=encoding_type)) - def _decode(self, x): + def _decode(self, x, mask): """ :param torch.FloatTensor x: [batch_size, max_len, tag_size] + :param torch.ByteTensor mask: [batch_size, max_len] :return torch.LongTensor, [batch_size, max_len] """ - tag_seq, _ = self.Crf.viterbi_decode(x, self.mask) + tag_seq, _ = self.Crf.viterbi_decode(x, mask) return tag_seq - def _internal_loss(self, x, y): + def _internal_loss(self, x, y, mask): """ Negative log likelihood loss. :param x: Tensor, [batch_size, max_len, tag_size] :param y: Tensor, [batch_size, max_len] + :param mask: Tensor, [batch_size, max_len] :return loss: a scalar Tensor """ x = x.float() y = y.long() - assert x.shape[:2] == y.shape - assert y.shape == self.mask.shape - total_loss = self.Crf(x, y, self.mask) + total_loss = self.Crf(x, y, mask) return torch.mean(total_loss) - def _make_mask(self, x, seq_len): - batch_size, max_len = x.size(0), x.size(1) - mask = seq_len_to_mask(seq_len) - mask = mask.view(batch_size, max_len) - mask = mask.to(x).float() - return mask - def _forward(self, words, seq_len, target=None): """ :param torch.LongTensor words: [batch_size, mex_len] @@ -236,15 +227,13 @@ class AdvSeqLabel(nn.Module): words = words.long() seq_len = seq_len.long() - self.mask = self._make_mask(words, seq_len) - - # seq_len = seq_len.long() + mask = seq_len_to_mask(seq_len, max_len=words.size(1)) + target = target.long() if target is not None else None if next(self.parameters()).is_cuda: words = words.cuda() - self.mask = self.mask.cuda() - + x = self.Embedding(words) x = self.norm1(x) # [batch_size, max_len, word_emb_dim] @@ -257,9 +246,9 @@ class AdvSeqLabel(nn.Module): x = self.drop(x) x = self.Linear2(x) if target is not None: - return {"loss": self._internal_loss(x, target)} + return {"loss": self._internal_loss(x, target, mask)} else: - return {"pred": self._decode(x)} + return {"pred": self._decode(x, mask)} def forward(self, words, seq_len, target): """ diff --git a/test/models/test_sequence_labeling.py b/test/models/test_sequence_labeling.py index 3a70e381..815d7047 100644 --- a/test/models/test_sequence_labeling.py +++ b/test/models/test_sequence_labeling.py @@ -3,9 +3,24 @@ import unittest from .model_runner import * -from fastNLP.models.sequence_labeling import SeqLabeling, AdvSeqLabel +from fastNLP.models.sequence_labeling import SeqLabeling, AdvSeqLabel, BiLSTMCRF from fastNLP.core.losses import LossInForward +class TestBiLSTM(unittest.TestCase): + def test_case1(self): + # 测试能否正常运行CNN + init_emb = (VOCAB_SIZE, 30) + model = BiLSTMCRF(init_emb, + hidden_size=30, + num_classes=NUM_CLS) + + data = RUNNER.prepare_pos_tagging_data() + data.set_input('target') + loss = LossInForward() + metric = AccuracyMetric(pred=C.OUTPUT, target=C.TARGET, seq_len=C.INPUT_LEN) + RUNNER.run_model(model, data, loss, metric) + + class TesSeqLabel(unittest.TestCase): def test_case1(self): # 测试能否正常运行CNN From 091f24e393f434eba66937af65adcbcd8ea3d3cf Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Sun, 1 Sep 2019 10:15:11 +0800 Subject: [PATCH 26/92] fix some bugs in test code. --- test/__init__.py | 3 +++ test/core/test_utils.py | 17 +++++++++++------ test/models/__init__.py | 0 test/models/test_bert.py | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 test/__init__.py create mode 100644 test/models/__init__.py diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..c7a5f082 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,3 @@ +import fastNLP + +__all__ = ["fastNLP"] diff --git a/test/core/test_utils.py b/test/core/test_utils.py index 363d5fa1..29645fb1 100644 --- a/test/core/test_utils.py +++ b/test/core/test_utils.py @@ -119,7 +119,8 @@ class TestCache(unittest.TestCase): def test_cache_save(self): try: start_time = time.time() - embed, vocab, d = process_data_1('test/data_for_tests/word2vec_test.txt', 'test/data_for_tests/cws_train') + embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', + 'test/data_for_tests/cws_train') end_time = time.time() pre_time = end_time - start_time with open('test/demo1.pkl', 'rb') as f: @@ -128,7 +129,8 @@ class TestCache(unittest.TestCase): for i in range(embed.shape[0]): self.assertListEqual(embed[i].tolist(), _embed[i].tolist()) start_time = time.time() - embed, vocab, d = process_data_1('test/data_for_tests/word2vec_test.txt', 'test/data_for_tests/cws_train') + embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', + 'test/data_for_tests/cws_train') end_time = time.time() read_time = end_time - start_time print("Read using {:.3f}, while prepare using:{:.3f}".format(read_time, pre_time)) @@ -139,7 +141,7 @@ class TestCache(unittest.TestCase): def test_cache_save_overwrite_path(self): try: start_time = time.time() - embed, vocab, d = process_data_1('test/data_for_tests/word2vec_test.txt', 'test/data_for_tests/cws_train', + embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', 'test/data_for_tests/cws_train', _cache_fp='test/demo_overwrite.pkl') end_time = time.time() pre_time = end_time - start_time @@ -149,7 +151,8 @@ class TestCache(unittest.TestCase): for i in range(embed.shape[0]): self.assertListEqual(embed[i].tolist(), _embed[i].tolist()) start_time = time.time() - embed, vocab, d = process_data_1('test/data_for_tests/word2vec_test.txt', 'test/data_for_tests/cws_train', + embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', + 'test/data_for_tests/cws_train', _cache_fp='test/demo_overwrite.pkl') end_time = time.time() read_time = end_time - start_time @@ -161,7 +164,8 @@ class TestCache(unittest.TestCase): def test_cache_refresh(self): try: start_time = time.time() - embed, vocab, d = process_data_1('test/data_for_tests/word2vec_test.txt', 'test/data_for_tests/cws_train', + embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', + 'test/data_for_tests/cws_train', _refresh=True) end_time = time.time() pre_time = end_time - start_time @@ -171,7 +175,8 @@ class TestCache(unittest.TestCase): for i in range(embed.shape[0]): self.assertListEqual(embed[i].tolist(), _embed[i].tolist()) start_time = time.time() - embed, vocab, d = process_data_1('test/data_for_tests/word2vec_test.txt', 'test/data_for_tests/cws_train', + embed, vocab, d = process_data_1('test/data_for_tests/embedding/small_static_embedding/word2vec_test.txt', + 'test/data_for_tests/cws_train', _refresh=True) end_time = time.time() read_time = end_time - start_time diff --git a/test/models/__init__.py b/test/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/models/test_bert.py b/test/models/test_bert.py index 2b310edf..969a8594 100644 --- a/test/models/test_bert.py +++ b/test/models/test_bert.py @@ -82,7 +82,7 @@ class TestBert(unittest.TestCase): def test_bert_5(self): vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) - embed = BertEmbedding(vocab, model_dir_or_name='./../data_for_tests/embedding/small_bert', + embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', include_cls_sep=True) model = BertForSentenceMatching(embed) From 1c2ee50c47b0b59b81a828838bf531c54fea5181 Mon Sep 17 00:00:00 2001 From: yunfan Date: Sun, 1 Sep 2019 10:31:14 +0800 Subject: [PATCH 27/92] [fix] EchoCallback --- fastNLP/core/callback.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index dde9a31a..5167b09f 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -1031,12 +1031,11 @@ class EchoCallback(Callback): def __init__(self, name, out=sys.stdout): super(EchoCallback, self).__init__() self.name = name - self.out = out + self.out = out # deprecated def __getattribute__(self, item): if item.startswith('on_'): - logger.info('{}.{} has been called at pid: {}'.format(self.name, item, os.getpid()), - file=self.out) + logger.info('{}.{} has been called at pid: {}'.format(self.name, item, os.getpid())) return super(EchoCallback, self).__getattribute__(item) From b9aa05f6cf371a9ceb99463c445fa000a724fa21 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Sun, 1 Sep 2019 11:22:42 +0800 Subject: [PATCH 28/92] add testing codes and data for loader and pipe. --- test/data_for_tests/io/cws_msra/dev.txt | 2 ++ test/data_for_tests/io/cws_msra/test.txt | 2 ++ test/data_for_tests/io/cws_msra/train.txt | 3 +++ test/data_for_tests/io/imdb/dev.txt | 2 ++ test/data_for_tests/io/imdb/test.txt | 2 ++ test/data_for_tests/io/imdb/train.txt | 2 ++ test/data_for_tests/io/rte/dev.tsv | 3 +++ test/data_for_tests/io/rte/test.tsv | 3 +++ test/data_for_tests/io/rte/train.tsv | 4 ++++ test/io/loader/test_classification_loader.py | 8 ++++++++ test/io/loader/test_conll_loader.py | 14 ++++++++++++-- test/io/loader/test_cws_loader.py | 13 ++++++++++++- test/io/loader/test_matching_loader.py | 8 ++++++++ test/io/pipe/test_classification.py | 8 ++++++++ test/io/pipe/test_conll.py | 14 ++++++++++++-- test/io/pipe/test_cws.py | 12 +++++++++++- test/io/pipe/test_matching.py | 8 ++++++++ 17 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 test/data_for_tests/io/cws_msra/dev.txt create mode 100644 test/data_for_tests/io/cws_msra/test.txt create mode 100644 test/data_for_tests/io/cws_msra/train.txt create mode 100644 test/data_for_tests/io/imdb/dev.txt create mode 100644 test/data_for_tests/io/imdb/test.txt create mode 100644 test/data_for_tests/io/imdb/train.txt create mode 100644 test/data_for_tests/io/rte/dev.tsv create mode 100644 test/data_for_tests/io/rte/test.tsv create mode 100644 test/data_for_tests/io/rte/train.tsv diff --git a/test/data_for_tests/io/cws_msra/dev.txt b/test/data_for_tests/io/cws_msra/dev.txt new file mode 100644 index 00000000..9c6b34ee --- /dev/null +++ b/test/data_for_tests/io/cws_msra/dev.txt @@ -0,0 +1,2 @@ +“ 人们 常 说 生活 是 一 部 教科书 ， 而 血 与 火 的 战争 更 是 不可多得 的 教科书 ， 她 确实 是 名副其实 的 ‘ 我 的 大学 ’ 。 +他 “ 严格要求 自己 ， 从 一个 科举 出身 的 进士 成为 一个 伟大 的 民主主义 者 ， 进而 成为 一 位 杰出 的 党外 共产主义 战士 ， 献身 于 崇高 的 共产主义 事业 。 diff --git a/test/data_for_tests/io/cws_msra/test.txt b/test/data_for_tests/io/cws_msra/test.txt new file mode 100644 index 00000000..8d5c6b3c --- /dev/null +++ b/test/data_for_tests/io/cws_msra/test.txt @@ -0,0 +1,2 @@ +扬帆 远东 做 与 中国 合作 的 先行 +希腊 的 经济 结构 较 特殊 。 diff --git a/test/data_for_tests/io/cws_msra/train.txt b/test/data_for_tests/io/cws_msra/train.txt new file mode 100644 index 00000000..35c2cad0 --- /dev/null +++ b/test/data_for_tests/io/cws_msra/train.txt @@ -0,0 +1,3 @@ +“ 心 静 渐 知 春 似 海 ， 花 深 每 觉 影 生 香 。 +“ 吃 屎 的 东西 ， 连 一 捆 麦 也 铡 不 动 呀 ？ +复旦大学 百年 校庆 。 \ No newline at end of file diff --git a/test/data_for_tests/io/imdb/dev.txt b/test/data_for_tests/io/imdb/dev.txt new file mode 100644 index 00000000..6b548a0c --- /dev/null +++ b/test/data_for_tests/io/imdb/dev.txt @@ -0,0 +1,2 @@ +neg It, at all, you have seen when harry met sally, then avoid this one. It will not only make you bang your head on the table as why can't bollywood even make a good remake; but also annoy you with the so called funny moments in it. The charm of the movie is missing. Ranee looks terrible. Saif tries to act like he is one hell of an actor. The plots that have been picked up from the original, don't look effective either. The part where both of them bring their friends along and they hit a note, it just doesn't look appealing. What can be more disastrous? you wanna waste some money, this is what you can get. Otherwise, put some more bucks, and watch the original. Its too good to miss.. +neg The monster from Enemy Mine somehow made his way into a small mountain community, where he has taken up residence. He's being hunted by a female doctor-turned-vigilante who is out to exterminate him. This female assassin, who looks like a refugee from a Motley Crue video, rides around on a motorcycle and tries to save a bunch of kids who have chosen to have a Big Chill weekend right smack dab in the middle of the monster's turf. Decapitations and lots of blood are primarily in place to draw attention away from the story which limps along like a bad version of the Island of Dr. Moreau (and yes, it's worse than the one with Val Kilmer). diff --git a/test/data_for_tests/io/imdb/test.txt b/test/data_for_tests/io/imdb/test.txt new file mode 100644 index 00000000..c9bfae74 --- /dev/null +++ b/test/data_for_tests/io/imdb/test.txt @@ -0,0 +1,2 @@ +neg Alan Rickman & Emma Thompson give good performances with southern/New Orleans accents in this detective flick. It's worth seeing for their scenes- and Rickman's scene with Hal Holbrook. These three actors mannage to entertain us no matter what the movie, it seems. The plot for the movie shows potential, but one gets the impression in watching the film that it was not pulled off as well as it could have been. The fact that it is cluttered by a rather uninteresting subplot and mostly uninteresting kidnappers really muddles things. The movie is worth a view- if for nothing more than entertaining performances by Rickman, Thompson, and Holbrook. +neg I have seen this movie and I did not care for this movie anyhow. I would not think about going to Paris because I do not like this country and its national capital. I do not like to learn french anyhow because I do not understand their language. Why would I go to France when I rather go to Germany or the United Kingdom? Germany and the United Kingdom are the nations I tolerate. Apparently the Olsen Twins do not understand the French language just like me. Therefore I will not bother the France trip no matter what. I might as well stick to the United Kingdom and meet single women and play video games if there is a video arcade. That is all. diff --git a/test/data_for_tests/io/imdb/train.txt b/test/data_for_tests/io/imdb/train.txt new file mode 100644 index 00000000..d6ac6b68 --- /dev/null +++ b/test/data_for_tests/io/imdb/train.txt @@ -0,0 +1,2 @@ +neg I'll try to use words to describe this on....

I saw the original, which was good in its own way, but back then I should have feared a sequel.

And I was 'afraid' when I picked this one up, but now that I've seen it, I have to say, it's even worse then I thought. Why these movies still get money still makes my mind spin.

Let's start with the actors;they aren't all that good, but it has to be said, some make heads turn by being just plain awful. But what can an actor do with a script like this one. It's trying to be a copy of the original only this time the places have changed, any form of story is gone and any attempt of actually coming up with something that hasn't been done before, fails miserably. In a futile attempt to get it up-to-date, they try to make it exciting by making use of the whole 'big-brother' theme , but that has been worn out ages ago and offers nothing but a filler for between the beginning and the end. An attempt was made to try to save the movie by making a ton of references to the '83 original, but it just ended up being plain funny and sometimes a bit sad. In conclusion, if you have nothing , and I mean nothing , to do... go watch it, or play Frisbee... with the DVD.... by yourself. It'll offer you the same amount of fun.. I promise +pos This movie is totally wicked! It's really great to see MJH in a different role than her Sabrina character! The plot is totally cool, and the characters are excellently written. Definitely one of the best movies!! diff --git a/test/data_for_tests/io/rte/dev.tsv b/test/data_for_tests/io/rte/dev.tsv new file mode 100644 index 00000000..725d7542 --- /dev/null +++ b/test/data_for_tests/io/rte/dev.tsv @@ -0,0 +1,3 @@ +index sentence1 sentence2 label +0 Dana Reeve, the widow of the actor Christopher Reeve, has died of lung cancer at age 44, according to the Christopher Reeve Foundation. Christopher Reeve had an accident. not_entailment +1 Yet, we now are discovering that antibiotics are losing their effectiveness against illness. Disease-causing bacteria are mutating faster than we can come up with new antibiotics to fight the new variations. Bacteria is winning the war against antibiotics. entailment diff --git a/test/data_for_tests/io/rte/test.tsv b/test/data_for_tests/io/rte/test.tsv new file mode 100644 index 00000000..aeceb467 --- /dev/null +++ b/test/data_for_tests/io/rte/test.tsv @@ -0,0 +1,3 @@ +index sentence1 sentence2 +0 Mangla was summoned after Madhumita's sister Nidhi Shukla, who was the first witness in the case. Shukla is related to Mangla. +1 Authorities in Brazil say that more than 200 people are being held hostage in a prison in the country's remote, Amazonian-jungle state of Rondonia. Authorities in Brazil hold 200 people as hostage. diff --git a/test/data_for_tests/io/rte/train.tsv b/test/data_for_tests/io/rte/train.tsv new file mode 100644 index 00000000..9f3dab6e --- /dev/null +++ b/test/data_for_tests/io/rte/train.tsv @@ -0,0 +1,4 @@ +index sentence1 sentence2 label +0 No Weapons of Mass Destruction Found in Iraq Yet. Weapons of Mass Destruction Found in Iraq. not_entailment +1 A place of sorrow, after Pope John Paul II died, became a place of celebration, as Roman Catholic faithful gathered in downtown Chicago to mark the installation of new Pope Benedict XVI. Pope Benedict XVI is the new leader of the Roman Catholic Church. entailment +2 Herceptin was already approved to treat the sickest breast cancer patients, and the company said, Monday, it will discuss with federal regulators the possibility of prescribing the drug for more breast cancer patients. Herceptin can be used to treat breast cancer. entailment diff --git a/test/io/loader/test_classification_loader.py b/test/io/loader/test_classification_loader.py index 28f08921..1438a014 100644 --- a/test/io/loader/test_classification_loader.py +++ b/test/io/loader/test_classification_loader.py @@ -17,3 +17,11 @@ class TestDownload(unittest.TestCase): for loader in [YelpFullLoader, YelpPolarityLoader, IMDBLoader, SST2Loader, SSTLoader]: data_bundle = loader().load() print(data_bundle) + + +class TestLoad(unittest.TestCase): + + def test_load(self): + for loader in [IMDBLoader]: + data_bundle = loader().load('test/data_for_tests/io/imdb') + print(data_bundle) diff --git a/test/io/loader/test_conll_loader.py b/test/io/loader/test_conll_loader.py index e44b8a2a..861de5a5 100644 --- a/test/io/loader/test_conll_loader.py +++ b/test/io/loader/test_conll_loader.py @@ -1,7 +1,9 @@ import unittest import os -from fastNLP.io.loader.conll import MsraNERLoader, PeopleDailyNERLoader, WeiboNERLoader +from fastNLP.io.loader.conll import MsraNERLoader, PeopleDailyNERLoader, WeiboNERLoader, \ + Conll2003Loader + class MSRANERTest(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") @@ -10,12 +12,20 @@ class MSRANERTest(unittest.TestCase): data_bundle = MsraNERLoader().load() print(data_bundle) + class PeopleDailyTest(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_download(self): PeopleDailyNERLoader().download() + class WeiboNERTest(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_download(self): - WeiboNERLoader().download() \ No newline at end of file + WeiboNERLoader().download() + + +class TestConll2003Loader(unittest.TestCase): + def test__load(self): + Conll2003Loader()._load('test/data_for_tests/conll_2003_example.txt') + diff --git a/test/io/loader/test_cws_loader.py b/test/io/loader/test_cws_loader.py index 6ad607c3..8b5d4081 100644 --- a/test/io/loader/test_cws_loader.py +++ b/test/io/loader/test_cws_loader.py @@ -10,4 +10,15 @@ class CWSLoaderTest(unittest.TestCase): for dataset_name in dataset_names: with self.subTest(dataset_name=dataset_name): data_bundle = CWSLoader(dataset_name=dataset_name).load() - print(data_bundle) \ No newline at end of file + print(data_bundle) + + +class RunCWSLoaderTest(unittest.TestCase): + def test_cws_loader(self): + dataset_names = ['msra'] + for dataset_name in dataset_names: + with self.subTest(dataset_name=dataset_name): + data_bundle = CWSLoader(dataset_name=dataset_name).load( + f'test/data_for_tests/io/cws_{dataset_name}' + ) + print(data_bundle) diff --git a/test/io/loader/test_matching_loader.py b/test/io/loader/test_matching_loader.py index 5c1a91f1..652cf161 100644 --- a/test/io/loader/test_matching_loader.py +++ b/test/io/loader/test_matching_loader.py @@ -20,3 +20,11 @@ class TestDownload(unittest.TestCase): data_bundle = loader().load() print(data_bundle) + +class TestLoad(unittest.TestCase): + + def test_load(self): + for loader in [RTELoader]: + data_bundle = loader().load('test/data_for_tests/io/rte') + print(data_bundle) + diff --git a/test/io/pipe/test_classification.py b/test/io/pipe/test_classification.py index 39dc71e0..c6e2005e 100644 --- a/test/io/pipe/test_classification.py +++ b/test/io/pipe/test_classification.py @@ -11,3 +11,11 @@ class TestPipe(unittest.TestCase): print(pipe) data_bundle = pipe(tokenizer='raw').process_from_file() print(data_bundle) + + +class TestRunPipe(unittest.TestCase): + + def test_load(self): + for pipe in [IMDBPipe]: + data_bundle = pipe(tokenizer='raw').process_from_file('test/data_for_tests/io/imdb') + print(data_bundle) diff --git a/test/io/pipe/test_conll.py b/test/io/pipe/test_conll.py index e8879d71..6f6c4fad 100644 --- a/test/io/pipe/test_conll.py +++ b/test/io/pipe/test_conll.py @@ -1,6 +1,7 @@ import unittest import os -from fastNLP.io import MsraNERPipe, PeopleDailyPipe, WeiboNERPipe +from fastNLP.io import MsraNERPipe, PeopleDailyPipe, WeiboNERPipe, Conll2003Pipe, Conll2003NERPipe + @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") class TestPipe(unittest.TestCase): @@ -9,4 +10,13 @@ class TestPipe(unittest.TestCase): with self.subTest(pipe=pipe): print(pipe) data_bundle = pipe().process_from_file() - print(data_bundle) \ No newline at end of file + print(data_bundle) + + +class TestRunPipe(unittest.TestCase): + def test_conll2003(self): + for pipe in [Conll2003Pipe, Conll2003NERPipe]: + with self.subTest(pipe=pipe): + print(pipe) + data_bundle = pipe().process_from_file('test/data_for_tests/conll_2003_example.txt') + print(data_bundle) diff --git a/test/io/pipe/test_cws.py b/test/io/pipe/test_cws.py index 2fc57ae2..dd901a25 100644 --- a/test/io/pipe/test_cws.py +++ b/test/io/pipe/test_cws.py @@ -3,6 +3,7 @@ import unittest import os from fastNLP.io.pipe.cws import CWSPipe + class CWSPipeTest(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_process_from_file(self): @@ -10,4 +11,13 @@ class CWSPipeTest(unittest.TestCase): for dataset_name in dataset_names: with self.subTest(dataset_name=dataset_name): data_bundle = CWSPipe(dataset_name=dataset_name).process_from_file() - print(data_bundle) \ No newline at end of file + print(data_bundle) + + +class RunCWSPipeTest(unittest.TestCase): + def test_process_from_file(self): + dataset_names = ['msra'] + for dataset_name in dataset_names: + with self.subTest(dataset_name=dataset_name): + data_bundle = CWSPipe().process_from_file(f'test/data_for_tests/io/cws_{dataset_name}') + print(data_bundle) diff --git a/test/io/pipe/test_matching.py b/test/io/pipe/test_matching.py index c057bb0c..33904e7a 100644 --- a/test/io/pipe/test_matching.py +++ b/test/io/pipe/test_matching.py @@ -24,3 +24,11 @@ class TestBertPipe(unittest.TestCase): print(pipe) data_bundle = pipe(tokenizer='raw').process_from_file() print(data_bundle) + + +class TestRunPipe(unittest.TestCase): + + def test_load(self): + for pipe in [RTEPipe, RTEBertPipe]: + data_bundle = pipe(tokenizer='raw').process_from_file('test/data_for_tests/io/rte') + print(data_bundle) From 1994029ab84fb70ee8d790732006747f5d918a02 Mon Sep 17 00:00:00 2001 From: yh Date: Mon, 2 Sep 2019 15:59:45 +0800 Subject: [PATCH 29/92] =?UTF-8?q?1.=E5=BD=93=E5=89=8D=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E7=9A=84encoding=5Ftype=E9=83=BD=E6=94=AF=E6=8C=81=E4=BB=8Etag?= =?UTF-8?q?=5Fvocab=E4=B8=AD=E8=87=AA=E5=8A=A8=E5=88=A4=E6=96=AD;=E9=81=BF?= =?UTF-8?q?=E5=85=8D=E8=A7=A6=E5=8F=91=E6=97=A0=E6=84=8F=E8=AF=86=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84metric=20bug;=202.=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E9=83=A8=E5=88=86inplace=E6=93=8D=E4=BD=9C=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E6=B1=82=E5=AF=BC=E7=9A=84=E9=97=AE=E9=A2=98;=203.Vocabulary?= =?UTF-8?q?=E5=B0=86=E4=B8=80=E4=BA=9B=E5=B1=9E=E6=80=A7=E9=80=9A=E8=BF=87?= =?UTF-8?q?property=E6=9A=B4=E9=9C=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/metrics.py | 85 +++++++++++++++++++------ fastNLP/core/vocabulary.py | 70 +++++++++++++-------- fastNLP/io/data_bundle.py | 43 ++++++++++++- fastNLP/io/pipe/conll.py | 2 +- fastNLP/models/biaffine_parser.py | 15 +++-- fastNLP/modules/decoder/crf.py | 38 ++++++++---- test/core/test_metrics.py | 41 +++++++++++- test/modules/decoder/test_CRF.py | 100 +++++++++++++++++++++++++++++- 8 files changed, 321 insertions(+), 73 deletions(-) diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index 0dc601a3..b06e5459 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -24,7 +24,7 @@ from .utils import seq_len_to_mask from .vocabulary import Vocabulary from abc import abstractmethod import warnings - +from typing import Union class MetricBase(object): """ @@ -337,15 +337,18 @@ class AccuracyMetric(MetricBase): raise TypeError(f"`seq_lens` in {_get_func_signature(self.evaluate)} must be torch.Tensor," f"got {type(seq_len)}.") - if seq_len is not None: - masks = seq_len_to_mask(seq_len=seq_len) + if seq_len is not None and target.dim()>1: + max_len = target.size(1) + masks = seq_len_to_mask(seq_len=seq_len, max_len=max_len) else: masks = None - if pred.size() == target.size(): + if pred.dim() == target.dim(): pass - elif len(pred.size()) == len(target.size()) + 1: + elif pred.dim() == target.dim() + 1: pred = pred.argmax(dim=-1) + if seq_len is None: + warnings.warn("You are not passing `seq_len` to exclude pad when calculate accuracy.") else: raise RuntimeError(f"In {_get_func_signature(self.evaluate)}, when pred have " f"size:{pred.size()}, target should have size: {pred.size()} or " @@ -493,20 +496,63 @@ def _bio_tag_to_spans(tags, ignore_labels=None): return [(span[0], (span[1][0], span[1][1] + 1)) for span in spans if span[0] not in ignore_labels] -def _check_tag_vocab_and_encoding_type(vocab:Vocabulary, encoding_type:str): +def _get_encoding_type_from_tag_vocab(tag_vocab:Union[Vocabulary, dict])->str: + """ + 给定Vocabulary自动判断是哪种类型的encoding, 支持判断bmes, bioes, bmeso, bio + + :param tag_vocab: 支持传入tag Vocabulary; 或者传入形如{0:"O", 1:"B-tag1"}，即index在前，tag在后的dict。 + :return: + """ + tag_set = set() + unk_token = '' + pad_token = '' + if isinstance(tag_vocab, Vocabulary): + unk_token = tag_vocab.unknown + pad_token = tag_vocab.padding + tag_vocab = tag_vocab.idx2word + for idx, tag in tag_vocab.items(): + if tag in (unk_token, pad_token): + continue + tag = tag[:1].lower() + tag_set.add(tag) + + bmes_tag_set = set('bmes') + if tag_set == bmes_tag_set: + return 'bmes' + bio_tag_set = set('bio') + if tag_set == bio_tag_set: + return 'bio' + bmeso_tag_set = set('bmeso') + if tag_set == bmeso_tag_set: + return 'bmeso' + bioes_tag_set = set('bioes') + if tag_set == bioes_tag_set: + return 'bioes' + raise RuntimeError("encoding_type cannot be inferred automatically. Only support " + "'bio', 'bmes', 'bmeso', 'bioes' type.") + + +def _check_tag_vocab_and_encoding_type(tag_vocab:Union[Vocabulary, dict], encoding_type:str): """ 检查vocab中的tag是否与encoding_type是匹配的 - :param vocab: target的Vocabulary + :param tag_vocab: 支持传入tag Vocabulary; 或者传入形如{0:"O", 1:"B-tag1"}，即index在前，tag在后的dict。 :param encoding_type: bio, bmes, bioes, bmeso :return: """ tag_set = set() - for tag, idx in vocab: - if idx in (vocab.unknown_idx, vocab.padding_idx): + unk_token = '' + pad_token = '' + if isinstance(tag_vocab, Vocabulary): + unk_token = tag_vocab.unknown + pad_token = tag_vocab.padding + tag_vocab = tag_vocab.idx2word + for idx, tag in tag_vocab.items(): + if tag in (unk_token, pad_token): continue tag = tag[:1].lower() tag_set.add(tag) + tags = encoding_type for tag in tag_set: assert tag in tags, f"{tag} is not a valid tag in encoding type:{encoding_type}. Please check your " \ @@ -549,7 +595,7 @@ class SpanFPreRecMetric(MetricBase): :param str pred: 用该key在evaluate()时从传入dict中取出prediction数据。 为None，则使用 `pred` 取数据 :param str target: 用该key在evaluate()时从传入dict中取出target数据。 为None，则使用 `target` 取数据 :param str seq_len: 用该key在evaluate()时从传入dict中取出sequence length数据。为None，则使用 `seq_len` 取数据。 - :param str encoding_type: 目前支持bio, bmes, bmeso, bioes + :param str encoding_type: 目前支持bio, bmes, bmeso, bioes。默认为None，通过tag_vocab自动判断. :param list ignore_labels: str 组成的list. 这个list中的class不会被用于计算。例如在POS tagging时传入['NN']，则不会计算'NN'这 个label :param bool only_gross: 是否只计算总的f1, precision, recall的值；如果为False，不仅返回总的f1, pre, rec, 还会返回每个 @@ -560,18 +606,21 @@ class SpanFPreRecMetric(MetricBase): 常用为beta=0.5, 1, 2. 若为0.5则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 """ - def __init__(self, tag_vocab, pred=None, target=None, seq_len=None, encoding_type='bio', ignore_labels=None, + def __init__(self, tag_vocab, pred=None, target=None, seq_len=None, encoding_type=None, ignore_labels=None, only_gross=True, f_type='micro', beta=1): - - encoding_type = encoding_type.lower() - + if not isinstance(tag_vocab, Vocabulary): raise TypeError("tag_vocab can only be fastNLP.Vocabulary, not {}.".format(type(tag_vocab))) if f_type not in ('micro', 'macro'): raise ValueError("f_type only supports `micro` or `macro`', got {}.".format(f_type)) - - self.encoding_type = encoding_type - _check_tag_vocab_and_encoding_type(tag_vocab, encoding_type) + + if encoding_type: + encoding_type = encoding_type.lower() + _check_tag_vocab_and_encoding_type(tag_vocab, encoding_type) + self.encoding_type = encoding_type + else: + self.encoding_type = _get_encoding_type_from_tag_vocab(tag_vocab) + if self.encoding_type == 'bmes': self.tag_to_span_func = _bmes_tag_to_spans elif self.encoding_type == 'bio': @@ -581,7 +630,7 @@ class SpanFPreRecMetric(MetricBase): elif self.encoding_type == 'bioes': self.tag_to_span_func = _bioes_tag_to_spans else: - raise ValueError("Only support 'bio', 'bmes', 'bmeso' type.") + raise ValueError("Only support 'bio', 'bmes', 'bmeso', 'bioes' type.") self.ignore_labels = ignore_labels self.f_type = f_type diff --git a/fastNLP/core/vocabulary.py b/fastNLP/core/vocabulary.py index cd4f2c0f..b0f9650a 100644 --- a/fastNLP/core/vocabulary.py +++ b/fastNLP/core/vocabulary.py @@ -39,7 +39,7 @@ def _check_build_vocab(func): @wraps(func) # to solve missing docstring def _wrapper(self, *args, **kwargs): - if self.word2idx is None or self.rebuild is True: + if self._word2idx is None or self.rebuild is True: self.build_vocab() return func(self, *args, **kwargs) @@ -95,12 +95,30 @@ class Vocabulary(object): self.word_count = Counter() self.unknown = unknown self.padding = padding - self.word2idx = None - self.idx2word = None + self._word2idx = None + self._idx2word = None self.rebuild = True # 用于承载不需要单独创建entry的词语，具体见from_dataset()方法 self._no_create_word = Counter() - + + @property + @_check_build_vocab + def word2idx(self): + return self._word2idx + + @word2idx.setter + def word2idx(self, value): + self._word2idx = value + + @property + @_check_build_vocab + def idx2word(self): + return self._idx2word + + @idx2word.setter + def idx2word(self, value): + self._word2idx = value + @_check_build_status def update(self, word_lst, no_create_entry=False): """依次增加序列中词在词典中的出现频率 @@ -187,21 +205,21 @@ class Vocabulary(object): 但已经记录在词典中的词, 不会改变对应的 `int` """ - if self.word2idx is None: - self.word2idx = {} + if self._word2idx is None: + self._word2idx = {} if self.padding is not None: - self.word2idx[self.padding] = len(self.word2idx) + self._word2idx[self.padding] = len(self._word2idx) if self.unknown is not None: - self.word2idx[self.unknown] = len(self.word2idx) + self._word2idx[self.unknown] = len(self._word2idx) max_size = min(self.max_size, len(self.word_count)) if self.max_size else None words = self.word_count.most_common(max_size) if self.min_freq is not None: words = filter(lambda kv: kv[1] >= self.min_freq, words) - if self.word2idx is not None: - words = filter(lambda kv: kv[0] not in self.word2idx, words) - start_idx = len(self.word2idx) - self.word2idx.update({w: i + start_idx for i, (w, _) in enumerate(words)}) + if self._word2idx is not None: + words = filter(lambda kv: kv[0] not in self._word2idx, words) + start_idx = len(self._word2idx) + self._word2idx.update({w: i + start_idx for i, (w, _) in enumerate(words)}) self.build_reverse_vocab() self.rebuild = False return self @@ -211,12 +229,12 @@ class Vocabulary(object): 基于 `word to index` dict, 构建 `index to word` dict. """ - self.idx2word = {i: w for w, i in self.word2idx.items()} + self._idx2word = {i: w for w, i in self._word2idx.items()} return self @_check_build_vocab def __len__(self): - return len(self.word2idx) + return len(self._word2idx) @_check_build_vocab def __contains__(self, item): @@ -226,7 +244,7 @@ class Vocabulary(object): :param item: the word :return: True or False """ - return item in self.word2idx + return item in self._word2idx def has_word(self, w): """ @@ -248,10 +266,10 @@ class Vocabulary(object): vocab[w] """ - if w in self.word2idx: - return self.word2idx[w] + if w in self._word2idx: + return self._word2idx[w] if self.unknown is not None: - return self.word2idx[self.unknown] + return self._word2idx[self.unknown] else: raise ValueError("word `{}` not in vocabulary".format(w)) @@ -405,7 +423,7 @@ class Vocabulary(object): """ if self.unknown is None: return None - return self.word2idx[self.unknown] + return self._word2idx[self.unknown] @property @_check_build_vocab @@ -415,7 +433,7 @@ class Vocabulary(object): """ if self.padding is None: return None - return self.word2idx[self.padding] + return self._word2idx[self.padding] @_check_build_vocab def to_word(self, idx): @@ -425,7 +443,7 @@ class Vocabulary(object): :param int idx: the index :return str word: the word """ - return self.idx2word[idx] + return self._idx2word[idx] def clear(self): """ @@ -434,8 +452,8 @@ class Vocabulary(object): :return: """ self.word_count.clear() - self.word2idx = None - self.idx2word = None + self._word2idx = None + self._idx2word = None self.rebuild = True self._no_create_word.clear() return self @@ -446,8 +464,8 @@ class Vocabulary(object): """ len(self) # make sure vocab has been built state = self.__dict__.copy() - # no need to pickle idx2word as it can be constructed from word2idx - del state['idx2word'] + # no need to pickle _idx2word as it can be constructed from _word2idx + del state['_idx2word'] return state def __setstate__(self, state): @@ -462,5 +480,5 @@ class Vocabulary(object): @_check_build_vocab def __iter__(self): - for word, index in self.word2idx.items(): + for word, index in self._word2idx.items(): yield word, index diff --git a/fastNLP/io/data_bundle.py b/fastNLP/io/data_bundle.py index f30add34..3e7f39d3 100644 --- a/fastNLP/io/data_bundle.py +++ b/fastNLP/io/data_bundle.py @@ -8,7 +8,7 @@ __all__ = [ from ..core.dataset import DataSet from ..core.vocabulary import Vocabulary - +from typing import Union class DataBundle: """ @@ -191,7 +191,7 @@ class DataBundle: raise KeyError(f"{field_name} not found DataSet:{name}.") return self - def rename_field(self, field_name, new_field_name, ignore_miss_dataset=True): + def rename_field(self, field_name, new_field_name, ignore_miss_dataset=True, rename_vocab=True): """ 将DataBundle中所有DataSet中名为field_name的field重命名为new_field_name. @@ -199,6 +199,7 @@ class DataBundle: :param str new_field_name: :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; 如果为False，则报错 + :param bool rename_vocab: 如果该field同时也存在于vocabs中，会将该field的名称对应修改 :return: self """ for name, dataset in self.datasets.items(): @@ -206,15 +207,20 @@ class DataBundle: dataset.rename_field(field_name=field_name, new_field_name=new_field_name) elif not ignore_miss_dataset: raise KeyError(f"{field_name} not found DataSet:{name}.") + if rename_vocab: + if field_name in self.vocabs: + self.vocabs[new_field_name] = self.vocabs.pop(field_name) + return self - def delete_field(self, field_name, ignore_miss_dataset=True): + def delete_field(self, field_name, ignore_miss_dataset=True, delete_vocab=True): """ 将DataBundle中所有DataSet中名为field_name的field删除掉. :param str field_name: :param bool ignore_miss_dataset: 当某个field名称在某个dataset不存在时，如果为True，则直接忽略该DataSet; 如果为False，则报错 + :param bool delete_vocab: 如果该field也在vocabs中存在，将该值也一并删除 :return: self """ for name, dataset in self.datasets.items(): @@ -222,8 +228,39 @@ class DataBundle: dataset.delete_field(field_name=field_name) elif not ignore_miss_dataset: raise KeyError(f"{field_name} not found DataSet:{name}.") + if delete_vocab: + if field_name in self.vocabs: + self.vocabs.pop(field_name) return self + def iter_datasets(self)->Union[str, DataSet]: + """ + 迭代data_bundle中的DataSet + + Example:: + + for name, dataset in data_bundle.iter_datasets(): + pass + + :return: + """ + for name, dataset in self.datasets.items(): + yield name, dataset + + def iter_vocabs(self)->Union[str, Vocabulary]: + """ + 迭代data_bundle中的DataSet + + Example: + + for field_name, vocab in data_bundle.iter_vocabs(): + pass + + :return: + """ + for field_name, vocab in self.vocabs.items(): + yield field_name, vocab + def apply_field(self, func, field_name:str, new_field_name:str, ignore_miss_dataset=True, **kwargs): """ 对DataBundle中所有的dataset使用apply_field方法 diff --git a/fastNLP/io/pipe/conll.py b/fastNLP/io/pipe/conll.py index eb7d4909..2edc9008 100644 --- a/fastNLP/io/pipe/conll.py +++ b/fastNLP/io/pipe/conll.py @@ -193,7 +193,7 @@ class OntoNotesNERPipe(_NERPipe): """ 处理OntoNotes的NER数据，处理之后DataSet中的field情况为 - .. csv-table:: Following is a demo layout of DataSet returned by Conll2003Loader + .. csv-table:: :header: "raw_words", "words", "target", "seq_len" "[Nadim, Ladki]", "[2, 3]", "[1, 2]", 2 diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index bead09fc..6b0829bd 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -207,7 +207,7 @@ class ArcBiaffine(nn.Module): output = dep.matmul(self.U) output = output.bmm(head.transpose(-1, -2)) if self.has_bias: - output += head.matmul(self.bias).unsqueeze(1) + output = output + head.matmul(self.bias).unsqueeze(1) return output @@ -234,7 +234,7 @@ class LabelBilinear(nn.Module): :return output: [batch, seq_len, num_cls] 每个元素对应类别的概率图 """ output = self.bilinear(x1, x2) - output += self.lin(torch.cat([x1, x2], dim=2)) + output = output + self.lin(torch.cat([x1, x2], dim=2)) return output @@ -363,7 +363,7 @@ class BiaffineParser(GraphParser): # print('forward {} {}'.format(batch_size, seq_len)) # get sequence mask - mask = seq_len_to_mask(seq_len).long() + mask = seq_len_to_mask(seq_len, max_len=length).long() word = self.word_embedding(words1) # [N,L] -> [N,L,C_0] pos = self.pos_embedding(words2) # [N,L] -> [N,L,C_1] @@ -435,10 +435,10 @@ class BiaffineParser(GraphParser): """ batch_size, length, _ = pred1.shape - mask = seq_len_to_mask(seq_len) + mask = seq_len_to_mask(seq_len, max_len=length) flip_mask = (mask == 0) _arc_pred = pred1.clone() - _arc_pred.masked_fill_(flip_mask.unsqueeze(1), -float('inf')) + _arc_pred = _arc_pred.masked_fill(flip_mask.unsqueeze(1), -float('inf')) arc_logits = F.log_softmax(_arc_pred, dim=2) label_logits = F.log_softmax(pred2, dim=2) batch_index = torch.arange(batch_size, device=arc_logits.device, dtype=torch.long).unsqueeze(1) @@ -446,9 +446,8 @@ class BiaffineParser(GraphParser): arc_loss = arc_logits[batch_index, child_index, target1] label_loss = label_logits[batch_index, child_index, target2] - byte_mask = flip_mask.byte() - arc_loss.masked_fill_(byte_mask, 0) - label_loss.masked_fill_(byte_mask, 0) + arc_loss = arc_loss.masked_fill(flip_mask, 0) + label_loss = label_loss.masked_fill(flip_mask, 0) arc_nll = -arc_loss.mean() label_nll = -label_loss.mean() return arc_nll + label_nll diff --git a/fastNLP/modules/decoder/crf.py b/fastNLP/modules/decoder/crf.py index f63d46e3..c13ea50c 100644 --- a/fastNLP/modules/decoder/crf.py +++ b/fastNLP/modules/decoder/crf.py @@ -10,33 +10,45 @@ from torch import nn from ..utils import initial_parameter from ...core.vocabulary import Vocabulary +from ...core.metrics import _get_encoding_type_from_tag_vocab, _check_tag_vocab_and_encoding_type +from typing import Union - -def allowed_transitions(id2target, encoding_type='bio', include_start_end=False): +def allowed_transitions(tag_vocab:Union[Vocabulary, dict], encoding_type=None, include_start_end=False): """ 别名：:class:`fastNLP.modules.allowed_transitions` :class:`fastNLP.modules.decoder.allowed_transitions` 给定一个id到label的映射表，返回所有可以跳转的(from_tag_id, to_tag_id)列表。 - :param dict, ~fastNLP.Vocabulary id2target: key是label的indices，value是str类型的tag或tag-label。value可以是只有tag的, 比如"B", "M"; 也可以是 - "B-NN", "M-NN", tag和label之间一定要用"-"隔开。一般可以通过Vocabulary.idx2word得到id2label。 - :param str encoding_type: 支持"bio", "bmes", "bmeso", "bioes"。 + :param ~fastNLP.Vocabulary,dict tag_vocab: 支持类型为tag或tag-label。只有tag的,比如"B", "M"; 也可以是"B-NN", "M-NN", + tag和label之间一定要用"-"隔开。如果传入dict，格式需要形如{0:"O", 1:"B-tag1"}，即index在前，tag在后。 + :param str encoding_type: 支持"bio", "bmes", "bmeso", "bioes"。默认为None，通过vocab自动推断 :param bool include_start_end: 是否包含开始与结尾的转换。比如在bio中，b/o可以在开头，但是i不能在开头； 为True，返回的结果中会包含(start_idx, b_idx), (start_idx, o_idx), 但是不包含(start_idx, i_idx); start_idx=len(id2label), end_idx=len(id2label)+1。为False, 返回的结果中不含与开始结尾相关的内容 :return: List[Tuple(int, int)]], 内部的Tuple是可以进行跳转的(from_tag_id, to_tag_id)。 """ - if isinstance(id2target, Vocabulary): - id2target = id2target.idx2word - num_tags = len(id2target) + if encoding_type is None: + encoding_type = _get_encoding_type_from_tag_vocab(tag_vocab) + else: + encoding_type = encoding_type.lower() + _check_tag_vocab_and_encoding_type(tag_vocab, encoding_type) + + pad_token = '' + unk_token = '' + + if isinstance(tag_vocab, Vocabulary): + id_label_lst = list(tag_vocab.idx2word.items()) + pad_token = tag_vocab.padding + unk_token = tag_vocab.unknown + else: + id_label_lst = list(tag_vocab.items()) + + num_tags = len(tag_vocab) start_idx = num_tags end_idx = num_tags + 1 - encoding_type = encoding_type.lower() allowed_trans = [] - id_label_lst = list(id2target.items()) if include_start_end: id_label_lst += [(start_idx, 'start'), (end_idx, 'end')] - def split_tag_label(from_label): from_label = from_label.lower() if from_label in ['start', 'end']: @@ -48,11 +60,11 @@ def allowed_transitions(id2target, encoding_type='bio', include_start_end=False) return from_tag, from_label for from_id, from_label in id_label_lst: - if from_label in ['', '']: + if from_label in [pad_token, unk_token]: continue from_tag, from_label = split_tag_label(from_label) for to_id, to_label in id_label_lst: - if to_label in ['', '']: + if to_label in [pad_token, unk_token]: continue to_tag, to_label = split_tag_label(to_label) if _is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label): diff --git a/test/core/test_metrics.py b/test/core/test_metrics.py index 5a7c55cf..8a472a62 100644 --- a/test/core/test_metrics.py +++ b/test/core/test_metrics.py @@ -11,6 +11,12 @@ from fastNLP.core.metrics import SpanFPreRecMetric, ExtractiveQAMetric def _generate_tags(encoding_type, number_labels=4): + """ + + :param encoding_type: 例如BIOES, BMES, BIO等 + :param number_labels: 多少个label，大于1 + :return: + """ vocab = {} for i in range(number_labels): label = str(i) @@ -184,7 +190,7 @@ class TestAccuracyMetric(unittest.TestCase): self.assertDictEqual(metric.get_metric(), {'acc': 1.}) -class SpanF1PreRecMetric(unittest.TestCase): +class SpanFPreRecMetricTest(unittest.TestCase): def test_case1(self): from fastNLP.core.metrics import _bmes_tag_to_spans from fastNLP.core.metrics import _bio_tag_to_spans @@ -338,6 +344,39 @@ class SpanF1PreRecMetric(unittest.TestCase): for key, value in expected_metric.items(): self.assertAlmostEqual(value, metric_value[key], places=5) + def test_auto_encoding_type_infer(self): + # 检查是否可以自动check encode的类型 + vocabs = {} + import random + for encoding_type in ['bio', 'bioes', 'bmeso']: + vocab = Vocabulary(unknown=None, padding=None) + for i in range(random.randint(10, 100)): + label = str(random.randint(1, 10)) + for tag in encoding_type: + if tag!='o': + vocab.add_word(f'{tag}-{label}') + else: + vocab.add_word('o') + vocabs[encoding_type] = vocab + for e in ['bio', 'bioes', 'bmeso']: + with self.subTest(e=e): + metric = SpanFPreRecMetric(tag_vocab=vocabs[e]) + assert metric.encoding_type == e + + bmes_vocab = _generate_tags('bmes') + vocab = Vocabulary() + for tag, index in bmes_vocab.items(): + vocab.add_word(tag) + metric = SpanFPreRecMetric(vocab) + assert metric.encoding_type == 'bmes' + + # 一些无法check的情况 + vocab = Vocabulary() + for i in range(10): + vocab.add_word(str(i)) + with self.assertRaises(Exception): + metric = SpanFPreRecMetric(vocab) + def test_encoding_type(self): # 检查传入的tag_vocab与encoding_type不符合时，是否会报错 vocabs = {} diff --git a/test/modules/decoder/test_CRF.py b/test/modules/decoder/test_CRF.py index 647af7d3..94b4ab7a 100644 --- a/test/modules/decoder/test_CRF.py +++ b/test/modules/decoder/test_CRF.py @@ -1,6 +1,6 @@ import unittest - +from fastNLP import Vocabulary class TestCRF(unittest.TestCase): def test_case1(self): @@ -14,7 +14,8 @@ class TestCRF(unittest.TestCase): id2label = {0: 'B', 1:'M', 2:'E', 3:'S'} expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 5), (3, 0), (3, 3), (3, 5), (4, 0), (4, 3)} - self.assertSetEqual(expected_res, set(allowed_transitions(id2label, encoding_type='BMES', include_start_end=True))) + self.assertSetEqual(expected_res, set( + allowed_transitions(id2label, encoding_type='BMES', include_start_end=True))) id2label = {0: 'B', 1: 'I', 2:'O', 3: '', 4:""} allowed_transitions(id2label, include_start_end=True) @@ -37,7 +38,100 @@ class TestCRF(unittest.TestCase): expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 4), (2, 7), (2, 9), (3, 0), (3, 3), (3, 4), (3, 7), (3, 9), (4, 5), (4, 6), (5, 5), (5, 6), (6, 0), (6, 3), (6, 4), (6, 7), (6, 9), (7, 0), (7, 3), (7, 4), (7, 7), (7, 9), (8, 0), (8, 3), (8, 4), (8, 7)} - self.assertSetEqual(expected_res, set(allowed_transitions(id2label, encoding_type='BMES', include_start_end=True))) + self.assertSetEqual(expected_res, set( + allowed_transitions(id2label, include_start_end=True))) + + def test_case11(self): + # 测试自动推断encoding类型 + from fastNLP.modules.decoder.crf import allowed_transitions + + id2label = {0: 'B', 1: 'I', 2: 'O'} + expected_res = {(0, 0), (0, 1), (0, 2), (0, 4), (1, 0), (1, 1), (1, 2), (1, 4), (2, 0), (2, 2), + (2, 4), (3, 0), (3, 2)} + self.assertSetEqual(expected_res, set(allowed_transitions(id2label, include_start_end=True))) + + id2label = {0: 'B', 1: 'M', 2: 'E', 3: 'S'} + expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 5), (3, 0), (3, 3), (3, 5), (4, 0), (4, 3)} + self.assertSetEqual(expected_res, set( + allowed_transitions(id2label, include_start_end=True))) + + id2label = {0: 'B', 1: 'I', 2: 'O', 3: '', 4: ""} + allowed_transitions(id2label, include_start_end=True) + + labels = ['O'] + for label in ['X', 'Y']: + for tag in 'BI': + labels.append('{}-{}'.format(tag, label)) + id2label = {idx: label for idx, label in enumerate(labels)} + expected_res = {(0, 0), (0, 1), (0, 3), (0, 6), (1, 0), (1, 1), (1, 2), (1, 3), (1, 6), (2, 0), (2, 1), + (2, 2), (2, 3), (2, 6), (3, 0), (3, 1), (3, 3), (3, 4), (3, 6), (4, 0), (4, 1), (4, 3), + (4, 4), (4, 6), (5, 0), (5, 1), (5, 3)} + self.assertSetEqual(expected_res, set(allowed_transitions(id2label, include_start_end=True))) + + labels = [] + for label in ['X', 'Y']: + for tag in 'BMES': + labels.append('{}-{}'.format(tag, label)) + id2label = {idx: label for idx, label in enumerate(labels)} + expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 4), (2, 7), (2, 9), (3, 0), (3, 3), (3, 4), + (3, 7), (3, 9), (4, 5), (4, 6), (5, 5), (5, 6), (6, 0), (6, 3), (6, 4), (6, 7), (6, 9), (7, 0), + (7, 3), (7, 4), (7, 7), (7, 9), (8, 0), (8, 3), (8, 4), (8, 7)} + self.assertSetEqual(expected_res, set( + allowed_transitions(id2label, include_start_end=True))) + + def test_case12(self): + # 测试能否通过vocab生成转移矩阵 + from fastNLP.modules.decoder.crf import allowed_transitions + + id2label = {0: 'B', 1: 'I', 2: 'O'} + vocab = Vocabulary(unknown=None, padding=None) + for idx, tag in id2label.items(): + vocab.add_word(tag) + expected_res = {(0, 0), (0, 1), (0, 2), (0, 4), (1, 0), (1, 1), (1, 2), (1, 4), (2, 0), (2, 2), + (2, 4), (3, 0), (3, 2)} + self.assertSetEqual(expected_res, set(allowed_transitions(vocab, include_start_end=True))) + + id2label = {0: 'B', 1: 'M', 2: 'E', 3: 'S'} + vocab = Vocabulary(unknown=None, padding=None) + for idx, tag in id2label.items(): + vocab.add_word(tag) + expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 5), (3, 0), (3, 3), (3, 5), (4, 0), (4, 3)} + self.assertSetEqual(expected_res, set( + allowed_transitions(vocab, include_start_end=True))) + + id2label = {0: 'B', 1: 'I', 2: 'O', 3: '', 4: ""} + vocab = Vocabulary() + for idx, tag in id2label.items(): + vocab.add_word(tag) + allowed_transitions(vocab, include_start_end=True) + + labels = ['O'] + for label in ['X', 'Y']: + for tag in 'BI': + labels.append('{}-{}'.format(tag, label)) + id2label = {idx: label for idx, label in enumerate(labels)} + expected_res = {(0, 0), (0, 1), (0, 3), (0, 6), (1, 0), (1, 1), (1, 2), (1, 3), (1, 6), (2, 0), (2, 1), + (2, 2), (2, 3), (2, 6), (3, 0), (3, 1), (3, 3), (3, 4), (3, 6), (4, 0), (4, 1), (4, 3), + (4, 4), (4, 6), (5, 0), (5, 1), (5, 3)} + vocab = Vocabulary(unknown=None, padding=None) + for idx, tag in id2label.items(): + vocab.add_word(tag) + self.assertSetEqual(expected_res, set(allowed_transitions(vocab, include_start_end=True))) + + labels = [] + for label in ['X', 'Y']: + for tag in 'BMES': + labels.append('{}-{}'.format(tag, label)) + id2label = {idx: label for idx, label in enumerate(labels)} + vocab = Vocabulary(unknown=None, padding=None) + for idx, tag in id2label.items(): + vocab.add_word(tag) + expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 4), (2, 7), (2, 9), (3, 0), (3, 3), (3, 4), + (3, 7), (3, 9), (4, 5), (4, 6), (5, 5), (5, 6), (6, 0), (6, 3), (6, 4), (6, 7), (6, 9), (7, 0), + (7, 3), (7, 4), (7, 7), (7, 9), (8, 0), (8, 3), (8, 4), (8, 7)} + self.assertSetEqual(expected_res, set( + allowed_transitions(vocab, include_start_end=True))) + def test_case2(self): # 测试CRF能否避免解码出非法跃迁, 使用allennlp做了验证。 From 53f744a87d9d48cc3beaa43a17010a9628261f72 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Mon, 2 Sep 2019 19:43:28 +0800 Subject: [PATCH 30/92] fix some bugs in docs --- docs/source/tutorials/tutorial_9_callback.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/tutorials/tutorial_9_callback.rst b/docs/source/tutorials/tutorial_9_callback.rst index 8e2742bb..dc50aca5 100644 --- a/docs/source/tutorials/tutorial_9_callback.rst +++ b/docs/source/tutorials/tutorial_9_callback.rst @@ -23,7 +23,7 @@ Callback的构建和使用 class LRDecay(fastNLP.Callback): def __init__(self): - super(MyCallback, self).__init__() + super(LRDecay, self).__init__() self.base_lrs = [] self.delta = [] From 8dae71ff08476c573e3df26c00e188a7745ace78 Mon Sep 17 00:00:00 2001 From: xxliu Date: Tue, 3 Sep 2019 14:19:29 +0800 Subject: [PATCH 31/92] pipeline --- fastNLP/io/loader/coreference.py | 19 +++++++- fastNLP/io/pipe/coreference.py | 45 ++++++++++++------- .../coreference_resolution/model/model_re.py | 11 ++++- .../model/softmax_loss.py | 8 ++-- .../coreference_resolution/test/__init__.py | 0 .../test/test_dataloader.py | 14 ------ reproduction/coreference_resolution/train.py | 2 +- 7 files changed, 63 insertions(+), 36 deletions(-) delete mode 100644 reproduction/coreference_resolution/test/__init__.py delete mode 100644 reproduction/coreference_resolution/test/test_dataloader.py diff --git a/fastNLP/io/loader/coreference.py b/fastNLP/io/loader/coreference.py index c8d9bbf5..2e4d72de 100644 --- a/fastNLP/io/loader/coreference.py +++ b/fastNLP/io/loader/coreference.py @@ -1,17 +1,34 @@ from ...core.dataset import DataSet from ..file_reader import _read_json from ...core.instance import Instance +from ...core.const import Const from .json import JsonLoader class CRLoader(JsonLoader): + """ + 原始数据中内容应该为, 每一行为一个json对象,其中doc_key包含文章的种类信息，speakers包含每句话的说话者信息，cluster是指向现实中同一个事物的聚集，sentences是文本信息内容。 + + Example:: + + {"doc_key":"bc/cctv/00/cctv_001", + "speakers":"[["Speaker1","Speaker1","Speaker1"],["Speaker1","Speaker1","Speaker1"]]", + "clusters":"[[[2,3],[4,5]],[7,8],[18,20]]]", + "sentences":[["I","have","an","apple"],["It","is","good"]] + } + + 读取预处理好的Conll2012数据。 + + """ def __init__(self, fields=None, dropna=False): super().__init__(fields, dropna) + self.fields = {"doc_key":Const.INPUTS(0),"speakers":Const.INPUTS(1),"clusters":Const.TARGET,"sentences":Const.INPUTS(2)} def _load(self, path): """ 加载数据 - :param path: + :param path: 数据文件路径，文件为json + :return: """ dataset = DataSet() diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py index bdf6a132..711e5919 100644 --- a/fastNLP/io/pipe/coreference.py +++ b/fastNLP/io/pipe/coreference.py @@ -6,12 +6,16 @@ __all__ = [ from .pipe import Pipe from ..data_bundle import DataBundle from ..loader.coreference import CRLoader +from ...core.const import Const from fastNLP.core.vocabulary import Vocabulary import numpy as np import collections class CoreferencePipe(Pipe): + """ + 对Coreference resolution问题进行处理，得到文章种类/说话者/字符级信息/序列长度。 + """ def __init__(self,config): super().__init__() @@ -19,28 +23,39 @@ class CoreferencePipe(Pipe): def process(self, data_bundle: DataBundle): genres = {g: i for i, g in enumerate(["bc", "bn", "mz", "nw", "pt", "tc", "wb"])} - vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name='sentences') + vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name=Const.INPUTS(2)) vocab.build_vocab() word2id = vocab.word2idx + data_bundle.vocabs = {"vocab":vocab} char_dict = get_char_dict(self.config.char_path) + for name, ds in data_bundle.datasets.items(): - ds.apply(lambda x: doc2numpy(x['sentences'], word2id, char_dict, max(self.config.filter), + # genre + ds.apply(lambda x: genres[x[Const.INPUTS(0)][:2]], new_field_name=Const.INPUTS(0)) + + # speaker_ids_np + ds.apply(lambda x: speaker2numpy(x[Const.INPUTS(1)], self.config.max_sentences, is_train=name == 'train'), + new_field_name=Const.INPUTS(1)) + + # doc_np + ds.apply(lambda x: doc2numpy(x[Const.INPUTS(2)], word2id, char_dict, max(self.config.filter), self.config.max_sentences, is_train=name == 'train')[0], - new_field_name='doc_np') - ds.apply(lambda x: doc2numpy(x['sentences'], word2id, char_dict, max(self.config.filter), + new_field_name=Const.INPUTS(3)) + # char_index + ds.apply(lambda x: doc2numpy(x[Const.INPUTS(2)], word2id, char_dict, max(self.config.filter), self.config.max_sentences, is_train=name == 'train')[1], - new_field_name='char_index') - ds.apply(lambda x: doc2numpy(x['sentences'], word2id, char_dict, max(self.config.filter), + new_field_name=Const.CHAR_INPUT) + # seq len + ds.apply(lambda x: doc2numpy(x[Const.INPUTS(2)], word2id, char_dict, max(self.config.filter), self.config.max_sentences, is_train=name == 'train')[2], - new_field_name='seq_len') - ds.apply(lambda x: speaker2numpy(x["speakers"], self.config.max_sentences, is_train=name == 'train'), - new_field_name='speaker_ids_np') - ds.apply(lambda x: genres[x["doc_key"][:2]], new_field_name='genre') - - ds.set_ignore_type('clusters') - ds.set_padder('clusters', None) - ds.set_input("sentences", "doc_np", "speaker_ids_np", "genre", "char_index", "seq_len") - ds.set_target("clusters") + new_field_name=Const.INPUT_LEN) + + + ds.set_ignore_type(Const.TARGET) + ds.set_padder(Const.TARGET, None) + ds.set_input(Const.INPUTS(0), Const.INPUTS(1), Const.INPUTS(2), Const.INPUTS(3), Const.CHAR_INPUT, Const.INPUT_LEN) + ds.set_target(Const.TARGET) + return data_bundle def process_from_file(self, paths): diff --git a/reproduction/coreference_resolution/model/model_re.py b/reproduction/coreference_resolution/model/model_re.py index 9dd90ec4..eaa2941b 100644 --- a/reproduction/coreference_resolution/model/model_re.py +++ b/reproduction/coreference_resolution/model/model_re.py @@ -8,6 +8,7 @@ from fastNLP.models.base_model import BaseModel from fastNLP.modules.encoder.variational_rnn import VarLSTM from reproduction.coreference_resolution.model import preprocess from fastNLP.io.embed_loader import EmbedLoader +from fastNLP.core.const import Const import random # 设置seed @@ -415,7 +416,7 @@ class Model(BaseModel): return predicted_clusters - def forward(self, sentences, doc_np, speaker_ids_np, genre, char_index, seq_len): + def forward(self, words1 , words2, words3, words4, chars, seq_len): """ 实际输入都是tensor :param sentences: 句子，被fastNLP转化成了numpy， @@ -426,6 +427,14 @@ class Model(BaseModel): :param seq_len: 被fastNLP转化成了Tensor :return: """ + + sentences = words3 + doc_np = words4 + speaker_ids_np = words2 + genre = words1 + char_index = chars + + # change for fastNLP sentences = sentences[0].tolist() doc_tensor = doc_np[0] diff --git a/reproduction/coreference_resolution/model/softmax_loss.py b/reproduction/coreference_resolution/model/softmax_loss.py index c75a31d6..1c1fcc69 100644 --- a/reproduction/coreference_resolution/model/softmax_loss.py +++ b/reproduction/coreference_resolution/model/softmax_loss.py @@ -11,18 +11,18 @@ class SoftmaxLoss(LossBase): 允许多标签分类 """ - def __init__(self, antecedent_scores=None, clusters=None, mention_start_tensor=None, mention_end_tensor=None): + def __init__(self, antecedent_scores=None, target=None, mention_start_tensor=None, mention_end_tensor=None): """ :param pred: :param target: """ super().__init__() - self._init_param_map(antecedent_scores=antecedent_scores, clusters=clusters, + self._init_param_map(antecedent_scores=antecedent_scores, target=target, mention_start_tensor=mention_start_tensor, mention_end_tensor=mention_end_tensor) - def get_loss(self, antecedent_scores, clusters, mention_start_tensor, mention_end_tensor): - antecedent_labels = get_labels(clusters[0], mention_start_tensor, mention_end_tensor, + def get_loss(self, antecedent_scores, target, mention_start_tensor, mention_end_tensor): + antecedent_labels = get_labels(target[0], mention_start_tensor, mention_end_tensor, Config().max_antecedents) antecedent_labels = torch.from_numpy(antecedent_labels*1).to(torch.device("cuda:" + Config().cuda)) diff --git a/reproduction/coreference_resolution/test/__init__.py b/reproduction/coreference_resolution/test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/reproduction/coreference_resolution/test/test_dataloader.py b/reproduction/coreference_resolution/test/test_dataloader.py deleted file mode 100644 index 6a3be520..00000000 --- a/reproduction/coreference_resolution/test/test_dataloader.py +++ /dev/null @@ -1,14 +0,0 @@ - - -import unittest -from fastNLP.io.pipe.coreference import CoreferencePipe -from reproduction.coreference_resolution.model.config import Config - -class Test_CRLoader(unittest.TestCase): - def test_cr_loader(self): - config = Config() - bundle = CoreferencePipe(config).process_from_file({'train': config.train_path, 'dev': config.dev_path,'test': config.test_path}) - - print(bundle.datasets['train'][0]) - print(bundle.datasets['dev'][0]) - print(bundle.datasets['test'][0]) diff --git a/reproduction/coreference_resolution/train.py b/reproduction/coreference_resolution/train.py index 6c26cf4c..790c7659 100644 --- a/reproduction/coreference_resolution/train.py +++ b/reproduction/coreference_resolution/train.py @@ -45,7 +45,7 @@ if __name__ == "__main__": print("数据集划分：\ntrain:", str(len(data_info.datasets["train"])), "\ndev:" + str(len(data_info.datasets["dev"])) + "\ntest:" + str(len(data_info.datasets["test"]))) # print(data_info) - model = Model(data_info.vocabs, config) + model = Model(data_info.vocabs['vocab'], config) print(model) loss = SoftmaxLoss() From b3718b10dcda636883f0267b76b264c904e807ff Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Tue, 3 Sep 2019 23:19:18 +0800 Subject: [PATCH 32/92] 1. rename init_embed to embed in models/*; 2. update documents in models/bert.py; 3. update tutorial six. --- docs/source/fastNLP.models.bert.rst | 6 + docs/source/fastNLP.models.rst | 3 +- .../fastNLP.models.sequence_labeling.rst | 2 +- .../tutorials/tutorial_6_seq_labeling.rst | 92 +++---- fastNLP/models/__init__.py | 10 +- fastNLP/models/bert.py | 241 +++++++++++------- fastNLP/models/biaffine_parser.py | 6 +- fastNLP/models/cnn_text_classification.py | 6 +- fastNLP/models/snli.py | 10 +- fastNLP/models/star_transformer.py | 24 +- test/models/test_bert.py | 84 +++++- test/models/test_biaffine_parser.py | 4 +- 12 files changed, 312 insertions(+), 176 deletions(-) create mode 100644 docs/source/fastNLP.models.bert.rst diff --git a/docs/source/fastNLP.models.bert.rst b/docs/source/fastNLP.models.bert.rst new file mode 100644 index 00000000..b0c813f9 --- /dev/null +++ b/docs/source/fastNLP.models.bert.rst @@ -0,0 +1,6 @@ +fastNLP.models.bert +=================== + +.. automodule:: fastNLP.models.bert + :members: BertForSequenceClassification, BertForSentenceMatching, BertForMultipleChoice, BertForTokenClassification, BertForQuestionAnswering + diff --git a/docs/source/fastNLP.models.rst b/docs/source/fastNLP.models.rst index fb782de1..21cf41a7 100644 --- a/docs/source/fastNLP.models.rst +++ b/docs/source/fastNLP.models.rst @@ -2,7 +2,7 @@ fastNLP.models ============== .. automodule:: fastNLP.models - :members: CNNText, SeqLabeling, AdvSeqLabel, ESIM, StarTransEnc, STSeqLabel, STNLICls, STSeqCls, BiaffineParser, GraphParser + :members: CNNText, SeqLabeling, AdvSeqLabel, ESIM, StarTransEnc, STSeqLabel, STNLICls, STSeqCls, BiaffineParser, GraphParser, BertForSequenceClassification, BertForSentenceMatching, BertForMultipleChoice, BertForTokenClassification, BertForQuestionAnswering 子模块 ------ @@ -10,6 +10,7 @@ fastNLP.models .. toctree:: :maxdepth: 1 + fastNLP.models.bert fastNLP.models.biaffine_parser fastNLP.models.cnn_text_classification fastNLP.models.sequence_labeling diff --git a/docs/source/fastNLP.models.sequence_labeling.rst b/docs/source/fastNLP.models.sequence_labeling.rst index f6551f8b..dcd1300e 100644 --- a/docs/source/fastNLP.models.sequence_labeling.rst +++ b/docs/source/fastNLP.models.sequence_labeling.rst @@ -2,5 +2,5 @@ fastNLP.models.sequence_labeling ================================ .. automodule:: fastNLP.models.sequence_labeling - :members: SeqLabeling, AdvSeqLabel + :members: SeqLabeling, AdvSeqLabel, BiLSTMCRF diff --git a/docs/source/tutorials/tutorial_6_seq_labeling.rst b/docs/source/tutorials/tutorial_6_seq_labeling.rst index 09a53cdc..7fcf97b3 100644 --- a/docs/source/tutorials/tutorial_6_seq_labeling.rst +++ b/docs/source/tutorials/tutorial_6_seq_labeling.rst @@ -3,64 +3,52 @@ ===================== 这一部分的内容主要展示如何使用fastNLP 实现序列标注任务。你可以使用fastNLP的各个组件快捷，方便地完成序列标注任务，达到出色的效果。 -在阅读这篇Tutorial前，希望你已经熟悉了fastNLP的基础使用，包括基本数据结构以及数据预处理，embedding的嵌入等，希望你对之前的教程有更进一步的掌握。 -我们将对CoNLL-03的英文数据集进行处理，展示如何完成命名实体标注任务整个训练的过程。 +在阅读这篇Tutorial前，希望你已经熟悉了fastNLP的基础使用，尤其是数据的载入以及模型的构建，通过这个小任务的能让你进一步熟悉fastNLP的使用。 +我们将对基于Weibo的中文社交数据集进行处理，展示如何完成命名实体标注任务的整个过程。 载入数据 =================================== -fastNLP可以方便地载入各种类型的数据。同时，针对常见的数据集，我们已经预先实现了载入方法，其中包含CoNLL-03数据集。 +fastNLP的数据载入主要是由Loader与Pipe两个基类衔接完成的。通过Loader可以方便地载入各种类型的数据。同时，针对常见的数据集，我们已经预先实现了载入方法，其中包含weibo数据集。 在设计dataloader时，以DataSetLoader为基类，可以改写并应用于其他数据集的载入。 .. code-block:: python - class Conll2003DataLoader(DataSetLoader): - def __init__(self, task:str='ner', encoding_type:str='bioes'): - assert task in ('ner', 'pos', 'chunk') - index = {'ner':3, 'pos':1, 'chunk':2}[task] - #ConllLoader是fastNLP内置的类 - self._loader = ConllLoader(headers=['raw_words', 'target'], indexes=[0, index]) - self._tag_converters = None - if task in ('ner', 'chunk'): - #iob和iob2bioes会对tag进行统一，标准化 - self._tag_converters = [iob2] - if encoding_type == 'bioes': - self._tag_converters.append(iob2bioes) - - def load(self, path: str): - dataset = self._loader.load(path) - def convert_tag_schema(tags): - for converter in self._tag_converters: - tags = converter(tags) - return tags - if self._tag_converters: - #使用apply实现convert_tag_schema函数，实际上也支持匿名函数 - dataset.apply_field(convert_tag_schema, field_name=Const.TARGET, new_field_name=Const.TARGET) - return dataset - -输出数据格式如： - - {'raw_words': ['on', 'Friday', ':'] type=list, - 'target': ['O', 'O', 'O'] type=list}, + from fastNLP.io import WeiboNERLoader + data_bundle = WeiboNERLoader().load() + + + +载入后的数据如 :: + + {'dev': DataSet( + {{'raw_chars': ['用', '最', '大', '努', '力', '去', '做''人', '生', '。', '哈', '哈', '哈', '哈', '哈', '哈', ' + 'target': ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O',, 'O', 'O', 'O', 'O', 'O', 'O'] type=list})} + + {'test': DataSet( + {{'raw_chars': ['感', '恩', '大', '回', '馈'] type=list, 'target': ['O', 'O', 'O', 'O', 'O'] type=list})} + + {'train': DataSet( + {'raw_chars': ['国', '安', '老', '球', '迷'] type=list, 'target': ['B-ORG.NAM', 'I-ORG.NAM', 'B-PER.NOM', 'I-PER.NOM', 'I-PER.NOM'] type=list})} + 数据处理 ---------------------------- -我们进一步处理数据。将数据和词表封装在 :class:`~fastNLP.DataBundle` 类中。data是DataBundle的实例。 -我们输入模型的数据包括char embedding，以及word embedding。在数据处理部分，我们尝试完成词表的构建。 -使用fastNLP中的Vocabulary类来构建词表。 +我们进一步处理数据。通过Pipe基类处理Loader载入的数据。 如果你还有印象，应该还能想起，实现自定义数据集的Pipe时，至少要编写process 函数或者process_from_file 函数。前者接受 :class:`~fastNLP.DataBundle` 类的数据，并返回该 :class:`~fastNLP.DataBundle` 。后者接收数据集所在文件夹为参数，读取并处理为 :class:`~fastNLP.DataBundle` 后，通过process 函数处理数据。 +这里我们已经实现通过Loader载入数据，并已返回 :class:`~fastNLP.DataBundle` 类的数据。我们编写process 函数以处理Loader载入后的数据。 .. code-block:: python - word_vocab = Vocabulary(min_freq=2) - word_vocab.from_dataset(data.datasets['train'], field_name=Const.INPUT) - word_vocab.index_dataset(*data.datasets.values(),field_name=Const.INPUT, new_field_name=Const.INPUT) + from fastNLP.io import ChineseNERPipe + data_bundle = ChineseNERPipe(encoding_type='bioes', bigram=True).process(data_bundle) -处理后的data对象内部为： +载入后的数据如下 :: - dataset - vocabs - dataset保存了train和test中的数据，并保存为dataset类型 - vocab保存了words，raw-words以及target的词表。 + {'raw_chars': ['用', '最', '大', '努', '力', '去', '做', '值', '得', '的', '事', '人', '生', '。', '哈', '哈', '哈', '哈', '哈', '哈', '我', '在'] type=list, + 'target': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] type=list, + 'chars': [97, 71, 34, 422, 104, 72, 144, 628, 66, 3, 158, 2, 9, 647, 485, 196, 2,19] type=list, + 'bigrams': [5948, 1950, 34840, 98, 8413, 3961, 34841, 631, 34842, 407, 462, 45, 3 1959, 1619, 3, 3, 3, 3, 3, 2663, 29, 90] type=list, + 'seq_len': 30 type=int} 模型构建 -------------------------------- @@ -69,27 +57,23 @@ fastNLP可以方便地载入各种类型的数据。同时，针对常见的数 模型的训练 首先实例化模型，导入所需的char embedding以及word embedding。Embedding的载入可以参考教程。 -也可以查看 :mod:`~fastNLP.modules.encoder.embedding` 使用所需的embedding 载入方法。 -fastNLP将模型的训练过程封装在了 :class:`~fastnlp.trainer` 类中。 +也可以查看 :mod:`~fastNLP.embedding` 使用所需的embedding 载入方法。 +fastNLP将模型的训练过程封装在了 :class:`~fastnlp.Trainer` 类中。 根据不同的任务调整trainer中的参数即可。通常，一个trainer实例需要有：指定的训练数据集，模型，优化器，loss函数，评测指标，以及指定训练的epoch数，batch size等参数。 .. code-block:: python #实例化模型 - model = CNNBiLSTMCRF(word_embed, char_embed, hidden_size=200, num_layers=1, tag_vocab=data.vocabs[Const.TARGET], encoding_type=encoding_type) - #定义优化器 - optimizer = Adam(model.parameters(), lr=0.005) + model = CNBiLSTMCRFNER(char_embed, num_classes=len(data_bundle.vocabs['target']), bigram_embed=bigram_embed) #定义评估指标 - Metrics=SpanFPreRecMetric(tag_vocab=data.vocabs[Const.TARGET], encoding_type=encoding_type) - #实例化trainer - trainer = Trainer(train_data=data.datasets['train'], model=model, optimizer=optimizer, dev_data=data.datasets['test'], batch_size=10, metrics=Metrics,callbacks=callbacks, n_epochs=100) - #开始训练 - trainer.train() + Metrics=SpanFPreRecMetric(data_bundle.vocabs['target'], encoding_type='bioes') + #实例化trainer并训练 + Trainer(data_bundle.datasets['train'], model, batch_size=20, metrics=Metrics, num_workers=2, dev_data=data_bundle. datasets['dev']).train() + 训练中会保存最优的参数配置。 -训练的结果如下： -.. code-block:: python +训练的结果如下 :: Evaluation on DataSet test: SpanFPreRecMetric: f=0.727661, pre=0.732293, rec=0.723088 diff --git a/fastNLP/models/__init__.py b/fastNLP/models/__init__.py index 14314049..a659e1d5 100644 --- a/fastNLP/models/__init__.py +++ b/fastNLP/models/__init__.py @@ -21,12 +21,18 @@ __all__ = [ "STSeqCls", "BiaffineParser", - "GraphParser" + "GraphParser", + + "BertForSequenceClassification", + "BertForSentenceMatching", + "BertForMultipleChoice", + "BertForTokenClassification", + "BertForQuestionAnswering" ] from .base_model import BaseModel from .bert import BertForMultipleChoice, BertForQuestionAnswering, BertForSequenceClassification, \ - BertForTokenClassification + BertForTokenClassification, BertForSentenceMatching from .biaffine_parser import BiaffineParser, GraphParser from .cnn_text_classification import CNNText from .sequence_labeling import SeqLabeling, AdvSeqLabel diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py index 08f16db2..4a04bd6d 100644 --- a/fastNLP/models/bert.py +++ b/fastNLP/models/bert.py @@ -1,9 +1,35 @@ -"""undocumented -bert.py is modified from huggingface/pytorch-pretrained-BERT, which is licensed under the Apache License 2.0. +""" +fastNLP提供了BERT应用到五个下游任务的模型代码，可以直接调用。这五个任务分别为 + + - 文本分类任务： :class:`~fastNLP.models.BertForSequenceClassification` + - Matching任务： :class:`~fastNLP.models.BertForSentenceMatching` + - 多选任务： :class:`~fastNLP.models.BertForMultipleChoice` + - 序列标注任务： :class:`~fastNLP.models.BertForTokenClassification` + - 抽取式QA任务： :class:`~fastNLP.models.BertForQuestionAnswering` + +每一个模型必须要传入一个名字为 `embed` 的 :class:`fastNLP.embeddings.BertEmbedding` ，这个参数包含了 +:class:`fastNLP.modules.encoder.BertModel` ，是下游模型的编码器(encoder)。 + +除此以外，还需要传入一个数字，这个数字在不同下游任务模型上的意义如下:: + + 下游任务模型 参数名称 含义 + BertForSequenceClassification num_labels 文本分类类别数目，默认值为2 + BertForSentenceMatching num_labels Matching任务类别数目，默认值为2 + BertForMultipleChoice num_choices 多选任务选项数目，默认值为2 + BertForTokenClassification num_labels 序列标注标签数目，无默认值 + BertForQuestionAnswering num_labels 抽取式QA列数，默认值为2(即第一列为start_span, 第二列为end_span) + +最后还可以传入dropout的大小，默认值为0.1。 """ -__all__ = [] +__all__ = [ + "BertForSequenceClassification", + "BertForSentenceMatching", + "BertForMultipleChoice", + "BertForTokenClassification", + "BertForQuestionAnswering" +] import warnings @@ -13,28 +39,40 @@ from torch import nn from .base_model import BaseModel from ..core.const import Const from ..core._logger import logger -from ..modules.encoder import BertModel -from ..modules.encoder.bert import BertConfig, CONFIG_FILE -from ..embeddings.bert_embedding import BertEmbedding +from ..embeddings import BertEmbedding class BertForSequenceClassification(BaseModel): - """BERT model for classification. """ - def __init__(self, init_embed: BertEmbedding, num_labels: int=2): + 别名： :class:`fastNLP.models.BertForSequenceClassification` + :class:`fastNLP.models.bert.BertForSequenceClassification` + + BERT model for classification. + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: 文本分类类别数目，默认值为2. + :param float dropout: dropout的大小，默认值为0.1. + """ + def __init__(self, embed: BertEmbedding, num_labels: int=2, dropout=0.1): super(BertForSequenceClassification, self).__init__() self.num_labels = num_labels - self.bert = init_embed - self.dropout = nn.Dropout(0.1) + self.bert = embed + self.dropout = nn.Dropout(p=dropout) self.classifier = nn.Linear(self.bert.embedding_dim, num_labels) if not self.bert.model.include_cls_sep: - warn_msg = "Bert for sequence classification excepts BertEmbedding `include_cls_sep` True, but got False." + self.bert.model.include_cls_sep = True + warn_msg = "Bert for sequence classification excepts BertEmbedding `include_cls_sep` True, " \ + "but got False. FastNLP has changed it to True." logger.warn(warn_msg) warnings.warn(warn_msg) def forward(self, words): + """ + :param torch.LongTensor words: [batch_size, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.Tensor [batch_size, num_labels] + """ hidden = self.dropout(self.bert(words)) cls_hidden = hidden[:, 0] logits = self.classifier(cls_hidden) @@ -42,172 +80,193 @@ class BertForSequenceClassification(BaseModel): return {Const.OUTPUT: logits} def predict(self, words): + """ + :param torch.LongTensor words: [batch_size, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.LongTensor [batch_size] + """ logits = self.forward(words)[Const.OUTPUT] return {Const.OUTPUT: torch.argmax(logits, dim=-1)} class BertForSentenceMatching(BaseModel): + """ + 别名： :class:`fastNLP.models.BertForSentenceMatching` + :class:`fastNLP.models.bert.BertForSentenceMatching` + + BERT model for sentence matching. - """BERT model for matching. + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: Matching任务类别数目，默认值为2. + :param float dropout: dropout的大小，默认值为0.1. """ - def __init__(self, init_embed: BertEmbedding, num_labels: int=2): + def __init__(self, embed: BertEmbedding, num_labels: int=2, dropout=0.1): super(BertForSentenceMatching, self).__init__() self.num_labels = num_labels - self.bert = init_embed - self.dropout = nn.Dropout(0.1) + self.bert = embed + self.dropout = nn.Dropout(p=dropout) self.classifier = nn.Linear(self.bert.embedding_dim, num_labels) if not self.bert.model.include_cls_sep: - error_msg = "Bert for sentence matching excepts BertEmbedding `include_cls_sep` True, but got False." - logger.error(error_msg) - raise RuntimeError(error_msg) + self.bert.model.include_cls_sep = True + warn_msg = "Bert for sentence matching excepts BertEmbedding `include_cls_sep` True, " \ + "but got False. FastNLP has changed it to True." + logger.warn(warn_msg) + warnings.warn(warn_msg) def forward(self, words): - hidden = self.dropout(self.bert(words)) - cls_hidden = hidden[:, 0] + """ + :param torch.LongTensor words: [batch_size, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.Tensor [batch_size, num_labels] + """ + hidden = self.bert(words) + cls_hidden = self.dropout(hidden[:, 0]) logits = self.classifier(cls_hidden) return {Const.OUTPUT: logits} def predict(self, words): + """ + :param torch.LongTensor words: [batch_size, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.LongTensor [batch_size] + """ logits = self.forward(words)[Const.OUTPUT] return {Const.OUTPUT: torch.argmax(logits, dim=-1)} class BertForMultipleChoice(BaseModel): - """BERT model for multiple choice tasks. """ - def __init__(self, init_embed: BertEmbedding, num_choices=2): + 别名： :class:`fastNLP.models.BertForMultipleChoice` + :class:`fastNLP.models.bert.BertForMultipleChoice` + + BERT model for multiple choice. + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_choices: 多选任务选项数目，默认值为2. + :param float dropout: dropout的大小，默认值为0.1. + """ + def __init__(self, embed: BertEmbedding, num_choices=2, dropout=0.1): super(BertForMultipleChoice, self).__init__() self.num_choices = num_choices - self.bert = init_embed - self.dropout = nn.Dropout(0.1) + self.bert = embed + self.dropout = nn.Dropout(p=dropout) self.classifier = nn.Linear(self.bert.embedding_dim, 1) - self.include_cls_sep = init_embed.model.include_cls_sep if not self.bert.model.include_cls_sep: - error_msg = "Bert for multiple choice excepts BertEmbedding `include_cls_sep` True, but got False." - logger.error(error_msg) - raise RuntimeError(error_msg) + self.bert.model.include_cls_sep = True + warn_msg = "Bert for multiple choice excepts BertEmbedding `include_cls_sep` True, " \ + "but got False. FastNLP has changed it to True." + logger.warn(warn_msg) + warnings.warn(warn_msg) def forward(self, words): """ - :param torch.Tensor words: [batch_size, num_choices, seq_len] - :return: [batch_size, num_labels] + :param torch.LongTensor words: [batch_size, num_choices, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.LongTensor [batch_size, num_choices] """ batch_size, num_choices, seq_len = words.size() input_ids = words.view(batch_size * num_choices, seq_len) hidden = self.bert(input_ids) - pooled_output = hidden[:, 0] - pooled_output = self.dropout(pooled_output) + pooled_output = self.dropout(hidden[:, 0]) logits = self.classifier(pooled_output) reshaped_logits = logits.view(-1, self.num_choices) return {Const.OUTPUT: reshaped_logits} def predict(self, words): + """ + :param torch.LongTensor words: [batch_size, num_choices, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.LongTensor [batch_size] + """ logits = self.forward(words)[Const.OUTPUT] return {Const.OUTPUT: torch.argmax(logits, dim=-1)} class BertForTokenClassification(BaseModel): - """BERT model for token-level classification. """ - def __init__(self, init_embed: BertEmbedding, num_labels): + 别名： :class:`fastNLP.models.BertForTokenClassification` + :class:`fastNLP.models.bert.BertForTokenClassification` + + BERT model for token classification. + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: 序列标注标签数目，无默认值. + :param float dropout: dropout的大小，默认值为0.1. + """ + def __init__(self, embed: BertEmbedding, num_labels, dropout=0.1): super(BertForTokenClassification, self).__init__() self.num_labels = num_labels - self.bert = init_embed - self.dropout = nn.Dropout(0.1) + self.bert = embed + self.dropout = nn.Dropout(p=dropout) self.classifier = nn.Linear(self.bert.embedding_dim, num_labels) - self.include_cls_sep = init_embed.model.include_cls_sep - if self.include_cls_sep: - warn_msg = "Bert for token classification excepts BertEmbedding `include_cls_sep` False, but got True." - warnings.warn(warn_msg) + if self.bert.model.include_cls_sep: + self.bert.model.include_cls_sep = False + warn_msg = "Bert for token classification excepts BertEmbedding `include_cls_sep` False, " \ + "but got True. FastNLP has changed it to False." logger.warn(warn_msg) + warnings.warn(warn_msg) def forward(self, words): """ - :param torch.Tensor words: [batch_size, seq_len] - :return: [batch_size, seq_len, num_labels] + :param torch.LongTensor words: [batch_size, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.Tensor [batch_size, seq_len, num_labels] """ - sequence_output = self.bert(words) - if self.include_cls_sep: - sequence_output = sequence_output[:, 1: -1] # [batch_size, seq_len, embed_dim] + sequence_output = self.bert(words) # [batch_size, seq_len, embed_dim] sequence_output = self.dropout(sequence_output) logits = self.classifier(sequence_output) return {Const.OUTPUT: logits} def predict(self, words): + """ + :param torch.LongTensor words: [batch_size, seq_len] + :return: { :attr:`fastNLP.Const.OUTPUT` : logits}: torch.LongTensor [batch_size, seq_len] + """ logits = self.forward(words)[Const.OUTPUT] return {Const.OUTPUT: torch.argmax(logits, dim=-1)} class BertForQuestionAnswering(BaseModel): - """BERT model for Question Answering (span extraction). - This module is composed of the BERT model with a linear layer on top of - the sequence output that computes start_logits and end_logits - Params: - `config`: a BertConfig class instance with the configuration to build a new model. - `bert_dir`: a dir which contains the bert parameters within file `pytorch_model.bin` - Inputs: - `input_ids`: a torch.LongTensor of shape [batch_size, sequence_length] - with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts - `extract_features.py`, `run_classifier.py` and `run_squad.py`) - `token_type_ids`: an optional torch.LongTensor of shape [batch_size, sequence_length] with the token - types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to - a `sentence B` token (see BERT paper for more details). - `attention_mask`: an optional torch.LongTensor of shape [batch_size, sequence_length] with indices - selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max - input sequence length in the current batch. It's the mask that we typically use for attention when - a batch has varying length sentences. - `start_positions`: position of the first token for the labeled span: torch.LongTensor of shape [batch_size]. - Positions are clamped to the length of the sequence and position outside of the sequence are not taken - into account for computing the loss. - `end_positions`: position of the last token for the labeled span: torch.LongTensor of shape [batch_size]. - Positions are clamped to the length of the sequence and position outside of the sequence are not taken - into account for computing the loss. - Outputs: - if `start_positions` and `end_positions` are not `None`: - Outputs the total_loss which is the sum of the CrossEntropy loss for the start and end token positions. - if `start_positions` or `end_positions` is `None`: - Outputs a tuple of start_logits, end_logits which are the logits respectively for the start and end - position tokens of shape [batch_size, sequence_length]. - Example usage: - ```python - # Already been converted into WordPiece token ids - input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]]) - input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]]) - token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]]) - config = BertConfig(vocab_size_or_config_json_file=32000, hidden_size=768, - num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072) - bert_dir = 'your-bert-file-dir' - model = BertForQuestionAnswering(config, bert_dir) - start_logits, end_logits = model(input_ids, token_type_ids, input_mask) - ``` """ - def __init__(self, init_embed: BertEmbedding, num_labels=2): + 别名： :class:`fastNLP.models.BertForQuestionAnswering` + :class:`fastNLP.models.bert.BertForQuestionAnswering` + + BERT model for classification. + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: 抽取式QA列数，默认值为2(即第一列为start_span, 第二列为end_span). + """ + def __init__(self, embed: BertEmbedding, num_labels=2): super(BertForQuestionAnswering, self).__init__() - self.bert = init_embed + self.bert = embed self.num_labels = num_labels self.qa_outputs = nn.Linear(self.bert.embedding_dim, self.num_labels) if not self.bert.model.include_cls_sep: - error_msg = "Bert for multiple choice excepts BertEmbedding `include_cls_sep` True, but got False." - logger.error(error_msg) - raise RuntimeError(error_msg) + self.bert.model.include_cls_sep = True + warn_msg = "Bert for question answering excepts BertEmbedding `include_cls_sep` True, " \ + "but got False. FastNLP has changed it to True." + logger.warn(warn_msg) + warnings.warn(warn_msg) def forward(self, words): + """ + :param torch.LongTensor words: [batch_size, seq_len] + :return: 一个包含num_labels个logit的dict，每一个logit的形状都是[batch_size, seq_len] + """ sequence_output = self.bert(words) logits = self.qa_outputs(sequence_output) # [batch_size, seq_len, num_labels] return {Const.OUTPUTS(i): logits[:, :, i] for i in range(self.num_labels)} def predict(self, words): + """ + :param torch.LongTensor words: [batch_size, seq_len] + :return: 一个包含num_labels个logit的dict，每一个logit的形状都是[batch_size] + """ logits = self.forward(words) return {Const.OUTPUTS(i): torch.argmax(logits[Const.OUTPUTS(i)], dim=-1) for i in range(self.num_labels)} diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index 6b0829bd..455d27a7 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -245,7 +245,7 @@ class BiaffineParser(GraphParser): Biaffine Dependency Parser 实现. 论文参考 `Deep Biaffine Attention for Neural Dependency Parsing (Dozat and Manning, 2016) `_ . - :param init_embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding :param pos_vocab_size: part-of-speech 词典大小 @@ -262,7 +262,7 @@ class BiaffineParser(GraphParser): """ def __init__(self, - init_embed, + embed, pos_vocab_size, pos_emb_dim, num_label, @@ -276,7 +276,7 @@ class BiaffineParser(GraphParser): super(BiaffineParser, self).__init__() rnn_out_size = 2 * rnn_hidden_size word_hid_dim = pos_hid_dim = rnn_hidden_size - self.word_embedding = get_embeddings(init_embed) + self.word_embedding = get_embeddings(embed) word_emb_dim = self.word_embedding.embedding_dim self.pos_embedding = nn.Embedding(num_embeddings=pos_vocab_size, embedding_dim=pos_emb_dim) self.word_fc = nn.Linear(word_emb_dim, word_hid_dim) diff --git a/fastNLP/models/cnn_text_classification.py b/fastNLP/models/cnn_text_classification.py index 37a60c35..4bf9c4d1 100644 --- a/fastNLP/models/cnn_text_classification.py +++ b/fastNLP/models/cnn_text_classification.py @@ -23,7 +23,7 @@ class CNNText(torch.nn.Module): 使用CNN进行文本分类的模型 'Yoon Kim. 2014. Convolution Neural Networks for Sentence Classification.' - :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray init_embed: Embedding的大小(传入tuple(int, int), + :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, Embedding, ndarray等则直接使用该值初始化Embedding :param int num_classes: 一共有多少类 :param int,tuple(int) out_channels: 输出channel的数量。如果为list，则需要与kernel_sizes的数量保持一致 @@ -31,7 +31,7 @@ class CNNText(torch.nn.Module): :param float dropout: Dropout的大小 """ - def __init__(self, init_embed, + def __init__(self, embed, num_classes, kernel_nums=(30, 40, 50), kernel_sizes=(1, 3, 5), @@ -39,7 +39,7 @@ class CNNText(torch.nn.Module): super(CNNText, self).__init__() # no support for pre-trained embedding currently - self.embed = embedding.Embedding(init_embed) + self.embed = embedding.Embedding(embed) self.conv_pool = encoder.ConvMaxpool( in_channels=self.embed.embedding_dim, out_channels=kernel_nums, diff --git a/fastNLP/models/snli.py b/fastNLP/models/snli.py index 5ca4052d..97a14e9f 100644 --- a/fastNLP/models/snli.py +++ b/fastNLP/models/snli.py @@ -24,21 +24,21 @@ class ESIM(BaseModel): ESIM model的一个PyTorch实现 论文参见： https://arxiv.org/pdf/1609.06038.pdf - :param init_embedding: 初始化的Embedding + :param embed: 初始化的Embedding :param int hidden_size: 隐藏层大小，默认值为Embedding的维度 :param int num_labels: 目标标签种类数量，默认值为3 :param float dropout_rate: dropout的比率，默认值为0.3 :param float dropout_embed: 对Embedding的dropout比率，默认值为0.1 """ - def __init__(self, init_embedding, hidden_size=None, num_labels=3, dropout_rate=0.3, + def __init__(self, embed, hidden_size=None, num_labels=3, dropout_rate=0.3, dropout_embed=0.1): super(ESIM, self).__init__() - if isinstance(init_embedding, TokenEmbedding) or isinstance(init_embedding, Embedding): - self.embedding = init_embedding + if isinstance(embed, TokenEmbedding) or isinstance(embed, Embedding): + self.embedding = embed else: - self.embedding = Embedding(init_embedding) + self.embedding = Embedding(embed) self.dropout_embed = EmbedDropout(p=dropout_embed) if hidden_size is None: hidden_size = self.embedding.embed_size diff --git a/fastNLP/models/star_transformer.py b/fastNLP/models/star_transformer.py index b95d1c25..7fe0d343 100644 --- a/fastNLP/models/star_transformer.py +++ b/fastNLP/models/star_transformer.py @@ -23,7 +23,7 @@ class StarTransEnc(nn.Module): 带word embedding的Star-Transformer Encoder - :param init_embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding :param hidden_size: 模型中特征维度. @@ -35,7 +35,7 @@ class StarTransEnc(nn.Module): :param dropout: 模型除词嵌入外的dropout概率. """ - def __init__(self, init_embed, + def __init__(self, embed, hidden_size, num_layers, num_head, @@ -44,7 +44,7 @@ class StarTransEnc(nn.Module): emb_dropout, dropout): super(StarTransEnc, self).__init__() - self.embedding = get_embeddings(init_embed) + self.embedding = get_embeddings(embed) emb_dim = self.embedding.embedding_dim self.emb_fc = nn.Linear(emb_dim, hidden_size) # self.emb_drop = nn.Dropout(emb_dropout) @@ -108,7 +108,7 @@ class STSeqLabel(nn.Module): 用于序列标注的Star-Transformer模型 - :param init_embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding :param num_cls: 输出类别个数 @@ -122,7 +122,7 @@ class STSeqLabel(nn.Module): :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 """ - def __init__(self, init_embed, num_cls, + def __init__(self, embed, num_cls, hidden_size=300, num_layers=4, num_head=8, @@ -132,7 +132,7 @@ class STSeqLabel(nn.Module): emb_dropout=0.1, dropout=0.1, ): super(STSeqLabel, self).__init__() - self.enc = StarTransEnc(init_embed=init_embed, + self.enc = StarTransEnc(embed=embed, hidden_size=hidden_size, num_layers=num_layers, num_head=num_head, @@ -173,7 +173,7 @@ class STSeqCls(nn.Module): 用于分类任务的Star-Transformer - :param init_embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding :param num_cls: 输出类别个数 @@ -187,7 +187,7 @@ class STSeqCls(nn.Module): :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 """ - def __init__(self, init_embed, num_cls, + def __init__(self, embed, num_cls, hidden_size=300, num_layers=4, num_head=8, @@ -197,7 +197,7 @@ class STSeqCls(nn.Module): emb_dropout=0.1, dropout=0.1, ): super(STSeqCls, self).__init__() - self.enc = StarTransEnc(init_embed=init_embed, + self.enc = StarTransEnc(embed=embed, hidden_size=hidden_size, num_layers=num_layers, num_head=num_head, @@ -238,7 +238,7 @@ class STNLICls(nn.Module): 用于自然语言推断(NLI)的Star-Transformer - :param init_embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding :param num_cls: 输出类别个数 @@ -252,7 +252,7 @@ class STNLICls(nn.Module): :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 """ - def __init__(self, init_embed, num_cls, + def __init__(self, embed, num_cls, hidden_size=300, num_layers=4, num_head=8, @@ -262,7 +262,7 @@ class STNLICls(nn.Module): emb_dropout=0.1, dropout=0.1, ): super(STNLICls, self).__init__() - self.enc = StarTransEnc(init_embed=init_embed, + self.enc = StarTransEnc(embed=embed, hidden_size=hidden_size, num_layers=num_layers, num_head=num_head, diff --git a/test/models/test_bert.py b/test/models/test_bert.py index 969a8594..9cab3a88 100644 --- a/test/models/test_bert.py +++ b/test/models/test_bert.py @@ -23,10 +23,25 @@ class TestBert(unittest.TestCase): self.assertTrue(Const.OUTPUT in pred) self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2, 2)) - pred = model.predict(input_ids) + pred = model(input_ids) self.assertTrue(isinstance(pred, dict)) self.assertTrue(Const.OUTPUT in pred) - self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2,)) + self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2, 2)) + + def test_bert_1_w(self): + vocab = Vocabulary().add_word_lst("this is a test .".split()) + embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', + include_cls_sep=False) + + with self.assertWarns(Warning): + model = BertForSequenceClassification(embed, 2) + + input_ids = torch.LongTensor([[1, 2, 3], [5, 6, 0]]) + + pred = model.predict(input_ids) + self.assertTrue(isinstance(pred, dict)) + self.assertTrue(Const.OUTPUT in pred) + self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2,)) def test_bert_2(self): @@ -44,6 +59,23 @@ class TestBert(unittest.TestCase): self.assertTrue(Const.OUTPUT in pred) self.assertEqual(tuple(pred[Const.OUTPUT].shape), (1, 2)) + def test_bert_2_w(self): + + vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) + embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', + include_cls_sep=False) + + with self.assertWarns(Warning): + model = BertForMultipleChoice(embed, 2) + + input_ids = torch.LongTensor([[[2, 6, 7], [1, 6, 5]]]) + print(input_ids.size()) + + pred = model.predict(input_ids) + self.assertTrue(isinstance(pred, dict)) + self.assertTrue(Const.OUTPUT in pred) + self.assertEqual(tuple(pred[Const.OUTPUT].shape), (1,)) + def test_bert_3(self): vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) @@ -58,6 +90,22 @@ class TestBert(unittest.TestCase): self.assertTrue(Const.OUTPUT in pred) self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2, 3, 7)) + def test_bert_3_w(self): + + vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) + embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', + include_cls_sep=True) + + with self.assertWarns(Warning): + model = BertForTokenClassification(embed, 7) + + input_ids = torch.LongTensor([[1, 2, 3], [6, 5, 0]]) + + pred = model.predict(input_ids) + self.assertTrue(isinstance(pred, dict)) + self.assertTrue(Const.OUTPUT in pred) + self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2, 3)) + def test_bert_4(self): vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) @@ -79,6 +127,22 @@ class TestBert(unittest.TestCase): self.assertTrue(isinstance(pred, dict)) self.assertEqual(len(pred), 7) + def test_bert_4_w(self): + + vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) + embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', + include_cls_sep=False) + + with self.assertWarns(Warning): + model = BertForQuestionAnswering(embed) + + input_ids = torch.LongTensor([[1, 2, 3], [6, 5, 0]]) + + pred = model.predict(input_ids) + self.assertTrue(isinstance(pred, dict)) + self.assertTrue(Const.OUTPUTS(1) in pred) + self.assertEqual(tuple(pred[Const.OUTPUTS(1)].shape), (2,)) + def test_bert_5(self): vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) @@ -93,3 +157,19 @@ class TestBert(unittest.TestCase): self.assertTrue(Const.OUTPUT in pred) self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2, 2)) + def test_bert_5_w(self): + + vocab = Vocabulary().add_word_lst("this is a test [SEP] .".split()) + embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', + include_cls_sep=False) + + with self.assertWarns(Warning): + model = BertForSentenceMatching(embed) + + input_ids = torch.LongTensor([[1, 2, 3], [6, 5, 0]]) + + pred = model.predict(input_ids) + self.assertTrue(isinstance(pred, dict)) + self.assertTrue(Const.OUTPUT in pred) + self.assertEqual(tuple(pred[Const.OUTPUT].shape), (2,)) + diff --git a/test/models/test_biaffine_parser.py b/test/models/test_biaffine_parser.py index 4f93b994..4b38d816 100644 --- a/test/models/test_biaffine_parser.py +++ b/test/models/test_biaffine_parser.py @@ -27,7 +27,7 @@ def prepare_parser_data(): class TestBiaffineParser(unittest.TestCase): def test_train(self): - model = BiaffineParser(init_embed=(VOCAB_SIZE, 10), + model = BiaffineParser(embed=(VOCAB_SIZE, 10), pos_vocab_size=VOCAB_SIZE, pos_emb_dim=10, rnn_hidden_size=10, arc_mlp_size=10, @@ -37,7 +37,7 @@ class TestBiaffineParser(unittest.TestCase): RUNNER.run_model(model, ds, loss=ParserLoss(), metrics=ParserMetric()) def test_train2(self): - model = BiaffineParser(init_embed=(VOCAB_SIZE, 10), + model = BiaffineParser(embed=(VOCAB_SIZE, 10), pos_vocab_size=VOCAB_SIZE, pos_emb_dim=10, rnn_hidden_size=16, arc_mlp_size=10, From d15ad75d96f3b72fe6b439ef8ce6e4829987ce0f Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Tue, 3 Sep 2019 23:33:10 +0800 Subject: [PATCH 33/92] fix a bug in test code --- test/modules/decoder/test_bert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/modules/decoder/test_bert.py b/test/modules/decoder/test_bert.py index 0fcf01e4..56946f5d 100644 --- a/test/modules/decoder/test_bert.py +++ b/test/modules/decoder/test_bert.py @@ -3,7 +3,7 @@ import unittest import torch -from fastNLP.models.bert import BertModel +from fastNLP.modules.encoder.bert import BertModel class TestBert(unittest.TestCase): From e903db0e70bb4cd9e9b45907fc33db4b4fce9765 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Wed, 4 Sep 2019 12:47:52 +0800 Subject: [PATCH 34/92] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E5=88=86=E7=B1=BBPipe;=E4=BD=BF=E7=94=A8=E7=9F=A9=E9=98=B5?= =?UTF-8?q?=E5=8A=A0=E9=80=9FBertEmbedding=E9=83=A8=E5=88=86pool=5Fmethod;?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=83=A8=E5=88=86=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E4=BE=8B=E5=90=8D=E7=A7=B0;=E4=BF=AE=E5=A4=8Dmetric=E4=B8=AD?= =?UTF-8?q?=E5=AF=B9warning=E7=9A=84=E8=AF=AF=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/losses.py | 4 +- fastNLP/core/metrics.py | 2 +- fastNLP/embeddings/bert_embedding.py | 42 +++++--- fastNLP/io/__init__.py | 4 +- fastNLP/io/data_bundle.py | 15 +-- fastNLP/io/file_utils.py | 7 +- fastNLP/io/loader/__init__.py | 3 +- fastNLP/io/loader/classification.py | 57 +++++++++++ fastNLP/io/pipe/__init__.py | 3 +- fastNLP/io/pipe/classification.py | 101 ++++++++++++++++++- fastNLP/io/pipe/conll.py | 40 ++++++-- test/embeddings/test_bert_embedding.py | 6 ++ test/io/loader/test_classification_loader.py | 6 +- test/io/loader/test_conll_loader.py | 6 +- test/io/loader/test_cws_loader.py | 4 +- test/io/loader/test_matching_loader.py | 5 +- test/io/pipe/test_classification.py | 13 ++- test/io/pipe/test_conll.py | 6 +- test/io/pipe/test_cws.py | 4 +- test/io/pipe/test_matching.py | 6 +- 20 files changed, 274 insertions(+), 60 deletions(-) diff --git a/fastNLP/core/losses.py b/fastNLP/core/losses.py index d5549cec..7402a568 100644 --- a/fastNLP/core/losses.py +++ b/fastNLP/core/losses.py @@ -238,8 +238,8 @@ class CrossEntropyLoss(LossBase): pred = pred.tranpose(-1, pred) pred = pred.reshape(-1, pred.size(-1)) target = target.reshape(-1) - if seq_len is not None: - mask = seq_len_to_mask(seq_len).reshape(-1).eq(0) + if seq_len is not None and target.dim()>1: + mask = seq_len_to_mask(seq_len, max_len=target.size(1)).reshape(-1).eq(0) target = target.masked_fill(mask, self.padding_idx) return F.cross_entropy(input=pred, target=target, diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index b06e5459..c0f14c90 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -347,7 +347,7 @@ class AccuracyMetric(MetricBase): pass elif pred.dim() == target.dim() + 1: pred = pred.argmax(dim=-1) - if seq_len is None: + if seq_len is None and target.dim()>1: warnings.warn("You are not passing `seq_len` to exclude pad when calculate accuracy.") else: raise RuntimeError(f"In {_get_func_signature(self.evaluate)}, when pred have " diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index f6c36623..08615fe0 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -68,7 +68,7 @@ class BertEmbedding(ContextualEmbedding): def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en-base-uncased', layers: str = '-1', pool_method: str = 'first', word_dropout=0, dropout=0, include_cls_sep: bool = False, - pooled_cls=True, requires_grad: bool = False, auto_truncate: bool = False): + pooled_cls=True, requires_grad: bool = True, auto_truncate: bool = False): super(BertEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) if model_dir_or_name.lower() in PRETRAINED_BERT_MODEL_DIR: @@ -165,7 +165,7 @@ class BertWordPieceEncoder(nn.Module): """ def __init__(self, model_dir_or_name: str = 'en-base-uncased', layers: str = '-1', pooled_cls: bool = False, - word_dropout=0, dropout=0, requires_grad: bool = False): + word_dropout=0, dropout=0, requires_grad: bool = True): super().__init__() self.model = _WordPieceBertModel(model_dir_or_name=model_dir_or_name, layers=layers, pooled_cls=pooled_cls) @@ -288,7 +288,7 @@ class _WordBertModel(nn.Module): self.auto_truncate = auto_truncate # 将所有vocab中word的wordpiece计算出来, 需要额外考虑[CLS]和[SEP] - logger.info("Start to generating word pieces for word.") + logger.info("Start to generate word pieces for word.") # 第一步统计出需要的word_piece, 然后创建新的embed和word_piece_vocab, 然后填入值 word_piece_dict = {'[CLS]': 1, '[SEP]': 1} # 用到的word_piece以及新增的 found_count = 0 @@ -374,7 +374,8 @@ class _WordBertModel(nn.Module): else: raise RuntimeError( "After split words into word pieces, the lengths of word pieces are longer than the " - f"maximum allowed sequence length:{self._max_position_embeddings} of bert.") + f"maximum allowed sequence length:{self._max_position_embeddings} of bert. You can set " + f"`auto_truncate=True` for BertEmbedding to automatically truncate overlong input.") # +2是由于需要加入[CLS]与[SEP] word_pieces = words.new_full((batch_size, min(word_piece_length + 2, self._max_position_embeddings)), @@ -407,15 +408,26 @@ class _WordBertModel(nn.Module): # output_layers = [self.layers] # len(self.layers) x batch_size x real_word_piece_length x hidden_size if self.include_cls_sep: - outputs = bert_outputs[-1].new_zeros(len(self.layers), batch_size, max_word_len + 2, - bert_outputs[-1].size(-1)) s_shift = 1 + outputs = bert_outputs[-1].new_zeros(len(self.layers), batch_size, max_word_len + 2, + bert_outputs[-1].size(-1)) + else: + s_shift = 0 outputs = bert_outputs[-1].new_zeros(len(self.layers), batch_size, max_word_len, bert_outputs[-1].size(-1)) - s_shift = 0 batch_word_pieces_cum_length = batch_word_pieces_length.new_zeros(batch_size, max_word_len + 1) batch_word_pieces_cum_length[:, 1:] = batch_word_pieces_length.cumsum(dim=-1) # batch_size x max_len + + if self.pool_method == 'first': + batch_word_pieces_cum_length = batch_word_pieces_cum_length[:, :seq_len.max()] + batch_word_pieces_cum_length.masked_fill_(batch_word_pieces_cum_length.ge(word_piece_length), 0) + batch_indexes = batch_indexes[:, None].expand((batch_size, batch_word_pieces_cum_length.size(1))) + elif self.pool_method == 'last': + batch_word_pieces_cum_length = batch_word_pieces_cum_length[:, 1:seq_len.max()+1] - 1 + batch_word_pieces_cum_length.masked_fill_(batch_word_pieces_cum_length.ge(word_piece_length), 0) + batch_indexes = batch_indexes[:, None].expand((batch_size, batch_word_pieces_cum_length.size(1))) + for l_index, l in enumerate(self.layers): output_layer = bert_outputs[l] real_word_piece_length = output_layer.size(1) - 2 @@ -426,16 +438,15 @@ class _WordBertModel(nn.Module): output_layer = torch.cat((output_layer, paddings), dim=1).contiguous() # 从word_piece collapse到word的表示 truncate_output_layer = output_layer[:, 1:-1] # 删除[CLS]与[SEP] batch_size x len x hidden_size - outputs_seq_len = seq_len + s_shift if self.pool_method == 'first': - for i in range(batch_size): - i_word_pieces_cum_length = batch_word_pieces_cum_length[i, :seq_len[i]] # 每个word的start位置 - outputs[l_index, i, s_shift:outputs_seq_len[i]] = truncate_output_layer[ - i, i_word_pieces_cum_length] # num_layer x batch_size x len x hidden_size + tmp = truncate_output_layer[batch_indexes, batch_word_pieces_cum_length] + tmp = tmp.masked_fill(word_mask[:, :batch_word_pieces_cum_length.size(1), None].eq(0), 0) + outputs[l_index, :, s_shift:batch_word_pieces_cum_length.size(1)+s_shift] = tmp + elif self.pool_method == 'last': - for i in range(batch_size): - i_word_pieces_cum_length = batch_word_pieces_cum_length[i, 1:seq_len[i] + 1] - 1 # 每个word的end - outputs[l_index, i, s_shift:outputs_seq_len[i]] = truncate_output_layer[i, i_word_pieces_cum_length] + tmp = truncate_output_layer[batch_indexes, batch_word_pieces_cum_length] + tmp = tmp.masked_fill(word_mask[:, :batch_word_pieces_cum_length.size(1), None].eq(0), 0) + outputs[l_index, :, s_shift:batch_word_pieces_cum_length.size(1)+s_shift] = tmp elif self.pool_method == 'max': for i in range(batch_size): for j in range(seq_len[i]): @@ -452,5 +463,6 @@ class _WordBertModel(nn.Module): else: outputs[l_index, :, 0] = output_layer[:, 0] outputs[l_index, batch_indexes, seq_len + s_shift] = output_layer[batch_indexes, seq_len + s_shift] + # 3. 最终的embedding结果 return outputs diff --git a/fastNLP/io/__init__.py b/fastNLP/io/__init__.py index 251b7292..6f727f05 100644 --- a/fastNLP/io/__init__.py +++ b/fastNLP/io/__init__.py @@ -24,6 +24,7 @@ __all__ = [ 'IMDBLoader', 'SSTLoader', 'SST2Loader', + "ChnSentiCorpLoader", 'ConllLoader', 'Conll2003Loader', @@ -52,8 +53,9 @@ __all__ = [ "SSTPipe", "SST2Pipe", "IMDBPipe", - "Conll2003Pipe", + "ChnSentiCorpPipe", + "Conll2003Pipe", "Conll2003NERPipe", "OntoNotesNERPipe", "MsraNERPipe", diff --git a/fastNLP/io/data_bundle.py b/fastNLP/io/data_bundle.py index 3e7f39d3..19b48828 100644 --- a/fastNLP/io/data_bundle.py +++ b/fastNLP/io/data_bundle.py @@ -306,12 +306,15 @@ class DataBundle: return self def __repr__(self): - _str = 'In total {} datasets:\n'.format(len(self.datasets)) - for name, dataset in self.datasets.items(): - _str += '\t{} has {} instances.\n'.format(name, len(dataset)) - _str += 'In total {} vocabs:\n'.format(len(self.vocabs)) - for name, vocab in self.vocabs.items(): - _str += '\t{} has {} entries.\n'.format(name, len(vocab)) + _str = '' + if len(self.datasets): + _str += 'In total {} datasets:\n'.format(len(self.datasets)) + for name, dataset in self.datasets.items(): + _str += '\t{} has {} instances.\n'.format(name, len(dataset)) + if len(self.vocabs): + _str += 'In total {} vocabs:\n'.format(len(self.vocabs)) + for name, vocab in self.vocabs.items(): + _str += '\t{} has {} entries.\n'.format(name, len(vocab)) return _str diff --git a/fastNLP/io/file_utils.py b/fastNLP/io/file_utils.py index 8ecdff25..f76bcd26 100644 --- a/fastNLP/io/file_utils.py +++ b/fastNLP/io/file_utils.py @@ -77,6 +77,9 @@ PRETRAIN_STATIC_FILES = { 'cn-tencent': "tencent_cn.zip", 'cn-fasttext': "cc.zh.300.vec.gz", 'cn-sgns-literature-word': 'sgns.literature.word.txt.zip', + 'cn-char-fastnlp-100d': "cn_char_fastnlp_100d.zip", + 'cn-bi-fastnlp-100d': "cn_bi_fastnlp_100d.zip", + "cn-tri-fastnlp-100d": "cn_tri_fastnlp_100d.zip" } DATASET_DIR = { @@ -96,7 +99,9 @@ DATASET_DIR = { "cws-pku": 'cws_pku.zip', "cws-cityu": "cws_cityu.zip", "cws-as": 'cws_as.zip', - "cws-msra": 'cws_msra.zip' + "cws-msra": 'cws_msra.zip', + + "chn-senti-corp":"chn_senti_corp.zip" } PRETRAIN_MAP = {'elmo': PRETRAINED_ELMO_MODEL_DIR, diff --git a/fastNLP/io/loader/__init__.py b/fastNLP/io/loader/__init__.py index 6c23f213..3ad1b47d 100644 --- a/fastNLP/io/loader/__init__.py +++ b/fastNLP/io/loader/__init__.py @@ -52,6 +52,7 @@ __all__ = [ 'IMDBLoader', 'SSTLoader', 'SST2Loader', + "ChnSentiCorpLoader", 'ConllLoader', 'Conll2003Loader', @@ -73,7 +74,7 @@ __all__ = [ "QNLILoader", "RTELoader" ] -from .classification import YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader +from .classification import YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader, ChnSentiCorpLoader from .conll import ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader from .csv import CSVLoader from .cws import CWSLoader diff --git a/fastNLP/io/loader/classification.py b/fastNLP/io/loader/classification.py index ec00d2b4..4ebd58e1 100644 --- a/fastNLP/io/loader/classification.py +++ b/fastNLP/io/loader/classification.py @@ -7,6 +7,7 @@ __all__ = [ "IMDBLoader", "SSTLoader", "SST2Loader", + "ChnSentiCorpLoader" ] import glob @@ -346,3 +347,59 @@ class SST2Loader(Loader): """ output_dir = self._get_dataset_path(dataset_name='sst-2') return output_dir + + +class ChnSentiCorpLoader(Loader): + """ + 支持读取的数据的格式为，第一行为标题(具体内容会被忽略)，之后一行为一个sample，第一个制表符之前被认为是label，第 + 一个制表符及之后认为是句子 + + Example:: + + label raw_chars + 1 這間酒店環境和服務態度亦算不錯,但房間空間太小~~ + 1 <荐书> 推荐所有喜欢<红楼>的红迷们一定要收藏这本书,要知道... + 0 商品的不足暂时还没发现，京东的订单处理速度实在.......周二就打包完成，周五才发货... + + 读取后的DataSet具有以下的field + + .. csv-table:: + :header: "raw_chars", "target" + + "這間酒店環境和服務態度亦算不錯,但房間空間太小~~", "1" + "<荐书> 推荐所有喜欢<红楼>...", "1" + "..." + + """ + def __init__(self): + super().__init__() + + def _load(self, path:str): + """ + 从path中读取数据 + + :param path: + :return: + """ + ds = DataSet() + with open(path, 'r', encoding='utf-8') as f: + f.readline() + for line in f: + line = line.strip() + tab_index = line.index('\t') + if tab_index!=-1: + target = line[:tab_index] + raw_chars = line[tab_index+1:] + if raw_chars: + ds.append(Instance(raw_chars=raw_chars, target=target)) + return ds + + def download(self)->str: + """ + 自动下载数据，该数据取自https://github.com/pengming617/bert_classification/tree/master/data，在 + https://arxiv.org/pdf/1904.09223.pdf与https://arxiv.org/pdf/1906.08101.pdf有使用 + + :return: + """ + output_dir = self._get_dataset_path('chn-senti-corp') + return output_dir diff --git a/fastNLP/io/pipe/__init__.py b/fastNLP/io/pipe/__init__.py index 048e4cfe..943709e7 100644 --- a/fastNLP/io/pipe/__init__.py +++ b/fastNLP/io/pipe/__init__.py @@ -17,6 +17,7 @@ __all__ = [ "SSTPipe", "SST2Pipe", "IMDBPipe", + "ChnSentiCorpPipe", "Conll2003NERPipe", "OntoNotesNERPipe", @@ -39,7 +40,7 @@ __all__ = [ "MNLIPipe", ] -from .classification import YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe +from .classification import YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe, ChnSentiCorpPipe from .conll import Conll2003NERPipe, OntoNotesNERPipe, MsraNERPipe, WeiboNERPipe, PeopleDailyPipe from .matching import MatchingBertPipe, RTEBertPipe, SNLIBertPipe, QuoraBertPipe, QNLIBertPipe, MNLIBertPipe, \ MatchingPipe, RTEPipe, SNLIPipe, QuoraPipe, QNLIPipe, MNLIPipe diff --git a/fastNLP/io/pipe/classification.py b/fastNLP/io/pipe/classification.py index 30c591a4..d1c7aa0e 100644 --- a/fastNLP/io/pipe/classification.py +++ b/fastNLP/io/pipe/classification.py @@ -5,7 +5,8 @@ __all__ = [ "YelpPolarityPipe", "SSTPipe", "SST2Pipe", - 'IMDBPipe' + 'IMDBPipe', + "ChnSentiCorpPipe" ] import re @@ -13,18 +14,18 @@ import re from nltk import Tree from .pipe import Pipe -from .utils import get_tokenizer, _indexize, _add_words_field, _drop_empty_instance +from .utils import get_tokenizer, _indexize, _add_words_field, _drop_empty_instance, _add_chars_field from ..data_bundle import DataBundle from ..loader.classification import IMDBLoader, YelpFullLoader, SSTLoader, SST2Loader, YelpPolarityLoader from ...core.const import Const from ...core.dataset import DataSet from ...core.instance import Instance from ...core.vocabulary import Vocabulary +from ..loader.classification import ChnSentiCorpLoader nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') - class _CLSPipe(Pipe): """ 分类问题的基类，负责对classification的数据进行tokenize操作。默认是对raw_words列操作，然后生成words列 @@ -457,3 +458,97 @@ class IMDBPipe(_CLSPipe): data_bundle = self.process(data_bundle) return data_bundle + + +class ChnSentiCorpPipe(Pipe): + """ + 处理之后的DataSet有以下的结构 + + .. csv-table:: + :header: "raw_chars", "chars", "target", "seq_len" + + "這間酒店環境和服務態度亦算不錯,但房間空間太小~~", "[2, 3, 4, 5, ...]", 1, 31 + "<荐书> 推荐所有喜欢<红楼>...", "[10, 21, ....]", 1, 25 + "..." + + 其中chars, seq_len是input，target是target + + :param bool bigrams: 是否增加一列bigrams. bigrams的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...]。如果 + 设置为True，返回的DataSet将有一列名为bigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('bigrams')获取. + :param bool trigrams: 是否增加一列trigrams. trigrams的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] + 。如果设置为True，返回的DataSet将有一列名为trigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('trigrams')获取. + """ + def __init__(self, bigrams=False, trigrams=False): + super().__init__() + + self.bigrams = bigrams + self.trigrams = trigrams + + def _tokenize(self, data_bundle): + """ + 将DataSet中的"复旦大学"拆分为["复", "旦", "大", "学"]. 未来可以通过扩展这个函数实现分词。 + + :param data_bundle: + :return: + """ + data_bundle.apply_field(list, field_name=Const.CHAR_INPUT, new_field_name=Const.CHAR_INPUT) + return data_bundle + + def process(self, data_bundle:DataBundle): + """ + 可以处理的DataSet应该具备以下的field + + .. csv-table:: + :header: "raw_chars", "target" + + "這間酒店環境和服務態度亦算不錯,但房間空間太小~~", "1" + "<荐书> 推荐所有喜欢<红楼>...", "1" + "..." + + :param data_bundle: + :return: + """ + _add_chars_field(data_bundle, lower=False) + + data_bundle = self._tokenize(data_bundle) + + input_field_names = [Const.CHAR_INPUT] + if self.bigrams: + for name, dataset in data_bundle.iter_datasets(): + dataset.apply_field(lambda chars: [c1 + c2 for c1, c2 in zip(chars, chars[1:] + [''])], + field_name=Const.CHAR_INPUT, new_field_name='bigrams') + input_field_names.append('bigrams') + if self.trigrams: + for name, dataset in data_bundle.iter_datasets(): + dataset.apply_field(lambda chars: [c1 + c2 + c3 for c1, c2, c3 in + zip(chars, chars[1:] + [''], chars[2:] + [''] * 2)], + field_name=Const.CHAR_INPUT, new_field_name='trigrams') + input_field_names.append('trigrams') + + # index + _indexize(data_bundle, input_field_names, Const.TARGET) + + input_fields = [Const.TARGET, Const.INPUT_LEN] + input_field_names + target_fields = [Const.TARGET] + + for name, dataset in data_bundle.datasets.items(): + dataset.add_seq_len(Const.CHAR_INPUT) + + data_bundle.set_input(*input_fields) + data_bundle.set_target(*target_fields) + + return data_bundle + + def process_from_file(self, paths=None): + """ + + :param paths: 支持路径类型参见 :class:`fastNLP.io.loader.Loader` 的load函数。 + :return: DataBundle + """ + # 读取数据 + data_bundle = ChnSentiCorpLoader().load(paths) + data_bundle = self.process(data_bundle) + + return data_bundle \ No newline at end of file diff --git a/fastNLP/io/pipe/conll.py b/fastNLP/io/pipe/conll.py index 2edc9008..a96b259a 100644 --- a/fastNLP/io/pipe/conll.py +++ b/fastNLP/io/pipe/conll.py @@ -222,14 +222,23 @@ class _CNNERPipe(Pipe): target。返回的DataSet中被设置为input有chars, target, seq_len; 设置为target有target, seq_len。 :param: str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 + :param bool bigrams: 是否增加一列bigrams. bigrams的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...]。如果 + 设置为True，返回的DataSet将有一列名为bigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('bigrams')获取. + :param bool trigrams: 是否增加一列trigrams. trigrams的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] + 。如果设置为True，返回的DataSet将有一列名为trigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('trigrams')获取. """ - def __init__(self, encoding_type: str = 'bio'): + def __init__(self, encoding_type: str = 'bio', bigrams=False, trigrams=False): if encoding_type == 'bio': self.convert_tag = iob2 else: self.convert_tag = lambda words: iob2bioes(iob2(words)) - + + self.bigrams = bigrams + self.trigrams = trigrams + def process(self, data_bundle: DataBundle) -> DataBundle: """ 支持的DataSet的field为 @@ -241,11 +250,11 @@ class _CNNERPipe(Pipe): "[青, 岛, 海, 牛, 队, 和, ...]", "[B-ORG, I-ORG, I-ORG, ...]" "[...]", "[...]" - raw_chars列为List[str], 是未转换的原始数据; chars列为List[int]，是转换为index的输入数据; target列是List[int]，是转换为index的 - target。返回的DataSet中被设置为input有chars, target, seq_len; 设置为target有target。 + raw_chars列为List[str], 是未转换的原始数据; chars列为List[int]，是转换为index的输入数据; target列是List[int]， + 是转换为index的target。返回的DataSet中被设置为input有chars, target, seq_len; 设置为target有target。 - :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]。 - 在传入DataBundle基础上原位修改。 + :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field + 的内容均为List[str]。在传入DataBundle基础上原位修改。 :return: DataBundle """ # 转换tag @@ -253,11 +262,24 @@ class _CNNERPipe(Pipe): dataset.apply_field(self.convert_tag, field_name=Const.TARGET, new_field_name=Const.TARGET) _add_chars_field(data_bundle, lower=False) - + + input_field_names = [Const.CHAR_INPUT] + if self.bigrams: + for name, dataset in data_bundle.datasets.items(): + dataset.apply_field(lambda chars: [c1 + c2 for c1, c2 in zip(chars, chars[1:] + [''])], + field_name=Const.CHAR_INPUT, new_field_name='bigrams') + input_field_names.append('bigrams') + if self.trigrams: + for name, dataset in data_bundle.datasets.items(): + dataset.apply_field(lambda chars: [c1 + c2 + c3 for c1, c2, c3 in + zip(chars, chars[1:] + [''], chars[2:] + [''] * 2)], + field_name=Const.CHAR_INPUT, new_field_name='trigrams') + input_field_names.append('trigrams') + # index - _indexize(data_bundle, input_field_names=Const.CHAR_INPUT, target_field_names=Const.TARGET) + _indexize(data_bundle, input_field_names, Const.TARGET) - input_fields = [Const.TARGET, Const.CHAR_INPUT, Const.INPUT_LEN] + input_fields = [Const.TARGET, Const.INPUT_LEN] + input_field_names target_fields = [Const.TARGET, Const.INPUT_LEN] for name, dataset in data_bundle.datasets.items(): diff --git a/test/embeddings/test_bert_embedding.py b/test/embeddings/test_bert_embedding.py index 46ad74c3..6a4a0ffa 100644 --- a/test/embeddings/test_bert_embedding.py +++ b/test/embeddings/test_bert_embedding.py @@ -13,6 +13,12 @@ class TestDownload(unittest.TestCase): words = torch.LongTensor([[2, 3, 4, 0]]) print(embed(words).size()) + for pool_method in ['first', 'last', 'max', 'avg']: + for include_cls_sep in [True, False]: + embed = BertEmbedding(vocab, model_dir_or_name='en', pool_method=pool_method, + include_cls_sep=include_cls_sep) + print(embed(words).size()) + def test_word_drop(self): vocab = Vocabulary().add_word_lst("This is a test .".split()) embed = BertEmbedding(vocab, model_dir_or_name='en', dropout=0.1, word_dropout=0.2) diff --git a/test/io/loader/test_classification_loader.py b/test/io/loader/test_classification_loader.py index 1438a014..f099c1b2 100644 --- a/test/io/loader/test_classification_loader.py +++ b/test/io/loader/test_classification_loader.py @@ -5,22 +5,22 @@ from fastNLP.io.loader.classification import YelpPolarityLoader from fastNLP.io.loader.classification import IMDBLoader from fastNLP.io.loader.classification import SST2Loader from fastNLP.io.loader.classification import SSTLoader +from fastNLP.io.loader.classification import ChnSentiCorpLoader import os @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") class TestDownload(unittest.TestCase): def test_download(self): - for loader in [YelpFullLoader, YelpPolarityLoader, IMDBLoader, SST2Loader, SSTLoader]: + for loader in [YelpFullLoader, YelpPolarityLoader, IMDBLoader, SST2Loader, SSTLoader, ChnSentiCorpLoader]: loader().download() def test_load(self): - for loader in [YelpFullLoader, YelpPolarityLoader, IMDBLoader, SST2Loader, SSTLoader]: + for loader in [YelpFullLoader, YelpPolarityLoader, IMDBLoader, SST2Loader, SSTLoader, ChnSentiCorpLoader]: data_bundle = loader().load() print(data_bundle) class TestLoad(unittest.TestCase): - def test_load(self): for loader in [IMDBLoader]: data_bundle = loader().load('test/data_for_tests/io/imdb') diff --git a/test/io/loader/test_conll_loader.py b/test/io/loader/test_conll_loader.py index 861de5a5..31859a6b 100644 --- a/test/io/loader/test_conll_loader.py +++ b/test/io/loader/test_conll_loader.py @@ -5,7 +5,7 @@ from fastNLP.io.loader.conll import MsraNERLoader, PeopleDailyNERLoader, WeiboNE Conll2003Loader -class MSRANERTest(unittest.TestCase): +class TestMSRANER(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_download(self): MsraNERLoader().download(re_download=False) @@ -13,13 +13,13 @@ class MSRANERTest(unittest.TestCase): print(data_bundle) -class PeopleDailyTest(unittest.TestCase): +class TestPeopleDaily(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_download(self): PeopleDailyNERLoader().download() -class WeiboNERTest(unittest.TestCase): +class TestWeiboNER(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_download(self): WeiboNERLoader().download() diff --git a/test/io/loader/test_cws_loader.py b/test/io/loader/test_cws_loader.py index 8b5d4081..55e48910 100644 --- a/test/io/loader/test_cws_loader.py +++ b/test/io/loader/test_cws_loader.py @@ -3,7 +3,7 @@ import os from fastNLP.io.loader import CWSLoader -class CWSLoaderTest(unittest.TestCase): +class TestCWSLoader(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_download(self): dataset_names = ['pku', 'cityu', 'as', 'msra'] @@ -13,7 +13,7 @@ class CWSLoaderTest(unittest.TestCase): print(data_bundle) -class RunCWSLoaderTest(unittest.TestCase): +class TestRunCWSLoader(unittest.TestCase): def test_cws_loader(self): dataset_names = ['msra'] for dataset_name in dataset_names: diff --git a/test/io/loader/test_matching_loader.py b/test/io/loader/test_matching_loader.py index 652cf161..cb1334e0 100644 --- a/test/io/loader/test_matching_loader.py +++ b/test/io/loader/test_matching_loader.py @@ -8,7 +8,7 @@ from fastNLP.io.loader.matching import MNLILoader import os @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") -class TestDownload(unittest.TestCase): +class TestMatchingDownload(unittest.TestCase): def test_download(self): for loader in [RTELoader, QNLILoader, SNLILoader, MNLILoader]: loader().download() @@ -21,8 +21,7 @@ class TestDownload(unittest.TestCase): print(data_bundle) -class TestLoad(unittest.TestCase): - +class TestMatchingLoad(unittest.TestCase): def test_load(self): for loader in [RTELoader]: data_bundle = loader().load('test/data_for_tests/io/rte') diff --git a/test/io/pipe/test_classification.py b/test/io/pipe/test_classification.py index c6e2005e..45c276a3 100644 --- a/test/io/pipe/test_classification.py +++ b/test/io/pipe/test_classification.py @@ -2,9 +2,10 @@ import unittest import os from fastNLP.io.pipe.classification import SSTPipe, SST2Pipe, IMDBPipe, YelpFullPipe, YelpPolarityPipe +from fastNLP.io.pipe.classification import ChnSentiCorpPipe @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") -class TestPipe(unittest.TestCase): +class TestClassificationPipe(unittest.TestCase): def test_process_from_file(self): for pipe in [YelpPolarityPipe, SST2Pipe, IMDBPipe, YelpFullPipe, SSTPipe]: with self.subTest(pipe=pipe): @@ -14,8 +15,16 @@ class TestPipe(unittest.TestCase): class TestRunPipe(unittest.TestCase): - def test_load(self): for pipe in [IMDBPipe]: data_bundle = pipe(tokenizer='raw').process_from_file('test/data_for_tests/io/imdb') print(data_bundle) + + +@unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") +class TestCNClassificationPipe(unittest.TestCase): + def test_process_from_file(self): + for pipe in [ChnSentiCorpPipe]: + with self.subTest(pipe=pipe): + data_bundle = pipe(bigrams=True, trigrams=True).process_from_file() + print(data_bundle) \ No newline at end of file diff --git a/test/io/pipe/test_conll.py b/test/io/pipe/test_conll.py index 6f6c4fad..4ecd7969 100644 --- a/test/io/pipe/test_conll.py +++ b/test/io/pipe/test_conll.py @@ -4,12 +4,14 @@ from fastNLP.io import MsraNERPipe, PeopleDailyPipe, WeiboNERPipe, Conll2003Pipe @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") -class TestPipe(unittest.TestCase): +class TestConllPipe(unittest.TestCase): def test_process_from_file(self): for pipe in [MsraNERPipe, PeopleDailyPipe, WeiboNERPipe]: with self.subTest(pipe=pipe): print(pipe) - data_bundle = pipe().process_from_file() + data_bundle = pipe(bigrams=True, trigrams=True).process_from_file() + print(data_bundle) + data_bundle = pipe(encoding_type='bioes').process_from_file() print(data_bundle) diff --git a/test/io/pipe/test_cws.py b/test/io/pipe/test_cws.py index dd901a25..063b6d9a 100644 --- a/test/io/pipe/test_cws.py +++ b/test/io/pipe/test_cws.py @@ -4,7 +4,7 @@ import os from fastNLP.io.pipe.cws import CWSPipe -class CWSPipeTest(unittest.TestCase): +class TestCWSPipe(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") def test_process_from_file(self): dataset_names = ['pku', 'cityu', 'as', 'msra'] @@ -14,7 +14,7 @@ class CWSPipeTest(unittest.TestCase): print(data_bundle) -class RunCWSPipeTest(unittest.TestCase): +class TestRunCWSPipe(unittest.TestCase): def test_process_from_file(self): dataset_names = ['msra'] for dataset_name in dataset_names: diff --git a/test/io/pipe/test_matching.py b/test/io/pipe/test_matching.py index 33904e7a..932d8289 100644 --- a/test/io/pipe/test_matching.py +++ b/test/io/pipe/test_matching.py @@ -7,7 +7,7 @@ from fastNLP.io.pipe.matching import SNLIBertPipe, RTEBertPipe, QNLIBertPipe, MN @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") -class TestPipe(unittest.TestCase): +class TestMatchingPipe(unittest.TestCase): def test_process_from_file(self): for pipe in [SNLIPipe, RTEPipe, QNLIPipe, MNLIPipe]: with self.subTest(pipe=pipe): @@ -17,7 +17,7 @@ class TestPipe(unittest.TestCase): @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") -class TestBertPipe(unittest.TestCase): +class TestMatchingBertPipe(unittest.TestCase): def test_process_from_file(self): for pipe in [SNLIBertPipe, RTEBertPipe, QNLIBertPipe, MNLIBertPipe]: with self.subTest(pipe=pipe): @@ -26,7 +26,7 @@ class TestBertPipe(unittest.TestCase): print(data_bundle) -class TestRunPipe(unittest.TestCase): +class TestRunMatchingPipe(unittest.TestCase): def test_load(self): for pipe in [RTEPipe, RTEBertPipe]: From 113ef8b11a34ca72fd0a1b6a1496dd42e272b94d Mon Sep 17 00:00:00 2001 From: ChenXin Date: Wed, 4 Sep 2019 14:31:45 +0800 Subject: [PATCH 35/92] add code to detect the defined location automatically --- fastNLP/__init__.py | 4 ++++ fastNLP/doc_utils.py | 21 +++++++++++++++++++++ fastNLP/embeddings/__init__.py | 4 ++++ fastNLP/io/__init__.py | 4 ++++ fastNLP/models/__init__.py | 4 ++++ fastNLP/modules/__init__.py | 4 ++++ 6 files changed, 41 insertions(+) create mode 100644 fastNLP/doc_utils.py diff --git a/fastNLP/__init__.py b/fastNLP/__init__.py index 19efac31..aceaf47f 100644 --- a/fastNLP/__init__.py +++ b/fastNLP/__init__.py @@ -70,3 +70,7 @@ from . import models from . import modules from .core import * from .io import loader, pipe + +import sys +from .doc_utils import doc_process +doc_process(sys.modules[__name__]) \ No newline at end of file diff --git a/fastNLP/doc_utils.py b/fastNLP/doc_utils.py new file mode 100644 index 00000000..924b7a6a --- /dev/null +++ b/fastNLP/doc_utils.py @@ -0,0 +1,21 @@ +import inspect +import sys + + +def doc_process(m): + for name, obj in inspect.getmembers(m): + if inspect.isclass(obj) or inspect.isfunction(obj): + if obj.__module__ != m.__name__: + if obj.__doc__ is None: + print(name, obj.__doc__) + else: + module_name = obj.__module__ + while 1: + defined_m = sys.modules[module_name] + if "undocumented" not in defined_m.__doc__ and name in defined_m.__all__: + obj.__doc__ = r"定义在 :class:`" + module_name + "." + name + "`\n" + obj.__doc__ + break + module_name = ".".join(module_name.split('.')[:-1]) + if module_name == m.__name__: + print(name, ": not found defined doc.") + break diff --git a/fastNLP/embeddings/__init__.py b/fastNLP/embeddings/__init__.py index 8a970e25..ea99154e 100644 --- a/fastNLP/embeddings/__init__.py +++ b/fastNLP/embeddings/__init__.py @@ -25,3 +25,7 @@ from .bert_embedding import BertEmbedding, BertWordPieceEncoder from .char_embedding import CNNCharEmbedding, LSTMCharEmbedding from .stack_embedding import StackEmbedding from .utils import get_embeddings + +import sys +from ..doc_utils import doc_process +doc_process(sys.modules[__name__]) \ No newline at end of file diff --git a/fastNLP/io/__init__.py b/fastNLP/io/__init__.py index 6f727f05..c8b3dfaa 100644 --- a/fastNLP/io/__init__.py +++ b/fastNLP/io/__init__.py @@ -88,3 +88,7 @@ from .model_io import ModelLoader, ModelSaver from .loader import * from .pipe import * + +import sys +from ..doc_utils import doc_process +doc_process(sys.modules[__name__]) \ No newline at end of file diff --git a/fastNLP/models/__init__.py b/fastNLP/models/__init__.py index a659e1d5..62adbf69 100644 --- a/fastNLP/models/__init__.py +++ b/fastNLP/models/__init__.py @@ -38,3 +38,7 @@ from .cnn_text_classification import CNNText from .sequence_labeling import SeqLabeling, AdvSeqLabel from .snli import ESIM from .star_transformer import StarTransEnc, STSeqCls, STNLICls, STSeqLabel + +import sys +from ..doc_utils import doc_process +doc_process(sys.modules[__name__]) \ No newline at end of file diff --git a/fastNLP/modules/__init__.py b/fastNLP/modules/__init__.py index 7959e454..769dc42a 100644 --- a/fastNLP/modules/__init__.py +++ b/fastNLP/modules/__init__.py @@ -54,3 +54,7 @@ from . import encoder from .decoder import * from .dropout import TimestepDropout from .encoder import * + +import sys +from ..doc_utils import doc_process +doc_process(sys.modules[__name__]) From 3651d61f41c267ef4801dc53e5ac359f8b71606f Mon Sep 17 00:00:00 2001 From: ChenXin Date: Wed, 4 Sep 2019 14:47:45 +0800 Subject: [PATCH 36/92] delete the alias in files. --- fastNLP/embeddings/bert_embedding.py | 2 -- fastNLP/embeddings/char_embedding.py | 4 ---- fastNLP/embeddings/elmo_embedding.py | 2 -- fastNLP/embeddings/embedding.py | 2 -- fastNLP/embeddings/stack_embedding.py | 2 -- fastNLP/embeddings/static_embedding.py | 2 -- fastNLP/modules/decoder/crf.py | 2 -- fastNLP/modules/decoder/mlp.py | 2 -- fastNLP/modules/decoder/utils.py | 2 -- fastNLP/modules/encoder/attention.py | 1 - fastNLP/modules/encoder/bert.py | 2 -- fastNLP/modules/encoder/char_encoder.py | 6 ------ fastNLP/modules/encoder/conv_maxpool.py | 2 -- fastNLP/modules/encoder/lstm.py | 2 -- fastNLP/modules/encoder/pooling.py | 8 -------- fastNLP/modules/encoder/star_transformer.py | 3 --- fastNLP/modules/encoder/transformer.py | 3 --- fastNLP/modules/encoder/variational_rnn.py | 6 ------ reproduction/text_classification/data/sstloader.py | 8 ++++---- reproduction/text_classification/model/awdlstm_module.py | 2 -- 20 files changed, 4 insertions(+), 59 deletions(-) diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index 08615fe0..17f6769d 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -26,8 +26,6 @@ from ..core import logger class BertEmbedding(ContextualEmbedding): """ - 别名：:class:`fastNLP.embeddings.BertEmbedding` :class:`fastNLP.embeddings.bert_embedding.BertEmbedding` - 使用BERT对words进行编码的Embedding。建议将输入的words长度限制在430以内，而不要使用512(根据预训练模型参数，可能有变化)。这是由于 预训练的bert模型长度限制为512个token，而因为输入的word是未进行word piece分割的(word piece的分割有BertEmbedding在输入word 时切分)，在分割之后长度可能会超过最大长度限制。 diff --git a/fastNLP/embeddings/char_embedding.py b/fastNLP/embeddings/char_embedding.py index 379d4eee..59109206 100644 --- a/fastNLP/embeddings/char_embedding.py +++ b/fastNLP/embeddings/char_embedding.py @@ -24,8 +24,6 @@ from ..core import logger class CNNCharEmbedding(TokenEmbedding): """ - 别名：:class:`fastNLP.embeddings.CNNCharEmbedding` :class:`fastNLP.embeddings.char_embedding.CNNCharEmbedding` - 使用CNN生成character embedding。CNN的结构为, embed(x) -> Dropout(x) -> CNN(x) -> activation(x) -> pool -> fc -> Dropout. 不同的kernel大小的fitler结果是concat起来然后通过一层fully connected layer, 然后输出word的表示。 @@ -179,8 +177,6 @@ class CNNCharEmbedding(TokenEmbedding): class LSTMCharEmbedding(TokenEmbedding): """ - 别名：:class:`fastNLP.embeddings.LSTMCharEmbedding` :class:`fastNLP.embeddings.char_embedding.LSTMCharEmbedding` - 使用LSTM的方式对character进行encode. embed(x) -> Dropout(x) -> LSTM(x) -> activation(x) -> pool -> Dropout Example:: diff --git a/fastNLP/embeddings/elmo_embedding.py b/fastNLP/embeddings/elmo_embedding.py index d82344e4..0ec0caa0 100644 --- a/fastNLP/embeddings/elmo_embedding.py +++ b/fastNLP/embeddings/elmo_embedding.py @@ -22,8 +22,6 @@ from ..core import logger class ElmoEmbedding(ContextualEmbedding): """ - 别名：:class:`fastNLP.embeddings.ElmoEmbedding` :class:`fastNLP.embeddings.elmo_embedding.ElmoEmbedding` - 使用ELMo的embedding。初始化之后，只需要传入words就可以得到对应的embedding。当前支持的使用名称初始化的模型有以下的这些(待补充) Example:: diff --git a/fastNLP/embeddings/embedding.py b/fastNLP/embeddings/embedding.py index 5e7b9803..255b0823 100644 --- a/fastNLP/embeddings/embedding.py +++ b/fastNLP/embeddings/embedding.py @@ -17,8 +17,6 @@ from .utils import get_embeddings class Embedding(nn.Module): """ - 别名：:class:`fastNLP.embeddings.Embedding` :class:`fastNLP.embeddings.embedding.Embedding` - 词向量嵌入，支持输入多种方式初始化. 可以通过self.num_embeddings获取词表大小; self.embedding_dim获取embedding的维度. Example:: diff --git a/fastNLP/embeddings/stack_embedding.py b/fastNLP/embeddings/stack_embedding.py index 14781945..e83a275c 100644 --- a/fastNLP/embeddings/stack_embedding.py +++ b/fastNLP/embeddings/stack_embedding.py @@ -17,8 +17,6 @@ from .embedding import TokenEmbedding class StackEmbedding(TokenEmbedding): """ - 别名：:class:`fastNLP.embeddings.StackEmbedding` :class:`fastNLP.embeddings.stack_embedding.StackEmbedding` - 支持将多个embedding集合成一个embedding。 Example:: diff --git a/fastNLP/embeddings/static_embedding.py b/fastNLP/embeddings/static_embedding.py index c768f32f..8249aa11 100644 --- a/fastNLP/embeddings/static_embedding.py +++ b/fastNLP/embeddings/static_embedding.py @@ -24,8 +24,6 @@ from ..core import logger class StaticEmbedding(TokenEmbedding): """ - 别名：:class:`fastNLP.embeddings.StaticEmbedding` :class:`fastNLP.embeddings.static_embedding.StaticEmbedding` - StaticEmbedding组件. 给定预训练embedding的名称或路径，根据vocab从embedding中抽取相应的数据(只会将出现在vocab中的词抽取出来， 如果没有找到，则会随机初始化一个值(但如果该word是被标记为no_create_entry的话，则不会单独创建一个值，而是会被指向unk的index))。 当前支持自动下载的预训练vector有以下的几种(待补充); diff --git a/fastNLP/modules/decoder/crf.py b/fastNLP/modules/decoder/crf.py index c13ea50c..e2a751f8 100644 --- a/fastNLP/modules/decoder/crf.py +++ b/fastNLP/modules/decoder/crf.py @@ -15,8 +15,6 @@ from typing import Union def allowed_transitions(tag_vocab:Union[Vocabulary, dict], encoding_type=None, include_start_end=False): """ - 别名：:class:`fastNLP.modules.allowed_transitions` :class:`fastNLP.modules.decoder.allowed_transitions` - 给定一个id到label的映射表，返回所有可以跳转的(from_tag_id, to_tag_id)列表。 :param ~fastNLP.Vocabulary,dict tag_vocab: 支持类型为tag或tag-label。只有tag的,比如"B", "M"; 也可以是"B-NN", "M-NN", diff --git a/fastNLP/modules/decoder/mlp.py b/fastNLP/modules/decoder/mlp.py index f6e687a7..3e594de1 100644 --- a/fastNLP/modules/decoder/mlp.py +++ b/fastNLP/modules/decoder/mlp.py @@ -12,8 +12,6 @@ from ..utils import initial_parameter class MLP(nn.Module): """ - 别名：:class:`fastNLP.modules.MLP` :class:`fastNLP.modules.decoder.MLP` - 多层感知器 :param List[int] size_layer: 一个int的列表，用来定义MLP的层数，列表中的数字为每一层是hidden数目。MLP的层数为 len(size_layer) - 1 diff --git a/fastNLP/modules/decoder/utils.py b/fastNLP/modules/decoder/utils.py index 118b1414..e0d2af68 100644 --- a/fastNLP/modules/decoder/utils.py +++ b/fastNLP/modules/decoder/utils.py @@ -8,8 +8,6 @@ import torch def viterbi_decode(logits, transitions, mask=None, unpad=False): r""" - 别名：:class:`fastNLP.modules.viterbi_decode` :class:`fastNLP.modules.decoder.viterbi_decode` - 给定一个特征矩阵以及转移分数矩阵，计算出最佳的路径以及对应的分数 :param torch.FloatTensor logits: batch_size x max_len x num_tags，特征矩阵。 diff --git a/fastNLP/modules/encoder/attention.py b/fastNLP/modules/encoder/attention.py index 6a973864..0d832653 100644 --- a/fastNLP/modules/encoder/attention.py +++ b/fastNLP/modules/encoder/attention.py @@ -45,7 +45,6 @@ class DotAttention(nn.Module): class MultiHeadAttention(nn.Module): """ - 别名：:class:`fastNLP.modules.MultiHeadAttention` :class:`fastNLP.modules.encoder.MultiHeadAttention` :param input_size: int, 输入维度的大小。同时也是输出维度的大小。 :param key_size: int, 每个head的维度大小。 diff --git a/fastNLP/modules/encoder/bert.py b/fastNLP/modules/encoder/bert.py index 6f6c4291..12379718 100644 --- a/fastNLP/modules/encoder/bert.py +++ b/fastNLP/modules/encoder/bert.py @@ -348,8 +348,6 @@ class BertPooler(nn.Module): class BertModel(nn.Module): """ - 别名：:class:`fastNLP.modules.BertModel` :class:`fastNLP.modules.encoder.BertModel` - BERT(Bidirectional Embedding Representations from Transformers). 用预训练权重矩阵来建立BERT模型:: diff --git a/fastNLP/modules/encoder/char_encoder.py b/fastNLP/modules/encoder/char_encoder.py index e40bd0dd..dc73f447 100644 --- a/fastNLP/modules/encoder/char_encoder.py +++ b/fastNLP/modules/encoder/char_encoder.py @@ -13,8 +13,6 @@ from ..utils import initial_parameter # from torch.nn.init import xavier_uniform class ConvolutionCharEncoder(nn.Module): """ - 别名：:class:`fastNLP.modules.ConvolutionCharEncoder` :class:`fastNLP.modules.encoder.ConvolutionCharEncoder` - char级别的卷积编码器. :param int char_emb_size: char级别embedding的维度. Default: 50 @@ -60,11 +58,7 @@ class ConvolutionCharEncoder(nn.Module): class LSTMCharEncoder(nn.Module): """ - 别名：:class:`fastNLP.modules.LSTMCharEncoder` :class:`fastNLP.modules.encoder.LSTMCharEncoder` - char级别基于LSTM的encoder. - - """ def __init__(self, char_emb_size=50, hidden_size=None, initial_method=None): diff --git a/fastNLP/modules/encoder/conv_maxpool.py b/fastNLP/modules/encoder/conv_maxpool.py index 68415189..bf629eba 100644 --- a/fastNLP/modules/encoder/conv_maxpool.py +++ b/fastNLP/modules/encoder/conv_maxpool.py @@ -10,8 +10,6 @@ import torch.nn.functional as F class ConvMaxpool(nn.Module): """ - 别名：:class:`fastNLP.modules.ConvMaxpool` :class:`fastNLP.modules.encoder.ConvMaxpool` - 集合了Convolution和Max-Pooling于一体的层。给定一个batch_size x max_len x input_size的输入，返回batch_size x sum(output_channels) 大小的matrix。在内部，是先使用CNN给输入做卷积，然后经过activation激活层，在通过在长度(max_len) 这一维进行max_pooling。最后得到每个sample的一个向量表示。 diff --git a/fastNLP/modules/encoder/lstm.py b/fastNLP/modules/encoder/lstm.py index 1f3eae6d..1dd1f0df 100644 --- a/fastNLP/modules/encoder/lstm.py +++ b/fastNLP/modules/encoder/lstm.py @@ -14,8 +14,6 @@ import torch.nn.utils.rnn as rnn class LSTM(nn.Module): """ - 别名：:class:`fastNLP.modules.LSTM` :class:`fastNLP.modules.encoder.LSTM` - LSTM 模块, 轻量封装的Pytorch LSTM. 在提供seq_len的情况下，将自动使用pack_padded_sequence; 同时默认将forget gate的bias初始化 为1; 且可以应对DataParallel中LSTM的使用问题。 diff --git a/fastNLP/modules/encoder/pooling.py b/fastNLP/modules/encoder/pooling.py index b1272284..c248601d 100644 --- a/fastNLP/modules/encoder/pooling.py +++ b/fastNLP/modules/encoder/pooling.py @@ -12,8 +12,6 @@ import torch.nn as nn class MaxPool(nn.Module): """ - 别名：:class:`fastNLP.modules.MaxPool` :class:`fastNLP.modules.encoder.MaxPool` - Max-pooling模块。 :param stride: 窗口移动大小，默认为kernel_size @@ -61,8 +59,6 @@ class MaxPool(nn.Module): class MaxPoolWithMask(nn.Module): """ - 别名：:class:`fastNLP.modules.MaxPoolWithMask` :class:`fastNLP.modules.encoder.MaxPoolWithMask` - 带mask矩阵的max pooling。在做max-pooling的时候不会考虑mask值为0的位置。 """ @@ -101,8 +97,6 @@ class KMaxPool(nn.Module): class AvgPool(nn.Module): """ - 别名：:class:`fastNLP.modules.AvgPool` :class:`fastNLP.modules.encoder.AvgPool` - 给定形如[batch_size, max_len, hidden_size]的输入，在最后一维进行avg pooling. 输出为[batch_size, hidden_size] """ @@ -128,8 +122,6 @@ class AvgPool(nn.Module): class AvgPoolWithMask(nn.Module): """ - 别名：:class:`fastNLP.modules.AvgPoolWithMask` :class:`fastNLP.modules.encoder.AvgPoolWithMask` - 给定形如[batch_size, max_len, hidden_size]的输入，在最后一维进行avg pooling. 输出为[batch_size, hidden_size], pooling 的时候只会考虑mask为1的位置 """ diff --git a/fastNLP/modules/encoder/star_transformer.py b/fastNLP/modules/encoder/star_transformer.py index 02d7a6a0..bb47d9b5 100644 --- a/fastNLP/modules/encoder/star_transformer.py +++ b/fastNLP/modules/encoder/star_transformer.py @@ -14,9 +14,6 @@ from torch.nn import functional as F class StarTransformer(nn.Module): """ - 别名：:class:`fastNLP.modules.StarTransformer` :class:`fastNLP.modules.encoder.StarTransformer` - - Star-Transformer 的encoder部分。 输入3d的文本输入, 返回相同长度的文本编码 paper: https://arxiv.org/abs/1902.09113 diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index d8a612a0..d29a10c3 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -10,9 +10,6 @@ from .attention import MultiHeadAttention class TransformerEncoder(nn.Module): """ - 别名：:class:`fastNLP.modules.TransformerEncoder` :class:`fastNLP.modules.encoder.TransformerEncoder` - - transformer的encoder模块，不包含embedding层 :param int num_layers: transformer的层数 diff --git a/fastNLP/modules/encoder/variational_rnn.py b/fastNLP/modules/encoder/variational_rnn.py index 933555c8..17e2ad23 100644 --- a/fastNLP/modules/encoder/variational_rnn.py +++ b/fastNLP/modules/encoder/variational_rnn.py @@ -223,8 +223,6 @@ class VarRNNBase(nn.Module): class VarLSTM(VarRNNBase): """ - 别名：:class:`fastNLP.modules.VarLSTM` :class:`fastNLP.modules.encoder.VarLSTM` - Variational Dropout LSTM. :param input_size: 输入 `x` 的特征维度 @@ -248,8 +246,6 @@ class VarLSTM(VarRNNBase): class VarRNN(VarRNNBase): """ - 别名：:class:`fastNLP.modules.VarRNN` :class:`fastNLP.modules.encoder.VarRNN` - Variational Dropout RNN. :param input_size: 输入 `x` 的特征维度 @@ -273,8 +269,6 @@ class VarRNN(VarRNNBase): class VarGRU(VarRNNBase): """ - 别名：:class:`fastNLP.modules.VarGRU` :class:`fastNLP.modules.encoder.VarGRU` - Variational Dropout GRU. :param input_size: 输入 `x` 的特征维度 diff --git a/reproduction/text_classification/data/sstloader.py b/reproduction/text_classification/data/sstloader.py index b635a14a..4e860279 100644 --- a/reproduction/text_classification/data/sstloader.py +++ b/reproduction/text_classification/data/sstloader.py @@ -11,11 +11,7 @@ from reproduction.utils import check_dataloader_paths, get_tokenizer class SSTLoader(DataSetLoader): - URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' - DATA_DIR = 'sst/' - """ - 别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader` 读取SST数据集, DataSet包含fields:: words: list(str) 需要分类的文本 target: str 文本的标签 @@ -23,6 +19,10 @@ class SSTLoader(DataSetLoader): :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False`` :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False`` """ + + URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip' + DATA_DIR = 'sst/' + def __init__(self, subtree=False, fine_grained=False): self.subtree = subtree tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral', diff --git a/reproduction/text_classification/model/awdlstm_module.py b/reproduction/text_classification/model/awdlstm_module.py index 87bfe730..a586ed2d 100644 --- a/reproduction/text_classification/model/awdlstm_module.py +++ b/reproduction/text_classification/model/awdlstm_module.py @@ -17,8 +17,6 @@ from .weight_drop import WeightDrop class LSTM(nn.Module): """ - 别名：:class:`fastNLP.modules.LSTM` :class:`fastNLP.modules.encoder.lstm.LSTM` - LSTM 模块, 轻量封装的Pytorch LSTM. 在提供seq_len的情况下，将自动使用pack_padded_sequence; 同时默认将forget gate的bias初始化 为1; 且可以应对DataParallel中LSTM的使用问题。 From 4caacadeae607ebd0699d05457213321874fb786 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Wed, 4 Sep 2019 14:51:50 +0800 Subject: [PATCH 37/92] delete the alias in files. --- fastNLP/core/batch.py | 2 -- fastNLP/core/callback.py | 23 +++-------------------- fastNLP/core/dataset.py | 2 -- fastNLP/core/field.py | 6 ------ fastNLP/core/instance.py | 2 -- fastNLP/core/losses.py | 12 ------------ fastNLP/core/metrics.py | 7 ------- fastNLP/core/optimizer.py | 5 ----- fastNLP/core/sampler.py | 9 --------- fastNLP/core/tester.py | 2 -- fastNLP/core/trainer.py | 2 -- fastNLP/core/utils.py | 2 -- fastNLP/core/vocabulary.py | 2 -- fastNLP/io/embed_loader.py | 2 -- fastNLP/io/loader/classification.py | 6 ------ fastNLP/io/loader/conll.py | 2 -- fastNLP/io/loader/csv.py | 2 -- fastNLP/io/loader/json.py | 2 -- fastNLP/io/model_io.py | 4 ---- fastNLP/io/pipe/classification.py | 2 -- fastNLP/io/pipe/pipe.py | 4 +++- fastNLP/models/bert.py | 15 --------------- fastNLP/models/biaffine_parser.py | 8 -------- fastNLP/models/cnn_text_classification.py | 2 -- fastNLP/models/sequence_labeling.py | 4 ---- fastNLP/models/snli.py | 2 -- fastNLP/models/star_transformer.py | 8 -------- fastNLP/modules/decoder/crf.py | 5 +---- 28 files changed, 7 insertions(+), 137 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index ff710b30..ad07341a 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -145,8 +145,6 @@ class BatchIter: class DataSetIter(BatchIter): """ - 别名：:class:`fastNLP.DataSetIter` :class:`fastNLP.core.batch.DataSetIter` - DataSetIter 用于从 `DataSet` 中按一定的顺序, 依次按 ``batch_size`` 的大小将数据取出， 组成 `x` 和 `y`:: diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index 5167b09f..3cdc0f8d 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -96,8 +96,6 @@ except: class Callback(object): """ - 别名：:class:`fastNLP.Callback` :class:`fastNLP.core.callback.Callback` - Callback是fastNLP中被设计用于增强 :class:`~fastNLP.Trainer` 的类。 如果Callback被传递给了 Trainer , 则 Trainer 会在对应的阶段调用Callback的函数， 具体调用时机可以通过 :doc:`trainer 模块` 查看。 @@ -436,8 +434,6 @@ class DistCallbackManager(CallbackManager): class GradientClipCallback(Callback): """ - 别名：:class:`fastNLP.GradientClipCallback` :class:`fastNLP.core.callback.GradientClipCallback` - 每次backward前，将parameter的gradient clip到某个范围。 :param None,torch.Tensor,List[torch.Tensor] parameters: 一般通过model.parameters()获得。 @@ -481,8 +477,6 @@ class GradientClipCallback(Callback): class EarlyStopCallback(Callback): """ - 别名：:class:`fastNLP.EarlyStopCallback` :class:`fastNLP.core.callback.EarlyStopCallback` - 多少个epoch没有变好就停止训练，相关类 :class:`EarlyStopError` :param int patience: epoch的数量 @@ -512,12 +506,10 @@ class EarlyStopCallback(Callback): class FitlogCallback(Callback): """ - 别名: :class:`fastNLP.FitlogCallback` :class:`fastNLP.core.callback.FitlogCallback` - 该callback可将loss和progress写入到fitlog中; 如果Trainer有dev的数据，将自动把dev的结果写入到log中; 同时还支持传入 - 一个(或多个)test数据集进行测试(只有在trainer具有dev时才能使用)，每次在dev上evaluate之后会在这些数据集上验证一下。 - 并将验证结果写入到fitlog中。这些数据集的结果是根据dev上最好的结果报道的，即如果dev在第3个epoch取得了最佳，则 - fitlog中记录的关于这些数据集的结果就是来自第三个epoch的结果。 + 一个(或多个)test数据集进行测试(只有在trainer具有dev时才能使用)，每次在dev上evaluate之后会在这些数据集上验证一下。 + 并将验证结果写入到fitlog中。这些数据集的结果是根据dev上最好的结果报道的，即如果dev在第3个epoch取得了最佳，则 + fitlog中记录的关于这些数据集的结果就是来自第三个epoch的结果。 :param ~fastNLP.DataSet,Dict[~fastNLP.DataSet] data: 传入DataSet对象，会使用多个Trainer中的metric对数据进行验证。如果需要 传入多个DataSet请通过dict的方式传入，dict的key将作为对应dataset的name传递给fitlog。data的结果的名称以'data'开头。 @@ -611,8 +603,6 @@ class FitlogCallback(Callback): class EvaluateCallback(Callback): """ - 别名: :class:`fastNLP.EvaluateCallback` :class:`fastNLP.core.callback.EvaluateCallback` - 该callback用于扩展Trainer训练过程中只能对dev数据进行验证的问题。 :param ~fastNLP.DataSet,Dict[~fastNLP.DataSet] data: 传入DataSet对象，会使用多个Trainer中的metric对数据进行验证。如果需要传入多个 @@ -673,8 +663,6 @@ class EvaluateCallback(Callback): class LRScheduler(Callback): """ - 别名：:class:`fastNLP.LRScheduler` :class:`fastNLP.core.callback.LRScheduler` - 对PyTorch LR Scheduler的包装以使得其可以被Trainer所使用 :param torch.optim.lr_scheduler._LRScheduler lr_scheduler: PyTorch的lr_scheduler @@ -695,7 +683,6 @@ class LRScheduler(Callback): class ControlC(Callback): """ - 别名：:class:`fastNLP.ControlC` :class:`fastNLP.core.callback.ControlC` :param bool quit_all: 若为True,则检测到control+C 直接退出程序；否则只退出Trainer """ @@ -732,8 +719,6 @@ class SmoothValue(object): class LRFinder(Callback): """ - 别名：:class:`fastNLP.LRFinder` :class:`fastNLP.core.callback.LRFinder` - 用第一个 epoch 找最佳的学习率，从第二个epoch开始应用它 :param float start_lr: 学习率下界 @@ -804,8 +789,6 @@ class LRFinder(Callback): class TensorboardCallback(Callback): """ - 别名：:class:`fastNLP.TensorboardCallback` :class:`fastNLP.core.callback.TensorboardCallback` - 接受以下一个或多个字符串作为参数： - "model" - "loss" diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 551cf1f8..441f9907 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -304,8 +304,6 @@ from ._logger import logger class DataSet(object): """ - 别名：:class:`fastNLP.DataSet` :class:`fastNLP.core.dataset.DataSet` - fastNLP的数据容器，详细的使用方法见文档 :doc:`fastNLP.core.dataset` :param data: 如果为dict类型，则每个key的value应该为等长的list; 如果为list， diff --git a/fastNLP/core/field.py b/fastNLP/core/field.py index 859dfb1f..468c248d 100644 --- a/fastNLP/core/field.py +++ b/fastNLP/core/field.py @@ -464,8 +464,6 @@ def _get_ele_type_and_dim(cell: Any, dim=0): class Padder: """ - 别名：:class:`fastNLP.Padder` :class:`fastNLP.core.field.Padder` - 所有padder都需要继承这个类，并覆盖__call__方法。 用于对batch进行padding操作。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前deepcopy一份。 @@ -534,8 +532,6 @@ class Padder: class AutoPadder(Padder): """ - 别名：:class:`fastNLP.AutoPadder` :class:`fastNLP.core.field.AutoPadder` - 根据contents的数据自动判定是否需要做padding。 1 如果元素类型(元素类型是指field中最里层元素的数据类型, 可以通过FieldArray.dtype查看，比如['This', 'is', ...]的元素类 @@ -628,8 +624,6 @@ class AutoPadder(Padder): class EngChar2DPadder(Padder): """ - 别名：:class:`fastNLP.EngChar2DPadder` :class:`fastNLP.core.field.EngChar2DPadder` - 用于为英语执行character级别的2D padding操作。对应的field内容应该类似[['T', 'h', 'i', 's'], ['a'], ['d', 'e', 'm', 'o']]， 但这个Padder只能处理index为int的情况。 diff --git a/fastNLP/core/instance.py b/fastNLP/core/instance.py index 9a5d9edf..2285e4a4 100644 --- a/fastNLP/core/instance.py +++ b/fastNLP/core/instance.py @@ -10,8 +10,6 @@ __all__ = [ class Instance(object): """ - 别名：:class:`fastNLP.Instance` :class:`fastNLP.core.instance.Instance` - Instance是fastNLP中对应一个sample的类。每个sample在fastNLP中是一个Instance对象。 Instance一般与 :class:`~fastNLP.DataSet` 一起使用, Instance的初始化如下面的Example所示:: diff --git a/fastNLP/core/losses.py b/fastNLP/core/losses.py index 7402a568..b2f5ce0a 100644 --- a/fastNLP/core/losses.py +++ b/fastNLP/core/losses.py @@ -167,8 +167,6 @@ class LossBase(object): class LossFunc(LossBase): """ - 别名：:class:`fastNLP.LossFunc` :class:`fastNLP.core.losses.LossFunc` - 提供给用户使用自定义损失函数的类 :param func: 用户自行定义的损失函数，应当为一个函数或者callable(func)为True的ojbect @@ -200,8 +198,6 @@ class LossFunc(LossBase): class CrossEntropyLoss(LossBase): """ - 别名：:class:`fastNLP.CrossEntropyLoss` :class:`fastNLP.core.losses.CrossEntropyLoss` - 交叉熵损失函数 :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` @@ -248,8 +244,6 @@ class CrossEntropyLoss(LossBase): class L1Loss(LossBase): """ - 别名：:class:`fastNLP.L1Loss` :class:`fastNLP.core.losses.L1Loss` - L1损失函数 :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` @@ -270,8 +264,6 @@ class L1Loss(LossBase): class BCELoss(LossBase): """ - 别名：:class:`fastNLP.BCELoss` :class:`fastNLP.core.losses.BCELoss` - 二分类交叉熵损失函数 :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` @@ -291,8 +283,6 @@ class BCELoss(LossBase): class NLLLoss(LossBase): """ - 别名：:class:`fastNLP.NLLLoss` :class:`fastNLP.core.losses.NLLLoss` - 负对数似然损失函数 :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` @@ -315,8 +305,6 @@ class NLLLoss(LossBase): class LossInForward(LossBase): """ - 别名：:class:`fastNLP.LossInForward` :class:`fastNLP.core.losses.LossInForward` - 从forward()函数返回结果中获取loss :param str loss_key: 在forward函数中loss的键名，默认为loss diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index c0f14c90..2dc6d9d8 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -294,9 +294,6 @@ class MetricBase(object): class AccuracyMetric(MetricBase): """ - - 别名：:class:`fastNLP.AccuracyMetric` :class:`fastNLP.core.metrics.AccuracyMetric` - 准确率Metric（其它的Metric参见 :doc:`fastNLP.core.metrics` ） :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` @@ -565,8 +562,6 @@ def _check_tag_vocab_and_encoding_type(tag_vocab:Union[Vocabulary, dict], encodi class SpanFPreRecMetric(MetricBase): r""" - 别名：:class:`fastNLP.SpanFPreRecMetric` :class:`fastNLP.core.metrics.SpanFPreRecMetric` - 在序列标注问题中，以span的方式计算F, pre, rec. 比如中文Part of speech中，会以character的方式进行标注，句子 `中国在亚洲` 对应的POS可能为(以BMES为例) ['B-NN', 'E-NN', 'S-DET', 'B-NN', 'E-NN']。该metric就是为类似情况下的F1计算。 @@ -832,8 +827,6 @@ def _pred_topk(y_prob, k=1): class ExtractiveQAMetric(MetricBase): r""" - 别名：:class:`fastNLP.ExtractiveQAMetric` :class:`fastNLP.core.metrics.ExtractiveQAMetric` - 抽取式QA（如SQuAD）的metric. :param pred1: 参数映射表中 `pred1` 的映射关系，None表示映射关系为 `pred1` -> `pred1` diff --git a/fastNLP/core/optimizer.py b/fastNLP/core/optimizer.py index e95047b4..c30c7e34 100644 --- a/fastNLP/core/optimizer.py +++ b/fastNLP/core/optimizer.py @@ -17,7 +17,6 @@ from torch.optim.optimizer import Optimizer as TorchOptimizer class Optimizer(object): """ - 别名：:class:`fastNLP.Optimizer` :class:`fastNLP.core.optimizer.Optimizer` :param model_params: a generator. E.g. ``model.parameters()`` for PyTorch models. :param kwargs: additional parameters. @@ -60,7 +59,6 @@ class NullOptimizer(Optimizer): class SGD(Optimizer): """ - 别名：:class:`fastNLP.SGD` :class:`fastNLP.core.optimizer.SGD` :param float lr: learning rate. Default: 0.01 :param float momentum: momentum. Default: 0 @@ -82,7 +80,6 @@ class SGD(Optimizer): class Adam(Optimizer): """ - 别名：:class:`fastNLP.Adam` :class:`fastNLP.core.optimizer.Adam` :param float lr: learning rate :param float weight_decay: @@ -105,8 +102,6 @@ class Adam(Optimizer): class AdamW(TorchOptimizer): r""" - 别名：:class:`fastNLP.AdamW` :class:`fastNLP.core.optimizer.AdamW` - 对AdamW的实现，该实现应该会在pytorch更高版本中出现，https://github.com/pytorch/pytorch/pull/21250。这里提前加入 .. todo:: diff --git a/fastNLP/core/sampler.py b/fastNLP/core/sampler.py index 9ca04fa0..d0df9129 100644 --- a/fastNLP/core/sampler.py +++ b/fastNLP/core/sampler.py @@ -15,9 +15,6 @@ import numpy as np class Sampler(object): """ - 别名：:class:`fastNLP.Sampler` :class:`fastNLP.core.sampler.Sampler` - - `Sampler` 类的基类. 规定以何种顺序取出data中的元素 子类必须实现 ``__call__`` 方法. 输入 `DataSet` 对象, 返回其中元素的下标序列 @@ -33,8 +30,6 @@ class Sampler(object): class SequentialSampler(Sampler): """ - 别名：:class:`fastNLP.SequentialSampler` :class:`fastNLP.core.sampler.SequentialSampler` - 顺序取出元素的 `Sampler` """ @@ -45,8 +40,6 @@ class SequentialSampler(Sampler): class RandomSampler(Sampler): """ - 别名：:class:`fastNLP.RandomSampler` :class:`fastNLP.core.sampler.RandomSampler` - 随机化取元素的 `Sampler` """ @@ -57,8 +50,6 @@ class RandomSampler(Sampler): class BucketSampler(Sampler): """ - 别名：:class:`fastNLP.BucketSampler` :class:`fastNLP.core.sampler.BucketSampler` - 带Bucket的 `Random Sampler`. 可以随机地取出长度相似的元素 :param int num_buckets: bucket的数量 diff --git a/fastNLP/core/tester.py b/fastNLP/core/tester.py index e549df81..344e24a8 100644 --- a/fastNLP/core/tester.py +++ b/fastNLP/core/tester.py @@ -65,8 +65,6 @@ __all__ = [ class Tester(object): """ - 别名：:class:`fastNLP.Tester` :class:`fastNLP.core.tester.Tester` - Tester是在提供数据，模型以及metric的情况下进行性能测试的类。需要传入模型，数据以及metric进行验证。 :param ~fastNLP.DataSet data: 需要测试的数据集 diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index a47f108b..9f262fb5 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -357,8 +357,6 @@ from ._logger import logger class Trainer(object): """ - 别名：:class:`fastNLP.Trainer` :class:`fastNLP.core.trainer.Trainer` - Trainer在fastNLP中用于组织单任务的训练过程，可以避免用户在不同训练任务中重复撰写 (1) epoch循环; (2) 将数据分成不同的Batch; diff --git a/fastNLP/core/utils.py b/fastNLP/core/utils.py index fcb2a07b..814e0bd5 100644 --- a/fastNLP/core/utils.py +++ b/fastNLP/core/utils.py @@ -66,8 +66,6 @@ def _prepare_cache_filepath(filepath): def cache_results(_cache_fp, _refresh=False, _verbose=1): """ - 别名：:class:`fastNLP.cache_results` :class:`fastNLP.core.uitls.cache_results` - cache_results是fastNLP中用于cache数据的装饰器。通过下面的例子看一下如何使用:: import time diff --git a/fastNLP/core/vocabulary.py b/fastNLP/core/vocabulary.py index b0f9650a..d4ff6077 100644 --- a/fastNLP/core/vocabulary.py +++ b/fastNLP/core/vocabulary.py @@ -66,8 +66,6 @@ def _check_build_status(func): class Vocabulary(object): """ - 别名：:class:`fastNLP.Vocabulary` :class:`fastNLP.core.vocabulary.Vocabulary` - 用于构建, 存储和使用 `str` 到 `int` 的一一映射:: vocab = Vocabulary() diff --git a/fastNLP/io/embed_loader.py b/fastNLP/io/embed_loader.py index a157901f..73a7a1de 100644 --- a/fastNLP/io/embed_loader.py +++ b/fastNLP/io/embed_loader.py @@ -33,8 +33,6 @@ class EmbeddingOption(Option): class EmbedLoader: """ - 别名：:class:`fastNLP.io.EmbedLoader` :class:`fastNLP.io.embed_loader.EmbedLoader` - 用于读取预训练的embedding, 读取结果可直接载入为模型参数。 """ diff --git a/fastNLP/io/loader/classification.py b/fastNLP/io/loader/classification.py index 4ebd58e1..9efcf5d2 100644 --- a/fastNLP/io/loader/classification.py +++ b/fastNLP/io/loader/classification.py @@ -24,8 +24,6 @@ from ...core.instance import Instance class YelpLoader(Loader): """ - 别名：:class:`fastNLP.io.YelpLoader` :class:`fastNLP.io.loader.YelpLoader` - 原始数据中内容应该为, 每一行为一个sample，第一个逗号之前为target，第一个逗号之后为文本内容。 Example:: @@ -164,8 +162,6 @@ class YelpPolarityLoader(YelpLoader): class IMDBLoader(Loader): """ - 别名：:class:`fastNLP.io.IMDBLoader` :class:`fastNLP.io.loader.IMDBLoader` - IMDBLoader读取后的数据将具有以下两列内容: raw_words: str, 需要分类的文本; target: str, 文本的标签 DataSet具备以下的结构: @@ -244,8 +240,6 @@ class IMDBLoader(Loader): class SSTLoader(Loader): """ - 别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.loader.SSTLoader` - 读取之后的DataSet具有以下的结构 .. csv-table:: 下面是使用SSTLoader读取的DataSet所具备的field diff --git a/fastNLP/io/loader/conll.py b/fastNLP/io/loader/conll.py index 1bd1b448..f30b031f 100644 --- a/fastNLP/io/loader/conll.py +++ b/fastNLP/io/loader/conll.py @@ -27,8 +27,6 @@ from ...core.instance import Instance class ConllLoader(Loader): """ - 别名：:class:`fastNLP.io.ConllLoader` :class:`fastNLP.io.loader.ConllLoader` - ConllLoader支持读取的数据格式: 以空行隔开两个sample，除了分割行，每一行用空格或者制表符隔开不同的元素。如下例所示: Example:: diff --git a/fastNLP/io/loader/csv.py b/fastNLP/io/loader/csv.py index 0d6e35fa..aaf38c00 100644 --- a/fastNLP/io/loader/csv.py +++ b/fastNLP/io/loader/csv.py @@ -12,8 +12,6 @@ from ...core.instance import Instance class CSVLoader(Loader): """ - 别名：:class:`fastNLP.io.CSVLoader` :class:`fastNLP.io.loader.CSVLoader` - 读取CSV格式的数据集, 返回 ``DataSet`` 。 :param List[str] headers: CSV文件的文件头.定义每一列的属性名称,即返回的DataSet中`field`的名称 diff --git a/fastNLP/io/loader/json.py b/fastNLP/io/loader/json.py index 012dee5a..671769fe 100644 --- a/fastNLP/io/loader/json.py +++ b/fastNLP/io/loader/json.py @@ -12,8 +12,6 @@ from ...core.instance import Instance class JsonLoader(Loader): """ - 别名：:class:`fastNLP.io.JsonLoader` :class:`fastNLP.io.loader.JsonLoader` - 读取json格式数据.数据必须按行存储,每行是一个包含各类属性的json对象 :param dict fields: 需要读入的json属性名称, 和读入后在DataSet中存储的field_name diff --git a/fastNLP/io/model_io.py b/fastNLP/io/model_io.py index a1899f51..9da921df 100644 --- a/fastNLP/io/model_io.py +++ b/fastNLP/io/model_io.py @@ -11,8 +11,6 @@ import torch class ModelLoader: """ - 别名：:class:`fastNLP.io.ModelLoader` :class:`fastNLP.io.model_io.ModelLoader` - 用于读取模型 """ @@ -41,8 +39,6 @@ class ModelLoader: class ModelSaver(object): """ - 别名：:class:`fastNLP.io.ModelSaver` :class:`fastNLP.io.model_io.ModelSaver` - 用于保存模型 Example:: diff --git a/fastNLP/io/pipe/classification.py b/fastNLP/io/pipe/classification.py index d1c7aa0e..3834a570 100644 --- a/fastNLP/io/pipe/classification.py +++ b/fastNLP/io/pipe/classification.py @@ -228,8 +228,6 @@ class YelpPolarityPipe(_CLSPipe): class SSTPipe(_CLSPipe): """ - 别名：:class:`fastNLP.io.SSTPipe` :class:`fastNLP.io.pipe.SSTPipe` - 经过该Pipe之后，DataSet中具备的field如下所示 .. csv-table:: 下面是使用SSTPipe处理后的DataSet所具备的field diff --git a/fastNLP/io/pipe/pipe.py b/fastNLP/io/pipe/pipe.py index 12d9c1cb..db65ece6 100644 --- a/fastNLP/io/pipe/pipe.py +++ b/fastNLP/io/pipe/pipe.py @@ -9,7 +9,9 @@ from .. import DataBundle class Pipe: """ - 别名：:class:`fastNLP.io.Pipe` :class:`fastNLP.io.pipe.Pipe` + .. todo:: + doc + """ def process(self, data_bundle: DataBundle) -> DataBundle: """ diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py index 4a04bd6d..85c3af8c 100644 --- a/fastNLP/models/bert.py +++ b/fastNLP/models/bert.py @@ -44,9 +44,6 @@ from ..embeddings import BertEmbedding class BertForSequenceClassification(BaseModel): """ - 别名： :class:`fastNLP.models.BertForSequenceClassification` - :class:`fastNLP.models.bert.BertForSequenceClassification` - BERT model for classification. :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). @@ -90,9 +87,6 @@ class BertForSequenceClassification(BaseModel): class BertForSentenceMatching(BaseModel): """ - 别名： :class:`fastNLP.models.BertForSentenceMatching` - :class:`fastNLP.models.bert.BertForSentenceMatching` - BERT model for sentence matching. :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). @@ -135,9 +129,6 @@ class BertForSentenceMatching(BaseModel): class BertForMultipleChoice(BaseModel): """ - 别名： :class:`fastNLP.models.BertForMultipleChoice` - :class:`fastNLP.models.bert.BertForMultipleChoice` - BERT model for multiple choice. :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). @@ -185,9 +176,6 @@ class BertForMultipleChoice(BaseModel): class BertForTokenClassification(BaseModel): """ - 别名： :class:`fastNLP.models.BertForTokenClassification` - :class:`fastNLP.models.bert.BertForTokenClassification` - BERT model for token classification. :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). @@ -231,9 +219,6 @@ class BertForTokenClassification(BaseModel): class BertForQuestionAnswering(BaseModel): """ - 别名： :class:`fastNLP.models.BertForQuestionAnswering` - :class:`fastNLP.models.bert.BertForQuestionAnswering` - BERT model for classification. :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index 455d27a7..5d094472 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -130,8 +130,6 @@ def _find_cycle(vertices, edges): class GraphParser(BaseModel): """ - 别名：:class:`fastNLP.models.GraphParser` :class:`fastNLP.models.baffine_parser.GraphParser` - 基于图的parser base class, 支持贪婪解码和最大生成树解码 """ @@ -240,8 +238,6 @@ class LabelBilinear(nn.Module): class BiaffineParser(GraphParser): """ - 别名：:class:`fastNLP.models.BiaffineParser` :class:`fastNLP.models.baffine_parser.BiaffineParser` - Biaffine Dependency Parser 实现. 论文参考 `Deep Biaffine Attention for Neural Dependency Parsing (Dozat and Manning, 2016) `_ . @@ -475,8 +471,6 @@ class BiaffineParser(GraphParser): class ParserLoss(LossFunc): """ - 别名：:class:`fastNLP.models.ParserLoss` :class:`fastNLP.models.baffine_parser.ParserLoss` - 计算parser的loss :param pred1: [batch_size, seq_len, seq_len] 边预测logits @@ -500,8 +494,6 @@ class ParserLoss(LossFunc): class ParserMetric(MetricBase): """ - 别名：:class:`fastNLP.models.ParserMetric` :class:`fastNLP.models.baffine_parser.ParserMetric` - 评估parser的性能 :param pred1: 边预测logits diff --git a/fastNLP/models/cnn_text_classification.py b/fastNLP/models/cnn_text_classification.py index 4bf9c4d1..65c20a55 100644 --- a/fastNLP/models/cnn_text_classification.py +++ b/fastNLP/models/cnn_text_classification.py @@ -18,8 +18,6 @@ from ..modules import encoder class CNNText(torch.nn.Module): """ - 别名：:class:`fastNLP.models.CNNText` :class:`fastNLP.models.cnn_text_classification.CNNText` - 使用CNN进行文本分类的模型 'Yoon Kim. 2014. Convolution Neural Networks for Sentence Classification.' diff --git a/fastNLP/models/sequence_labeling.py b/fastNLP/models/sequence_labeling.py index 6e839bea..d5bc250b 100644 --- a/fastNLP/models/sequence_labeling.py +++ b/fastNLP/models/sequence_labeling.py @@ -77,8 +77,6 @@ class BiLSTMCRF(BaseModel): class SeqLabeling(BaseModel): """ - 别名：:class:`fastNLP.models.SeqLabeling` :class:`fastNLP.models.sequence_labeling.SeqLabeling` - 一个基础的Sequence labeling的模型。 用于做sequence labeling的基础类。结构包含一层Embedding，一层LSTM(单向，一层)，一层FC，以及一层CRF。 @@ -156,8 +154,6 @@ class SeqLabeling(BaseModel): class AdvSeqLabel(nn.Module): """ - 别名：:class:`fastNLP.models.AdvSeqLabel` :class:`fastNLP.models.sequence_labeling.AdvSeqLabel` - 更复杂的Sequence Labelling模型。结构为Embedding, LayerNorm, 双向LSTM(两层)，FC，LayerNorm，DropOut，FC，CRF。 :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), diff --git a/fastNLP/models/snli.py b/fastNLP/models/snli.py index 97a14e9f..07303ddc 100644 --- a/fastNLP/models/snli.py +++ b/fastNLP/models/snli.py @@ -19,8 +19,6 @@ from ..embeddings.embedding import TokenEmbedding, Embedding class ESIM(BaseModel): """ - 别名：:class:`fastNLP.models.ESIM` :class:`fastNLP.models.snli.ESIM` - ESIM model的一个PyTorch实现 论文参见： https://arxiv.org/pdf/1609.06038.pdf diff --git a/fastNLP/models/star_transformer.py b/fastNLP/models/star_transformer.py index 7fe0d343..e4d5af84 100644 --- a/fastNLP/models/star_transformer.py +++ b/fastNLP/models/star_transformer.py @@ -19,8 +19,6 @@ from ..core.const import Const class StarTransEnc(nn.Module): """ - 别名：:class:`fastNLP.models.StarTransEnc` :class:`fastNLP.models.star_transformer.StarTransEnc` - 带word embedding的Star-Transformer Encoder :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 @@ -104,8 +102,6 @@ class _NLICls(nn.Module): class STSeqLabel(nn.Module): """ - 别名：:class:`fastNLP.models.STSeqLabel` :class:`fastNLP.models.star_transformer.STSeqLabel` - 用于序列标注的Star-Transformer模型 :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 @@ -169,8 +165,6 @@ class STSeqLabel(nn.Module): class STSeqCls(nn.Module): """ - 别名：:class:`fastNLP.models.STSeqCls` :class:`fastNLP.models.star_transformer.STSeqCls` - 用于分类任务的Star-Transformer :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 @@ -234,8 +228,6 @@ class STSeqCls(nn.Module): class STNLICls(nn.Module): """ - 别名：:class:`fastNLP.models.STNLICls` :class:`fastNLP.models.star_transformer.STNLICls` - 用于自然语言推断(NLI)的Star-Transformer :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 diff --git a/fastNLP/modules/decoder/crf.py b/fastNLP/modules/decoder/crf.py index e2a751f8..aeb73d76 100644 --- a/fastNLP/modules/decoder/crf.py +++ b/fastNLP/modules/decoder/crf.py @@ -166,10 +166,7 @@ def _is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label class ConditionalRandomField(nn.Module): """ - 别名：:class:`fastNLP.modules.ConditionalRandomField` :class:`fastNLP.modules.decoder.ConditionalRandomField` - - 条件随机场。 - 提供forward()以及viterbi_decode()两个方法，分别用于训练与inference。 + 条件随机场。提供forward()以及viterbi_decode()两个方法，分别用于训练与inference。 :param int num_tags: 标签的数量 :param bool include_start_end_trans: 是否考虑各个tag作为开始以及结尾的分数。 From a2e31584883abb68e4d7354ca0c95fc250e35605 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Wed, 4 Sep 2019 15:50:01 +0800 Subject: [PATCH 38/92] update the auto alias tool --- fastNLP/doc_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastNLP/doc_utils.py b/fastNLP/doc_utils.py index 924b7a6a..5801dd53 100644 --- a/fastNLP/doc_utils.py +++ b/fastNLP/doc_utils.py @@ -13,7 +13,8 @@ def doc_process(m): while 1: defined_m = sys.modules[module_name] if "undocumented" not in defined_m.__doc__ and name in defined_m.__all__: - obj.__doc__ = r"定义在 :class:`" + module_name + "." + name + "`\n" + obj.__doc__ + obj.__doc__ = r"别名 :class:`" + m.__name__ + "." + name + "`" \ + + " :class:`" + module_name + "." + name + "`\n" + obj.__doc__ break module_name = ".".join(module_name.split('.')[:-1]) if module_name == m.__name__: From b1fe5f5321a1953b41c544c92d074becde003194 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Wed, 4 Sep 2019 16:53:31 +0800 Subject: [PATCH 39/92] split the class's doc & __init__'s doc (core part) --- fastNLP/core/batch.py | 36 ++++++------ fastNLP/core/callback.py | 115 +++++++++++++++++++++---------------- fastNLP/core/dataset.py | 21 +++---- fastNLP/core/field.py | 12 ++-- fastNLP/core/instance.py | 3 +- fastNLP/core/losses.py | 23 ++++---- fastNLP/core/metrics.py | 66 ++++++++++----------- fastNLP/core/optimizer.py | 56 +++++++++++------- fastNLP/core/predictor.py | 6 +- fastNLP/core/sampler.py | 12 ++-- fastNLP/core/tester.py | 49 ++++++++-------- fastNLP/core/trainer.py | 98 +++++++++++++++---------------- fastNLP/core/vocabulary.py | 28 ++++----- 13 files changed, 286 insertions(+), 239 deletions(-) diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index ad07341a..b14b21de 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -9,15 +9,16 @@ __all__ = [ ] import atexit +from numbers import Number import numpy as np import torch import torch.utils.data -from numbers import Number -from .sampler import SequentialSampler -from .dataset import DataSet from ._logger import logger +from .dataset import DataSet +from .sampler import SequentialSampler + _python_is_exit = False @@ -153,23 +154,26 @@ class DataSetIter(BatchIter): for batch_x, batch_y in batch: # do stuff ... - :param dataset: :class:`~fastNLP.DataSet` 对象, 数据集 - :param int batch_size: 取出的batch大小 - :param sampler: 规定使用的 :class:`~fastNLP.Sampler` 方式. 若为 ``None`` , 使用 :class:`~fastNLP.SequentialSampler`. - - Default: ``None`` - :param bool as_numpy: 若为 ``True`` , 输出batch为 numpy.array. 否则为 :class:`torch.Tensor`. - - Default: ``False`` - :param int num_workers: 使用多少个进程来预处理数据 - :param bool pin_memory: 是否将产生的tensor使用pin memory, 可能会加快速度。 - :param bool drop_last: 如果最后一个batch没有batch_size这么多sample，就扔掉最后一个 - :param timeout: - :param worker_init_fn: 在每个worker启动时调用该函数，会传入一个值，该值是worker的index。 """ def __init__(self, dataset, batch_size=1, sampler=None, as_numpy=False, num_workers=0, pin_memory=False, drop_last=False, timeout=0, worker_init_fn=None): + """ + + :param dataset: :class:`~fastNLP.DataSet` 对象, 数据集 + :param int batch_size: 取出的batch大小 + :param sampler: 规定使用的 :class:`~fastNLP.Sampler` 方式. 若为 ``None`` , 使用 :class:`~fastNLP.SequentialSampler`. + + Default: ``None`` + :param bool as_numpy: 若为 ``True`` , 输出batch为 numpy.array. 否则为 :class:`torch.Tensor`. + + Default: ``False`` + :param int num_workers: 使用多少个进程来预处理数据 + :param bool pin_memory: 是否将产生的tensor使用pin memory, 可能会加快速度。 + :param bool drop_last: 如果最后一个batch没有batch_size这么多sample，就扔掉最后一个 + :param timeout: + :param worker_init_fn: 在每个worker启动时调用该函数，会传入一个值，该值是worker的index。 + """ super().__init__() assert isinstance(dataset, DataSet) if not isinstance(sampler, torch.utils.data.Sampler): diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index 3cdc0f8d..fe198acc 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -317,9 +317,11 @@ def _transfer(func): class CallbackManager(Callback): + """ + 内部使用的Callback管理类 + """ def __init__(self, env, callbacks=None): """ - 内部使用的Callback管理类 :param dict env: The key is the name of the Trainer attribute(str). The value is the attribute itself. :param List[Callback] callbacks: @@ -435,23 +437,23 @@ class DistCallbackManager(CallbackManager): class GradientClipCallback(Callback): """ 每次backward前，将parameter的gradient clip到某个范围。 - - :param None,torch.Tensor,List[torch.Tensor] parameters: 一般通过model.parameters()获得。 - 如果为None则默认对Trainer的model中所有参数进行clip - :param float clip_value: 将gradient 限制到[-clip_value, clip_value]。clip_value应该为正数 - :param str clip_type: 支持'norm', 'value' - 两种:: - - 1 'norm', 将gradient的norm rescale到[-clip_value, clip_value] - - 2 'value', 将gradient限制在[-clip_value, clip_value], - 小于-clip_value的gradient被赋值为-clip_value; - 大于clip_value的gradient被赋值为clip_value. - """ def __init__(self, parameters=None, clip_value=1, clip_type='norm'): + """ + :param None,torch.Tensor,List[torch.Tensor] parameters: 一般通过model.parameters()获得。 + 如果为None则默认对Trainer的model中所有参数进行clip + :param float clip_value: 将gradient 限制到[-clip_value, clip_value]。clip_value应该为正数 + :param str clip_type: 支持'norm', 'value' + 两种:: + + 1 'norm', 将gradient的norm rescale到[-clip_value, clip_value] + + 2 'value', 将gradient限制在[-clip_value, clip_value], + 小于-clip_value的gradient被赋值为-clip_value; + 大于clip_value的gradient被赋值为clip_value. + """ super().__init__() from torch import nn @@ -477,12 +479,14 @@ class GradientClipCallback(Callback): class EarlyStopCallback(Callback): """ - 多少个epoch没有变好就停止训练，相关类 :class:`EarlyStopError` - - :param int patience: epoch的数量 + 多少个epoch没有变好就停止训练，相关类 :class:`~fastNLP.core.callback.EarlyStopError` """ def __init__(self, patience): + """ + + :param int patience: epoch的数量 + """ super(EarlyStopCallback, self).__init__() self.patience = patience self.wait = 0 @@ -510,17 +514,19 @@ class FitlogCallback(Callback): 一个(或多个)test数据集进行测试(只有在trainer具有dev时才能使用)，每次在dev上evaluate之后会在这些数据集上验证一下。 并将验证结果写入到fitlog中。这些数据集的结果是根据dev上最好的结果报道的，即如果dev在第3个epoch取得了最佳，则 fitlog中记录的关于这些数据集的结果就是来自第三个epoch的结果。 - - :param ~fastNLP.DataSet,Dict[~fastNLP.DataSet] data: 传入DataSet对象，会使用多个Trainer中的metric对数据进行验证。如果需要 - 传入多个DataSet请通过dict的方式传入，dict的key将作为对应dataset的name传递给fitlog。data的结果的名称以'data'开头。 - :param ~fastNLP.Tester,Dict[~fastNLP.Tester] tester: Tester对象，将在on_valid_end时调用。tester的结果的名称以'tester'开头 - :param int log_loss_every: 多少个step记录一次loss(记录的是这几个batch的loss平均值)，如果数据集较大建议将该值设置得 - 大一些，不然会导致log文件巨大。默认为0, 即不要记录loss。 - :param int verbose: 是否在终端打印evaluation的结果，0不打印。 - :param bool log_exception: fitlog是否记录发生的exception信息 """ def __init__(self, data=None, tester=None, log_loss_every=0, verbose=0, log_exception=False): + """ + + :param ~fastNLP.DataSet,Dict[~fastNLP.DataSet] data: 传入DataSet对象，会使用多个Trainer中的metric对数据进行验证。如果需要 + 传入多个DataSet请通过dict的方式传入，dict的key将作为对应dataset的name传递给fitlog。data的结果的名称以'data'开头。 + :param ~fastNLP.Tester,Dict[~fastNLP.Tester] tester: Tester对象，将在on_valid_end时调用。tester的结果的名称以'tester'开头 + :param int log_loss_every: 多少个step记录一次loss(记录的是这几个batch的loss平均值)，如果数据集较大建议将该值设置得 + 大一些，不然会导致log文件巨大。默认为0, 即不要记录loss。 + :param int verbose: 是否在终端打印evaluation的结果，0不打印。 + :param bool log_exception: fitlog是否记录发生的exception信息 + """ super().__init__() self.datasets = {} self.testers = {} @@ -604,13 +610,14 @@ class FitlogCallback(Callback): class EvaluateCallback(Callback): """ 该callback用于扩展Trainer训练过程中只能对dev数据进行验证的问题。 - - :param ~fastNLP.DataSet,Dict[~fastNLP.DataSet] data: 传入DataSet对象，会使用多个Trainer中的metric对数据进行验证。如果需要传入多个 - DataSet请通过dict的方式传入。 - :param ~fastNLP.Tester,Dict[~fastNLP.DataSet] tester: Tester对象，将在on_valid_end时调用。 """ def __init__(self, data=None, tester=None): + """ + :param ~fastNLP.DataSet,Dict[~fastNLP.DataSet] data: 传入DataSet对象，会使用多个Trainer中的metric对数据进行验证。如果需要传入多个 + DataSet请通过dict的方式传入。 + :param ~fastNLP.Tester,Dict[~fastNLP.DataSet] tester: Tester对象，将在on_valid_end时调用。 + """ super().__init__() self.datasets = {} self.testers = {} @@ -664,12 +671,12 @@ class EvaluateCallback(Callback): class LRScheduler(Callback): """ 对PyTorch LR Scheduler的包装以使得其可以被Trainer所使用 - - :param torch.optim.lr_scheduler._LRScheduler lr_scheduler: PyTorch的lr_scheduler """ def __init__(self, lr_scheduler): - + """ + :param torch.optim.lr_scheduler._LRScheduler lr_scheduler: PyTorch的lr_scheduler + """ super(LRScheduler, self).__init__() import torch.optim if isinstance(lr_scheduler, torch.optim.lr_scheduler._LRScheduler): @@ -683,12 +690,13 @@ class LRScheduler(Callback): class ControlC(Callback): """ - - :param bool quit_all: 若为True,则检测到control+C 直接退出程序；否则只退出Trainer + 检测到 control+C 时的反馈 """ def __init__(self, quit_all): - + """ + :param bool quit_all: 若为True,则检测到control+C 直接退出程序；否则只退出Trainer + """ super(ControlC, self).__init__() if type(quit_all) != bool: raise ValueError("In KeyBoardInterrupt, quit_all arguemnt must be a bool.") @@ -720,13 +728,14 @@ class SmoothValue(object): class LRFinder(Callback): """ 用第一个 epoch 找最佳的学习率，从第二个epoch开始应用它 - - :param float start_lr: 学习率下界 - :param float end_lr: 学习率上界 """ def __init__(self, start_lr=1e-6, end_lr=10): + """ + :param float start_lr: 学习率下界 + :param float end_lr: 学习率上界 + """ super(LRFinder, self).__init__() self.start_lr, self.end_lr = start_lr, end_lr @@ -864,13 +873,15 @@ class TensorboardCallback(Callback): class WarmupCallback(Callback): """ 按一定的周期调节Learning rate的大小。 - - :param int,float warmup: 如果warmup为int，则在该step之前，learning rate根据schedule的策略变化; 如果warmup为float， - 如0.1, 则前10%的step是按照schedule策略调整learning rate。 - :param str schedule: 以哪种方式调整。linear: 前warmup的step上升到指定的learning rate(从Trainer中的optimizer处获取的), 后 - warmup的step下降到0； constant前warmup的step上升到指定learning rate，后面的step保持learning rate. """ def __init__(self, warmup=0.1, schedule='constant'): + """ + + :param int,float warmup: 如果warmup为int，则在该step之前，learning rate根据schedule的策略变化; 如果warmup为float， + 如0.1, 则前10%的step是按照schedule策略调整learning rate。 + :param str schedule: 以哪种方式调整。linear: 前warmup的step上升到指定的learning rate(从Trainer中的optimizer处获取的), 后 + warmup的step下降到0； constant前warmup的step上升到指定learning rate，后面的step保持learning rate. + """ super().__init__() self.warmup = max(warmup, 0.) @@ -920,13 +931,15 @@ class SaveModelCallback(Callback): -epoch:1_step:40_{metric_key}:{evaluate_performance}.pt -2019-07-03-15-10-00 -epoch:0_step:20_{metric_key}:{evaluate_performance}.pt # metric是给定的metric_key, evaluate_perfomance是性能 - - :param str save_dir: 将模型存放在哪个目录下，会在该目录下创建以时间戳命名的目录，并存放模型 - :param int top: 保存dev表现top多少模型。-1为保存所有模型。 - :param bool only_param: 是否只保存模型d饿权重。 - :param save_on_exception: 发生exception时，是否保存一份发生exception的模型。模型名称为epoch:x_step:x_Exception:{exception_name}. """ def __init__(self, save_dir, top=3, only_param=False, save_on_exception=False): + """ + + :param str save_dir: 将模型存放在哪个目录下，会在该目录下创建以时间戳命名的目录，并存放模型 + :param int top: 保存dev表现top多少模型。-1为保存所有模型。 + :param bool only_param: 是否只保存模型d饿权重。 + :param save_on_exception: 发生exception时，是否保存一份发生exception的模型。模型名称为epoch:x_step:x_Exception:{exception_name}. + """ super().__init__() if not os.path.isdir(save_dir): @@ -992,11 +1005,13 @@ class SaveModelCallback(Callback): class CallbackException(BaseException): """ 当需要通过callback跳出训练的时候可以通过抛出CallbackException并在on_exception中捕获这个值。 - - :param str msg: Exception的信息。 """ def __init__(self, msg): + """ + + :param str msg: Exception的信息。 + """ super(CallbackException, self).__init__(msg) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 441f9907..ebdc780f 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -288,30 +288,31 @@ __all__ = [ ] import _pickle as pickle -import warnings +from copy import deepcopy import numpy as np -from copy import deepcopy +from ._logger import logger +from .const import Const +from .field import AppendToTargetOrInputException from .field import AutoPadder from .field import FieldArray +from .field import SetInputOrTargetException from .instance import Instance from .utils import _get_func_signature -from .field import AppendToTargetOrInputException -from .field import SetInputOrTargetException -from .const import Const -from ._logger import logger + class DataSet(object): """ fastNLP的数据容器，详细的使用方法见文档 :doc:`fastNLP.core.dataset` - - :param data: 如果为dict类型，则每个key的value应该为等长的list; 如果为list， - 每个元素应该为具有相同field的 :class:`~fastNLP.Instance` 。 - """ def __init__(self, data=None): + """ + + :param data: 如果为dict类型，则每个key的value应该为等长的list; 如果为list， + 每个元素应该为具有相同field的 :class:`~fastNLP.Instance` 。 + """ self.field_arrays = {} if data is not None: if isinstance(data, dict): diff --git a/fastNLP/core/field.py b/fastNLP/core/field.py index 468c248d..82fcc523 100644 --- a/fastNLP/core/field.py +++ b/fastNLP/core/field.py @@ -468,18 +468,18 @@ class Padder: 用于对batch进行padding操作。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前deepcopy一份。 .. py:function:: __call__(self, contents, field_name, field_ele_dtype): + + """ + + def __init__(self, pad_val=0, **kwargs): + """ - 传入的是List内容。假设有以下的DataSet。 - :param List[Any] contents: 传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前 deepcopy一份。 :param str, field_name: field的名称。 :param np.int64,np.float64,np.str,None, field_ele_dtype: 该field的内层元素的类型。如果该field的ignore_type为True，该这个值为None。 :return: np.array([padded_element]) - - """ - - def __init__(self, pad_val=0, **kwargs): + """ self.pad_val = pad_val def set_pad_val(self, pad_val): diff --git a/fastNLP/core/instance.py b/fastNLP/core/instance.py index 2285e4a4..9460b5e4 100644 --- a/fastNLP/core/instance.py +++ b/fastNLP/core/instance.py @@ -37,7 +37,8 @@ class Instance(object): def items(self): """ 返回一个迭代器，迭代器返回两个内容，第一个内容是field_name, 第二个内容是field_value - :return: + + :return: 一个迭代器 """ return self.fields.items() diff --git a/fastNLP/core/losses.py b/fastNLP/core/losses.py index b2f5ce0a..9b32babb 100644 --- a/fastNLP/core/losses.py +++ b/fastNLP/core/losses.py @@ -20,7 +20,6 @@ from collections import defaultdict import torch import torch.nn.functional as F -from ..core.const import Const from .utils import _CheckError from .utils import _CheckRes from .utils import _build_args @@ -28,7 +27,7 @@ from .utils import _check_arg_dict_list from .utils import _check_function_or_method from .utils import _get_func_signature from .utils import seq_len_to_mask -import warnings +from ..core.const import Const class LossBase(object): @@ -284,15 +283,17 @@ class BCELoss(LossBase): class NLLLoss(LossBase): """ 负对数似然损失函数 - - :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` - :param target: 参数映射表中 `target` 的映射关系，None表示映射关系为 `target` -> `target` - :param ignore_idx: ignore的index，在计算loss时将忽略target中标号为ignore_idx的内容, 可以通过该值代替 - 传入seq_len. - :param str reduction: 支持 `mean` ，`sum` 和 `none` . """ def __init__(self, pred=None, target=None, ignore_idx=-100, reduction='mean'): + """ + + :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` + :param target: 参数映射表中 `target` 的映射关系，None表示映射关系为 `target` -> `target` + :param ignore_idx: ignore的index，在计算loss时将忽略target中标号为ignore_idx的内容, 可以通过该值代替 + 传入seq_len. + :param str reduction: 支持 `mean` ，`sum` 和 `none` . + """ super(NLLLoss, self).__init__() self._init_param_map(pred=pred, target=target) assert reduction in ('mean', 'sum', 'none') @@ -306,11 +307,13 @@ class NLLLoss(LossBase): class LossInForward(LossBase): """ 从forward()函数返回结果中获取loss - - :param str loss_key: 在forward函数中loss的键名，默认为loss """ def __init__(self, loss_key=Const.LOSS): + """ + + :param str loss_key: 在forward函数中loss的键名，默认为loss + """ super().__init__() if not isinstance(loss_key, str): raise TypeError(f"Only str allowed for loss_key, got {type(loss_key)}.") diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index 2dc6d9d8..ec1a1864 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -10,7 +10,10 @@ __all__ = [ ] import inspect +import warnings +from abc import abstractmethod from collections import defaultdict +from typing import Union import numpy as np import torch @@ -22,9 +25,7 @@ from .utils import _check_arg_dict_list from .utils import _get_func_signature from .utils import seq_len_to_mask from .vocabulary import Vocabulary -from abc import abstractmethod -import warnings -from typing import Union + class MetricBase(object): """ @@ -295,13 +296,15 @@ class MetricBase(object): class AccuracyMetric(MetricBase): """ 准确率Metric（其它的Metric参见 :doc:`fastNLP.core.metrics` ） - - :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` - :param target: 参数映射表中 `target` 的映射关系，None表示映射关系为 `target` -> `target` - :param seq_len: 参数映射表中 `seq_len` 的映射关系，None表示映射关系为 `seq_len` -> `seq_len` """ def __init__(self, pred=None, target=None, seq_len=None): + """ + + :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` + :param target: 参数映射表中 `target` 的映射关系，None表示映射关系为 `target` -> `target` + :param seq_len: 参数映射表中 `seq_len` 的映射关系，None表示映射关系为 `seq_len` -> `seq_len` + """ super().__init__() @@ -584,25 +587,23 @@ class SpanFPreRecMetric(MetricBase): 'rec-label':xxx, ... } - - :param tag_vocab: 标签的 :class:`~fastNLP.Vocabulary` 。支持的标签为"B"(没有label)；或"B-xxx"(xxx为某种label，比如POS中的NN)， - 在解码时，会将相同xxx的认为是同一个label，比如['B-NN', 'E-NN']会被合并为一个'NN'. - :param str pred: 用该key在evaluate()时从传入dict中取出prediction数据。 为None，则使用 `pred` 取数据 - :param str target: 用该key在evaluate()时从传入dict中取出target数据。 为None，则使用 `target` 取数据 - :param str seq_len: 用该key在evaluate()时从传入dict中取出sequence length数据。为None，则使用 `seq_len` 取数据。 - :param str encoding_type: 目前支持bio, bmes, bmeso, bioes。默认为None，通过tag_vocab自动判断. - :param list ignore_labels: str 组成的list. 这个list中的class不会被用于计算。例如在POS tagging时传入['NN']，则不会计算'NN'这 - 个label - :param bool only_gross: 是否只计算总的f1, precision, recall的值；如果为False，不仅返回总的f1, pre, rec, 还会返回每个 - label的f1, pre, rec - :param str f_type: `micro` 或 `macro` . `micro` :通过先计算总体的TP，FN和FP的数量，再计算f, precision, recall; `macro` : - 分布计算每个类别的f, precision, recall，然后做平均（各类别f的权重相同） - :param float beta: f_beta分数， :math:`f_{beta} = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}` . - 常用为beta=0.5, 1, 2. 若为0.5则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 """ def __init__(self, tag_vocab, pred=None, target=None, seq_len=None, encoding_type=None, ignore_labels=None, only_gross=True, f_type='micro', beta=1): + r""" + + :param tag_vocab: 标签的 :class:`~fastNLP.Vocabulary` 。支持的标签为"B"(没有label)；或"B-xxx"(xxx为某种label，比如POS中的NN)， + 在解码时，会将相同xxx的认为是同一个label，比如['B-NN', 'E-NN']会被合并为一个'NN'. + :param str pred: 用该key在evaluate()时从传入dict中取出prediction数据。 为None，则使用 `pred` 取数据 + :param str target: 用该key在evaluate()时从传入dict中取出target数据。 为None，则使用 `target` 取数据 + :param str seq_len: 用该key在evaluate()时从传入dict中取出sequence length数据。为None，则使用 `seq_len` 取数据。 + :param str encoding_type: 目前支持bio, bmes, bmeso, bioes。默认为None，通过tag_vocab自动判断. + :param list ignore_labels: str 组成的list. 这个list中的class不会被用于计算。例如在POS tagging时传入['NN']，则不会计算'NN'个label + :param bool only_gross: 是否只计算总的f1, precision, recall的值；如果为False，不仅返回总的f1, pre, rec, 还会返回每个label的f1, pre, rec + :param str f_type: `micro` 或 `macro` . `micro` :通过先计算总体的TP，FN和FP的数量，再计算f, precision, recall; `macro` : 分布计算每个类别的f, precision, recall，然后做平均（各类别f的权重相同） + :param float beta: f_beta分数， :math:`f_{beta} = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}` . 常用为 `beta=0.5, 1, 2` 若为0.5则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 + """ if not isinstance(tag_vocab, Vocabulary): raise TypeError("tag_vocab can only be fastNLP.Vocabulary, not {}.".format(type(tag_vocab))) @@ -829,20 +830,21 @@ class ExtractiveQAMetric(MetricBase): r""" 抽取式QA（如SQuAD）的metric. - :param pred1: 参数映射表中 `pred1` 的映射关系，None表示映射关系为 `pred1` -> `pred1` - :param pred2: 参数映射表中 `pred2` 的映射关系，None表示映射关系为 `pred2` -> `pred2` - :param target1: 参数映射表中 `target1` 的映射关系，None表示映射关系为 `target1` -> `target1` - :param target2: 参数映射表中 `target2` 的映射关系，None表示映射关系为 `target2` -> `target2` - :param float beta: f_beta分数， :math:`f_{beta} = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}` . - 常用为beta=0.5, 1, 2. 若为0.5则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 - :param bool right_open: right_open为true表示start跟end指针指向一个左闭右开区间，为false表示指向一个左闭右闭区间。 - :param bool print_predict_stat: True则输出预测答案是否为空与正确答案是否为空的统计信息, False则不输出 - """ def __init__(self, pred1=None, pred2=None, target1=None, target2=None, beta=1, right_open=True, print_predict_stat=False): - + r""" + + :param pred1: 参数映射表中 `pred1` 的映射关系，None表示映射关系为 `pred1` -> `pred1` + :param pred2: 参数映射表中 `pred2` 的映射关系，None表示映射关系为 `pred2` -> `pred2` + :param target1: 参数映射表中 `target1` 的映射关系，None表示映射关系为 `target1` -> `target1` + :param target2: 参数映射表中 `target2` 的映射关系，None表示映射关系为 `target2` -> `target2` + :param float beta: f_beta分数， :math:`f_{beta} = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}` . + 常用为beta=0.5, 1, 2. 若为0.5则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 + :param bool right_open: right_open为true表示start跟end指针指向一个左闭右开区间，为false表示指向一个左闭右闭区间。 + :param bool print_predict_stat: True则输出预测答案是否为空与正确答案是否为空的统计信息, False则不输出 + """ super(ExtractiveQAMetric, self).__init__() self._init_param_map(pred1=pred1, pred2=pred2, target1=target1, target2=target2) diff --git a/fastNLP/core/optimizer.py b/fastNLP/core/optimizer.py index c30c7e34..5e7c1cba 100644 --- a/fastNLP/core/optimizer.py +++ b/fastNLP/core/optimizer.py @@ -9,20 +9,23 @@ __all__ = [ "AdamW" ] -import torch import math + import torch from torch.optim.optimizer import Optimizer as TorchOptimizer class Optimizer(object): """ - - :param model_params: a generator. E.g. ``model.parameters()`` for PyTorch models. - :param kwargs: additional parameters. + Optimizer """ def __init__(self, model_params, **kwargs): + """ + + :param model_params: a generator. E.g. ``model.parameters()`` for PyTorch models. + :param kwargs: additional parameters. + """ if model_params is not None and not hasattr(model_params, "__next__"): raise RuntimeError("model parameters should be a generator, rather than {}.".format(type(model_params))) self.model_params = model_params @@ -59,13 +62,15 @@ class NullOptimizer(Optimizer): class SGD(Optimizer): """ - - :param float lr: learning rate. Default: 0.01 - :param float momentum: momentum. Default: 0 - :param model_params: a generator. E.g. ``model.parameters()`` for PyTorch models. + SGD """ def __init__(self, lr=0.001, momentum=0, model_params=None): + """ + :param float lr: learning rate. Default: 0.01 + :param float momentum: momentum. Default: 0 + :param model_params: a generator. E.g. ``model.parameters()`` for PyTorch models. + """ if not isinstance(lr, float): raise TypeError("learning rate has to be float.") super(SGD, self).__init__(model_params, lr=lr, momentum=momentum) @@ -81,12 +86,17 @@ class SGD(Optimizer): class Adam(Optimizer): """ - :param float lr: learning rate - :param float weight_decay: - :param model_params: a generator. E.g. ``model.parameters()`` for PyTorch models. """ def __init__(self, lr=0.001, weight_decay=0, betas=(0.9, 0.999), eps=1e-8, amsgrad=False, model_params=None): + """ + + :param float lr: learning rate + :param float weight_decay: + :param eps: + :param amsgrad: + :param model_params: a generator. E.g. ``model.parameters()`` for PyTorch models. + """ if not isinstance(lr, float): raise TypeError("learning rate has to be float.") super(Adam, self).__init__(model_params, lr=lr, betas=betas, eps=eps, amsgrad=amsgrad, @@ -110,17 +120,6 @@ class AdamW(TorchOptimizer): The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_. The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_. - :param params (iterable): iterable of parameters to optimize or dicts defining - parameter groups - :param lr (float, optional): learning rate (default: 1e-3) - :param betas (Tuple[float, float], optional): coefficients used for computing - running averages of gradient and its square (default: (0.9, 0.99)) - :param eps (float, optional): term added to the denominator to improve - numerical stability (default: 1e-8) - :param weight_decay (float, optional): weight decay coefficient (default: 1e-2) - algorithm from the paper `On the Convergence of Adam and Beyond`_ - (default: False) - .. _Adam\: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980 .. _Decoupled Weight Decay Regularization: @@ -131,6 +130,19 @@ class AdamW(TorchOptimizer): def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=1e-2, amsgrad=False): + """ + + :param params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + :param lr (float, optional): learning rate (default: 1e-3) + :param betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.99)) + :param eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + :param weight_decay (float, optional): weight decay coefficient (default: 1e-2) + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + """ if not 0.0 <= lr: raise ValueError("Invalid learning rate: {}".format(lr)) if not 0.0 <= eps: diff --git a/fastNLP/core/predictor.py b/fastNLP/core/predictor.py index c6b8fc90..e4112d5f 100644 --- a/fastNLP/core/predictor.py +++ b/fastNLP/core/predictor.py @@ -20,11 +20,13 @@ class Predictor(object): 与测试器（Tester）不同的是，predictor不关心模型性能的评价指标，只做inference。 这是一个fastNLP调用的高级模型包装器。它与Trainer、Tester不共享任何操作。 - - :param torch.nn.Module network: 用来完成预测任务的模型 """ def __init__(self, network): + """ + + :param torch.nn.Module network: 用来完成预测任务的模型 + """ if not isinstance(network, torch.nn.Module): raise ValueError( "Only fastNLP.models.BaseModel or torch.nn,Module is allowed, not {}".format(type(network))) diff --git a/fastNLP/core/sampler.py b/fastNLP/core/sampler.py index d0df9129..6e025688 100644 --- a/fastNLP/core/sampler.py +++ b/fastNLP/core/sampler.py @@ -51,14 +51,16 @@ class RandomSampler(Sampler): class BucketSampler(Sampler): """ 带Bucket的 `Random Sampler`. 可以随机地取出长度相似的元素 - - :param int num_buckets: bucket的数量 - :param int batch_size: batch的大小. 默认为None，Trainer在调用BucketSampler时，会将该值正确设置，如果是非Trainer场景使用，需 - 要显示传递该值 - :param str seq_len_field_name: 对应序列长度的 `field` 的名字 """ def __init__(self, num_buckets=10, batch_size=None, seq_len_field_name='seq_len'): + """ + + :param int num_buckets: bucket的数量 + :param int batch_size: batch的大小. 默认为None，Trainer在调用BucketSampler时，会将该值正确设置，如果是非Trainer场景使用，需 + 要显示传递该值 + :param str seq_len_field_name: 对应序列长度的 `field` 的名字 + """ self.num_buckets = num_buckets self.batch_size = batch_size self.seq_len_field_name = seq_len_field_name diff --git a/fastNLP/core/tester.py b/fastNLP/core/tester.py index 344e24a8..d1d5d41e 100644 --- a/fastNLP/core/tester.py +++ b/fastNLP/core/tester.py @@ -66,30 +66,32 @@ __all__ = [ class Tester(object): """ Tester是在提供数据，模型以及metric的情况下进行性能测试的类。需要传入模型，数据以及metric进行验证。 - - :param ~fastNLP.DataSet data: 需要测试的数据集 - :param torch.nn.module model: 使用的模型 - :param ~fastNLP.core.metrics.MetricBase,List[~fastNLP.core.metrics.MetricBase] metrics: 测试时使用的metrics - :param int batch_size: evaluation时使用的batch_size有多大。 - :param str,int,torch.device,list(int) device: 将模型load到哪个设备。默认为None，即Trainer不对模型 - 的计算位置进行管理。支持以下的输入: - - 1. str: ['cpu', 'cuda', 'cuda:0', 'cuda:1', ...] 依次为'cpu'中, 可见的第一个GPU中,可见的第一个GPU中,可见的第二个GPU中; - - 2. torch.device：将模型装载到torch.device上。 - - 3. int: 将使用device_id为该值的gpu进行训练 - - 4. list(int)：如果多于1个device，将使用torch.nn.DataParallel包裹model, 并使用传入的device。 - - 5. None. 为None则不对模型进行任何处理，如果传入的model为torch.nn.DataParallel该值必须为None。 - - 如果模型是通过predict()进行预测的话，那么将不能使用多卡(DataParallel)进行验证，只会使用第一张卡上的模型。 - :param int verbose: 如果为0不输出任何信息; 如果为1，打印出验证结果。 - :param bool use_tqdm: 是否使用tqdm来显示测试进度; 如果为False，则不会显示任何内容。 """ def __init__(self, data, model, metrics, batch_size=16, num_workers=0, device=None, verbose=1, use_tqdm=True): + """ + + :param ~fastNLP.DataSet data: 需要测试的数据集 + :param torch.nn.module model: 使用的模型 + :param ~fastNLP.core.metrics.MetricBase,List[~fastNLP.core.metrics.MetricBase] metrics: 测试时使用的metrics + :param int batch_size: evaluation时使用的batch_size有多大。 + :param str,int,torch.device,list(int) device: 将模型load到哪个设备。默认为None，即Trainer不对模型 + 的计算位置进行管理。支持以下的输入: + + 1. str: ['cpu', 'cuda', 'cuda:0', 'cuda:1', ...] 依次为'cpu'中, 可见的第一个GPU中,可见的第一个GPU中,可见的第二个GPU中; + + 2. torch.device：将模型装载到torch.device上。 + + 3. int: 将使用device_id为该值的gpu进行训练 + + 4. list(int)：如果多于1个device，将使用torch.nn.DataParallel包裹model, 并使用传入的device。 + + 5. None. 为None则不对模型进行任何处理，如果传入的model为torch.nn.DataParallel该值必须为None。 + + 如果模型是通过predict()进行预测的话，那么将不能使用多卡(DataParallel)进行验证，只会使用第一张卡上的模型。 + :param int verbose: 如果为0不输出任何信息; 如果为1，打印出验证结果。 + :param bool use_tqdm: 是否使用tqdm来显示测试进度; 如果为False，则不会显示任何内容。 + """ super(Tester, self).__init__() if not isinstance(model, nn.Module): @@ -137,10 +139,9 @@ class Tester(object): self._predict_func_wrapper = self._model.forward def test(self): - """开始进行验证，并返回验证结果。 + r"""开始进行验证，并返回验证结果。 - :return Dict[Dict] : dict的二层嵌套结构，dict的第一层是metric的名称; 第二层是这个metric的指标。 - 一个AccuracyMetric的例子为{'AccuracyMetric': {'acc': 1.0}}。 + :return Dict[Dict]: dict的二层嵌套结构，dict的第一层是metric的名称; 第二层是这个metric的指标。一个AccuracyMetric的例子为{'AccuracyMetric': {'acc': 1.0}}。 """ # turn on the testing mode; clean up the history self._model_device = _get_model_device(self._model) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 9f262fb5..a2c3b1f7 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -365,54 +365,6 @@ class Trainer(object): (5) 保存获得更好验证性能的模型等。 详细的介绍参见 :doc:`fastNLP.core.trainer` - - :param train_data: 训练集， :class:`~fastNLP.DataSet` 类型。 - :param nn.modules model: 待训练的模型 - :param optimizer: `torch.optim.Optimizer` 优化器。如果为None，则Trainer使用默认的Adam(model.parameters(), lr=4e-3)这个优化器 - :param int batch_size: 训练和验证的时候的batch大小。 - :param loss: 使用的 :class:`~fastNLP.core.losses.LossBase` 对象。当为None时，默认使用 :class:`~fastNLP.LossInForward` - :param sampler: Batch数据生成的顺序， :class:`~fastNLP.Sampler` 类型。如果为None，默认使用 :class:`~fastNLP.RandomSampler` - :param drop_last: 如果最后一个batch没有正好为batch_size这么多数据，就扔掉最后一个batch - :param num_workers: int, 有多少个线程来进行数据pad处理。 - :param update_every: int, 多少步更新一次梯度。用于希望累计梯度的场景，比如需要128的batch_size, 但是直接设为128 - 会导致内存不足，通过设置batch_size=32, update_every=4达到目的。当optimizer为None时，该参数无效。 - :param int n_epochs: 需要优化迭代多少次。 - :param int print_every: 多少次反向传播更新tqdm显示的loss; 如果use_tqdm=False, 则多少次反向传播打印loss。 - :param dev_data: 用于做验证的DataSet， :class:`~fastNLP.DataSet` 类型。 - :param metrics: 验证的评估函数。可以只使用一个 :class:`Metric` ， - 也可以使用多个 :class:`Metric` ，通过列表传入。 - 如验证时取得了更好的验证结果(如果有多个Metric，以列表中第一个Metric为准)，且save_path不为None， - 则保存当前模型。Metric种类详见 :doc:`metrics模块 ` 。仅在传入dev_data时有效。 - :param str,None metric_key: :class:`Metric` 有时会有多个指标， - 比如 :class:`~fastNLP.core.metrics.SpanFPreRecMetric` 中包含了'f', 'pre', 'rec'。此时需 - 要指定以哪个指标为准。另外有些指标是越小效果越好，比如语言模型的困惑度，这种情况下，在key前面增加一个'-'来表 - 明验证时，值越小越好(比如: "-ppl")。仅在传入dev_data时有效。 - :param int validate_every: 多少个step在验证集上验证一次; 如果为-1，则每个epoch结束验证一次。仅在传入dev_data时有效。 - :param str,None save_path: 将模型保存路径，如果路径不存在，将自动创建文件夹。如果为None，则不保存模型。如果dev_data为None，则保存 - 最后一次迭代的模型。保存的时候不仅保存了参数，还保存了模型结构。即便使用DataParallel，这里也只保存模型。 - :param bool use_tqdm: 是否使用tqdm来显示训练进度; 如果为False，则将loss打印在终端中。 - :param str,int,torch.device,list(int) device: 将模型load到哪个设备。默认为None，即Trainer不对模型 - 的计算位置进行管理。支持以下的输入: - - 1. str: ['cpu', 'cuda', 'cuda:0', 'cuda:1', ...] 依次为'cpu'中, 可见的第一个GPU中, 可见的第一个GPU中, - 可见的第二个GPU中; - - 2. torch.device：将模型装载到torch.device上。 - - 3. int: 将使用device_id为该值的gpu进行训练 - - 4. list(int)：如果多于1个device，将使用torch.nn.DataParallel包裹model, 并使用传入的device。 - - 5. None. 为None则不对模型进行任何处理，如果传入的model为torch.nn.DataParallel该值必须为None。 - - 已知可能会出现的问题：Adagrad优化器可能无法正常使用这个参数，请手动管理模型位置。 - - :param list(callbacks) callbacks: 用于在train过程中起调节作用的回调函数。比如early stop，negative sampling等可以 - 通过callback机制实现。 可使用的callback参见 :doc:`callback模块 ` - :param int check_code_level: 模型检查等级. -1: 不进行检查; 0: 仅出现错误时停止; 1: 如果有field没有被使用， - 报告警告信息; 2: 有任何field没有被使用都报错. 检查的原理是通过使用很小的batch(默认2个sample)来运行代码，但是 - 这个过程理论上不会修改任何参数，只是会检查能否运行。但如果(1)模型中存在将batch_size写为某个固定值的情况； - (2)模型中存在累加前向计算次数的，可能会多计算1次。以上情况建议将check_code_level设置为-1。 """ def __init__(self, train_data, model, optimizer=None, loss=None, @@ -421,6 +373,56 @@ class Trainer(object): dev_data=None, metrics=None, metric_key=None, validate_every=-1, save_path=None, use_tqdm=True, device=None, callbacks=None, check_code_level=0, **kwargs): + """ + + :param train_data: 训练集， :class:`~fastNLP.DataSet` 类型。 + :param nn.modules model: 待训练的模型 + :param optimizer: `torch.optim.Optimizer` 优化器。如果为None，则Trainer使用默认的Adam(model.parameters(), lr=4e-3)这个优化器 + :param int batch_size: 训练和验证的时候的batch大小。 + :param loss: 使用的 :class:`~fastNLP.core.losses.LossBase` 对象。当为None时，默认使用 :class:`~fastNLP.LossInForward` + :param sampler: Batch数据生成的顺序， :class:`~fastNLP.Sampler` 类型。如果为None，默认使用 :class:`~fastNLP.RandomSampler` + :param drop_last: 如果最后一个batch没有正好为batch_size这么多数据，就扔掉最后一个batch + :param num_workers: int, 有多少个线程来进行数据pad处理。 + :param update_every: int, 多少步更新一次梯度。用于希望累计梯度的场景，比如需要128的batch_size, 但是直接设为128 + 会导致内存不足，通过设置batch_size=32, update_every=4达到目的。当optimizer为None时，该参数无效。 + :param int n_epochs: 需要优化迭代多少次。 + :param int print_every: 多少次反向传播更新tqdm显示的loss; 如果use_tqdm=False, 则多少次反向传播打印loss。 + :param dev_data: 用于做验证的DataSet， :class:`~fastNLP.DataSet` 类型。 + :param metrics: 验证的评估函数。可以只使用一个 :class:`Metric` ， + 也可以使用多个 :class:`Metric` ，通过列表传入。 + 如验证时取得了更好的验证结果(如果有多个Metric，以列表中第一个Metric为准)，且save_path不为None， + 则保存当前模型。Metric种类详见 :doc:`metrics模块 ` 。仅在传入dev_data时有效。 + :param str,None metric_key: :class:`Metric` 有时会有多个指标， + 比如 :class:`~fastNLP.core.metrics.SpanFPreRecMetric` 中包含了'f', 'pre', 'rec'。此时需 + 要指定以哪个指标为准。另外有些指标是越小效果越好，比如语言模型的困惑度，这种情况下，在key前面增加一个'-'来表 + 明验证时，值越小越好(比如: "-ppl")。仅在传入dev_data时有效。 + :param int validate_every: 多少个step在验证集上验证一次; 如果为-1，则每个epoch结束验证一次。仅在传入dev_data时有效。 + :param str,None save_path: 将模型保存路径，如果路径不存在，将自动创建文件夹。如果为None，则不保存模型。如果dev_data为None，则保存 + 最后一次迭代的模型。保存的时候不仅保存了参数，还保存了模型结构。即便使用DataParallel，这里也只保存模型。 + :param bool use_tqdm: 是否使用tqdm来显示训练进度; 如果为False，则将loss打印在终端中。 + :param str,int,torch.device,list(int) device: 将模型load到哪个设备。默认为None，即Trainer不对模型 + 的计算位置进行管理。支持以下的输入: + + 1. str: ['cpu', 'cuda', 'cuda:0', 'cuda:1', ...] 依次为'cpu'中, 可见的第一个GPU中, 可见的第一个GPU中, + 可见的第二个GPU中; + + 2. torch.device：将模型装载到torch.device上。 + + 3. int: 将使用device_id为该值的gpu进行训练 + + 4. list(int)：如果多于1个device，将使用torch.nn.DataParallel包裹model, 并使用传入的device。 + + 5. None. 为None则不对模型进行任何处理，如果传入的model为torch.nn.DataParallel该值必须为None。 + + 已知可能会出现的问题：Adagrad优化器可能无法正常使用这个参数，请手动管理模型位置。 + + :param list(callbacks) callbacks: 用于在train过程中起调节作用的回调函数。比如early stop，negative sampling等可以 + 通过callback机制实现。 可使用的callback参见 :doc:`callback模块 ` + :param int check_code_level: 模型检查等级. -1: 不进行检查; 0: 仅出现错误时停止; 1: 如果有field没有被使用， + 报告警告信息; 2: 有任何field没有被使用都报错. 检查的原理是通过使用很小的batch(默认2个sample)来运行代码，但是 + 这个过程理论上不会修改任何参数，只是会检查能否运行。但如果(1)模型中存在将batch_size写为某个固定值的情况； + (2)模型中存在累加前向计算次数的，可能会多计算1次。以上情况建议将check_code_level设置为-1。 + """ super(Trainer, self).__init__() if not isinstance(model, nn.Module): raise TypeError(f"The type of model must be torch.nn.Module, got {type(model)}.") diff --git a/fastNLP/core/vocabulary.py b/fastNLP/core/vocabulary.py index d4ff6077..6d530eb6 100644 --- a/fastNLP/core/vocabulary.py +++ b/fastNLP/core/vocabulary.py @@ -73,21 +73,23 @@ class Vocabulary(object): vocab.update(word_list) vocab["word"] # str to int vocab.to_word(5) # int to str - - :param int max_size: `Vocabulary` 的最大大小, 即能存储词的最大数量 - 若为 ``None`` , 则不限制大小. Default: ``None`` - :param int min_freq: 能被记录下的词在文本中的最小出现频率, 应大于或等于 1. - 若小于该频率, 词语将被视为 `unknown`. 若为 ``None`` , 所有文本中的词都被记录. Default: ``None`` - :param str optional padding: padding的字符. 如果设置为 ``None`` , - 则vocabulary中不考虑padding, 也不计入词表大小，为 ``None`` 的情况多在为label建立Vocabulary的情况. - Default: '' - :param str optional unknown: unknown的字符，所有未被记录的词在转为 `int` 时将被视为unknown. - 如果设置为 ``None`` ,则vocabulary中不考虑unknow, 也不计入词表大小. - 为 ``None`` 的情况多在为label建立Vocabulary的情况. - Default: '' """ def __init__(self, max_size=None, min_freq=None, padding='', unknown=''): + """ + + :param int max_size: `Vocabulary` 的最大大小, 即能存储词的最大数量 + 若为 ``None`` , 则不限制大小. Default: ``None`` + :param int min_freq: 能被记录下的词在文本中的最小出现频率, 应大于或等于 1. + 若小于该频率, 词语将被视为 `unknown`. 若为 ``None`` , 所有文本中的词都被记录. Default: ``None`` + :param str optional padding: padding的字符. 如果设置为 ``None`` , + 则vocabulary中不考虑padding, 也不计入词表大小，为 ``None`` 的情况多在为label建立Vocabulary的情况. + Default: '' + :param str optional unknown: unknown的字符，所有未被记录的词在转为 `int` 时将被视为unknown. + 如果设置为 ``None`` ,则vocabulary中不考虑unknow, 也不计入词表大小. + 为 ``None`` 的情况多在为label建立Vocabulary的情况. + Default: '' + """ self.max_size = max_size self.min_freq = min_freq self.word_count = Counter() @@ -402,7 +404,7 @@ class Vocabulary(object): def to_index(self, w): """ - 将词转为数字. 若词不再词典中被记录, 将视为 unknown, 若 ``unknown=None`` , 将抛出``ValueError``:: + 将词转为数字. 若词不再词典中被记录, 将视为 unknown, 若 ``unknown=None`` , 将抛出 ``ValueError`` :: index = vocab.to_index('abc') # equals to From 60a535db08be4621e8b2f52bb83caad81c693075 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Wed, 4 Sep 2019 17:15:21 +0800 Subject: [PATCH 40/92] fix a little error in doc. TODO: fix the bug doc of the class which inherit the class from outer space --- fastNLP/core/metrics.py | 1 + fastNLP/core/optimizer.py | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index ec1a1864..72380fd6 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -152,6 +152,7 @@ class MetricBase(object): def get_metric_name(self): """ 返回metric的名称 + :return: """ return self._metric_name diff --git a/fastNLP/core/optimizer.py b/fastNLP/core/optimizer.py index 5e7c1cba..b782cfa6 100644 --- a/fastNLP/core/optimizer.py +++ b/fastNLP/core/optimizer.py @@ -120,12 +120,11 @@ class AdamW(TorchOptimizer): The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_. The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_. - .. _Adam\: A Method for Stochastic Optimization: - https://arxiv.org/abs/1412.6980 - .. _Decoupled Weight Decay Regularization: - https://arxiv.org/abs/1711.05101 - .. _On the Convergence of Adam and Beyond: - https://openreview.net/forum?id=ryQu7f-RZ + .. _Adam\: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980 + + .. _Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101 + + .. _On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ """ def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, From 14d048f3406fd05a79e9e61b8e05c410bf8882f0 Mon Sep 17 00:00:00 2001 From: yh Date: Thu, 5 Sep 2019 00:21:46 +0800 Subject: [PATCH 41/92] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbert=20embedding?= =?UTF-8?q?=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/embeddings/bert_embedding.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index 17f6769d..05351cbd 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -420,11 +420,11 @@ class _WordBertModel(nn.Module): if self.pool_method == 'first': batch_word_pieces_cum_length = batch_word_pieces_cum_length[:, :seq_len.max()] batch_word_pieces_cum_length.masked_fill_(batch_word_pieces_cum_length.ge(word_piece_length), 0) - batch_indexes = batch_indexes[:, None].expand((batch_size, batch_word_pieces_cum_length.size(1))) + _batch_indexes = batch_indexes[:, None].expand((batch_size, batch_word_pieces_cum_length.size(1))) elif self.pool_method == 'last': batch_word_pieces_cum_length = batch_word_pieces_cum_length[:, 1:seq_len.max()+1] - 1 batch_word_pieces_cum_length.masked_fill_(batch_word_pieces_cum_length.ge(word_piece_length), 0) - batch_indexes = batch_indexes[:, None].expand((batch_size, batch_word_pieces_cum_length.size(1))) + _batch_indexes = batch_indexes[:, None].expand((batch_size, batch_word_pieces_cum_length.size(1))) for l_index, l in enumerate(self.layers): output_layer = bert_outputs[l] @@ -437,12 +437,12 @@ class _WordBertModel(nn.Module): # 从word_piece collapse到word的表示 truncate_output_layer = output_layer[:, 1:-1] # 删除[CLS]与[SEP] batch_size x len x hidden_size if self.pool_method == 'first': - tmp = truncate_output_layer[batch_indexes, batch_word_pieces_cum_length] + tmp = truncate_output_layer[_batch_indexes, batch_word_pieces_cum_length] tmp = tmp.masked_fill(word_mask[:, :batch_word_pieces_cum_length.size(1), None].eq(0), 0) outputs[l_index, :, s_shift:batch_word_pieces_cum_length.size(1)+s_shift] = tmp elif self.pool_method == 'last': - tmp = truncate_output_layer[batch_indexes, batch_word_pieces_cum_length] + tmp = truncate_output_layer[_batch_indexes, batch_word_pieces_cum_length] tmp = tmp.masked_fill(word_mask[:, :batch_word_pieces_cum_length.size(1), None].eq(0), 0) outputs[l_index, :, s_shift:batch_word_pieces_cum_length.size(1)+s_shift] = tmp elif self.pool_method == 'max': From 880e3ad96953bb2ac6ed19b1a54efc06835dfc04 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Thu, 5 Sep 2019 01:26:22 +0800 Subject: [PATCH 42/92] 1. add mini_elmo.pkl and test codes for testing ElmoEmbedding; 2. update bert testing codes --- fastNLP/embeddings/elmo_embedding.py | 5 +- fastNLP/models/bert.py | 2 +- .../embedding/small_elmo/char.dic | 229 ++++++++++++++++++ .../elmo_1x16_16_32cnn_1xhighway_options.json | 29 +++ .../small_elmo/elmo_mini_for_testing.pkl | Bin 0 -> 37695 bytes test/embeddings/test_bert_embedding.py | 7 +- test/embeddings/test_elmo_embedding.py | 15 ++ 7 files changed, 282 insertions(+), 5 deletions(-) create mode 100644 test/data_for_tests/embedding/small_elmo/char.dic create mode 100644 test/data_for_tests/embedding/small_elmo/elmo_1x16_16_32cnn_1xhighway_options.json create mode 100644 test/data_for_tests/embedding/small_elmo/elmo_mini_for_testing.pkl diff --git a/fastNLP/embeddings/elmo_embedding.py b/fastNLP/embeddings/elmo_embedding.py index 0ec0caa0..d19a3577 100644 --- a/fastNLP/embeddings/elmo_embedding.py +++ b/fastNLP/embeddings/elmo_embedding.py @@ -69,6 +69,7 @@ class ElmoEmbedding(ContextualEmbedding): else: raise ValueError(f"Cannot recognize {model_dir_or_name}.") self.model = _ElmoModel(model_dir, vocab, cache_word_reprs=cache_word_reprs) + num_layers = self.model.encoder.num_layers if layers == 'mix': self.layer_weights = nn.Parameter(torch.zeros(self.model.config['lstm']['n_layers'] + 1), @@ -78,9 +79,9 @@ class ElmoEmbedding(ContextualEmbedding): self._embed_size = self.model.config['lstm']['projection_dim'] * 2 else: layers = list(map(int, layers.split(','))) - assert len(layers) > 0, "Must choose one output" + assert len(layers) > 0, "Must choose at least one output, but got None." for layer in layers: - assert 0 <= layer <= 2, "Layer index should be in range [0, 2]." + assert 0 <= layer <= num_layers, f"Layer index should be in range [0, {num_layers}], but got {layer}." self.layers = layers self._get_outputs = self._get_layer_outputs self._embed_size = len(self.layers) * self.model.config['lstm']['projection_dim'] * 2 diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py index 85c3af8c..30ed0cd8 100644 --- a/fastNLP/models/bert.py +++ b/fastNLP/models/bert.py @@ -241,7 +241,7 @@ class BertForQuestionAnswering(BaseModel): def forward(self, words): """ :param torch.LongTensor words: [batch_size, seq_len] - :return: 一个包含num_labels个logit的dict，每一个logit的形状都是[batch_size, seq_len] + :return: 一个包含num_labels个logit的dict，每一个logit的形状都是[batch_size, seq_len + 2] """ sequence_output = self.bert(words) logits = self.qa_outputs(sequence_output) # [batch_size, seq_len, num_labels] diff --git a/test/data_for_tests/embedding/small_elmo/char.dic b/test/data_for_tests/embedding/small_elmo/char.dic new file mode 100644 index 00000000..74285f34 --- /dev/null +++ b/test/data_for_tests/embedding/small_elmo/char.dic @@ -0,0 +1,229 @@ +! 33 +" 34 +# 35 +$ 36 +% 37 +& 38 +' 39 +( 40 +) 41 +* 42 ++ 43 +, 44 +- 45 +. 46 +/ 47 +0 48 +1 49 +2 50 +3 51 +4 52 +5 53 +6 54 +7 55 +8 56 +9 57 +: 58 +; 59 +< 60 += 61 +> 62 +? 63 +@ 64 +A 65 +B 66 +C 67 +D 68 +E 69 +F 70 +G 71 +H 72 +I 73 +J 74 +K 75 +L 76 +M 77 +N 78 +O 79 +P 80 +Q 81 +R 82 +S 83 +T 84 +U 85 +V 86 +W 87 +X 88 +Y 89 +Z 90 +[ 91 +\ 92 +] 93 +^ 94 +_ 95 +` 96 +a 97 +b 98 +c 99 +d 100 +e 101 +f 102 +g 103 +h 104 +i 105 +j 106 +k 107 +l 108 +m 109 +n 110 +o 111 +p 112 +q 113 +r 114 +s 115 +t 116 +u 117 +v 118 +w 119 +x 120 +y 121 +z 122 +{ 123 +| 124 +} 125 +~ 126 + 127 +€ 128 + 129 +‚ 130 +ƒ 131 +„ 132 +† 134 +‡ 135 +ˆ 136 +‰ 137 +Š 138 +‹ 139 +Œ 140 + 141 +Ž 142 + 143 + 144 +‘ 145 +’ 146 +“ 147 +” 148 +• 149 +– 150 +— 151 +˜ 152 +™ 153 +š 154 +› 155 +œ 156 + 157 +ž 158 +Ÿ 159 +  160 +¡ 161 +¢ 162 +£ 163 +¤ 164 +¥ 165 +¦ 166 +§ 167 +¨ 168 +© 169 +ª 170 +« 171 +¬ 172 +­ 173 +® 174 +¯ 175 +° 176 +± 177 +² 178 +³ 179 +´ 180 +µ 181 +¶ 182 +· 183 +¸ 184 +¹ 185 +º 186 +» 187 +¼ 188 +½ 189 +¾ 190 +¿ 191 +À 192 +Á 193 + 194 +à 195 +Ä 196 +Å 197 +Æ 198 +Ç 199 +È 200 +É 201 +Ê 202 +Ë 203 +Ì 204 +Í 205 +Î 206 +Ï 207 +Ð 208 +Ñ 209 +Ò 210 +Ó 211 +Ô 212 +Õ 213 +Ö 214 +× 215 +Ø 216 +Ù 217 +Ú 218 +Û 219 +Ü 220 +Ý 221 +Þ 222 +ß 223 +à 224 +á 225 +â 226 +ã 227 +ä 228 +å 229 +æ 230 +ç 231 +è 232 +é 233 +ê 234 +ë 235 +ì 236 +í 237 +î 238 +ï 239 +ð 240 +ñ 241 +ò 242 +ó 243 +ô 244 +õ 245 +ö 246 +÷ 247 +ø 248 +ù 249 +ú 250 +û 251 +ü 252 +ý 253 +þ 254 +ÿ 255 + 256 + 257 + 258 + 259 + 260 + 1 + -1 diff --git a/test/data_for_tests/embedding/small_elmo/elmo_1x16_16_32cnn_1xhighway_options.json b/test/data_for_tests/embedding/small_elmo/elmo_1x16_16_32cnn_1xhighway_options.json new file mode 100644 index 00000000..9c02ef72 --- /dev/null +++ b/test/data_for_tests/embedding/small_elmo/elmo_1x16_16_32cnn_1xhighway_options.json @@ -0,0 +1,29 @@ +{ + "lstm": { + "use_skip_connections": true, + "projection_dim": 16, + "cell_clip": 3, + "proj_clip": 3, + "dim": 16, + "n_layers": 1 + }, + "char_cnn": { + "activation": "relu", + "filters": [ + [ + 1, + 16 + ], + [ + 2, + 16 + ] + ], + "n_highway": 1, + "embedding": { + "dim": 4 + }, + "n_characters": 262, + "max_characters_per_token": 50 + } +} diff --git a/test/data_for_tests/embedding/small_elmo/elmo_mini_for_testing.pkl b/test/data_for_tests/embedding/small_elmo/elmo_mini_for_testing.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4c72f3d51b692e456b55db006d0f65ccf3a0e8a0 GIT binary patch literal 37695 zcmc$_d00P>E(4BBbek);^ii zpoCNsD)T&xgx?#_{d~U9{kfmt_5JUAu1iJ`?2qjN$O3InAh~a@7!Xh}rE&?8cy#ED-L~ul0ggk_P z@z@X&z!7y3ij4dV^EWj-Vm*i7kB1uU?-#)X4VQ2WTkRj_zj{WHUxWllK;NAssOK?> z$6(Edu#LW9t9^ofH~EM87#jtJghodA1P6uq`-TNYY%<#D9~2N6!4dL{*bwFyXyg+a z5fmIQ;S=V+Dl#Z|wNHe9Nce^@pD0raj&krsk$*);2cg78a&9mK?F)%km42oM zf5|Kz!I5$2$o`|?;Ss(O{{LHWxj>#=fg*u_$2GGt<%|jZ7oPmT@y7na8}|oKA%dgm z&QbE1_z#NuJKm}w-*ArdKcZQhm{^!{R073a?0JhD{~K&4N7aRYCuhPRAhigNx;sb1 zWAc9m;WomA0%+dO_GJYPI_AfBGUz*hUv$BbQ0CgicliWFze*>=a z^;`FU;{$sC1<=&knxp@hDh>XvO2a>ZQ~m%>jo=u$bEf?>;NNxmFQ+s9XFwBEb8C*t zUx23n1~mHvX#NM#B7$S-&av{)<*D?4t?=)Fzl~@8&u|vzR;C=Ae;LpAU&gchWjyK+ z5{lqpcMk2L_urBJxApA*8O+?+%#`Esm-VLq+j@?F);8l0;>-w+lRIZtWDIAvhd2+_ zXT5)f?`q!&Uyk$dIGLm8!K3nVTw*wLf5+rsxyRLoKior>H}to#=gf=YxCQ)hmYM{b#z?KY$zkFJYI(aF+j`9~r*NLyY%|U!ZT8k6%a#XT@(z z_~>~k{6+O&zv@q#{VRf3{-Y0A*jSoaa(sC{p!_QruHyNCs|de~P$I|AMd(*DT>Zxh z{3AGP+&KXr@_+I7+Yl1P%i#YCp1^-*Zf(U&MM3}a=CyzPi^nyIv+fu3U%v$ZVZT0t z6XMR<@OuyYv#3Ape}_uwKQp(n=A{A-kGULA@UY*^cXGmig-XO9rjZexD0j}rzsYR! zm(0=s$kx)>%-Wo@iN`jU$9D5yG7BVewz!D72>gmT+p5u9!Aob7*;*~I9#%(4H- z+``7j*qjshFPV4zOXm1rGAI0Dx-){4=*~&<7|+wQPatnM+~~V0+~|)b!;Su}@8o}E zZ)t39ZOlpe70_Zl;ZuL{w|CJ@*vZNMwSb&I%=boc_PKNR|5m!s{{-4!N$PF2EB!?~m4b zg6IDc{NNwn1reM>?wrHFHU59{{w;9ff8lJ+^Y5a6X}tJf8ZY@JaOoeWWf7cmcg_)B zdaDR@{vAd~^#Xah?AV{kl=;iy{~b`A<1w5Qe+DK08vGLtoXQx^$$uI#;hc)$RQ(zG zokIVzEa!9#=gc4an18Yc=WGn8`p@{-e;PM3=A4V+)cg|8KNkrV#iW?Uxe&)RF&w_jfTR^q=!Aeib^x|4I9u14RCl_B-kP ztw{PE&SHO?_bXh)|AY1?J|+G}`{nnN|4I8@21)&m_G>ej{!g0euZlwEZ?s=!jqHEY zes7j?|4IA3TaWQKvgaxF*YkQji$Z)Z0%y87j>?@#9`?(ULWLQ~ymAiW8&2Y8X6yjC zXM*473y^`0rc9;~my-WD4D;3YP;sVH$sL`sxYukP@#=Ym=J!UUV+LQ)E`@Mth_8b{ z*DuVJJsI$P&LHaPP{4h@eei6%4purSgng=ou>GO~c6@xX^#9X;0)O-%_Bfxq{iql2 zwl{+Koo0wVHv!pKcfpq&e-Pm>N7dt0;HAh$&@MfKR%jlDrvnkN%T)+YX?MX#VF%>0 zWD>69#!`C6kHL++5NIzbf-Q%Y@%OSeNZPEAQw<)$^!6tx_@_L`&C{gz+^vUAi?y)v z>t?1~y&mxm-D)^!Ktq=85E`{|M)YTW+$E?@6ejE9wueHvdAlcCUSNV}JNcvUpK@Vc zh#;1IW`aN0_n`Xl8=$GH1F>aKKy&P3bjs2Ix0%Xfhc70i?5iHW>N*P==uhavJ`XT@ z^98DB^AWktd_?e>2I(>z2QRi5W2*B8h`a9r0ofijWquBtv40h+EtkPge7g`;r9tGj z&Z8z>8bpQbmQtGqwu1h3GhF?w9o}WAVo5nQY^vRjY`vnP{nQvTF5??Se-VblL)pmG zuM`z}%!M=!Gu$k!N67m}$X>65>3s2->g;N!-u7-m!OfA#P;3XByq1U*Pc)!=JvWhr z^mlYd{WzMmeGkMI%z&k42IRDwFsVJRjU~O5@!PgCX6hw3=K??(04y zzV{Q*u(CQXtg@x(qX;WQU>(!HnwOrN|*H!@$L zS({|=tCL!oU-~`FSZoP(16xpa!5lDk{fTB9P9eoXmr#|+WRkW{4R6j(MT3+Yek3>v z=Qr__1s8R(BKin#$4aA?U>z)Go{h}im5_b?8j6_op@MV~Qc!aq4sO{8e4G0lRzCAZ zntI;Ms=@DwU)dPT897+e<>0#nzH!A@t2 zQQeY*z|GXfNyTD#ncNg2_j4?Dc*z}jQzn3u-}f-PKN=C;sr)$eYzd=s(}XNpkqbWx zZ=iW4GDJcy4mccR%!swYfho>l{L=t^n_UAt9z8{8R@9)aV%f;9#S}Nri05ivSjq@F zj3x8@o}&i4C&>Aj7J2IUiJBuEht@2(kH+OMfkLHIb_shVp?u>GnBgW$8ZKN#%0E+> zgf+DS+fY%-N3a+FK&|k6ij;M|k%g%QR%#K! z)}eirb9kggZ{A>i2lWCf`^9j=VG&ep{~8v5H^M&T4KylpAX9_OmC%N1}tHM0Nz~>nlX+K1_UD83;q!Yfn>0)=8LiE1+8@j0Y0E`Et5jWZp z?;kHm-ilX&X0;BqMYe!kDPc@YTTuKgo(~l2z(`s%YRp#W-uDq8YR{+P_w_#!-<%E` zfeniQAODGR>vi#z_)*Y%bqTu9?nJL=bfHGH!LI8nKPYI*Ph89xf3u2PtE1jO$5ZwjpR1wZ$twHJD5fbOs(SiV*W!_95q#&ymXs|2A@tL z&2nqdhYST$=lY7;`eZ3&6hB5jSrS-QNDp@jn-gWV3P@hl!`!u!!iBeExwC?$@tjNF z(WOa7;KCNeO9ek%ICa9jmr*F=>ubl!1H#rviBCZmu-aDc;L>6Rz{86TbOyf@4`_>Gzqw-kijugpBsfRbEQ*e0hFm$D;B99miG}0_d z&KH~E@(D>mCGe4zW2R6mI&Y)ZpA^X@KT%ZNbPOq4w<68Fdv=eTuA*&fqaZhs&Ags7 zfcUcHxQp&@LmMXX;_87Re!Em2Mey?0%9e?goP`l&MrpuowJw;(Q6l!<(RO+R-LT}) zIP#@s9q{=^HN5-w1@vN?U_h^tSu6AeU9F+fue0>8C%izHWWHL&1o8>M>^TPH7^Oor z&unD|o=qUnHtOKy4SML#V{shqc#oPr;}&E8MhCt9NHcN@bHQ|`7UBEVU*tWp*Q+3D zjxv51vJo49JB(DVHz0qz*=W~xH4^$x8uz?EhV})Bpc(BHN;tWTDLr3~dIPvf*QjVe2QbMCk|CyDRw!I-HYpeKC0}3xTH!nqZ`yMaCP#b1*)6 zzf{bfbQ@kePQ;=H7wz7T5yI;!SMu7&4g^>2fsw>4be_KjvD^3=8((cwEW3{6IGB^- z4?W5iL_^Dih{{kiRsLlQoV~4zl=AnX z2g8{lV#|jg>F{b2ksADZmkIh(Gzav;G^nlbD{+$aJF5E19A?#PL6XfJLIvmNK|qiZ z8sR$(iB^@UAa*(&6PSQhyL!3TYGR>&J8u=QE#ayjM)Ha7a3XOCE_fDlC7x-*Y^#l2 z$C6rjxJm>Ja+}bjmmis`j3@AY(gvh_RTQ5IO9N$VPg1$xiySX0pi*{CA~_?P%^*u1FItUNSWFZf2d1Tu>1D$Np!vPCFqpiF> z)O&X-ik7N;Z8QXry%c+Fd3+hnco`uNGI|->JD4`XPHsaKQVBEKG6dCM&%T!v3 zlPjS|QPIVvhRh5nk=!D1~#?cOtp;Jj78|L*7Tzn8mZ!;dTDiz}!Cp*&g!v z!5CtvxbiWiHf5ug%k0Qlvka(qL-^CD6V#mQ2X=EfA1Ph4Of>epKT|R89PnK^4Uw+# zgstC;N{i!&WI-D`@?|}mG$PL3q~u7wbN`9tj)hafAMCNs6a!)yV?_eC$0Dr?dor=p zhvD9r!B_MbfXs71-?(RxYX1OwKmH{MeEUJgo!f>M*C~@ZJ6s{ZGM?xfZ6%t4SLzSU zdCu$?RmI2LF9K6hK)qX1z^t?pB9^P?llnn7NDN;9fy`o@oTx-r_THq5?|(r`t*_x} zjv0{AA_y6*Lsl>3NX7DlRE_^W=2c2F+SaH5o0uD#CFn@nGZd^h4X565By(+K2s3pXO5JM+=}Qx!YmEt!mKvp``zHa3 zJr0Je_BYg~)^giVFDAqG*BGyW>8N#i4*p;jhgJ3Sn8!4=8YG` zDAkbgRh)!_juPaD;TX_eW4`rbfZ~U3qvEj^{ z%_S%-d_QV=-^8`rau%IUS7+W#%ZADDgbBVa2}ooGHn^BbHCTGX6lwqoO*je_E@f~j zGnrgm+kj$D4MAC`m7RTB4|>#JjczsNVy&ejWQBhbwat1zY_)HL@RW3>A#f?GWp|?; zQ?=2yyZzJ=3x1q6q<}B;Z1yua9=zr^Aou;>q4A9dx-H#+t~*V`<|kC}xah}lxpfx) zHk6G%cwC~K)+*wz+eJ*@I*bl(=!RwcZXvzsVo-1l#@2Az?&ZOGc#=UjLEH1;WJn2| zPH@F8yT@XmswY&zup}C{AsMV;JbD&pPintEU@Q(V!0Ll9VDGf6NXa9L3cCFj*{6;{ zPU|wjv2X^y(yB`y*xBQKtJHD*dv(f%x4-D7K1OHadm!)qAmv!F9N+FQhDVK-pl6ha zYS!wIp0BFp>%9=FX!!-^p+GN;GyjYd_@kgNcRe!wt_>YwHY8j`2Hq^Ggf*Tf*l_kG z(7e(L)T#5}5%3KiKH!D4mFJ-=Wiv?ATuUZm>pHx1nIo=rsKH#Wn^_Q99%LEI3v1mxkBJzs5iT|oGFaRC*O{t*TrbU~F$8pJ-T zL#tmlF=N7IxnmBifwb%%TzK;=$T!>KZ_zG{@!scPysVqMQ!pKRZDfi2xKQ%sdO4b7 z^NgD7x|0Z1$Kp57I&DK2o@TyHde3|}>V!!BJ^0nS*AR1PGq{gmg+BQeQNm{zLAA0L z*q)L`vx8nk;rIkXp528LlXsxb*M@jY@I7dH9*hs1nS*PZx>=* z0Y@Jpv@joIB{?C|7T<@|PhUshuk#V>d0aGkvH|p5F+ipF4WT7`4d|FnCre_IsHnJO zO#4QHat%}QkvdQE^{N6ss-uO*W;^m~*9^FR#}K+-OXJ9JMYPuR6qV2D;q2+V$wGY> zd@ued8WZ<|3ZJzDtTvZ2EoTByPv~6I{3@lBfLD+u= zX)L11=o1lie#Lp@AgDl;vzL(mH^S)B(INCM_aimNVHM~G>?7*+_Nax~Nj@v`k-<@U zGDAoNfBy6W*}4T`Z6uH(v4^z98KMyMQW?UZrm zij5LW?ZyNXDo1Z=-3)8_~ChaP*R{K#_;Pf{0i=9Jd{ddQaa6$GjYT*l=e9 zg?>=Cmrugk`}&!iF=gaRts5$HSO9(T46|NzBT|{$My4%YjiYK=P?aVgj(0<~YdX-I40$A*JdP}1(#lQTc7dT^oMfCj{cyjdHU8u^ zlN{B^WNcdGh}X#+=3;^_ez;&9@qFWhJor`7_dW9Xd(jj!xZ@6rPJ9FdSe%?$NyFk% zemw56E4fi8N<17FK&Adi1kG`1jQbq))_5H8=vYDC=}bfck27IW|72pKy^;8IisEUl z4wUKf9n|Qm$v7!{1jHh4qOWsv88x?TyA}~0C`;79KUYT){=rIwdU^e&v31~*d5*$9 zeAKke1!Uo}O{jTUGo{2=f}V*O6Wf`6+=%y2(cXm{(XGTZOgU(v-DoRvUQ>c>Us$2j z!LcB-xCL6Xg|P0aRw&_{V0X=Ul#xN2jDNvTP_hmoOI=stbB+d3IqNREH}C+S^3LOn z&w3C&6PC%2_J$kg320DHn9!eCsIgNfE_#nBYrZZBy>3h9cE5m|DMqBtY&nGMOkjFj z`S3315JpJMlni;+LE_44WZk|C_HNE*KJ1{#-b4Ik%aALqr&S1%v_yqs&d?U1O}@YW z#+a0eW4CSEyD#VuGvm1N34mKLJyj-umghkXhHR(Rn*?3 zWK#F|pq<>lI7W3*F>|AN2IKx?6`FNtB0kofi#rrM86k5AG~(h)W@d9i_j)AyT5uZL z*G_^IJw2r0vke>)l}WSuNoG-gBFGG%1i?02T(#;MiuJBRdx$T-7;ud#`8l7E)TNM{ zSBL~{3^7M1o@#sR4XTTT@qvp7zwk(;dPV^)OWO-0J3Z`#6JJoe3Z9Hb-z2PbZ8kMA z)`kp-sBtr%>yS)oNyLr(fr|OG@j<~L{L-Qcn#86NI^-F;XrWH2TYHgjHkNquQGW7X zycBIzZiM1cU+fzvjz2{VGkuvBkf@$cMna3A_}LheqW6V+Mm+-`yO#|?=i{+{vKDz> zyMrqEq=N75Ho$IK(quH+9W-Tq$cWBP>c%@8vPR(zrSyCuvLCaAd`!qgeCCmOs`6NT z>&0|3Q-TZWO1&U+eg z6)5baG|=^LkcV#}T)ACgH^(uJS{^P;cx`cN#vNg@@8nEUXC*{-I84R`Wo4*oqcPg6 za15qQo{br&P`uuADPl91lQ&XhLAvn+>d}^hQ;wzZ;pY`}A;lDhXq^WGy*{{TEl8Zg z&57J+YhqWR4Bl5tD4(I(j9vLOd|>-wTeaO+P}vVXazIxAe;TT%RwfJMPdDW;_lqN( zPyA+g#YrFQ?zYEXqMy;S^XVwdWGlYhq>2uIThXvrb31I$>VlQn1Pj+^Qa4`Eq^?;A zD_9DUw%ca-={H04Luv$`5_O!Htxe*tbaDUc188!q(eZ3S;xJZ-Tv;uH*U(L{qm9A= z=Hq~q8HNs@A>dTp&b%0ofQeSAFz?3~rhcL+Ru=E4HuL)LB_E!Ho$zaTKRp3La?Q!g z6I03V6B|jx9wq#uLK~{5^rJ;zA4AjmIoyR2I)q=(hy;#(&wMf*0(E~YvS{Z3`ozBv z%XzgyrJ^)G8#l9IbVwhUCG19v*BOJdbOKBc>_Fqb?uNi0&5UZ7Dz^4dMn#TFAM4r_*tG~ZmCb?Ob5)>m=WMb$;s`o?)(e+94#O-5NoL&-Es~Lc3q)-n zQgy}7_?6xus%=q1FS2)H!AuFPTRaAdd#2fqmr}$(d`eKsWeqA-M;E6y_aM`h>13I; zEgpA15Kn#CfqZf%V%Qmrr9EO$XY6CDqCyC(&DZB@7SDyzu5=_Pl7V%^%PG%!#dw8L zE>*PP4xIbA4BZYBAep0zl>WIubR_OO+;kX4t7t1qL30<>eo?B57OsuMVor&0mm`65yNb|(3#w#e-8RPeYyo3tymV3O8ZpguTdO zfeQ#;xrjD-nLywDUUEB252k3a+ia$RYb346mz!H!`j8 za)Aj{I4wfES}RbH-Z*6Jo{GdXm*Plko^G#akvB$zc79_oF=lInaV^K5h~{Xa8`L8t zy_11B&0M4vQ30n<2w+)faePdBBDR)M!(yZNm^&iNNOtrN>?J3Kr=7IMH|NjCBv*x4 z?u#cv442Dq@rK)ZW**Sn_klxc6{PrYhw^I)SZ`J*GumAP?{>d}K^HUfHbD$@Z=Zzq zvdby4C^_?2hgv z(<|RHW(yOc`pI0leYMlh_8#y1kmN7~M65*DR6R*PufNr9x0kG5vJIa*I~iTIP$h1I z0pQXvNaR&lgWv-fbcTBi-D&6bfNnfRQaP^RmM{awAJ>K<>4P{z_!>-l;zHad7vpP0 z506&YG4yg_q@!7nHd_L6UGN3=>urJ~!lHI+v@awymmxLdDde7xIF8FUz|Th~5&2i| z8M@a28`n(jdxq684{JZ|DEomgETu(UOUw-l2JAN!f>+cP1d zb}sH7J`J(K`qTzBX;Rl|h0h-mCdW3$Q)jJ2n6&#fX!_Dquxe{PQge6)^F^jI2F@Aq zKKdm!QTj5kE)XWl`G|ClJAqO+O5l*molqL@faTl7aH`ZHcv-`fDOZ;4ZW#l9Ul(%k z;%CfDNnso{AssF+X9=X5AQ630B9m7NuQZHpk9v5b<2P2JV66zo-Qxpw>Fqv9aCL&6 z*;Np06-i41#vX_*iIt2CxVORix|u8 zCFr8=PpVRT0-o3*k5A1$&h$CVCPvSuV9gahFkZqMSKZQw^O^!U&B_MK0#eYyt0|zB zyOGScy@Gc1#4#TQ`SAPnQ*ggt5rVItf;+q(=YcM3();iU!)*D2jKu|j#Myepl& zYu?W&mQ>=)Mmx~L%0A#cIDwLn?xpNw4}qz~Fgji@iUnL+kocR6C5R{p3cLPH`r$&v*l!wgon6YEi+utyF%9Ihe{w;QU|_JX`M@6%aTUk3GDM z6qk$vlg>EY9QYngWdj(mbRjhOI-e1#^&~4Y5IOm1m9>FIF@Baam1(MPM~9%$ZjstS zR4D5Pw=T59`y?&cM<-H66E$%_>sMa?J`&s(m7|t(Ik3QECCTDWvzt^YL3#T3!47UR z$vQU`XO*o+0_tDUuH%MCs9l{{Cwrky2g8}rNr#xVg?o{FjV_T@@8_NvV3?NeMNHAW zF>otz4$S$!1-^_m!egI`Vhh@U9ExRNc;FDacVsppx0+!_VIJ^X%8*z2S5Z!%|Z?AXH}P@DUFtY4B2NpgJP_Z)LewhIBfrI0$9B|y9br18ZHVN%&)OIlS@ zA%CzDR8+*#W)|Rm+8NX#xDPQO^zccUd??&hZMW2Q3z9gdfepm=qZFU*%*JooL_>W5 zY~3Wt!ydUq6syGvr*lsY@(Ra1$p?gH+^<40ig zn{3=uE{HCAN1zRAW66%Wp_qvRv|vpdrEm5W@*^&S>x1$5-kUB6V^z>Q`G+vlkj=C-U(vXL3&ilIpQmcPyy4iXFx#>@Q^@!nQ7U5JEaICkhgXVa!Obd3X6Vsu@+~6C#z<3^MBxgM zK6Z+U+P@yfPFhKBNDJV{vua`8mNhU@MvSP`S|hy)>nVYs&tS`}$>`hI*?94u188~D zDqF3fT9hI%gIOX{&dj%*1ggD-s3B#D@u7^-Zjmd{@F9$_ULR3@>JT?7Sq-Wd7F~+17U<%vcTpg!)Crrzo0%)Mr?9|^0%WmOoZ6mS0ir)HGk(vMv8?%I zJmYjQbl>l_yQ4S?{6+Toc^5xEw|4{4UtUQ$EY2o5ns?9|T8s>I{6yY9BS5C7Li;Qy zeE8NUD3N)GnkR+eG~xuxM>NQwt|vL`*9pQ)_oMH}X_CDtgS*&77H+@G=Y3;8h(2}p zf$FO`;&^`zWAoetN(N%>20r-USsVd;`F;X2el-S845UNbjjPOUI0{==UW7dvdhky0 zAu_hR4=1wsgTvZloWEF$WM5#wUH=J+j~7G26UQT`M{f~xx81Jt(GD~+;|eoTei`>J z|6~-fst3L8%ApQr@WauCEb?A^5#=0{L)>l0(e&wI%-vlI*zXdrhjm3ADb|#LxMDK( zvdIjmtk=R*a{bV^uVD9GClJTo_=3Kz@dxRhCg2?V3VHL*z&q|wAl6G4;D93sU~qQ| z(mvS^9p;0ev(1sw&dr9}(<8`M;SP$`QMbLc;1sB4(`2Ba*X}c4Db(@K=65a%ke|D@ zll5-~kxs=lV)eKS?mw3xkrs*I|Hc5De7Z{si3pHeVdvoO%??H@LJY4bC*g6J1@RH} z#lv#q#QH@DBY!XpS_&o+{uB>N*jb1~NC$#pmN2Q#{|u!uAMN~#b|N+3?I8bcGF~Qn znpwI-l&BkB;hyM-BtOPT+im1Kjq>^CfST4)vTlkx^Uj9>&%4Qxd+{oC$FkTfjYdh; zs}Rfkj*50LldrBqTgo{S?qu3C}Wu1@6sj1O=0dXIz?(vY;=W%QObp$QLOA-t*|4lP~D z)hQojXl=e6T;ljBunQLu~Z;MqX=V;(8j^y2j>_o956b=(ie$I;?hrD(Q{ z1-a{V34&6@h$|Zca#Gis<|U;hWP&~Q?R_d7e7_VceU3xybS=Dr_+!T-kGP7vTkRb5 zg^7;tS-T336%v^J3f-BK1z%EP(H&E7bm~qrrF3mCiXV%P7P=_Ls$E+!B7q7;c} zxGUou;Ei8g^dk1ZZv*i0^-ga@v4&2#u-6@hZmWTjA1n#+xJ~TupJYDD@5SZ|&f&Kw zSP+veC0E~ClLNkKkb1WaYhNkHhk8ax$AmLv{ssl~SCg@dL_Ok{th2>%A?by z4&om=`NS{I8b3=DW4~K(#}iM<(-Imd@R6Httj@{~QiaR0NAwf+c4-?)$h?MiuC%lA zYmhzP)EqXzHjM0>pU>`G+Q%wL-J}D~s?w|C^662zY`VzMnr?Hhz{=10*yStN)8|kk z+sca5=fx@Zsk-K{7Ln3gQ+h;vC4Jo6i`7!7 zqa8Xqyt}Ob`8+&Y!G`LHU`IDYimBklKvT|LXXkj%syL{ z#6C}n#n({~*%9~+JDd~7JzXc+qMc7@p+jnH`g;lc2mNblv80Q%e!y2&@Z}!--DEyJ zUTXsVY|Dc^>ilE@!FcSRIgIQ9ZhNK`UKcZ^JfGMQqLmd#X>gjie_#+Aln>z?OE` zuq~V4lAC*VXveShI7_(!kEt?m9Nea8zxuf#+kSW!edy+9HYzio{0yH-Cg_XV&)D#q zZOfR?#yPE~r_;+=l~2m-=8Y`vx@#Hx;6OiJJGu{-t9Ovwde_*RhlNn9{)}!;>B2z) zM@_Wdz?Pv4o zX4V1MPJcw*W)m5!gCFsuT^01X8(6&6ll^)N(?bVV z(udmf+34wS*@nCes(tg|XN(PoH%U zdBx^Owvq9r&20DjN;37j4trPWG<$d2LiWs)Qo6Nd44bk31RnEw37xI4$WAj%A#UA5 z_IZx!wEsaJI%<6YYks~BJFnn<&*&0sJmtNLwQf5zf6xz za}4PhYHE%1&}&>>HHA&9nb5f0dK>-R>;d_5sE{_BTuJ*nu4jkDRax(OU)eDV^H`TB z9<0BID{XK(g5I>Zmt7GPL}YWTsUHuMsO5PfgtM)eywT?;?aATnv_qA2g32N6x>nHM zeM$rV`nUkk()Ob544T<;`$b@Evkr2W2-rVsieu|Gx6q9GMOxx*KCXRzhwgEX#exTR zvv;0nktBgncq7|{>3ij5+_u%s%7cpJjIRUzbEBaBVBTHyuF;s573l?~;}_U0YZ-cI zDWMZbt?7&d%h&{d1-jLKjQxjsU9@`BF}!M#VB?K%uK47m^F*3@L?+ctq6Ad-$g z-AZaz28sEChi z;3&~PtEE_GTLC+|XeNF=u!#=$Iz{wGgBZ!6mt$?51V$^k=0B@N=dEiL%{D zJOYzgy&z9oy26a!Gi18@G zG>-M7?$H??!`Lj~5l*lAMh8aMvmMtru_rqP$bRG1Y>ozhqsPx~HrdLYjq;VTKeqKI zR%c$&x=WIYAG?FhkDSRa(`sOE+T^gCaV9RkB*40mnQWC>39F&lz;d6}upU#Tu}DWQ zM*MH+zMMC(Q7e?~5r2g|qkGx7{G)W({p4wl!%;QO(Vt4Uv!`f1{dAf+ z#JgX}9%ZR@nXHn_1H3?;kInIUNo<^@@y0-Yx_y2K+wGT3kDR_m4l0$fKNlZnt2W!P z13JsFO20T;A9|eLxokQ8EPoz-NjRLfZCk~T5xfskvEJBYzZ7lKb%DO#UY#{rrbwqOlWjbd?n-0p9kiTZ7JYd2TbwZZl)Y15 z$CmI(v$hM)(&^{C===LKXmPh`?5L`-MU$ z@W%chc#rrkw(mwW-5kAw7U(Xg-AotYRd=Kt&)%8EUij|Ler}eq-zTrf*6{r#BFFFH zmD5Y`b|ooxz=+R2Yfme!{$dI1_H>Zdojiy2>8@pf7u;P#PvY6A zWR3#a<8p=Q@4ic&7~W4?c6+i<4}4)0T8oJH`(Rubc^zju#o_VhgG?OXJ50tnV&e-( zU}~!xZSe3GX*S=^rn|`53!Gj=JKVOR$w%J4(?7nE-}Mr$rTmaJ%Z#TT?Z=SJ;8NB< zuaMk0=FB#g?O>N{3)-jNi^M|r3)pe;%J{X*QgSi5jkd0uN3^Q}SG_c0w<(sB&c{dC zn!|;-edf|f>-6ZuuYZusC+q3+uBYjZkB6}K>I7Q*h$3Bl?G$}> zTN3v4DaS<*zLChy8T4mIWBk0}96cgYLw~w^hMuq2{HyfR&J1?>!eS!wr~?(ee}PAz zn2=YWO6bc=^4RQ`3HU-~4-Rk}!LAn%&>9QO*ohM#k=?kOo|QVpuHKMBE2+HW^)jW| zk&Maqa_)8b?$9R^byrD*qg_~Nn?7=oo!sbC)<*V|o=4|A!>BtBe`-F<18jdC!-mFx9sHl3diE1yrlm*~KSNrxcw zpca4>xoG&wt+m}oW7+mu7eZm-v* zY1EosQf`1x_|}u2nNF;<-#I!uriHkFAI2ZPZzGPX33$Pna(3;3v5hC@9)}sL$J6b> zV_63YCpyF|0ejUrvgXe#p*<;to%vJAUhqI7>!1A-Un%^8ZZu2L(z}ft!)8ab-?uqq z-+LMKg#LS2Q1lr6CVD&0m&&9yzVgm=n`K#}2bJs#dM*9Bnq~Lo$+22D20&n(ex?s+4T1dAcd&9r7(4r?8qH29rynkvXy2XOLQZW^r!8;oAv3e8 z@%_`nboL5KdX~N(%_PmHJEa0>l;XyYdfTxJ>s9DW)vsx_gj?)|c74|2susKAg)#lU zJQmDLo}r$2;FAfNVarBmI~;jC>M3&EMR z!ekCBEVzvJ94%lazo)X^0o`)8R6Wj)c`MJxe7;IJ5~uJuvFCJmk}bQbdw@MO zr-$5*wIbuMEX1F-9>gWRtes59bb5x`JeDyThHCt*ugT2{h~e01{o%i&-%-80EeOXU5H}?g>uMFmpIx? z%+y|gRFq!hA4c<;Euf87D%%HedpdIICieynvbm`dd)lWs#iIrac?H+dYnW;_xPZXnkA%ADg*PqH>M`4 zSCL*jY5W(TfpfCVmc@s0&bBgA-64rDY4iFaGlbC>(NHwM>mJ%@EJ0o+NRqd@Jxs;q zGw7{~IjURq0}bFPCSw`KvHxlZ;OyCC<){6y@VhZt7*USH4pfqN4kz*MY2|oS_c~-g zYeI{J^PujTCuxlvi?=6Lk_X8NcxB^BQtuW=`qv3zb-CFjz_^IydN>gW`8YgboGh+0 z7Dw`z1MvJu&tax)KJH#)gLmyxBnK{jLbpOZ2pds`HyZK2P3J1$1AQwYWRDe2`IL;m zzRJV0BcdesFh642ipgP1Phv5Wga;nw!V?>Qvdiu&@Rb*G-S33q`1WjE)yKl6=@;P` zwHaUi`~zY|^theUCg{@na3W^Cm2B00fI>HpLZfOiI+iF!VuQ-KvYRqU{}Xc}Iwl?O z(l;j84OECr@B$*?W`U3Syaut^3(0{yN#vz*Js3&vB-Wj~(Cl4PQR84BdfWbiioR%y zG(sx~sGQ=}VHz;bx*oL(Na4Dh*2L<kN6}<8!lc*z`1l1mOFHYO75s8lRuus zkvDSjdyPnPYu{edY8j3ruf<`$C05*%zFV>FDQ6sIxe6Z|HNbN!G_m_j4g8kbhhC2C zCeo9<3GL}dOwN@4f2^JPH;4(t8{_nx)RS?m08o^{qa``P=mKkxVZwaH_{ zw-CJUDXtSTB=p5*5+xNyEet2(I!zuCj&LPHACHpwu0!NeP#OteWI?@7^+LaA4DnM? zpw-JfNbKyRH1rhL%j>z1y#EQm;2z8b8G1=wl8C95!Hz49*t>5EeHX4xj;nM1mrxV3 zyi1*&d&6Saa51>6IO4LFuWZ~>6Z+tp427jiB;(#8AhofW@?M)pztE%kDR02yA7L|N zKB2GMXP7XSfUyk|=;_}&Wcel?lD%yqlkBZe1Jm8v=)#A%)Nvx|WU5%Vy6fN>Vgo6@ zufew14rt&YRC7A|!42l*Tw^;-owEk_`iT(b*Rj}@&*@Q`@nxaRq_q9XZi5v)d;mIVft%cSwQSwVS5wu!P z;f7f{MELu4aO59{k-7~iUYCRk^ZqizvG+i?U5aE>TOu!Ol;Pbfv42viPKL!!Lc6sj z<;mOO_~H5JS*XHXZm(dx+GgN+9}%LXa}EAiLNIWWq083QGwvqGAXV0! zjvVO4>NoLlIkz08PZRh(Vg}ibEdsfUxh$_TpMT-F8A*D+gtb?0f;7oawrS)MbbObi z6J^w4FhG(?zbZ**FVv;SEah;`hH3PPyE^eZQiv066zJ%Y3gqQv3fyh=P^dti+|P-J z$vjbV)XSJKyT)NzzA74y6hjCvj3xFTfTxzvh=v};O<(otz?Bp{FRxARDX7q@=~@h& zG$wJfqab(hAmcfCB28pkaeJCE(Y!s5D2pD0z=^^jG7yAbn`}V0Lzp_A8pc^SbcwKB zC!`GR!OaI2V}Qv|c-CS_%TipRdJG|Qu@c=?cNkt|kE8k9L}`z&6gpZ*^Tm9kVIrq9 zocs6>o`5{HfBgWGZpXpxRVUE!mO16s#gv!mKVbZF`21fPq&eE+ z<${0g4yF4z+ABpZ7Q~|6-f^_*w+_p;sld+_vSecP9$Z)GhoKQ$;bHY}j9!(1Rn2?& zJ(aqcz4;wGXi~~d7`lX&_w=cpTnbES&}373~A!P}1(q|8#8DtG2pm4h>|Ab0#_ckt zat{P}(^HN-EXsg0i5ir*aVC3e!WK}wUkb&-b#P@pml4zbhn{VdSnHG5G0;#H0tJ^@ zy^`mkqw|~}{I7@cdSFH!_e*iP4J)$QRsn{D3SjWmBK*W&s;=%;}vfbFyfV%c0E>CD}^PSc~;-?6BWb*zYDziaK}09N8Sqi!h=?L3g=a z{2W}QszaPi?*p;8j&Gf%sPLu|m@@W_%b=Y_3(p~p<#Os(BhH|pe+EJ}gfrK-n9wpK zajd>`0_&pv@%g1TEVc4L`M27{A#O66e^8gqD^sEq-VLGZni5vhA`^<5l<=Ld2rcWG zgLUQ&7|fTXj_vEwOV|+~rx?>IzUkn2P?pAyC1U6&b5!?Kr&_az(B@VWW{)moeM`qt zN2y;-fZ8i~c}j~k*+s*Zn;oEf`VG$c@EC6FHzAAT%fP<)6C_w#f@)R`Q*KwziYc4Y zS@YxYbHOB)a-T>$ zKd)yZ#lExca(d*Aj|m-;HK36S3;C)Qp7^Wt9sJ;Qz=Z=*Skb3OL>`*M0m*MLsML)& zM5d730aIdj{XEvqm`F2@Nzu-?g{VDXLiB3;v3<1#Y`(V+opVEqI_mFWE=HRY_PjBc$t%(X zHig}jxf$h$4T;C6b13ZJ4Xz%?SU=ZEbgjHTnf|K)?S?O5q)(+iZ=V@sr6*5D9b1_T zVH(6!tDn(PD`4_o<)ERn4DDF!2fRx<^y_*#(sU#pR!UfqI|t>6#pn!(PRRgX;!=U( z@+$C4kfoohr7+3Q4VM^;(bn}sq->1{#KjbV#oHLBa@=`{?Kp}{Jno{V=mb*sWewD( z=i;TElgRT1LmKsUB8eOP0xzs9IFH3~Y*F-wG~0=Er>_jn`YuLzK7RHFGpA8@-AwrE zdlK_d1@`lei0wIV)X6Sq1J~(bw_ZK?y)c9ui#5ohOL(6cx;DnrUTb*vAJgy`7eN2t6k2j^#xoeQeoM7K4973HXIgHlHIEWOI;j4z(W8jpj zG`&QRnvH4zvvE6)yQxgqMYNz}_$0y3gOVhq`ydm!;xRu~c`AB&zd`%^>Xc{VYM1qRzpSI6-(S`P-I&vk&Nzq#duAo@6;H_i8CKt*Hii*DqA&-*1HEcT$|U zY7lxKa`VEy3N*X75Uy`X0^S^f-QXb}z2N&2C5^vBYGV!_YyS_c%)|r%I^{U*vIVa9 zeS|fJw(#l`_q((mfv1trAYN06BuG!An+hv%O`-*t+u(R27QuXlua}W`;FFBvLF0x2a#lvVGZTZvBYU?RHeo{qPoVw_V2Q!G2^G z@X`P1VSF6RWw2K~0H1AY)O?yYiTIYy1|}-ti(+favvCnr<=+%^hGYx0JpX}Sh8Hf5 zL+pIo#AZi67euCQXK##5B~5BmsN6eLhZ(SzS#pR`j*O*eZ6s~5Ul83zh zwGeb}4;r?Z(0s+4p!cT-qPaXmruR=6dd#i$fnfq)&d>B>sF2N>c@vg7#j)2m7?VTO zUgPk;)yT`R;-ir@Nt5nnhV33;?*v&ATd)PD#irrm_tqpyUKb<#n{bf*!G5hUA-sGS zMpCr`{^aY^o0$^iu;C|Ma#xvnF1g6=ihK?~s#U1+Pa)d6#)A5RG|h|UVUX5(^tRKc zog>yTDd0VR6dOlXB_z?!trGldUt;JXGrZ;XA8vo3LErrOiG#*Uz+5;1u06AG#-TcB zPSK@>en!N#H2_p!nSocJDMmi+WEQB)(O2;XRPsqW?C5=nzvOSDWKkg`GxCI2H=E7Y z4u{4G(}1UE!?&5HNC#_@SRIe$%&rF&(B>YGzBk9w;3w;#v)G6U-Za2Q9F>Mfg=F~c zcmUi2g{Z|bb*SoSVy;Wx0+mq>5;1p>Ev3L{DG1SqV^&n$MUQ++RfQ2B9$iv0nI@!O zN4ZLE;7#Ze>`YOn@w{sMGPjTOBYr^F(3_0m_B?pDOqP67SPvWK>Jr`l=~Q_14Ggo^ zoGmksgyl)`*`XU-TKUWEOGYaf8+oHfF=LQDWSYX_S0%(((O0Vzz1Oa#QK`Uwu za*PjQ&fX4?UAP53CvqA~ln5F1&SB%GO`{`Yg%I$d7@|g|kfc>nu=}|=oh^D7dupU{ z%eXp>{_hiha$gFWJAOcoRW$slS0M{Z>XFxDAQ;rCU>#r069~)Q#zjXh$UG@A@@(ZT z6!q68>E5lFxH1LqGoPS5!v>QTjmW<3zp*jO9D`rJVJ@GZNG*+W;PclET;XUyzBFCO ztGNOcX&Ay#9nSx})SSMYDND+WGVoV2S7VqP)9gQ1bp4hgJRN97(#2lmj17sC#F+Z>qX6EIm{obb*B!c+$_T6c}x8#`|R=3gM@bG(^5yYz_e-AN?m3Xh5Q z%Y-uferB2=YU@KYzBg<|PLTYcI!4d&} zk~c-SQ3+a^Do@*9mtbYsOK@~d=F4Wv(<3&|KwlwonsVI^d5yy&=9$EXRB@ zkS8yzCy+qdR)*It0ea6;kjHbe&!~KdZz{y-RPhP4qij4>e>)8!vqG79S2OUhZ!SDu zC4qsqJ%Vb-TF_M2B6*)o!AM1cUR@JvPaH;sges*-2- z6iN1kzgU%4$Sm&ahabL5bk+MuSY#$l4ffu{zdb$BU2zB8jg*jQ_k%w);~RK-w6V>d z!c^j>B5{nAf;R@^=*w19S_k8~+Vv^cHk;GNck7wv7%A%H*9v{hu7Jwvhgh@sHKsYK zLX?F%Imq#?Mz$BgnI+@N!C^6CXUw9nrWE}(+=|&&lUW~$PQ2NsLNiu`LC2s9eJlPI zJAY^Lv-fb<{;kFi6mVQ8c7UDItwjB=jwd@@vr+q@EDinT13AKXu+UPHyezRG=A+Yz zb+-=w*HMddZSxu4%CSne+YCkxN|^TQQY`ZOj4=tybZFl(*kAS(&vE(Olnrv!YJ)Cy zaMmL?rH-MR$syPyTL^m6zhTWob%@>_(i_<{B}1PQKfc;Dv_G)Ie(ZO#(J zwY`ktos+5JBMVw|wi`#Y^O-k;zd=~J4c`wsV_H}aEAD4bBkVu2C7&meaMcf3 zxJHz?Cai~|B@-baxEVSmML_LIDYV-dP-cu{g`H5N6IfFcDbZe8pYQ}Di&wHt)N5Sd zEJJk)ji|rWGn}=P^GYo|C(t`P0ta<|V|m_Au-z1nBkj3pwrdvSPWaxhkL5vF}LpYncM@Ng@oblxXDk*5yqed5xPR< zAu2yV4Z{8z@cDEv82s(WB}Td=WZZIeEziPFo1($}qc2!oT*@1tfj>?CCq~;U&vdxn8Wc^}Vq{4BPmjsDXu%^sw8LD@%5F0l;Gnc-K z(8%*q{NIc8VNPfk9BqAq%;IE7(zIqv^@G^+$|iJdnlWv>ew;}qy3~697pzIp2cMh9 z)JV;Ml)jOJp0Q*c(2^qy?oK5_idS&shBMG3)WPjlSPD5f zk@MoEJr#n%E3#BcQkMijTgP#p-hi)04-7w>izb|ZomV!!LM~#M)p>iLzmX}1Hp7Wz z>8r_ft*ZuosGkilr-;yNnN}qFILBQWb7q#B8Pk5wclx=Z72GQRu=P z2`Pkq5?q~IO4!xn+C<(wKDPo-aus-WP^_P! zNn+hJ(3T&|j-Rf9ms~!e`!|4wqZ(xY6f<(fONS0|xtA~DEm(Z(CU_=aXS&~AfyT-I zv5p&xaYckYkyA)uCDW~G@#iVj^HeKyljHhTJwD9n%1z&gl#~ZZu~8{ZypJ5~*zGqYgo*{A>0y6VGuPq6Mx251B7r<4Kax zPFC10ljCt%(ej{ZR>#SbsR`0Tx9|5cYnu#7ey2tcCdd*||541SnFO1+n?tg#C9#NC zp}bRj?A!{BX|28}si}E~KYxgjKi?EdLdimIED96bs0220um{&FC~X!J zqL;>9z`6`qnCfu}jy($mPrVsPX6jJA0tI^C=@xidSYWtiubyY*p1B#+$x9R+n+-x+cx8>SbHBd^->pkb*lQIof#ie}bCxw`@_R`)TFm3gRN z#W5fsaek(mkMYHsQg&s7C0U@WNlNxiCqF@fR$BBT8y?_sdn z666^O?8pC1U_87tAV>TS#Mzw1?Dug3xxfI-asC4{Jg&gqTt)Kx-81~V;vH=5&4=y} z|3SR@6dDm`17_RJXs4ALBrKK2%We)hsxggz{W+CX72HOTy2n7||3c)Kv&>U0ft)Az zk*p0v{fbk#mc57>;>%!7>L5lA5ay$b2npU+i^3QBA#HXME8~)l#mj3UqOh8=JmrTX z2h{1fKQg3ORh169tYL+s^09SuJvIzAV0Moz2IkI2y}#Aq>LgFI2ix)P(R{S^uV6Zc zM^Jf*9NpcnL|!l-+2_X;srR2Th_IF5a)@nMrMnH6eBl@v`P$@f;}q%@77EhmPhwx! z9|)0ZL6?&ka83Ua_+WS)^Na%E_1$q~XXPby4PFGDV|{FD*gbf#W+GXfRt=F$Gz3G5qT&tZ#7R;~3YgretW{Vm|PO%b{mdjGD8xeTN zO`h~O$k6DS3iReu9vOeR3xD_eK&GxHZQ7lISw$0wXTAjF#o1u+hrf*DHWLUq`wTYr zn9zR-mZZ`97JE6w9abhhMQs}{do-X!nbs5t9R1Ds>}kXEKcwk&vq_}MVi;WO`WY=Q ztLpyO87|LiW7G5&L3zP_Ca3Evn0Q`6Cg?aArbolMX}QRLm`=xY?`xWkQJhSm8SG_D z^=g2Uea57$?>H=%rkMTv5xdT*4K=DIX~s@TOew5|*>^tRkE_Nc&+HgRmds>NSBsIP zhEUcx@C@V2arfd67?H{DC0NgsAR7LAu&2)*dDasd@gOy-v1dQrb5JFfiNEl&HMB6XtD?7+Ek3#)o7 zS^LjdVD6bwoYY@{I-8={u(dZZWYmhPPL!a{z1PvQE(|TrHsOc$lc~Ms7YO}N3GJ_R zLe)`k#?f>!p$p_R)?AnEx;pYSEtZu=$1&ZWh*i~4z@(@Vx@P(>#ax{J50y^I0G_Ih{@M6_9 z%=*dIZ5t)1sz?^os3-)fR$MMT*b8hnaQm02dVy%9B^4{k1i7?_xUYISsDl|jx2+I1 zubl)RTVA4$@ex6v-yvLf=``cpSPFIX#?f&+Et1;B)sq@u!E=yBH&JzJ*Bg&KpS@70 z62s2DZAQtp0T}Qyru)Z)XynHy?B*RZu%SwsbUQ3VXJJ=(={kjoZ5%~Mnb-QXIejXL37tyj)crtlE(0A}E{5Y79)v`PFvk|` zMAyj!%z@K?kbSI*&R|I){%aVK9XZFZqgQvOn;ZQV|I0>ye0ZTew+cNPMhJNka8S zHdGdH@a{V1K+HWHYcV3jUn?+hKc8jxxq)}MGTjmJ0iLb8f-}~Kq4QJ)`Y%?QtoAV> zyDlBZZ=xEcr`HzQLzYxcI1Pqx>A{1l$;2W{iPf%v*Z^^i=x6crzbv zC7p%uT+e??q&4YNIth_4zgM=YSrVR8gWW$XDdN1|7+04V(XIveLE-9FE)Qu**Quz| zf;Llf+f9n-WNu+cj78`-B~2oIX*=#-R*1h(PJrx_NiaV4F6$Y7hn=(MCB7ChqJ{}g z@XPNPR-L-cT6;-=#kMsV5u;E0QesiTOO{sO{)0Q)Ea@_xKDM?@lUg5>1dWZeVAkp^ zJn(HgElS%9&vd7f`!c_|`}2q30=IwsCwms>4QrF3seAG922-En;1e!Rx1Oy+zNsj!lB!`3jvvCr^)F%B$Th}X^cyY>eF+xH=%U59OY%i*tsFPyvj9@J-ZYu2=5 zOfQ#NF4l~ME@Dg{`rd-b{}_IR&LHa;ZpzjTe_}bcfVxf-!KqJgLgSO|OjVp3%iFP* zzaY<$=*W!6R81wiZq_Jx95klGw?3lEr%5E%GaBw+*P>O1`ux(_u29!@iaE*gjsMP7 zqCNYTL7~DA)Ojb$aZA6$*g|#kOiGRL*3W`qS5f47|6(JfZ?L1UmO)itJwM$}nB?|r z(0b8}=)LDARM~a2wZ{zT!@deIek@NNH;U81mHXHY!pRu1KL8d~Dv%@J?|_K97LMge za{Z#|82PW=e%^p030W`29$54cy3Dx!+IlJGL7fx{Xuc16=1*aHk~OiI84sU4y)h{I zJG?IA_EzOrv2<$()2Wmxh+Lt8bG^h!M^hBXzR+d<^Z9|YlexaFw>8lk$b-$lMexP@ zi_Db#*RV5)(|SD?!s)wKR7{}&ro0YASwDm|k*9FBR2F)T+Cs%CJ>vMkTmb(+ULv`y z$^Xmq|HpUuf4Ii~$pi6%Z3Vr0au|GE3zmcou;Fdupqx07?lqqcwqsxL_52a$?4sLX zuyH@kp0QU@oMB8HLNWwa&jQH3hbvIz&^o5CsS#{hBM`NXZS zsegUQz-l3O?(bBVvDBv{_69V#*n^fW0vhW&na;V;&1_zoj)C2suvTggIdkCzs!d1* z?N4uUuWr784C^s%hwGq$M55`9C_Fjj47R0?)P8gkdAFhj>Rufus~Zw=zV&Z*QR_VX zec~wl@8AuHFe=4|lW*gaJS95tR}4b>U!mHZR;c>sOZ;qJqJxht;l?riozD>agn#hB zWHD8D$-&vfvoPzDHM~gM2{}RC?2U!fq3GczDxw{LI`8#ireh^O?M@=QGondllLJPm z-)0u;EMg-k_>;E_HiF16YpmK7gVS!Y*przE1G9F)p5nJ`)_7suJ~s(_U3p~Hl%3?K zo)hcb{fJfeoE5-9RMkd~ZgWpy?sgl} znRCN^k*usVkdeEoq|3UryIQHVU!}O^CESLzU zWPMRFPR>%I;ZE})qUJn)ynF+t>SoYHCl)rmOanWKeSFX7GIXYBB2hI`BzJGkq3OGX zsOIq?99WozGxq9I;f*RJJ|vnfxt@h=q&*dL65x@y)o}Y?0K^rHC$}?01S`l)ScbE3 z*QP!6QHu%eZgnI(=TO+r>6|CDGU3m1O*X$n29|GAp|{`Y(_@3(VB6_O)?C|)BXOVk zLT#_`ZdW&ipPfLOEpy?Zs4}yxV=stry-(g6X5#6s#pGLwIT?BM0rMBi!ML}<=o{}u zpPbo^8#6@VjDr|n4NRjut)5`#wh8o3+F3l1w;2nKG9lrTIIYbfxU;Ggo<;p(w|!ZR zQT2;SW1;|$Rfn;Lb8{i;j1?YoD`4eMMUf`aPQlqLJ?ubBK69w507AsBf_KDZ{ODK) zEoCM&eBU9`QYb@Kt*!x=ldjCMtB0v-U>(j((O?a2)o3K=S@6Gf2lA70VDiveVl?$B z$YhOz=koKoQ{c=r&NC#At3_!2!VqSek1Mpjc0$qfPAH}rM#I#!@Q<@Lo%Q=4D-kh; zD$o7QIQZw`Wy>GT=P3qY(f%DBgtNdWHW)*=n&s%eGt}s?BYc>gK+k8{q59T%yc2Il zc=J=Be!412G5yNE`T88jE~he{vt?=SuQTwBw85cWqiA2em-#cW4-%Y(h>qTUrZ2LA zj`!Why2QK%_uJCM?o$^0EPsp|dA6|li!My9{ed?7H85fNRy;EQ0#4Rw<?t)K$eBkho7I=Sl5i=IN8+Z4p(!3K*=yF4ZXmIt} z_rDTk&(TMCWzk3GlEPJzANL%plPVZd+d;@pRcGekV`$PJE(f$I3zXzOz=p=N@UUbV zzA%^#?mH?lm&>cp48M=xkH3XO2@Y^hGnPh8>4o&G7qMcZ1D=un59U1kjvD+p$d-*E zBVzi*Zu3HTxBDr>p7W)s=I`2V3p;+Al27J!@b~%@nls0S{>$ou|Gvi4a)oo`L%)FCB777y7M0?Lnr}?W zrBA4BWe!CVK>|1J@BD*G3`zGKW^US9(bz@*v8a!(!McmMc+^Ml+$ zOxBbZI937HF9C(Ld5|;@$eV!64Eu9)Wm4@LlB*L>8-`eDugy0{T4iX zImWyVzl$XnQq-JvB4?I3(TSYT@baN4cv7|)Vhqf2-4B*+_v85P701Y_6c0AxZ$D&u z5{|(-mxld|faPJQD&NmSNV{>AD7u(~$npp1CZSC7re45zLn-XfEmm-IumeKpn!lgJmGwE>2(4u)~tiIu2*2W@(osG%?mvKWfI%hR)f)w zxvcEQrvf4E)9Ca(6Ib#}n1R|H6m2cS$>wWukLz7V_~1Xdr4bH^>Sh$5UBp#E*YNl6 zVc6GkguE^?rZJ+@q;YpR{qry%4n29q?9D&Nb~$`xG+&s~riJHluS5-)v;sZZcO7q6 zXfwyJ`xANDCv34*CR}^6g7OZY!}te_NlnmZyj;5iKCZmPsE6+&{5kpHztoUfL`CVN zg$pXX0^Txn-YCJNq3>vw&oWAH))BDfI1)Px=t?V1(BfAz?}Se<{`-^Z0H?J~+Pw?t z8)0hot_@arX2B7T9h`T%9q)Jwkz4aiIIVvYV^-s1?=g2bI&{6n3&)O=E#Ad;9PJm5 zo)RUBS6-k&&n=K%eiZBb8u4~}8ffc|-R8PcZO(8CQUanZY2>TL87^|!d;%=i)qaR^!oL zgyVFqKZFjauE8m*+h{&11zi1y*-;l?x`98!7+>EAk01l<&YBRN>ZSCUiXG~5|9g+{ zY4Cbj49jnN&<{p$(JI1|nCZGv_ms~ll79-ipY~#qQ8eDXB}Mb@B*PzPfjk0^6%PTa8V?)I>wRmik`>GX}UR{!oI3lGPh<2hE-m}$6sdCN#^ruSzbHnwIwr~hw~xXONbiVd*So& z1*F5J8FRe^;8t%Czx4Lp30>LYs8%_ob0< z&6!zq7Qr=n$;u@47`pm`Ciu+Gf-$v59P4c4H)pOuA?tbwT$l*Qp0`1M_C4_Yxq&F= zT?9MgK=1H$aQl}ge0#YC|5=s5>ob!%j_`SIUR%!f0CHib?Rbbdae~oKd=KkmE|RkG zlJxewbGRfq4?W*_Gwj$hVrG2`4!0`7($^#K^|cUnQ5pt?=3?BwMV5-xR)YBkKQMkD z3CqfjX@h1ku5J{k2?bGD=7sFsqz;S}xj_F&e1oUW33MPJ9<1}9vMUd+C4mJSNcMj= zWN5!4xrH3Fso#NY9aq9UnX!>Q#y;WZWDn9^bO>bnH^Px+eYo<3CmWX=0}fJrI(17w zrlbql3o|d^eYs*bqdS1UdAy#?buDELB|q}VqE9h9#_tA+k8g2dP9KyW=)!-O4im@P zy*SR~2HZ&*fDcP#(RA^8d{eU;U!Gn^3qNq$_V^I0f9V=@L=>{uc;P4!=SUUL|Kl%p z$b>5|#i@6A6iW75LT&RF95mA-Mbq6eXI%m{Xj%*Zc6!nIE(Pc?{x#eRAH?EeZCG;T z6Mw|e8R+w5d=lk^r{1lB{O<E0_8`5*I_t_J6PM{)vbE*NDR#m8dcN)Hgw?ew~Y83t< zNgdK-QAM(VyckhIEZ)S({JOwIw4Py1=ZF*WultGk#Q+dXHUUp@FSaeclu<}NLMpZ7 z;onnvrb#{?JO*XRON9PfAp z37!^YhEEY=Bkw{lv1Vkj1W~L(0?p@TlAoLdyG}-(uCWo|l|&=BH<&^9*+)@L7e!pQ z#|O3ig@{qa88&Fb9oA4Qj`Y;lLD!WW=6atg{*K(jM6cB(Gq#A*6Ec!m;+=?7Io^E` zU4<9I=P>V%F?b|Z9dFH;hO1BVG4dLLSI_w*N8OZNdp@2_`<}znzrPqui5u|TBZB@A z+<@|xBao_mTwro1756ucpqsu272V5YRkn)}jgXaiCdQG>*xQ10rMeh{M+`b_a3Qe+ zy?_hPb8OXa$WG>wnv+owATyQLay{v{`rl!AMhx@k;%e&ZW59j38pc-qWHpMuV5j1L ztbg!0V((GI-t`nF=^-nj@!tkIaft}y@{K#sWbQxMdrJe*W*rITc=5qu<;?hEMcN-0 z2FdeE;l!UMSUyjMPBoUsmT!@``3lFpza>u^rcZ~g+YF6JTu1j+-e6nKo0%Y*SMJxPI0rA|*c5S27D%44i9a}O)j5O^LHj)cTJT(*Plc+E6s7| z{h8$ZacgXU8U`sM!gRv6<5>Oh7OZ;O0zrEO?5YC|g4FvmBxr2`3XPm2Im*JM#e@fi z?H%|zcqNSJVc7b85X!Ap`5m%>=&N%b z?zy^Rn+YOP{HYgfz!D{?YrrO>ybgx%6{Ad?r_KZ1`cv=R{2S)koL0nHZ-i=P!cnVVP z2vPDQA=;51jJx=LUrwr{UM(7zXXB2ER_r{V+wgBPVLp8>hQA#b zprWQ%5VY|uyIH`U6R?ZZpZ)ErlP?@)@NFbqI*d1l)TM zTJkkuO8-1!{ND>yo%a~`ty}^&XU&Pn#yyyGV+M(Gm`^MG&w&?SVyenC=tQ4~EZglt zpBbdVA$u9}$MhSj%&*~VoXKXJ1xLxhwVK$1-`L-ck|b*WbFhpYPaOPgKu<=y+}`yX zdtgRAl<9oK^0^zB5Enh_7EIYk$<>wTT94!Dqhn}U8VZXea==e<2AP*73%3U(m=h~< z$(oy-zI$^MW`DT_eVJ`oyeFHHR`Q2N^VjTk6=~wAIh}sm$Z>+dUSd8pZKbla0_gkT z+wi7q1Kq2pNO>v)@U&E$JoU?h1@p3St6wFndVe|XS7)g37iE5AL?7GTEkk}s<+1bD zon)1-=@6~D61ZUV9xu#`p)o3^sIk+Od|n{KZn`5v=e|8`|Dg5)Xub8tukL-|5SI=y zYtxZkxriLOUsiePm@a*??HRP~QzPr#!w^>XLcO#XrE24`E`>+-zR_bsCmIN*cNlZp zj5Bm@eh3c9-gqJXA+yqIDfxKzprHG79J5x%gE>C_GoJW0ht>Qro{9yg;)lyzz9%Gr zXg-ib>n0~UrDO?N{VJ3!T`xh+9fDXHj&prX*aR85Dt4UYSLTT!%ls9-0=2H%%-#_N zk{pdpX?-jNq+i33#%)|(mWdMDqV!cmJ+9ZCj`d?K9hkVz?7Lj9eEsp}ad;^Cl1p6%{L z&tr~6Zr=uad8$92lviM-@I9XVS`S5OoHkcqC$-G?VYgoVGsCh>PLPxp&vSC!1ufHaE<1$vj-AL zMPPGfnaOK5uOXe&fUD8LWFu9yRU!emL!rh)1V%UB#@?;ov@*V$ZR)%YV`(B}(5`^Y z&%cLnKD#rQXNqv1_{lW0^b1@6Q3C_E_d#EXB0K%*PI6#yCTWm~fV|#Rs#qgNc5cz9 zM^vssh3ZPCaZLhfKV3mSc%2~+ei-1LUPqjAS^$#5yFfhfJlcC)U}mpzr=KJw>8|YS zU@CioX|nr{CUqkV`c?auzndA6;vVfpo z2IT@jz}~G}a7+29ZPv~u=oC_i7C()M|1x9h6uAhl?9yV}RwC|DV(1+r%=O%q@#eE% zu&`f-T2<%qyFA}OQd%*#_4ut$Umdk1M`b_2j(3Plhtff;Y9e*`os8QTU%;PdSCA|>WAfmy2z~ZZk`(;a zr8RVjdD>wC<~2oZW}7rQCEbD-6jE^CG9KvE=+V$a_u-I)0!cHDCYEP(K|3Ilh>b{- z{c+-iJ>o#L%umyP*Indobsx7TM!})x@x(pg4jk8e%x-)lLT2yX$5O7Y-(PSO!h`p* zK1%WMujm`ooG{9K(;H6%3oZ!USL>0V_TnThx&oB=VXT(QS+tTjW6v2bBAWXq!(^KV z;6^{ql$weUtbXJBx@&k()BujOMuKU+E9p7352KEKXOdNxa(ka{SQxyPJZL_I#Tuu$ zoS6!3F`q;REFW`h$aie6+dE8;`hi=6!oW7mjb1-tLc3FJh@gBCZkd>kbGw zuAn(YPx1u{Za>Bx?#xH?C4RJ$ISfCIM2JzBHOxy%VUIqaJ#%056vV_<+*PB1vo)8H zeGw0srrpQD{GI__#c{pWmK|o_a&z>7Ux(oV%q4rzsl$`GrO>_F1%KJO&|e1MFmdT* z`tsOHdVg#keJi;UtFsTH$=^u)eDx~Y#0}w_7wd4)Kpy|oY{26dzd_CaIFZhcAs6G@ zG3KW_JN3dOZ2zN8Bm#Virr$7D+_fTdaRA@af|#>+w8=()DX!MGf@7cg5Ul)#ow4x< zbGCMr@l`B<-R;R(AJN9()lgWpRi19`lqPBienPl&HwjqY0F@^u(c-)_}O`vkPW%#Kgh9K$}z;)kS^SG413g8z`A2k z?2fq@LuTCpTzBa+y|+9bU72;6gsUMos0=E}44D_%fO#Ufg^ay{YuJJNn9&QHRh zK_4&HrE}gpkoaY_jN%spL;EX$_w6KGAZfrXJ?l$NMUIe7HBM+gO^GkMay5OhDF@}G zN?^S2WYB&r4Zpr@B8NAAh8nHO#ADhee3O&Sta@xi%ojSL^V8ihuyQedChtL?@3J7q z3J0O>g%b>XmE>P}{~Xd!)v${F5v+gZS8%x{U`9j#!N4{2KkIQc0Vlv-Gnwk zk=e}X&k#5)HJuio*P%^c<(ckl$1z;{3m#Z-j%i#W#BST|h1REqh~zna=J=31+-Z}g zM{S}Yb-`>w*qUs9?Vg`ZS^i7bX3I7F`{q2ztq8{aG6v#&6=CMlzu<6WBR%WGV(CRY zvfcg?qz*-qJ8Jh>t!`7&de4mD=NwE+|A@7mr*~`YT7k)`1kPjr4INwv*`Opw)kS(L zA0(ooPYZG&QkF1`2wZw53z8MIooXM0pV5!_-Oe=;wzC48Trl3 z2?Gl@#@2(Fw9J5rH#4dC@5^BMEE(pkkf-0~Y-BATF`%t@54MeMgV4%xWdGp`u;O@Y zZ{s;m^ZS4BqH7-Bt6G5T>Qy12|188zSB0J2_#QoZRN#NI733RNQ|*|Gc=>w_)fEvU zS1J)t#lM018>-pE4-;4`FID2F{RwqUir`_xORRmAgLim`*vmIxVqdN_Xd6ed((dQ5 zzo1c2;OfSXdwm1nD}R84?K_a~la0EXSJ~OtE@*uGB`nqWher%pNUj`5^Ri!|m+1l4 z&?*RJ!UHhmoDP2fy%8@R{f`;#oJ>`7AFvNX%y5!-3cGAo0{`MvF(MjKftgn{h|0@L zFtx~&s(+U!OF}I{sc19QOC{k@=w^EN)je2OXNJeC7Q(OScc}VNm+atl><obs9BF%He&l9X*c=A$)-=ZZA%U{m&A{q&1A*plYQb zc?CPI&ST*0f1D=v4sN)QCy|A^bbgd9=qHYWQo}8>{lrqB64!~|2-jqF@q)|u8<-o* zgy@}%s%-sS7Ge)L)6P31IB{+rKW)Qo@@iHw({p4dIhj_E53Wez)x-O77yFyN+dc!O zx4efxNAjTL+#%xpITJP=s=%J}`{??pC73c&jAH-IC0h?(z*e=>;G@)l1&0>GtxwPK zCGCQJ*W%ccbSWC&wFiSrZ}FcFi4&{;UN9pm@0s~(0pwfD572H?pe>_GXxKatnQd*% zx&a&7FXTWH8;!{0&wA7&VH~ZIpGh8R+{8~S<%v`KRZz6N&072}fLC12H-B>&7OF&% z^nzUU*!~|Ab3=nry;=x*8IQR&I$%#nx6Q3IS4Hd^%s5?d6NtBuP{&D3HE>Hxc6e5*S}{jb5!dnXg)tk z1aZyK^-~udKgY7`KlH;t8Kxq%b~~Idwx>`#g`8V-2#u~y0-yA7Hc#~*b{@IL*T)FP zj?=0p9ra=UW=|!xI#DFUZ6>?ofClq;E4G|Nn*~bzmnG?@) z@FAl@GIY4JCTB#^9N7^Zmuf{TCN@R$;n6-=TJ ztR}&qI|jHq>^C@kn9ojm;6coGMAPWk#IVa%PlrU41i& zhg399aWG;m?$2V{=RSn$8B$czE(RyLakX2vBD5RNBWuusH0P&Mp3wjH5`;z_2HWJN z;CJvo1m&%yPsc3ir+hgYxxf=jms^te-$t;!kHG|KZtuR%nhqQIkTgFHdcA8N4g9YH zYwmJq&3HS~bE6B1@*5s~6=6uryT-sG${zg6^k~xUS6oI_hQx;VV8{IP@Z!y3ESiXD z5ImWd8^6Jx9p-2(t_i9fr%q!5m$g3yBv?8L^70)?TFOjf(O%82+38Fj13J;-g9}9R z-?BO(IS~1AGwXNu4meBo!Y&tMvhnP0Qd-Jofvo4!_d)%@(^_1)Zdn+0T$;=BEK`{p z86cTwwvt$V6;k@QkNx}6j>t`MLWHlC@ ziNeeNr_e&`6 z`79h+1ELU&;vr_MfPfoVSBfhvvelIUp|Bz`Xpple5h>saX{aK6^LFJJqZBDLO>?mlSlR?YH+*P~h6YhZ7 zLo;G5YOMFAYTXMIv~irZh8BW$!*B7HpNXtF&CI3cO}O{;Uy#>12$%D6F?CH1j9r_9 z+WlwQ(7WE~5M6Py;gI0i{Y|Ike3)=-+=%AvG>WO%i;n7RAZ2YOJ@*QPw3mXVsU?uIS8MgWwt*JFqiZ$ zP~Wt%nejR(@83pU9Wqqr^1nX*wvqy-^QmqA8nSeUGwp)+q2<*`G}?bjKaLKw_9aEU z{+NJ1j|mnZlS`Vy1vERDL2d=Pcqu_7<#vly>dE~Su12({7D{7{QDkcAhizggPMt46 zoe&~eYPsim>k`r#QrUrDWz*D`NM2L3aO@#cqq-1=(+@FQl_$hUwn6K@6HK%1D3lMZ zq@Yti1SEfQN!M&q#a6VUHiffkb!_bTsY3PFL=SSXo-*D<{~4gcK-rq|Chy)gpA zpKQmD^rrjydSMq3W{0M2bIgWkmWZS_c^Pdwj zBsh`e#fVPtCDK2?TuYOFzAVr(2HhLvq?5Y~sh!cJ+Bgkwn3rP7Vv(+FE(O`*4h`yk(n0*Q9Gok0dm3C6Ed9YH)3~pAU5{sV17N{2a3)zt7jHgu1`c^T#YKfx40{t zJM+rtl3gl+_K9v#e8TKw$-hJ70hwMc*6~FDb~|v{+HeGqzK}o3!NTr zn`u`_Rh~%o-Ov;cR9q*hWAj=9VVW%WF`V znE_9)uP1NscvK!Z!ZO>MnS$RH=cgB9M}a4Kz2FRrs6HK8LmB z7&oNI>C8c>zZi+Vz00UQUr(2=dZ6Ve zXF+@=OwtM|%#;_y>YfEqPxDRCsLEkxg9&R@Cqeg(zm%5p5oi?4DQO@Yb+=}v9M4@; zd%TM!57|I@ArAY_z0TCG%gFMuOtQ1n5?SA&3i(yo6#aKW=J#QzY;B>S){O|>+MO8m x#U+rvc0EEpG$ZvT%}TAuswM3_&g$R!o+Z}}GVy1ih5y_2=P;80>HGg}{{}98;D!JI literal 0 HcmV?d00001 diff --git a/test/embeddings/test_bert_embedding.py b/test/embeddings/test_bert_embedding.py index 6a4a0ffa..71511458 100644 --- a/test/embeddings/test_bert_embedding.py +++ b/test/embeddings/test_bert_embedding.py @@ -29,8 +29,11 @@ class TestDownload(unittest.TestCase): class TestBertEmbedding(unittest.TestCase): def test_bert_embedding_1(self): - vocab = Vocabulary().add_word_lst("this is a test .".split()) - embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert') + vocab = Vocabulary().add_word_lst("this is a test . [SEP]".split()) + embed = BertEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_bert', word_dropout=0.1) + requires_grad = embed.requires_grad + embed.requires_grad = not requires_grad + embed.train() words = torch.LongTensor([[2, 3, 4, 0]]) result = embed(words) self.assertEqual(result.size(), (1, 4, 16)) diff --git a/test/embeddings/test_elmo_embedding.py b/test/embeddings/test_elmo_embedding.py index a087f0a4..bfb31659 100644 --- a/test/embeddings/test_elmo_embedding.py +++ b/test/embeddings/test_elmo_embedding.py @@ -18,4 +18,19 @@ class TestDownload(unittest.TestCase): # 首先保证所有权重可以加载；上传权重；验证可以下载 +class TestRunElmo(unittest.TestCase): + def test_elmo_embedding(self): + vocab = Vocabulary().add_word_lst("This is a test .".split()) + elmo_embed = ElmoEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_elmo', layers='0,1') + words = torch.LongTensor([[0, 1, 2]]) + hidden = elmo_embed(words) + print(hidden.size()) + + def test_elmo_embedding_layer_assertion(self): + vocab = Vocabulary().add_word_lst("This is a test .".split()) + try: + elmo_embed = ElmoEmbedding(vocab, model_dir_or_name='test/data_for_tests/embedding/small_elmo', + layers='0,1,2') + except AssertionError as e: + print(e) From ea5fbc8881dc763a1ac13c0422da07fb199d6fc1 Mon Sep 17 00:00:00 2001 From: xxliu Date: Thu, 5 Sep 2019 05:07:52 +0800 Subject: [PATCH 43/92] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=B3=A8=E9=87=8A=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=B5=8B=E8=AF=95=E6=96=87=E4=BB=B6=E5=8F=8A?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=A0=B7=E4=BE=8B=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E9=83=A8=E5=88=86=E5=8F=98=E9=87=8F=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/io/loader/coreference.py | 5 +- fastNLP/io/pipe/coreference.py | 48 +++++++++++++++++-- reproduction/coreference_resolution/train.py | 16 +++---- .../coreference/coreference_dev.json | 2 + .../coreference/coreference_test.json | 2 + .../coreference/coreference_train.json | 2 + test/io/loader/test_coreference_loader.py | 16 +++++++ test/io/pipe/test_coreference.py | 24 ++++++++++ 8 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 test/data_for_tests/coreference/coreference_dev.json create mode 100644 test/data_for_tests/coreference/coreference_test.json create mode 100644 test/data_for_tests/coreference/coreference_train.json create mode 100644 test/io/loader/test_coreference_loader.py create mode 100644 test/io/pipe/test_coreference.py diff --git a/fastNLP/io/loader/coreference.py b/fastNLP/io/loader/coreference.py index 2e4d72de..b4493571 100644 --- a/fastNLP/io/loader/coreference.py +++ b/fastNLP/io/loader/coreference.py @@ -22,7 +22,10 @@ class CRLoader(JsonLoader): """ def __init__(self, fields=None, dropna=False): super().__init__(fields, dropna) - self.fields = {"doc_key":Const.INPUTS(0),"speakers":Const.INPUTS(1),"clusters":Const.TARGET,"sentences":Const.INPUTS(2)} + # self.fields = {"doc_key":Const.INPUTS(0),"speakers":Const.INPUTS(1),"clusters":Const.TARGET,"sentences":Const.INPUTS(2)} + # TODO check 1 + self.fields = {"doc_key": "raw_key", "speakers": "raw_speakers", "clusters": "raw_clusters", + "sentences": "raw_words"} def _load(self, path): """ diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py index 711e5919..baa616f1 100644 --- a/fastNLP/io/pipe/coreference.py +++ b/fastNLP/io/pipe/coreference.py @@ -22,21 +22,56 @@ class CoreferencePipe(Pipe): self.config = config def process(self, data_bundle: DataBundle): + """ + 对load进来的数据进一步处理 + 原始数据包含：raw_key,raw_speaker,raw_words,raw_clusters + .. csv-table:: + :header: "raw_key", "raw_speaker","raw_words","raw_clusters" + + "bc/cctv/00/cctv_0000_0", "[["Speaker#1", "Speaker#1"],[]]","[["I","am"],[]]","[[[2,3],[6,7]],[[10,12],[20,22]]]" + "bc/cctv/00/cctv_0000_1"", "[["Speaker#1", "Speaker#1"],[]]","[["He","is"],[]]","[[[2,3],[6,7]],[[10,12],[20,22]]]" + "[...]", "[...]","[...]","[...]" + + 处理完成后数据包含文章类别、speaker信息、句子信息、句子对应的index、char、句子长度、target： + .. csv-table:: + :header: "words1", "words2","words3","words4","chars","seq_len","target" + + "bc", "[[0,0],[1,1]]","[["I","am"],[]]",[[1,2],[]],[[[1],[2,3]],[]],[2,3],"[[[2,3],[6,7]],[[10,12],[20,22]]]" + "[...]", "[...]","[...]","[...]","[...]","[...]","[...]" + + + :param data_bundle: + :return: + """ genres = {g: i for i, g in enumerate(["bc", "bn", "mz", "nw", "pt", "tc", "wb"])} - vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name=Const.INPUTS(2)) + vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name="raw_words") vocab.build_vocab() word2id = vocab.word2idx - data_bundle.vocabs = {"vocab":vocab} - char_dict = get_char_dict(self.config.char_path) + data_bundle.set_vocab(vocab,"vocab") + if self.config.char_path: + char_dict = get_char_dict(self.config.char_path) + else: + char_set = set() + for i,w in enumerate(word2id): + if i < 2: + continue + for c in w: + char_set.add(c) + + char_dict = collections.defaultdict(int) + char_dict.update({c: i for i, c in enumerate(char_set)}) for name, ds in data_bundle.datasets.items(): # genre - ds.apply(lambda x: genres[x[Const.INPUTS(0)][:2]], new_field_name=Const.INPUTS(0)) + ds.apply(lambda x: genres[x["raw_key"][:2]], new_field_name=Const.INPUTS(0)) # speaker_ids_np - ds.apply(lambda x: speaker2numpy(x[Const.INPUTS(1)], self.config.max_sentences, is_train=name == 'train'), + ds.apply(lambda x: speaker2numpy(x["raw_speakers"], self.config.max_sentences, is_train=name == 'train'), new_field_name=Const.INPUTS(1)) + # sentences + ds.rename_field("raw_words",Const.INPUTS(2)) + # doc_np ds.apply(lambda x: doc2numpy(x[Const.INPUTS(2)], word2id, char_dict, max(self.config.filter), self.config.max_sentences, is_train=name == 'train')[0], @@ -50,6 +85,9 @@ class CoreferencePipe(Pipe): self.config.max_sentences, is_train=name == 'train')[2], new_field_name=Const.INPUT_LEN) + # clusters + ds.rename_field("raw_clusters", Const.TARGET) + ds.set_ignore_type(Const.TARGET) ds.set_padder(Const.TARGET, None) diff --git a/reproduction/coreference_resolution/train.py b/reproduction/coreference_resolution/train.py index 790c7659..c91f7109 100644 --- a/reproduction/coreference_resolution/train.py +++ b/reproduction/coreference_resolution/train.py @@ -37,15 +37,15 @@ if __name__ == "__main__": print(config) - @cache_results('cache.pkl') + # @cache_results('cache.pkl') def cache(): - bundle = CoreferencePipe(Config()).process_from_file({'train': config.train_path, 'dev': config.dev_path,'test': config.test_path}) + bundle = CoreferencePipe(config).process_from_file({'train': config.train_path, 'dev': config.dev_path,'test': config.test_path}) return bundle - data_info = cache() - print("数据集划分：\ntrain:", str(len(data_info.datasets["train"])), - "\ndev:" + str(len(data_info.datasets["dev"])) + "\ntest:" + str(len(data_info.datasets["test"]))) + data_bundle = cache() + print("数据集划分：\ntrain:", str(len(data_bundle.get_dataset("train"))), + "\ndev:" + str(len(data_bundle.get_dataset("dev"))) + "\ntest:" + str(len(data_bundle.get_dataset('test')))) # print(data_info) - model = Model(data_info.vocabs['vocab'], config) + model = Model(data_bundle.vocabs['vocab'], config) print(model) loss = SoftmaxLoss() @@ -56,8 +56,8 @@ if __name__ == "__main__": lr_decay_callback = LRCallback(optim.param_groups, config.lr_decay) - trainer = Trainer(model=model, train_data=data_info.datasets["train"], dev_data=data_info.datasets["dev"], - loss=loss, metrics=metric, check_code_level=-1,sampler=None, + trainer = Trainer(model=model, train_data=data_bundle.datasets["train"], dev_data=data_bundle.datasets["dev"], + loss=loss, metrics=metric, check_code_level=-1, sampler=None, batch_size=1, device=torch.device("cuda:" + config.cuda), metric_key='f', n_epochs=config.epoch, optimizer=optim, save_path='/remote-home/xxliu/pycharm/fastNLP/fastNLP/reproduction/coreference_resolution/save', diff --git a/test/data_for_tests/coreference/coreference_dev.json b/test/data_for_tests/coreference/coreference_dev.json new file mode 100644 index 00000000..9322ed30 --- /dev/null +++ b/test/data_for_tests/coreference/coreference_dev.json @@ -0,0 +1,2 @@ +{"doc_key": "bc/cctv/00/cctv_0000_0", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"]], "clusters": [[[70, 70], [485, 486], [500, 500], [73, 73], [55, 55], [153, 154], [366, 366]], [[307, 312], [255, 256]], [[198, 199], [163, 164]], [[289, 290], [318, 318], [494, 497], [129, 131], [261, 261], [86, 86], [387, 387], [278, 278], [122, 124], [51, 56], [221, 225], [353, 355], [292, 292], [299, 299], [322, 322], [348, 348], [311, 312], [251, 253]], [[143, 144], [138, 138]], [[155, 176], [213, 214], [183, 184], [195, 195]], [[398, 398], [403, 403], [335, 335], [390, 390]], [[28, 28], [32, 37]], [[337, 338], [372, 373]], [[129, 130], [488, 489], [122, 123], [108, 109], [147, 148], [191, 192], [41, 42], [23, 24], [251, 252]], [[208, 208], [201, 204]], [[377, 379], [411, 413]]], "sentences": [["In", "the", "summer", "of", "2005", ",", "a", "picture", "that", "people", "have", "long", "been", "looking", "forward", "to", "started", "emerging", "with", "frequency", "in", "various", "major", "Hong", "Kong", "media", "."], ["With", "their", "unique", "charm", ",", "these", "well", "-", "known", "cartoon", "images", "once", "again", "caused", "Hong", "Kong", "to", "be", "a", "focus", "of", "worldwide", "attention", "."], ["The", "world", "'s", "fifth", "Disney", "park", "will", "soon", "open", "to", "the", "public", "here", "."], ["The", "most", "important", "thing", "about", "Disney", "is", "that", "it", "is", "a", "global", "brand", "."], ["Well", ",", "for", "several", "years", ",", "although", "it", "was", "still", "under", "construction", "and", ",", "er", ",", "not", "yet", "open", ",", "it", "can", "be", "said", "that", "many", "people", "have", "viewed", "Hong", "Kong", "with", "new", "respect", "."], ["Then", "welcome", "to", "the", "official", "writing", "ceremony", "of", "Hong", "Kong", "Disneyland", "."], ["The", "construction", "of", "Hong", "Kong", "Disneyland", "began", "two", "years", "ago", ",", "in", "2003", "."], ["In", "January", "of", "that", "year", ",", "the", "Hong", "Kong", "government", "turned", "over", "to", "Disney", "Corporation", "200", "hectares", "of", "land", "at", "the", "foot", "of", "Lantau", "Island", "that", "was", "obtained", "following", "the", "largest", "land", "reclamation", "project", "in", "recent", "years", "."], ["One", "."], ["Since", "then", ",", "this", "area", "has", "become", "a", "prohibited", "zone", "in", "Hong", "Kong", "."], ["As", "its", "neighbor", "on", "Lantau", "Island", ",", "Hong", "Kong", "International", "Airport", "had", "to", "change", "its", "flight", "routes", "to", "make", "this", "area", "a", "no", "-", "fly", "zone", "."], ["Mickey", "Mouse", "'s", "new", "home", ",", "settling", "on", "Chinese", "land", "for", "the", "first", "time", ",", "has", "captured", "worldwide", "attention", "."], ["There", "'s", "only", "one", "month", "left", "before", "the", "opening", "of", "Hong", "Kong", "Disneyland", "on", "September", "12", "."], ["The", "subway", "to", "Disney", "has", "already", "been", "constructed", "."], ["At", "subway", "stations", ",", "passengers", "will", "frequently", "press", "the", "station", "for", "Disney", "on", "ticket", "machines", ",", "trying", "to", "purchase", "tickets", "to", "enjoy", "the", "park", "when", "it", "first", "opens", "."], ["Meanwhile", ",", "the", "Disney", "subway", "station", "is", "scheduled", "to", "open", "on", "the", "same", "day", "as", "the", "park", "."], ["For", "two", "years", ",", "Disney", "has", "constantly", "maintained", "its", "mystery", "."], ["No", "media", "have", "been", "allowed", "to", "enter", "for", "photos", "."], ["We", "took", "a", "taxi", "along", "the", "path", "of", "the", "highway", "that", "heads", "toward", "Disney", ",", "trying", "to", "experience", "this", "mysterious", "park", "from", "close", "by", "."], ["However", ",", "before", "any", "of", "the", "Disney", "symbols", "were", "in", "sight", ",", "the", "car", "was", "stopped", "by", "a", "security", "guard", "at", "the", "intersection", "of", "the", "road", "towards", "Disney", "."], ["On", "our", "way", "back", ",", "the", "taxi", "driver", "gave", "us", "an", "explanation", "after", "understanding", "our", "intentions", "."], ["Er", ",", "according", "to", "what", "the", "security", "guard", "said", ",", "for", "the", "time", "before", "everything", "is", "officially", ",", "opened", ",", ",", "no", "cars", "can", "enter", "unless", "they", "have", "special", "permission", "."], ["No", "one", "can", "enter", "otherwise", "."], ["Video", "recording", "is", "especially", "forbidden", "."], ["Ah", ",", "everything", "is", "top", "secret", "."], ["If", "pictures", "are", "taken", "without", "permission", ",", "%pw", "that", "is", "to", "say", ",", "it", "will", "at", "all", "times", "be", "pursued", "by", "legal", "action", ",", "a", "big", "hassle", "."], ["Although", "Disney", "Corporation", "chose", "Hong", "Kong", "as", "the", "venue", "for", "the", "Chinese", "Disney", "park", ",", "what", "they", "are", "actually", "most", "excited", "about", "is", "the", "mainland", "China", "tourist", "market", "."]]} +{"doc_key": "bc/cctv/00/cctv_0000_1", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"]], "clusters": [[[24, 25], [121, 122], [44, 45], [83, 84], [9, 10], [233, 235], [199, 200]]], "sentences": [["Since", "the", "implementation", "of", "the", "Individual", "Visit", "Scheme", "between", "Hong", "Kong", "and", "the", "mainland", ",", "more", "and", "more", "mainland", "tourists", "are", "coming", "to", "visit", "Hong", "Kong", "."], ["From", "the", "beginning", "up", "till", "now", ",", "more", "than", "seven", "million", "individual", "tourists", ",", "have", "come", "to", "Hong", "Kong", "."], ["Well", ",", "we", "now", ",", "er", ",", "believe", "more", "will", "be", "coming", "."], ["At", "this", "point", ",", "it", "has", "been", "about", "two", "years", "."], ["Also", ",", "the", "current", "number", "of", "34", "cities", "will", "be", "increased", "."], ["Hong", "Kong", "was", "developed", "from", "a", "fishing", "harbor", "one", "hundred", "years", "ago", "to", "become", "today", "'s", "international", "metropolis", "."], ["Here", ",", "eastern", "and", "western", "cultures", "have", "gathered", ",", "and", "the", "new", "and", "the", "old", "coexist", "."], ["When", "in", "Hong", "Kong", ",", "you", "can", "wander", "among", "skyscrapers", ",", "heartily", "enjoy", "shopping", "sprees", "in", "well", "-", "known", "stores", "and", "malls", "for", "goods", "from", "various", "countries", ",", "and", "taste", "delicious", "snacks", "from", "all", "over", "the", "world", "at", "tea", "shops", "or", "at", "street", "stands", "in", "Mong", "Kok", "."], ["You", "can", "go", "to", "burn", "incense", "and", "make", "a", "vow", "at", "the", "Repulse", "Bay", ",", "where", "all", "deities", "gather", "."], ["You", "can", "enjoy", "the", "most", "charming", "sun", "-", "filled", "sandy", "beaches", "in", "Hong", "Kong", "."], ["You", "can", "ascend", "Victoria", "Peak", "to", "get", "a", "panoramic", "view", "of", "Victoria", "Harbor", "'s", "beautiful", "scenery", "."], ["Or", "hop", "onto", "a", "trolley", "with", "over", "a", "century", "of", "history", ",", "and", "feel", "the", "city", "'s", "blend", "of", "the", "old", "and", "the", "modern", "in", "slow", "motion", "."]]} diff --git a/test/data_for_tests/coreference/coreference_test.json b/test/data_for_tests/coreference/coreference_test.json new file mode 100644 index 00000000..399b8cc5 --- /dev/null +++ b/test/data_for_tests/coreference/coreference_test.json @@ -0,0 +1,2 @@ +{"doc_key": "bc/cctv/00/cctv_0005_0", "speakers": [["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"], ["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"], ["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"]], "clusters": [[[57, 59], [25, 27], [42, 44]], [[19, 23], [16, 16]], [[83, 83], [82, 82]]], "sentences": [["--", "basically", ",", "it", "was", "unanimously", "agreed", "upon", "by", "the", "various", "relevant", "parties", "."], ["To", "express", "its", "determination", ",", "the", "Chinese", "securities", "regulatory", "department", "compares", "this", "stock", "reform", "to", "a", "die", "that", "has", "been", "cast", "."], ["It", "takes", "time", "to", "prove", "whether", "the", "stock", "reform", "can", "really", "meet", "expectations", ",", "and", "whether", "any", "deviations", "that", "arise", "during", "the", "stock", "reform", "can", "be", "promptly", "corrected", "."], ["Dear", "viewers", ",", "the", "China", "News", "program", "will", "end", "here", "."], ["This", "is", "Xu", "Li", "."], ["Thank", "you", "everyone", "for", "watching", "."], ["Coming", "up", "is", "the", "Focus", "Today", "program", "hosted", "by", "Wang", "Shilin", "."], ["Good-bye", ",", "dear", "viewers", "."]]} +{"doc_key": "bc/cctv/00/cctv_0005_1", "speakers": [["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Wang_shilin", "Wang_shilin"], ["Zhou_hanhua", "Zhou_hanhua"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"]], "clusters": [[[233, 234], [7, 8]], [[253, 254], [438, 439]], [[411, 412], [64, 67], [18, 30], [259, 260], [516, 516]], [[432, 433], [190, 204], [272, 272], [325, 325], [314, 314], [292, 292], [281, 281], [334, 334]], [[310, 311], [299, 300], [321, 321]], [[172, 172], [10, 10]], [[372, 373], [392, 393], [216, 219], [418, 419]], [[29, 30], [108, 109], [112, 113]], [[72, 73], [59, 60], [27, 27]], [[305, 305], [377, 377]], [[502, 503], [444, 447], [459, 460]], [[352, 353], [387, 387], [362, 362], [408, 408], [210, 219], [375, 375], [360, 360], [350, 350]], [[182, 185], [166, 168], [247, 250], [224, 226]], [[383, 384], [51, 60]], [[367, 368], [268, 268], [35, 36], [256, 260]], [[523, 523], [500, 500], [493, 493], [435, 435], [238, 238]], [[228, 229], [187, 188], [170, 171]]], "sentences": [["Hello", ",", "dear", "viewers", "."], ["Welcome", "to", "Focus", "Today", "."], ["Today", ",", "let", "'s", "turn", "our", "attention", "to", "a", "road", "cave", "-", "in", "accident", "that", "happened", "in", "Beijing", "over", "the", "holiday", "."], ["Before", "dawn", "on", "January", "3", ",", "a", "sewage", "pipe", "leakage", "accident", "occurred", "at", "the", "main", "and", "side", "roads", "of", "Jingguang", "Bridge", ",", "East", "Third", "Ring", "Road", ",", "Beijing", "Municipality", ",", "resulting", "in", "the", "road", "caving", "in", "."], ["Relevant", "departments", "from", "Beijing", "Municipality", "promptly", "activated", "emergency", "contingency", "plans", "."], ["The", "traffic", "administration", "department", "carried", "out", "traffic", "supervision", "near", "the", "accident", "scene", "."], ["Well", ",", "how", "did", "the", "emergency", "response", "mechanisms", "activated", "by", "governmental", "departments", "operate", "effectively", "during", "the", "holiday", "?"], ["After", "the", "holiday", ",", "what", "will", "be", "done", "to", "handle", "citizens", "'", "peak", "commute", "?"], ["In", "addition", ",", "what", "measures", "did", "relevant", "departments", "take", "to", "resolve", "issues", "such", "as", "waste", "discharge", ",", "heating", ",", "and", "communication", ",", "in", "order", "to", "ensure", "that", "the", "lives", "of", "citizens", "were", "not", "affected", "?"], ["Well", ",", "we", "have", "invited", "two", "honorable", "guests", "to", "the", "studio", "today", "to", "follow", "this", "topic", "with", "us", "."], ["One", "of", "the", "two", "honorable", "guests", "in", "the", "studio", "is", "Professor", "Zhou", "Hanhua", "from", "the", "Institute", "of", "Law", "of", "the", "Chinese", "Academy", "of", "Social", "Sciences", "."], ["Hello", "."], ["Next", "is", "Yang", "Yang", ",", "a", "host", "of", "Beijing", "Traffic", "Radio", "Station", "."], ["Hello", "."], ["Welcome", "both", "of", "you", "to", "the", "studio", "to", "participate", "in", "our", "program", "."], ["Well", ",", "I", "especially", "want", "to", "know", ",", "ha", ",", "how", "the", "two", "of", "you", "found", "out", "the", "news", "on", "the", "day", "of", "the", "accident", "?"], ["Ah", ",", ",", "about", "11:00", "m.", "yesterday", ",", "ah", ",", "I", "happened", "to", "find", "out", "through", "an", "SMS", "when", "I", "was", "outside", "."], ["Uh-huh", "."], ["Uh-huh", "."], ["It", "happened", "that", "I", "was", "going", "to", "have", "lunch", "with", "a", "friend", ",", "um", ",", "at", "noon", "."], ["And", "then", ",", "the", "friend", "first", "sent", "me", "an", "SMS", ",", "Uh-huh", ".", "saying", "he", "would", "come", "pick", "me", "up", "to", "go", "together", "."], ["After", "that", ",", "I", "received", "an", "SMS", "from", "1860", "."], ["Uh-huh", ",", "it", "was", "through", "an", "SMS", "."], ["And", "you", ",", "Yang", "Yang", "?"], ["A", "friend", "happened", "to", "call", "me", "."], ["You", "were", "not", "at", "work", "that", "day", "?"], ["No", "."], ["The", "station", "called", "me", "at", "noon", "and", "said", "something", "happened", "at", "Jingguang", "Bridge", "and", "that", "I", "had", "to", "go", "to", "the", "station", "immediately", "to", "research", "the", "upcoming", "program", "."], ["Uh-huh", ",", "that", "means", ",", "er", ",", "you", "found", "out", "the", "accident", "through", "an", "information", "source", "at", "the", "station", "."], ["Right", ",", "right", ",", "right", "."], ["Uh-huh", "."], ["Well", ",", "like", "Professor", "Zhou", ",", "I", "also", "received", "this", "news", ",", "ha", ",", "through", "a", "mobile", "phone", "SMS", "."], ["At", "that", "time", ",", ",", "it", "can", "be", "said", "that", "this", "SMS", "was", "among", "the", "many", ",", "ha", ",", "SMS", "containing", "New", "Year", "wishes", ",", "like", "Happy", "New", "Year", ",", "received", "after", "the", "start", "of", "the", "New", "Year", "."], ["Uh-huh", "."], ["Ah", ",", "actually", "I", "felt", "a", "lot", "of", "warmth", "when", "I", "received", "that", "SMS", "."], ["Although", "we", "live", "in", "the", "west", "instead", "of", "the", "east", "and", "it", "did", "not", "affect", "us", "much", ",", "I", "think", "it", "is", "very", "useful", ",", "ah", ",", "to", "inform", "people", "of", "this", "kind", "of", "news", "."], ["Yes", ",", "exceptionally", "."], ["Yes", ",", "exceptionally", "."]]} diff --git a/test/data_for_tests/coreference/coreference_train.json b/test/data_for_tests/coreference/coreference_train.json new file mode 100644 index 00000000..6932bbb7 --- /dev/null +++ b/test/data_for_tests/coreference/coreference_train.json @@ -0,0 +1,2 @@ +{"doc_key": "bc/cctv/00/cctv_0001_0", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"]], "clusters": [[[113, 114], [42, 45], [88, 91]], [[288, 288], [293, 293]], [[185, 189], [162, 165], [101, 104]], [[232, 233], [209, 209], [253, 253]], [[36, 37], [31, 32]], [[55, 56], [79, 81]], [[283, 283], [269, 275]], [[39, 45], [47, 47]], [[285, 285], [298, 298], [235, 237], [258, 260], [117, 120], [267, 267]], [[75, 77], [51, 53]], [[310, 310], [289, 289], [295, 295]], [[135, 136], [273, 273], [26, 26]], [[200, 201], [182, 183], [179, 180]]], "sentences": [["What", "kind", "of", "memory", "?"], ["We", "respectfully", "invite", "you", "to", "watch", "a", "special", "edition", "of", "Across", "China", "."], ["WW", "II", "Landmarks", "on", "the", "Great", "Earth", "of", "China", ":", "Eternal", "Memories", "of", "Taihang", "Mountain"], ["Standing", "tall", "on", "Taihang", "Mountain", "is", "the", "Monument", "to", "the", "Hundred", "Regiments", "Offensive", "."], ["It", "is", "composed", "of", "a", "primary", "stele", ",", "secondary", "steles", ",", "a", "huge", "round", "sculpture", "and", "beacon", "tower", ",", "and", "the", "Great", "Wall", ",", "among", "other", "things", "."], ["A", "primary", "stele", ",", "three", "secondary", "steles", ",", "and", "two", "inscribed", "steles", "."], ["The", "Hundred", "Regiments", "Offensive", "was", "the", "campaign", "of", "the", "largest", "scale", "launched", "by", "the", "Eighth", "Route", "Army", "during", "the", "War", "of", "Resistance", "against", "Japan", "."], ["This", "campaign", "broke", "through", "the", "Japanese", "army", "'s", "blockade", "to", "reach", "base", "areas", "behind", "enemy", "lines", ",", "stirring", "up", "anti-Japanese", "spirit", "throughout", "the", "nation", "and", "influencing", "the", "situation", "of", "the", "anti-fascist", "war", "of", "the", "people", "worldwide", "."], ["This", "is", "Zhuanbi", "Village", ",", "Wuxiang", "County", "of", "Shanxi", "Province", ",", "where", "the", "Eighth", "Route", "Army", "was", "headquartered", "back", "then", "."], ["On", "a", "wall", "outside", "the", "headquarters", "we", "found", "a", "map", "."], ["This", "map", "was", "the", "Eighth", "Route", "Army", "'s", "depiction", "of", "the", "Mediterranean", "Sea", "situation", "at", "that", "time", "."], ["This", "map", "reflected", "the", "European", "battlefield", "situation", "."], ["In", "1940", ",", "the", "German", "army", "invaded", "and", "occupied", "Czechoslovakia", ",", "Poland", ",", "the", "Netherlands", ",", "Belgium", ",", "and", "France", "."], ["It", "was", "during", "this", "year", "that", "the", "Japanese", "army", "developed", "a", "strategy", "to", "rapidly", "force", "the", "Chinese", "people", "into", "submission", "by", "the", "end", "of", "1940", "."], ["In", "May", ",", "the", "Japanese", "army", "launched", "--"], ["From", "one", "side", ",", "it", "seized", "an", "important", "city", "in", "China", "called", "Yichang", "."], ["Um", ",", ",", "uh", ",", "through", "Yichang", ",", "it", "could", "directly", "reach", "Chongqing", "."], ["Ah", ",", "that", "threatened", "Chongqing", "."], ["Then", "they", "would", ",", "ah", ",", "bomb", "these", "large", "rear", "areas", "such", "as", "Chongqing", "."], ["So", ",", "along", "with", "the", "coordinated", ",", "er", ",", "economic", "blockade", ",", "military", "offensives", ",", "and", "strategic", "bombings", ",", "er", ",", "a", "simultaneous", "attack", "was", "launched", "in", "Hong", "Kong", "to", "lure", "the", "KMT", "government", "into", "surrender", "."], ["The", "progress", "of", "this", "coordinated", "offensive", "was", "already", "very", "entrenched", "by", "then", "."]]} +{"doc_key": "bc/cctv/00/cctv_0001_1", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1"]], "clusters": [[[129, 131], [167, 169]], [[495, 496], [446, 447], [183, 186]], [[433, 438], [314, 316], [318, 318]], [[154, 157], [531, 534], [436, 438], [139, 142], [43, 45]], [[560, 561], [547, 554], [279, 288]], [[309, 309], [374, 374], [21, 23], [9, 9], [312, 312], [385, 385]], [[212, 213], [193, 197]], [[577, 578], [581, 582]], [[262, 267], [591, 592], [523, 524], [565, 568], [424, 431]], [[255, 256], [28, 32]], [[492, 493], [175, 181], [443, 444]], [[124, 127], [449, 451], [250, 253], [29, 31], [188, 191], [407, 416], [71, 74], [510, 513], [129, 129]], [[63, 67], [139, 146], [76, 78]], [[443, 452], [175, 191]], [[485, 487], [596, 598]], [[517, 524], [556, 556], [526, 526]], [[81, 98], [133, 134]], [[47, 48], [109, 112]], [[348, 353], [365, 365], [388, 390]], [[1, 1], [477, 477], [267, 267]], [[550, 551], [288, 288], [3, 4], [18, 18]]], "sentences": [["By", "1940", ",", "China", "'s", "War", "of", "Resistance", "against", "Japan", "had", "entered", "a", "stalemate", "."], ["The", "situation", "on", "our", "side", "and", "the", "enemy", "'s", "side", "was", "intertwined", "."], ["The", "Eighth", "Route", "Army", "guerrillas", "were", "extraordinarily", "active", ",", "creating", "more", "and", "more", "trouble", "for", "the", "Japanese", "army", "in", "North", "China", "."], ["Hayao", "Tada", ",", "commander", "of", "the", "Japanese", "North", "China", "Area", "Army", ",", "adopted", "a", "strategy", "of", "siege", "warfare", "to", "deal", "with", "the", "Eighth", "Route", "Army", "."], ["The", "specific", "method", "was", "building", "a", "closely", "connected", "transport", "network", ",", "with", "a", "road", "for", "every", "village", "and", "defensive", "towers", "on", "every", "road", "."], ["Roads", "and", "railways", "were", "used", "as", "links", "to", "connect", "all", "of", "North", "China", "into", "a", "solid", ",", "widespread", "siege", ",", "in", "order", "to", "strangle", "the", "Eighth", "Route", "Army", "and", "its", "base", "areas", "in", "this", "net", "."], ["As", "part", "of", "the", "Japanese", "army", "'s", "strategy", "of", "siege", "warfare", ",", "railways", "and", "roads", "had", "actually", "become", "the", "Japanese", "army", "'s", "weapons", "of", "war", ",", "becoming", "a", "great", "threat", "to", "the", "base", "areas", "."], ["In", "December", "1939", ",", "Commander", "-", "in", "-", "chief", "Zhu", "De", "and", "Vice", "Commander", "Peng", "Dehuai", "of", "the", "Eighth", "Route", "Army", "received", "a", "top", "-", "secret", "telegram", "from", "Commander", "Lu", "Zhengcao", "of", "the", "Jizhong", "Military", "District", ",", "among", "other", "people", "."], ["The", "telegram", "said", "that", "the", "Japanese", "troops", "were", "building", "blockade", "trenches", "and", "chessboard", "-", "like", "roads", "to", "divide", "the", "Jizhong", "base", "area", "into", "small", "isolated", "blocks", "without", "the", "ability", "to", "mutually", "communicate", "and", "support", "each", "other", ",", "causing", "the", "Eighth", "Route", "Army", "and", "the", "guerrillas", "to", "lose", "maneuverability", "."], ["Before", "the", "Hundred", "Regiments", "Offensive", "in", "1940", ",", "an", "inclination", "to", "compromise", ",", "ah", ",", "surrender", ",", "was", "an", "extremely", "serious", "crisis", "in", "the", "frontline", "situation", "in", "China", "."], ["Well", ",", "on", "the", "battlefield", "behind", "enemy", "lines", ",", "in", "order", "to", "take", "over", ",", "consolidate", "the", "area", "under", "its", "occupation", ",", "Japan", "began", "a", "new", "strategy", "."], ["That", "was", "to", "use", "railways", "as", "a", "pillar", ",", "roads", "as", "a", "chain", ",", "and", "strongholds", "as", "a", "lock", ",", "to", "carry", "out", "siege", "warfare", "in", "an", "attempt", "to", "divide", "the", "base", "areas", "behind", "enemy", "lines", ",", "ah", ",", "so", "as", ",", "er", ",", "to", "cut", "off", "their", "communication", "with", "one", "another", "."], ["In", "addition", ",", "it", "relied", "on", "this", "cage", ",", "ah", ",", "to", "further", "strengthen", "its", "assaults", "against", "the", "base", "areas", "."], ["Er", "."], ["So", ",", "it", "was", "amidst", "such", "a", "grave", "international", "and", "domestic", "situation", "that", "the", "Eighth", "Route", "Army", "led", "by", "the", "Chinese", "Communist", "Party", ",", "ah", ",", "launched", ",", "ah", ",", "a", "strategic", "offensive", "called", "the", "Hundred", "Regiments", "Offensive", "."], ["This", "plot", "of", "the", "Japanese", "army", "drew", "great", "attention", "from", "Zhu", "De", "and", "Peng", "Dehuai", "of", "Eighth", "Route", "Army", "headquarters", "."], ["After", "meticulous", "studies", "and", "painstaking", "preparations", "by", "many", "parties", ",", "a", "battle", "plan", "based", "on", "surprise", "was", "formulated", "."], ["On", "July", "22", ",", "1940", ",", "a", "campaign", "preparation", "order", "to", "attack", "the", "Zhengtai", "Railway", ",", "jointly", "signed", "by", "Zhu", "De", ",", "Peng", "Dehuai", ",", "and", "Zuo", "Quan", ",", "was", "sent", "to", "Yan'an", "and", "all", "units", "of", "the", "Eighth", "Route", "Army", "."], ["What", "was", "the", ",", "purpose", "and", "goal", "of", "this", "campaign", "?"], ["It", "was", "to", "break", "through", "the", "Japanese", "army", "'s", "siege", "policy", "against", "base", "areas", "behind", "enemy", "lines", ",", "and", "to", "avert", "the", "crisis", "of", "China", "'s", "compromise", "and", "surrender", "."], ["It", "was", "to", "overcome", "this", "crisis", "."], ["Well", ",", "the", "Hundred", "Regiments", "Offensive", "was", "divided", "into", "three", "phases", "."], ["Beginning", "from", "August", "20", ",", "from", "August", "20", "to", "September", "10", ",", "the", "main", "purpose", "of", "the", "campaign", "was", "to", "sabotage", "the", "Zhengtai", "Railway", "."]]} diff --git a/test/io/loader/test_coreference_loader.py b/test/io/loader/test_coreference_loader.py new file mode 100644 index 00000000..48551f3e --- /dev/null +++ b/test/io/loader/test_coreference_loader.py @@ -0,0 +1,16 @@ +from fastNLP.io.loader.coreference import CRLoader +import unittest + +class TestCR(unittest.TestCase): + def test_load(self): + + test_root = "../../data_for_tests/coreference/" + train_path = test_root+"coreference_train.json" + dev_path = test_root+"coreference_dev.json" + test_path = test_root+"coreference_test.json" + paths = {"train": train_path,"dev":dev_path,"test":test_path} + + bundle1 = CRLoader().load(paths) + bundle2 = CRLoader().load(test_root) + print(bundle1) + print(bundle2) \ No newline at end of file diff --git a/test/io/pipe/test_coreference.py b/test/io/pipe/test_coreference.py new file mode 100644 index 00000000..1c53f2b0 --- /dev/null +++ b/test/io/pipe/test_coreference.py @@ -0,0 +1,24 @@ +import unittest +from fastNLP.io.pipe.coreference import CoreferencePipe + + +class TestCR(unittest.TestCase): + + def test_load(self): + class Config(): + max_sentences = 50 + filter = [3, 4, 5] + char_path = None + config = Config() + + file_root_path = "../../data_for_tests/coreference/" + train_path = file_root_path + "coreference_train.json" + dev_path = file_root_path + "coreference_dev.json" + test_path = file_root_path + "coreference_test.json" + + paths = {"train": train_path, "dev": dev_path, "test": test_path} + + bundle1 = CoreferencePipe(config).process_from_file(paths) + bundle2 = CoreferencePipe(config).process_from_file(file_root_path) + print(bundle1) + print(bundle2) \ No newline at end of file From b5a7db0b669f6956a98300799e060977f8a45a55 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Thu, 5 Sep 2019 14:31:38 +0800 Subject: [PATCH 44/92] delete the output part in dot-utils --- fastNLP/doc_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fastNLP/doc_utils.py b/fastNLP/doc_utils.py index 5801dd53..5f293d3f 100644 --- a/fastNLP/doc_utils.py +++ b/fastNLP/doc_utils.py @@ -1,3 +1,7 @@ +"""undocumented""" + +__all__ = [] + import inspect import sys @@ -7,7 +11,8 @@ def doc_process(m): if inspect.isclass(obj) or inspect.isfunction(obj): if obj.__module__ != m.__name__: if obj.__doc__ is None: - print(name, obj.__doc__) + # print(name, obj.__doc__) + pass else: module_name = obj.__module__ while 1: @@ -18,5 +23,5 @@ def doc_process(m): break module_name = ".".join(module_name.split('.')[:-1]) if module_name == m.__name__: - print(name, ": not found defined doc.") + # print(name, ": not found defined doc.") break From 5b7e9b6572ff980c9b536b3b8a8b5ea526bd2ad6 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Thu, 5 Sep 2019 14:32:37 +0800 Subject: [PATCH 45/92] update the ChnSentiCorpPipe in docs --- docs/source/fastNLP.io.loader.rst | 2 +- docs/source/fastNLP.io.pipe.rst | 2 +- docs/source/fastNLP.io.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/fastNLP.io.loader.rst b/docs/source/fastNLP.io.loader.rst index 060b5450..c1af6c0c 100644 --- a/docs/source/fastNLP.io.loader.rst +++ b/docs/source/fastNLP.io.loader.rst @@ -2,6 +2,6 @@ fastNLP.io.loader ================= .. automodule:: fastNLP.io.loader - :members: Loader, YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader, ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader, MsraNERLoader, PeopleDailyNERLoader, WeiboNERLoader, CSVLoader, JsonLoader, CWSLoader, MNLILoader, QuoraLoader, SNLILoader, QNLILoader, RTELoader + :members: Loader, YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader, ChnSentiCorpLoader, ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader, MsraNERLoader, PeopleDailyNERLoader, WeiboNERLoader, CSVLoader, JsonLoader, CWSLoader, MNLILoader, QuoraLoader, SNLILoader, QNLILoader, RTELoader :inherited-members: diff --git a/docs/source/fastNLP.io.pipe.rst b/docs/source/fastNLP.io.pipe.rst index d35d2ddc..3ef9b5a8 100644 --- a/docs/source/fastNLP.io.pipe.rst +++ b/docs/source/fastNLP.io.pipe.rst @@ -2,6 +2,6 @@ fastNLP.io.pipe =============== .. automodule:: fastNLP.io.pipe - :members: Pipe, CWSPipe, YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe, Conll2003NERPipe, OntoNotesNERPipe, MsraNERPipe, WeiboNERPipe, PeopleDailyPipe, Conll2003Pipe, MatchingBertPipe, RTEBertPipe, SNLIBertPipe, QuoraBertPipe, QNLIBertPipe, MNLIBertPipe, MatchingPipe, RTEPipe, SNLIPipe, QuoraPipe, QNLIPipe, MNLIPipe + :members: Pipe, CWSPipe, YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe, ChnSentiCorpPipe, Conll2003NERPipe, OntoNotesNERPipe, MsraNERPipe, WeiboNERPipe, PeopleDailyPipe, Conll2003Pipe, MatchingBertPipe, RTEBertPipe, SNLIBertPipe, QuoraBertPipe, QNLIBertPipe, MNLIBertPipe, MatchingPipe, RTEPipe, SNLIPipe, QuoraPipe, QNLIPipe, MNLIPipe :inherited-members: diff --git a/docs/source/fastNLP.io.rst b/docs/source/fastNLP.io.rst index 96df9d6c..7118039d 100644 --- a/docs/source/fastNLP.io.rst +++ b/docs/source/fastNLP.io.rst @@ -2,7 +2,7 @@ fastNLP.io ========== .. automodule:: fastNLP.io - :members: DataBundle, EmbedLoader, Loader, YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader, ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader, MsraNERLoader, WeiboNERLoader, PeopleDailyNERLoader, CSVLoader, JsonLoader, CWSLoader, MNLILoader, QuoraLoader, SNLILoader, QNLILoader, RTELoader, Pipe, YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe, Conll2003Pipe, Conll2003NERPipe, OntoNotesNERPipe, MsraNERPipe, PeopleDailyPipe, WeiboNERPipe, CWSPipe, MatchingBertPipe, RTEBertPipe, SNLIBertPipe, QuoraBertPipe, QNLIBertPipe, MNLIBertPipe, MatchingPipe, RTEPipe, SNLIPipe, QuoraPipe, QNLIPipe, MNLIPipe, ModelLoader, ModelSaver + :members: DataBundle, EmbedLoader, Loader, YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader, ChnSentiCorpLoader, ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader, MsraNERLoader, WeiboNERLoader, PeopleDailyNERLoader, CSVLoader, JsonLoader, CWSLoader, MNLILoader, QuoraLoader, SNLILoader, QNLILoader, RTELoader, Pipe, YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe, ChnSentiCorpPipe, Conll2003Pipe, Conll2003NERPipe, OntoNotesNERPipe, MsraNERPipe, PeopleDailyPipe, WeiboNERPipe, CWSPipe, MatchingBertPipe, RTEBertPipe, SNLIBertPipe, QuoraBertPipe, QNLIBertPipe, MNLIBertPipe, MatchingPipe, RTEPipe, SNLIPipe, QuoraPipe, QNLIPipe, MNLIPipe, ModelLoader, ModelSaver :inherited-members: 子模块 From f004a070b4606fa509f6d55ea70a8ac9a82766af Mon Sep 17 00:00:00 2001 From: ChenXin Date: Thu, 5 Sep 2019 15:13:08 +0800 Subject: [PATCH 46/92] update the doc tool --- docs/count.py | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/docs/count.py b/docs/count.py index 6a5d256b..7118216a 100644 --- a/docs/count.py +++ b/docs/count.py @@ -23,6 +23,13 @@ def _colored_string(string: str, color: str or int) -> str: return "\033[%dm%s\033[0m" % (color, string) +def gr(string, flag): + if flag: + return _colored_string(string, "green") + else: + return _colored_string(string, "red") + + def find_all_modules(): modules = {} children = {} @@ -79,20 +86,46 @@ def create_rst_file(modules, name, children): def check_file(m, name): + names = name.split('.') + test_name = "test." + ".".join(names[1:-1]) + ".test_" + names[-1] + try: + __import__(test_name) + tm = sys.modules[test_name] + except ModuleNotFoundError: + tm = None + tested = tm is not None + funcs = {} + classes = {} for item, obj in inspect.getmembers(m): - if inspect.isclass(obj) and obj.__module__ == name: - print(obj) - if inspect.isfunction(obj) and obj.__module__ == name: - print("FUNC", obj) + if inspect.isclass(obj) and obj.__module__ == name and not obj.__name__.startswith('_'): + this = (obj.__doc__ is not None, tested and obj.__name__ in dir(tm), {}) + for i in dir(obj): + func = getattr(obj, i) + if inspect.isfunction(func) and not i.startswith('_'): + this[2][i] = (func.__doc__ is not None, False) + classes[obj.__name__] = this + if inspect.isfunction(obj) and obj.__module__ == name and not obj.__name__.startswith('_'): + this = (obj.__doc__ is not None, tested and obj.__name__ in dir(tm)) # docs + funcs[obj.__name__] = this + return funcs, classes -def check_files(modules): +def check_files(modules, out=sys.stdout): for name in sorted(modules.keys()): - if name == 'fastNLP.core.utils': - check_file(modules[name], name) + print(name, file=out) + funcs, classes = check_file(modules[name], name) + for f in funcs: + print("%-30s \t %s \t %s" % (f, gr("文档", funcs[f][0]), gr("测试", funcs[f][1])), file=out) + for c in classes: + print("%-30s \t %s \t %s" % (c, gr("文档", classes[c][0]), gr("测试", classes[c][1])), file=out) + methods = classes[c][2] + for f in methods: + print(" %-28s \t %s" % (f, gr("文档", methods[f][0])), file=out) + print(file=out) def main(): + sys.path.append("..") print(_colored_string('Getting modules...', "Blue")) modules, to_doc, children = find_all_modules() print(_colored_string('Done!', "Green")) From 5bbfb92a300d8d9aeba7f45ae4e2bf8dad19fcb4 Mon Sep 17 00:00:00 2001 From: xxliu Date: Fri, 6 Sep 2019 13:08:57 +0800 Subject: [PATCH 47/92] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E8=A7=84=E8=8C=83?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E4=BF=AE=E6=94=B9=E6=B5=8B=E8=AF=95=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=B7=AF=E5=BE=84=E4=BB=A5=E5=8C=B9=E9=85=8Dgithub?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- reproduction/coreference_resolution/train.py | 4 ++-- test/io/loader/test_coreference_loader.py | 2 +- test/io/pipe/test_coreference.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/reproduction/coreference_resolution/train.py b/reproduction/coreference_resolution/train.py index c91f7109..cd4b65a5 100644 --- a/reproduction/coreference_resolution/train.py +++ b/reproduction/coreference_resolution/train.py @@ -45,7 +45,7 @@ if __name__ == "__main__": print("数据集划分：\ntrain:", str(len(data_bundle.get_dataset("train"))), "\ndev:" + str(len(data_bundle.get_dataset("dev"))) + "\ntest:" + str(len(data_bundle.get_dataset('test')))) # print(data_info) - model = Model(data_bundle.vocabs['vocab'], config) + model = Model(data_bundle.get_vocab("vocab"), config) print(model) loss = SoftmaxLoss() @@ -60,7 +60,7 @@ if __name__ == "__main__": loss=loss, metrics=metric, check_code_level=-1, sampler=None, batch_size=1, device=torch.device("cuda:" + config.cuda), metric_key='f', n_epochs=config.epoch, optimizer=optim, - save_path='/remote-home/xxliu/pycharm/fastNLP/fastNLP/reproduction/coreference_resolution/save', + save_path= None, callbacks=[lr_decay_callback, GradientClipCallback(clip_value=5)]) print() diff --git a/test/io/loader/test_coreference_loader.py b/test/io/loader/test_coreference_loader.py index 48551f3e..d827e947 100644 --- a/test/io/loader/test_coreference_loader.py +++ b/test/io/loader/test_coreference_loader.py @@ -4,7 +4,7 @@ import unittest class TestCR(unittest.TestCase): def test_load(self): - test_root = "../../data_for_tests/coreference/" + test_root = "test/data_for_tests/coreference/" train_path = test_root+"coreference_train.json" dev_path = test_root+"coreference_dev.json" test_path = test_root+"coreference_test.json" diff --git a/test/io/pipe/test_coreference.py b/test/io/pipe/test_coreference.py index 1c53f2b0..517be993 100644 --- a/test/io/pipe/test_coreference.py +++ b/test/io/pipe/test_coreference.py @@ -11,7 +11,7 @@ class TestCR(unittest.TestCase): char_path = None config = Config() - file_root_path = "../../data_for_tests/coreference/" + file_root_path = "test/data_for_tests/coreference/" train_path = file_root_path + "coreference_train.json" dev_path = file_root_path + "coreference_dev.json" test_path = file_root_path + "coreference_test.json" From 2fbc1d78518d6f75080da8bdab6ddaecd5d3cd87 Mon Sep 17 00:00:00 2001 From: unknown <793736331@qq.com> Date: Sat, 7 Sep 2019 15:22:43 +0800 Subject: [PATCH 48/92] change the print format for dataset and instance --- fastNLP/core/dataset.py | 101 ++++++++++++++-------------- fastNLP/core/instance.py | 20 +++--- fastNLP/core/utils.py | 138 +++++++++++++++++++++++++++------------ 3 files changed, 155 insertions(+), 104 deletions(-) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index ebdc780f..36852b93 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -300,13 +300,14 @@ from .field import FieldArray from .field import SetInputOrTargetException from .instance import Instance from .utils import _get_func_signature +from .utils import pretty_table_printer class DataSet(object): """ fastNLP的数据容器，详细的使用方法见文档 :doc:`fastNLP.core.dataset` """ - + def __init__(self, data=None): """ @@ -326,26 +327,26 @@ class DataSet(object): for ins in data: assert isinstance(ins, Instance), "Must be Instance type, not {}.".format(type(ins)) self.append(ins) - + else: raise ValueError("data only be dict or list type.") - + def __contains__(self, item): return item in self.field_arrays - + def __iter__(self): def iter_func(): for idx in range(len(self)): yield self[idx] - + return iter_func() - + def _inner_iter(self): class Iter_ptr: def __init__(self, dataset, idx): self.dataset = dataset self.idx = idx - + def __getitem__(self, item): assert item in self.dataset.field_arrays, "no such field:{} in Instance {}".format(item, self.dataset[ self.idx]) @@ -358,13 +359,13 @@ class DataSet(object): def __repr__(self): return self.dataset[self.idx].__repr__() - + def inner_iter_func(): for idx in range(len(self)): yield Iter_ptr(self, idx) - + return inner_iter_func() - + def __getitem__(self, idx): """给定int的index，返回一个Instance; 给定slice，返回包含这个slice内容的新的DataSet。 @@ -397,20 +398,20 @@ class DataSet(object): return dataset else: raise KeyError("Unrecognized type {} for idx in __getitem__ method".format(type(idx))) - + def __getattr__(self, item): # Not tested. Don't use !! if item == "field_arrays": raise AttributeError if isinstance(item, str) and item in self.field_arrays: return self.field_arrays[item] - + def __setstate__(self, state): self.__dict__ = state - + def __getstate__(self): return self.__dict__ - + def __len__(self): """Fetch the length of the dataset. @@ -420,16 +421,10 @@ class DataSet(object): return 0 field = iter(self.field_arrays.values()).__next__() return len(field) - - def __inner_repr__(self): - if len(self) < 20: - return ",\n".join([ins.__repr__() for ins in self]) - else: - return self[:5].__inner_repr__() + "\n...\n" + self[-5:].__inner_repr__() - + def __repr__(self): - return "DataSet(" + self.__inner_repr__() + ")" - + return str(pretty_table_printer(self)) + def append(self, instance): """ 将一个instance对象append到DataSet后面。 @@ -454,7 +449,7 @@ class DataSet(object): except AppendToTargetOrInputException as e: logger.error(f"Cannot append to field:{name}.") raise e - + def add_fieldarray(self, field_name, fieldarray): """ 将fieldarray添加到DataSet中. @@ -469,7 +464,7 @@ class DataSet(object): raise RuntimeError(f"The field to add must have the same size as dataset. " f"Dataset size {len(self)} != field size {len(fieldarray)}") self.field_arrays[field_name] = fieldarray - + def add_field(self, field_name, fields, padder=AutoPadder(), is_input=False, is_target=False, ignore_type=False): """ 新增一个field @@ -481,14 +476,14 @@ class DataSet(object): :param bool is_target: 新加入的field是否是target :param bool ignore_type: 是否忽略对新加入的field的类型检查 """ - + if len(self.field_arrays) != 0: if len(self) != len(fields): raise RuntimeError(f"The field to add must have the same size as dataset. " f"Dataset size {len(self)} != field size {len(fields)}") self.field_arrays[field_name] = FieldArray(field_name, fields, is_target=is_target, is_input=is_input, padder=padder, ignore_type=ignore_type) - + def delete_instance(self, index): """ 删除第index个instance @@ -504,7 +499,7 @@ class DataSet(object): for field in self.field_arrays.values(): field.pop(index) return self - + def delete_field(self, field_name): """ 删除名为field_name的field @@ -538,7 +533,7 @@ class DataSet(object): if isinstance(field_name, str): return field_name in self.field_arrays return False - + def get_field(self, field_name): """ 获取field_name这个field @@ -549,7 +544,7 @@ class DataSet(object): if field_name not in self.field_arrays: raise KeyError("Field name {} not found in DataSet".format(field_name)) return self.field_arrays[field_name] - + def get_all_fields(self): """ 返回一个dict，key为field_name, value为对应的 :class:`~fastNLP.FieldArray` @@ -557,7 +552,7 @@ class DataSet(object): :return dict: 返回如上所述的字典 """ return self.field_arrays - + def get_field_names(self) -> list: """ 返回一个list，包含所有 field 的名字 @@ -565,7 +560,7 @@ class DataSet(object): :return list: 返回如上所述的列表 """ return sorted(self.field_arrays.keys()) - + def get_length(self): """ 获取DataSet的元素数量 @@ -573,7 +568,7 @@ class DataSet(object): :return: int: DataSet中Instance的个数。 """ return len(self) - + def rename_field(self, field_name, new_field_name): """ 将某个field重新命名. @@ -587,7 +582,7 @@ class DataSet(object): else: raise KeyError("DataSet has no field named {}.".format(field_name)) return self - + def set_target(self, *field_names, flag=True, use_1st_ins_infer_dim_type=True): """ 将field_names的field设置为target @@ -614,7 +609,7 @@ class DataSet(object): else: raise KeyError("{} is not a valid field name.".format(name)) return self - + def set_input(self, *field_names, flag=True, use_1st_ins_infer_dim_type=True): """ 将field_names的field设置为input:: @@ -638,7 +633,7 @@ class DataSet(object): else: raise KeyError("{} is not a valid field name.".format(name)) return self - + def set_ignore_type(self, *field_names, flag=True): """ 将field设置为忽略类型状态。当某个field被设置了ignore_type, 则在被设置为target或者input时将不进行类型检查， @@ -655,7 +650,7 @@ class DataSet(object): else: raise KeyError("{} is not a valid field name.".format(name)) return self - + def set_padder(self, field_name, padder): """ 为field_name设置padder:: @@ -671,7 +666,7 @@ class DataSet(object): raise KeyError("There is no field named {}.".format(field_name)) self.field_arrays[field_name].set_padder(padder) return self - + def set_pad_val(self, field_name, pad_val): """ 为某个field设置对应的pad_val. @@ -683,7 +678,7 @@ class DataSet(object): raise KeyError("There is no field named {}.".format(field_name)) self.field_arrays[field_name].set_pad_val(pad_val) return self - + def get_input_name(self): """ 返回所有is_input被设置为True的field名称 @@ -691,7 +686,7 @@ class DataSet(object): :return list: 里面的元素为被设置为input的field名称 """ return [name for name, field in self.field_arrays.items() if field.is_input] - + def get_target_name(self): """ 返回所有is_target被设置为True的field名称 @@ -699,7 +694,7 @@ class DataSet(object): :return list: 里面的元素为被设置为target的field名称 """ return [name for name, field in self.field_arrays.items() if field.is_target] - + def apply_field(self, func, field_name, new_field_name=None, **kwargs): """ 将DataSet中的每个instance中的名为 `field_name` 的field传给func，并获取它的返回值。 @@ -728,16 +723,16 @@ class DataSet(object): results.append(func(ins[field_name])) except Exception as e: if idx != -1: - logger.error("Exception happens at the `{}`th(from 1) instance.".format(idx+1)) + logger.error("Exception happens at the `{}`th(from 1) instance.".format(idx + 1)) raise e if not (new_field_name is None) and len(list(filter(lambda x: x is not None, results))) == 0: # all None raise ValueError("{} always return None.".format(_get_func_signature(func=func))) - + if new_field_name is not None: self._add_apply_field(results, new_field_name, kwargs) - + return results - + def _add_apply_field(self, results, new_field_name, kwargs): """ 将results作为加入到新的field中，field名称为new_field_name @@ -769,7 +764,7 @@ class DataSet(object): self.add_field(field_name=new_field_name, fields=results, is_input=extra_param.get("is_input", None), is_target=extra_param.get("is_target", None), ignore_type=extra_param.get("ignore_type", False)) - + def apply(self, func, new_field_name=None, **kwargs): """ 将DataSet中每个instance传入到func中，并获取它的返回值. @@ -801,13 +796,13 @@ class DataSet(object): # results = [func(ins) for ins in self._inner_iter()] if not (new_field_name is None) and len(list(filter(lambda x: x is not None, results))) == 0: # all None raise ValueError("{} always return None.".format(_get_func_signature(func=func))) - + if new_field_name is not None: self._add_apply_field(results, new_field_name, kwargs) - + return results - def add_seq_len(self, field_name:str, new_field_name=Const.INPUT_LEN): + def add_seq_len(self, field_name: str, new_field_name=Const.INPUT_LEN): """ 将使用len()直接对field_name中每个元素作用，将其结果作为seqence length, 并放入seq_len这个field。 @@ -844,7 +839,7 @@ class DataSet(object): return dataset else: return DataSet() - + def split(self, ratio, shuffle=True): """ 将DataSet按照ratio的比例拆分，返回两个DataSet @@ -870,9 +865,9 @@ class DataSet(object): for field_name in self.field_arrays: train_set.field_arrays[field_name].to(self.field_arrays[field_name]) dev_set.field_arrays[field_name].to(self.field_arrays[field_name]) - + return train_set, dev_set - + def save(self, path): """ 保存DataSet. @@ -881,7 +876,7 @@ class DataSet(object): """ with open(path, 'wb') as f: pickle.dump(self, f) - + @staticmethod def load(path): r""" diff --git a/fastNLP/core/instance.py b/fastNLP/core/instance.py index 9460b5e4..3cf7ab45 100644 --- a/fastNLP/core/instance.py +++ b/fastNLP/core/instance.py @@ -3,10 +3,13 @@ instance 模块实现了Instance 类在fastNLP中对应sample。一个sample可 便于理解的例子可以参考文档 :doc:`fastNLP.core.dataset` 中的表格 """ + __all__ = [ "Instance" ] +from .utils import pretty_table_printer + class Instance(object): """ @@ -20,11 +23,11 @@ class Instance(object): >>>ins.add_field("field_3", [3, 3, 3]) >>>ins = Instance(**{'x1': 1, 'x2':np.zeros((3, 4))}) """ - + def __init__(self, **fields): - + self.fields = fields - + def add_field(self, field_name, field): """ 向Instance中增加一个field @@ -41,18 +44,15 @@ class Instance(object): :return: 一个迭代器 """ return self.fields.items() - + def __getitem__(self, name): if name in self.fields: return self.fields[name] else: raise KeyError("{} not found".format(name)) - + def __setitem__(self, name, field): return self.add_field(name, field) - + def __repr__(self): - s = '\'' - return "{" + ",\n".join( - "\'" + field_name + "\': " + str(self.fields[field_name]) + \ - f" type={(str(type(self.fields[field_name]))).split(s)[1]}" for field_name in self.fields) + "}" + return str(pretty_table_printer(self)) diff --git a/fastNLP/core/utils.py b/fastNLP/core/utils.py index 814e0bd5..dd2afab7 100644 --- a/fastNLP/core/utils.py +++ b/fastNLP/core/utils.py @@ -1,6 +1,7 @@ """ utils模块实现了 fastNLP 内部和外部所需的很多工具。其中用户可以使用的是 :func:`cache_results` 修饰器。 """ + __all__ = [ "cache_results", "seq_len_to_mask", @@ -12,12 +13,12 @@ import inspect import os import warnings from collections import Counter, namedtuple - import numpy as np import torch import torch.nn as nn from typing import List from ._logger import logger +from prettytable import PrettyTable _CheckRes = namedtuple('_CheckRes', ['missing', 'unused', 'duplicated', 'required', 'all_needed', 'varargs']) @@ -25,27 +26,27 @@ _CheckRes = namedtuple('_CheckRes', ['missing', 'unused', 'duplicated', 'require class Option(dict): """a dict can treat keys as attributes""" - + def __getattr__(self, item): try: return self.__getitem__(item) except KeyError: raise AttributeError(item) - + def __setattr__(self, key, value): if key.startswith('__') and key.endswith('__'): raise AttributeError(key) self.__setitem__(key, value) - + def __delattr__(self, item): try: self.pop(item) except KeyError: raise AttributeError(item) - + def __getstate__(self): return self - + def __setstate__(self, state): self.update(state) @@ -112,13 +113,13 @@ def cache_results(_cache_fp, _refresh=False, _verbose=1): :param int _verbose: 是否打印cache的信息。 :return: """ - + def wrapper_(func): signature = inspect.signature(func) for key, _ in signature.parameters.items(): if key in ('_cache_fp', '_refresh', '_verbose'): raise RuntimeError("The function decorated by cache_results cannot have keyword `{}`.".format(key)) - + def wrapper(*args, **kwargs): if '_cache_fp' in kwargs: cache_filepath = kwargs.pop('_cache_fp') @@ -136,7 +137,7 @@ def cache_results(_cache_fp, _refresh=False, _verbose=1): else: verbose = _verbose refresh_flag = True - + if cache_filepath is not None and refresh is False: # load data if os.path.exists(cache_filepath): @@ -145,7 +146,7 @@ def cache_results(_cache_fp, _refresh=False, _verbose=1): if verbose == 1: logger.info("Read cache from {}.".format(cache_filepath)) refresh_flag = False - + if refresh_flag: results = func(*args, **kwargs) if cache_filepath is not None: @@ -155,11 +156,11 @@ def cache_results(_cache_fp, _refresh=False, _verbose=1): with open(cache_filepath, 'wb') as f: _pickle.dump(results, f) logger.info("Save cache to {}.".format(cache_filepath)) - + return results - + return wrapper - + return wrapper_ @@ -187,6 +188,7 @@ def _save_model(model, model_name, save_dir, only_param=False): torch.save(model, model_path) model.to(_model_device) + def _move_model_to_device(model, device): """ 将model移动到device @@ -211,7 +213,7 @@ def _move_model_to_device(model, device): """ # if isinstance(model, torch.nn.parallel.DistributedDataParallel): # raise RuntimeError("model of `torch.nn.parallel.DistributedDataParallel` is not supported right now.") - + if device is None: if isinstance(model, torch.nn.DataParallel): model.cuda() @@ -220,10 +222,10 @@ def _move_model_to_device(model, device): if not torch.cuda.is_available() and ( device != 'cpu' or (isinstance(device, torch.device) and device.type != 'cpu')): raise ValueError("There is no usable gpu. set `device` as `cpu` or `None`.") - + if isinstance(model, torch.nn.DataParallel): raise RuntimeError("When model is `torch.nn.DataParallel`, the device has to be `None`.") - + if isinstance(device, int): assert device > -1, "device can only be non-negative integer" assert torch.cuda.device_count() > device, "Only has {} gpus, cannot use device {}.".format( @@ -267,7 +269,7 @@ def _get_model_device(model): """ # TODO 这个函数存在一定的风险，因为同一个模型可能存在某些parameter不在显卡中，比如BertEmbedding. 或者跨显卡 assert isinstance(model, nn.Module) - + parameters = list(model.parameters()) if len(parameters) == 0: return None @@ -427,10 +429,10 @@ def _move_dict_value_to_device(*args, device: torch.device, non_blocking=False): """ if not torch.cuda.is_available(): return - + if not isinstance(device, torch.device): raise TypeError(f"device must be `torch.device`, got `{type(device)}`") - + for arg in args: if isinstance(arg, dict): for key, value in arg.items(): @@ -445,10 +447,10 @@ class _CheckError(Exception): _CheckError. Used in losses.LossBase, metrics.MetricBase. """ - + def __init__(self, check_res: _CheckRes, func_signature: str): errs = [f'Problems occurred when calling `{func_signature}`'] - + if check_res.varargs: errs.append(f"\tvarargs: {check_res.varargs}(Does not support pass positional arguments, please delete it)") if check_res.missing: @@ -457,9 +459,9 @@ class _CheckError(Exception): errs.append(f"\tduplicated param: {check_res.duplicated}") if check_res.unused: errs.append(f"\tunused param: {check_res.unused}") - + Exception.__init__(self, '\n'.join(errs)) - + self.check_res = check_res self.func_signature = func_signature @@ -479,7 +481,7 @@ def _check_loss_evaluate(prev_func_signature: str, func_signature: str, check_re # if check_res.varargs: # errs.append(f"\tvarargs: *{check_res.varargs}") # suggestions.append(f"Does not support pass positional arguments, please delete *{check_res.varargs}.") - + if check_res.unused: for _unused in check_res.unused: if _unused in target_dict: @@ -490,7 +492,7 @@ def _check_loss_evaluate(prev_func_signature: str, func_signature: str, check_re unuseds.append(f"\tunused field: {_unused_field}") if _unused_param: unuseds.append(f"\tunused param: {_unused_param}") # output from predict or forward - + module_name = func_signature.split('.')[0] if check_res.missing: errs.append(f"\tmissing param: {check_res.missing}") @@ -511,7 +513,7 @@ def _check_loss_evaluate(prev_func_signature: str, func_signature: str, check_re mapped_missing.append(_miss) else: unmapped_missing.append(_miss) - + for _miss in mapped_missing + unmapped_missing: if _miss in dataset: suggestions.append(f"Set `{_miss}` as target.") @@ -524,17 +526,17 @@ def _check_loss_evaluate(prev_func_signature: str, func_signature: str, check_re else: _tmp = f'Provide `{_miss}` in DataSet or output of {prev_func_signature}.' suggestions.append(_tmp) - + if check_res.duplicated: errs.append(f"\tduplicated param: {check_res.duplicated}.") suggestions.append(f"Delete {check_res.duplicated} in the output of " f"{prev_func_signature} or do not set {check_res.duplicated} as targets. ") - + if len(errs) > 0: errs.extend(unuseds) elif check_level == STRICT_CHECK_LEVEL: errs.extend(unuseds) - + if len(errs) > 0: errs.insert(0, f'Problems occurred when calling {func_signature}') sugg_str = "" @@ -561,11 +563,11 @@ def _check_loss_evaluate(prev_func_signature: str, func_signature: str, check_re def _check_forward_error(forward_func, batch_x, dataset, check_level): check_res = _check_arg_dict_list(forward_func, batch_x) func_signature = _get_func_signature(forward_func) - + errs = [] suggestions = [] _unused = [] - + # if check_res.varargs: # errs.append(f"\tvarargs: {check_res.varargs}") # suggestions.append(f"Does not support pass positional arguments, please delete *{check_res.varargs}.") @@ -586,14 +588,14 @@ def _check_forward_error(forward_func, batch_x, dataset, check_level): # _tmp += f"Or you might find it in `unused field:`, you can use DataSet.rename_field() to " \ # f"rename the field in `unused field:`." suggestions.append(_tmp) - + if check_res.unused: _unused = [f"\tunused field: {check_res.unused}"] if len(errs) > 0: errs.extend(_unused) elif check_level == STRICT_CHECK_LEVEL: errs.extend(_unused) - + if len(errs) > 0: errs.insert(0, f'Problems occurred when calling {func_signature}') sugg_str = "" @@ -641,7 +643,7 @@ def seq_len_to_mask(seq_len, max_len=None): max_len = int(max_len) if max_len else int(seq_len.max()) broad_cast_seq_len = np.tile(np.arange(max_len), (len(seq_len), 1)) mask = broad_cast_seq_len < seq_len.reshape(-1, 1) - + elif isinstance(seq_len, torch.Tensor): assert seq_len.dim() == 1, f"seq_len can only have one dimension, got {seq_len.dim() == 1}." batch_size = seq_len.size(0) @@ -650,7 +652,7 @@ def seq_len_to_mask(seq_len, max_len=None): mask = broad_cast_seq_len.lt(seq_len.unsqueeze(1)) else: raise TypeError("Only support 1-d numpy.ndarray or 1-d torch.Tensor.") - + return mask @@ -658,24 +660,25 @@ class _pseudo_tqdm: """ 当无法引入tqdm，或者Trainer中设置use_tqdm为false的时候，用该方法打印数据 """ + def __init__(self, **kwargs): self.logger = logger - + def write(self, info): self.logger.info(info) - + def set_postfix_str(self, info): self.logger.info(info) - + def __getattr__(self, item): def pass_func(*args, **kwargs): pass - + return pass_func - + def __enter__(self): return self - + def __exit__(self, exc_type, exc_val, exc_tb): del self @@ -749,3 +752,56 @@ def get_seq_len(words, pad_value=0): """ mask = words.ne(pad_value) return mask.sum(dim=-1) + + +def pretty_table_printer(dataset_or_ins) -> PrettyTable: + """ + :param dataset_or_ins: 传入一个dataSet或者instance + ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2], field_3=["a", "b", "c"]) + +-----------+-----------+-----------------+ + | field_1 | field_2 | field_3 | + +-----------+-----------+-----------------+ + | [1, 1, 1] | [2, 2, 2] | ['a', 'b', 'c'] | + +-----------+-----------+-----------------+ + :return: 以 pretty table的形式返回根据terminal大小进行自动截断 + """ + x = PrettyTable() + try: + sz = os.get_terminal_size() + column = sz.columns + row = sz.lines + except OSError: + column = 144 + row = 11 + if type(dataset_or_ins).__name__ == "DataSet": + x.field_names = list(dataset_or_ins.field_arrays.keys()) + c_size = len(x.field_names) + for ins in dataset_or_ins: + x.add_row([sub_column(ins[k], column, c_size, k) for k in x.field_names]) + row -= 1 + if row < 0: + x.add_row(["..." for _ in range(c_size)]) + break + elif type(dataset_or_ins).__name__ == "Instance": + x.field_names = list(dataset_or_ins.fields.keys()) + c_size = len(x.field_names) + x.add_row([sub_column(dataset_or_ins[k], column, c_size, k) for k in x.field_names]) + + else: + raise Exception("only accept DataSet and Instance") + return x + + +def sub_column(string: str, c: int, c_size: int, title: str) -> str: + """ + :param string: 要被截断的字符串 + :param c: 命令行列数 + :param c_size: instance或dataset field数 + :param title: 列名 + :return: 对一个过长的列进行截断的结果 + """ + avg = max(int(c / c_size), len(title)) + string = str(string) + if len(string) > avg: + string = string[:(avg - 3)] + "..." + return string From 04a54df226763fa4f7b3ddb88f779008206698d8 Mon Sep 17 00:00:00 2001 From: yhcc Date: Sat, 7 Sep 2019 16:06:35 +0800 Subject: [PATCH 49/92] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f71e2223..db0b89ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ numpy>=1.14.2 torch>=1.0.0 tqdm>=4.28.1 nltk>=3.4.1 +prettytable>=0.7.2 requests spacy From 8c8e22cc9baa08a1c8ee9ba887717db41cce57b5 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Sat, 7 Sep 2019 18:47:03 +0800 Subject: [PATCH 50/92] =?UTF-8?q?DataSet=E4=B8=AD=E5=A2=9E=E5=8A=A0print?= =?UTF-8?q?=5Ffield=5Fmeta=E6=96=B9=E6=B3=95=EF=BC=8C=E4=BD=BF=E5=BE=97?= =?UTF-8?q?=E5=85=B6=E5=8F=AF=E4=BB=A5=E8=8E=B7=E5=8F=96field=E7=9A=84inpu?= =?UTF-8?q?t=E5=92=8Ctarget=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/dataset.py | 57 +++++++++++++++++++++++++++++++++++++++ fastNLP/core/field.py | 7 +++-- requirements.txt | 1 + test/core/test_dataset.py | 15 ++++++++++- 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 36852b93..2b548f22 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -301,6 +301,7 @@ from .field import SetInputOrTargetException from .instance import Instance from .utils import _get_func_signature from .utils import pretty_table_printer +from prettytable import PrettyTable class DataSet(object): @@ -425,6 +426,62 @@ class DataSet(object): def __repr__(self): return str(pretty_table_printer(self)) + def print_field_meta(self): + """ + 输出当前field的meta信息, 形似下列的输出 + + +-------------+-------+-------+ + | field_names | x | y | + +-------------+-------+-------+ + | is_input | True | False | + | is_target | False | False | + | ignore_type | False | | + | pad_value | 0 | | + +-------------+-------+-------+ + + field_names: DataSet中field的名称 + is_input: field是否为input + is_target: field是否为target + ignore_type: 是否忽略该field的type, 一般仅在该field至少为input或target时才有意义 + pad_value: 该field的pad的值，仅在该field为input或target时有意义 + + :return: + """ + if len(self.field_arrays)>0: + field_names = ['field_names'] + is_inputs = ['is_input'] + is_targets = ['is_target'] + pad_values = ['pad_value'] + ignore_types = ['ignore_type'] + + for name, field_array in self.field_arrays.items(): + field_names.append(name) + if field_array.is_input: + is_inputs.append(True) + else: + is_inputs.append(False) + if field_array.is_target: + is_targets.append(True) + else: + is_targets.append(False) + + if (field_array.is_input or field_array.is_target) and field_array.padder is not None: + pad_values.append(field_array.padder.get_pad_val()) + else: + pad_values.append(' ') + + if field_array._ignore_type: + ignore_types.append(True) + elif field_array.is_input or field_array.is_target: + ignore_types.append(False) + else: + ignore_types.append(' ') + table = PrettyTable(field_names=field_names) + fields = [is_inputs, is_targets, ignore_types, pad_values] + for field in fields: + table.add_row(field) + logger.info(table) + def append(self, instance): """ 将一个instance对象append到DataSet后面。 diff --git a/fastNLP/core/field.py b/fastNLP/core/field.py index 82fcc523..1835bafa 100644 --- a/fastNLP/core/field.py +++ b/fastNLP/core/field.py @@ -53,7 +53,7 @@ class FieldArray: self.content = _content self._ignore_type = ignore_type # 根据input的情况设置input，target等 - self._cell_ndim = None # 多少维度 + self._cell_ndim = None # 多少维度， 如果value是1, dim为0; 如果value是[1, 2], dim=2 self.dtype = None # 最内层的element都是什么类型的 self._use_1st_ins_infer_dim_type = bool(use_1st_ins_infer_dim_type) self._is_input = False @@ -484,7 +484,10 @@ class Padder: def set_pad_val(self, pad_val): self.pad_val = pad_val - + + def get_pad_val(self): + return self.pad_val + @abstractmethod def __call__(self, contents, field_name, field_ele_dtype, dim: int): """ diff --git a/requirements.txt b/requirements.txt index f71e2223..bdd4a9e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ tqdm>=4.28.1 nltk>=3.4.1 requests spacy +prettytable>=0.7.2 \ No newline at end of file diff --git a/test/core/test_dataset.py b/test/core/test_dataset.py index 059d52d2..9820eff6 100644 --- a/test/core/test_dataset.py +++ b/test/core/test_dataset.py @@ -229,4 +229,17 @@ class TestDataSetIter(unittest.TestCase): def test__repr__(self): ds = DataSet({"x": [[1, 2, 3, 4]] * 10, "y": [[5, 6]] * 10}) for iter in ds: - self.assertEqual(iter.__repr__(), "{'x': [1, 2, 3, 4] type=list,\n'y': [5, 6] type=list}") + self.assertEqual(iter.__repr__(), """+--------------+--------+ +| x | y | ++--------------+--------+ +| [1, 2, 3, 4] | [5, 6] | ++--------------+--------+""") + + +class TestDataSetFieldMeta(unittest.TestCase): + def test_print_field_meta(self): + ds = DataSet({"x": [[1, 2, 3, 4]] * 10, "y": [[5, 6]] * 10}) + ds.print_field_meta() + + ds.set_input('x') + ds.print_field_meta() From 53bcc0b26a9b4e5560946ef2a4b7134bc589a7e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E7=9A=84=E8=8F=9C=E9=B8=A1=E7=BA=A2?= =?UTF-8?q?=E7=91=9E?= Date: Sun, 8 Sep 2019 16:28:15 +0800 Subject: [PATCH 51/92] loader-pr --- fastNLP/io/file_reader.py | 40 +++++++++++++++++++-------------------- fastNLP/io/test.csv | 6 ++++++ 2 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 fastNLP/io/test.csv diff --git a/fastNLP/io/file_reader.py b/fastNLP/io/file_reader.py index 0ae0a319..17a0a6ca 100644 --- a/fastNLP/io/file_reader.py +++ b/fastNLP/io/file_reader.py @@ -2,6 +2,7 @@ 此模块用于给其它模块提供读取文件的函数，没有为用户提供 API """ import json +import csv def _read_csv(path, encoding='utf-8', headers=None, sep=',', dropna=True): @@ -16,27 +17,26 @@ def _read_csv(path, encoding='utf-8', headers=None, sep=',', dropna=True): :if False, raise ValueError when reading invalid data. default: True :return: generator, every time yield (line number, csv item) """ - with open(path, 'r', encoding=encoding) as f: - start_idx = 0 - if headers is None: - headers = f.readline().rstrip('\r\n') - headers = headers.split(sep) - start_idx += 1 - elif not isinstance(headers, (list, tuple)): - raise TypeError("headers should be list or tuple, not {}." \ + f = csv.reader(open(path, encoding=encoding), delimiter=sep) + start_idx = 0 + if headers is None: + headers = next(f) + start_idx += 1 + elif not isinstance(headers, (list, tuple)): + raise TypeError("headers should be list or tuple, not {}." \ .format(type(headers))) - for line_idx, line in enumerate(f, start_idx): - contents = line.rstrip('\r\n').split(sep) - if len(contents) != len(headers): - if dropna: - continue - else: - raise ValueError("Line {} has {} parts, while header has {} parts." \ - .format(line_idx, len(contents), len(headers))) - _dict = {} - for header, content in zip(headers, contents): - _dict[header] = content - yield line_idx, _dict + for line_idx, line in enumerate(f, start_idx): + contents = line + if len(contents) != len(headers): + if dropna: + continue + else: + raise ValueError("Line {} has {} parts, while header has {} parts." \ + .format(line_idx, len(contents), len(headers))) + _dict = {} + for header, content in zip(headers, contents): + _dict[header] = content + yield line_idx, _dict def _read_json(path, encoding='utf-8', fields=None, dropna=True): diff --git a/fastNLP/io/test.csv b/fastNLP/io/test.csv new file mode 100644 index 00000000..88293b2f --- /dev/null +++ b/fastNLP/io/test.csv @@ -0,0 +1,6 @@ +a b +1 "Contrary to other reviews, I have zero complaints about the service or the prices. I have been getting tire service here for the past 5 years now, and compared to my experience with places like Pep Boys, these guys are experienced and know what they're doing. \nAlso, this is one place that I do not feel like I am being taken advantage of, just because of my gender. Other auto mechanics have been notorious for capitalizing on my ignorance of cars, and have sucked my bank account dry. But here, my service and road coverage has all been well explained - and let up to me to decide. \nAnd they just renovated the waiting room. It looks a lot better than it did in previous years." +2 "Last summer I had an appointment to get new tires and had to wait a super long time. I also went in this week for them to fix a minor problem with a tire they put on. They \""fixed\"" it for free, and the very next morning I had the same issue. I called to complain, and the \""manager\"" didn't even apologize!!! So frustrated. Never going back. They seem overpriced, too." +3 "Friendly staff, same starbucks fair you get anywhere else. Sometimes the lines can get long." +4 "The food is good. Unfortunately the service is very hit or miss. The main issue seems to be with the kitchen, the waiters and waitresses are often very apologetic for the long waits and it's pretty obvious that some of them avoid the tables after taking the initial order to avoid hearing complaints." +5 "Even when we didn't have a car Filene's Basement was worth the bus trip to the Waterfront. I always find something (usually I find 3-4 things and spend about $60) and better still, I am always still wearing the clothes and shoes 3 months later. \n\nI kind of suspect this is the best shopping in Pittsburgh; it's much better than the usual department stores, better than Marshall's and TJ Maxx and better than the Saks downtown, even when it has a sale. Selection, bargains AND quality.\n\nI like this Filene's better than Gabriel Brothers, which are harder to get to. Gabriel Brothers are a real discount shopper's challenge and I'm afraid I didn't live in Pittsburgh long enough to develop the necessary skills . . . Filene's was still up and running in June 2007 when I left town." \ No newline at end of file From 776840439f7f313a97900a52ce57c05cb72ca42f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=A6=E4=B9=A0=E7=9A=84=E8=8F=9C=E9=B8=A1=E7=BA=A2?= =?UTF-8?q?=E7=91=9E?= Date: Sun, 8 Sep 2019 16:28:53 +0800 Subject: [PATCH 52/92] loader-pr --- fastNLP/io/test.csv | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 fastNLP/io/test.csv diff --git a/fastNLP/io/test.csv b/fastNLP/io/test.csv deleted file mode 100644 index 88293b2f..00000000 --- a/fastNLP/io/test.csv +++ /dev/null @@ -1,6 +0,0 @@ -a b -1 "Contrary to other reviews, I have zero complaints about the service or the prices. I have been getting tire service here for the past 5 years now, and compared to my experience with places like Pep Boys, these guys are experienced and know what they're doing. \nAlso, this is one place that I do not feel like I am being taken advantage of, just because of my gender. Other auto mechanics have been notorious for capitalizing on my ignorance of cars, and have sucked my bank account dry. But here, my service and road coverage has all been well explained - and let up to me to decide. \nAnd they just renovated the waiting room. It looks a lot better than it did in previous years." -2 "Last summer I had an appointment to get new tires and had to wait a super long time. I also went in this week for them to fix a minor problem with a tire they put on. They \""fixed\"" it for free, and the very next morning I had the same issue. I called to complain, and the \""manager\"" didn't even apologize!!! So frustrated. Never going back. They seem overpriced, too." -3 "Friendly staff, same starbucks fair you get anywhere else. Sometimes the lines can get long." -4 "The food is good. Unfortunately the service is very hit or miss. The main issue seems to be with the kitchen, the waiters and waitresses are often very apologetic for the long waits and it's pretty obvious that some of them avoid the tables after taking the initial order to avoid hearing complaints." -5 "Even when we didn't have a car Filene's Basement was worth the bus trip to the Waterfront. I always find something (usually I find 3-4 things and spend about $60) and better still, I am always still wearing the clothes and shoes 3 months later. \n\nI kind of suspect this is the best shopping in Pittsburgh; it's much better than the usual department stores, better than Marshall's and TJ Maxx and better than the Saks downtown, even when it has a sale. Selection, bargains AND quality.\n\nI like this Filene's better than Gabriel Brothers, which are harder to get to. Gabriel Brothers are a real discount shopper's challenge and I'm afraid I didn't live in Pittsburgh long enough to develop the necessary skills . . . Filene's was still up and running in June 2007 when I left town." \ No newline at end of file From 1caa83d0cafbb5df6470627fab8dea86b56df36a Mon Sep 17 00:00:00 2001 From: ZikaiGuo <634500098@qq.com> Date: Sun, 8 Sep 2019 14:54:31 +0200 Subject: [PATCH 53/92] Update transformer.py --- fastNLP/modules/encoder/transformer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index d29a10c3..3d97c306 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -40,6 +40,8 @@ class TransformerEncoder(nn.Module): :param seq_mask: [batch, seq_len] :return: [batch, seq_len, model_size] """ + if seq_mask is None: # 防止后续乘法时出错 + seq_mask = 1 input = self.norm1(input) attention = self.atte(input, input, input, atte_mask_out) input = input + self.dropout(attention) From 917cedf808d2c03d1a2be4099ba7d1ef894f47d9 Mon Sep 17 00:00:00 2001 From: yh Date: Tue, 10 Sep 2019 10:38:02 +0800 Subject: [PATCH 54/92] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=B0=E7=9A=84tutor?= =?UTF-8?q?ial;=20=E5=88=A0=E9=99=A4=E5=90=84embedding=E4=B8=ADrequires=5F?= =?UTF-8?q?grad=E7=9A=84=E8=AE=BE=E7=BD=AE=EF=BC=8C=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E5=88=B0=E5=9F=BA=E7=B1=BB=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ..._callback.rst => tutorial_10_callback.rst} | 0 ...l_10_fitlog.rst => tutorial_11_fitlog.rst} | 0 .../tutorials/tutorial_1_data_preprocess.rst | 74 ++- .../tutorials/tutorial_2_load_dataset.rst | 150 ------ .../tutorials/tutorial_2_vocabulary.rst | 131 ++++++ .../source/tutorials/tutorial_3_embedding.rst | 437 +++++++++++++++--- .../tutorials/tutorial_4_load_dataset.rst | 219 +++++++++ ...izer.rst => tutorial_6_loss_optimizer.rst} | 0 ...l_8_metrics.rst => tutorial_7_metrics.rst} | 0 ...dels.rst => tutorial_8_modules_models.rst} | 0 ...beling.rst => tutorial_9_seq_labeling.rst} | 0 docs/source/user/tutorials.rst | 15 +- fastNLP/embeddings/bert_embedding.py | 38 -- fastNLP/embeddings/char_embedding.py | 58 --- fastNLP/embeddings/elmo_embedding.py | 23 +- fastNLP/embeddings/embedding.py | 4 + fastNLP/embeddings/stack_embedding.py | 26 +- fastNLP/embeddings/static_embedding.py | 26 +- fastNLP/io/loader/classification.py | 1 - fastNLP/io/loader/loader.py | 26 +- 20 files changed, 804 insertions(+), 424 deletions(-) rename docs/source/tutorials/{tutorial_9_callback.rst => tutorial_10_callback.rst} (100%) rename docs/source/tutorials/{tutorial_10_fitlog.rst => tutorial_11_fitlog.rst} (100%) delete mode 100644 docs/source/tutorials/tutorial_2_load_dataset.rst create mode 100644 docs/source/tutorials/tutorial_2_vocabulary.rst create mode 100644 docs/source/tutorials/tutorial_4_load_dataset.rst rename docs/source/tutorials/{tutorial_4_loss_optimizer.rst => tutorial_6_loss_optimizer.rst} (100%) rename docs/source/tutorials/{tutorial_8_metrics.rst => tutorial_7_metrics.rst} (100%) rename docs/source/tutorials/{tutorial_7_modules_models.rst => tutorial_8_modules_models.rst} (100%) rename docs/source/tutorials/{tutorial_6_seq_labeling.rst => tutorial_9_seq_labeling.rst} (100%) diff --git a/docs/source/tutorials/tutorial_9_callback.rst b/docs/source/tutorials/tutorial_10_callback.rst similarity index 100% rename from docs/source/tutorials/tutorial_9_callback.rst rename to docs/source/tutorials/tutorial_10_callback.rst diff --git a/docs/source/tutorials/tutorial_10_fitlog.rst b/docs/source/tutorials/tutorial_11_fitlog.rst similarity index 100% rename from docs/source/tutorials/tutorial_10_fitlog.rst rename to docs/source/tutorials/tutorial_11_fitlog.rst diff --git a/docs/source/tutorials/tutorial_1_data_preprocess.rst b/docs/source/tutorials/tutorial_1_data_preprocess.rst index 0ec63f87..dfc3bbbe 100644 --- a/docs/source/tutorials/tutorial_1_data_preprocess.rst +++ b/docs/source/tutorials/tutorial_1_data_preprocess.rst @@ -1,21 +1,20 @@ ============================== -使用DataSet预处理文本 +DataSet ============================== -:class:`~fastNLP.DataSet` 是fastNLP中用于承载数据的容器。可以将DataSet看做是一个表格， -每一行是一个sample (在fastNLP中被称为 :mod:`~fastNLP.core.instance` )， -每一列是一个feature (在fastNLP中称为 :mod:`~fastNLP.core.field` )。 +:class:`~fastNLP.DataSet` 是fastNLP用于承载数据的类，一般训练集、验证集和测试集会被加载为三个单独的:class:`~fastNLP.DataSet`对象。 + +:class:`~fastNLP.DataSet`中的数据组织形式类似一个表格，比如下面 :class:`~fastNLP.DataSet` 一共有3列，列在fastNLP中被称为field。 .. csv-table:: - :header: "sentence", "words", "seq_len" + :header: "raw_chars", "chars", "seq_len" - "This is the first instance .", "[This, is, the, first, instance, .]", 6 - "Second instance .", "[Second, instance, .]", 3 + "历任公司副总经理、总工程师，", "[历 任 公 司 副 总 经 理 、 总 工 程 师 ，]", 6 "Third instance .", "[Third, instance, .]", 3 "...", "[...]", "..." -上面是一个样例数据中 DataSet 的存储结构。其中它的每一行是一个 :class:`~fastNLP.Instance` 对象； 每一列是一个 :class:`~fastNLP.FieldArray` 对象。 - +每一行是一个instance (在fastNLP中被称为 :mod:`~fastNLP.core.Instance` )， +每一列是一个field (在fastNLP中称为 :mod:`~fastNLP.core.FieldArray` )。 ----------------------------- 数据集构建和删除 @@ -26,11 +25,23 @@ .. code-block:: python from fastNLP import DataSet - data = {'sentence':["This is the first instance .", "Second instance .", "Third instance ."], + data = {'raw_words':["This is the first instance .", "Second instance .", "Third instance ."], 'words': [['this', 'is', 'the', 'first', 'instance', '.'], ['Second', 'instance', '.'], ['Third', 'instance', '.']], 'seq_len': [6, 3, 3]} dataset = DataSet(data) # 传入的dict的每个key的value应该为具有相同长度的list + print(dataset) + +输出为:: + + +------------------------------+------------------------------------------------+---------+ + | raw_words | words | seq_len | + +------------------------------+------------------------------------------------+---------+ + | This is the first instance . | ['this', 'is', 'the', 'first', 'instance', ... | 6 | + | Second instance . | ['Second', 'instance', '.'] | 3 | + | Third instance . | ['Third', 'instance', '.'] | 3 | + +------------------------------+------------------------------------------------+---------+ + 我们还可以使用 :func:`~fastNLP.DataSet.append` 方法向数据集内增加数据 @@ -39,7 +50,7 @@ from fastNLP import DataSet from fastNLP import Instance dataset = DataSet() - instance = Instance(sentence="This is the first instance", + instance = Instance(raw_words="This is the first instance", words=['this', 'is', 'the', 'first', 'instance', '.'], seq_len=6) dataset.append(instance) @@ -52,10 +63,10 @@ from fastNLP import DataSet from fastNLP import Instance dataset = DataSet([ - Instance(sentence="This is the first instance", + Instance(raw_words="This is the first instance", words=['this', 'is', 'the', 'first', 'instance', '.'], seq_len=6), - Instance(sentence="Second instance .", + Instance(raw_words="Second instance .", words=['Second', 'instance', '.'], seq_len=3) ]) @@ -106,24 +117,49 @@ FastNLP 同样提供了多种删除数据的方法 :func:`~fastNLP.DataSet.drop` .. code-block:: python from fastNLP import DataSet - data = {'sentence':["This is the first instance .", "Second instance .", "Third instance ."]} + data = {'raw_words':["This is the first instance .", "Second instance .", "Third instance ."]} dataset = DataSet(data) # 将句子分成单词形式, 详见DataSet.apply()方法 - dataset.apply(lambda ins: ins['sentence'].split(), new_field_name='words') + dataset.apply(lambda ins: ins['raw_words'].split(), new_field_name='words') # 或使用DataSet.apply_field() - dataset.apply_field(lambda sent:sent.split(), field_name='sentence', new_field_name='words') + dataset.apply_field(lambda sent:sent.split(), field_name='raw_words', new_field_name='words') # 除了匿名函数，也可以定义函数传递进去 def get_words(instance): - sentence = instance['sentence'] + sentence = instance['raw_words'] words = sentence.split() return words dataset.apply(get_words, new_field_name='words') -除了手动处理数据集之外，你还可以使用 fastNLP 提供的各种 :class:`~fastNLP.io.base_loader.DataSetLoader` 来进行数据处理。 -详细请参考这篇教程 :doc:`使用DataSetLoader加载数据集 ` 。 +除了手动处理数据集之外，你还可以使用 fastNLP 提供的各种 :class:`~fastNLP.io.Loader`和:class:`~fastNLP.io.Pipe` 来进行数据处理。 +详细请参考这篇教程 :doc:`使用Loader和Pipe处理数据 ` 。 + +----------------------------- +fastNLP中field的命名习惯 +----------------------------- + +在英文任务中，fastNLP常用的field名称有: + + - raw_words: 表示的是原始的str。例如"This is a demo sentence ."。存在多个raw_words的情况，例如matching任务，它们会被定义为 + raw_words0, raw_words1。但在conll格式下，raw_words列也可能为["This", "is", "a", "demo", "sentence", "."]的形式。 + - words: 表示的是已经tokenize后的词语。例如["This", "is", "a", "demo", "sentence"], 但由于str并不能直接被神经网络所使用， + 所以words中的内容往往被转换为int，如[3, 10, 4, 2, 7, ...]等。多列words的情况，会被命名为words0, words1 + - target: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列。 + - seq_len: 一般用于表示words列的长度 + +在中文任务中，fastNLP常用的field名称有: + + - raw_chars: 表示的是原始的连续汉字序列。例如"这是一个示例。" + - chars: 表示已经切分为单独的汉字的序列。例如["这", "是", "一", "个", "示", "例", "。"]。但由于神经网络不能识别汉字，所以一般 + 该列会被转为int形式，如[3, 4, 5, 6, ...]。 + - raw_words: 如果原始汉字序列中已经包含了词语的边界，则该列称为raw_words。如"上海 浦东 开发 与 法制 建设 同步"。 + - words: 表示单独的汉字词语序列。例如["上海", "", "浦东", "开发", "与", "法制", "建设", ...]或[2, 3, 4, ...] + - target: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列。 + - seq_len: 表示输入序列的长度 + +# TODO 这一段移动到datasetiter那里 ----------------------------- DataSet与pad diff --git a/docs/source/tutorials/tutorial_2_load_dataset.rst b/docs/source/tutorials/tutorial_2_load_dataset.rst deleted file mode 100644 index 17ad6baf..00000000 --- a/docs/source/tutorials/tutorial_2_load_dataset.rst +++ /dev/null @@ -1,150 +0,0 @@ -======================================= -使用Loader和Pipe加载并处理数据集 -======================================= - -这一部分是一个关于如何加载数据集的教程 - -教程目录： - - - `Part I: 数据集容器DataBundle`_ - - `Part II: 加载数据集的基类Loader`_ - - `Part III: 不同格式类型的基础Loader`_ - - `Part IV: 使用Pipe对数据集进行预处理`_ - - `Part V: fastNLP封装好的Loader和Pipe`_ - - ------------------------------------- -Part I: 数据集容器DataBundle ------------------------------------- - -在fastNLP中，我们使用 :class:`~fastNLP.io.data_bundle.DataBundle` 来存储数据集信息。 -:class:`~fastNLP.io.data_bundle.DataBundle` 类包含了两个重要内容： `datasets` 和 `vocabs` 。 - -`datasets` 是一个 `key` 为数据集名称（如 `train` ， `dev` ，和 `test` 等）， `value` 为 :class:`~fastNLP.DataSet` 的字典。 - -`vocabs` 是一个 `key` 为词表名称（如 :attr:`fastNLP.Const.INPUT` 表示输入文本的词表名称， :attr:`fastNLP.Const.TARGET` 表示目标 -的真实标签词表的名称，等等）， `value` 为词表内容（ :class:`~fastNLP.Vocabulary` ）的字典。 - -------------------------------------- -Part II: 加载数据集的基类Loader -------------------------------------- - -在fastNLP中，我们采用 :class:`~fastNLP.io.loader.Loader` 来作为加载数据集的基类。 -:class:`~fastNLP.io.loader.Loader` 定义了各种Loader所需的API接口，开发者应该继承它实现各种的Loader。 -在各种数据集的Loader当中，至少应该编写如下内容: - - - _load 函数：从一个数据文件中读取数据，返回一个 :class:`~fastNLP.DataSet` - - load 函数：从文件或者文件夹中读取数据并组装成 :class:`~fastNLP.io.data_bundle.DataBundle` - -Loader的load函数返回的 :class:`~fastNLP.io.data_bundle.DataBundle` 里面包含了数据集的原始数据。 - --------------------------------------------------------- -Part III: 不同格式类型的基础Loader --------------------------------------------------------- - -:class:`~fastNLP.io.loader.CSVLoader` - 读取CSV类型的数据集文件。例子如下： - - .. code-block:: python - - from fastNLP.io.loader import CSVLoader - data_set_loader = CSVLoader( - headers=('words', 'target'), sep='\t' - ) - # 表示将CSV文件中每一行的第一项填入'words' field，第二项填入'target' field。 - # 其中每两项之间由'\t'分割开来 - - data_set = data_set_loader._load('path/to/your/file') - - 数据集内容样例如下 :: - - But it does not leave you with much . 1 - You could hate it for the same reason . 1 - The performances are an absolute joy . 4 - - -:class:`~fastNLP.io.loader.JsonLoader` - 读取Json类型的数据集文件，数据必须按行存储，每行是一个包含各类属性的Json对象。例子如下： - - .. code-block:: python - - from fastNLP.io.loader import JsonLoader - oader = JsonLoader( - fields={'sentence1': 'words1', 'sentence2': 'words2', 'gold_label': 'target'} - ) - # 表示将Json对象中'sentence1'、'sentence2'和'gold_label'对应的值赋给'words1'、'words2'、'target'这三个fields - - data_set = loader._load('path/to/your/file') - - 数据集内容样例如下 :: - - {"annotator_labels": ["neutral"], "captionID": "3416050480.jpg#4", "gold_label": "neutral", "pairID": "3416050480.jpg#4r1n", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is training his horse for a competition.", "sentence2_binary_parse": "( ( A person ) ( ( is ( ( training ( his horse ) ) ( for ( a competition ) ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (VP (VBG training) (NP (PRP$ his) (NN horse)) (PP (IN for) (NP (DT a) (NN competition))))) (. .)))"} - {"annotator_labels": ["contradiction"], "captionID": "3416050480.jpg#4", "gold_label": "contradiction", "pairID": "3416050480.jpg#4r1c", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is at a diner, ordering an omelette.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is ( at ( a diner ) ) ) , ) ( ordering ( an omelette ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (PP (IN at) (NP (DT a) (NN diner))) (, ,) (S (VP (VBG ordering) (NP (DT an) (NN omelette))))) (. .)))"} - {"annotator_labels": ["entailment"], "captionID": "3416050480.jpg#4", "gold_label": "entailment", "pairID": "3416050480.jpg#4r1e", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is outdoors, on a horse.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is outdoors ) , ) ( on ( a horse ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (ADVP (RB outdoors)) (, ,) (PP (IN on) (NP (DT a) (NN horse)))) (. .)))"} - ------------------------------------------- -Part IV: 使用Pipe对数据集进行预处理 ------------------------------------------- - -在fastNLP中，我们采用 :class:`~fastNLP.io.pipe.Pipe` 来作为加载数据集的基类。 -:class:`~fastNLP.io.pipe.Pipe` 定义了各种Pipe所需的API接口，开发者应该继承它实现各种的Pipe。 -在各种数据集的Pipe当中，至少应该编写如下内容: - - - process 函数：对输入的 :class:`~fastNLP.io.data_bundle.DataBundle` 进行处理（如构建词表、 - 将dataset的文本内容转成index等等），然后返回该 :class:`~fastNLP.io.data_bundle.DataBundle` - - process_from_file 函数：输入数据集所在文件夹，读取内容并组装成 :class:`~fastNLP.io.data_bundle.DataBundle` ， - 然后调用相对应的process函数对数据进行预处理 - -以SNLI数据集为例，写一个自定义Pipe的例子如下： - -.. code-block:: python - - from fastNLP.io.loader import SNLILoader - from fastNLP.io.pipe import MatchingPipe - - class MySNLIPipe(MatchingPipe): - - def process(self, data_bundle): - data_bundle = super(MySNLIPipe, self).process(data_bundle) - # MatchingPipe类里封装了一个关于matching任务的process函数，可以直接继承使用 - # 如果有需要进行额外的预处理操作可以在这里加入您的代码 - return data_bundle - - def process_from_file(self, paths=None): - data_bundle = SNLILoader().load(paths) # 使用SNLILoader读取原始数据集 - # SNLILoader的load函数中，paths如果为None则会自动下载 - return self.process(data_bundle) # 调用相对应的process函数对data_bundle进行处理 - -调用Pipe示例： - -.. code-block:: python - - from fastNLP.io.pipe import SNLIBertPipe - data_bundle = SNLIBertPipe(lower=True, tokenizer=arg.tokenizer).process_from_file() - print(data_bundle) - -输出的内容是:: - - In total 3 datasets: - train has 549367 instances. - dev has 9842 instances. - test has 9824 instances. - In total 2 vocabs: - words has 34184 entries. - target has 3 entries. - -这里表示一共有3个数据集和2个词表。其中： - - - 3个数据集分别为train、dev、test数据集，分别有549367、9842、9824个instance - - 2个词表分别为words词表与target词表。其中words词表为句子文本所构建的词表，一共有34184个单词； - target词表为目标标签所构建的词表，一共有3种标签。（注：如果有多个输入，则句子文本所构建的词表将 - 会被命名为words1以对应相对应的列名） - ------------------------------------------- -Part V: fastNLP封装好的Loader和Pipe ------------------------------------------- - -fastNLP封装了多种任务/数据集的Loader和Pipe并提供自动下载功能，具体参见文档 - -`fastNLP可加载的embedding与数据集 `_ - diff --git a/docs/source/tutorials/tutorial_2_vocabulary.rst b/docs/source/tutorials/tutorial_2_vocabulary.rst new file mode 100644 index 00000000..9656e4ec --- /dev/null +++ b/docs/source/tutorials/tutorial_2_vocabulary.rst @@ -0,0 +1,131 @@ + +============================== +Vocabulary +============================== + + :class:`~fastNLP.Vocabulary`是包含字或词与index关系的类，用于将文本转换为index。 + +----------------------------- +构建Vocabulary +----------------------------- + +.. code-block:: python + + from fastNLP import Vocabulary + + vocab = Vocabulary() + vocab.add_word_lst(['复', '旦', '大', '学']) # 加入新的字 + vocab.add_word('上海') # `上海`会作为一个整体 + vocab.to_index('复') # 应该会为3 + vocab.to_index('我') # 会输出1，Vocabulary中默认pad的index为0, unk(没有找到的词)的index为1 + + # 在构建target的Vocabulary时，词表中应该用不上pad和unk，可以通过以下的初始化 + vocab = Vocabulary(unknown=None, pad=None) + vocab.add_word_lst(['positive', 'negative']) + vocab.to_index('positive') # 输出0 + vocab.to_index('neutral') # 会报错 + +除了通过以上的方式建立词表，Vocabulary还可以通过使用下面的函数直从 :class:`~fastNLP.DataSet` 中的某一列建立词表以及将该列转换为index + +.. code-block:: python + + from fastNLP import Vocabulary + from fastNLP import DataSet + + dataset = DataSet({'chars': [ + ['今', '天', '天', '气', '很', '好', '。'], + ['被', '这', '部', '电', '影', '浪', '费', '了', '两', '个', '小', '时', '。'] + ], + 'target': ['neutral', 'negative'] + }) + + vocab = Vocabulary() + vocab.from_dataset(dataset, field_name='chars') + vocab.index_dataset(dataset, field_name='chars') + + target_vocab = Vocabulary(padding=None, unknown=None) + target_vocab.from_dataset(dataset, field_name='target') + target_vocab.index_dataset(dataset, field_name='target') + print(dataset) + +输出内容为:: + + +---------------------------------------------------+--------+ + | chars | target | + +---------------------------------------------------+--------+ + | [4, 2, 2, 5, 6, 7, 3] | 0 | + | [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 3] | 1 | + +---------------------------------------------------+--------+ + + +----------------------------- +一些使用tips +----------------------------- + +在通过使用from_dataset()函数在DataSet上建立词表时，将测试集和验证集放入参数no_create_entry_dataset中，如下所示 + +.. code-block:: python + + from fastNLP import Vocabulary + from fastNLP import DataSet + + tr_data = DataSet({'chars': [ + ['今', '天', '心', '情', '很', '好', '。'], + ['被', '这', '部', '电', '影', '浪', '费', '了', '两', '个', '小', '时', '。'] + ], + 'target': ['positive', 'negative'] + }) + dev_data = DataSet({'chars': [ + ['住', '宿', '条', '件', '还', '不', '错'], + ['糟', '糕', '的', '天', '气', '，', '无', '法', '出', '行', '。'] + ], + 'target': ['positive', 'negative'] + }) + + vocab = Vocabulary() + # 将验证集或者测试集在建立词表是放入no_create_entry_dataset这个参数中。 + vocab.from_dataset(tr_data, field_name='chars', no_create_entry_dataset=[dev_data]) + + +:class:`~fastNLP.Vocabulary` 中的`no_create_entry`, 建议在添加来自于测试集和验证集的词的时候将该参数置为True, 或将验证集和测试集 +传入`no_create_entry_dataset`参数。它们的意义是在接下来的模型会使用pretrain的embedding(包括glove, word2vec, elmo与bert)且会finetune的 +情况下，如果仅使用来自于train的数据建立vocabulary，会导致只出现在test与dev中的词语无法充分利用到来自于预训练embedding的信息(因为他们 +会被认为是unk)，所以在建立词表的时候将test与dev考虑进来会使得最终的结果更好。通过与fastNLP中的各种Embedding配合使用，会有如下的效果， +如果一个词出现在了train中，但是没在预训练模型中，embedding会为随机初始化，且它单独的一个vector，如果finetune embedding的话， +这个词在更新之后可能会有更好的表示; 而如果这个词仅出现在了dev或test中，那么就不能为它们单独建立vector，而应该让它指向unk这个vector的 +值(当unk的值更新时，这个词也使用的是更新之后的vector)。所以被认为是no_create_entry的token，将首先从预训练的词表中寻找它的表示，如 +果找到了，就使用该表示; 如果没有找到，则认为该词的表示应该为unk的表示。 + +下面我们结合部分:code:`~fastNLP.embeddings.StaticEmbedding`的例子来说明下该值造成的影响，如果您对 +:code:`~fastNLP.embeddings.StaticEmbedding`不太了解，您可以先参考\{Embedding教程的引用}部分再来阅读该部分 + +.. code-block:: python + + import torch + from fastNLP.embeddings import StaticEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary() + vocab.add_word('train') + vocab.add_word('only_in_train') # 仅在train出现，但肯定在预训练词表中不存在 + vocab.add_word('test', no_create_entry=True) # 该词只在dev或test中出现 + vocab.add_word('only_in_test', no_create_entry=True) # 这个词肯定在预训练中找不到 + + embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d') + print(embed(torch.LongTensor([vocab.to_index('train')]))) + print(embed(torch.LongTensor([vocab.to_index('only_in_train')]))) + print(embed(torch.LongTensor([vocab.to_index('test')]))) + print(embed(torch.LongTensor([vocab.to_index('only_in_test')]))) + print(embed(torch.LongTensor([vocab.unknown_idx]))) + +输出结果(只截取了部分vector):: + + tensor([[ 0.9497, 0.3433, 0.8450, -0.8852, ...]], grad_fn=) # train + tensor([[ 0.0540, -0.0557, -0.0514, -0.1688, ...]], grad_fn=) # only_in_train + tensor([[ 0.1318, -0.2552, -0.0679, 0.2619, ...]], grad_fn=) # test + tensor([[0., 0., 0., 0., 0., ...]], grad_fn=) # only_in_test + tensor([[0., 0., 0., 0., 0., ...]], grad_fn=) # unk + +首先train和test都能够从预训练中找到对应的vector，所以它们是各自的vector表示; only_in_train在预训练中找不到，StaticEmbedding为它 +新建了一个entry，所以它有一个单独的vector; 而only_in_dev在预训练中找不到被指向了unk的值(fastNLP用零向量初始化unk)，与最后一行unk的 +表示相同。 \ No newline at end of file diff --git a/docs/source/tutorials/tutorial_3_embedding.rst b/docs/source/tutorials/tutorial_3_embedding.rst index 07dc30bc..4e29efed 100644 --- a/docs/source/tutorials/tutorial_3_embedding.rst +++ b/docs/source/tutorials/tutorial_3_embedding.rst @@ -7,161 +7,446 @@ 教程目录： - `Part I: embedding介绍`_ - - `Part II: 使用随机初始化的embedding`_ - - `Part III: 使用预训练的静态embedding`_ - - `Part IV: 使用预训练的Contextual Embedding(ELMo & BERT)`_ - - `Part V: 使用character-level的embedding`_ - - `Part VI: 叠加使用多个embedding`_ - - `Part VII: fastNLP支持的预训练Embedding`_ - - + - `Part II: 使用预训练的静态embedding`_ + - `Part III: 使用随机初始化的embedding`_ + - `Part IV: ELMo Embedding`_ + - `Part V: Bert Embedding`_ + - `Part VI: 使用character-level的embedding`_ + - `Part VII: 叠加使用多个embedding`_ + - `Part VIII: Embedding的其它说明`_ + - `Part IX: StaticEmbedding的使用建议`_ --------------------------------------- Part I: embedding介绍 --------------------------------------- -与torch.nn.Embedding类似，fastNLP的embedding接受的输入是一个被index好的序列，输出的内容是这个序列的embedding结果。 - -fastNLP的embedding包括了预训练embedding和随机初始化embedding。 +Embedding是一种词嵌入技术，可以将字或者词转换为实向量。目前使用较多的预训练词嵌入有word2vec, fasttext, glove, character embedding, +elmo以及bert。 +但使用这些词嵌入方式的时候都需要做一些加载上的处理，比如预训练的word2vec, fasttext以及glove都有着超过几十万个词语的表示，但一般任务大概 +只会用到其中几万个词，如果直接加载所有的词汇，会导致内存占用变大以及运行速度变慢，需要从预训练文件中抽取本次实验的用到的词汇；而对于英文的 +elmo和character embedding, 需要将word拆分成character才能使用；Bert的使用更是涉及到了Byte pair encoding(BPE)相关的内容。为了方便 +大家的使用，fastNLP通过:class:`~fastNLP.Vocabulary`统一了不同embedding的使用。下面我们将讲述一些例子来说明一下 --------------------------------------- -Part II: 使用随机初始化的embedding +Part II: 使用预训练的静态embedding --------------------------------------- -使用随机初始化的embedding参见 :class:`~fastNLP.embeddings.embedding.Embedding` 。 - -可以传入词表大小和embedding维度： +在fastNLP中，加载预训练的word2vec, glove以及fasttext都使用的是 :class:`~fastNLP.embeddings.StaticEmbedding`。另外，为了方便大家的 +使用，fastNLP提供了多种静态词向量的自动下载并缓存(默认缓存到~/.fastNLP/embeddings文件夹下)的功能，支持自动下载的预训练向量可以在 +``_ +查看。 .. code-block:: python - from fastNLP import Embedding - embed = Embedding(10000, 50) + import torch + from fastNLP.embeddings import StaticEmbedding + from fastNLP import Vocabulary -也可以传入一个初始化的参数矩阵： + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) -.. code-block:: python + embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d', requires_grad=True) + + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]]) + print(embed(words).size()) - from fastNLP import Embedding - embed = Embedding(init_embed) +输出为:: -其中的init_embed可以是torch.FloatTensor、torch.nn.Embedding或者numpy.ndarray。 + torch.Size([1, 5, 50]) +fastNLP的StaticEmbedding在初始化之后，就和pytorch中的Embedding是类似的了。:class:`~fastNLP.embeddings.StaticEmbedding`的初始化 +主要是从model_dir_or_name提供的词向量中抽取出:class:`~fastNLP.Vocabulary`中词语的vector。 + +除了可以通过使用预先提供的Embedding，:class:`~fastNLP.embeddings.StaticEmbedding`也支持加载本地的预训练词向量，glove, word2vec以及 +fasttext格式的。通过将model_dir_or_name修改为本地的embedding文件路径，即可使用本地的embedding。 --------------------------------------- -Part III: 使用预训练的静态embedding +Part III: 使用随机初始化的embedding --------------------------------------- -在使用预训练的embedding之前，需要根据数据集的内容构建一个词表 :class:`~fastNLP.core.vocabulary.Vocabulary` ，在 -预训练embedding类初始化的时候需要将这个词表作为参数传入。 - -在fastNLP中，我们提供了 :class:`~fastNLP.embeddings.StaticEmbedding` 这一个类。 -通过 :class:`~fastNLP.embeddings.StaticEmbedding` 可以加载预训练好的静态 -Embedding，例子如下： +有时候需要使用随机初始化的Embedding，也可以通过使用 :class:`~fastNLP.embeddings.StaticEmbedding`获得。只需要将model_dir_or_name +置为None，且传入embedding_dim，如下例所示 .. code-block:: python - from fastNLP import StaticEmbedding - embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50', requires_grad=True) + from fastNLP.embeddings import StaticEmbedding + from fastNLP import Vocabulary -vocab为根据数据集构建的词表，model_dir_or_name可以是一个路径，也可以是embedding模型的名称： + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) - 1 如果传入的是路径，那么fastNLP将会根据该路径来读取预训练的权重文件并将embedding加载进来(glove - 和word2vec类型的权重文件都支持) + embed = StaticEmbedding(vocab, model_dir_or_name=None, embedding_dim=30) - 2 如果传入的是模型名称，那么fastNLP将会根据名称查找embedding模型，如果在cache目录下找到模型则会 - 自动加载；如果找不到则会自动下载到cache目录。默认的cache目录为 `~/.fastNLP` 文件夹。可以通过环境 - 变量 ``FASTNLP_CACHE_DIR`` 来自定义cache目录，如:: + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]]) + print(embed(words).size()) - $ FASTNLP_CACHE_DIR=~/fastnlp_cache_dir python your_python_file.py +输出为:: + + torch.Size([1, 5, 30]) -这个命令表示fastNLP将会在 `~/fastnlp_cache_dir` 这个目录下寻找模型，找不到则会自动将模型下载到这个目录 ----------------------------------------------------------- -Part IV: 使用预训练的Contextual Embedding(ELMo & BERT) +Part IV: ELMo Embedding ----------------------------------------------------------- 在fastNLP中，我们提供了ELMo和BERT的embedding： :class:`~fastNLP.embeddings.ElmoEmbedding` -和 :class:`~fastNLP.embeddings.BertEmbedding` 。 +和 :class:`~fastNLP.embeddings.BertEmbedding` 。可自动下载的ElmoEmbedding可以 +从``_找到。 与静态embedding类似，ELMo的使用方法如下： .. code-block:: python - from fastNLP import ElmoEmbedding - embed = ElmoEmbedding(vocab, model_dir_or_name='small', requires_grad=False) + from fastNLP.embeddings import ElmoEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) + + embed = ElmoEmbedding(vocab, model_dir_or_name='en-small', requires_grad=False) + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]]) + print(embed(words).size()) + +输出为:: + + torch.Size([1, 5, 256]) + +也可以输出多层的ELMo结果，fastNLP将在不同层的结果在最后一维上拼接，下面的代码需要在上面的代码执行结束之后执行 + +.. code-block:: python + + embed = ElmoEmbedding(vocab, model_dir_or_name='en-small', requires_grad=False, layers='1,2') + print(embed(words).size()) + +输出为:: + + torch.Size([1, 5, 512]) + +另外，根据``_，不同层之间使用可学习的权重可以使得ELMo的效果更好，在fastNLP中可以通过以下的初始化 +实现3层输出的结果通过可学习的权重进行加法融合。 + +.. code-block:: python + + embed = ElmoEmbedding(vocab, model_dir_or_name='en-small', requires_grad=True, layers='mix') + print(embed(words).size()) + +输出为:: + + torch.Size([1, 5, 256]) + + +----------------------------------------------------------- +Part V: Bert Embedding +----------------------------------------------------------- -BERT-embedding的使用方法如下： +虽然Bert并不算严格意义上的Embedding，但通过将Bert封装成Embedding的形式将极大减轻使用的复杂程度。可自动下载的Bert Embedding可以 +从``_找到。我们将使用下面的例子讲述一下 +BertEmbedding的使用 .. code-block:: python - from fastNLP import BertEmbedding - embed = BertEmbedding( - vocab, model_dir_or_name='en-base-cased', requires_grad=False, layers='4,-2,-1' - ) + from fastNLP.embeddings import BertEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) + + embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased') + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]]) + print(embed(words).size()) + +输出为:: + + torch.Size([1, 5, 768]) + +可以通过申明使用指定层数的output也可以使用多层的output，下面的代码需要在上面的代码执行结束之后执行 + +.. code-block:: python + + # 使用后面两层的输出 + embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased', layers='10,11') + print(embed(words).size()) # 结果将是在最后一维做拼接 + +输出为:: + + torch.Size([1, 5, 1536]) + +在Bert中还存在两个特殊的字符[CLS]和[SEP]，默认情况下这两个字符是自动加入并且在计算结束之后会自动删除，以使得输入的序列长度和输出的序列 +长度是一致的，但是有些分类的情况，必须需要使用[CLS]的表示，这种情况可以通过在初始化时申明一下需要保留[CLS]的表示，如下例所示 + +.. code-block:: python + + embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased', layers='-1', include_cls_sep=True) + print(embed(words).size()) # 结果将在序列维度上增加2 + # 取出句子的cls表示 + cls_reps = embed(words)[:, 0] # shape: [batch_size, 768] + +输出为:: + + torch.Size([1, 7, 768]) + +在英文Bert模型中，一个英文单词可能会被切分为多个subword，例如"fairness"会被拆分为["fair", "##ness"]，这样一个word对应的将有两个输出， +:class:`~fastNLP.embeddings.BertEmbedding`会使用pooling方法将一个word的subword的表示合并成一个vector，通过pool_method可以控制 +该pooling方法，支持的有"first"(即使用fair的表示作为fairness的表示), "last"(使用##ness的表示作为fairness的表示), "max"(对fair和 +##ness在每一维上做max),"avg"(对fair和##ness每一维做average)。 + +.. code-block:: python + + embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased', layers='-1', pool_method='max') + print(embed(words).size()) + +输出为:: + + torch.Size([1, 5, 768]) + +另外，根据``_ ，Bert的还存在一种用法，句子之间通过[SEP]拼接起来，前一句话的token embedding为0， +后一句话的token embedding为1。BertEmbedding能够自动识别句子中间的[SEP]来正确设置对应的token_type_id的。 + +.. code-block:: python + + vocab = Vocabulary() + vocab.add_word_lst("this is a demo . [SEP] another sentence .".split()) + + embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased', layers='-1', pool_method='max') + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo . [SEP] another sentence .".split()]]) + print(embed(words).size()) + +输出为:: -其中layers变量表示需要取哪几层的encode结果。 + torch.Size([1, 9, 768]) + +在多个[SEP]的情况下，将会使token_type_id不断0，1循环。比如"first sentence [SEP] second sentence [SEP] third sentence", 它们的 +token_type_id将是[0, 0, 0, 1, 1, 1, 0, 0]。但请注意[SEP]一定要大写的，不能是[sep]，否则无法识别。 + +更多:class:`~fastNLP.embedding.BertEmbedding`的使用，请参考\ref{找人写一篇BertEmbedding的使用教程} ----------------------------------------------------- -Part V: 使用character-level的embedding +Part VI: 使用character-level的embedding ----------------------------------------------------- -除了预训练的embedding以外，fastNLP还提供了CharEmbedding： :class:`~fastNLP.embeddings.CNNCharEmbedding` 和 -:class:`~fastNLP.embeddings.LSTMCharEmbedding` 。 +除了预训练的embedding以外，fastNLP还提供了两种Character Embedding： :class:`~fastNLP.embeddings.CNNCharEmbedding` 和 +:class:`~fastNLP.embeddings.LSTMCharEmbedding` 。一般在使用character embedding时，需要在预处理的时候将word拆分成character，这 +会使得预处理过程变得非常繁琐。在fastNLP中，使用character embedding也只需要传入:class:`~fastNLP.Vocabulary`即可，而且该 +Vocabulary与其它Embedding使用的Vocabulary是一致的，如下面的例子所示 CNNCharEmbedding的使用例子如下： .. code-block:: python - from fastNLP import CNNCharEmbedding - embed = CNNCharEmbedding(vocab, embed_size=100, char_emb_size=50) + from fastNLP.embeddings import CNNCharEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) + + # character的embedding维度大小为50，返回的embedding结果维度大小为64。 + embed = CNNCharEmbedding(vocab, embed_size=64, char_emb_size=50) + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]]) + print(embed(words).size()) -这表示这个CNNCharEmbedding当中character的embedding维度大小为50，返回的embedding结果维度大小为100。 +输出为:: + + torch.Size([1, 5, 64]) 与CNNCharEmbedding类似，LSTMCharEmbedding的使用例子如下： .. code-block:: python - from fastNLP import LSTMCharEmbedding - embed = LSTMCharEmbedding(vocab, embed_size=100, char_emb_size=50) + from fastNLP.embeddings import LSTMCharEmbeddding + from fastNLP import Vocabulary + + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) -这表示这个LSTMCharEmbedding当中character的embedding维度大小为50，返回的embedding结果维度大小为100。 + # character的embedding维度大小为50，返回的embedding结果维度大小为64。 + embed = LSTMCharEmbeddding(vocab, embed_size=64, char_emb_size=50) + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]]) + print(embed(words).size()) +输出为:: + + torch.Size([1, 5, 64]) ----------------------------------------------------- -Part VI: 叠加使用多个embedding +Part VII: 叠加使用多个embedding ----------------------------------------------------- -在fastNLP中，我们使用 :class:`~fastNLP.embeddings.StackEmbedding` 来叠加多个embedding +单独使用Character Embedding往往效果并不是很好，需要同时结合word embedding。在fastNLP中可以通过:class:`~fastNLP.embeddings.StackEmbedding` +来叠加embedding，具体的例子如下所示 + +.. code-block:: python + + from fastNLP.embeddings import StaticEmbedding, StackEmbedding, CNNCharEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) + + word_embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d') + char_embed = CNNCharEmbedding(vocab, embed_size=64, char_emb_size=50) + embed = StackEmbedding([word_embed, char_embed]) + + words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]]) + print(embed(words).size()) # 输出embedding的维度为50+64=114 + +输出为:: + + torch.Size([1, 5, 114]) -例子如下： +:class:`~fastNLP.embeddings.StaticEmbedding`, :class:`~fastNLP.embeddings.ElmoEmbedding`, +:class:`~fastNLP.embeddings.CNNCharEmbedding`, :class:`~fastNLP.embeddings.BertEmbedding`等都可以互相拼接。 +:class:`~fastNLP.embeddings.StackEmbedding`的使用也是和其它Embedding是一致的，即输出index返回对应的表示。但能够拼接起来的Embedding +必须使用同样的:class:`~fastNLP.Vocabulary`，因为只有使用同样的:class:`~fastNLP.Vocabulary`才能保证同一个index指向的是同一个词或字 + +----------------------------------------------------------- +Part VIII: Embedding的其它说明 +----------------------------------------------------------- + +(1) 获取各种Embedding的dimension .. code-block:: python - from fastNLP import StaticEmbedding, StackEmbedding - embed_1 = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50', requires_grad=True) - embed_2 = StaticEmbedding(vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) + from fastNLP.embeddings import * - stack_embed = StackEmbedding([embed_1, embed_2]) + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) -StackEmbedding会把多个embedding的结果拼接起来，如上面例子的stack_embed返回的embedding维度为350维。 + static_embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d') + print(static_embed.embedding_dim) # 50 + char_embed = CNNCharEmbedding(vocab, embed_size=30) + print(char_embed.embedding_dim) # 30 + elmo_embed_1 = ElmoEmbedding(vocab, model_dir_or_name='en-small', layers='2') + print(elmo_embed_1.embedding_dim) # 256 + elmo_embed_2 = ElmoEmbedding(vocab, model_dir_or_name='en-small', layers='1,2') + print(elmo_embed_2.embedding_dim) # 512 + bert_embed_1 = BertEmbedding(vocab, layers='-1', model_dir_or_name='en-base-cased') + print(bert_embed_1.embedding_dim) # 768 + bert_embed_2 = BertEmbedding(vocab, layers='2,-1', model_dir_or_name='en-base-cased') + print(bert_embed_2.embedding_dim) # 1536 + stack_embed = StackEmbedding([static_embed, char_embed]) + print(stack_embed.embedding_dim) # 80 -除此以外，还可以把静态embedding跟上下文相关的embedding拼接起来： +(2) 设置Embedding的权重是否更新 .. code-block:: python - from fastNLP import StaticEmbedding, StackEmbedding, ElmoEmbedding - elmo_embedding = ElmoEmbedding(vocab, model_dir_or_name='medium', layers='0,1,2', requires_grad=False) - glove_embedding = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50', requires_grad=True) + from fastNLP.embeddings import * + + vocab = Vocabulary() + vocab.add_word_lst("this is a demo .".split()) + + embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased') + embed.requires_grad = False # BertEmbedding不更新 + +(3) 各种Embedding中word_dropout与dropout的说明 + +fastNLP中所有的Embedding都支持传入word_dropout和dropout参数，word_dropout指示的是以多大概率将输入的word置为unk的index，这样既可以 +是的unk得到训练，也可以有一定的regularize效果; dropout参数是在获取到word的表示之后，以多大概率将一些维度的表示置为0。 + +如果使用:class:`~fastNLP.embeddings.StackEmbedding`且需要用到word_dropout，建议将word_dropout设置在:class:`~fastNLP.embeddings.StackEmbedding`。 + + +----------------------------------------------------------- +Part IX: StaticEmbedding的使用建议 +----------------------------------------------------------- + +在英文的命名实体识别(NER)任务中，由``_ 指出，同时使用cnn character embedding和word embedding +会使得NER的效果有比较大的提升。正如你在\ref{引用第七节}看到的那样，fastNLP支持将:class:`~fastNLP.embeddings.CNNCharacterEmbedding` +与:class:`~fastNLP.embeddings.StaticEmbedding`拼成一个:class:`~fastNLP.embeddings.StackEmbedding`。如果通过这种方式使用，需要 +在预处理文本时，不要将词汇小写化(因为Character Embedding需要利用词语中的大小写信息)且不要将出现频次低于某个阈值的word设置为unk(因为 +Character embedding需要利用字形信息)；但:class:`~fastNLP.embeddings.StaticEmbedding`使用的某些预训练词嵌入的词汇表中只有小写的词 +语, 且某些低频词并未在预训练中出现需要被剔除。即(1) character embedding需要保留大小写，而某些static embedding不需要保留大小写。(2) +character embedding需要保留所有的字形, 而static embedding需要设置一个最低阈值以学到更好的表示。 + +(1) fastNLP如何解决关于大小写的问题 + +fastNLP通过在:class:`~fastNLP.embeddings.StaticEmbedding`增加了一个lower参数解决该问题。如下面的例子所示 + +.. code-block:: python + + from fastNLP.embeddings import StaticEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary().add_word_lst("The the a A".split()) + # 下面用随机的StaticEmbedding演示，但与使用预训练时效果是一致的 + embed = StaticEmbedding(vocab, model_name_or_dir=None, embedding_dim=5) + print(embed(torch.LongTensor([vocab.to_index('The')]))) + print(embed(torch.LongTensor([vocab.to_index('the')]))) + +输出为:: + + tensor([[-0.4685, 0.4572, 0.5159, -0.2618, -0.6871]], grad_fn=) + tensor([[ 0.2615, 0.1490, -0.2491, 0.4009, -0.3842]], grad_fn=) + +可以看到"The"与"the"的vector是不一致的。但如果我们在初始化:class:`~fastNLP.embeddings.StaticEmbedding`将lower设置为True，效果将 +如下所示 + +.. code-block:: python + + from fastNLP.embeddings import StaticEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary().add_word_lst("The the a A".split()) + # 下面用随机的StaticEmbedding演示，但与使用预训练时效果是一致的 + embed = StaticEmbedding(vocab, model_name_or_dir=None, embedding_dim=5, lower=True) + print(embed(torch.LongTensor([vocab.to_index('The')]))) + print(embed(torch.LongTensor([vocab.to_index('the')]))) + +输出为:: + + tensor([[-0.2237, 0.6825, -0.3459, -0.1795, 0.7516]], grad_fn=) + tensor([[-0.2237, 0.6825, -0.3459, -0.1795, 0.7516]], grad_fn=) + +可以看到"The"与"the"的vector是一致的。他们实际上也是引用的同一个vector。通过将lower设置为True，可以在:class:`~fastNLP.embeddings.StaticEmbedding` +实现类似具备相同小写结果的词语引用同一个vector。 + +(2) fastNLP如何解决min_freq的问题 + +fastNLP通过在:class:`~fastNLP.embeddings.StaticEmbedding`增加了一个min_freq参数解决该问题。如下面的例子所示 + +.. code-block:: python + + from fastNLP.embeddings import StaticEmbedding + from fastNLP import Vocabulary + + vocab = Vocabulary().add_word_lst("the the the a".split()) + # 下面用随机的StaticEmbedding演示，但与使用预训练时效果是一致的 + embed = StaticEmbedding(vocab, model_name_or_dir=None, embedding_dim=5, min_freq=2) + print(embed(torch.LongTensor([vocab.to_index('the')]))) + print(embed(torch.LongTensor([vocab.to_index('a')]))) + print(embed(torch.LongTensor([vocab.unknown_idx]))) + +输出为:: + + tensor([[ 0.0454, 0.3375, 0.6758, -0.2026, -0.4715]], grad_fn=) + tensor([[-0.7602, 0.0149, 0.2733, 0.3974, 0.7371]], grad_fn=) + tensor([[-0.7602, 0.0149, 0.2733, 0.3974, 0.7371]], grad_fn=) + +其中最后一行为unknown值的vector，可以看到a的vector表示与unknown是一样的，这是由于a的频次低于了2，所以被指向了unknown的表示；而the由于 +词频超过了2次，所以它是单独的表示。 + +在计算min_freq时，也会考虑到lower的作用，比如 + +.. code-block:: python - stack_embed = StackEmbedding([elmo_embedding, glove_embedding]) + from fastNLP.embeddings import StaticEmbedding + from fastNLP import Vocabulary ------------------------------------------- -Part VII: fastNLP支持的预训练Embedding ------------------------------------------- + vocab = Vocabulary().add_word_lst("the the the a A".split()) + # 下面用随机的StaticEmbedding演示，但与使用预训练时效果是一致的 + embed = StaticEmbedding(vocab, model_name_or_dir=None, embedding_dim=5, min_freq=2, lower=True) + print(embed(torch.LongTensor([vocab.to_index('the')]))) + print(embed(torch.LongTensor([vocab.to_index('a')]))) + print(embed(torch.LongTensor([vocab.to_index('A')]))) + print(embed(torch.LongTensor([vocab.unknown_idx]))) -fastNLP支持多种预训练Embedding并提供自动下载功能，具体参见文档 +输出为:: -`fastNLP可加载的embedding与数据集 `_ + tensor([[-0.7453, -0.5542, 0.5039, 0.6195, -0.4723]], grad_fn=) # the + tensor([[ 0.0170, -0.0995, -0.5743, -0.2469, -0.2095]], grad_fn=) # a + tensor([[ 0.0170, -0.0995, -0.5743, -0.2469, -0.2095]], grad_fn=) # A + tensor([[ 0.6707, -0.5786, -0.6967, 0.0111, 0.1209]], grad_fn=) # unk +可以看到a不再和最后一行的unknown共享一个表示了，这是由于a与A都算入了a的词频，且A的表示也是a的表示。 diff --git a/docs/source/tutorials/tutorial_4_load_dataset.rst b/docs/source/tutorials/tutorial_4_load_dataset.rst new file mode 100644 index 00000000..c7e49fac --- /dev/null +++ b/docs/source/tutorials/tutorial_4_load_dataset.rst @@ -0,0 +1,219 @@ +======================================= +使用Loader和Pipe加载并处理数据集 +======================================= + +这一部分是一个关于如何加载数据集的教程 + +教程目录： + + - `Part I: 数据集容器DataBundle`_ + - `Part II: 加载的各种数据集的Loader`_ + - `Part III: 使用Pipe对数据集进行预处理`_ + - `Part IV: fastNLP封装好的Loader和Pipe`_ + - `Part V: 不同格式类型的基础Loader`_ + + +------------------------------------ +Part I: 数据集容器DataBundle +------------------------------------ + +而由于对于同一个任务，训练集，验证集和测试集会共用同一个词表以及具有相同的目标值，所以在fastNLP中我们使用了 :class:`~fastNLP.io.DataBundle` +来承载同一个任务的多个数据集 :class:`~fastNLP.DataSet` 以及它们的词表 :class:`~fastNLP.Vocabulary`。下面会有例子介绍:class:`~fastNLP.io.DataBundle` +的相关使用。 + +:class: `~fastNLP.io.DataBundle` 在fastNLP中主要在各个 :class: `~fastNLP.io.Loader` 和 :class: `~fastNLP.io.Pipe` 中被使用。 +下面我们将先介绍一下 :class: `~fastNLP.io.Loader` 和 :class: `~fastNLP.io.Pipe`, 之后我们将给出相应的例子。 + +------------------------------------- +Part II: 加载的各种数据集的Loader +------------------------------------- + +在fastNLP中，所有的数据Loader都可以通过其文档判断其支持读取的数据格式，以及读取之后返回的 :class:`~fastNLP.DataSet` 的格式。例如 +\ref 加个引用。 + + - download 函数：自动将该数据集下载到缓存地址，默认缓存地址为~/.fastNLP/datasets/。由于版权等原因，不是所有的Loader都实现了该方法。 + 该方法会返回下载后文件所处的缓存地址。可以查看对应Loader的download的方法的文档来判断该Loader加载的数据。 + - _load 函数：从一个数据文件中读取数据，返回一个 :class:`~fastNLP.DataSet`。返回的DataSet的格式可从Loader文档判断。 + - load 函数：从文件或者文件夹中读取数据并组装成 :class:`~fastNLP.io.DataBundle`。支持接受的参数类型有以下的几种 + - None, 将尝试读取自动缓存的数据，仅支持提供了自动下载数据的Loader + - 文件夹路径, 默认将尝试在该路径下匹配文件名中含有`train`, `test`, `dev`的文件，如果有多个文件含有这相同的关键字，将无法通过 + 该方式读取 + - dict, 例如{'train':"/path/to/tr.conll", 'dev':"/to/validate.conll", "test":"/to/te.conll"} + +.. code-block:: python + + from fastNLP.io import CWSLoader + + loader = CWSLoader(dataset_name='pku') + data_bundle = loader.load() + print(data_bundle) + +输出内容为:: + + In total 3 datasets: + dev has 1831 instances. + train has 17223 instances. + test has 1944 instances. + +这里表示一共有3个数据集。其中： + + - 3个数据集分别为train、dev、test数据集，分别有17223、1831、1944个instance + +也可以取出DataSet并DataSet中的具体内容 + +.. code-block:: python + + tr_data = data_bundle.get_dataset('train') + print(tr_data[:2]) + + 输出为:: + + +--------------------------------------------------------------------------------------+ + | raw_words | + +--------------------------------------------------------------------------------------+ + | 迈向 充满 希望 的 新 世纪 —— 一九九八年 新年 讲话 （ 附 图片 １ 张 ） | + | 中共中央 总书记 、 国家 主席 江 泽民 | + +--------------------------------------------------------------------------------------+ + +------------------------------------------ +Part III: 使用Pipe对数据集进行预处理 +------------------------------------------ +通过:class:`~fastNLP.io.Loader` 可以将文本数据读入，但并不能直接被神经网络使用，还需要进行一定的预处理。 + +在fastNLP中，我们使用 :class:`~fastNLP.io.Pipe`的子类作为数据预处理的类，Pipe和Loader一般具备一一对应的关系，该关系可以从其名称判断， +例如:class:`~fastNLP.io.CWSLoader`与:class:`~fastNLP.io.CWSPipe`是一一对应的。一般情况下Pipe处理包含以下的几个过程，(1)将raw_words或 +raw_chars进行tokenize以切分成不同的词或字; (2) 再建立词或字的 :class:`~fastNLP.Vocabulary`, 并将词或字转换为index; (3)将target +列建立词表并将target列转为index; + +所有的Pipe都可通过其文档查看通过该Pipe之后DataSet中的field的情况; 如 \ref{TODO 添加对例子的引用} + +各种数据集的Pipe当中，都包含了以下的两个函数: + + - process 函数：对输入的 :class:`~fastNLP.io.DataBundle` 进行处理, 然后返回处理之后的 :class:`~fastNLP.io.DataBundle`。 + process函数的文档中包含了该Pipe支持处理的DataSet的格式。 + - process_from_file 函数：输入数据集所在文件夹，使用对应的Loader读取数据(所以该函数支持的参数类型是由于其对应的Loader的load函数 + 决定的)，然后调用相对应的process函数对数据进行预处理。相当于是把Load和process放在一个函数中执行。 + +接着上面CWSLoader的例子，我们展示一下CWSPipe的功能： + +.. code-block:: python + + from fastNLP.io import CWSPipe + + data_bundle = CWSPipe().process(data_bundle) + print(data_bundle) + +输出内容为:: + + In total 3 datasets: + dev has 1831 instances. + train has 17223 instances. + test has 1944 instances. + In total 2 vocabs: + chars has 4777 entries. + target has 4 entries. + +表示一共有3个数据集和2个词表。其中： + + - 3个数据集分别为train、dev、test数据集，分别有17223、1831、1944个instance + - 2个词表分别为chars词表与target词表。其中chars词表为句子文本所构建的词表，一共有4777个字； + target词表为目标标签所构建的词表，一共有4种标签。 + +相较于之前CWSLoader读取的DataBundle，新增了两个Vocabulary。 我们可以打印一下处理之后的DataSet + +.. code-block:: python + + tr_data = data_bundle.get_dataset('train') + print(tr_data[:2]) + +输出为:: + + +---------------------------------------------------+------------------------------------+------------------------------------+---------+ + | raw_words | chars | target | seq_len | + +---------------------------------------------------+------------------------------------+------------------------------------+---------+ + | 迈向 充满 希望 的 新 世纪 —— 一九九八年... | [1224, 178, 674, 544, 573, 435,... | [0, 1, 0, 1, 0, 1, 2, 2, 0, 1, ... | 29 | + | 中共中央 总书记 、 国家 主席 江 泽民 | [11, 212, 11, 335, 124, 256, 10... | [0, 3, 3, 1, 0, 3, 1, 2, 0, 1, ... | 15 | + +---------------------------------------------------+------------------------------------+------------------------------------+---------+ + +可以看到有两列为int的field: chars和target。这两列的名称同时也是DataBundle中的Vocabulary的名称。可以通过下列的代码获取并查看Vocabulary的 +信息 + +.. code-block:: python + + vocab = data_bundle.get_vocab('target') + print(vocab) + +输出为:: + + Vocabulary(['B', 'E', 'S', 'M']...) + +------------------------------------------ +Part IV: fastNLP封装好的Loader和Pipe +------------------------------------------ + +fastNLP封装了多种任务/数据集的Loader和Pipe并提供自动下载功能，具体参见文档 + +`fastNLP可加载数据集 `_ + +-------------------------------------------------------- +Part V: 不同格式类型的基础Loader +-------------------------------------------------------- + +除了上面提到的针对具体任务的Loader，我们还提供了CSV格式和JSON格式的Loader + +:class:`~fastNLP.io.loader.CSVLoader` + 读取CSV类型的数据集文件。例子如下： + + .. code-block:: python + + from fastNLP.io.loader import CSVLoader + data_set_loader = CSVLoader( + headers=('raw_words', 'target'), sep='\t' + ) + # 表示将CSV文件中每一行的第一项填入'words' field，第二项填入'target' field。 + # 其中项之间由'\t'分割开来 + + data_set = data_set_loader._load('path/to/your/file') + + 数据集内容样例如下 :: + + But it does not leave you with much . 1 + You could hate it for the same reason . 1 + The performances are an absolute joy . 4 + + 读取之后的DataSet具有以下的field + + .. csv-table:: + :header: raw_words, target + + "But it does not leave you with much .", "1" + "You could hate it for the same reason .", "1" + "The performances are an absolute joy .", "4" + +:class:`~fastNLP.io.loader.JsonLoader` + 读取Json类型的数据集文件，数据必须按行存储，每行是一个包含各类属性的Json对象。例子如下： + + .. code-block:: python + + from fastNLP.io.loader import JsonLoader + oader = JsonLoader( + fields={'sentence1': 'raw_words1', 'sentence2': 'raw_words2', 'gold_label': 'target'} + ) + # 表示将Json对象中'sentence1'、'sentence2'和'gold_label'对应的值赋给'raw_words1'、'raw_words2'、'target'这三个fields + + data_set = loader._load('path/to/your/file') + + 数据集内容样例如下 :: + + {"annotator_labels": ["neutral"], "captionID": "3416050480.jpg#4", "gold_label": "neutral", "pairID": "3416050480.jpg#4r1n", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is training his horse for a competition.", "sentence2_binary_parse": "( ( A person ) ( ( is ( ( training ( his horse ) ) ( for ( a competition ) ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (VP (VBG training) (NP (PRP$ his) (NN horse)) (PP (IN for) (NP (DT a) (NN competition))))) (. .)))"} + {"annotator_labels": ["contradiction"], "captionID": "3416050480.jpg#4", "gold_label": "contradiction", "pairID": "3416050480.jpg#4r1c", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is at a diner, ordering an omelette.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is ( at ( a diner ) ) ) , ) ( ordering ( an omelette ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (PP (IN at) (NP (DT a) (NN diner))) (, ,) (S (VP (VBG ordering) (NP (DT an) (NN omelette))))) (. .)))"} + {"annotator_labels": ["entailment"], "captionID": "3416050480.jpg#4", "gold_label": "entailment", "pairID": "3416050480.jpg#4r1e", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is outdoors, on a horse.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is outdoors ) , ) ( on ( a horse ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (ADVP (RB outdoors)) (, ,) (PP (IN on) (NP (DT a) (NN horse)))) (. .)))"} + + 读取之后的DataSet具有以下的field + + .. csv-table:: + :header: raw_words0, raw_words1, target + + "A person on a horse jumps over a broken down airplane.", "A person is training his horse for a competition.", "neutral" + "A person on a horse jumps over a broken down airplane.", "A person is at a diner, ordering an omelette.", "contradiction" + "A person on a horse jumps over a broken down airplane.", "A person is outdoors, on a horse.", "entailment" diff --git a/docs/source/tutorials/tutorial_4_loss_optimizer.rst b/docs/source/tutorials/tutorial_6_loss_optimizer.rst similarity index 100% rename from docs/source/tutorials/tutorial_4_loss_optimizer.rst rename to docs/source/tutorials/tutorial_6_loss_optimizer.rst diff --git a/docs/source/tutorials/tutorial_8_metrics.rst b/docs/source/tutorials/tutorial_7_metrics.rst similarity index 100% rename from docs/source/tutorials/tutorial_8_metrics.rst rename to docs/source/tutorials/tutorial_7_metrics.rst diff --git a/docs/source/tutorials/tutorial_7_modules_models.rst b/docs/source/tutorials/tutorial_8_modules_models.rst similarity index 100% rename from docs/source/tutorials/tutorial_7_modules_models.rst rename to docs/source/tutorials/tutorial_8_modules_models.rst diff --git a/docs/source/tutorials/tutorial_6_seq_labeling.rst b/docs/source/tutorials/tutorial_9_seq_labeling.rst similarity index 100% rename from docs/source/tutorials/tutorial_6_seq_labeling.rst rename to docs/source/tutorials/tutorial_9_seq_labeling.rst diff --git a/docs/source/user/tutorials.rst b/docs/source/user/tutorials.rst index 3e9e1b54..e19f252b 100644 --- a/docs/source/user/tutorials.rst +++ b/docs/source/user/tutorials.rst @@ -8,13 +8,14 @@ fastNLP 详细使用教程 :maxdepth: 1 使用DataSet预处理文本 - 使用Loader和Pipe加载并处理数据集 + 使用Vocabulary转换文本与index 使用Embedding模块将文本转成向量 - 动手实现一个文本分类器I-使用Trainer和Tester快速训练和测试 + 使用Loader和Pipe加载并处理数据集 动手实现一个文本分类器II-使用DataSetIter实现自定义训练过程 - 快速实现序列标注模型 - 使用Modules和Models快速搭建自定义模型 - 使用Metric快速评测你的模型 - 使用Callback自定义你的训练过程 - 使用fitlog 辅助 fastNLP 进行科研 + 动手实现一个文本分类器I-使用Trainer和Tester快速训练和测试 + 使用Metric快速评测你的模型 + 使用Modules和Models快速搭建自定义模型 + 快速实现序列标注模型 + 使用Callback自定义你的训练过程 + 使用fitlog 辅助 fastNLP 进行科研 diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index 05351cbd..aa998801 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -126,27 +126,6 @@ class BertEmbedding(ContextualEmbedding): if self._word_sep_index: words.masked_fill_(sep_mask, self._word_sep_index) return words - - @property - def requires_grad(self): - """ - Embedding的参数是否允许优化。True: 所有参数运行优化; False: 所有参数不允许优化; None: 部分允许优化、部分不允许 - - :return: - """ - requires_grads = set([param.requires_grad for name, param in self.named_parameters() - if 'word_pieces_lengths' not in name]) - if len(requires_grads) == 1: - return requires_grads.pop() - else: - return None - - @requires_grad.setter - def requires_grad(self, value): - for name, param in self.named_parameters(): - if 'word_pieces_lengths' in name: # 这个不能加入到requires_grad中 - continue - param.requires_grad = value class BertWordPieceEncoder(nn.Module): @@ -175,23 +154,6 @@ class BertWordPieceEncoder(nn.Module): self.word_dropout = word_dropout self.dropout_layer = nn.Dropout(dropout) - @property - def requires_grad(self): - """ - Embedding的参数是否允许优化。True: 所有参数运行优化; False: 所有参数不允许优化; None: 部分允许优化、部分不允许 - :return: - """ - requires_grads = set([param.requires_grad for name, param in self.named_parameters()]) - if len(requires_grads) == 1: - return requires_grads.pop() - else: - return None - - @requires_grad.setter - def requires_grad(self, value): - for name, param in self.named_parameters(): - param.requires_grad = value - @property def embed_size(self): return self._embed_size diff --git a/fastNLP/embeddings/char_embedding.py b/fastNLP/embeddings/char_embedding.py index 59109206..2492b6d7 100644 --- a/fastNLP/embeddings/char_embedding.py +++ b/fastNLP/embeddings/char_embedding.py @@ -139,40 +139,6 @@ class CNNCharEmbedding(TokenEmbedding): chars = torch.sum(conv_chars, dim=-2) / chars_masks.eq(0).sum(dim=-1, keepdim=True).float() chars = self.fc(chars) return self.dropout(chars) - - @property - def requires_grad(self): - """ - Embedding的参数是否允许优化。True: 所有参数运行优化; False: 所有参数不允许优化; None: 部分允许优化、部分不允许 - :return: - """ - params = [] - for name, param in self.named_parameters(): - if 'words_to_chars_embedding' not in name and 'word_lengths' not in name: - params.append(param.requires_grad) - requires_grads = set(params) - if len(requires_grads) == 1: - return requires_grads.pop() - else: - return None - - @requires_grad.setter - def requires_grad(self, value): - for name, param in self.named_parameters(): - if 'words_to_chars_embedding' in name or 'word_lengths' in name: # 这个不能加入到requires_grad中 - continue - param.requires_grad = value - - def reset_parameters(self): - for name, param in self.named_parameters(): - if 'words_to_chars_embedding' in name or 'word_lengths' in name: # 这个不能reset - continue - if 'char_embedding' in name: - continue - if param.data.dim() > 1: - nn.init.xavier_uniform_(param, 1) - else: - nn.init.uniform_(param, -1, 1) class LSTMCharEmbedding(TokenEmbedding): @@ -293,27 +259,3 @@ class LSTMCharEmbedding(TokenEmbedding): chars = self.fc(chars) return self.dropout(chars) - - @property - def requires_grad(self): - """ - Embedding的参数是否允许优化。True: 所有参数运行优化; False: 所有参数不允许优化; None: 部分允许优化、部分不允许 - - :return: - """ - params = [] - for name, param in self.named_parameters(): - if 'words_to_chars_embedding' not in name and 'word_lengths' not in name: - params.append(param) - requires_grads = set(params) - if len(requires_grads) == 1: - return requires_grads.pop() - else: - return None - - @requires_grad.setter - def requires_grad(self, value): - for name, param in self.named_parameters(): - if 'words_to_chars_embedding' in name or 'word_lengths' in name: # 这个不能加入到requires_grad中 - continue - param.requires_grad = value diff --git a/fastNLP/embeddings/elmo_embedding.py b/fastNLP/embeddings/elmo_embedding.py index d19a3577..57842c33 100644 --- a/fastNLP/embeddings/elmo_embedding.py +++ b/fastNLP/embeddings/elmo_embedding.py @@ -55,7 +55,7 @@ class ElmoEmbedding(ContextualEmbedding): 并删除character encoder，之后将直接使用cache的embedding。默认为False。 """ - def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en', layers: str = '2', requires_grad: bool = False, + def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en', layers: str = '2', requires_grad: bool = True, word_dropout=0.0, dropout=0.0, cache_word_reprs: bool = False): super(ElmoEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) @@ -136,27 +136,6 @@ class ElmoEmbedding(ContextualEmbedding): for name in ['layers', 'model', 'layer_weights', 'gamma']: if hasattr(self, name): delattr(self, name) - - @property - def requires_grad(self): - """ - Embedding的参数是否允许优化。True: 所有参数运行优化; False: 所有参数不允许优化; None: 部分允许优化、部分不允许 - - :return: - """ - requires_grads = set([param.requires_grad for name, param in self.named_parameters() - if 'words_to_chars_embedding' not in name and 'words_to_words' not in name]) - if len(requires_grads) == 1: - return requires_grads.pop() - else: - return None - - @requires_grad.setter - def requires_grad(self, value): - for name, param in self.named_parameters(): - if 'words_to_chars_embedding' in name or 'words_to_words' in name: # 这个不能加入到requires_grad中 - continue - param.requires_grad = value class _ElmoModel(nn.Module): diff --git a/fastNLP/embeddings/embedding.py b/fastNLP/embeddings/embedding.py index 255b0823..e82ef0b4 100644 --- a/fastNLP/embeddings/embedding.py +++ b/fastNLP/embeddings/embedding.py @@ -115,6 +115,10 @@ class Embedding(nn.Module): class TokenEmbedding(nn.Module): + """ + fastNLP中各种Embedding的基类 + + """ def __init__(self, vocab, word_dropout=0.0, dropout=0.0): super(TokenEmbedding, self).__init__() if vocab.rebuild: diff --git a/fastNLP/embeddings/stack_embedding.py b/fastNLP/embeddings/stack_embedding.py index e83a275c..91702ec2 100644 --- a/fastNLP/embeddings/stack_embedding.py +++ b/fastNLP/embeddings/stack_embedding.py @@ -22,10 +22,11 @@ class StackEmbedding(TokenEmbedding): Example:: >>> from fastNLP import Vocabulary - >>> from fastNLP.embeddings import StaticEmbedding + >>> from fastNLP.embeddings import StaticEmbedding, StackEmbedding >>> vocab = Vocabulary().add_word_lst("The whether is good .".split()) >>> embed_1 = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d', requires_grad=True) >>> embed_2 = StaticEmbedding(vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) + >>> embed = StackEmbedding([embed_1, embed_2]) :param embeds: 一个由若干个TokenEmbedding组成的list，要求每一个TokenEmbedding的词表都保持一致 :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。不同embedidng会在相同的位置 @@ -57,35 +58,26 @@ class StackEmbedding(TokenEmbedding): :return: """ assert isinstance(embed, TokenEmbedding) + self._embed_size += embed.embed_size self.embeds.append(embed) + return self def pop(self): """ 弹出最后一个embed :return: """ - return self.embeds.pop() + embed = self.embeds.pop() + self._embed_size -= embed.embed_size + return embed @property def embed_size(self): - return self._embed_size - - @property - def requires_grad(self): """ - Embedding的参数是否允许优化。True: 所有参数运行优化; False: 所有参数不允许优化; None: 部分允许优化、部分不允许 + 该Embedding输出的vector的最后一维的维度。 :return: """ - requires_grads = set([embed.requires_grad for embed in self.embeds()]) - if len(requires_grads) == 1: - return requires_grads.pop() - else: - return None - - @requires_grad.setter - def requires_grad(self, value): - for embed in self.embeds(): - embed.requires_grad = value + return self._embed_size def forward(self, words): """ diff --git a/fastNLP/embeddings/static_embedding.py b/fastNLP/embeddings/static_embedding.py index 8249aa11..399191dc 100644 --- a/fastNLP/embeddings/static_embedding.py +++ b/fastNLP/embeddings/static_embedding.py @@ -54,13 +54,16 @@ class StaticEmbedding(TokenEmbedding): 如果输入为None则使用embedding_dim的维度随机初始化一个embedding。 :param int embedding_dim: 随机初始化的embedding的维度，当该值为大于0的值时，将忽略model_dir_or_name。 :param bool requires_grad: 是否需要gradient. 默认为True - :param callable init_method: 如何初始化没有找到的值。可以使用torch.nn.init.*中各种方法。调用该方法时传入一个tensor对 + :param callable init_method: 如何初始化没有找到的值。可以使用torch.nn.init.*中各种方法, 传入的方法应该接受一个tensor，并 + inplace地修改其值。 :param bool lower: 是否将vocab中的词语小写后再和预训练的词表进行匹配。如果你的词表中包含大写的词语，或者就是需要单独 为大写的词语开辟一个vector表示，则将lower设置为False。 :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 :param bool normalize: 是否对vector进行normalize，使得每个vector的norm为1。 :param int min_freq: Vocabulary词频数小于这个数量的word将被指向unk。 + :param dict **kwarngs: only_train_min_freq, 仅对train中的词语使用min_freq筛选; only_norm_found_vector是否仅对在预训练中 + 找到的词语使用normalize。 """ def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en', embedding_dim=-1, requires_grad: bool = True, @@ -183,27 +186,6 @@ class StaticEmbedding(TokenEmbedding): return embed - @property - def requires_grad(self): - """ - Embedding的参数是否允许优化。True: 所有参数运行优化; False: 所有参数不允许优化; None: 部分允许优化、部分不允许 - - :return: - """ - requires_grads = set([param.requires_grad for name, param in self.named_parameters() - if 'words_to_words' not in name]) - if len(requires_grads) == 1: - return requires_grads.pop() - else: - return None - - @requires_grad.setter - def requires_grad(self, value): - for name, param in self.named_parameters(): - if 'words_to_words' in name: - continue - param.requires_grad = value - def _load_with_vocab(self, embed_filepath, vocab, dtype=np.float32, padding='', unknown='', error='ignore', init_method=None): """ diff --git a/fastNLP/io/loader/classification.py b/fastNLP/io/loader/classification.py index 9efcf5d2..53bc6789 100644 --- a/fastNLP/io/loader/classification.py +++ b/fastNLP/io/loader/classification.py @@ -31,7 +31,6 @@ class YelpLoader(Loader): "1","I got 'new' tires from the..." "1","Don't waste your time..." - 读取YelpFull, YelpPolarity的数据。可以通过xxx下载并预处理数据。 读取的DataSet将具备以下的数据结构 .. csv-table:: diff --git a/fastNLP/io/loader/loader.py b/fastNLP/io/loader/loader.py index 22636a27..baf2874e 100644 --- a/fastNLP/io/loader/loader.py +++ b/fastNLP/io/loader/loader.py @@ -34,29 +34,27 @@ class Loader: """ 从指定一个或多个路径中的文件中读取数据，返回 :class:`~fastNLP.io.DataBundle` 。 - 读取的field根据ConllLoader初始化时传入的headers决定。 - :param Union[str, Dict[str, str]] paths: 支持以下的几种输入方式 (0) 如果为None，则先查看本地是否有缓存，如果没有则自动下载并缓存。 (1) 传入一个目录, 该目录下名称包含train的被认为是train，包含test的被认为是test，包含dev的被认为是dev，如果检测到多个文件 名包含'train'、 'dev'、 'test'则会报错:: - data_bundle = ConllLoader().load('/path/to/dir') # 返回的DataBundle中datasets根据目录下是否检测到train、 - # dev、 test等有所变化，可以通过以下的方式取出DataSet - tr_data = data_bundle.datasets['train'] - te_data = data_bundle.datasets['test'] # 如果目录下有文件包含test这个字段 + data_bundle = xxxLoader().load('/path/to/dir') # 返回的DataBundle中datasets根据目录下是否检测到train、 + # dev、 test等有所变化，可以通过以下的方式取出DataSet + tr_data = data_bundle.get_dataset('train') + te_data = data_bundle.get_dataset('test') # 如果目录下有文件包含test这个字段 - (2) 传入文件路径:: + (2) 传入一个dict，比如train，dev，test不在同一个目录下，或者名称中不包含train, dev, test:: - data_bundle = ConllLoader().load("/path/to/a/train.conll") # 返回DataBundle对象, datasets中仅包含'train' - tr_data = data_bundle.datasets['train'] # 可以通过以下的方式取出DataSet + paths = {'train':"/path/to/tr.conll", 'dev':"/to/validate.conll", "test":"/to/te.conll"} + data_bundle = xxxLoader().load(paths) # 返回的DataBundle中的dataset中包含"train", "dev", "test" + dev_data = data_bundle.get_dataset('dev') - (3) 传入一个dict，比如train，dev，test不在同一个目录下，或者名称中不包含train, dev, test:: + (3) 传入文件路径:: - paths = {'train':"/path/to/tr.conll", 'dev':"/to/validate.conll", "test":"/to/te.conll"} - data_bundle = ConllLoader().load(paths) # 返回的DataBundle中的dataset中包含"train", "dev", "test" - dev_data = data_bundle.datasets['dev'] + data_bundle = xxxLoader().load("/path/to/a/train.conll") # 返回DataBundle对象, datasets中仅包含'train' + tr_data = data_bundle.get_dataset('train') # 取出DataSet :return: 返回的 :class:`~fastNLP.io.DataBundle` """ @@ -78,7 +76,7 @@ class Loader: @staticmethod def _get_dataset_path(dataset_name): """ - 传入dataset的名称，获取读取数据的目录。如果数据不存在，会尝试自动下载并缓存 + 传入dataset的名称，获取读取数据的目录。如果数据不存在，会尝试自动下载并缓存（如果支持的话） :param str dataset_name: 数据集的名称 :return: str, 数据集的目录地址。直接到该目录下读取相应的数据即可。 From 587edd54382f9dac7c638eb7adbde73b34830f78 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 11:08:27 +0800 Subject: [PATCH 55/92] update the doc-checking tool --- docs/count.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/count.py b/docs/count.py index 7118216a..0830c7cc 100644 --- a/docs/count.py +++ b/docs/count.py @@ -110,18 +110,34 @@ def check_file(m, name): return funcs, classes -def check_files(modules, out=sys.stdout): +def check_files(modules, out=None): for name in sorted(modules.keys()): print(name, file=out) funcs, classes = check_file(modules[name], name) - for f in funcs: - print("%-30s \t %s \t %s" % (f, gr("文档", funcs[f][0]), gr("测试", funcs[f][1])), file=out) - for c in classes: - print("%-30s \t %s \t %s" % (c, gr("文档", classes[c][0]), gr("测试", classes[c][1])), file=out) - methods = classes[c][2] - for f in methods: - print(" %-28s \t %s" % (f, gr("文档", methods[f][0])), file=out) - print(file=out) + if out is None: + for f in funcs: + print("%-30s \t %s \t %s" % (f, gr("文档", funcs[f][0]), gr("测试", funcs[f][1]))) + for c in classes: + print("%-30s \t %s \t %s" % (c, gr("文档", classes[c][0]), gr("测试", classes[c][1]))) + methods = classes[c][2] + for f in methods: + print(" %-28s \t %s" % (f, gr("文档", methods[f][0]))) + else: + for f in funcs: + if not funcs[f][0]: + print("缺少文档 %s" % (f), file=out) + if not funcs[f][1]: + print("缺少测试 %s" % (f), file=out) + for c in classes: + if not classes[c][0]: + print("缺少文档 %s" % (c), file=out) + if not classes[c][1]: + print("缺少测试 %s" % (c), file=out) + methods = classes[c][2] + for f in methods: + if not methods[f][0]: + print("缺少文档 %s" % (c + "." + f), file=out) + print(file=out) def main(): @@ -134,7 +150,7 @@ def main(): create_rst_file(modules, name, children) print(_colored_string('Done!', "Green")) print(_colored_string('Checking all files...', "Blue")) - check_files(modules) + check_files(modules, out=open("results.txt", "w")) print(_colored_string('Done!', "Green")) From 9b8265dc7e4df64de427b57334c8e6f4e10cebaf Mon Sep 17 00:00:00 2001 From: yh Date: Tue, 10 Sep 2019 12:39:57 +0800 Subject: [PATCH 56/92] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dchar=5Fembedding=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/embeddings/char_embedding.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fastNLP/embeddings/char_embedding.py b/fastNLP/embeddings/char_embedding.py index 2492b6d7..a0328525 100644 --- a/fastNLP/embeddings/char_embedding.py +++ b/fastNLP/embeddings/char_embedding.py @@ -106,8 +106,7 @@ class CNNCharEmbedding(TokenEmbedding): for i in range(len(kernel_sizes))]) self._embed_size = embed_size self.fc = nn.Linear(sum(filter_nums), embed_size) - self.reset_parameters() - + def forward(self, words): """ 输入words的index后，生成对应的words的表示。 From c38d815c49916c5011f74474723ff40ee3cb7422 Mon Sep 17 00:00:00 2001 From: xxliu Date: Tue, 10 Sep 2019 13:17:45 +0800 Subject: [PATCH 57/92] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/io/pipe/coreference.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py index baa616f1..836b251d 100644 --- a/fastNLP/io/pipe/coreference.py +++ b/fastNLP/io/pipe/coreference.py @@ -28,15 +28,15 @@ class CoreferencePipe(Pipe): .. csv-table:: :header: "raw_key", "raw_speaker","raw_words","raw_clusters" - "bc/cctv/00/cctv_0000_0", "[["Speaker#1", "Speaker#1"],[]]","[["I","am"],[]]","[[[2,3],[6,7]],[[10,12],[20,22]]]" - "bc/cctv/00/cctv_0000_1"", "[["Speaker#1", "Speaker#1"],[]]","[["He","is"],[]]","[[[2,3],[6,7]],[[10,12],[20,22]]]" + "bc/cctv/00/cctv_0000_0", "[[Speaker#1, Speaker#1],[]]","[['I','am'],[]]","[[[2,3],[6,7]],[[10,12],[20,22]]]" + "bc/cctv/00/cctv_0000_1", "[['Speaker#1', 'peaker#1'],[]]","[['He','is'],[]]","[[[2,3],[6,7]],[[10,12],[20,22]]]" "[...]", "[...]","[...]","[...]" 处理完成后数据包含文章类别、speaker信息、句子信息、句子对应的index、char、句子长度、target： .. csv-table:: :header: "words1", "words2","words3","words4","chars","seq_len","target" - "bc", "[[0,0],[1,1]]","[["I","am"],[]]",[[1,2],[]],[[[1],[2,3]],[]],[2,3],"[[[2,3],[6,7]],[[10,12],[20,22]]]" + "bc", "[[0,0],[1,1]]","[['I','am'],[]]","[[1,2],[]]","[[[1],[2,3]],[]]","[2,3]","[[[2,3],[6,7]],[[10,12],[20,22]]]" "[...]", "[...]","[...]","[...]","[...]","[...]","[...]" From c9883bcb30a5d0c5c01180d5ca4848ad006a6544 Mon Sep 17 00:00:00 2001 From: xxliu Date: Tue, 10 Sep 2019 15:06:23 +0800 Subject: [PATCH 58/92] undocumented --- fastNLP/io/loader/coreference.py | 2 ++ fastNLP/io/pipe/coreference.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/fastNLP/io/loader/coreference.py b/fastNLP/io/loader/coreference.py index b4493571..6e2344d2 100644 --- a/fastNLP/io/loader/coreference.py +++ b/fastNLP/io/loader/coreference.py @@ -1,3 +1,5 @@ +"""undocumented""" + from ...core.dataset import DataSet from ..file_reader import _read_json from ...core.instance import Instance diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py index 836b251d..bb40ca55 100644 --- a/fastNLP/io/pipe/coreference.py +++ b/fastNLP/io/pipe/coreference.py @@ -1,3 +1,5 @@ +"""undocumented""" + __all__ = [ "CoreferencePipe" From 9e3251dff4ab39d56f50fb9aa8a8f5406e190076 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 14:47:14 +0800 Subject: [PATCH 59/92] fix a lot of doc bugs in codes --- fastNLP/core/dataset.py | 35 +++++++++++++------------- fastNLP/core/optimizer.py | 3 ++- fastNLP/embeddings/static_embedding.py | 15 ++++++----- fastNLP/io/loader/loader.py | 16 ++++++------ fastNLP/io/pipe/conll.py | 16 ++++++------ fastNLP/io/pipe/pipe.py | 2 +- 6 files changed, 43 insertions(+), 44 deletions(-) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 2b548f22..1fa0fd10 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -291,6 +291,7 @@ import _pickle as pickle from copy import deepcopy import numpy as np +from prettytable import PrettyTable from ._logger import logger from .const import Const @@ -301,7 +302,6 @@ from .field import SetInputOrTargetException from .instance import Instance from .utils import _get_func_signature from .utils import pretty_table_printer -from prettytable import PrettyTable class DataSet(object): @@ -428,23 +428,22 @@ class DataSet(object): def print_field_meta(self): """ - 输出当前field的meta信息, 形似下列的输出 - - +-------------+-------+-------+ - | field_names | x | y | - +-------------+-------+-------+ - | is_input | True | False | - | is_target | False | False | - | ignore_type | False | | - | pad_value | 0 | | - +-------------+-------+-------+ - - field_names: DataSet中field的名称 - is_input: field是否为input - is_target: field是否为target - ignore_type: 是否忽略该field的type, 一般仅在该field至少为input或target时才有意义 - pad_value: 该field的pad的值，仅在该field为input或target时有意义 - + 输出当前field的meta信息, 形似下列的输出:: + + +-------------+-------+-------+ + | field_names | x | y | + +=============+=======+=======+ + | is_input | True | False | + | is_target | False | False | + | ignore_type | False | | + | pad_value | 0 | | + +-------------+-------+-------+ + + :param field_names: DataSet中field的名称 + :param is_input: field是否为input + :param is_target: field是否为target + :param ignore_type: 是否忽略该field的type, 一般仅在该field至少为input或target时才有意义 + :param pad_value: 该field的pad的值，仅在该field为input或target时有意义 :return: """ if len(self.field_arrays)>0: diff --git a/fastNLP/core/optimizer.py b/fastNLP/core/optimizer.py index b782cfa6..b534a72a 100644 --- a/fastNLP/core/optimizer.py +++ b/fastNLP/core/optimizer.py @@ -37,6 +37,7 @@ class Optimizer(object): def _get_require_grads_param(self, params): """ 将params中不需要gradient的删除 + :param iterable params: parameters :return: list(nn.Parameters) """ @@ -85,7 +86,7 @@ class SGD(Optimizer): class Adam(Optimizer): """ - + Adam """ def __init__(self, lr=0.001, weight_decay=0, betas=(0.9, 0.999), eps=1e-8, amsgrad=False, model_params=None): diff --git a/fastNLP/embeddings/static_embedding.py b/fastNLP/embeddings/static_embedding.py index 399191dc..3d2471e6 100644 --- a/fastNLP/embeddings/static_embedding.py +++ b/fastNLP/embeddings/static_embedding.py @@ -7,19 +7,19 @@ __all__ = [ "StaticEmbedding" ] import os +import warnings +from collections import defaultdict +from copy import deepcopy +import numpy as np import torch import torch.nn as nn -import numpy as np -import warnings +from .embedding import TokenEmbedding +from ..core import logger from ..core.vocabulary import Vocabulary from ..io.file_utils import PRETRAIN_STATIC_FILES, _get_embedding_url, cached_path -from .embedding import TokenEmbedding from ..modules.utils import _get_file_name_base_on_postfix -from copy import deepcopy -from collections import defaultdict -from ..core import logger class StaticEmbedding(TokenEmbedding): @@ -62,8 +62,7 @@ class StaticEmbedding(TokenEmbedding): :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 :param bool normalize: 是否对vector进行normalize，使得每个vector的norm为1。 :param int min_freq: Vocabulary词频数小于这个数量的word将被指向unk。 - :param dict **kwarngs: only_train_min_freq, 仅对train中的词语使用min_freq筛选; only_norm_found_vector是否仅对在预训练中 - 找到的词语使用normalize。 + :param dict kwarngs: only_train_min_freq, 仅对train中的词语使用min_freq筛选; only_norm_found_vector是否仅对在预训练中找到的词语使用normalize。 """ def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en', embedding_dim=-1, requires_grad: bool = True, diff --git a/fastNLP/io/loader/loader.py b/fastNLP/io/loader/loader.py index baf2874e..384125a8 100644 --- a/fastNLP/io/loader/loader.py +++ b/fastNLP/io/loader/loader.py @@ -31,27 +31,27 @@ class Loader: raise NotImplementedError def load(self, paths: Union[str, Dict[str, str]] = None) -> DataBundle: - """ + r""" 从指定一个或多个路径中的文件中读取数据，返回 :class:`~fastNLP.io.DataBundle` 。 - :param Union[str, Dict[str, str]] paths: 支持以下的几种输入方式 - (0) 如果为None，则先查看本地是否有缓存，如果没有则自动下载并缓存。 + :param Union[str, Dict[str, str]] paths: 支持以下的几种输入方式： + + 0.如果为None，则先查看本地是否有缓存，如果没有则自动下载并缓存。 - (1) 传入一个目录, 该目录下名称包含train的被认为是train，包含test的被认为是test，包含dev的被认为是dev，如果检测到多个文件 - 名包含'train'、 'dev'、 'test'则会报错:: + 1.传入一个目录, 该目录下名称包含train的被认为是train，包含test的被认为是test，包含dev的被认为是dev，如果检测到多个文件名包含'train'、 'dev'、 'test'则会报错:: - data_bundle = xxxLoader().load('/path/to/dir') # 返回的DataBundle中datasets根据目录下是否检测到train、 + data_bundle = xxxLoader().load('/path/to/dir') # 返回的DataBundle中datasets根据目录下是否检测到train # dev、 test等有所变化，可以通过以下的方式取出DataSet tr_data = data_bundle.get_dataset('train') te_data = data_bundle.get_dataset('test') # 如果目录下有文件包含test这个字段 - (2) 传入一个dict，比如train，dev，test不在同一个目录下，或者名称中不包含train, dev, test:: + 2.传入一个dict，比如train，dev，test不在同一个目录下，或者名称中不包含train, dev, test:: paths = {'train':"/path/to/tr.conll", 'dev':"/to/validate.conll", "test":"/to/te.conll"} data_bundle = xxxLoader().load(paths) # 返回的DataBundle中的dataset中包含"train", "dev", "test" dev_data = data_bundle.get_dataset('dev') - (3) 传入文件路径:: + 3.传入文件路径:: data_bundle = xxxLoader().load("/path/to/a/train.conll") # 返回DataBundle对象, datasets中仅包含'train' tr_data = data_bundle.get_dataset('train') # 取出DataSet diff --git a/fastNLP/io/pipe/conll.py b/fastNLP/io/pipe/conll.py index a96b259a..12a94d20 100644 --- a/fastNLP/io/pipe/conll.py +++ b/fastNLP/io/pipe/conll.py @@ -28,12 +28,14 @@ class _NERPipe(Pipe): raw_words列为List[str], 是未转换的原始数据; words列为List[int]，是转换为index的输入数据; target列是List[int]，是转换为index的 target。返回的DataSet中被设置为input有words, target, seq_len; 设置为target有target, seq_len。 - - :param: str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 - :param bool lower: 是否将words小写化后再建立词表，绝大多数情况都不需要设置为True。 """ def __init__(self, encoding_type: str = 'bio', lower: bool = False): + """ + + :param: str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 + :param bool lower: 是否将words小写化后再建立词表，绝大多数情况都不需要设置为True。 + """ if encoding_type == 'bio': self.convert_tag = iob2 else: @@ -51,9 +53,8 @@ class _NERPipe(Pipe): "[AL-AIN, United, Arab, ...]", "[B-LOC, B-LOC, I-LOC, ...]" "[...]", "[...]" - :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]。 - 在传入DataBundle基础上原位修改。 - :return: DataBundle + :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]在传入DataBundle基础上原位修改。 + :return DataBundle: """ # 转换tag for name, dataset in data_bundle.datasets.items(): @@ -253,8 +254,7 @@ class _CNNERPipe(Pipe): raw_chars列为List[str], 是未转换的原始数据; chars列为List[int]，是转换为index的输入数据; target列是List[int]， 是转换为index的target。返回的DataSet中被设置为input有chars, target, seq_len; 设置为target有target。 - :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field - 的内容均为List[str]。在传入DataBundle基础上原位修改。 + :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]。在传入DataBundle基础上原位修改。 :return: DataBundle """ # 转换tag diff --git a/fastNLP/io/pipe/pipe.py b/fastNLP/io/pipe/pipe.py index db65ece6..07735d91 100644 --- a/fastNLP/io/pipe/pipe.py +++ b/fastNLP/io/pipe/pipe.py @@ -24,7 +24,7 @@ class Pipe: def process_from_file(self, paths) -> DataBundle: """ - 传入文件路径，生成处理好的DataBundle对象。paths支持的路径形式可以参考 `fastNLP.io.loader.Loader.load()` + 传入文件路径，生成处理好的DataBundle对象。paths支持的路径形式可以参考 ：:meth:`fastNLP.io.Loader.load()` :param paths: :return: DataBundle From df4c5a3d1667c57f58a495ba79990626f9181b2b Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 15:52:03 +0800 Subject: [PATCH 60/92] fix a lot of doc bugs in codes TODO: split __init__'s doc and classes' doc --- docs/README.md | 1 - .../tutorials/tutorial_1_data_preprocess.rst | 18 +++--- .../tutorials/tutorial_2_vocabulary.rst | 11 ++-- .../source/tutorials/tutorial_3_embedding.rst | 64 ++++++++++--------- .../tutorials/tutorial_4_load_dataset.rst | 36 +++++------ .../tutorials/tutorial_5_datasetiter.rst | 2 +- fastNLP/core/callback.py | 2 +- fastNLP/core/dataset.py | 2 +- 8 files changed, 64 insertions(+), 72 deletions(-) diff --git a/docs/README.md b/docs/README.md index 15dcccda..2bb6953c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,7 +32,6 @@ Serving HTTP on 0.0.0.0 port 8000 (http://0.0.0.0:8000/) ... 我们在[这里](./source/user/example.rst)列举了fastNLP文档经常用到的reStructuredText语法（网页查看请结合Raw模式）， 您可以通过阅读它进行快速上手。FastNLP大部分的文档都是写在代码中通过Sphinx工具进行抽取生成的， -您还可以参考这篇[未完成的文章](./source/user/docs_in_code.rst)了解代码内文档编写的规范。 ## 文档维护人员 diff --git a/docs/source/tutorials/tutorial_1_data_preprocess.rst b/docs/source/tutorials/tutorial_1_data_preprocess.rst index dfc3bbbe..47ad3b3f 100644 --- a/docs/source/tutorials/tutorial_1_data_preprocess.rst +++ b/docs/source/tutorials/tutorial_1_data_preprocess.rst @@ -2,9 +2,9 @@ DataSet ============================== -:class:`~fastNLP.DataSet` 是fastNLP用于承载数据的类，一般训练集、验证集和测试集会被加载为三个单独的:class:`~fastNLP.DataSet`对象。 +:class:`~fastNLP.DataSet` 是fastNLP用于承载数据的类，一般训练集、验证集和测试集会被加载为三个单独的 :class:`~fastNLP.DataSet` 对象。 -:class:`~fastNLP.DataSet`中的数据组织形式类似一个表格，比如下面 :class:`~fastNLP.DataSet` 一共有3列，列在fastNLP中被称为field。 +:class:`~fastNLP.DataSet` 中的数据组织形式类似一个表格，比如下面 :class:`~fastNLP.DataSet` 一共有3列，列在fastNLP中被称为field。 .. csv-table:: :header: "raw_chars", "chars", "seq_len" @@ -134,7 +134,7 @@ FastNLP 同样提供了多种删除数据的方法 :func:`~fastNLP.DataSet.drop` dataset.apply(get_words, new_field_name='words') 除了手动处理数据集之外，你还可以使用 fastNLP 提供的各种 :class:`~fastNLP.io.Loader`和:class:`~fastNLP.io.Pipe` 来进行数据处理。 -详细请参考这篇教程 :doc:`使用Loader和Pipe处理数据 ` 。 +详细请参考这篇教程 :doc:`使用Loader和Pipe处理数据 ` 。 ----------------------------- fastNLP中field的命名习惯 @@ -142,24 +142,22 @@ fastNLP中field的命名习惯 在英文任务中，fastNLP常用的field名称有: - - raw_words: 表示的是原始的str。例如"This is a demo sentence ."。存在多个raw_words的情况，例如matching任务，它们会被定义为 - raw_words0, raw_words1。但在conll格式下，raw_words列也可能为["This", "is", "a", "demo", "sentence", "."]的形式。 - - words: 表示的是已经tokenize后的词语。例如["This", "is", "a", "demo", "sentence"], 但由于str并不能直接被神经网络所使用， - 所以words中的内容往往被转换为int，如[3, 10, 4, 2, 7, ...]等。多列words的情况，会被命名为words0, words1 + - raw_words: 表示的是原始的str。例如"This is a demo sentence ."。存在多个raw_words的情况，例如matching任务，它们会被定义为raw_words0, raw_words1。但在conll格式下，raw_words列也可能为["This", "is", "a", "demo", "sentence", "."]的形式。 + - words: 表示的是已经tokenize后的词语。例如["This", "is", "a", "demo", "sentence"], 但由于str并不能直接被神经网络所使用，所以words中的内容往往被转换为int，如[3, 10, 4, 2, 7, ...]等。多列words的情况，会被命名为words0, words1 - target: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列。 - seq_len: 一般用于表示words列的长度 在中文任务中，fastNLP常用的field名称有: - raw_chars: 表示的是原始的连续汉字序列。例如"这是一个示例。" - - chars: 表示已经切分为单独的汉字的序列。例如["这", "是", "一", "个", "示", "例", "。"]。但由于神经网络不能识别汉字，所以一般 - 该列会被转为int形式，如[3, 4, 5, 6, ...]。 + - chars: 表示已经切分为单独的汉字的序列。例如["这", "是", "一", "个", "示", "例", "。"]。但由于神经网络不能识别汉字，所以一般该列会被转为int形式，如[3, 4, 5, 6, ...]。 - raw_words: 如果原始汉字序列中已经包含了词语的边界，则该列称为raw_words。如"上海 浦东 开发 与 法制 建设 同步"。 - words: 表示单独的汉字词语序列。例如["上海", "", "浦东", "开发", "与", "法制", "建设", ...]或[2, 3, 4, ...] - target: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列。 - seq_len: 表示输入序列的长度 -# TODO 这一段移动到datasetiter那里 +.. todo:: + 这一段移动到datasetiter那里 ----------------------------- DataSet与pad diff --git a/docs/source/tutorials/tutorial_2_vocabulary.rst b/docs/source/tutorials/tutorial_2_vocabulary.rst index 9656e4ec..d5bb9b7f 100644 --- a/docs/source/tutorials/tutorial_2_vocabulary.rst +++ b/docs/source/tutorials/tutorial_2_vocabulary.rst @@ -1,9 +1,8 @@ - ============================== Vocabulary ============================== - :class:`~fastNLP.Vocabulary`是包含字或词与index关系的类，用于将文本转换为index。 +:class:`~fastNLP.Vocabulary` 是包含字或词与index关系的类，用于将文本转换为index。 ----------------------------- 构建Vocabulary @@ -87,8 +86,8 @@ Vocabulary vocab.from_dataset(tr_data, field_name='chars', no_create_entry_dataset=[dev_data]) -:class:`~fastNLP.Vocabulary` 中的`no_create_entry`, 建议在添加来自于测试集和验证集的词的时候将该参数置为True, 或将验证集和测试集 -传入`no_create_entry_dataset`参数。它们的意义是在接下来的模型会使用pretrain的embedding(包括glove, word2vec, elmo与bert)且会finetune的 +:class:`~fastNLP.Vocabulary` 中的 `no_create_entry` , 建议在添加来自于测试集和验证集的词的时候将该参数置为True, 或将验证集和测试集 +传入 `no_create_entry_dataset` 参数。它们的意义是在接下来的模型会使用pretrain的embedding(包括glove, word2vec, elmo与bert)且会finetune的 情况下，如果仅使用来自于train的数据建立vocabulary，会导致只出现在test与dev中的词语无法充分利用到来自于预训练embedding的信息(因为他们 会被认为是unk)，所以在建立词表的时候将test与dev考虑进来会使得最终的结果更好。通过与fastNLP中的各种Embedding配合使用，会有如下的效果， 如果一个词出现在了train中，但是没在预训练模型中，embedding会为随机初始化，且它单独的一个vector，如果finetune embedding的话， @@ -96,8 +95,8 @@ Vocabulary 值(当unk的值更新时，这个词也使用的是更新之后的vector)。所以被认为是no_create_entry的token，将首先从预训练的词表中寻找它的表示，如 果找到了，就使用该表示; 如果没有找到，则认为该词的表示应该为unk的表示。 -下面我们结合部分:code:`~fastNLP.embeddings.StaticEmbedding`的例子来说明下该值造成的影响，如果您对 -:code:`~fastNLP.embeddings.StaticEmbedding`不太了解，您可以先参考\{Embedding教程的引用}部分再来阅读该部分 +下面我们结合部分 :class:`~fastNLP.embeddings.StaticEmbedding` 的例子来说明下该值造成的影响，如果您对 +:class:`~fastNLP.embeddings.StaticEmbedding` 不太了解，您可以先参考 :doc:`tutorial_3_embedding` 部分再来阅读该部分 .. code-block:: python diff --git a/docs/source/tutorials/tutorial_3_embedding.rst b/docs/source/tutorials/tutorial_3_embedding.rst index 4e29efed..9a6d00d2 100644 --- a/docs/source/tutorials/tutorial_3_embedding.rst +++ b/docs/source/tutorials/tutorial_3_embedding.rst @@ -26,17 +26,16 @@ elmo以及bert。 但使用这些词嵌入方式的时候都需要做一些加载上的处理，比如预训练的word2vec, fasttext以及glove都有着超过几十万个词语的表示，但一般任务大概 只会用到其中几万个词，如果直接加载所有的词汇，会导致内存占用变大以及运行速度变慢，需要从预训练文件中抽取本次实验的用到的词汇；而对于英文的 elmo和character embedding, 需要将word拆分成character才能使用；Bert的使用更是涉及到了Byte pair encoding(BPE)相关的内容。为了方便 -大家的使用，fastNLP通过:class:`~fastNLP.Vocabulary`统一了不同embedding的使用。下面我们将讲述一些例子来说明一下 +大家的使用，fastNLP通过 :class:`~fastNLP.Vocabulary` 统一了不同embedding的使用。下面我们将讲述一些例子来说明一下 --------------------------------------- Part II: 使用预训练的静态embedding --------------------------------------- -在fastNLP中，加载预训练的word2vec, glove以及fasttext都使用的是 :class:`~fastNLP.embeddings.StaticEmbedding`。另外，为了方便大家的 +在fastNLP中，加载预训练的word2vec, glove以及fasttext都使用的是 :class:`~fastNLP.embeddings.StaticEmbedding` 。另外，为了方便大家的 使用，fastNLP提供了多种静态词向量的自动下载并缓存(默认缓存到~/.fastNLP/embeddings文件夹下)的功能，支持自动下载的预训练向量可以在 -``_ -查看。 +`此处 `_ 查看。 .. code-block:: python @@ -56,17 +55,17 @@ Part II: 使用预训练的静态embedding torch.Size([1, 5, 50]) -fastNLP的StaticEmbedding在初始化之后，就和pytorch中的Embedding是类似的了。:class:`~fastNLP.embeddings.StaticEmbedding`的初始化 -主要是从model_dir_or_name提供的词向量中抽取出:class:`~fastNLP.Vocabulary`中词语的vector。 +fastNLP的StaticEmbedding在初始化之后，就和pytorch中的Embedding是类似的了。 :class:`~fastNLP.embeddings.StaticEmbedding` 的初始化 +主要是从model_dir_or_name提供的词向量中抽取出 :class:`~fastNLP.Vocabulary` 中词语的vector。 -除了可以通过使用预先提供的Embedding，:class:`~fastNLP.embeddings.StaticEmbedding`也支持加载本地的预训练词向量，glove, word2vec以及 +除了可以通过使用预先提供的Embedding, :class:`~fastNLP.embeddings.StaticEmbedding` 也支持加载本地的预训练词向量，glove, word2vec以及 fasttext格式的。通过将model_dir_or_name修改为本地的embedding文件路径，即可使用本地的embedding。 --------------------------------------- Part III: 使用随机初始化的embedding --------------------------------------- -有时候需要使用随机初始化的Embedding，也可以通过使用 :class:`~fastNLP.embeddings.StaticEmbedding`获得。只需要将model_dir_or_name +有时候需要使用随机初始化的Embedding，也可以通过使用 :class:`~fastNLP.embeddings.StaticEmbedding` 获得。只需要将model_dir_or_name 置为None，且传入embedding_dim，如下例所示 .. code-block:: python @@ -93,7 +92,7 @@ Part IV: ELMo Embedding 在fastNLP中，我们提供了ELMo和BERT的embedding： :class:`~fastNLP.embeddings.ElmoEmbedding` 和 :class:`~fastNLP.embeddings.BertEmbedding` 。可自动下载的ElmoEmbedding可以 -从``_找到。 +从 `此处 `_ 找到。 与静态embedding类似，ELMo的使用方法如下： @@ -124,7 +123,7 @@ Part IV: ELMo Embedding torch.Size([1, 5, 512]) -另外，根据``_，不同层之间使用可学习的权重可以使得ELMo的效果更好，在fastNLP中可以通过以下的初始化 +另外，根据 `这篇文章 `_ ，不同层之间使用可学习的权重可以使得ELMo的效果更好，在fastNLP中可以通过以下的初始化 实现3层输出的结果通过可学习的权重进行加法融合。 .. code-block:: python @@ -142,7 +141,7 @@ Part V: Bert Embedding ----------------------------------------------------------- 虽然Bert并不算严格意义上的Embedding，但通过将Bert封装成Embedding的形式将极大减轻使用的复杂程度。可自动下载的Bert Embedding可以 -从``_找到。我们将使用下面的例子讲述一下 +从 `此处 `_ 找到。我们将使用下面的例子讲述一下 BertEmbedding的使用 .. code-block:: python @@ -187,8 +186,8 @@ BertEmbedding的使用 torch.Size([1, 7, 768]) -在英文Bert模型中，一个英文单词可能会被切分为多个subword，例如"fairness"会被拆分为["fair", "##ness"]，这样一个word对应的将有两个输出， -:class:`~fastNLP.embeddings.BertEmbedding`会使用pooling方法将一个word的subword的表示合并成一个vector，通过pool_method可以控制 +在英文Bert模型中，一个英文单词可能会被切分为多个subword，例如"fairness"会被拆分为 ``["fair", "##ness"]`` ，这样一个word对应的将有两个输出， +:class:`~fastNLP.embeddings.BertEmbedding` 会使用pooling方法将一个word的subword的表示合并成一个vector，通过pool_method可以控制 该pooling方法，支持的有"first"(即使用fair的表示作为fairness的表示), "last"(使用##ness的表示作为fairness的表示), "max"(对fair和 ##ness在每一维上做max),"avg"(对fair和##ness每一维做average)。 @@ -201,7 +200,7 @@ BertEmbedding的使用 torch.Size([1, 5, 768]) -另外，根据``_ ，Bert的还存在一种用法，句子之间通过[SEP]拼接起来，前一句话的token embedding为0， +另外，根据 `文章 `_ ，Bert的还存在一种用法，句子之间通过[SEP]拼接起来，前一句话的token embedding为0， 后一句话的token embedding为1。BertEmbedding能够自动识别句子中间的[SEP]来正确设置对应的token_type_id的。 .. code-block:: python @@ -220,7 +219,10 @@ BertEmbedding的使用 在多个[SEP]的情况下，将会使token_type_id不断0，1循环。比如"first sentence [SEP] second sentence [SEP] third sentence", 它们的 token_type_id将是[0, 0, 0, 1, 1, 1, 0, 0]。但请注意[SEP]一定要大写的，不能是[sep]，否则无法识别。 -更多:class:`~fastNLP.embedding.BertEmbedding`的使用，请参考\ref{找人写一篇BertEmbedding的使用教程} +更多 :class:`~fastNLP.embedding.BertEmbedding` 的使用，请参考BertEmbedding的使用教程 + +.. todo:: + 找人写一篇BertEmbedding的使用教程 ----------------------------------------------------- Part VI: 使用character-level的embedding @@ -228,8 +230,8 @@ Part VI: 使用character-level的embedding 除了预训练的embedding以外，fastNLP还提供了两种Character Embedding： :class:`~fastNLP.embeddings.CNNCharEmbedding` 和 :class:`~fastNLP.embeddings.LSTMCharEmbedding` 。一般在使用character embedding时，需要在预处理的时候将word拆分成character，这 -会使得预处理过程变得非常繁琐。在fastNLP中，使用character embedding也只需要传入:class:`~fastNLP.Vocabulary`即可，而且该 -Vocabulary与其它Embedding使用的Vocabulary是一致的，如下面的例子所示 +会使得预处理过程变得非常繁琐。在fastNLP中，使用character embedding也只需要传入 :class:`~fastNLP.Vocabulary` 即可，而且该 +Vocabulary与其它Embedding使用的Vocabulary是一致的，下面我们看两个例子。 CNNCharEmbedding的使用例子如下： @@ -273,7 +275,7 @@ CNNCharEmbedding的使用例子如下： Part VII: 叠加使用多个embedding ----------------------------------------------------- -单独使用Character Embedding往往效果并不是很好，需要同时结合word embedding。在fastNLP中可以通过:class:`~fastNLP.embeddings.StackEmbedding` +单独使用Character Embedding往往效果并不是很好，需要同时结合word embedding。在fastNLP中可以通过 :class:`~fastNLP.embeddings.StackEmbedding` 来叠加embedding，具体的例子如下所示 .. code-block:: python @@ -295,10 +297,10 @@ Part VII: 叠加使用多个embedding torch.Size([1, 5, 114]) -:class:`~fastNLP.embeddings.StaticEmbedding`, :class:`~fastNLP.embeddings.ElmoEmbedding`, -:class:`~fastNLP.embeddings.CNNCharEmbedding`, :class:`~fastNLP.embeddings.BertEmbedding`等都可以互相拼接。 -:class:`~fastNLP.embeddings.StackEmbedding`的使用也是和其它Embedding是一致的，即输出index返回对应的表示。但能够拼接起来的Embedding -必须使用同样的:class:`~fastNLP.Vocabulary`，因为只有使用同样的:class:`~fastNLP.Vocabulary`才能保证同一个index指向的是同一个词或字 +:class:`~fastNLP.embeddings.StaticEmbedding` , :class:`~fastNLP.embeddings.ElmoEmbedding` , +:class:`~fastNLP.embeddings.CNNCharEmbedding` , :class:`~fastNLP.embeddings.BertEmbedding` 等都可以互相拼接。 +:class:`~fastNLP.embeddings.StackEmbedding` 的使用也是和其它Embedding是一致的，即输出index返回对应的表示。但能够拼接起来的Embedding +必须使用同样的 :class:`~fastNLP.Vocabulary` ，因为只有使用同样的 :class:`~fastNLP.Vocabulary` 才能保证同一个index指向的是同一个词或字 ----------------------------------------------------------- Part VIII: Embedding的其它说明 @@ -345,24 +347,24 @@ Part VIII: Embedding的其它说明 fastNLP中所有的Embedding都支持传入word_dropout和dropout参数，word_dropout指示的是以多大概率将输入的word置为unk的index，这样既可以 是的unk得到训练，也可以有一定的regularize效果; dropout参数是在获取到word的表示之后，以多大概率将一些维度的表示置为0。 -如果使用:class:`~fastNLP.embeddings.StackEmbedding`且需要用到word_dropout，建议将word_dropout设置在:class:`~fastNLP.embeddings.StackEmbedding`。 +如果使用 :class:`~fastNLP.embeddings.StackEmbedding` 且需要用到word_dropout，建议将word_dropout设置在 :class:`~fastNLP.embeddings.StackEmbedding` 。 ----------------------------------------------------------- Part IX: StaticEmbedding的使用建议 ----------------------------------------------------------- -在英文的命名实体识别(NER)任务中，由``_ 指出，同时使用cnn character embedding和word embedding -会使得NER的效果有比较大的提升。正如你在\ref{引用第七节}看到的那样，fastNLP支持将:class:`~fastNLP.embeddings.CNNCharacterEmbedding` -与:class:`~fastNLP.embeddings.StaticEmbedding`拼成一个:class:`~fastNLP.embeddings.StackEmbedding`。如果通过这种方式使用，需要 +在英文的命名实体识别(NER)任务中，由 `论文 `_ 指出，同时使用cnn character embedding和word embedding +会使得NER的效果有比较大的提升。正如你在上节中看到的那样，fastNLP支持将 :class:`~fastNLP.embeddings.CNNCharEmbedding` +与 :class:`~fastNLP.embeddings.StaticEmbedding` 拼成一个 :class:`~fastNLP.embeddings.StackEmbedding` 。如果通过这种方式使用，需要 在预处理文本时，不要将词汇小写化(因为Character Embedding需要利用词语中的大小写信息)且不要将出现频次低于某个阈值的word设置为unk(因为 -Character embedding需要利用字形信息)；但:class:`~fastNLP.embeddings.StaticEmbedding`使用的某些预训练词嵌入的词汇表中只有小写的词 +Character embedding需要利用字形信息)；但 :class:`~fastNLP.embeddings.StaticEmbedding` 使用的某些预训练词嵌入的词汇表中只有小写的词 语, 且某些低频词并未在预训练中出现需要被剔除。即(1) character embedding需要保留大小写，而某些static embedding不需要保留大小写。(2) character embedding需要保留所有的字形, 而static embedding需要设置一个最低阈值以学到更好的表示。 (1) fastNLP如何解决关于大小写的问题 -fastNLP通过在:class:`~fastNLP.embeddings.StaticEmbedding`增加了一个lower参数解决该问题。如下面的例子所示 +fastNLP通过在 :class:`~fastNLP.embeddings.StaticEmbedding` 增加了一个lower参数解决该问题。如下面的例子所示 .. code-block:: python @@ -380,7 +382,7 @@ fastNLP通过在:class:`~fastNLP.embeddings.StaticEmbedding`增加了一个lower tensor([[-0.4685, 0.4572, 0.5159, -0.2618, -0.6871]], grad_fn=) tensor([[ 0.2615, 0.1490, -0.2491, 0.4009, -0.3842]], grad_fn=) -可以看到"The"与"the"的vector是不一致的。但如果我们在初始化:class:`~fastNLP.embeddings.StaticEmbedding`将lower设置为True，效果将 +可以看到"The"与"the"的vector是不一致的。但如果我们在初始化 :class:`~fastNLP.embeddings.StaticEmbedding` 将lower设置为True，效果将 如下所示 .. code-block:: python @@ -399,12 +401,12 @@ fastNLP通过在:class:`~fastNLP.embeddings.StaticEmbedding`增加了一个lower tensor([[-0.2237, 0.6825, -0.3459, -0.1795, 0.7516]], grad_fn=) tensor([[-0.2237, 0.6825, -0.3459, -0.1795, 0.7516]], grad_fn=) -可以看到"The"与"the"的vector是一致的。他们实际上也是引用的同一个vector。通过将lower设置为True，可以在:class:`~fastNLP.embeddings.StaticEmbedding` +可以看到"The"与"the"的vector是一致的。他们实际上也是引用的同一个vector。通过将lower设置为True，可以在 :class:`~fastNLP.embeddings.StaticEmbedding` 实现类似具备相同小写结果的词语引用同一个vector。 (2) fastNLP如何解决min_freq的问题 -fastNLP通过在:class:`~fastNLP.embeddings.StaticEmbedding`增加了一个min_freq参数解决该问题。如下面的例子所示 +fastNLP通过在 :class:`~fastNLP.embeddings.StaticEmbedding` 增加了一个min_freq参数解决该问题。如下面的例子所示 .. code-block:: python diff --git a/docs/source/tutorials/tutorial_4_load_dataset.rst b/docs/source/tutorials/tutorial_4_load_dataset.rst index c7e49fac..f5f8dbd4 100644 --- a/docs/source/tutorials/tutorial_4_load_dataset.rst +++ b/docs/source/tutorials/tutorial_4_load_dataset.rst @@ -18,11 +18,11 @@ Part I: 数据集容器DataBundle ------------------------------------ 而由于对于同一个任务，训练集，验证集和测试集会共用同一个词表以及具有相同的目标值，所以在fastNLP中我们使用了 :class:`~fastNLP.io.DataBundle` -来承载同一个任务的多个数据集 :class:`~fastNLP.DataSet` 以及它们的词表 :class:`~fastNLP.Vocabulary`。下面会有例子介绍:class:`~fastNLP.io.DataBundle` +来承载同一个任务的多个数据集 :class:`~fastNLP.DataSet` 以及它们的词表 :class:`~fastNLP.Vocabulary`。下面会有例子介绍 :class:`~fastNLP.io.DataBundle` 的相关使用。 -:class: `~fastNLP.io.DataBundle` 在fastNLP中主要在各个 :class: `~fastNLP.io.Loader` 和 :class: `~fastNLP.io.Pipe` 中被使用。 -下面我们将先介绍一下 :class: `~fastNLP.io.Loader` 和 :class: `~fastNLP.io.Pipe`, 之后我们将给出相应的例子。 +:class:`~fastNLP.io.DataBundle` 在fastNLP中主要在各个 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` 中被使用。 +下面我们将先介绍一下 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` , 之后我们将给出相应的例子。 ------------------------------------- Part II: 加载的各种数据集的Loader @@ -31,13 +31,12 @@ Part II: 加载的各种数据集的Loader 在fastNLP中，所有的数据Loader都可以通过其文档判断其支持读取的数据格式，以及读取之后返回的 :class:`~fastNLP.DataSet` 的格式。例如 \ref 加个引用。 - - download 函数：自动将该数据集下载到缓存地址，默认缓存地址为~/.fastNLP/datasets/。由于版权等原因，不是所有的Loader都实现了该方法。 - 该方法会返回下载后文件所处的缓存地址。可以查看对应Loader的download的方法的文档来判断该Loader加载的数据。 - - _load 函数：从一个数据文件中读取数据，返回一个 :class:`~fastNLP.DataSet`。返回的DataSet的格式可从Loader文档判断。 + - download 函数：自动将该数据集下载到缓存地址，默认缓存地址为~/.fastNLP/datasets/。由于版权等原因，不是所有的Loader都实现了该方法。该方法会返回下载后文件所处的缓存地址。可以查看对应Loader的download的方法的文档来判断该Loader加载的数据。 + - _load 函数：从一个数据文件中读取数据，返回一个 :class:`~fastNLP.DataSet` 。返回的DataSet的格式可从Loader文档判断。 - load 函数：从文件或者文件夹中读取数据并组装成 :class:`~fastNLP.io.DataBundle`。支持接受的参数类型有以下的几种 + - None, 将尝试读取自动缓存的数据，仅支持提供了自动下载数据的Loader - - 文件夹路径, 默认将尝试在该路径下匹配文件名中含有`train`, `test`, `dev`的文件，如果有多个文件含有这相同的关键字，将无法通过 - 该方式读取 + - 文件夹路径, 默认将尝试在该路径下匹配文件名中含有 `train` , `test` , `dev` 的python文件，如果有多个文件含有这相同的关键字，将无法通过该方式读取 - dict, 例如{'train':"/path/to/tr.conll", 'dev':"/to/validate.conll", "test":"/to/te.conll"} .. code-block:: python @@ -66,7 +65,7 @@ Part II: 加载的各种数据集的Loader tr_data = data_bundle.get_dataset('train') print(tr_data[:2]) - 输出为:: +输出为:: +--------------------------------------------------------------------------------------+ | raw_words | @@ -81,18 +80,16 @@ Part III: 使用Pipe对数据集进行预处理 通过:class:`~fastNLP.io.Loader` 可以将文本数据读入，但并不能直接被神经网络使用，还需要进行一定的预处理。 在fastNLP中，我们使用 :class:`~fastNLP.io.Pipe`的子类作为数据预处理的类，Pipe和Loader一般具备一一对应的关系，该关系可以从其名称判断， -例如:class:`~fastNLP.io.CWSLoader`与:class:`~fastNLP.io.CWSPipe`是一一对应的。一般情况下Pipe处理包含以下的几个过程，(1)将raw_words或 -raw_chars进行tokenize以切分成不同的词或字; (2) 再建立词或字的 :class:`~fastNLP.Vocabulary`, 并将词或字转换为index; (3)将target +例如 :class:`~fastNLP.io.CWSLoader` 与 :class:`~fastNLP.io.CWSPipe` 是一一对应的。一般情况下Pipe处理包含以下的几个过程，(1)将raw_words或 +raw_chars进行tokenize以切分成不同的词或字; (2) 再建立词或字的 :class:`~fastNLP.Vocabulary` , 并将词或字转换为index; (3)将target 列建立词表并将target列转为index; 所有的Pipe都可通过其文档查看通过该Pipe之后DataSet中的field的情况; 如 \ref{TODO 添加对例子的引用} 各种数据集的Pipe当中，都包含了以下的两个函数: - - process 函数：对输入的 :class:`~fastNLP.io.DataBundle` 进行处理, 然后返回处理之后的 :class:`~fastNLP.io.DataBundle`。 - process函数的文档中包含了该Pipe支持处理的DataSet的格式。 - - process_from_file 函数：输入数据集所在文件夹，使用对应的Loader读取数据(所以该函数支持的参数类型是由于其对应的Loader的load函数 - 决定的)，然后调用相对应的process函数对数据进行预处理。相当于是把Load和process放在一个函数中执行。 + - process 函数：对输入的 :class:`~fastNLP.io.DataBundle` 进行处理, 然后返回处理之后的 :class:`~fastNLP.io.DataBundle` 。process函数的文档中包含了该Pipe支持处理的DataSet的格式。 + - process_from_file 函数：输入数据集所在文件夹，使用对应的Loader读取数据(所以该函数支持的参数类型是由于其对应的Loader的load函数决定的)，然后调用相对应的process函数对数据进行预处理。相当于是把Load和process放在一个函数中执行。 接着上面CWSLoader的例子，我们展示一下CWSPipe的功能： @@ -116,8 +113,7 @@ raw_chars进行tokenize以切分成不同的词或字; (2) 再建立词或字的 表示一共有3个数据集和2个词表。其中： - 3个数据集分别为train、dev、test数据集，分别有17223、1831、1944个instance - - 2个词表分别为chars词表与target词表。其中chars词表为句子文本所构建的词表，一共有4777个字； - target词表为目标标签所构建的词表，一共有4种标签。 + - 2个词表分别为chars词表与target词表。其中chars词表为句子文本所构建的词表，一共有4777个字；target词表为目标标签所构建的词表，一共有4种标签。 相较于之前CWSLoader读取的DataBundle，新增了两个Vocabulary。 我们可以打印一下处理之后的DataSet @@ -161,8 +157,7 @@ Part V: 不同格式类型的基础Loader 除了上面提到的针对具体任务的Loader，我们还提供了CSV格式和JSON格式的Loader -:class:`~fastNLP.io.loader.CSVLoader` - 读取CSV类型的数据集文件。例子如下： +:class:`~fastNLP.io.loader.CSVLoader` 读取CSV类型的数据集文件。例子如下： .. code-block:: python @@ -190,8 +185,7 @@ Part V: 不同格式类型的基础Loader "You could hate it for the same reason .", "1" "The performances are an absolute joy .", "4" -:class:`~fastNLP.io.loader.JsonLoader` - 读取Json类型的数据集文件，数据必须按行存储，每行是一个包含各类属性的Json对象。例子如下： +:class:`~fastNLP.io.JsonLoader` 读取Json类型的数据集文件，数据必须按行存储，每行是一个包含各类属性的Json对象。例子如下： .. code-block:: python diff --git a/docs/source/tutorials/tutorial_5_datasetiter.rst b/docs/source/tutorials/tutorial_5_datasetiter.rst index 2ec753c3..6076214f 100644 --- a/docs/source/tutorials/tutorial_5_datasetiter.rst +++ b/docs/source/tutorials/tutorial_5_datasetiter.rst @@ -4,7 +4,7 @@ 我们使用和 :doc:`/user/quickstart` 中一样的任务来进行详细的介绍。给出一段评价性文字，预测其情感倾向是积极（label=1）、 消极（label=0）还是中性（label=2），使用 :class:`~fastNLP.DataSetIter` 类来编写自己的训练过程。 -自己编写训练过程之前的内容与 :doc:`/tutorials/tutorial_4_loss_optimizer` 中的完全一样，如已经阅读过可以跳过。 +自己编写训练过程之前的内容与 :doc:`/tutorials/tutorial_6_loss_optimizer` 中的完全一样，如已经阅读过可以跳过。 -------------- 数据处理 diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index fe198acc..520ea733 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -805,7 +805,7 @@ class TensorboardCallback(Callback): .. warning:: fastNLP 已停止对此功能的维护，请等待 fastNLP 兼容 PyTorch1.1 的下一个版本。 - 或者使用和 fastNLP 高度配合的 fitlog（参见 :doc:`/tutorials/tutorial_10_fitlog` ）。 + 或者使用和 fastNLP 高度配合的 fitlog（参见 :doc:`/tutorials/tutorial_11_fitlog` ）。 """ diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 1fa0fd10..38395e57 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -86,7 +86,7 @@ dataset.append(Instance(sentence=sent, label=label)) .. note:: - 直接读取特定数据集的数据请参考 :doc:`/tutorials/tutorial_2_load_dataset` + 直接读取特定数据集的数据请参考 :doc:`/tutorials/tutorial_4_load_dataset` 2.2 对DataSet中的内容处理 -------------------------------------- From d481c84abc4e8631dc03ddce170809e4f93128fc Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 15:53:41 +0800 Subject: [PATCH 61/92] delete a out-date file --- docs/source/user/docs_in_code.rst | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 docs/source/user/docs_in_code.rst diff --git a/docs/source/user/docs_in_code.rst b/docs/source/user/docs_in_code.rst deleted file mode 100644 index a0b9576f..00000000 --- a/docs/source/user/docs_in_code.rst +++ /dev/null @@ -1,3 +0,0 @@ -=============== -在代码中写文档 -=============== \ No newline at end of file From b015cc149cb5c0688a882118410099d6b84bf78d Mon Sep 17 00:00:00 2001 From: xxliu Date: Tue, 10 Sep 2019 16:14:18 +0800 Subject: [PATCH 62/92] undocumented --- fastNLP/io/loader/coreference.py | 4 ++-- fastNLP/io/pipe/coreference.py | 12 ++++++------ reproduction/coreference_resolution/train.py | 3 ++- reproduction/coreference_resolution/valid.py | 2 +- test/data_for_tests/coreference/coreference_dev.json | 3 +-- .../data_for_tests/coreference/coreference_test.json | 3 +-- .../coreference/coreference_train.json | 3 +-- 7 files changed, 14 insertions(+), 16 deletions(-) diff --git a/fastNLP/io/loader/coreference.py b/fastNLP/io/loader/coreference.py index 6e2344d2..714b11e5 100644 --- a/fastNLP/io/loader/coreference.py +++ b/fastNLP/io/loader/coreference.py @@ -26,8 +26,8 @@ class CRLoader(JsonLoader): super().__init__(fields, dropna) # self.fields = {"doc_key":Const.INPUTS(0),"speakers":Const.INPUTS(1),"clusters":Const.TARGET,"sentences":Const.INPUTS(2)} # TODO check 1 - self.fields = {"doc_key": "raw_key", "speakers": "raw_speakers", "clusters": "raw_clusters", - "sentences": "raw_words"} + self.fields = {"doc_key": Const.RAW_WORDS(0), "speakers": Const.RAW_WORDS(1), "clusters": Const.RAW_WORDS(2), + "sentences": Const.RAW_WORDS(3)} def _load(self, path): """ diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py index bb40ca55..b6d88998 100644 --- a/fastNLP/io/pipe/coreference.py +++ b/fastNLP/io/pipe/coreference.py @@ -46,10 +46,10 @@ class CoreferencePipe(Pipe): :return: """ genres = {g: i for i, g in enumerate(["bc", "bn", "mz", "nw", "pt", "tc", "wb"])} - vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name="raw_words") + vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name= Const.RAW_WORDS(3)) vocab.build_vocab() word2id = vocab.word2idx - data_bundle.set_vocab(vocab,"vocab") + data_bundle.set_vocab(vocab,Const.INPUT) if self.config.char_path: char_dict = get_char_dict(self.config.char_path) else: @@ -65,14 +65,14 @@ class CoreferencePipe(Pipe): for name, ds in data_bundle.datasets.items(): # genre - ds.apply(lambda x: genres[x["raw_key"][:2]], new_field_name=Const.INPUTS(0)) + ds.apply(lambda x: genres[x[Const.RAW_WORDS(0)][:2]], new_field_name=Const.INPUTS(0)) # speaker_ids_np - ds.apply(lambda x: speaker2numpy(x["raw_speakers"], self.config.max_sentences, is_train=name == 'train'), + ds.apply(lambda x: speaker2numpy(x[Const.RAW_WORDS(1)], self.config.max_sentences, is_train=name == 'train'), new_field_name=Const.INPUTS(1)) # sentences - ds.rename_field("raw_words",Const.INPUTS(2)) + ds.rename_field(Const.RAW_WORDS(3),Const.INPUTS(2)) # doc_np ds.apply(lambda x: doc2numpy(x[Const.INPUTS(2)], word2id, char_dict, max(self.config.filter), @@ -88,7 +88,7 @@ class CoreferencePipe(Pipe): new_field_name=Const.INPUT_LEN) # clusters - ds.rename_field("raw_clusters", Const.TARGET) + ds.rename_field(Const.RAW_WORDS(2), Const.TARGET) ds.set_ignore_type(Const.TARGET) diff --git a/reproduction/coreference_resolution/train.py b/reproduction/coreference_resolution/train.py index cd4b65a5..23ba5d5b 100644 --- a/reproduction/coreference_resolution/train.py +++ b/reproduction/coreference_resolution/train.py @@ -8,6 +8,7 @@ from fastNLP.core.callback import Callback, GradientClipCallback from fastNLP.core.trainer import Trainer from fastNLP.io.pipe.coreference import CoreferencePipe +from fastNLP.core.const import Const from reproduction.coreference_resolution.model.config import Config from reproduction.coreference_resolution.model.model_re import Model @@ -45,7 +46,7 @@ if __name__ == "__main__": print("数据集划分：\ntrain:", str(len(data_bundle.get_dataset("train"))), "\ndev:" + str(len(data_bundle.get_dataset("dev"))) + "\ntest:" + str(len(data_bundle.get_dataset('test')))) # print(data_info) - model = Model(data_bundle.get_vocab("vocab"), config) + model = Model(data_bundle.get_vocab(Const.INPUT), config) print(model) loss = SoftmaxLoss() diff --git a/reproduction/coreference_resolution/valid.py b/reproduction/coreference_resolution/valid.py index 454629e1..a528ea06 100644 --- a/reproduction/coreference_resolution/valid.py +++ b/reproduction/coreference_resolution/valid.py @@ -17,7 +17,7 @@ if __name__=='__main__': {'train': config.train_path, 'dev': config.dev_path, 'test': config.test_path}) metirc = CRMetric() model = torch.load(args.path) - tester = Tester(bundle.datasets['test'],model,metirc,batch_size=1,device="cuda:0") + tester = Tester(bundle.get_dataset("test"),model,metirc,batch_size=1,device="cuda:0") tester.test() print('test over') diff --git a/test/data_for_tests/coreference/coreference_dev.json b/test/data_for_tests/coreference/coreference_dev.json index 9322ed30..bb6592d3 100644 --- a/test/data_for_tests/coreference/coreference_dev.json +++ b/test/data_for_tests/coreference/coreference_dev.json @@ -1,2 +1 @@ -{"doc_key": "bc/cctv/00/cctv_0000_0", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2", "Speaker#2"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"]], "clusters": [[[70, 70], [485, 486], [500, 500], [73, 73], [55, 55], [153, 154], [366, 366]], [[307, 312], [255, 256]], [[198, 199], [163, 164]], [[289, 290], [318, 318], [494, 497], [129, 131], [261, 261], [86, 86], [387, 387], [278, 278], [122, 124], [51, 56], [221, 225], [353, 355], [292, 292], [299, 299], [322, 322], [348, 348], [311, 312], [251, 253]], [[143, 144], [138, 138]], [[155, 176], [213, 214], [183, 184], [195, 195]], [[398, 398], [403, 403], [335, 335], [390, 390]], [[28, 28], [32, 37]], [[337, 338], [372, 373]], [[129, 130], [488, 489], [122, 123], [108, 109], [147, 148], [191, 192], [41, 42], [23, 24], [251, 252]], [[208, 208], [201, 204]], [[377, 379], [411, 413]]], "sentences": [["In", "the", "summer", "of", "2005", ",", "a", "picture", "that", "people", "have", "long", "been", "looking", "forward", "to", "started", "emerging", "with", "frequency", "in", "various", "major", "Hong", "Kong", "media", "."], ["With", "their", "unique", "charm", ",", "these", "well", "-", "known", "cartoon", "images", "once", "again", "caused", "Hong", "Kong", "to", "be", "a", "focus", "of", "worldwide", "attention", "."], ["The", "world", "'s", "fifth", "Disney", "park", "will", "soon", "open", "to", "the", "public", "here", "."], ["The", "most", "important", "thing", "about", "Disney", "is", "that", "it", "is", "a", "global", "brand", "."], ["Well", ",", "for", "several", "years", ",", "although", "it", "was", "still", "under", "construction", "and", ",", "er", ",", "not", "yet", "open", ",", "it", "can", "be", "said", "that", "many", "people", "have", "viewed", "Hong", "Kong", "with", "new", "respect", "."], ["Then", "welcome", "to", "the", "official", "writing", "ceremony", "of", "Hong", "Kong", "Disneyland", "."], ["The", "construction", "of", "Hong", "Kong", "Disneyland", "began", "two", "years", "ago", ",", "in", "2003", "."], ["In", "January", "of", "that", "year", ",", "the", "Hong", "Kong", "government", "turned", "over", "to", "Disney", "Corporation", "200", "hectares", "of", "land", "at", "the", "foot", "of", "Lantau", "Island", "that", "was", "obtained", "following", "the", "largest", "land", "reclamation", "project", "in", "recent", "years", "."], ["One", "."], ["Since", "then", ",", "this", "area", "has", "become", "a", "prohibited", "zone", "in", "Hong", "Kong", "."], ["As", "its", "neighbor", "on", "Lantau", "Island", ",", "Hong", "Kong", "International", "Airport", "had", "to", "change", "its", "flight", "routes", "to", "make", "this", "area", "a", "no", "-", "fly", "zone", "."], ["Mickey", "Mouse", "'s", "new", "home", ",", "settling", "on", "Chinese", "land", "for", "the", "first", "time", ",", "has", "captured", "worldwide", "attention", "."], ["There", "'s", "only", "one", "month", "left", "before", "the", "opening", "of", "Hong", "Kong", "Disneyland", "on", "September", "12", "."], ["The", "subway", "to", "Disney", "has", "already", "been", "constructed", "."], ["At", "subway", "stations", ",", "passengers", "will", "frequently", "press", "the", "station", "for", "Disney", "on", "ticket", "machines", ",", "trying", "to", "purchase", "tickets", "to", "enjoy", "the", "park", "when", "it", "first", "opens", "."], ["Meanwhile", ",", "the", "Disney", "subway", "station", "is", "scheduled", "to", "open", "on", "the", "same", "day", "as", "the", "park", "."], ["For", "two", "years", ",", "Disney", "has", "constantly", "maintained", "its", "mystery", "."], ["No", "media", "have", "been", "allowed", "to", "enter", "for", "photos", "."], ["We", "took", "a", "taxi", "along", "the", "path", "of", "the", "highway", "that", "heads", "toward", "Disney", ",", "trying", "to", "experience", "this", "mysterious", "park", "from", "close", "by", "."], ["However", ",", "before", "any", "of", "the", "Disney", "symbols", "were", "in", "sight", ",", "the", "car", "was", "stopped", "by", "a", "security", "guard", "at", "the", "intersection", "of", "the", "road", "towards", "Disney", "."], ["On", "our", "way", "back", ",", "the", "taxi", "driver", "gave", "us", "an", "explanation", "after", "understanding", "our", "intentions", "."], ["Er", ",", "according", "to", "what", "the", "security", "guard", "said", ",", "for", "the", "time", "before", "everything", "is", "officially", ",", "opened", ",", ",", "no", "cars", "can", "enter", "unless", "they", "have", "special", "permission", "."], ["No", "one", "can", "enter", "otherwise", "."], ["Video", "recording", "is", "especially", "forbidden", "."], ["Ah", ",", "everything", "is", "top", "secret", "."], ["If", "pictures", "are", "taken", "without", "permission", ",", "%pw", "that", "is", "to", "say", ",", "it", "will", "at", "all", "times", "be", "pursued", "by", "legal", "action", ",", "a", "big", "hassle", "."], ["Although", "Disney", "Corporation", "chose", "Hong", "Kong", "as", "the", "venue", "for", "the", "Chinese", "Disney", "park", ",", "what", "they", "are", "actually", "most", "excited", "about", "is", "the", "mainland", "China", "tourist", "market", "."]]} -{"doc_key": "bc/cctv/00/cctv_0000_1", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi", "Zhou_liangshuyi"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"]], "clusters": [[[24, 25], [121, 122], [44, 45], [83, 84], [9, 10], [233, 235], [199, 200]]], "sentences": [["Since", "the", "implementation", "of", "the", "Individual", "Visit", "Scheme", "between", "Hong", "Kong", "and", "the", "mainland", ",", "more", "and", "more", "mainland", "tourists", "are", "coming", "to", "visit", "Hong", "Kong", "."], ["From", "the", "beginning", "up", "till", "now", ",", "more", "than", "seven", "million", "individual", "tourists", ",", "have", "come", "to", "Hong", "Kong", "."], ["Well", ",", "we", "now", ",", "er", ",", "believe", "more", "will", "be", "coming", "."], ["At", "this", "point", ",", "it", "has", "been", "about", "two", "years", "."], ["Also", ",", "the", "current", "number", "of", "34", "cities", "will", "be", "increased", "."], ["Hong", "Kong", "was", "developed", "from", "a", "fishing", "harbor", "one", "hundred", "years", "ago", "to", "become", "today", "'s", "international", "metropolis", "."], ["Here", ",", "eastern", "and", "western", "cultures", "have", "gathered", ",", "and", "the", "new", "and", "the", "old", "coexist", "."], ["When", "in", "Hong", "Kong", ",", "you", "can", "wander", "among", "skyscrapers", ",", "heartily", "enjoy", "shopping", "sprees", "in", "well", "-", "known", "stores", "and", "malls", "for", "goods", "from", "various", "countries", ",", "and", "taste", "delicious", "snacks", "from", "all", "over", "the", "world", "at", "tea", "shops", "or", "at", "street", "stands", "in", "Mong", "Kok", "."], ["You", "can", "go", "to", "burn", "incense", "and", "make", "a", "vow", "at", "the", "Repulse", "Bay", ",", "where", "all", "deities", "gather", "."], ["You", "can", "enjoy", "the", "most", "charming", "sun", "-", "filled", "sandy", "beaches", "in", "Hong", "Kong", "."], ["You", "can", "ascend", "Victoria", "Peak", "to", "get", "a", "panoramic", "view", "of", "Victoria", "Harbor", "'s", "beautiful", "scenery", "."], ["Or", "hop", "onto", "a", "trolley", "with", "over", "a", "century", "of", "history", ",", "and", "feel", "the", "city", "'s", "blend", "of", "the", "old", "and", "the", "modern", "in", "slow", "motion", "."]]} +{"doc_key": "bc/cctv/00/cctv_0000_0", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"]], "clusters": [[[70, 70], [485, 486], [500, 500], [73, 73], [55, 55], [153, 154], [366, 366]]], "sentences": [["In", "the", "summer", "of", "2005", ",", "a", "picture", "that", "people", "have", "long", "been", "looking", "forward", "to", "started", "emerging", "with", "frequency", "in", "various", "major", "Hong", "Kong", "media", "."], ["With", "their", "unique", "charm", ",", "these", "well", "-", "known", "cartoon", "images", "once", "again", "caused", "Hong", "Kong", "to", "be", "a", "focus", "of", "worldwide", "attention", "."]]} diff --git a/test/data_for_tests/coreference/coreference_test.json b/test/data_for_tests/coreference/coreference_test.json index 399b8cc5..9577da0e 100644 --- a/test/data_for_tests/coreference/coreference_test.json +++ b/test/data_for_tests/coreference/coreference_test.json @@ -1,2 +1 @@ -{"doc_key": "bc/cctv/00/cctv_0005_0", "speakers": [["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"], ["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"], ["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"], ["Xu_li", "Xu_li", "Xu_li", "Xu_li", "Xu_li"]], "clusters": [[[57, 59], [25, 27], [42, 44]], [[19, 23], [16, 16]], [[83, 83], [82, 82]]], "sentences": [["--", "basically", ",", "it", "was", "unanimously", "agreed", "upon", "by", "the", "various", "relevant", "parties", "."], ["To", "express", "its", "determination", ",", "the", "Chinese", "securities", "regulatory", "department", "compares", "this", "stock", "reform", "to", "a", "die", "that", "has", "been", "cast", "."], ["It", "takes", "time", "to", "prove", "whether", "the", "stock", "reform", "can", "really", "meet", "expectations", ",", "and", "whether", "any", "deviations", "that", "arise", "during", "the", "stock", "reform", "can", "be", "promptly", "corrected", "."], ["Dear", "viewers", ",", "the", "China", "News", "program", "will", "end", "here", "."], ["This", "is", "Xu", "Li", "."], ["Thank", "you", "everyone", "for", "watching", "."], ["Coming", "up", "is", "the", "Focus", "Today", "program", "hosted", "by", "Wang", "Shilin", "."], ["Good-bye", ",", "dear", "viewers", "."]]} -{"doc_key": "bc/cctv/00/cctv_0005_1", "speakers": [["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Wang_shilin", "Wang_shilin"], ["Zhou_hanhua", "Zhou_hanhua"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua", "Zhou_hanhua"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"], ["Yang_yang", "Yang_yang", "Yang_yang", "Yang_yang"], ["Wang_shilin", "Wang_shilin", "Wang_shilin", "Wang_shilin"]], "clusters": [[[233, 234], [7, 8]], [[253, 254], [438, 439]], [[411, 412], [64, 67], [18, 30], [259, 260], [516, 516]], [[432, 433], [190, 204], [272, 272], [325, 325], [314, 314], [292, 292], [281, 281], [334, 334]], [[310, 311], [299, 300], [321, 321]], [[172, 172], [10, 10]], [[372, 373], [392, 393], [216, 219], [418, 419]], [[29, 30], [108, 109], [112, 113]], [[72, 73], [59, 60], [27, 27]], [[305, 305], [377, 377]], [[502, 503], [444, 447], [459, 460]], [[352, 353], [387, 387], [362, 362], [408, 408], [210, 219], [375, 375], [360, 360], [350, 350]], [[182, 185], [166, 168], [247, 250], [224, 226]], [[383, 384], [51, 60]], [[367, 368], [268, 268], [35, 36], [256, 260]], [[523, 523], [500, 500], [493, 493], [435, 435], [238, 238]], [[228, 229], [187, 188], [170, 171]]], "sentences": [["Hello", ",", "dear", "viewers", "."], ["Welcome", "to", "Focus", "Today", "."], ["Today", ",", "let", "'s", "turn", "our", "attention", "to", "a", "road", "cave", "-", "in", "accident", "that", "happened", "in", "Beijing", "over", "the", "holiday", "."], ["Before", "dawn", "on", "January", "3", ",", "a", "sewage", "pipe", "leakage", "accident", "occurred", "at", "the", "main", "and", "side", "roads", "of", "Jingguang", "Bridge", ",", "East", "Third", "Ring", "Road", ",", "Beijing", "Municipality", ",", "resulting", "in", "the", "road", "caving", "in", "."], ["Relevant", "departments", "from", "Beijing", "Municipality", "promptly", "activated", "emergency", "contingency", "plans", "."], ["The", "traffic", "administration", "department", "carried", "out", "traffic", "supervision", "near", "the", "accident", "scene", "."], ["Well", ",", "how", "did", "the", "emergency", "response", "mechanisms", "activated", "by", "governmental", "departments", "operate", "effectively", "during", "the", "holiday", "?"], ["After", "the", "holiday", ",", "what", "will", "be", "done", "to", "handle", "citizens", "'", "peak", "commute", "?"], ["In", "addition", ",", "what", "measures", "did", "relevant", "departments", "take", "to", "resolve", "issues", "such", "as", "waste", "discharge", ",", "heating", ",", "and", "communication", ",", "in", "order", "to", "ensure", "that", "the", "lives", "of", "citizens", "were", "not", "affected", "?"], ["Well", ",", "we", "have", "invited", "two", "honorable", "guests", "to", "the", "studio", "today", "to", "follow", "this", "topic", "with", "us", "."], ["One", "of", "the", "two", "honorable", "guests", "in", "the", "studio", "is", "Professor", "Zhou", "Hanhua", "from", "the", "Institute", "of", "Law", "of", "the", "Chinese", "Academy", "of", "Social", "Sciences", "."], ["Hello", "."], ["Next", "is", "Yang", "Yang", ",", "a", "host", "of", "Beijing", "Traffic", "Radio", "Station", "."], ["Hello", "."], ["Welcome", "both", "of", "you", "to", "the", "studio", "to", "participate", "in", "our", "program", "."], ["Well", ",", "I", "especially", "want", "to", "know", ",", "ha", ",", "how", "the", "two", "of", "you", "found", "out", "the", "news", "on", "the", "day", "of", "the", "accident", "?"], ["Ah", ",", ",", "about", "11:00", "m.", "yesterday", ",", "ah", ",", "I", "happened", "to", "find", "out", "through", "an", "SMS", "when", "I", "was", "outside", "."], ["Uh-huh", "."], ["Uh-huh", "."], ["It", "happened", "that", "I", "was", "going", "to", "have", "lunch", "with", "a", "friend", ",", "um", ",", "at", "noon", "."], ["And", "then", ",", "the", "friend", "first", "sent", "me", "an", "SMS", ",", "Uh-huh", ".", "saying", "he", "would", "come", "pick", "me", "up", "to", "go", "together", "."], ["After", "that", ",", "I", "received", "an", "SMS", "from", "1860", "."], ["Uh-huh", ",", "it", "was", "through", "an", "SMS", "."], ["And", "you", ",", "Yang", "Yang", "?"], ["A", "friend", "happened", "to", "call", "me", "."], ["You", "were", "not", "at", "work", "that", "day", "?"], ["No", "."], ["The", "station", "called", "me", "at", "noon", "and", "said", "something", "happened", "at", "Jingguang", "Bridge", "and", "that", "I", "had", "to", "go", "to", "the", "station", "immediately", "to", "research", "the", "upcoming", "program", "."], ["Uh-huh", ",", "that", "means", ",", "er", ",", "you", "found", "out", "the", "accident", "through", "an", "information", "source", "at", "the", "station", "."], ["Right", ",", "right", ",", "right", "."], ["Uh-huh", "."], ["Well", ",", "like", "Professor", "Zhou", ",", "I", "also", "received", "this", "news", ",", "ha", ",", "through", "a", "mobile", "phone", "SMS", "."], ["At", "that", "time", ",", ",", "it", "can", "be", "said", "that", "this", "SMS", "was", "among", "the", "many", ",", "ha", ",", "SMS", "containing", "New", "Year", "wishes", ",", "like", "Happy", "New", "Year", ",", "received", "after", "the", "start", "of", "the", "New", "Year", "."], ["Uh-huh", "."], ["Ah", ",", "actually", "I", "felt", "a", "lot", "of", "warmth", "when", "I", "received", "that", "SMS", "."], ["Although", "we", "live", "in", "the", "west", "instead", "of", "the", "east", "and", "it", "did", "not", "affect", "us", "much", ",", "I", "think", "it", "is", "very", "useful", ",", "ah", ",", "to", "inform", "people", "of", "this", "kind", "of", "news", "."], ["Yes", ",", "exceptionally", "."], ["Yes", ",", "exceptionally", "."]]} +{"doc_key": "bc/cctv/00/cctv_0005_0", "speakers": [["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"], ["speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1", "speaker#1"]], "clusters": [[[57, 59], [25, 27], [42, 44]]], "sentences": [["--", "basically", ",", "it", "was", "unanimously", "agreed", "upon", "by", "the", "various", "relevant", "parties", "."], ["To", "express", "its", "determination", ",", "the", "Chinese", "securities", "regulatory", "department", "compares", "this", "stock", "reform", "to", "a", "die", "that", "has", "been", "cast", "."]]} \ No newline at end of file diff --git a/test/data_for_tests/coreference/coreference_train.json b/test/data_for_tests/coreference/coreference_train.json index 6932bbb7..0c2940df 100644 --- a/test/data_for_tests/coreference/coreference_train.json +++ b/test/data_for_tests/coreference/coreference_train.json @@ -1,2 +1 @@ -{"doc_key": "bc/cctv/00/cctv_0001_0", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"]], "clusters": [[[113, 114], [42, 45], [88, 91]], [[288, 288], [293, 293]], [[185, 189], [162, 165], [101, 104]], [[232, 233], [209, 209], [253, 253]], [[36, 37], [31, 32]], [[55, 56], [79, 81]], [[283, 283], [269, 275]], [[39, 45], [47, 47]], [[285, 285], [298, 298], [235, 237], [258, 260], [117, 120], [267, 267]], [[75, 77], [51, 53]], [[310, 310], [289, 289], [295, 295]], [[135, 136], [273, 273], [26, 26]], [[200, 201], [182, 183], [179, 180]]], "sentences": [["What", "kind", "of", "memory", "?"], ["We", "respectfully", "invite", "you", "to", "watch", "a", "special", "edition", "of", "Across", "China", "."], ["WW", "II", "Landmarks", "on", "the", "Great", "Earth", "of", "China", ":", "Eternal", "Memories", "of", "Taihang", "Mountain"], ["Standing", "tall", "on", "Taihang", "Mountain", "is", "the", "Monument", "to", "the", "Hundred", "Regiments", "Offensive", "."], ["It", "is", "composed", "of", "a", "primary", "stele", ",", "secondary", "steles", ",", "a", "huge", "round", "sculpture", "and", "beacon", "tower", ",", "and", "the", "Great", "Wall", ",", "among", "other", "things", "."], ["A", "primary", "stele", ",", "three", "secondary", "steles", ",", "and", "two", "inscribed", "steles", "."], ["The", "Hundred", "Regiments", "Offensive", "was", "the", "campaign", "of", "the", "largest", "scale", "launched", "by", "the", "Eighth", "Route", "Army", "during", "the", "War", "of", "Resistance", "against", "Japan", "."], ["This", "campaign", "broke", "through", "the", "Japanese", "army", "'s", "blockade", "to", "reach", "base", "areas", "behind", "enemy", "lines", ",", "stirring", "up", "anti-Japanese", "spirit", "throughout", "the", "nation", "and", "influencing", "the", "situation", "of", "the", "anti-fascist", "war", "of", "the", "people", "worldwide", "."], ["This", "is", "Zhuanbi", "Village", ",", "Wuxiang", "County", "of", "Shanxi", "Province", ",", "where", "the", "Eighth", "Route", "Army", "was", "headquartered", "back", "then", "."], ["On", "a", "wall", "outside", "the", "headquarters", "we", "found", "a", "map", "."], ["This", "map", "was", "the", "Eighth", "Route", "Army", "'s", "depiction", "of", "the", "Mediterranean", "Sea", "situation", "at", "that", "time", "."], ["This", "map", "reflected", "the", "European", "battlefield", "situation", "."], ["In", "1940", ",", "the", "German", "army", "invaded", "and", "occupied", "Czechoslovakia", ",", "Poland", ",", "the", "Netherlands", ",", "Belgium", ",", "and", "France", "."], ["It", "was", "during", "this", "year", "that", "the", "Japanese", "army", "developed", "a", "strategy", "to", "rapidly", "force", "the", "Chinese", "people", "into", "submission", "by", "the", "end", "of", "1940", "."], ["In", "May", ",", "the", "Japanese", "army", "launched", "--"], ["From", "one", "side", ",", "it", "seized", "an", "important", "city", "in", "China", "called", "Yichang", "."], ["Um", ",", ",", "uh", ",", "through", "Yichang", ",", "it", "could", "directly", "reach", "Chongqing", "."], ["Ah", ",", "that", "threatened", "Chongqing", "."], ["Then", "they", "would", ",", "ah", ",", "bomb", "these", "large", "rear", "areas", "such", "as", "Chongqing", "."], ["So", ",", "along", "with", "the", "coordinated", ",", "er", ",", "economic", "blockade", ",", "military", "offensives", ",", "and", "strategic", "bombings", ",", "er", ",", "a", "simultaneous", "attack", "was", "launched", "in", "Hong", "Kong", "to", "lure", "the", "KMT", "government", "into", "surrender", "."], ["The", "progress", "of", "this", "coordinated", "offensive", "was", "already", "very", "entrenched", "by", "then", "."]]} -{"doc_key": "bc/cctv/00/cctv_0001_1", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang", "Luo_huanzhang"], ["Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1", "Luo_huanzhang,Speaker#1"]], "clusters": [[[129, 131], [167, 169]], [[495, 496], [446, 447], [183, 186]], [[433, 438], [314, 316], [318, 318]], [[154, 157], [531, 534], [436, 438], [139, 142], [43, 45]], [[560, 561], [547, 554], [279, 288]], [[309, 309], [374, 374], [21, 23], [9, 9], [312, 312], [385, 385]], [[212, 213], [193, 197]], [[577, 578], [581, 582]], [[262, 267], [591, 592], [523, 524], [565, 568], [424, 431]], [[255, 256], [28, 32]], [[492, 493], [175, 181], [443, 444]], [[124, 127], [449, 451], [250, 253], [29, 31], [188, 191], [407, 416], [71, 74], [510, 513], [129, 129]], [[63, 67], [139, 146], [76, 78]], [[443, 452], [175, 191]], [[485, 487], [596, 598]], [[517, 524], [556, 556], [526, 526]], [[81, 98], [133, 134]], [[47, 48], [109, 112]], [[348, 353], [365, 365], [388, 390]], [[1, 1], [477, 477], [267, 267]], [[550, 551], [288, 288], [3, 4], [18, 18]]], "sentences": [["By", "1940", ",", "China", "'s", "War", "of", "Resistance", "against", "Japan", "had", "entered", "a", "stalemate", "."], ["The", "situation", "on", "our", "side", "and", "the", "enemy", "'s", "side", "was", "intertwined", "."], ["The", "Eighth", "Route", "Army", "guerrillas", "were", "extraordinarily", "active", ",", "creating", "more", "and", "more", "trouble", "for", "the", "Japanese", "army", "in", "North", "China", "."], ["Hayao", "Tada", ",", "commander", "of", "the", "Japanese", "North", "China", "Area", "Army", ",", "adopted", "a", "strategy", "of", "siege", "warfare", "to", "deal", "with", "the", "Eighth", "Route", "Army", "."], ["The", "specific", "method", "was", "building", "a", "closely", "connected", "transport", "network", ",", "with", "a", "road", "for", "every", "village", "and", "defensive", "towers", "on", "every", "road", "."], ["Roads", "and", "railways", "were", "used", "as", "links", "to", "connect", "all", "of", "North", "China", "into", "a", "solid", ",", "widespread", "siege", ",", "in", "order", "to", "strangle", "the", "Eighth", "Route", "Army", "and", "its", "base", "areas", "in", "this", "net", "."], ["As", "part", "of", "the", "Japanese", "army", "'s", "strategy", "of", "siege", "warfare", ",", "railways", "and", "roads", "had", "actually", "become", "the", "Japanese", "army", "'s", "weapons", "of", "war", ",", "becoming", "a", "great", "threat", "to", "the", "base", "areas", "."], ["In", "December", "1939", ",", "Commander", "-", "in", "-", "chief", "Zhu", "De", "and", "Vice", "Commander", "Peng", "Dehuai", "of", "the", "Eighth", "Route", "Army", "received", "a", "top", "-", "secret", "telegram", "from", "Commander", "Lu", "Zhengcao", "of", "the", "Jizhong", "Military", "District", ",", "among", "other", "people", "."], ["The", "telegram", "said", "that", "the", "Japanese", "troops", "were", "building", "blockade", "trenches", "and", "chessboard", "-", "like", "roads", "to", "divide", "the", "Jizhong", "base", "area", "into", "small", "isolated", "blocks", "without", "the", "ability", "to", "mutually", "communicate", "and", "support", "each", "other", ",", "causing", "the", "Eighth", "Route", "Army", "and", "the", "guerrillas", "to", "lose", "maneuverability", "."], ["Before", "the", "Hundred", "Regiments", "Offensive", "in", "1940", ",", "an", "inclination", "to", "compromise", ",", "ah", ",", "surrender", ",", "was", "an", "extremely", "serious", "crisis", "in", "the", "frontline", "situation", "in", "China", "."], ["Well", ",", "on", "the", "battlefield", "behind", "enemy", "lines", ",", "in", "order", "to", "take", "over", ",", "consolidate", "the", "area", "under", "its", "occupation", ",", "Japan", "began", "a", "new", "strategy", "."], ["That", "was", "to", "use", "railways", "as", "a", "pillar", ",", "roads", "as", "a", "chain", ",", "and", "strongholds", "as", "a", "lock", ",", "to", "carry", "out", "siege", "warfare", "in", "an", "attempt", "to", "divide", "the", "base", "areas", "behind", "enemy", "lines", ",", "ah", ",", "so", "as", ",", "er", ",", "to", "cut", "off", "their", "communication", "with", "one", "another", "."], ["In", "addition", ",", "it", "relied", "on", "this", "cage", ",", "ah", ",", "to", "further", "strengthen", "its", "assaults", "against", "the", "base", "areas", "."], ["Er", "."], ["So", ",", "it", "was", "amidst", "such", "a", "grave", "international", "and", "domestic", "situation", "that", "the", "Eighth", "Route", "Army", "led", "by", "the", "Chinese", "Communist", "Party", ",", "ah", ",", "launched", ",", "ah", ",", "a", "strategic", "offensive", "called", "the", "Hundred", "Regiments", "Offensive", "."], ["This", "plot", "of", "the", "Japanese", "army", "drew", "great", "attention", "from", "Zhu", "De", "and", "Peng", "Dehuai", "of", "Eighth", "Route", "Army", "headquarters", "."], ["After", "meticulous", "studies", "and", "painstaking", "preparations", "by", "many", "parties", ",", "a", "battle", "plan", "based", "on", "surprise", "was", "formulated", "."], ["On", "July", "22", ",", "1940", ",", "a", "campaign", "preparation", "order", "to", "attack", "the", "Zhengtai", "Railway", ",", "jointly", "signed", "by", "Zhu", "De", ",", "Peng", "Dehuai", ",", "and", "Zuo", "Quan", ",", "was", "sent", "to", "Yan'an", "and", "all", "units", "of", "the", "Eighth", "Route", "Army", "."], ["What", "was", "the", ",", "purpose", "and", "goal", "of", "this", "campaign", "?"], ["It", "was", "to", "break", "through", "the", "Japanese", "army", "'s", "siege", "policy", "against", "base", "areas", "behind", "enemy", "lines", ",", "and", "to", "avert", "the", "crisis", "of", "China", "'s", "compromise", "and", "surrender", "."], ["It", "was", "to", "overcome", "this", "crisis", "."], ["Well", ",", "the", "Hundred", "Regiments", "Offensive", "was", "divided", "into", "three", "phases", "."], ["Beginning", "from", "August", "20", ",", "from", "August", "20", "to", "September", "10", ",", "the", "main", "purpose", "of", "the", "campaign", "was", "to", "sabotage", "the", "Zhengtai", "Railway", "."]]} +{"doc_key": "bc/cctv/00/cctv_0001_0", "speakers": [["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"], ["Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1", "Speaker#1"]], "clusters": [[[113, 114], [42, 45], [88, 91]]], "sentences": [["What", "kind", "of", "memory", "?"], ["We", "respectfully", "invite", "you", "to", "watch", "a", "special", "edition", "of", "Across", "China", "."]]} From 45fbbac79ee76410b601b6216014d2c8f8e65f2e Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 19:11:18 +0800 Subject: [PATCH 63/92] split the docs in embeddings --- fastNLP/embeddings/bert_embedding.py | 71 ++++++++++++++------------ fastNLP/embeddings/char_embedding.py | 67 +++++++++++++----------- fastNLP/embeddings/elmo_embedding.py | 37 ++++++++------ fastNLP/embeddings/embedding.py | 17 +++--- fastNLP/embeddings/stack_embedding.py | 12 +++-- fastNLP/embeddings/static_embedding.py | 33 ++++++------ 6 files changed, 130 insertions(+), 107 deletions(-) diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index aa998801..84105444 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -8,20 +8,19 @@ __all__ = [ "BertWordPieceEncoder" ] -import os import collections +import warnings +from itertools import chain -from torch import nn -import torch import numpy as np -from itertools import chain +import torch +from torch import nn +from .contextual_embedding import ContextualEmbedding +from ..core import logger from ..core.vocabulary import Vocabulary from ..io.file_utils import PRETRAINED_BERT_MODEL_DIR from ..modules.encoder.bert import _WordPieceBertModel, BertModel, BertTokenizer -from .contextual_embedding import ContextualEmbedding -import warnings -from ..core import logger class BertEmbedding(ContextualEmbedding): @@ -43,30 +42,32 @@ class BertEmbedding(ContextualEmbedding): >>> outputs = embed(words) >>> outputs.size() >>> # torch.Size([1, 5, 2304]) - - :param ~fastNLP.Vocabulary vocab: 词表 - :param str model_dir_or_name: 模型所在目录或者模型的名称。当传入模型所在目录时，目录中应该包含一个词表文件(以.txt作为后缀名), - 权重文件(以.bin作为文件后缀名), 配置文件(以.json作为后缀名)。 - :param str layers: 输出embedding表示来自于哪些层，不同层的结果按照layers中的顺序在最后一维concat起来。以','隔开层数，层的序号是 - 从0开始，可以以负数去索引倒数几层。 - :param str pool_method: 因为在bert中，每个word会被表示为多个word pieces, 当获取一个word的表示的时候，怎样从它的word pieces - 中计算得到它对应的表示。支持 ``last`` , ``first`` , ``avg`` , ``max``。 - :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 - :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 - :param bool include_cls_sep: bool，在bert计算句子的表示的时候，需要在前面加上[CLS]和[SEP], 是否在结果中保留这两个内容。 这样 - 会使得word embedding的结果比输入的结果长两个token。如果该值为True，则在使用 :class::StackEmbedding 可能会与其它类型的 - embedding长度不匹配。 - :param bool pooled_cls: 返回的[CLS]是否使用预训练中的BertPool映射一下，仅在include_cls_sep时有效。如果下游任务只取[CLS]做预测， - 一般该值为True。 - :param bool requires_grad: 是否需要gradient以更新Bert的权重。 - :param bool auto_truncate: 当句子words拆分为word pieces长度超过bert最大允许长度(一般为512), 自动截掉拆分后的超过510个 - word pieces后的内容，并将第512个word piece置为[SEP]。超过长度的部分的encode结果直接全部置零。一般仅有只使用[CLS] - 来进行分类的任务将auto_truncate置为True。 """ def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en-base-uncased', layers: str = '-1', pool_method: str = 'first', word_dropout=0, dropout=0, include_cls_sep: bool = False, pooled_cls=True, requires_grad: bool = True, auto_truncate: bool = False): + """ + + :param ~fastNLP.Vocabulary vocab: 词表 + :param str model_dir_or_name: 模型所在目录或者模型的名称。当传入模型所在目录时，目录中应该包含一个词表文件(以.txt作为后缀名), + 权重文件(以.bin作为文件后缀名), 配置文件(以.json作为后缀名)。 + :param str layers: 输出embedding表示来自于哪些层，不同层的结果按照layers中的顺序在最后一维concat起来。以','隔开层数，层的序号是 + 从0开始，可以以负数去索引倒数几层。 + :param str pool_method: 因为在bert中，每个word会被表示为多个word pieces, 当获取一个word的表示的时候，怎样从它的word pieces + 中计算得到它对应的表示。支持 ``last`` , ``first`` , ``avg`` , ``max``。 + :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 + :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 + :param bool include_cls_sep: bool，在bert计算句子的表示的时候，需要在前面加上[CLS]和[SEP], 是否在结果中保留这两个内容。 这样 + 会使得word embedding的结果比输入的结果长两个token。如果该值为True，则在使用 :class::StackEmbedding 可能会与其它类型的 + embedding长度不匹配。 + :param bool pooled_cls: 返回的[CLS]是否使用预训练中的BertPool映射一下，仅在include_cls_sep时有效。如果下游任务只取[CLS]做预测， + 一般该值为True。 + :param bool requires_grad: 是否需要gradient以更新Bert的权重。 + :param bool auto_truncate: 当句子words拆分为word pieces长度超过bert最大允许长度(一般为512), 自动截掉拆分后的超过510个 + word pieces后的内容，并将第512个word piece置为[SEP]。超过长度的部分的encode结果直接全部置零。一般仅有只使用[CLS] + 来进行分类的任务将auto_truncate置为True。 + """ super(BertEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) if model_dir_or_name.lower() in PRETRAINED_BERT_MODEL_DIR: @@ -131,18 +132,20 @@ class BertEmbedding(ContextualEmbedding): class BertWordPieceEncoder(nn.Module): """ 读取bert模型，读取之后调用index_dataset方法在dataset中生成word_pieces这一列。 - - :param str model_dir_or_name: 模型所在目录或者模型的名称。默认值为 ``en-base-uncased`` - :param str layers: 最终结果中的表示。以','隔开层数，可以以负数去索引倒数几层 - :param bool pooled_cls: 返回的句子开头的[CLS]是否使用预训练中的BertPool映射一下，仅在include_cls_sep时有效。如果下游任务只取 - [CLS]做预测，一般该值为True。 - :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 - :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 - :param bool requires_grad: 是否需要gradient。 """ def __init__(self, model_dir_or_name: str = 'en-base-uncased', layers: str = '-1', pooled_cls: bool = False, word_dropout=0, dropout=0, requires_grad: bool = True): + """ + + :param str model_dir_or_name: 模型所在目录或者模型的名称。默认值为 ``en-base-uncased`` + :param str layers: 最终结果中的表示。以','隔开层数，可以以负数去索引倒数几层 + :param bool pooled_cls: 返回的句子开头的[CLS]是否使用预训练中的BertPool映射一下，仅在include_cls_sep时有效。如果下游任务只取 + [CLS]做预测，一般该值为True。 + :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 + :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 + :param bool requires_grad: 是否需要gradient。 + """ super().__init__() self.model = _WordPieceBertModel(model_dir_or_name=model_dir_or_name, layers=layers, pooled_cls=pooled_cls) diff --git a/fastNLP/embeddings/char_embedding.py b/fastNLP/embeddings/char_embedding.py index a0328525..72a33e97 100644 --- a/fastNLP/embeddings/char_embedding.py +++ b/fastNLP/embeddings/char_embedding.py @@ -8,18 +8,19 @@ __all__ = [ "LSTMCharEmbedding" ] +from typing import List + import torch import torch.nn as nn import torch.nn.functional as F -from typing import List -from .static_embedding import StaticEmbedding -from ..modules.encoder.lstm import LSTM -from ..core.vocabulary import Vocabulary from .embedding import TokenEmbedding +from .static_embedding import StaticEmbedding from .utils import _construct_char_vocab_from_vocab from .utils import get_embeddings from ..core import logger +from ..core.vocabulary import Vocabulary +from ..modules.encoder.lstm import LSTM class CNNCharEmbedding(TokenEmbedding): @@ -39,24 +40,27 @@ class CNNCharEmbedding(TokenEmbedding): >>> outputs.size() >>> # torch.Size([1, 5，50]) - :param vocab: 词表 - :param embed_size: 该CNNCharEmbedding的输出维度大小，默认值为50. - :param char_emb_size: character的embed的维度。character是从vocab中生成的。默认值为50. - :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 - :param float dropout: 以多大的概率drop分布式表示与char embedding的输出。 - :param filter_nums: filter的数量. 长度需要和kernels一致。默认值为[40, 30, 20]. - :param kernel_sizes: kernel的大小. 默认值为[5, 3, 1]. - :param pool_method: character的表示在合成一个表示时所使用的pool方法，支持'avg', 'max'. - :param activation: CNN之后使用的激活方法，支持'relu', 'sigmoid', 'tanh' 或者自定义函数. - :param min_char_freq: character的最少出现次数。默认值为2. - :param pre_train_char_embed: 可以有两种方式调用预训练好的character embedding：第一种是传入embedding文件夹 - (文件夹下应该只有一个以.txt作为后缀的文件)或文件路径；第二种是传入embedding的名称，第二种情况将自动查看缓存中是否存在该模型， - 没有的话将自动下载。如果输入为None则使用embedding_dim的维度随机初始化一个embedding. """ def __init__(self, vocab: Vocabulary, embed_size: int = 50, char_emb_size: int = 50, word_dropout: float = 0, dropout: float = 0, filter_nums: List[int] = (40, 30, 20), kernel_sizes: List[int] = (5, 3, 1), pool_method: str = 'max', activation='relu', min_char_freq: int = 2, pre_train_char_embed: str = None): + """ + + :param vocab: 词表 + :param embed_size: 该CNNCharEmbedding的输出维度大小，默认值为50. + :param char_emb_size: character的embed的维度。character是从vocab中生成的。默认值为50. + :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 + :param float dropout: 以多大的概率drop分布式表示与char embedding的输出。 + :param filter_nums: filter的数量. 长度需要和kernels一致。默认值为[40, 30, 20]. + :param kernel_sizes: kernel的大小. 默认值为[5, 3, 1]. + :param pool_method: character的表示在合成一个表示时所使用的pool方法，支持'avg', 'max'. + :param activation: CNN之后使用的激活方法，支持'relu', 'sigmoid', 'tanh' 或者自定义函数. + :param min_char_freq: character的最少出现次数。默认值为2. + :param pre_train_char_embed: 可以有两种方式调用预训练好的character embedding：第一种是传入embedding文件夹 + (文件夹下应该只有一个以.txt作为后缀的文件)或文件路径；第二种是传入embedding的名称，第二种情况将自动查看缓存中是否存在该模型， + 没有的话将自动下载。如果输入为None则使用embedding_dim的维度随机初始化一个embedding. + """ super(CNNCharEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) for kernel in kernel_sizes: @@ -156,25 +160,28 @@ class LSTMCharEmbedding(TokenEmbedding): >>> outputs.size() >>> # torch.Size([1, 5，50]) - :param vocab: 词表 - :param embed_size: LSTMCharEmbedding的输出维度。默认值为50. - :param char_emb_size: character的embedding的维度。默认值为50. - :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 - :param dropout: 以多大概率drop character embedding的输出以及最终的word的输出。 - :param hidden_size: LSTM的中间hidden的大小，如果为bidirectional的，hidden会除二，默认为50. - :param pool_method: 支持'max', 'avg'。 - :param activation: 激活函数，支持'relu', 'sigmoid', 'tanh', 或者自定义函数. - :param min_char_freq: character的最小出现次数。默认值为2. - :param bidirectional: 是否使用双向的LSTM进行encode。默认值为True。 - :param pre_train_char_embed: 可以有两种方式调用预训练好的character embedding：第一种是传入embedding文件夹 - (文件夹下应该只有一个以.txt作为后缀的文件)或文件路径；第二种是传入embedding的名称，第二种情况将自动查看缓存中是否存在该模型， - 没有的话将自动下载。如果输入为None则使用embedding_dim的维度随机初始化一个embedding. """ def __init__(self, vocab: Vocabulary, embed_size: int = 50, char_emb_size: int = 50, word_dropout: float = 0, dropout: float = 0, hidden_size=50, pool_method: str = 'max', activation='relu', min_char_freq: int = 2, bidirectional=True, pre_train_char_embed: str = None): + """ + + :param vocab: 词表 + :param embed_size: LSTMCharEmbedding的输出维度。默认值为50. + :param char_emb_size: character的embedding的维度。默认值为50. + :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 + :param dropout: 以多大概率drop character embedding的输出以及最终的word的输出。 + :param hidden_size: LSTM的中间hidden的大小，如果为bidirectional的，hidden会除二，默认为50. + :param pool_method: 支持'max', 'avg'。 + :param activation: 激活函数，支持'relu', 'sigmoid', 'tanh', 或者自定义函数. + :param min_char_freq: character的最小出现次数。默认值为2. + :param bidirectional: 是否使用双向的LSTM进行encode。默认值为True。 + :param pre_train_char_embed: 可以有两种方式调用预训练好的character embedding：第一种是传入embedding文件夹 + (文件夹下应该只有一个以.txt作为后缀的文件)或文件路径；第二种是传入embedding的名称，第二种情况将自动查看缓存中是否存在该模型， + 没有的话将自动下载。如果输入为None则使用embedding_dim的维度随机初始化一个embedding. + """ super(LSTMCharEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) assert hidden_size % 2 == 0, "Only even kernel is allowed." diff --git a/fastNLP/embeddings/elmo_embedding.py b/fastNLP/embeddings/elmo_embedding.py index 57842c33..f2d643f7 100644 --- a/fastNLP/embeddings/elmo_embedding.py +++ b/fastNLP/embeddings/elmo_embedding.py @@ -7,18 +7,20 @@ __all__ = [ "ElmoEmbedding" ] +import codecs +import json import os + import torch import torch.nn as nn import torch.nn.functional as F -import json -import codecs +from .contextual_embedding import ContextualEmbedding +from ..core import logger from ..core.vocabulary import Vocabulary from ..io.file_utils import cached_path, _get_embedding_url, PRETRAINED_ELMO_MODEL_DIR from ..modules.encoder._elmo import ElmobiLm, ConvTokenEmbedder -from .contextual_embedding import ContextualEmbedding -from ..core import logger + class ElmoEmbedding(ContextualEmbedding): """ @@ -41,22 +43,25 @@ class ElmoEmbedding(ContextualEmbedding): >>> embed = ElmoEmbedding(vocab, model_dir_or_name='en', layers='mix', requires_grad=False) >>> embed.set_mix_weights_requires_grad() # 使得weighted的权重是可以学习的，但ELMO的LSTM部分是不更新 - :param vocab: 词表 - :param model_dir_or_name: 可以有两种方式调用预训练好的ELMo embedding：第一种是传入ELMo所在文件夹，该文件夹下面应该有两个文件， - 其中一个是以json为后缀的配置文件，另一个是以pkl为后缀的权重文件；第二种是传入ELMo版本的名称，将自动查看缓存中是否存在该模型， - 没有的话将自动下载并缓存。 - :param layers: str, 指定返回的层数(从0开始), 以,隔开不同的层。如果要返回第二层的结果'2', 返回后两层的结果'1,2'。不同的层的结果 - 按照这个顺序concat起来，默认为'2'。'mix'会使用可学习的权重结合不同层的表示(权重是否可训练与requires_grad保持一致， - 初始化权重对三层结果进行mean-pooling, 可以通过ElmoEmbedding.set_mix_weights_requires_grad()方法只将mix weights设置为可学习。) - :param requires_grad: bool, 该层是否需要gradient, 默认为False. - :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 - :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 - :param cache_word_reprs: 可以选择对word的表示进行cache; 设置为True的话，将在初始化的时候为每个word生成对应的embedding， - 并删除character encoder，之后将直接使用cache的embedding。默认为False。 """ def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en', layers: str = '2', requires_grad: bool = True, word_dropout=0.0, dropout=0.0, cache_word_reprs: bool = False): + """ + + :param vocab: 词表 + :param model_dir_or_name: 可以有两种方式调用预训练好的ELMo embedding：第一种是传入ELMo所在文件夹，该文件夹下面应该有两个文件， + 其中一个是以json为后缀的配置文件，另一个是以pkl为后缀的权重文件；第二种是传入ELMo版本的名称，将自动查看缓存中是否存在该模型， + 没有的话将自动下载并缓存。 + :param layers: str, 指定返回的层数(从0开始), 以,隔开不同的层。如果要返回第二层的结果'2', 返回后两层的结果'1,2'。不同的层的结果 + 按照这个顺序concat起来，默认为'2'。'mix'会使用可学习的权重结合不同层的表示(权重是否可训练与requires_grad保持一致， + 初始化权重对三层结果进行mean-pooling, 可以通过ElmoEmbedding.set_mix_weights_requires_grad()方法只将mix weights设置为可学习。) + :param requires_grad: bool, 该层是否需要gradient, 默认为False. + :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 + :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 + :param cache_word_reprs: 可以选择对word的表示进行cache; 设置为True的话，将在初始化的时候为每个word生成对应的embedding， + 并删除character encoder，之后将直接使用cache的embedding。默认为False。 + """ super(ElmoEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) # 根据model_dir_or_name检查是否存在并下载 diff --git a/fastNLP/embeddings/embedding.py b/fastNLP/embeddings/embedding.py index e82ef0b4..08921f33 100644 --- a/fastNLP/embeddings/embedding.py +++ b/fastNLP/embeddings/embedding.py @@ -8,9 +8,10 @@ __all__ = [ "TokenEmbedding" ] -import torch.nn as nn from abc import abstractmethod + import torch +import torch.nn as nn from .utils import get_embeddings @@ -28,16 +29,18 @@ class Embedding(nn.Module): >>> init_embed = np.zeros((2000, 100)) >>> embed = Embedding(init_embed) # 使用numpy.ndarray的值作为初始化值初始化一个Embedding - :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray init_embed: 支持传入Embedding的大小(传入tuple(int, int), - 第一个int为vocab_zie, 第二个int为embed_dim); 或传入Tensor, Embedding, numpy.ndarray等则直接使用该值初始化Embedding; - :param float word_dropout: 按照一定概率随机将word设置为unk_index，这样可以使得unk这个token得到足够的训练, 且会对网络有 - 一定的regularize的作用。设置该值时，必须同时设置unk_index - :param float dropout: 对Embedding的输出的dropout。 - :param int unk_index: drop word时替换为的index。fastNLP的Vocabulary的unk_index默认为1。 """ def __init__(self, init_embed, word_dropout=0, dropout=0.0, unk_index=None): + """ + :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray init_embed: 支持传入Embedding的大小(传入tuple(int, int), + 第一个int为vocab_zie, 第二个int为embed_dim); 或传入Tensor, Embedding, numpy.ndarray等则直接使用该值初始化Embedding; + :param float word_dropout: 按照一定概率随机将word设置为unk_index，这样可以使得unk这个token得到足够的训练, 且会对网络有 + 一定的regularize的作用。设置该值时，必须同时设置unk_index + :param float dropout: 对Embedding的输出的dropout。 + :param int unk_index: drop word时替换为的index。fastNLP的Vocabulary的unk_index默认为1。 + """ super(Embedding, self).__init__() self.embed = get_embeddings(init_embed) diff --git a/fastNLP/embeddings/stack_embedding.py b/fastNLP/embeddings/stack_embedding.py index 91702ec2..21a06b5f 100644 --- a/fastNLP/embeddings/stack_embedding.py +++ b/fastNLP/embeddings/stack_embedding.py @@ -28,14 +28,16 @@ class StackEmbedding(TokenEmbedding): >>> embed_2 = StaticEmbedding(vocab, model_dir_or_name='en-word2vec-300', requires_grad=True) >>> embed = StackEmbedding([embed_1, embed_2]) - :param embeds: 一个由若干个TokenEmbedding组成的list，要求每一个TokenEmbedding的词表都保持一致 - :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。不同embedidng会在相同的位置 - 被设置为unknown。如果这里设置了dropout，则组成的embedding就不要再设置dropout了。 - :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 - """ def __init__(self, embeds: List[TokenEmbedding], word_dropout=0, dropout=0): + """ + + :param embeds: 一个由若干个TokenEmbedding组成的list，要求每一个TokenEmbedding的词表都保持一致 + :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。不同embedidng会在相同的位置 + 被设置为unknown。如果这里设置了dropout，则组成的embedding就不要再设置dropout了。 + :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 + """ vocabs = [] for embed in embeds: if hasattr(embed, 'get_word_vocab'): diff --git a/fastNLP/embeddings/static_embedding.py b/fastNLP/embeddings/static_embedding.py index 3d2471e6..f519e705 100644 --- a/fastNLP/embeddings/static_embedding.py +++ b/fastNLP/embeddings/static_embedding.py @@ -48,25 +48,28 @@ class StaticEmbedding(TokenEmbedding): [ 0.5773, 0.7251, -0.3104, 0.0777, 0.4849]]], grad_fn=) # 每种word的输出是一致的。 - :param vocab: Vocabulary. 若该项为None则会读取所有的embedding。 - :param model_dir_or_name: 可以有两种方式调用预训练好的static embedding：第一种是传入embedding文件夹(文件夹下应该只有一个 - 以.txt作为后缀的文件)或文件路径；第二种是传入embedding的名称，第二种情况将自动查看缓存中是否存在该模型，没有的话将自动下载。 - 如果输入为None则使用embedding_dim的维度随机初始化一个embedding。 - :param int embedding_dim: 随机初始化的embedding的维度，当该值为大于0的值时，将忽略model_dir_or_name。 - :param bool requires_grad: 是否需要gradient. 默认为True - :param callable init_method: 如何初始化没有找到的值。可以使用torch.nn.init.*中各种方法, 传入的方法应该接受一个tensor，并 - inplace地修改其值。 - :param bool lower: 是否将vocab中的词语小写后再和预训练的词表进行匹配。如果你的词表中包含大写的词语，或者就是需要单独 - 为大写的词语开辟一个vector表示，则将lower设置为False。 - :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 - :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 - :param bool normalize: 是否对vector进行normalize，使得每个vector的norm为1。 - :param int min_freq: Vocabulary词频数小于这个数量的word将被指向unk。 - :param dict kwarngs: only_train_min_freq, 仅对train中的词语使用min_freq筛选; only_norm_found_vector是否仅对在预训练中找到的词语使用normalize。 """ def __init__(self, vocab: Vocabulary, model_dir_or_name: str = 'en', embedding_dim=-1, requires_grad: bool = True, init_method=None, lower=False, dropout=0, word_dropout=0, normalize=False, min_freq=1, **kwargs): + """ + + :param vocab: Vocabulary. 若该项为None则会读取所有的embedding。 + :param model_dir_or_name: 可以有两种方式调用预训练好的static embedding：第一种是传入embedding文件夹(文件夹下应该只有一个 + 以.txt作为后缀的文件)或文件路径；第二种是传入embedding的名称，第二种情况将自动查看缓存中是否存在该模型，没有的话将自动下载。 + 如果输入为None则使用embedding_dim的维度随机初始化一个embedding。 + :param int embedding_dim: 随机初始化的embedding的维度，当该值为大于0的值时，将忽略model_dir_or_name。 + :param bool requires_grad: 是否需要gradient. 默认为True + :param callable init_method: 如何初始化没有找到的值。可以使用torch.nn.init.*中各种方法, 传入的方法应该接受一个tensor，并 + inplace地修改其值。 + :param bool lower: 是否将vocab中的词语小写后再和预训练的词表进行匹配。如果你的词表中包含大写的词语，或者就是需要单独 + 为大写的词语开辟一个vector表示，则将lower设置为False。 + :param float dropout: 以多大的概率对embedding的表示进行Dropout。0.1即随机将10%的值置为0。 + :param float word_dropout: 以多大的概率将一个词替换为unk。这样既可以训练unk也是一定的regularize。 + :param bool normalize: 是否对vector进行normalize，使得每个vector的norm为1。 + :param int min_freq: Vocabulary词频数小于这个数量的word将被指向unk。 + :param dict kwarngs: only_train_min_freq, 仅对train中的词语使用min_freq筛选; only_norm_found_vector是否仅对在预训练中找到的词语使用normalize。 + """ super(StaticEmbedding, self).__init__(vocab, word_dropout=word_dropout, dropout=dropout) if embedding_dim > 0: model_dir_or_name = None From 4a9cd850b2bbe319cbd0998cda020f2d18cfd4c1 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 22:01:39 +0800 Subject: [PATCH 64/92] split the docs in io subpackage --- fastNLP/io/data_bundle.py | 11 ++++-- fastNLP/io/loader/conll.py | 10 +++-- fastNLP/io/loader/csv.py | 13 ++++--- fastNLP/io/loader/cws.py | 7 +++- fastNLP/io/loader/json.py | 15 ++++--- fastNLP/io/loader/matching.py | 2 +- fastNLP/io/pipe/classification.py | 65 ++++++++++++++++++++----------- fastNLP/io/pipe/conll.py | 47 +++++++++++----------- fastNLP/io/pipe/coreference.py | 13 ++++--- fastNLP/io/pipe/cws.py | 17 ++++---- fastNLP/io/pipe/matching.py | 15 ++++--- fastNLP/io/pipe/pipe.py | 1 + 12 files changed, 132 insertions(+), 84 deletions(-) diff --git a/fastNLP/io/data_bundle.py b/fastNLP/io/data_bundle.py index 19b48828..1f05cf68 100644 --- a/fastNLP/io/data_bundle.py +++ b/fastNLP/io/data_bundle.py @@ -6,9 +6,11 @@ __all__ = [ 'DataBundle', ] +from typing import Union + from ..core.dataset import DataSet from ..core.vocabulary import Vocabulary -from typing import Union + class DataBundle: """ @@ -22,11 +24,14 @@ class DataBundle: train_data = data_bundle.datasets['train'] dev_data = data_bundle.datasets['train'] - :param vocabs: 从名称(字符串)到 :class:`~fastNLP.Vocabulary` 类型的dict - :param datasets: 从名称(字符串)到 :class:`~fastNLP.DataSet` 类型的dict """ def __init__(self, vocabs: dict = None, datasets: dict = None): + """ + + :param vocabs: 从名称(字符串)到 :class:`~fastNLP.Vocabulary` 类型的dict + :param datasets: 从名称(字符串)到 :class:`~fastNLP.DataSet` 类型的dict + """ self.vocabs = vocabs or {} self.datasets = datasets or {} diff --git a/fastNLP/io/loader/conll.py b/fastNLP/io/loader/conll.py index f30b031f..0526628d 100644 --- a/fastNLP/io/loader/conll.py +++ b/fastNLP/io/loader/conll.py @@ -53,13 +53,15 @@ class ConllLoader(Loader): 数据中以"-DOCSTART-"开头的行将被忽略，因为该符号在conll 2003中被用为文档分割符。 - :param list headers: 每一列数据的名称，需为List or Tuple of str。``header`` 与 ``indexes`` 一一对应 - :param list indexes: 需要保留的数据列下标，从0开始。若为 ``None`` ，则所有列都保留。Default: ``None`` - :param bool dropna: 是否忽略非法数据，若 ``False`` ，遇到非法数据时抛出 ``ValueError`` 。Default: ``True`` - """ def __init__(self, headers, indexes=None, dropna=True): + """ + + :param list headers: 每一列数据的名称，需为List or Tuple of str。``header`` 与 ``indexes`` 一一对应 + :param list indexes: 需要保留的数据列下标，从0开始。若为 ``None`` ，则所有列都保留。Default: ``None`` + :param bool dropna: 是否忽略非法数据，若 ``False`` ，遇到非法数据时抛出 ``ValueError`` 。Default: ``True`` + """ super(ConllLoader, self).__init__() if not isinstance(headers, (list, tuple)): raise TypeError( diff --git a/fastNLP/io/loader/csv.py b/fastNLP/io/loader/csv.py index aaf38c00..6f35efbe 100644 --- a/fastNLP/io/loader/csv.py +++ b/fastNLP/io/loader/csv.py @@ -14,14 +14,17 @@ class CSVLoader(Loader): """ 读取CSV格式的数据集, 返回 ``DataSet`` 。 - :param List[str] headers: CSV文件的文件头.定义每一列的属性名称,即返回的DataSet中`field`的名称 - 若为 ``None`` ,则将读入文件的第一行视作 ``headers`` . Default: ``None`` - :param str sep: CSV文件中列与列之间的分隔符. Default: "," - :param bool dropna: 是否忽略非法数据,若 ``True`` 则忽略,若 ``False`` ,在遇到非法数据时,抛出 ``ValueError`` . - Default: ``False`` """ def __init__(self, headers=None, sep=",", dropna=False): + """ + + :param List[str] headers: CSV文件的文件头.定义每一列的属性名称,即返回的DataSet中`field`的名称 + 若为 ``None`` ,则将读入文件的第一行视作 ``headers`` . Default: ``None`` + :param str sep: CSV文件中列与列之间的分隔符. Default: "," + :param bool dropna: 是否忽略非法数据,若 ``True`` 则忽略,若 ``False`` ,在遇到非法数据时,抛出 ``ValueError`` . + Default: ``False`` + """ super().__init__() self.headers = headers self.sep = sep diff --git a/fastNLP/io/loader/cws.py b/fastNLP/io/loader/cws.py index 2fbb1091..887bb545 100644 --- a/fastNLP/io/loader/cws.py +++ b/fastNLP/io/loader/cws.py @@ -33,10 +33,13 @@ class CWSLoader(Loader): "上海 浦东 开发 与 法制 建设 同步" "新华社 上海 二月 十日 电 （ 记者 谢金虎 、 张持坚 ）" "..." - - :param: str dataset_name: data的名称，支持pku, msra, cityu(繁体), as(繁体), None + """ def __init__(self, dataset_name:str=None): + """ + + :param str dataset_name: data的名称，支持pku, msra, cityu(繁体), as(繁体), None + """ super().__init__() datanames = {'pku': 'cws-pku', 'msra':'cws-msra', 'as':'cws-as', 'cityu':'cws-cityu'} if dataset_name in datanames: diff --git a/fastNLP/io/loader/json.py b/fastNLP/io/loader/json.py index 671769fe..6e988baa 100644 --- a/fastNLP/io/loader/json.py +++ b/fastNLP/io/loader/json.py @@ -14,15 +14,18 @@ class JsonLoader(Loader): """ 读取json格式数据.数据必须按行存储,每行是一个包含各类属性的json对象 - :param dict fields: 需要读入的json属性名称, 和读入后在DataSet中存储的field_name - ``fields`` 的 `key` 必须是json对象的属性名. ``fields`` 的 `value` 为读入后在DataSet存储的 `field_name` , - `value` 也可为 ``None`` , 这时读入后的 `field_name` 与json对象对应属性同名 - ``fields`` 可为 ``None`` , 这时,json对象所有属性都保存在DataSet中. Default: ``None`` - :param bool dropna: 是否忽略非法数据,若 ``True`` 则忽略,若 ``False`` ,在遇到非法数据时,抛出 ``ValueError`` . - Default: ``False`` """ def __init__(self, fields=None, dropna=False): + """ + + :param dict fields: 需要读入的json属性名称, 和读入后在DataSet中存储的field_name + ``fields`` 的 `key` 必须是json对象的属性名. ``fields`` 的 `value` 为读入后在DataSet存储的 `field_name` , + `value` 也可为 ``None`` , 这时读入后的 `field_name` 与json对象对应属性同名 + ``fields`` 可为 ``None`` , 这时,json对象所有属性都保存在DataSet中. Default: ``None`` + :param bool dropna: 是否忽略非法数据,若 ``True`` 则忽略,若 ``False`` ,在遇到非法数据时,抛出 ``ValueError`` . + Default: ``False`` + """ super(JsonLoader, self).__init__() self.dropna = dropna self.fields = None diff --git a/fastNLP/io/loader/matching.py b/fastNLP/io/loader/matching.py index a21d0845..b713fc9a 100644 --- a/fastNLP/io/loader/matching.py +++ b/fastNLP/io/loader/matching.py @@ -128,7 +128,7 @@ class SNLILoader(JsonLoader): def load(self, paths: Union[str, Dict[str, str]] = None) -> DataBundle: """ - 从指定一个或多个路径中的文件中读取数据，返回:class:`~fastNLP.io.DataBundle` 。 + 从指定一个或多个路径中的文件中读取数据，返回 :class:`~fastNLP.io.DataBundle` 。 读取的field根据ConllLoader初始化时传入的headers决定。 diff --git a/fastNLP/io/pipe/classification.py b/fastNLP/io/pipe/classification.py index 3834a570..450c2058 100644 --- a/fastNLP/io/pipe/classification.py +++ b/fastNLP/io/pipe/classification.py @@ -16,12 +16,12 @@ from nltk import Tree from .pipe import Pipe from .utils import get_tokenizer, _indexize, _add_words_field, _drop_empty_instance, _add_chars_field from ..data_bundle import DataBundle +from ..loader.classification import ChnSentiCorpLoader from ..loader.classification import IMDBLoader, YelpFullLoader, SSTLoader, SST2Loader, YelpPolarityLoader from ...core.const import Const from ...core.dataset import DataSet from ...core.instance import Instance from ...core.vocabulary import Vocabulary -from ..loader.classification import ChnSentiCorpLoader nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') @@ -33,6 +33,7 @@ class _CLSPipe(Pipe): """ def __init__(self, tokenizer: str = 'spacy', lang='en'): + self.tokenizer = get_tokenizer(tokenizer, lang=lang) def _tokenize(self, data_bundle, field_name=Const.INPUT, new_field_name=None): @@ -98,13 +99,16 @@ class YelpFullPipe(_CLSPipe): "Offers that ...", "[20, 40, ...]", 1, 21 "...", "[...]", ., . - :param bool lower: 是否对输入进行小写化。 - :param int granularity: 支持2, 3, 5。若为2, 则认为是2分类问题，将1、2归为1类，4、5归为一类，丢掉2；若为3, 则有3分类问题，将 - 1、2归为1类，3归为1类，4、5归为1类；若为5, 则有5分类问题。 - :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 """ def __init__(self, lower: bool = False, granularity=5, tokenizer: str = 'spacy'): + """ + + :param bool lower: 是否对输入进行小写化。 + :param int granularity: 支持2, 3, 5。若为2, 则认为是2分类问题，将1、2归为1类，4、5归为一类，丢掉2；若为3, 则有3分类问题，将 + 1、2归为1类，3归为1类，4、5归为1类；若为5, 则有5分类问题。 + :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 + """ super().__init__(tokenizer=tokenizer, lang='en') self.lower = lower assert granularity in (2, 3, 5), "granularity can only be 2,3,5." @@ -191,11 +195,14 @@ class YelpPolarityPipe(_CLSPipe): "Offers that ...", "[20, 40, ...]", 1, 21 "...", "[...]", ., . - :param bool lower: 是否对输入进行小写化。 - :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 """ def __init__(self, lower: bool = False, tokenizer: str = 'spacy'): + """ + + :param bool lower: 是否对输入进行小写化。 + :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 + """ super().__init__(tokenizer=tokenizer, lang='en') self.lower = lower @@ -237,15 +244,18 @@ class SSTPipe(_CLSPipe): "Offers that ...", "[20, 40, ...]", 1, 18 "...", "[...]", ., . - :param bool subtree: 是否将train, test, dev数据展开为子树，扩充数据量。 Default: ``False`` - :param bool train_subtree: 是否将train集通过子树扩展数据。 - :param bool lower: 是否对输入进行小写化。 - :param int granularity: 支持2, 3, 5。若为2, 则认为是2分类问题，将0、1归为1类，3、4归为一类，丢掉2；若为3, 则有3分类问题，将 - 0、1归为1类，2归为1类，3、4归为1类；若为5, 则有5分类问题。 - :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 """ def __init__(self, subtree=False, train_subtree=True, lower=False, granularity=5, tokenizer='spacy'): + """ + + :param bool subtree: 是否将train, test, dev数据展开为子树，扩充数据量。 Default: ``False`` + :param bool train_subtree: 是否将train集通过子树扩展数据。 + :param bool lower: 是否对输入进行小写化。 + :param int granularity: 支持2, 3, 5。若为2, 则认为是2分类问题，将0、1归为1类，3、4归为一类，丢掉2；若为3, 则有3分类问题，将 + 0、1归为1类，2归为1类，3、4归为1类；若为5, 则有5分类问题。 + :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 + """ super().__init__(tokenizer=tokenizer, lang='en') self.subtree = subtree self.train_tree = train_subtree @@ -327,11 +337,14 @@ class SST2Pipe(_CLSPipe): "unflinchingly bleak and...", "[10, 11, 7,...]", 1, 21 "...", "...", ., . - :param bool lower: 是否对输入进行小写化。 - :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 """ def __init__(self, lower=False, tokenizer='spacy'): + """ + + :param bool lower: 是否对输入进行小写化。 + :param str tokenizer: 使用哪种tokenize方式将数据切成单词。支持'spacy'和'raw'。raw使用空格作为切分。 + """ super().__init__(tokenizer=tokenizer, lang='en') self.lower = lower @@ -402,11 +415,14 @@ class IMDBPipe(_CLSPipe): 其中raw_words为str类型，是原文; words是转换为index的输入; target是转换为index的目标值; words列被设置为input; target列被设置为target。 - :param bool lower: 是否将words列的数据小写。 - :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。 """ def __init__(self, lower: bool = False, tokenizer: str = 'spacy'): + """ + + :param bool lower: 是否将words列的数据小写。 + :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。 + """ super().__init__(tokenizer=tokenizer, lang='en') self.lower = lower @@ -471,14 +487,17 @@ class ChnSentiCorpPipe(Pipe): 其中chars, seq_len是input，target是target - :param bool bigrams: 是否增加一列bigrams. bigrams的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...]。如果 - 设置为True，返回的DataSet将有一列名为bigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 - data_bundle.get_vocab('bigrams')获取. - :param bool trigrams: 是否增加一列trigrams. trigrams的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] - 。如果设置为True，返回的DataSet将有一列名为trigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 - data_bundle.get_vocab('trigrams')获取. """ def __init__(self, bigrams=False, trigrams=False): + """ + + :param bool bigrams: 是否增加一列bigrams. bigrams的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...]。如果 + 设置为True，返回的DataSet将有一列名为bigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('bigrams')获取. + :param bool trigrams: 是否增加一列trigrams. trigrams的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] + 。如果设置为True，返回的DataSet将有一列名为trigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('trigrams')获取. + """ super().__init__() self.bigrams = bigrams diff --git a/fastNLP/io/pipe/conll.py b/fastNLP/io/pipe/conll.py index 12a94d20..24ba2ba0 100644 --- a/fastNLP/io/pipe/conll.py +++ b/fastNLP/io/pipe/conll.py @@ -54,7 +54,7 @@ class _NERPipe(Pipe): "[...]", "[...]" :param ~fastNLP.DataBundle data_bundle: 传入的DataBundle中的DataSet必须包含raw_words和ner两个field，且两个field的内容均为List[str]在传入DataBundle基础上原位修改。 - :return DataBundle: + :return DataBundle: """ # 转换tag for name, dataset in data_bundle.datasets.items(): @@ -94,8 +94,6 @@ class Conll2003NERPipe(_NERPipe): raw_words列为List[str], 是未转换的原始数据; words列为List[int]，是转换为index的输入数据; target列是List[int]，是转换为index的 target。返回的DataSet中被设置为input有words, target, seq_len; 设置为target有target。 - :param: str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 - :param bool lower: 是否将words小写化后再建立词表，绝大多数情况都不需要设置为True。 """ def process_from_file(self, paths) -> DataBundle: @@ -112,18 +110,21 @@ class Conll2003NERPipe(_NERPipe): class Conll2003Pipe(Pipe): - def __init__(self, chunk_encoding_type='bioes', ner_encoding_type='bioes', lower: bool = False): - """ - 经过该Pipe后，DataSet中的内容如下 + r""" + 经过该Pipe后，DataSet中的内容如下 - .. csv-table:: - :header: "raw_words", "words", "pos", "chunk", "ner", "seq_len" + .. csv-table:: + :header: "raw_words" , "words", "pos", "chunk", "ner", "seq_len" - "[Nadim, Ladki]", "[2, 3]", "[0, 0]", "[1, 2]", "[1, 2]", 2 - "[AL-AIN, United, Arab, ...]", "[4, 5, 6,...]", "[1, 2...]", "[3, 4...]", "[3, 4...]", 6 - "[...]", "[...]", "[...]", "[...]", "[...]". + "[Nadim, Ladki]", "[2, 3]", "[0, 0]", "[1, 2]", "[1, 2]", 2 + "[AL-AIN, United, Arab, ...]", "[4, 5, 6,...]", "[1, 2...]", "[3, 4...]", "[3, 4...]", 6 + "[...]", "[...]", "[...]", "[...]", "[...]", . - 其中words, seq_len是input; pos, chunk, ner, seq_len是target + 其中words, seq_len是input; pos, chunk, ner, seq_len是target + + """ + def __init__(self, chunk_encoding_type='bioes', ner_encoding_type='bioes', lower: bool = False): + """ :param str chunk_encoding_type: 支持bioes, bio。 :param str ner_encoding_type: 支持bioes, bio。 @@ -148,7 +149,7 @@ class Conll2003Pipe(Pipe): "[Nadim, Ladki]", "[NNP, NNP]", "[B-NP, I-NP]", "[B-PER, I-PER]" "[AL-AIN, United, Arab, ...]", "[NNP, NNP...]", "[B-NP, B-NP, ...]", "[B-LOC, B-LOC,...]" - "[...]", "[...]", "[...]", "[...]". + "[...]", "[...]", "[...]", "[...]", . :param data_bundle: :return: 传入的DataBundle @@ -204,8 +205,6 @@ class OntoNotesNERPipe(_NERPipe): raw_words列为List[str], 是未转换的原始数据; words列为List[int]，是转换为index的输入数据; target列是List[int]，是转换为index的 target。返回的DataSet中被设置为input有words, target, seq_len; 设置为target有target。 - :param: str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 - :param bool lower: 是否将words小写化后再建立词表，绝大多数情况都不需要设置为True。 """ def process_from_file(self, paths): @@ -222,16 +221,19 @@ class _CNNERPipe(Pipe): raw_chars列为List[str], 是未转换的原始数据; chars列为List[int]，是转换为index的输入数据; target列是List[int]，是转换为index的 target。返回的DataSet中被设置为input有chars, target, seq_len; 设置为target有target, seq_len。 - :param: str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 - :param bool bigrams: 是否增加一列bigrams. bigrams的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...]。如果 - 设置为True，返回的DataSet将有一列名为bigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 - data_bundle.get_vocab('bigrams')获取. - :param bool trigrams: 是否增加一列trigrams. trigrams的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] - 。如果设置为True，返回的DataSet将有一列名为trigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 - data_bundle.get_vocab('trigrams')获取. """ def __init__(self, encoding_type: str = 'bio', bigrams=False, trigrams=False): + """ + + :param str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 + :param bool bigrams: 是否增加一列bigrams. bigrams的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...]。如果 + 设置为True，返回的DataSet将有一列名为bigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('bigrams')获取. + :param bool trigrams: 是否增加一列trigrams. trigrams的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] + 。如果设置为True，返回的DataSet将有一列名为trigrams, 且已经转换为了index并设置为input，对应的vocab可以通过 + data_bundle.get_vocab('trigrams')获取. + """ if encoding_type == 'bio': self.convert_tag = iob2 else: @@ -346,7 +348,6 @@ class WeiboNERPipe(_CNNERPipe): raw_chars列为List[str], 是未转换的原始数据; chars列为List[int]，是转换为index的输入数据; target列是List[int]，是转换为index的 target。返回的DataSet中被设置为input有chars, target, seq_len; 设置为target有target。 - :param: str encoding_type: target列使用什么类型的encoding方式，支持bioes, bio两种。 """ def process_from_file(self, paths=None) -> DataBundle: diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py index b6d88998..3c171507 100644 --- a/fastNLP/io/pipe/coreference.py +++ b/fastNLP/io/pipe/coreference.py @@ -5,13 +5,15 @@ __all__ = [ ] +import collections + +import numpy as np + +from fastNLP.core.vocabulary import Vocabulary from .pipe import Pipe from ..data_bundle import DataBundle from ..loader.coreference import CRLoader from ...core.const import Const -from fastNLP.core.vocabulary import Vocabulary -import numpy as np -import collections class CoreferencePipe(Pipe): @@ -25,8 +27,8 @@ class CoreferencePipe(Pipe): def process(self, data_bundle: DataBundle): """ - 对load进来的数据进一步处理 - 原始数据包含：raw_key,raw_speaker,raw_words,raw_clusters + 对load进来的数据进一步处理原始数据包含：raw_key,raw_speaker,raw_words,raw_clusters + .. csv-table:: :header: "raw_key", "raw_speaker","raw_words","raw_clusters" @@ -35,6 +37,7 @@ class CoreferencePipe(Pipe): "[...]", "[...]","[...]","[...]" 处理完成后数据包含文章类别、speaker信息、句子信息、句子对应的index、char、句子长度、target： + .. csv-table:: :header: "words1", "words2","words3","words4","chars","seq_len","target" diff --git a/fastNLP/io/pipe/cws.py b/fastNLP/io/pipe/cws.py index 748cf10a..1fc64785 100644 --- a/fastNLP/io/pipe/cws.py +++ b/fastNLP/io/pipe/cws.py @@ -146,15 +146,18 @@ class CWSPipe(Pipe): 其中bigrams仅当bigrams列为True的时候为真 - :param str,None dataset_name: 支持'pku', 'msra', 'cityu', 'as', None - :param str encoding_type: 可以选择'bmes', 'segapp'两种。"我 来自 复旦大学...", bmes的tag为[S, B, E, B, M, M, E...]; segapp - 的tag为[seg, app, seg, app, app, app, seg, ...] - :param bool replace_num_alpha: 是否将数字和字母用特殊字符替换。 - :param bool bigrams: 是否增加一列bigram. bigram的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...] - :param bool trigrams: 是否增加一列trigram. trigram的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] """ def __init__(self, dataset_name=None, encoding_type='bmes', replace_num_alpha=True, bigrams=False, trigrams=False): + """ + + :param str,None dataset_name: 支持'pku', 'msra', 'cityu', 'as', None + :param str encoding_type: 可以选择'bmes', 'segapp'两种。"我 来自 复旦大学...", bmes的tag为[S, B, E, B, M, M, E...]; segapp + 的tag为[seg, app, seg, app, app, app, seg, ...] + :param bool replace_num_alpha: 是否将数字和字母用特殊字符替换。 + :param bool bigrams: 是否增加一列bigram. bigram的构成是['复', '旦', '大', '学', ...]->["复旦", "旦大", ...] + :param bool trigrams: 是否增加一列trigram. trigram的构成是 ['复', '旦', '大', '学', ...]->["复旦大", "旦大学", ...] + """ if encoding_type == 'bmes': self.word_lens_to_tags = _word_lens_to_bmes else: @@ -253,7 +256,7 @@ class CWSPipe(Pipe): def process_from_file(self, paths=None) -> DataBundle: """ - + :param str paths: :return: """ diff --git a/fastNLP/io/pipe/matching.py b/fastNLP/io/pipe/matching.py index 747e7b44..3db79aef 100644 --- a/fastNLP/io/pipe/matching.py +++ b/fastNLP/io/pipe/matching.py @@ -37,11 +37,14 @@ class MatchingBertPipe(Pipe): words列被设置为input，target列被设置为target和input(设置为input以方便在forward函数中计算loss， 如果不在forward函数中计算loss也不影响，fastNLP将根据forward函数的形参名进行传参). - :param bool lower: 是否将word小写化。 - :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。 """ def __init__(self, lower=False, tokenizer: str = 'raw'): + """ + + :param bool lower: 是否将word小写化。 + :param str tokenizer: 使用什么tokenizer来将句子切分为words. 支持spacy, raw两种。raw即使用空格拆分。 + """ super().__init__() self.lower = bool(lower) @@ -163,12 +166,14 @@ class MatchingPipe(Pipe): words1是premise，words2是hypothesis。其中words1,words2,seq_len1,seq_len2被设置为input；target被设置为target 和input(设置为input以方便在forward函数中计算loss，如果不在forward函数中计算loss也不影响，fastNLP将根据forward函数 的形参名进行传参)。 - - :param bool lower: 是否将所有raw_words转为小写。 - :param str tokenizer: 将原始数据tokenize的方式。支持spacy, raw. spacy是使用spacy切分，raw就是用空格切分。 """ def __init__(self, lower=False, tokenizer: str = 'raw'): + """ + + :param bool lower: 是否将所有raw_words转为小写。 + :param str tokenizer: 将原始数据tokenize的方式。支持spacy, raw. spacy是使用spacy切分，raw就是用空格切分。 + """ super().__init__() self.lower = bool(lower) diff --git a/fastNLP/io/pipe/pipe.py b/fastNLP/io/pipe/pipe.py index 07735d91..20bafe01 100644 --- a/fastNLP/io/pipe/pipe.py +++ b/fastNLP/io/pipe/pipe.py @@ -13,6 +13,7 @@ class Pipe: doc """ + def process(self, data_bundle: DataBundle) -> DataBundle: """ 对输入的DataBundle进行处理，然后返回该DataBundle。 From 32e99e3696f3287bf40c8a3d687fbd3eb79e5a1b Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 22:22:33 +0800 Subject: [PATCH 65/92] split the docs in models --- fastNLP/models/bert.py | 45 ++++++---- fastNLP/models/biaffine_parser.py | 94 ++++++++++--------- fastNLP/models/cnn_text_classification.py | 15 ++-- fastNLP/models/sequence_labeling.py | 49 +++++----- fastNLP/models/snli.py | 13 +-- fastNLP/models/star_transformer.py | 104 ++++++++++++---------- 6 files changed, 185 insertions(+), 135 deletions(-) diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py index 30ed0cd8..2bd15eb0 100644 --- a/fastNLP/models/bert.py +++ b/fastNLP/models/bert.py @@ -37,8 +37,8 @@ import torch from torch import nn from .base_model import BaseModel -from ..core.const import Const from ..core._logger import logger +from ..core.const import Const from ..embeddings import BertEmbedding @@ -46,11 +46,14 @@ class BertForSequenceClassification(BaseModel): """ BERT model for classification. - :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). - :param int num_labels: 文本分类类别数目，默认值为2. - :param float dropout: dropout的大小，默认值为0.1. """ def __init__(self, embed: BertEmbedding, num_labels: int=2, dropout=0.1): + """ + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: 文本分类类别数目，默认值为2. + :param float dropout: dropout的大小，默认值为0.1. + """ super(BertForSequenceClassification, self).__init__() self.num_labels = num_labels @@ -89,11 +92,14 @@ class BertForSentenceMatching(BaseModel): """ BERT model for sentence matching. - :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). - :param int num_labels: Matching任务类别数目，默认值为2. - :param float dropout: dropout的大小，默认值为0.1. """ def __init__(self, embed: BertEmbedding, num_labels: int=2, dropout=0.1): + """ + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: Matching任务类别数目，默认值为2. + :param float dropout: dropout的大小，默认值为0.1. + """ super(BertForSentenceMatching, self).__init__() self.num_labels = num_labels self.bert = embed @@ -131,11 +137,14 @@ class BertForMultipleChoice(BaseModel): """ BERT model for multiple choice. - :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). - :param int num_choices: 多选任务选项数目，默认值为2. - :param float dropout: dropout的大小，默认值为0.1. """ def __init__(self, embed: BertEmbedding, num_choices=2, dropout=0.1): + """ + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_choices: 多选任务选项数目，默认值为2. + :param float dropout: dropout的大小，默认值为0.1. + """ super(BertForMultipleChoice, self).__init__() self.num_choices = num_choices @@ -178,11 +187,14 @@ class BertForTokenClassification(BaseModel): """ BERT model for token classification. - :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). - :param int num_labels: 序列标注标签数目，无默认值. - :param float dropout: dropout的大小，默认值为0.1. """ def __init__(self, embed: BertEmbedding, num_labels, dropout=0.1): + """ + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: 序列标注标签数目，无默认值. + :param float dropout: dropout的大小，默认值为0.1. + """ super(BertForTokenClassification, self).__init__() self.num_labels = num_labels @@ -221,10 +233,13 @@ class BertForQuestionAnswering(BaseModel): """ BERT model for classification. - :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). - :param int num_labels: 抽取式QA列数，默认值为2(即第一列为start_span, 第二列为end_span). """ def __init__(self, embed: BertEmbedding, num_labels=2): + """ + + :param fastNLP.embeddings.BertEmbedding embed: 下游模型的编码器(encoder). + :param int num_labels: 抽取式QA列数，默认值为2(即第一列为start_span, 第二列为end_span). + """ super(BertForQuestionAnswering, self).__init__() self.bert = embed diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index 5d094472..45f8adb7 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -6,23 +6,23 @@ __all__ = [ "GraphParser" ] +from collections import defaultdict + import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from collections import defaultdict - +from .base_model import BaseModel from ..core.const import Const as C from ..core.losses import LossFunc from ..core.metrics import MetricBase +from ..core.utils import seq_len_to_mask +from ..embeddings.utils import get_embeddings from ..modules.dropout import TimestepDropout from ..modules.encoder.transformer import TransformerEncoder from ..modules.encoder.variational_rnn import VarLSTM from ..modules.utils import initial_parameter -from ..embeddings.utils import get_embeddings -from .base_model import BaseModel -from ..core.utils import seq_len_to_mask def _mst(scores): @@ -181,11 +181,14 @@ class ArcBiaffine(nn.Module): """ Biaffine Dependency Parser 的子模块, 用于构建预测边的图 - :param hidden_size: 输入的特征维度 - :param bias: 是否使用bias. Default: ``True`` """ def __init__(self, hidden_size, bias=True): + """ + + :param hidden_size: 输入的特征维度 + :param bias: 是否使用bias. Default: ``True`` + """ super(ArcBiaffine, self).__init__() self.U = nn.Parameter(torch.Tensor(hidden_size, hidden_size), requires_grad=True) self.has_bias = bias @@ -213,13 +216,16 @@ class LabelBilinear(nn.Module): """ Biaffine Dependency Parser 的子模块, 用于构建预测边类别的图 - :param in1_features: 输入的特征1维度 - :param in2_features: 输入的特征2维度 - :param num_label: 边类别的个数 - :param bias: 是否使用bias. Default: ``True`` """ def __init__(self, in1_features, in2_features, num_label, bias=True): + """ + + :param in1_features: 输入的特征1维度 + :param in2_features: 输入的特征2维度 + :param num_label: 边类别的个数 + :param bias: 是否使用bias. Default: ``True`` + """ super(LabelBilinear, self).__init__() self.bilinear = nn.Bilinear(in1_features, in2_features, num_label, bias=bias) self.lin = nn.Linear(in1_features + in2_features, num_label, bias=False) @@ -241,20 +247,6 @@ class BiaffineParser(GraphParser): Biaffine Dependency Parser 实现. 论文参考 `Deep Biaffine Attention for Neural Dependency Parsing (Dozat and Manning, 2016) `_ . - :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 - embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, - 此时就以传入的对象作为embedding - :param pos_vocab_size: part-of-speech 词典大小 - :param pos_emb_dim: part-of-speech 向量维度 - :param num_label: 边的类别个数 - :param rnn_layers: rnn encoder的层数 - :param rnn_hidden_size: rnn encoder 的隐状态维度 - :param arc_mlp_size: 边预测的MLP维度 - :param label_mlp_size: 类别预测的MLP维度 - :param dropout: dropout概率. - :param encoder: encoder类别, 可选 ('lstm', 'var-lstm', 'transformer'). Default: lstm - :param use_greedy_infer: 是否在inference时使用贪心算法. - 若 ``False`` , 使用更加精确但相对缓慢的MST算法. Default: ``False`` """ def __init__(self, @@ -269,6 +261,23 @@ class BiaffineParser(GraphParser): dropout=0.3, encoder='lstm', use_greedy_infer=False): + """ + + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, + 此时就以传入的对象作为embedding + :param pos_vocab_size: part-of-speech 词典大小 + :param pos_emb_dim: part-of-speech 向量维度 + :param num_label: 边的类别个数 + :param rnn_layers: rnn encoder的层数 + :param rnn_hidden_size: rnn encoder 的隐状态维度 + :param arc_mlp_size: 边预测的MLP维度 + :param label_mlp_size: 类别预测的MLP维度 + :param dropout: dropout概率. + :param encoder: encoder类别, 可选 ('lstm', 'var-lstm', 'transformer'). Default: lstm + :param use_greedy_infer: 是否在inference时使用贪心算法. + 若 ``False`` , 使用更加精确但相对缓慢的MST算法. Default: ``False`` + """ super(BiaffineParser, self).__init__() rnn_out_size = 2 * rnn_hidden_size word_hid_dim = pos_hid_dim = rnn_hidden_size @@ -473,17 +482,20 @@ class ParserLoss(LossFunc): """ 计算parser的loss - :param pred1: [batch_size, seq_len, seq_len] 边预测logits - :param pred2: [batch_size, seq_len, num_label] label预测logits - :param target1: [batch_size, seq_len] 真实边的标注 - :param target2: [batch_size, seq_len] 真实类别的标注 - :param seq_len: [batch_size, seq_len] 真实目标的长度 - :return loss: scalar """ def __init__(self, pred1=None, pred2=None, target1=None, target2=None, seq_len=None): + """ + + :param pred1: [batch_size, seq_len, seq_len] 边预测logits + :param pred2: [batch_size, seq_len, num_label] label预测logits + :param target1: [batch_size, seq_len] 真实边的标注 + :param target2: [batch_size, seq_len] 真实类别的标注 + :param seq_len: [batch_size, seq_len] 真实目标的长度 + :return loss: scalar + """ super(ParserLoss, self).__init__(BiaffineParser.loss, pred1=pred1, pred2=pred2, @@ -496,20 +508,22 @@ class ParserMetric(MetricBase): """ 评估parser的性能 - :param pred1: 边预测logits - :param pred2: label预测logits - :param target1: 真实边的标注 - :param target2: 真实类别的标注 - :param seq_len: 序列长度 - :return dict: 评估结果:: - - UAS: 不带label时, 边预测的准确率 - LAS: 同时预测边和label的准确率 """ def __init__(self, pred1=None, pred2=None, target1=None, target2=None, seq_len=None): + """ + :param pred1: 边预测logits + :param pred2: label预测logits + :param target1: 真实边的标注 + :param target2: 真实类别的标注 + :param seq_len: 序列长度 + :return dict: 评估结果:: + + UAS: 不带label时, 边预测的准确率 + LAS: 同时预测边和label的准确率 + """ super().__init__() self._init_param_map(pred1=pred1, pred2=pred2, target1=target1, target2=target2, diff --git a/fastNLP/models/cnn_text_classification.py b/fastNLP/models/cnn_text_classification.py index 65c20a55..863c4941 100644 --- a/fastNLP/models/cnn_text_classification.py +++ b/fastNLP/models/cnn_text_classification.py @@ -21,12 +21,6 @@ class CNNText(torch.nn.Module): 使用CNN进行文本分类的模型 'Yoon Kim. 2014. Convolution Neural Networks for Sentence Classification.' - :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), - 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, Embedding, ndarray等则直接使用该值初始化Embedding - :param int num_classes: 一共有多少类 - :param int,tuple(int) out_channels: 输出channel的数量。如果为list，则需要与kernel_sizes的数量保持一致 - :param int,tuple(int) kernel_sizes: 输出channel的kernel大小。 - :param float dropout: Dropout的大小 """ def __init__(self, embed, @@ -34,6 +28,15 @@ class CNNText(torch.nn.Module): kernel_nums=(30, 40, 50), kernel_sizes=(1, 3, 5), dropout=0.5): + """ + + :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), + 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, Embedding, ndarray等则直接使用该值初始化Embedding + :param int num_classes: 一共有多少类 + :param int,tuple(int) out_channels: 输出channel的数量。如果为list，则需要与kernel_sizes的数量保持一致 + :param int,tuple(int) kernel_sizes: 输出channel的kernel大小。 + :param float dropout: Dropout的大小 + """ super(CNNText, self).__init__() # no support for pre-trained embedding currently diff --git a/fastNLP/models/sequence_labeling.py b/fastNLP/models/sequence_labeling.py index d5bc250b..560599d1 100644 --- a/fastNLP/models/sequence_labeling.py +++ b/fastNLP/models/sequence_labeling.py @@ -25,16 +25,19 @@ class BiLSTMCRF(BaseModel): """ 结构为embedding + BiLSTM + FC + Dropout + CRF. - :param embed: 支持(1)fastNLP的各种Embedding, (2) tuple, 指明num_embedding, dimension, 如(1000, 100) - :param num_classes: 一共多少个类 - :param num_layers: BiLSTM的层数 - :param hidden_size: BiLSTM的hidden_size，实际hidden size为该值的两倍(前向、后向) - :param dropout: dropout的概率，0为不dropout - :param target_vocab: Vocabulary对象，target与index的对应关系 - :param encoding_type: encoding的类型，支持'bioes', 'bmes', 'bio', 'bmeso'等 """ def __init__(self, embed, num_classes, num_layers=1, hidden_size=100, dropout=0.5, target_vocab=None, encoding_type=None): + """ + + :param embed: 支持(1)fastNLP的各种Embedding, (2) tuple, 指明num_embedding, dimension, 如(1000, 100) + :param num_classes: 一共多少个类 + :param num_layers: BiLSTM的层数 + :param hidden_size: BiLSTM的hidden_size，实际hidden size为该值的两倍(前向、后向) + :param dropout: dropout的概率，0为不dropout + :param target_vocab: Vocabulary对象，target与index的对应关系 + :param encoding_type: encoding的类型，支持'bioes', 'bmes', 'bio', 'bmeso'等 + """ super().__init__() self.embed = get_embeddings(embed) @@ -80,13 +83,16 @@ class SeqLabeling(BaseModel): 一个基础的Sequence labeling的模型。 用于做sequence labeling的基础类。结构包含一层Embedding，一层LSTM(单向，一层)，一层FC，以及一层CRF。 - :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), - 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, embedding, ndarray等则直接使用该值初始化Embedding - :param int hidden_size: LSTM隐藏层的大小 - :param int num_classes: 一共有多少类 """ def __init__(self, embed, hidden_size, num_classes): + """ + + :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), + 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, embedding, ndarray等则直接使用该值初始化Embedding + :param int hidden_size: LSTM隐藏层的大小 + :param int num_classes: 一共有多少类 + """ super(SeqLabeling, self).__init__() self.embedding = get_embeddings(embed) @@ -155,20 +161,21 @@ class SeqLabeling(BaseModel): class AdvSeqLabel(nn.Module): """ 更复杂的Sequence Labelling模型。结构为Embedding, LayerNorm, 双向LSTM(两层)，FC，LayerNorm，DropOut，FC，CRF。 - - :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), - 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, Embedding, ndarray等则直接使用该值初始化Embedding - :param int hidden_size: LSTM的隐层大小 - :param int num_classes: 有多少个类 - :param float dropout: LSTM中以及DropOut层的drop概率 - :param dict id2words: tag id转为其tag word的表。用于在CRF解码时防止解出非法的顺序，比如'BMES'这个标签规范中，'S' - 不能出现在'B'之后。这里也支持类似与'B-NN'，即'-'前为标签类型的指示，后面为具体的tag的情况。这里不但会保证 - 'B-NN'后面不为'S-NN'还会保证'B-NN'后面不会出现'M-xx'(任何非'M-NN'和'E-NN'的情况。) - :param str encoding_type: 支持"BIO", "BMES", "BEMSO", 只有在id2words不为None的情况有用。 """ def __init__(self, embed, hidden_size, num_classes, dropout=0.3, id2words=None, encoding_type='bmes'): + """ + :param tuple(int,int),torch.FloatTensor,nn.Embedding,numpy.ndarray embed: Embedding的大小(传入tuple(int, int), + 第一个int为vocab_zie, 第二个int为embed_dim); 如果为Tensor, Embedding, ndarray等则直接使用该值初始化Embedding + :param int hidden_size: LSTM的隐层大小 + :param int num_classes: 有多少个类 + :param float dropout: LSTM中以及DropOut层的drop概率 + :param dict id2words: tag id转为其tag word的表。用于在CRF解码时防止解出非法的顺序，比如'BMES'这个标签规范中，'S' + 不能出现在'B'之后。这里也支持类似与'B-NN'，即'-'前为标签类型的指示，后面为具体的tag的情况。这里不但会保证 + 'B-NN'后面不为'S-NN'还会保证'B-NN'后面不会出现'M-xx'(任何非'M-NN'和'E-NN'的情况。) + :param str encoding_type: 支持"BIO", "BMES", "BEMSO", 只有在id2words不为None的情况有用。 + """ super().__init__() self.Embedding = get_embeddings(embed) diff --git a/fastNLP/models/snli.py b/fastNLP/models/snli.py index 07303ddc..9a48f967 100644 --- a/fastNLP/models/snli.py +++ b/fastNLP/models/snli.py @@ -22,15 +22,18 @@ class ESIM(BaseModel): ESIM model的一个PyTorch实现 论文参见： https://arxiv.org/pdf/1609.06038.pdf - :param embed: 初始化的Embedding - :param int hidden_size: 隐藏层大小，默认值为Embedding的维度 - :param int num_labels: 目标标签种类数量，默认值为3 - :param float dropout_rate: dropout的比率，默认值为0.3 - :param float dropout_embed: 对Embedding的dropout比率，默认值为0.1 """ def __init__(self, embed, hidden_size=None, num_labels=3, dropout_rate=0.3, dropout_embed=0.1): + """ + + :param embed: 初始化的Embedding + :param int hidden_size: 隐藏层大小，默认值为Embedding的维度 + :param int num_labels: 目标标签种类数量，默认值为3 + :param float dropout_rate: dropout的比率，默认值为0.3 + :param float dropout_embed: 对Embedding的dropout比率，默认值为0.1 + """ super(ESIM, self).__init__() if isinstance(embed, TokenEmbedding) or isinstance(embed, Embedding): diff --git a/fastNLP/models/star_transformer.py b/fastNLP/models/star_transformer.py index e4d5af84..117a63a2 100644 --- a/fastNLP/models/star_transformer.py +++ b/fastNLP/models/star_transformer.py @@ -11,26 +11,16 @@ __all__ = [ import torch from torch import nn -from ..modules.encoder.star_transformer import StarTransformer +from ..core.const import Const from ..core.utils import seq_len_to_mask from ..embeddings.utils import get_embeddings -from ..core.const import Const +from ..modules.encoder.star_transformer import StarTransformer class StarTransEnc(nn.Module): """ 带word embedding的Star-Transformer Encoder - :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 - embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, - 此时就以传入的对象作为embedding - :param hidden_size: 模型中特征维度. - :param num_layers: 模型层数. - :param num_head: 模型中multi-head的head个数. - :param head_dim: 模型中multi-head中每个head特征维度. - :param max_len: 模型能接受的最大输入长度. - :param emb_dropout: 词嵌入的dropout概率. - :param dropout: 模型除词嵌入外的dropout概率. """ def __init__(self, embed, @@ -41,6 +31,18 @@ class StarTransEnc(nn.Module): max_len, emb_dropout, dropout): + """ + + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象,此时就以传入的对象作为embedding + :param hidden_size: 模型中特征维度. + :param num_layers: 模型层数. + :param num_head: 模型中multi-head的head个数. + :param head_dim: 模型中multi-head中每个head特征维度. + :param max_len: 模型能接受的最大输入长度. + :param emb_dropout: 词嵌入的dropout概率. + :param dropout: 模型除词嵌入外的dropout概率. + """ super(StarTransEnc, self).__init__() self.embedding = get_embeddings(embed) emb_dim = self.embedding.embedding_dim @@ -104,18 +106,6 @@ class STSeqLabel(nn.Module): """ 用于序列标注的Star-Transformer模型 - :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 - embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, - 此时就以传入的对象作为embedding - :param num_cls: 输出类别个数 - :param hidden_size: 模型中特征维度. Default: 300 - :param num_layers: 模型层数. Default: 4 - :param num_head: 模型中multi-head的head个数. Default: 8 - :param head_dim: 模型中multi-head中每个head特征维度. Default: 32 - :param max_len: 模型能接受的最大输入长度. Default: 512 - :param cls_hidden_size: 分类器隐层维度. Default: 600 - :param emb_dropout: 词嵌入的dropout概率. Default: 0.1 - :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 """ def __init__(self, embed, num_cls, @@ -127,6 +117,20 @@ class STSeqLabel(nn.Module): cls_hidden_size=600, emb_dropout=0.1, dropout=0.1, ): + """ + + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding + :param num_cls: 输出类别个数 + :param hidden_size: 模型中特征维度. Default: 300 + :param num_layers: 模型层数. Default: 4 + :param num_head: 模型中multi-head的head个数. Default: 8 + :param head_dim: 模型中multi-head中每个head特征维度. Default: 32 + :param max_len: 模型能接受的最大输入长度. Default: 512 + :param cls_hidden_size: 分类器隐层维度. Default: 600 + :param emb_dropout: 词嵌入的dropout概率. Default: 0.1 + :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 + """ super(STSeqLabel, self).__init__() self.enc = StarTransEnc(embed=embed, hidden_size=hidden_size, @@ -167,18 +171,6 @@ class STSeqCls(nn.Module): """ 用于分类任务的Star-Transformer - :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 - embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, - 此时就以传入的对象作为embedding - :param num_cls: 输出类别个数 - :param hidden_size: 模型中特征维度. Default: 300 - :param num_layers: 模型层数. Default: 4 - :param num_head: 模型中multi-head的head个数. Default: 8 - :param head_dim: 模型中multi-head中每个head特征维度. Default: 32 - :param max_len: 模型能接受的最大输入长度. Default: 512 - :param cls_hidden_size: 分类器隐层维度. Default: 600 - :param emb_dropout: 词嵌入的dropout概率. Default: 0.1 - :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 """ def __init__(self, embed, num_cls, @@ -190,6 +182,20 @@ class STSeqCls(nn.Module): cls_hidden_size=600, emb_dropout=0.1, dropout=0.1, ): + """ + + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding + :param num_cls: 输出类别个数 + :param hidden_size: 模型中特征维度. Default: 300 + :param num_layers: 模型层数. Default: 4 + :param num_head: 模型中multi-head的head个数. Default: 8 + :param head_dim: 模型中multi-head中每个head特征维度. Default: 32 + :param max_len: 模型能接受的最大输入长度. Default: 512 + :param cls_hidden_size: 分类器隐层维度. Default: 600 + :param emb_dropout: 词嵌入的dropout概率. Default: 0.1 + :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 + """ super(STSeqCls, self).__init__() self.enc = StarTransEnc(embed=embed, hidden_size=hidden_size, @@ -230,18 +236,6 @@ class STNLICls(nn.Module): """ 用于自然语言推断(NLI)的Star-Transformer - :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 - embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, - 此时就以传入的对象作为embedding - :param num_cls: 输出类别个数 - :param hidden_size: 模型中特征维度. Default: 300 - :param num_layers: 模型层数. Default: 4 - :param num_head: 模型中multi-head的head个数. Default: 8 - :param head_dim: 模型中multi-head中每个head特征维度. Default: 32 - :param max_len: 模型能接受的最大输入长度. Default: 512 - :param cls_hidden_size: 分类器隐层维度. Default: 600 - :param emb_dropout: 词嵌入的dropout概率. Default: 0.1 - :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 """ def __init__(self, embed, num_cls, @@ -253,6 +247,20 @@ class STNLICls(nn.Module): cls_hidden_size=600, emb_dropout=0.1, dropout=0.1, ): + """ + + :param embed: 单词词典, 可以是 tuple, 包括(num_embedings, embedding_dim), 即 + embedding的大小和每个词的维度. 也可以传入 nn.Embedding 对象, 此时就以传入的对象作为embedding + :param num_cls: 输出类别个数 + :param hidden_size: 模型中特征维度. Default: 300 + :param num_layers: 模型层数. Default: 4 + :param num_head: 模型中multi-head的head个数. Default: 8 + :param head_dim: 模型中multi-head中每个head特征维度. Default: 32 + :param max_len: 模型能接受的最大输入长度. Default: 512 + :param cls_hidden_size: 分类器隐层维度. Default: 600 + :param emb_dropout: 词嵌入的dropout概率. Default: 0.1 + :param dropout: 模型除词嵌入外的dropout概率. Default: 0.1 + """ super(STNLICls, self).__init__() self.enc = StarTransEnc(embed=embed, hidden_size=hidden_size, From 202bde4bfd3ac26e1155f7f6e2c6e021b5cfa3d7 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 22:56:50 +0800 Subject: [PATCH 66/92] split the docs over~ --- fastNLP/modules/decoder/crf.py | 22 +++--- fastNLP/modules/decoder/mlp.py | 15 ++-- fastNLP/modules/encoder/attention.py | 27 ++++--- fastNLP/modules/encoder/char_encoder.py | 13 +-- fastNLP/modules/encoder/conv_maxpool.py | 11 ++- fastNLP/modules/encoder/lstm.py | 21 ++--- fastNLP/modules/encoder/pooling.py | 16 ++-- fastNLP/modules/encoder/star_transformer.py | 19 +++-- fastNLP/modules/encoder/transformer.py | 17 ++-- fastNLP/modules/encoder/variational_rnn.py | 88 ++++++++++++--------- 10 files changed, 145 insertions(+), 104 deletions(-) diff --git a/fastNLP/modules/decoder/crf.py b/fastNLP/modules/decoder/crf.py index aeb73d76..669501e9 100644 --- a/fastNLP/modules/decoder/crf.py +++ b/fastNLP/modules/decoder/crf.py @@ -5,13 +5,15 @@ __all__ = [ "allowed_transitions" ] +from typing import Union + import torch from torch import nn from ..utils import initial_parameter -from ...core.vocabulary import Vocabulary from ...core.metrics import _get_encoding_type_from_tag_vocab, _check_tag_vocab_and_encoding_type -from typing import Union +from ...core.vocabulary import Vocabulary + def allowed_transitions(tag_vocab:Union[Vocabulary, dict], encoding_type=None, include_start_end=False): """ @@ -168,17 +170,19 @@ class ConditionalRandomField(nn.Module): """ 条件随机场。提供forward()以及viterbi_decode()两个方法，分别用于训练与inference。 - :param int num_tags: 标签的数量 - :param bool include_start_end_trans: 是否考虑各个tag作为开始以及结尾的分数。 - :param List[Tuple[from_tag_id(int), to_tag_id(int)]] allowed_transitions: 内部的Tuple[from_tag_id(int), - to_tag_id(int)]视为允许发生的跃迁，其他没有包含的跃迁认为是禁止跃迁，可以通过 - allowed_transitions()函数得到；如果为None，则所有跃迁均为合法 - :param str initial_method: 初始化方法。见initial_parameter """ def __init__(self, num_tags, include_start_end_trans=False, allowed_transitions=None, initial_method=None): - + """ + + :param int num_tags: 标签的数量 + :param bool include_start_end_trans: 是否考虑各个tag作为开始以及结尾的分数。 + :param List[Tuple[from_tag_id(int), to_tag_id(int)]] allowed_transitions: 内部的Tuple[from_tag_id(int), + to_tag_id(int)]视为允许发生的跃迁，其他没有包含的跃迁认为是禁止跃迁，可以通过 + allowed_transitions()函数得到；如果为None，则所有跃迁均为合法 + :param str initial_method: 初始化方法。见initial_parameter + """ super(ConditionalRandomField, self).__init__() self.include_start_end_trans = include_start_end_trans diff --git a/fastNLP/modules/decoder/mlp.py b/fastNLP/modules/decoder/mlp.py index 3e594de1..0f23f481 100644 --- a/fastNLP/modules/decoder/mlp.py +++ b/fastNLP/modules/decoder/mlp.py @@ -14,12 +14,6 @@ class MLP(nn.Module): """ 多层感知器 - :param List[int] size_layer: 一个int的列表，用来定义MLP的层数，列表中的数字为每一层是hidden数目。MLP的层数为 len(size_layer) - 1 - :param Union[str,func,List[str]] activation: 一个字符串或者函数的列表，用来定义每一个隐层的激活函数，字符串包括relu，tanh和 - sigmoid，默认值为relu - :param Union[str,func] output_activation: 字符串或者函数，用来定义输出层的激活函数，默认值为None，表示输出层没有激活函数 - :param str initial_method: 参数初始化方式 - :param float dropout: dropout概率，默认值为0 .. note:: 隐藏层的激活函数通过activation定义。一个str/function或者一个str/function的list可以被传入activation。 @@ -42,6 +36,15 @@ class MLP(nn.Module): """ def __init__(self, size_layer, activation='relu', output_activation=None, initial_method=None, dropout=0.0): + """ + + :param List[int] size_layer: 一个int的列表，用来定义MLP的层数，列表中的数字为每一层是hidden数目。MLP的层数为 len(size_layer) - 1 + :param Union[str,func,List[str]] activation: 一个字符串或者函数的列表，用来定义每一个隐层的激活函数，字符串包括relu，tanh和 + sigmoid，默认值为relu + :param Union[str,func] output_activation: 字符串或者函数，用来定义输出层的激活函数，默认值为None，表示输出层没有激活函数 + :param str initial_method: 参数初始化方式 + :param float dropout: dropout概率，默认值为0 + """ super(MLP, self).__init__() self.hiddens = nn.ModuleList() self.output = None diff --git a/fastNLP/modules/encoder/attention.py b/fastNLP/modules/encoder/attention.py index 0d832653..32f59c22 100644 --- a/fastNLP/modules/encoder/attention.py +++ b/fastNLP/modules/encoder/attention.py @@ -46,14 +46,17 @@ class DotAttention(nn.Module): class MultiHeadAttention(nn.Module): """ - :param input_size: int, 输入维度的大小。同时也是输出维度的大小。 - :param key_size: int, 每个head的维度大小。 - :param value_size: int，每个head中value的维度。 - :param num_head: int，head的数量。 - :param dropout: float。 """ def __init__(self, input_size, key_size, value_size, num_head, dropout=0.1): + """ + + :param input_size: int, 输入维度的大小。同时也是输出维度的大小。 + :param key_size: int, 每个head的维度大小。 + :param value_size: int，每个head中value的维度。 + :param num_head: int，head的数量。 + :param dropout: float。 + """ super(MultiHeadAttention, self).__init__() self.input_size = input_size self.key_size = key_size @@ -169,15 +172,17 @@ class BiAttention(nn.Module): class SelfAttention(nn.Module): """ Self Attention Module. - - :param int input_size: 输入tensor的hidden维度 - :param int attention_unit: 输出tensor的hidden维度 - :param int attention_hops: - :param float drop: dropout概率，默认值为0.5 - :param str initial_method: 初始化参数方法 """ def __init__(self, input_size, attention_unit=300, attention_hops=10, drop=0.5, initial_method=None, ): + """ + + :param int input_size: 输入tensor的hidden维度 + :param int attention_unit: 输出tensor的hidden维度 + :param int attention_hops: + :param float drop: dropout概率，默认值为0.5 + :param str initial_method: 初始化参数方法 + """ super(SelfAttention, self).__init__() self.attention_hops = attention_hops diff --git a/fastNLP/modules/encoder/char_encoder.py b/fastNLP/modules/encoder/char_encoder.py index dc73f447..786a2467 100644 --- a/fastNLP/modules/encoder/char_encoder.py +++ b/fastNLP/modules/encoder/char_encoder.py @@ -15,14 +15,17 @@ class ConvolutionCharEncoder(nn.Module): """ char级别的卷积编码器. - :param int char_emb_size: char级别embedding的维度. Default: 50 - :例: 有26个字符, 每一个的embedding是一个50维的向量, 所以输入的向量维度为50. - :param tuple feature_maps: 一个由int组成的tuple. tuple的长度是char级别卷积操作的数目, 第`i`个int表示第`i`个卷积操作的filter. - :param tuple kernels: 一个由int组成的tuple. tuple的长度是char级别卷积操作的数目, 第`i`个int表示第`i`个卷积操作的卷积核. - :param initial_method: 初始化参数的方式, 默认为`xavier normal` """ def __init__(self, char_emb_size=50, feature_maps=(40, 30, 30), kernels=(1, 3, 5), initial_method=None): + """ + + :param int char_emb_size: char级别embedding的维度. Default: 50 + :例: 有26个字符, 每一个的embedding是一个50维的向量, 所以输入的向量维度为50. + :param tuple feature_maps: 一个由int组成的tuple. tuple的长度是char级别卷积操作的数目, 第`i`个int表示第`i`个卷积操作的filter. + :param tuple kernels: 一个由int组成的tuple. tuple的长度是char级别卷积操作的数目, 第`i`个int表示第`i`个卷积操作的卷积核. + :param initial_method: 初始化参数的方式, 默认为`xavier normal` + """ super(ConvolutionCharEncoder, self).__init__() self.convs = nn.ModuleList([ nn.Conv2d(1, feature_maps[i], kernel_size=(char_emb_size, kernels[i]), bias=True, diff --git a/fastNLP/modules/encoder/conv_maxpool.py b/fastNLP/modules/encoder/conv_maxpool.py index bf629eba..f19a92f3 100644 --- a/fastNLP/modules/encoder/conv_maxpool.py +++ b/fastNLP/modules/encoder/conv_maxpool.py @@ -14,13 +14,16 @@ class ConvMaxpool(nn.Module): sum(output_channels) 大小的matrix。在内部，是先使用CNN给输入做卷积，然后经过activation激活层，在通过在长度(max_len) 这一维进行max_pooling。最后得到每个sample的一个向量表示。 - :param int in_channels: 输入channel的大小，一般是embedding的维度; 或encoder的output维度 - :param int,tuple(int) out_channels: 输出channel的数量。如果为list，则需要与kernel_sizes的数量保持一致 - :param int,tuple(int) kernel_sizes: 输出channel的kernel大小。 - :param str activation: Convolution后的结果将通过该activation后再经过max-pooling。支持relu, sigmoid, tanh """ def __init__(self, in_channels, out_channels, kernel_sizes, activation="relu"): + """ + + :param int in_channels: 输入channel的大小，一般是embedding的维度; 或encoder的output维度 + :param int,tuple(int) out_channels: 输出channel的数量。如果为list，则需要与kernel_sizes的数量保持一致 + :param int,tuple(int) kernel_sizes: 输出channel的kernel大小。 + :param str activation: Convolution后的结果将通过该activation后再经过max-pooling。支持relu, sigmoid, tanh + """ super(ConvMaxpool, self).__init__() for kernel_size in kernel_sizes: diff --git a/fastNLP/modules/encoder/lstm.py b/fastNLP/modules/encoder/lstm.py index 1dd1f0df..06b437ef 100644 --- a/fastNLP/modules/encoder/lstm.py +++ b/fastNLP/modules/encoder/lstm.py @@ -15,20 +15,23 @@ import torch.nn.utils.rnn as rnn class LSTM(nn.Module): """ LSTM 模块, 轻量封装的Pytorch LSTM. 在提供seq_len的情况下，将自动使用pack_padded_sequence; 同时默认将forget gate的bias初始化 - 为1; 且可以应对DataParallel中LSTM的使用问题。 + 为1; 且可以应对DataParallel中LSTM的使用问题。 - :param input_size: 输入 `x` 的特征维度 - :param hidden_size: 隐状态 `h` 的特征维度. - :param num_layers: rnn的层数. Default: 1 - :param dropout: 层间dropout概率. Default: 0 - :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` - :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 - :(batch, seq, feature). Default: ``False`` - :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` """ def __init__(self, input_size, hidden_size=100, num_layers=1, dropout=0.0, batch_first=True, bidirectional=False, bias=True): + """ + + :param input_size: 输入 `x` 的特征维度 + :param hidden_size: 隐状态 `h` 的特征维度. + :param num_layers: rnn的层数. Default: 1 + :param dropout: 层间dropout概率. Default: 0 + :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` + :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 + :(batch, seq, feature). Default: ``False`` + :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` + """ super(LSTM, self).__init__() self.batch_first = batch_first self.lstm = nn.LSTM(input_size, hidden_size, num_layers, bias=bias, batch_first=batch_first, diff --git a/fastNLP/modules/encoder/pooling.py b/fastNLP/modules/encoder/pooling.py index c248601d..789b6d26 100644 --- a/fastNLP/modules/encoder/pooling.py +++ b/fastNLP/modules/encoder/pooling.py @@ -14,16 +14,18 @@ class MaxPool(nn.Module): """ Max-pooling模块。 - :param stride: 窗口移动大小，默认为kernel_size - :param padding: padding的内容，默认为0 - :param dilation: 控制窗口内元素移动距离的大小 - :param dimension: MaxPool的维度，支持1，2，3维。 - :param kernel_size: max pooling的窗口大小，默认为tensor最后k维，其中k为dimension - :param ceil_mode: """ def __init__(self, stride=None, padding=0, dilation=1, dimension=1, kernel_size=None, ceil_mode=False): - + """ + + :param stride: 窗口移动大小，默认为kernel_size + :param padding: padding的内容，默认为0 + :param dilation: 控制窗口内元素移动距离的大小 + :param dimension: MaxPool的维度，支持1，2，3维。 + :param kernel_size: max pooling的窗口大小，默认为tensor最后k维，其中k为dimension + :param ceil_mode: + """ super(MaxPool, self).__init__() assert (1 <= dimension) and (dimension <= 3) self.dimension = dimension diff --git a/fastNLP/modules/encoder/star_transformer.py b/fastNLP/modules/encoder/star_transformer.py index bb47d9b5..d4cc66f7 100644 --- a/fastNLP/modules/encoder/star_transformer.py +++ b/fastNLP/modules/encoder/star_transformer.py @@ -18,17 +18,20 @@ class StarTransformer(nn.Module): paper: https://arxiv.org/abs/1902.09113 - :param int hidden_size: 输入维度的大小。同时也是输出维度的大小。 - :param int num_layers: star-transformer的层数 - :param int num_head: head的数量。 - :param int head_dim: 每个head的维度大小。 - :param float dropout: dropout 概率. Default: 0.1 - :param int max_len: int or None, 如果为int，输入序列的最大长度， - 模型会为输入序列加上position embedding。 - 若为`None`，忽略加上position embedding的步骤. Default: `None` """ def __init__(self, hidden_size, num_layers, num_head, head_dim, dropout=0.1, max_len=None): + """ + + :param int hidden_size: 输入维度的大小。同时也是输出维度的大小。 + :param int num_layers: star-transformer的层数 + :param int num_head: head的数量。 + :param int head_dim: 每个head的维度大小。 + :param float dropout: dropout 概率. Default: 0.1 + :param int max_len: int or None, 如果为int，输入序列的最大长度， + 模型会为输入序列加上position embedding。 + 若为`None`，忽略加上position embedding的步骤. Default: `None` + """ super(StarTransformer, self).__init__() self.iters = num_layers diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index 3d97c306..323091b0 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -12,13 +12,6 @@ class TransformerEncoder(nn.Module): """ transformer的encoder模块，不包含embedding层 - :param int num_layers: transformer的层数 - :param int model_size: 输入维度的大小。同时也是输出维度的大小。 - :param int inner_size: FFN层的hidden大小 - :param int key_size: 每个head的维度大小。 - :param int value_size: 每个head中value的维度。 - :param int num_head: head的数量。 - :param float dropout: dropout概率. Default: 0.1 """ class SubLayer(nn.Module): @@ -53,6 +46,16 @@ class TransformerEncoder(nn.Module): return input def __init__(self, num_layers, **kargs): + """ + + :param int num_layers: transformer的层数 + :param int model_size: 输入维度的大小。同时也是输出维度的大小。 + :param int inner_size: FFN层的hidden大小 + :param int key_size: 每个head的维度大小。 + :param int value_size: 每个head中value的维度。 + :param int num_head: head的数量。 + :param float dropout: dropout概率. Default: 0.1 + """ super(TransformerEncoder, self).__init__() self.layers = nn.ModuleList([self.SubLayer(**kargs) for _ in range(num_layers)]) self.norm = nn.LayerNorm(kargs['model_size'], eps=1e-6) diff --git a/fastNLP/modules/encoder/variational_rnn.py b/fastNLP/modules/encoder/variational_rnn.py index 17e2ad23..5f4a5534 100644 --- a/fastNLP/modules/encoder/variational_rnn.py +++ b/fastNLP/modules/encoder/variational_rnn.py @@ -106,22 +106,25 @@ class VarRNNBase(nn.Module): 论文参考: `A Theoretically Grounded Application of Dropout in Recurrent Neural Networks (Yarin Gal and Zoubin Ghahramani, 2016) https://arxiv.org/abs/1512.05287`. - :param mode: rnn 模式, (lstm or not) - :param Cell: rnn cell 类型, (lstm, gru, etc) - :param input_size: 输入 `x` 的特征维度 - :param hidden_size: 隐状态 `h` 的特征维度 - :param num_layers: rnn的层数. Default: 1 - :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` - :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 - (batch, seq, feature). Default: ``False`` - :param input_dropout: 对输入的dropout概率. Default: 0 - :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 - :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` """ def __init__(self, mode, Cell, input_size, hidden_size, num_layers=1, bias=True, batch_first=False, input_dropout=0, hidden_dropout=0, bidirectional=False): + """ + + :param mode: rnn 模式, (lstm or not) + :param Cell: rnn cell 类型, (lstm, gru, etc) + :param input_size: 输入 `x` 的特征维度 + :param hidden_size: 隐状态 `h` 的特征维度 + :param num_layers: rnn的层数. Default: 1 + :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` + :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 + (batch, seq, feature). Default: ``False`` + :param input_dropout: 对输入的dropout概率. Default: 0 + :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 + :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` + """ super(VarRNNBase, self).__init__() self.mode = mode self.input_size = input_size @@ -225,18 +228,21 @@ class VarLSTM(VarRNNBase): """ Variational Dropout LSTM. - :param input_size: 输入 `x` 的特征维度 - :param hidden_size: 隐状态 `h` 的特征维度 - :param num_layers: rnn的层数. Default: 1 - :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` - :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 - (batch, seq, feature). Default: ``False`` - :param input_dropout: 对输入的dropout概率. Default: 0 - :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 - :param bidirectional: 若为 ``True``, 使用双向的LSTM. Default: ``False`` """ def __init__(self, *args, **kwargs): + """ + + :param input_size: 输入 `x` 的特征维度 + :param hidden_size: 隐状态 `h` 的特征维度 + :param num_layers: rnn的层数. Default: 1 + :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` + :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 + (batch, seq, feature). Default: ``False`` + :param input_dropout: 对输入的dropout概率. Default: 0 + :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 + :param bidirectional: 若为 ``True``, 使用双向的LSTM. Default: ``False`` + """ super(VarLSTM, self).__init__( mode="LSTM", Cell=nn.LSTMCell, *args, **kwargs) @@ -248,18 +254,21 @@ class VarRNN(VarRNNBase): """ Variational Dropout RNN. - :param input_size: 输入 `x` 的特征维度 - :param hidden_size: 隐状态 `h` 的特征维度 - :param num_layers: rnn的层数. Default: 1 - :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` - :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 - (batch, seq, feature). Default: ``False`` - :param input_dropout: 对输入的dropout概率. Default: 0 - :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 - :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` """ def __init__(self, *args, **kwargs): + """ + + :param input_size: 输入 `x` 的特征维度 + :param hidden_size: 隐状态 `h` 的特征维度 + :param num_layers: rnn的层数. Default: 1 + :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` + :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 + (batch, seq, feature). Default: ``False`` + :param input_dropout: 对输入的dropout概率. Default: 0 + :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 + :param bidirectional: 若为 ``True``, 使用双向的RNN. Default: ``False`` + """ super(VarRNN, self).__init__( mode="RNN", Cell=nn.RNNCell, *args, **kwargs) @@ -271,18 +280,21 @@ class VarGRU(VarRNNBase): """ Variational Dropout GRU. - :param input_size: 输入 `x` 的特征维度 - :param hidden_size: 隐状态 `h` 的特征维度 - :param num_layers: rnn的层数. Default: 1 - :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` - :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 - (batch, seq, feature). Default: ``False`` - :param input_dropout: 对输入的dropout概率. Default: 0 - :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 - :param bidirectional: 若为 ``True``, 使用双向的GRU. Default: ``False`` """ def __init__(self, *args, **kwargs): + """ + + :param input_size: 输入 `x` 的特征维度 + :param hidden_size: 隐状态 `h` 的特征维度 + :param num_layers: rnn的层数. Default: 1 + :param bias: 如果为 ``False``, 模型将不会使用bias. Default: ``True`` + :param batch_first: 若为 ``True``, 输入和输出 ``Tensor`` 形状为 + (batch, seq, feature). Default: ``False`` + :param input_dropout: 对输入的dropout概率. Default: 0 + :param hidden_dropout: 对每个隐状态的dropout概率. Default: 0 + :param bidirectional: 若为 ``True``, 使用双向的GRU. Default: ``False`` + """ super(VarGRU, self).__init__( mode="GRU", Cell=nn.GRUCell, *args, **kwargs) From 9214cc6e363705e86ea553c54264428968588ce1 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 10 Sep 2019 23:41:12 +0800 Subject: [PATCH 67/92] add the auto baseclass doc --- fastNLP/__init__.py | 22 +++++++++++++++------- fastNLP/doc_utils.py | 12 ++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/fastNLP/__init__.py b/fastNLP/__init__.py index aceaf47f..8800a5a7 100644 --- a/fastNLP/__init__.py +++ b/fastNLP/__init__.py @@ -14,7 +14,6 @@ __all__ = [ "Instance", "FieldArray", - "DataSetIter", "BatchIter", "TorchLoaderIter", @@ -29,10 +28,18 @@ __all__ = [ "Callback", "GradientClipCallback", "EarlyStopCallback", - "TensorboardCallback", + "FitlogCallback", + "EvaluateCallback", "LRScheduler", "ControlC", "LRFinder", + "TensorboardCallback", + "WarmupCallback", + 'SaveModelCallback', + "EchoCallback", + "TesterCallback", + "CallbackException", + "EarlyStopError", "Padder", "AutoPadder", @@ -46,7 +53,7 @@ __all__ = [ "SGD", "Adam", "AdamW", - + "Sampler", "SequentialSampler", "BucketSampler", @@ -60,17 +67,18 @@ __all__ = [ "LossInForward", "cache_results", - + 'logger' ] __version__ = '0.4.5' +import sys + from . import embeddings from . import models from . import modules from .core import * +from .doc_utils import doc_process from .io import loader, pipe -import sys -from .doc_utils import doc_process -doc_process(sys.modules[__name__]) \ No newline at end of file +doc_process(sys.modules[__name__]) diff --git a/fastNLP/doc_utils.py b/fastNLP/doc_utils.py index 5f293d3f..52e347b9 100644 --- a/fastNLP/doc_utils.py +++ b/fastNLP/doc_utils.py @@ -25,3 +25,15 @@ def doc_process(m): if module_name == m.__name__: # print(name, ": not found defined doc.") break + + if inspect.isclass(obj): + for base in obj.__bases__: + if base.__module__.startswith("fastNLP"): + parts = base.__module__.split(".") + [] + module_name, i = "fastNLP", 1 + for i in range(len(parts) - 1): + defined_m = sys.modules[module_name] + if "undocumented" not in defined_m.__doc__ and name in defined_m.__all__: + obj.__doc__ = r"基类 :class:`" + defined_m.__name__ + "." + base.__name__ + "` \n\n" + obj.__doc__ + break + module_name += "." + parts[i + 1] From 5e6e35dbe31dbb584e0a5ff7d949a81bcde440a2 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 02:13:12 +0800 Subject: [PATCH 68/92] update README.md --- README.md | 16 +++++++++------- reproduction/README.md | 8 ++++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 531fbc83..6651041e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ fastNLP 依赖以下包: + nltk>=3.4.1 + requests + spacy ++ prettytable>=0.7.2 其中torch的安装可能与操作系统及 CUDA 的版本相关，请参见 [PyTorch 官网](https://pytorch.org/) 。 在依赖包安装完成后，您可以在命令行执行如下指令完成安装 @@ -45,15 +46,16 @@ fastNLP0.5.0版本将在近期推出，请密切关注。 - [0. 快速入门](https://fastnlp.readthedocs.io/zh/latest/user/quickstart.html) - [1. 使用DataSet预处理文本](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_1_data_preprocess.html) -- [2. 使用Loader和Pipe加载并处理数据集](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_2_load_dataset.html) +- [2. 使用Vocabulary转换文本与index](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_2_vocabulary.html) - [3. 使用Embedding模块将文本转成向量](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_3_embedding.html) -- [4. 动手实现一个文本分类器I-使用Trainer和Tester快速训练和测试](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_4_loss_optimizer.html) +- [4. 使用Loader和Pipe加载并处理数据集](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_4_load_dataset.html) - [5. 动手实现一个文本分类器II-使用DataSetIter实现自定义训练过程](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_5_datasetiter.html) -- [6. 快速实现序列标注模型](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_6_seq_labeling.html) -- [7. 使用Modules和Models快速搭建自定义模型](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_7_modules_models.html) -- [8. 使用Metric快速评测你的模型](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_8_metrics.html) -- [9. 使用Callback自定义你的训练过程](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_9_callback.html) -- [10. 使用fitlog 辅助 fastNLP 进行科研](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_10_fitlog.html) +- [6. 动手实现一个文本分类器I-使用Trainer和Tester快速训练和测试](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_6_loss_optimizer.html) +- [7. 使用Metric快速评测你的模型](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_7_metrics.html) +- [8. 使用Modules和Models快速搭建自定义模型](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_8_modules_models.html) +- [9. 快速实现序列标注模型](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_9_seq_labeling.html) +- [10. 使用Callback自定义你的训练过程](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_10_callback.html) +- [11. 使用fitlog 辅助 fastNLP 进行科研](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_11_fitlog.html) diff --git a/reproduction/README.md b/reproduction/README.md index c2478713..ce623dbe 100644 --- a/reproduction/README.md +++ b/reproduction/README.md @@ -3,8 +3,8 @@ 复现的模型有: - [Star-Transformer](Star_transformer) -- [Biaffine](https://github.com/fastnlp/fastNLP/blob/999a14381747068e9e6a7cc370037b320197db00/fastNLP/models/biaffine_parser.py#L239) -- [CNNText](https://github.com/fastnlp/fastNLP/blob/999a14381747068e9e6a7cc370037b320197db00/fastNLP/models/cnn_text_classification.py#L12) +- [Biaffine](https://github.com/fastnlp/fastNLP/blob/master/fastNLP/models/biaffine_parser.py) +- [CNNText](https://github.com/fastnlp/fastNLP/blob/master/fastNLP/models/cnn_text_classification.py) - ... # 任务复现 @@ -20,8 +20,8 @@ - [NER](seqence_labelling/ner) -## Coreference Resolution (共指消解) -- [Coreference Resolution 共指消解任务复现](coreference_resolution) +## Coreference Resolution (指代消解) +- [Coreference Resolution 指代消解任务复现](coreference_resolution) ## Summarization (摘要) From 143bf8ed32185de16ab0ac4f35ce4b2747051f49 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 02:14:12 +0800 Subject: [PATCH 69/92] update DataBundle and add two property: num_dataset and num_vocab --- fastNLP/io/data_bundle.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/fastNLP/io/data_bundle.py b/fastNLP/io/data_bundle.py index 1f05cf68..ba275e61 100644 --- a/fastNLP/io/data_bundle.py +++ b/fastNLP/io/data_bundle.py @@ -10,6 +10,7 @@ from typing import Union from ..core.dataset import DataSet from ..core.vocabulary import Vocabulary +from ..core._logger import logger class DataBundle: @@ -47,13 +48,14 @@ class DataBundle: self.vocabs[field_name] = vocab return self - def set_dataset(self, dataset, name): + def set_dataset(self, dataset, name: str): """ :param ~fastNLP.DataSet dataset: 传递给DataBundle的DataSet :param str name: dataset的名称 :return: self """ + assert isinstance(dataset, DataSet), "Only fastNLP.DataSet supports." self.datasets[name] = dataset return self @@ -64,7 +66,13 @@ class DataBundle: :param str name: dataset的名称，一般为'train', 'dev', 'test' :return: DataSet """ - return self.datasets[name] + if name in self.datasets.keys(): + return self.datasets[name] + else: + error_msg = f'DataBundle do NOT have DataSet named {name}. ' \ + f'It should be one of {self.datasets.keys()}.' + logger.error(error_msg) + raise KeyError(error_msg) def delete_dataset(self, name: str): """ @@ -83,7 +91,13 @@ class DataBundle: :param str field_name: 名称 :return: Vocabulary """ - return self.vocabs[field_name] + if field_name in self.vocabs.keys(): + return self.vocabs[field_name] + else: + error_msg = f'DataBundle do NOT have Vocabulary named {field_name}. ' \ + f'It should be one of {self.vocabs.keys()}.' + logger.error(error_msg) + raise KeyError(error_msg) def delete_vocab(self, field_name: str): """ @@ -94,6 +108,14 @@ class DataBundle: self.vocabs.pop(field_name, None) return self + @property + def num_dataset(self): + return len(self.datasets) + + @property + def num_vocab(self): + return len(self.vocabs) + def set_input(self, *field_names, flag=True, use_1st_ins_infer_dim_type=True, ignore_miss_dataset=True): """ 将field_names中的field设置为input, 对data_bundle中所有的dataset执行该操作:: @@ -238,7 +260,7 @@ class DataBundle: self.vocabs.pop(field_name) return self - def iter_datasets(self)->Union[str, DataSet]: + def iter_datasets(self) -> Union[str, DataSet]: """ 迭代data_bundle中的DataSet @@ -252,7 +274,7 @@ class DataBundle: for name, dataset in self.datasets.items(): yield name, dataset - def iter_vocabs(self)->Union[str, Vocabulary]: + def iter_vocabs(self) -> Union[str, Vocabulary]: """ 迭代data_bundle中的DataSet @@ -266,7 +288,7 @@ class DataBundle: for field_name, vocab in self.vocabs.items(): yield field_name, vocab - def apply_field(self, func, field_name:str, new_field_name:str, ignore_miss_dataset=True, **kwargs): + def apply_field(self, func, field_name: str, new_field_name: str, ignore_miss_dataset=True, **kwargs): """ 对DataBundle中所有的dataset使用apply_field方法 @@ -313,11 +335,11 @@ class DataBundle: def __repr__(self): _str = '' if len(self.datasets): - _str += 'In total {} datasets:\n'.format(len(self.datasets)) + _str += 'In total {} datasets:\n'.format(self.num_dataset) for name, dataset in self.datasets.items(): _str += '\t{} has {} instances.\n'.format(name, len(dataset)) if len(self.vocabs): - _str += 'In total {} vocabs:\n'.format(len(self.vocabs)) + _str += 'In total {} vocabs:\n'.format(self.num_vocab) for name, vocab in self.vocabs.items(): _str += '\t{} has {} entries.\n'.format(name, len(vocab)) return _str From 753327d214e296b96e00b19ba0d267c61d7d5d7d Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 02:16:57 +0800 Subject: [PATCH 70/92] fix code style in coreference task and related codes --- fastNLP/io/loader/__init__.py | 4 ++-- fastNLP/io/loader/coreference.py | 12 ++++++---- fastNLP/io/pipe/__init__.py | 4 ++-- fastNLP/io/pipe/coreference.py | 12 ++++------ reproduction/coreference_resolution/train.py | 24 +++++++------------ reproduction/coreference_resolution/valid.py | 4 ++-- .../{ => io}/coreference/coreference_dev.json | 0 .../coreference/coreference_test.json | 0 .../coreference/coreference_train.json | 0 test/io/loader/test_coreference_loader.py | 22 ++++++++++++----- test/io/pipe/test_coreference.py | 19 +++++++++++---- 11 files changed, 57 insertions(+), 44 deletions(-) rename test/data_for_tests/{ => io}/coreference/coreference_dev.json (100%) rename test/data_for_tests/{ => io}/coreference/coreference_test.json (100%) rename test/data_for_tests/{ => io}/coreference/coreference_train.json (100%) diff --git a/fastNLP/io/loader/__init__.py b/fastNLP/io/loader/__init__.py index cf88e8c0..06ad57c3 100644 --- a/fastNLP/io/loader/__init__.py +++ b/fastNLP/io/loader/__init__.py @@ -74,7 +74,7 @@ __all__ = [ "QNLILoader", "RTELoader", - "CRLoader" + "CoReferenceLoader" ] from .classification import YelpLoader, YelpFullLoader, YelpPolarityLoader, IMDBLoader, SSTLoader, SST2Loader, ChnSentiCorpLoader from .conll import ConllLoader, Conll2003Loader, Conll2003NERLoader, OntoNotesNERLoader, CTBLoader @@ -84,4 +84,4 @@ from .json import JsonLoader from .loader import Loader from .matching import MNLILoader, QuoraLoader, SNLILoader, QNLILoader, RTELoader from .conll import MsraNERLoader, PeopleDailyNERLoader, WeiboNERLoader -from .coreference import CRLoader \ No newline at end of file +from .coreference import CoReferenceLoader \ No newline at end of file diff --git a/fastNLP/io/loader/coreference.py b/fastNLP/io/loader/coreference.py index 714b11e5..4293f65a 100644 --- a/fastNLP/io/loader/coreference.py +++ b/fastNLP/io/loader/coreference.py @@ -1,5 +1,9 @@ """undocumented""" +__all__ = [ + "CoReferenceLoader", +] + from ...core.dataset import DataSet from ..file_reader import _read_json from ...core.instance import Instance @@ -7,7 +11,7 @@ from ...core.const import Const from .json import JsonLoader -class CRLoader(JsonLoader): +class CoReferenceLoader(JsonLoader): """ 原始数据中内容应该为, 每一行为一个json对象,其中doc_key包含文章的种类信息，speakers包含每句话的说话者信息，cluster是指向现实中同一个事物的聚集，sentences是文本信息内容。 @@ -24,8 +28,8 @@ class CRLoader(JsonLoader): """ def __init__(self, fields=None, dropna=False): super().__init__(fields, dropna) - # self.fields = {"doc_key":Const.INPUTS(0),"speakers":Const.INPUTS(1),"clusters":Const.TARGET,"sentences":Const.INPUTS(2)} - # TODO check 1 + # self.fields = {"doc_key":Const.INPUTS(0),"speakers":Const.INPUTS(1), + # "clusters":Const.TARGET,"sentences":Const.INPUTS(2)} self.fields = {"doc_key": Const.RAW_WORDS(0), "speakers": Const.RAW_WORDS(1), "clusters": Const.RAW_WORDS(2), "sentences": Const.RAW_WORDS(3)} @@ -43,4 +47,4 @@ class CRLoader(JsonLoader): else: ins = d dataset.append(Instance(**ins)) - return dataset \ No newline at end of file + return dataset diff --git a/fastNLP/io/pipe/__init__.py b/fastNLP/io/pipe/__init__.py index f3534cc2..0ddb1f2d 100644 --- a/fastNLP/io/pipe/__init__.py +++ b/fastNLP/io/pipe/__init__.py @@ -39,7 +39,7 @@ __all__ = [ "QNLIPipe", "MNLIPipe", - "CoreferencePipe" + "CoReferencePipe" ] from .classification import YelpFullPipe, YelpPolarityPipe, SSTPipe, SST2Pipe, IMDBPipe, ChnSentiCorpPipe @@ -49,4 +49,4 @@ from .matching import MatchingBertPipe, RTEBertPipe, SNLIBertPipe, QuoraBertPipe from .pipe import Pipe from .conll import Conll2003Pipe from .cws import CWSPipe -from .coreference import CoreferencePipe +from .coreference import CoReferencePipe diff --git a/fastNLP/io/pipe/coreference.py b/fastNLP/io/pipe/coreference.py index 3c171507..c1b218a5 100644 --- a/fastNLP/io/pipe/coreference.py +++ b/fastNLP/io/pipe/coreference.py @@ -1,8 +1,7 @@ """undocumented""" __all__ = [ - "CoreferencePipe" - + "CoReferencePipe" ] import collections @@ -12,11 +11,11 @@ import numpy as np from fastNLP.core.vocabulary import Vocabulary from .pipe import Pipe from ..data_bundle import DataBundle -from ..loader.coreference import CRLoader +from ..loader.coreference import CoReferenceLoader from ...core.const import Const -class CoreferencePipe(Pipe): +class CoReferencePipe(Pipe): """ 对Coreference resolution问题进行处理，得到文章种类/说话者/字符级信息/序列长度。 """ @@ -52,7 +51,7 @@ class CoreferencePipe(Pipe): vocab = Vocabulary().from_dataset(*data_bundle.datasets.values(), field_name= Const.RAW_WORDS(3)) vocab.build_vocab() word2id = vocab.word2idx - data_bundle.set_vocab(vocab,Const.INPUT) + data_bundle.set_vocab(vocab, Const.INPUTS(0)) if self.config.char_path: char_dict = get_char_dict(self.config.char_path) else: @@ -93,7 +92,6 @@ class CoreferencePipe(Pipe): # clusters ds.rename_field(Const.RAW_WORDS(2), Const.TARGET) - ds.set_ignore_type(Const.TARGET) ds.set_padder(Const.TARGET, None) ds.set_input(Const.INPUTS(0), Const.INPUTS(1), Const.INPUTS(2), Const.INPUTS(3), Const.CHAR_INPUT, Const.INPUT_LEN) @@ -102,7 +100,7 @@ class CoreferencePipe(Pipe): return data_bundle def process_from_file(self, paths): - bundle = CRLoader().load(paths) + bundle = CoReferenceLoader().load(paths) return self.process(bundle) diff --git a/reproduction/coreference_resolution/train.py b/reproduction/coreference_resolution/train.py index 23ba5d5b..d5445cd5 100644 --- a/reproduction/coreference_resolution/train.py +++ b/reproduction/coreference_resolution/train.py @@ -1,5 +1,3 @@ -import sys -sys.path.append('../..') import torch from torch.optim import Adam @@ -7,20 +5,15 @@ from torch.optim import Adam from fastNLP.core.callback import Callback, GradientClipCallback from fastNLP.core.trainer import Trainer -from fastNLP.io.pipe.coreference import CoreferencePipe +from fastNLP.io.pipe.coreference import CoReferencePipe from fastNLP.core.const import Const from reproduction.coreference_resolution.model.config import Config from reproduction.coreference_resolution.model.model_re import Model from reproduction.coreference_resolution.model.softmax_loss import SoftmaxLoss from reproduction.coreference_resolution.model.metric import CRMetric -from fastNLP import SequentialSampler -from fastNLP import cache_results -# torch.backends.cudnn.benchmark = False -# torch.backends.cudnn.deterministic = True - class LRCallback(Callback): def __init__(self, parameters, decay_rate=1e-3): super().__init__() @@ -38,15 +31,13 @@ if __name__ == "__main__": print(config) - # @cache_results('cache.pkl') def cache(): - bundle = CoreferencePipe(config).process_from_file({'train': config.train_path, 'dev': config.dev_path,'test': config.test_path}) + bundle = CoReferencePipe(config).process_from_file({'train': config.train_path, 'dev': config.dev_path, + 'test': config.test_path}) return bundle data_bundle = cache() - print("数据集划分：\ntrain:", str(len(data_bundle.get_dataset("train"))), - "\ndev:" + str(len(data_bundle.get_dataset("dev"))) + "\ntest:" + str(len(data_bundle.get_dataset('test')))) - # print(data_info) - model = Model(data_bundle.get_vocab(Const.INPUT), config) + print(data_bundle) + model = Model(data_bundle.get_vocab(Const.INPUTS(0)), config) print(model) loss = SoftmaxLoss() @@ -59,9 +50,10 @@ if __name__ == "__main__": trainer = Trainer(model=model, train_data=data_bundle.datasets["train"], dev_data=data_bundle.datasets["dev"], loss=loss, metrics=metric, check_code_level=-1, sampler=None, - batch_size=1, device=torch.device("cuda:" + config.cuda), metric_key='f', n_epochs=config.epoch, + batch_size=1, device=torch.device("cuda:" + config.cuda) if torch.cuda.is_available() else None, + metric_key='f', n_epochs=config.epoch, optimizer=optim, - save_path= None, + save_path=None, callbacks=[lr_decay_callback, GradientClipCallback(clip_value=5)]) print() diff --git a/reproduction/coreference_resolution/valid.py b/reproduction/coreference_resolution/valid.py index a528ea06..e79642b8 100644 --- a/reproduction/coreference_resolution/valid.py +++ b/reproduction/coreference_resolution/valid.py @@ -1,7 +1,7 @@ import torch from reproduction.coreference_resolution.model.config import Config from reproduction.coreference_resolution.model.metric import CRMetric -from fastNLP.io.pipe.coreference import CoreferencePipe +from fastNLP.io.pipe.coreference import CoReferencePipe from fastNLP import Tester import argparse @@ -13,7 +13,7 @@ if __name__=='__main__': args = parser.parse_args() config = Config() - bundle = CoreferencePipe(Config()).process_from_file( + bundle = CoReferencePipe(Config()).process_from_file( {'train': config.train_path, 'dev': config.dev_path, 'test': config.test_path}) metirc = CRMetric() model = torch.load(args.path) diff --git a/test/data_for_tests/coreference/coreference_dev.json b/test/data_for_tests/io/coreference/coreference_dev.json similarity index 100% rename from test/data_for_tests/coreference/coreference_dev.json rename to test/data_for_tests/io/coreference/coreference_dev.json diff --git a/test/data_for_tests/coreference/coreference_test.json b/test/data_for_tests/io/coreference/coreference_test.json similarity index 100% rename from test/data_for_tests/coreference/coreference_test.json rename to test/data_for_tests/io/coreference/coreference_test.json diff --git a/test/data_for_tests/coreference/coreference_train.json b/test/data_for_tests/io/coreference/coreference_train.json similarity index 100% rename from test/data_for_tests/coreference/coreference_train.json rename to test/data_for_tests/io/coreference/coreference_train.json diff --git a/test/io/loader/test_coreference_loader.py b/test/io/loader/test_coreference_loader.py index d827e947..02f3a1c5 100644 --- a/test/io/loader/test_coreference_loader.py +++ b/test/io/loader/test_coreference_loader.py @@ -1,16 +1,26 @@ -from fastNLP.io.loader.coreference import CRLoader +from fastNLP.io.loader.coreference import CoReferenceLoader import unittest + class TestCR(unittest.TestCase): def test_load(self): - test_root = "test/data_for_tests/coreference/" + test_root = "test/data_for_tests/io/coreference/" train_path = test_root+"coreference_train.json" dev_path = test_root+"coreference_dev.json" test_path = test_root+"coreference_test.json" - paths = {"train": train_path,"dev":dev_path,"test":test_path} + paths = {"train": train_path, "dev": dev_path, "test": test_path} - bundle1 = CRLoader().load(paths) - bundle2 = CRLoader().load(test_root) + bundle1 = CoReferenceLoader().load(paths) + bundle2 = CoReferenceLoader().load(test_root) print(bundle1) - print(bundle2) \ No newline at end of file + print(bundle2) + + self.assertEqual(bundle1.num_dataset, 3) + self.assertEqual(bundle2.num_dataset, 3) + self.assertEqual(bundle1.num_vocab, 0) + self.assertEqual(bundle2.num_vocab, 0) + + self.assertEqual(len(bundle1.get_dataset('train')), 1) + self.assertEqual(len(bundle1.get_dataset('dev')), 1) + self.assertEqual(len(bundle1.get_dataset('test')), 1) diff --git a/test/io/pipe/test_coreference.py b/test/io/pipe/test_coreference.py index 517be993..3a492419 100644 --- a/test/io/pipe/test_coreference.py +++ b/test/io/pipe/test_coreference.py @@ -1,5 +1,5 @@ import unittest -from fastNLP.io.pipe.coreference import CoreferencePipe +from fastNLP.io.pipe.coreference import CoReferencePipe class TestCR(unittest.TestCase): @@ -11,14 +11,23 @@ class TestCR(unittest.TestCase): char_path = None config = Config() - file_root_path = "test/data_for_tests/coreference/" + file_root_path = "test/data_for_tests/io/coreference/" train_path = file_root_path + "coreference_train.json" dev_path = file_root_path + "coreference_dev.json" test_path = file_root_path + "coreference_test.json" paths = {"train": train_path, "dev": dev_path, "test": test_path} - bundle1 = CoreferencePipe(config).process_from_file(paths) - bundle2 = CoreferencePipe(config).process_from_file(file_root_path) + bundle1 = CoReferencePipe(config).process_from_file(paths) + bundle2 = CoReferencePipe(config).process_from_file(file_root_path) print(bundle1) - print(bundle2) \ No newline at end of file + print(bundle2) + self.assertEqual(bundle1.num_dataset, 3) + self.assertEqual(bundle2.num_dataset, 3) + self.assertEqual(bundle1.num_vocab, 1) + self.assertEqual(bundle2.num_vocab, 1) + + self.assertEqual(len(bundle1.get_dataset('train')), 1) + self.assertEqual(len(bundle1.get_dataset('dev')), 1) + self.assertEqual(len(bundle1.get_dataset('test')), 1) + self.assertEqual(len(bundle1.get_vocab('words1')), 84) From f22991698ad02a459e0a22ef3f486e29fc112f72 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 02:19:27 +0800 Subject: [PATCH 71/92] add assert in test_elmo_embedding --- test/embeddings/test_elmo_embedding.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/embeddings/test_elmo_embedding.py b/test/embeddings/test_elmo_embedding.py index bfb31659..ed6910b4 100644 --- a/test/embeddings/test_elmo_embedding.py +++ b/test/embeddings/test_elmo_embedding.py @@ -25,6 +25,7 @@ class TestRunElmo(unittest.TestCase): words = torch.LongTensor([[0, 1, 2]]) hidden = elmo_embed(words) print(hidden.size()) + self.assertEqual(hidden.size(), (1, 3, elmo_embed.embedding_dim)) def test_elmo_embedding_layer_assertion(self): vocab = Vocabulary().add_word_lst("This is a test .".split()) From a3b4d5e76e10f0817588d7301f2e52be32b84fd0 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 14:10:01 +0800 Subject: [PATCH 72/92] remove tensorboard logs after testing TensorboardCallback --- test/core/test_callbacks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/core/test_callbacks.py b/test/core/test_callbacks.py index 909295c0..5fc8cbf8 100644 --- a/test/core/test_callbacks.py +++ b/test/core/test_callbacks.py @@ -85,6 +85,9 @@ class TestCallback(unittest.TestCase): metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False, callbacks=[TensorboardCallback("loss", "metric")], check_code_level=2) trainer.train() + import os + import shutil + shutil.rmtree(os.path.join("./", 'tensorboard_logs_{}'.format(trainer.start_time))) def test_readonly_property(self): from fastNLP.core.callback import Callback From 7726c16959de87057966162bfbfdf0e0eae14a7c Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 16:48:55 +0800 Subject: [PATCH 73/92] Update tester.py --- fastNLP/core/tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastNLP/core/tester.py b/fastNLP/core/tester.py index d1d5d41e..d9c2e2f7 100644 --- a/fastNLP/core/tester.py +++ b/fastNLP/core/tester.py @@ -182,8 +182,8 @@ class Tester(object): pbar.close() end_time = time.time() test_str = f'Evaluate data in {round(end_time - start_time, 2)} seconds!' - # pbar.write(test_str) - self.logger.info(test_str) + if self.verbose >= 0: + self.logger.info(test_str) except _CheckError as e: prev_func_signature = _get_func_signature(self._predict_func) _check_loss_evaluate(prev_func_signature=prev_func_signature, func_signature=e.func_signature, From 33cbb5b540ecf2837d93f28650fd679e39f0dd27 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 16:51:19 +0800 Subject: [PATCH 74/92] Create bert_embedding_tutorial.ipynb --- tutorials/bert_embedding_tutorial.ipynb | 470 ++++++++++++++++++++++++ 1 file changed, 470 insertions(+) create mode 100644 tutorials/bert_embedding_tutorial.ipynb diff --git a/tutorials/bert_embedding_tutorial.ipynb b/tutorials/bert_embedding_tutorial.ipynb new file mode 100644 index 00000000..a893fef0 --- /dev/null +++ b/tutorials/bert_embedding_tutorial.ipynb @@ -0,0 +1,470 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BertEmbedding的各种用法\n", + "fastNLP的BertEmbedding以pytorch-transformer.BertModel的代码为基础，是一个使用BERT对words进行编码的Embedding。\n", + "\n", + "使用BertEmbedding和fastNLP.models.bert里面模型可以搭建BERT应用到五种下游任务的模型。\n", + "\n", + "*预训练好的Embedding参数及数据集的介绍和自动下载功能见 [Embedding教程](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_3_embedding.html) 和 [数据处理教程](https://fastnlp.readthedocs.io/zh/latest/tutorials/tutorial_4_load_dataset.html)。*\n", + "\n", + "## 1. BERT for Squence Classification\n", + "在文本分类任务中，我们采用SST数据集作为例子来介绍BertEmbedding的使用方法。" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "import torch\n", + "warnings.filterwarnings(\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "In total 3 datasets:\n", + "\ttest has 2210 instances.\n", + "\ttrain has 8544 instances.\n", + "\tdev has 1101 instances.\n", + "In total 2 vocabs:\n", + "\twords has 21701 entries.\n", + "\ttarget has 5 entries." + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 载入数据集\n", + "from fastNLP.io import SSTPipe\n", + "data_bundle = SSTPipe(subtree=False, train_subtree=False, lower=False, tokenizer='raw').process_from_file()\n", + "data_bundle" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading vocabulary file /remote-home/source/fastnlp_caches/embedding/bert-base-cased/vocab.txt\n", + "Load pre-trained BERT parameters from file /remote-home/source/fastnlp_caches/embedding/bert-base-cased/pytorch_model.bin.\n", + "Start to generate word pieces for word.\n", + "Found(Or segment into word pieces) 21701 words out of 21701.\n" + ] + } + ], + "source": [ + "# 载入BertEmbedding\n", + "from fastNLP.embeddings import BertEmbedding\n", + "embed = BertEmbedding(data_bundle.get_vocab('words'), model_dir_or_name='en-base-cased', include_cls_sep=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# 载入模型\n", + "from fastNLP.models import BertForSequenceClassification\n", + "model = BertForSequenceClassification(embed, len(data_bundle.get_vocab('target')))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input fields after batch(if batch size is 2):\n", + "\twords: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 37]) \n", + "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "target fields after batch(if batch size is 2):\n", + "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2019-09-11-17-35-26\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=268), HTML(value='')), layout=Layout(display=…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=18), HTML(value='')), layout=Layout(display='…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluate data in 2.08 seconds!\n", + "Evaluation on dev at Epoch 1/2. Step:134/268: \n", + "AccuracyMetric: acc=0.459582\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=18), HTML(value='')), layout=Layout(display='…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluate data in 2.2 seconds!\n", + "Evaluation on dev at Epoch 2/2. Step:268/268: \n", + "AccuracyMetric: acc=0.468665\n", + "\n", + "\n", + "In Epoch:2/Step:268, got best dev performance:\n", + "AccuracyMetric: acc=0.468665\n", + "Reloaded the best model.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'best_eval': {'AccuracyMetric': {'acc': 0.468665}},\n", + " 'best_epoch': 2,\n", + " 'best_step': 268,\n", + " 'seconds': 114.5}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 训练模型\n", + "from fastNLP import Trainer, CrossEntropyLoss, AccuracyMetric, Adam\n", + "trainer = Trainer(data_bundle.get_dataset('train'), model, \n", + " optimizer=Adam(model_params=model.parameters(), lr=2e-5), \n", + " loss=CrossEntropyLoss(), device=[0],\n", + " batch_size=64, dev_data=data_bundle.get_dataset('dev'), \n", + " metrics=AccuracyMetric(), n_epochs=2, print_every=1)\n", + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=18), HTML(value='')), layout=Layout(display='…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 4.52 seconds!\n", + "[tester] \n", + "AccuracyMetric: acc=0.504072\n" + ] + }, + { + "data": { + "text/plain": [ + "{'AccuracyMetric': {'acc': 0.504072}}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 测试结果并删除模型\n", + "from fastNLP import Tester\n", + "tester = Tester(data_bundle.get_dataset('test'), model, batch_size=128, metrics=AccuracyMetric())\n", + "tester.test()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## 2. BERT for Sentence Matching\n", + "在Matching任务中，我们采用RTE数据集作为例子来介绍BertEmbedding的使用方法。" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "In total 3 datasets:\n", + "\ttest has 3000 instances.\n", + "\ttrain has 2490 instances.\n", + "\tdev has 277 instances.\n", + "In total 2 vocabs:\n", + "\twords has 41281 entries.\n", + "\ttarget has 2 entries." + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 载入数据集\n", + "from fastNLP.io import RTEBertPipe\n", + "data_bundle = RTEBertPipe(lower=False, tokenizer='raw').process_from_file()\n", + "data_bundle" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading vocabulary file /remote-home/source/fastnlp_caches/embedding/bert-base-cased/vocab.txt\n", + "Load pre-trained BERT parameters from file /remote-home/source/fastnlp_caches/embedding/bert-base-cased/pytorch_model.bin.\n", + "Start to generate word pieces for word.\n", + "Found(Or segment into word pieces) 41279 words out of 41281.\n" + ] + } + ], + "source": [ + "# 载入BertEmbedding\n", + "from fastNLP.embeddings import BertEmbedding\n", + "embed = BertEmbedding(data_bundle.get_vocab('words'), model_dir_or_name='en-base-cased', include_cls_sep=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# 载入模型\n", + "from fastNLP.models import BertForSentenceMatching\n", + "model = BertForSentenceMatching(embed, len(data_bundle.get_vocab('target')))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input fields after batch(if batch size is 2):\n", + "\twords: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 45]) \n", + "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "target fields after batch(if batch size is 2):\n", + "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2019-09-11-17-37-36\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=312), HTML(value='')), layout=Layout(display=…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=18), HTML(value='')), layout=Layout(display='…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluate data in 1.72 seconds!\n", + "Evaluation on dev at Epoch 1/2. Step:156/312: \n", + "AccuracyMetric: acc=0.624549\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=18), HTML(value='')), layout=Layout(display='…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluate data in 1.74 seconds!\n", + "Evaluation on dev at Epoch 2/2. Step:312/312: \n", + "AccuracyMetric: acc=0.649819\n", + "\n", + "\n", + "In Epoch:2/Step:312, got best dev performance:\n", + "AccuracyMetric: acc=0.649819\n", + "Reloaded the best model.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'best_eval': {'AccuracyMetric': {'acc': 0.649819}},\n", + " 'best_epoch': 2,\n", + " 'best_step': 312,\n", + " 'seconds': 109.87}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 训练模型\n", + "from fastNLP import Trainer, CrossEntropyLoss, AccuracyMetric, Adam\n", + "trainer = Trainer(data_bundle.get_dataset('train'), model, \n", + " optimizer=Adam(model_params=model.parameters(), lr=2e-5), \n", + " loss=CrossEntropyLoss(), device=[0],\n", + " batch_size=16, dev_data=data_bundle.get_dataset('dev'), \n", + " metrics=AccuracyMetric(), n_epochs=2, print_every=1)\n", + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 38d5b36be1b8daf177901affc83fc9b51625d6f3 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Wed, 11 Sep 2019 17:41:46 +0800 Subject: [PATCH 75/92] Update test_callbacks.py --- test/core/test_callbacks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/core/test_callbacks.py b/test/core/test_callbacks.py index 5fc8cbf8..98dd422d 100644 --- a/test/core/test_callbacks.py +++ b/test/core/test_callbacks.py @@ -87,7 +87,9 @@ class TestCallback(unittest.TestCase): trainer.train() import os import shutil - shutil.rmtree(os.path.join("./", 'tensorboard_logs_{}'.format(trainer.start_time))) + path = os.path.join("./", 'tensorboard_logs_{}'.format(trainer.start_time)) + if os.path.exists(path): + shutil.rmtree(path) def test_readonly_property(self): from fastNLP.core.callback import Callback From 64fc8bc1e5d897e494ddb87cb6703a567d7dff2d Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Thu, 12 Sep 2019 02:59:45 +0800 Subject: [PATCH 76/92] 1. update classification and matching loader and pipe; 2. add data and test codes for testing classification and matching loader and pipe. --- fastNLP/io/pipe/classification.py | 14 ++++++- fastNLP/io/pipe/matching.py | 29 +++++++++++++- fastNLP/io/pipe/utils.py | 14 ++++++- test/data_for_tests/io/MNLI/dev_matched.tsv | 6 +++ .../data_for_tests/io/MNLI/dev_mismatched.tsv | 6 +++ test/data_for_tests/io/MNLI/test_matched.tsv | 6 +++ .../io/MNLI/test_mismatched.tsv | 6 +++ test/data_for_tests/io/MNLI/train.tsv | 7 ++++ test/data_for_tests/io/QNLI/dev.tsv | 6 +++ test/data_for_tests/io/QNLI/test.tsv | 6 +++ test/data_for_tests/io/QNLI/train.tsv | 6 +++ test/data_for_tests/io/RTE/dev.tsv | 6 +++ test/data_for_tests/io/RTE/test.tsv | 6 +++ test/data_for_tests/io/RTE/train.tsv | 6 +++ .../data_for_tests/io/SNLI/snli_1.0_dev.jsonl | 5 +++ .../io/SNLI/snli_1.0_test.jsonl | 5 +++ .../io/SNLI/snli_1.0_train.jsonl | 5 +++ test/data_for_tests/io/SST-2/dev.tsv | 6 +++ test/data_for_tests/io/SST-2/test.tsv | 6 +++ test/data_for_tests/io/SST-2/train.tsv | 6 +++ test/data_for_tests/io/SST/dev.txt | 6 +++ test/data_for_tests/io/SST/test.txt | 6 +++ test/data_for_tests/io/SST/train.txt | 6 +++ test/data_for_tests/io/imdb/dev.txt | 8 +++- test/data_for_tests/io/imdb/test.txt | 4 ++ test/data_for_tests/io/imdb/train.txt | 6 ++- .../io/yelp_review_full/dev.csv | 6 +++ .../io/yelp_review_full/test.csv | 6 +++ .../io/yelp_review_full/train.csv | 6 +++ .../io/yelp_review_polarity/dev.csv | 6 +++ .../io/yelp_review_polarity/test.csv | 6 +++ .../io/yelp_review_polarity/train.csv | 6 +++ test/io/loader/test_classification_loader.py | 30 +++++++++++++-- test/io/loader/test_matching_loader.py | 24 ++++++++++-- test/io/pipe/test_classification.py | 35 ++++++++++++++++- test/io/pipe/test_matching.py | 38 +++++++++++++++++-- 36 files changed, 338 insertions(+), 18 deletions(-) create mode 100755 test/data_for_tests/io/MNLI/dev_matched.tsv create mode 100755 test/data_for_tests/io/MNLI/dev_mismatched.tsv create mode 100755 test/data_for_tests/io/MNLI/test_matched.tsv create mode 100755 test/data_for_tests/io/MNLI/test_mismatched.tsv create mode 100755 test/data_for_tests/io/MNLI/train.tsv create mode 100755 test/data_for_tests/io/QNLI/dev.tsv create mode 100755 test/data_for_tests/io/QNLI/test.tsv create mode 100755 test/data_for_tests/io/QNLI/train.tsv create mode 100644 test/data_for_tests/io/RTE/dev.tsv create mode 100644 test/data_for_tests/io/RTE/test.tsv create mode 100644 test/data_for_tests/io/RTE/train.tsv create mode 100755 test/data_for_tests/io/SNLI/snli_1.0_dev.jsonl create mode 100755 test/data_for_tests/io/SNLI/snli_1.0_test.jsonl create mode 100755 test/data_for_tests/io/SNLI/snli_1.0_train.jsonl create mode 100755 test/data_for_tests/io/SST-2/dev.tsv create mode 100755 test/data_for_tests/io/SST-2/test.tsv create mode 100755 test/data_for_tests/io/SST-2/train.tsv create mode 100755 test/data_for_tests/io/SST/dev.txt create mode 100755 test/data_for_tests/io/SST/test.txt create mode 100755 test/data_for_tests/io/SST/train.txt create mode 100755 test/data_for_tests/io/yelp_review_full/dev.csv create mode 100755 test/data_for_tests/io/yelp_review_full/test.csv create mode 100755 test/data_for_tests/io/yelp_review_full/train.csv create mode 100755 test/data_for_tests/io/yelp_review_polarity/dev.csv create mode 100755 test/data_for_tests/io/yelp_review_polarity/test.csv create mode 100755 test/data_for_tests/io/yelp_review_polarity/train.csv diff --git a/fastNLP/io/pipe/classification.py b/fastNLP/io/pipe/classification.py index 450c2058..b1d150aa 100644 --- a/fastNLP/io/pipe/classification.py +++ b/fastNLP/io/pipe/classification.py @@ -10,6 +10,7 @@ __all__ = [ ] import re +import warnings from nltk import Tree @@ -22,6 +23,7 @@ from ...core.const import Const from ...core.dataset import DataSet from ...core.instance import Instance from ...core.vocabulary import Vocabulary +from ...core._logger import logger nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') @@ -373,7 +375,17 @@ class SST2Pipe(_CLSPipe): src_vocab.index_dataset(*data_bundle.datasets.values(), field_name=Const.INPUT) tgt_vocab = Vocabulary(unknown=None, padding=None) - tgt_vocab.from_dataset(data_bundle.datasets['train'], field_name=Const.TARGET) + tgt_vocab.from_dataset(*[ds for name, ds in data_bundle.iter_datasets() if 'train' in name], + field_name=Const.TARGET, + no_create_entry_dataset=[ds for name, ds in data_bundle.iter_datasets() + if ('train' not in name) and (ds.has_field(Const.TARGET))] + ) + if len(tgt_vocab._no_create_word) > 0: + warn_msg = f"There are {len(tgt_vocab._no_create_word)} target labels" \ + f" in {[name for name in data_bundle.datasets.keys() if 'train' not in name]} " \ + f"data set but not in train data set!." + warnings.warn(warn_msg) + logger.warn(warn_msg) datasets = [] for name, dataset in data_bundle.datasets.items(): if dataset.has_field(Const.TARGET): diff --git a/fastNLP/io/pipe/matching.py b/fastNLP/io/pipe/matching.py index 3db79aef..7620a556 100644 --- a/fastNLP/io/pipe/matching.py +++ b/fastNLP/io/pipe/matching.py @@ -15,11 +15,14 @@ __all__ = [ "MNLIPipe", ] +import warnings + from .pipe import Pipe from .utils import get_tokenizer from ..loader.matching import SNLILoader, MNLILoader, QNLILoader, RTELoader, QuoraLoader from ...core.const import Const from ...core.vocabulary import Vocabulary +from ...core._logger import logger class MatchingBertPipe(Pipe): @@ -101,7 +104,18 @@ class MatchingBertPipe(Pipe): word_vocab.index_dataset(*data_bundle.datasets.values(), field_name=Const.INPUT) target_vocab = Vocabulary(padding=None, unknown=None) - target_vocab.from_dataset(data_bundle.datasets['train'], field_name=Const.TARGET) + target_vocab.from_dataset(*[ds for name, ds in data_bundle.iter_datasets() if 'train' in name], + field_name=Const.TARGET, + no_create_entry_dataset=[ds for name, ds in data_bundle.iter_datasets() + if ('train' not in name) and (ds.has_field(Const.TARGET))] + ) + if len(target_vocab._no_create_word) > 0: + warn_msg = f"There are {len(tgt_vocab._no_create_word)} target labels" \ + f" in {[name for name in data_bundle.datasets.keys() if 'train' not in name]} " \ + f"data set but not in train data set!." + warnings.warn(warn_msg) + logger.warn(warn_msg) + has_target_datasets = [dataset for name, dataset in data_bundle.datasets.items() if dataset.has_field(Const.TARGET)] target_vocab.index_dataset(*has_target_datasets, field_name=Const.TARGET) @@ -227,7 +241,18 @@ class MatchingPipe(Pipe): word_vocab.index_dataset(*data_bundle.datasets.values(), field_name=[Const.INPUTS(0), Const.INPUTS(1)]) target_vocab = Vocabulary(padding=None, unknown=None) - target_vocab.from_dataset(data_bundle.datasets['train'], field_name=Const.TARGET) + target_vocab.from_dataset(*[ds for name, ds in data_bundle.iter_datasets() if 'train' in name], + field_name=Const.TARGET, + no_create_entry_dataset=[ds for name, ds in data_bundle.iter_datasets() + if ('train' not in name) and (ds.has_field(Const.TARGET))] + ) + if len(target_vocab._no_create_word) > 0: + warn_msg = f"There are {len(tgt_vocab._no_create_word)} target labels" \ + f" in {[name for name in data_bundle.datasets.keys() if 'train' not in name]} " \ + f"data set but not in train data set!." + warnings.warn(warn_msg) + logger.warn(warn_msg) + has_target_datasets = [dataset for name, dataset in data_bundle.datasets.items() if dataset.has_field(Const.TARGET)] target_vocab.index_dataset(*has_target_datasets, field_name=Const.TARGET) diff --git a/fastNLP/io/pipe/utils.py b/fastNLP/io/pipe/utils.py index ea7e0aa8..3db9c4fe 100644 --- a/fastNLP/io/pipe/utils.py +++ b/fastNLP/io/pipe/utils.py @@ -7,9 +7,11 @@ __all__ = [ ] from typing import List +import warnings from ...core.const import Const from ...core.vocabulary import Vocabulary +from ...core._logger import logger def iob2(tags: List[str]) -> List[str]: @@ -111,7 +113,17 @@ def _indexize(data_bundle, input_field_names=Const.INPUT, target_field_names=Con for target_field_name in target_field_names: tgt_vocab = Vocabulary(unknown=None, padding=None) - tgt_vocab.from_dataset(data_bundle.datasets['train'], field_name=target_field_name) + tgt_vocab.from_dataset(*[ds for name, ds in data_bundle.iter_datasets() if 'train' in name], + field_name=Const.TARGET, + no_create_entry_dataset=[ds for name, ds in data_bundle.iter_datasets() + if ('train' not in name) and (ds.has_field(Const.TARGET))] + ) + if len(tgt_vocab._no_create_word) > 0: + warn_msg = f"There are {len(tgt_vocab._no_create_word)} target labels" \ + f" in {[name for name in data_bundle.datasets.keys() if 'train' not in name]} " \ + f"data set but not in train data set!." + warnings.warn(warn_msg) + logger.warn(warn_msg) tgt_vocab.index_dataset(*data_bundle.datasets.values(), field_name=target_field_name) data_bundle.set_vocab(tgt_vocab, target_field_name) diff --git a/test/data_for_tests/io/MNLI/dev_matched.tsv b/test/data_for_tests/io/MNLI/dev_matched.tsv new file mode 100755 index 00000000..ace2dd27 --- /dev/null +++ b/test/data_for_tests/io/MNLI/dev_matched.tsv @@ -0,0 +1,6 @@ +index promptID pairID genre sentence1_binary_parse sentence2_binary_parse sentence1_parse sentence2_parse sentence1 sentence2 label1 label2 label3 label4 label5 gold_label +0 63735 63735n slate ( ( The ( new rights ) ) ( are ( nice enough ) ) ) ( Everyone ( really ( likes ( the ( newest benefits ) ) ) ) ) (ROOT (S (NP (DT The) (JJ new) (NNS rights)) (VP (VBP are) (ADJP (JJ nice) (RB enough))))) (ROOT (S (NP (NN Everyone)) (VP (ADVP (RB really)) (VBZ likes) (NP (DT the) (JJS newest) (NNS benefits))))) The new rights are nice enough Everyone really likes the newest benefits neutral entailment neutral neutral neutral neutral +1 91383 91383c government ( ( This site ) ( ( includes ( ( ( ( a list ) ( of ( all ( award winners ) ) ) ) and ) ( ( a ( searchable database ) ) ( of ( Government ( Executive articles ) ) ) ) ) ) . ) ) ( ( ( The ( Government ( Executive articles ) ) ) ( housed ( on ( the website ) ) ) ) ( ( ( are not ) ( able ( to ( be searched ) ) ) ) . ) ) (ROOT (S (NP (DT This) (NN site)) (VP (VBZ includes) (NP (NP (NP (DT a) (NN list)) (PP (IN of) (NP (DT all) (NN award) (NNS winners)))) (CC and) (NP (NP (DT a) (JJ searchable) (NN database)) (PP (IN of) (NP (NNP Government) (NNP Executive) (NNS articles)))))) (. .))) (ROOT (S (NP (NP (DT The) (NNP Government) (NNP Executive) (NNS articles)) (VP (VBN housed) (PP (IN on) (NP (DT the) (NN website))))) (VP (VBP are) (RB not) (ADJP (JJ able) (S (VP (TO to) (VP (VB be) (ADJP (JJ searched))))))) (. .))) This site includes a list of all award winners and a searchable database of Government Executive articles. The Government Executive articles housed on the website are not able to be searched. contradiction contradiction contradiction contradiction contradiction contradiction +2 755 755e telephone ( ( ( ( uh ( i ( ( do n't ) ( know ( ( i i ) ( have ( ( mixed emotions ) ( about ( him ( ( uh sometimes ) ( i ( like him ) ) ) ) ) ) ) ) ) ) ) ) but ) ( ( at ( the ( same times ) ) ) ( i ( love ( to ( see somebody ) ) ) ) ) ) ( beat him ) ) ( I ( ( ( ( ( ( like him ) ( for ( the ( most part ) ) ) ) , ) but ) ( ( would still ) ( enjoy ( seeing ( someone ( beat him ) ) ) ) ) ) . ) ) (ROOT (SINV (S (S (INTJ (UH uh)) (NP (FW i)) (VP (VBP do) (RB n't) (VP (VB know) (NP (NP (FW i) (FW i)) (SBAR (S (VP (VBP have) (VP (VBN mixed) (NP (NNS emotions)) (PP (IN about) (S (NP (PRP him)) (VP (VBG uh) (ADVP (RB sometimes)) (NP (NP (FW i)) (PP (IN like) (NP (PRP him))))))))))))))) (CC but) (S (PP (IN at) (NP (DT the) (JJ same) (NNS times))) (NP (FW i)) (VP (VBP love) (S (VP (TO to) (VP (VB see) (NP (NN somebody)))))))) (VP (VBD beat)) (NP (PRP him)))) (ROOT (S (NP (PRP I)) (VP (VP (VBP like) (NP (PRP him)) (PP (IN for) (NP (DT the) (JJS most) (NN part)))) (, ,) (CC but) (VP (MD would) (ADVP (RB still)) (VP (VB enjoy) (S (VP (VBG seeing) (S (NP (NN someone)) (VP (VB beat) (NP (PRP him))))))))) (. .))) uh i don't know i i have mixed emotions about him uh sometimes i like him but at the same times i love to see somebody beat him I like him for the most part, but would still enjoy seeing someone beat him. entailment entailment entailment entailment entailment entailment +3 78013 78013c telephone ( yeah ( ( i i ) ( think ( ( my ( favorite restaurant ) ) ( ( is always ) ( been ( ( the ( one closest ) ) ( you ( ( know ( the closest ) ) ( ( as long ) ( as ( it ( 's ( it ( meets ( ( the ( minimum criteria ) ) ( you ( know ( of ( good food ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ( ( My ( favorite restaurants ) ) ( ( ( ( are always ) ( ( ( ( ( at least ) a ) hundred ) miles ) away ) ) ( from ( my house ) ) ) . ) ) (ROOT (S (VP (VB yeah) (NP (NP (FW i) (FW i)) (SBAR (S (VP (VBP think) (SBAR (S (NP (PRP$ my) (JJ favorite) (NN restaurant)) (VP (VBZ is) (ADVP (RB always)) (VP (VBN been) (NP (NP (DT the) (CD one) (JJS closest)) (SBAR (S (NP (PRP you)) (VP (VBP know) (NP (DT the) (JJS closest)) (ADVP (ADVP (RB as) (RB long)) (SBAR (IN as) (S (NP (PRP it)) (VP (VBZ 's) (SBAR (S (NP (PRP it)) (VP (VBZ meets) (NP (NP (DT the) (JJ minimum) (NNS criteria)) (SBAR (S (NP (PRP you)) (VP (VBP know) (PP (IN of) (NP (JJ good) (NN food))))))))))))))))))))))))))))) (ROOT (S (NP (PRP$ My) (JJ favorite) (NNS restaurants)) (VP (VBP are) (ADVP (RB always)) (ADVP (NP (QP (IN at) (JJS least) (DT a) (CD hundred)) (NNS miles)) (RB away)) (PP (IN from) (NP (PRP$ my) (NN house)))) (. .))) yeah i i think my favorite restaurant is always been the one closest you know the closest as long as it's it meets the minimum criteria you know of good food My favorite restaurants are always at least a hundred miles away from my house. contradiction contradiction contradiction contradiction contradiction contradiction +4 96377 96377c telephone ( i ( ( do n't ) ( know ( um ( do ( you ( do ( ( a lot ) ( of camping ) ) ) ) ) ) ) ) ) ( I ( ( know exactly ) . ) ) (ROOT (S (NP (FW i)) (VP (VBP do) (RB n't) (VP (VB know) (SBAR (S (NP (NN um)) (VP (VBP do) (SBAR (S (NP (PRP you)) (VP (VBP do) (NP (NP (DT a) (NN lot)) (PP (IN of) (NP (NN camping)))))))))))))) (ROOT (S (NP (PRP I)) (VP (VBP know) (ADVP (RB exactly))) (. .))) i don't know um do you do a lot of camping I know exactly. contradiction contradiction contradiction contradiction contradiction contradiction diff --git a/test/data_for_tests/io/MNLI/dev_mismatched.tsv b/test/data_for_tests/io/MNLI/dev_mismatched.tsv new file mode 100755 index 00000000..a1da8897 --- /dev/null +++ b/test/data_for_tests/io/MNLI/dev_mismatched.tsv @@ -0,0 +1,6 @@ +index promptID pairID genre sentence1_binary_parse sentence2_binary_parse sentence1_parse sentence2_parse sentence1 sentence2 label1 label2 label3 label4 label5 gold_label +0 75290 75290c letters ( ( Your contribution ) ( ( helped ( make ( it ( possible ( for ( us ( to ( ( provide ( our students ) ) ( with ( a ( quality education ) ) ) ) ) ) ) ) ) ) ) . ) ) ( ( Your contributions ) ( ( were ( of ( ( no help ) ( with ( ( our ( students ' ) ) education ) ) ) ) ) . ) ) (ROOT (S (NP (PRP$ Your) (NN contribution)) (VP (VBD helped) (VP (VB make) (S (NP (PRP it)) (ADJP (JJ possible)) (SBAR (IN for) (S (NP (PRP us)) (VP (TO to) (VP (VB provide) (NP (PRP$ our) (NNS students)) (PP (IN with) (NP (DT a) (NN quality) (NN education)))))))))) (. .))) (ROOT (S (NP (PRP$ Your) (NNS contributions)) (VP (VBD were) (PP (IN of) (NP (NP (DT no) (NN help)) (PP (IN with) (NP (NP (PRP$ our) (NNS students) (POS ')) (NN education)))))) (. .))) Your contribution helped make it possible for us to provide our students with a quality education. Your contributions were of no help with our students' education. contradiction contradiction contradiction contradiction contradiction contradiction +1 133794 133794c verbatim ( ( ( ( ( ( The answer ) ( ( ( ( has nothing ) ( to ( do ( with ( their cause ) ) ) ) ) , ) however ) ) , ) but ) ( ( with ( ( ( ( ( ( ( ( the ( simple fact ) ) ( that ( dictionaries ( ( are not ) ( exercises ( in ( bi-unique substitutability ) ) ) ) ) ) ) ; ) ( in ( ( ( other words ) , ) ( if ( ( one ( of ( ( the senses ) ( of run ) ) ) ) ( ( is ` ) ( ( ( ( operate ' ) -LRB- ) ( as ( in ( She ( runs ( an ( engine factory ) ) ) ) ) ) ) -RRB- ) ) ) ) ) ) ) , ) ( that ( ( does not ) ( ( make it ) ( ( valid ( to ( assume ( that ( one ( can ( substitute ( ( operate ( for run ) ) ( in ( We ( ( run ( in ( ( the marathon ) ( every year ) ) ) ) . ) ) ) ) ) ) ) ) ) ) ) ( Although ( ( ( ( recognizing this ) ( as ( ( a shortcoming ) ( of dictionaries ) ) ) ) and ) ( ( ( assigning it ) arbitrarily ) ( to ( what ( , ( ( for ( lack ( of ( a ( better term ) ) ) ) ) ( , ( we ( might ( call ( ( the genius ) ( of ( the language ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) , ) ( might ( seem ( trivial ( to ( the ( casual observer ) ) ) ) ) ) ) ) ( , ( it ( is ( ( a ( valid matter ) ) ( for ( concern ( in ( ( the realm ) ( of lexicology ) ) ) ) ) ) ) ) ) ) ) . ) ( Dictionaries ( ( ( are indeed ) ( exercises ( in ( bi-unique substitutability ) ) ) ) . ) ) (ROOT (S (S (NP (DT The) (NN answer)) (VP (VBZ has) (ADVP (NN nothing)) (S (VP (TO to) (VP (VB do) (PP (IN with) (NP (PRP$ their) (NN cause)))))) (, ,) (ADVP (RB however)))) (, ,) (CC but) (S (SBAR (IN with) (S (NP (NP (DT the) (JJ simple) (NN fact)) (SBAR (IN that) (S (NP (NNS dictionaries)) (VP (VBP are) (RB not) (NP (NP (NNS exercises)) (PP (IN in) (NP (JJ bi-unique) (NN substitutability))))))) (: ;) (PP (IN in) (NP (NP (JJ other) (NNS words)) (, ,) (SBAR (IN if) (S (NP (NP (CD one)) (PP (IN of) (NP (NP (DT the) (NNS senses)) (PP (IN of) (NP (NN run)))))) (VP (VBZ is) (`` `) (VP (VB operate) ('' ') (-LRB- -LRB-) (SBAR (RB as) (IN in) (S (NP (PRP She)) (VP (VBZ runs) (NP (DT an) (NN engine) (NN factory))))) (-RRB- -RRB-))))))) (, ,) (SBAR (WHNP (WDT that)) (S (VP (VBZ does) (RB not) (VP (VB make) (NP (PRP it)) (S (ADJP (JJ valid) (S (VP (TO to) (VP (VB assume) (SBAR (IN that) (S (NP (PRP one)) (VP (MD can) (VP (VB substitute) (VP (VB operate) (PP (IN for) (NP (NN run))) (SBAR (IN in) (S (NP (PRP We)) (VP (VB run) (PP (IN in) (NP (NP (DT the) (NN marathon)) (NP (DT every) (NN year)))) (. .))))))))))))) (SBAR (IN Although) (S (S (VP (VBG recognizing) (NP (DT this)) (PP (IN as) (NP (NP (DT a) (NN shortcoming)) (PP (IN of) (NP (NNS dictionaries))))))) (CC and) (S (VP (VBG assigning) (NP (PRP it)) (ADVP (RB arbitrarily)) (PP (TO to) (SBAR (WHNP (WP what)) (S (, ,) (PP (IN for) (NP (NP (NN lack)) (PP (IN of) (NP (DT a) (JJR better) (NN term))))) (, ,) (NP (PRP we)) (VP (MD might) (VP (VB call) (NP (NP (DT the) (NN genius)) (PP (IN of) (NP (DT the) (NN language)))))))))))))))))) (, ,)) (VP (MD might) (VP (VB seem) (ADJP (JJ trivial) (PP (TO to) (NP (DT the) (JJ casual) (NN observer)))))))) (, ,) (NP (PRP it)) (VP (VBZ is) (NP (NP (DT a) (JJ valid) (NN matter)) (PP (IN for) (NP (NP (NN concern)) (PP (IN in) (NP (NP (DT the) (NN realm)) (PP (IN of) (NP (NN lexicology)))))))))) (. .))) (ROOT (S (NP (NNS Dictionaries)) (VP (VBP are) (ADVP (RB indeed)) (NP (NP (NNS exercises)) (PP (IN in) (NP (JJ bi-unique) (NN substitutability))))) (. .))) The answer has nothing to do with their cause, however, but with the simple fact that dictionaries are not exercises in bi-unique substitutability; in other words, if one of the senses of run is `operate' (as in She runs an engine factory ), that does not make it valid to assume that one can substitute operate for run in We run in the marathon every year . Although recognizing this as a shortcoming of dictionaries and assigning it arbitrarily to what, for lack of a better term, we might call the genius of the language, might seem trivial to the casual observer, it is a valid matter for concern in the realm of lexicology. Dictionaries are indeed exercises in bi-unique substitutability. contradiction contradiction contradiction contradiction contradiction contradiction +2 3628 3628c verbatim ( We ( ( serve ( ( a ( classic ( Tuscan meal ) ) ) ( that ( includes ( ( a ( Florentine terrine ) ) ( made ( with ( dick ( and ( chicken livers ) ) ) ) ) ) ) ) ) ) . ) ) ( We ( ( serve ( ( a meal ) ( of ( Florentine terrine ) ) ) ) . ) ) (ROOT (S (NP (PRP We)) (VP (VBP serve) (NP (NP (DT a) (JJ classic) (NNP Tuscan) (NN meal)) (SBAR (WHNP (WDT that)) (S (VP (VBZ includes) (NP (NP (DT a) (JJ Florentine) (NN terrine)) (VP (VBN made) (PP (IN with) (NP (NN dick) (CC and) (NN chicken) (NNS livers)))))))))) (. .))) (ROOT (S (NP (PRP We)) (VP (VBP serve) (NP (NP (DT a) (NN meal)) (PP (IN of) (NP (NNP Florentine) (NN terrine))))) (. .))) We serve a classic Tuscan meal that includes a Florentine terrine made with dick and chicken livers. We serve a meal of Florentine terrine. contradiction neutral entailment entailment entailment entailment +3 89411 89411c letters ( ( ( A ( few months ) ) ago ) ( , ( ( ( ( Carl Newton ) and ) I ) ( ( ( wrote ( a letter ) ) ( asking ( you ( to ( ( consider ( a ( financial contribution ) ) ) ( to ( ( graduate Endodontics ) ( at ( Indiana University ) ) ) ) ) ) ) ) ) . ) ) ) ) ( ( ( ( Carl Newton ) and ) I ) ( ( ( have never ) ( ( had ( any ( other ( previous contact ) ) ) ) ( with you ) ) ) . ) ) (ROOT (S (ADVP (NP (DT A) (JJ few) (NNS months)) (RB ago)) (, ,) (NP (NP (NNP Carl) (NNP Newton)) (CC and) (NP (PRP I))) (VP (VBD wrote) (NP (DT a) (NN letter)) (S (VP (VBG asking) (S (NP (PRP you)) (VP (TO to) (VP (VB consider) (NP (DT a) (JJ financial) (NN contribution)) (PP (TO to) (NP (NP (JJ graduate) (NNS Endodontics)) (PP (IN at) (NP (NNP Indiana) (NNP University))))))))))) (. .))) (ROOT (S (NP (NP (NNP Carl) (NNP Newton)) (CC and) (NP (PRP I))) (VP (VBP have) (ADVP (RB never)) (VP (VBN had) (NP (DT any) (JJ other) (JJ previous) (NN contact)) (PP (IN with) (NP (PRP you))))) (. .))) A few months ago, Carl Newton and I wrote a letter asking you to consider a financial contribution to graduate Endodontics at Indiana University. Carl Newton and I have never had any other previous contact with you. contradiction contradiction contradiction contradiction contradiction contradiction +4 136158 136158e facetoface ( I ( ( was ( on ( ( this earth ) ( you ( know ( ( , ( ( I ( 've ( lived ( on ( ( this earth ) ( for ( some reason ) ) ) ) ) ) ) , ) ) ( I ( just ( ( do n't ) ( know ( what ( it ( is yet ) ) ) ) ) ) ) ) ) ) ) ) ) . ) ) ( I ( ( ( ( do n't ) yet ) ( ( know ( the reason ) ) ( why ( I ( have ( lived ( on earth ) ) ) ) ) ) ) . ) ) (ROOT (S (NP (PRP I)) (VP (VBD was) (PP (IN on) (NP (NP (DT this) (NN earth)) (SBAR (S (NP (PRP you)) (VP (VBP know) (SBAR (S (PRN (, ,) (S (NP (PRP I)) (VP (VBP 've) (VP (VBN lived) (PP (IN on) (NP (NP (DT this) (NN earth)) (PP (IN for) (NP (DT some) (NN reason)))))))) (, ,)) (NP (PRP I)) (ADVP (RB just)) (VP (VBP do) (RB n't) (VP (VB know) (SBAR (WHNP (WP what)) (S (NP (PRP it)) (VP (VBZ is) (ADVP (RB yet))))))))))))))) (. .))) (ROOT (S (NP (PRP I)) (VP (VBP do) (RB n't) (ADVP (RB yet)) (VP (VB know) (NP (DT the) (NN reason)) (SBAR (WHADVP (WRB why)) (S (NP (PRP I)) (VP (VBP have) (VP (VBN lived) (PP (IN on) (NP (NN earth))))))))) (. .))) I was on this earth you know, I've lived on this earth for some reason, I just don't know what it is yet. I don't yet know the reason why I have lived on earth. entailment entailment entailment entailment entailment entailment diff --git a/test/data_for_tests/io/MNLI/test_matched.tsv b/test/data_for_tests/io/MNLI/test_matched.tsv new file mode 100755 index 00000000..b90c2d2a --- /dev/null +++ b/test/data_for_tests/io/MNLI/test_matched.tsv @@ -0,0 +1,6 @@ +index promptID pairID genre sentence1_binary_parse sentence2_binary_parse sentence1_parse sentence2_parse sentence1 sentence2 +0 31493 31493 travel ( ( ( ( ( ( ( ( Hierbas , ) ( ans seco ) ) , ) ( ans dulce ) ) , ) and ) frigola ) ( ( ( are just ) ( ( a ( few names ) ) ( worth ( ( keeping ( a look-out ) ) for ) ) ) ) . ) ) ( Hierbas ( ( is ( ( a name ) ( worth ( ( looking out ) for ) ) ) ) . ) ) (ROOT (S (NP (NP (NNS Hierbas)) (, ,) (NP (NN ans) (NN seco)) (, ,) (NP (NN ans) (NN dulce)) (, ,) (CC and) (NP (NN frigola))) (VP (VBP are) (ADVP (RB just)) (NP (NP (DT a) (JJ few) (NNS names)) (PP (JJ worth) (S (VP (VBG keeping) (NP (DT a) (NN look-out)) (PP (IN for))))))) (. .))) (ROOT (S (NP (NNS Hierbas)) (VP (VBZ is) (NP (NP (DT a) (NN name)) (PP (JJ worth) (S (VP (VBG looking) (PRT (RP out)) (PP (IN for))))))) (. .))) Hierbas, ans seco, ans dulce, and frigola are just a few names worth keeping a look-out for. Hierbas is a name worth looking out for. +1 92164 92164 government ( ( ( The extent ) ( of ( the ( behavioral effects ) ) ) ) ( ( would ( ( depend ( in ( part ( on ( ( the structure ) ( of ( ( ( the ( individual ( account program ) ) ) and ) ( any limits ) ) ) ) ) ) ) ) ( on ( accessing ( the funds ) ) ) ) ) . ) ) ( ( Many people ) ( ( would ( be ( very ( unhappy ( to ( ( loose control ) ( over ( their ( own money ) ) ) ) ) ) ) ) ) . ) ) (ROOT (S (NP (NP (DT The) (NN extent)) (PP (IN of) (NP (DT the) (JJ behavioral) (NNS effects)))) (VP (MD would) (VP (VB depend) (PP (IN in) (NP (NP (NN part)) (PP (IN on) (NP (NP (DT the) (NN structure)) (PP (IN of) (NP (NP (DT the) (JJ individual) (NN account) (NN program)) (CC and) (NP (DT any) (NNS limits)))))))) (PP (IN on) (S (VP (VBG accessing) (NP (DT the) (NNS funds))))))) (. .))) (ROOT (S (NP (JJ Many) (NNS people)) (VP (MD would) (VP (VB be) (ADJP (RB very) (JJ unhappy) (PP (TO to) (NP (NP (JJ loose) (NN control)) (PP (IN over) (NP (PRP$ their) (JJ own) (NN money)))))))) (. .))) The extent of the behavioral effects would depend in part on the structure of the individual account program and any limits on accessing the funds. Many people would be very unhappy to loose control over their own money. +2 9662 9662 government ( ( ( Timely access ) ( to information ) ) ( ( is ( in ( ( the ( best interests ) ) ( of ( ( ( both GAO ) and ) ( the agencies ) ) ) ) ) ) . ) ) ( It ( ( ( is ( in ( ( everyone 's ) ( best interest ) ) ) ) ( to ( ( have access ) ( to ( information ( in ( a ( timely manner ) ) ) ) ) ) ) ) . ) ) (ROOT (S (NP (NP (JJ Timely) (NN access)) (PP (TO to) (NP (NN information)))) (VP (VBZ is) (PP (IN in) (NP (NP (DT the) (JJS best) (NNS interests)) (PP (IN of) (NP (NP (DT both) (NNP GAO)) (CC and) (NP (DT the) (NNS agencies))))))) (. .))) (ROOT (S (NP (PRP It)) (VP (VBZ is) (PP (IN in) (NP (NP (NN everyone) (POS 's)) (JJS best) (NN interest))) (S (VP (TO to) (VP (VB have) (NP (NN access)) (PP (TO to) (NP (NP (NN information)) (PP (IN in) (NP (DT a) (JJ timely) (NN manner))))))))) (. .))) Timely access to information is in the best interests of both GAO and the agencies. It is in everyone's best interest to have access to information in a timely manner. +3 5991 5991 travel ( ( Based ( in ( ( the ( Auvergnat ( spa town ) ) ) ( of Vichy ) ) ) ) ( , ( ( the ( French government ) ) ( often ( ( ( ( proved ( more zealous ) ) ( than ( its masters ) ) ) ( in ( ( ( suppressing ( civil liberties ) ) and ) ( ( drawing up ) ( anti-Jewish legislation ) ) ) ) ) . ) ) ) ) ) ( ( The ( French government ) ) ( ( passed ( ( anti-Jewish laws ) ( aimed ( at ( helping ( the Nazi ) ) ) ) ) ) . ) ) (ROOT (S (PP (VBN Based) (PP (IN in) (NP (NP (DT the) (NNP Auvergnat) (NN spa) (NN town)) (PP (IN of) (NP (NNP Vichy)))))) (, ,) (NP (DT the) (JJ French) (NN government)) (ADVP (RB often)) (VP (VBD proved) (NP (JJR more) (NNS zealous)) (PP (IN than) (NP (PRP$ its) (NNS masters))) (PP (IN in) (S (VP (VP (VBG suppressing) (NP (JJ civil) (NNS liberties))) (CC and) (VP (VBG drawing) (PRT (RP up)) (NP (JJ anti-Jewish) (NN legislation))))))) (. .))) (ROOT (S (NP (DT The) (JJ French) (NN government)) (VP (VBD passed) (NP (NP (JJ anti-Jewish) (NNS laws)) (VP (VBN aimed) (PP (IN at) (S (VP (VBG helping) (NP (DT the) (JJ Nazi)))))))) (. .))) Based in the Auvergnat spa town of Vichy, the French government often proved more zealous than its masters in suppressing civil liberties and drawing up anti-Jewish legislation. The French government passed anti-Jewish laws aimed at helping the Nazi. +4 50156 50156 travel ( ( ( ( ( Built ( in 1870 ) ) ( , ( ( ( its canopy ) ( of ( stained ( glass ( and ( cast iron ) ) ) ) ) ) ( is ( ( the oldest ) ( in Dublin ) ) ) ) ) ) ; ) ( ( its ( enthusiastic ( interior decoration ) ) ) ( ( is also ) ( typical ( of ( the era ) ) ) ) ) ) . ) ( It ( ( ( ( was ( constructed ( in 1870 ) ) ) and ) ( has ( ( the ( oldest canopy ) ) ( in Dublin ) ) ) ) . ) ) (ROOT (S (S (S (VP (VBN Built) (PP (IN in) (NP (CD 1870))))) (, ,) (NP (NP (PRP$ its) (NN canopy)) (PP (IN of) (NP (JJ stained) (NN glass) (CC and) (NN cast) (NN iron)))) (VP (VBZ is) (NP (NP (DT the) (JJS oldest)) (PP (IN in) (NP (NNP Dublin)))))) (: ;) (S (NP (PRP$ its) (JJ enthusiastic) (JJ interior) (NN decoration)) (VP (VBZ is) (ADVP (RB also)) (ADJP (JJ typical) (PP (IN of) (NP (DT the) (NN era)))))) (. .))) (ROOT (S (NP (PRP It)) (VP (VP (VBD was) (VP (VBN constructed) (PP (IN in) (NP (CD 1870))))) (CC and) (VP (VBZ has) (NP (NP (DT the) (JJS oldest) (NN canopy)) (PP (IN in) (NP (NNP Dublin)))))) (. .))) Built in 1870, its canopy of stained glass and cast iron is the oldest in Dublin; its enthusiastic interior decoration is also typical of the era. It was constructed in 1870 and has the oldest canopy in Dublin. diff --git a/test/data_for_tests/io/MNLI/test_mismatched.tsv b/test/data_for_tests/io/MNLI/test_mismatched.tsv new file mode 100755 index 00000000..798cd395 --- /dev/null +++ b/test/data_for_tests/io/MNLI/test_mismatched.tsv @@ -0,0 +1,6 @@ +index promptID pairID genre sentence1_binary_parse sentence2_binary_parse sentence1_parse sentence2_parse sentence1 sentence2 +0 16130 16130 facetoface ( ( What ( have ( you decided ) ) ) ( , ( what ( ( ( are you ) ( going ( to do ) ) ) ? ) ) ) ) ( So ( what ( ( 's ( your decision ) ) ? ) ) ) (ROOT (SBARQ (SBAR (WHNP (WP What)) (S (VP (VBP have) (S (NP (PRP you)) (VP (VBD decided)))))) (, ,) (WHNP (WP what)) (SQ (VBP are) (NP (PRP you)) (VP (VBG going) (S (VP (TO to) (VP (VB do)))))) (. ?))) (ROOT (SBARQ (RB So) (WHNP (WP what)) (SQ (VBZ 's) (NP (PRP$ your) (NN decision))) (. ?))) What have you decided, what are you going to do? So what's your decision? +1 128269 128269 oup ( ( ( Women 's ) clothing ) ( ( is ( characterized ( by ( ( great diversity ) ( in ( ( styles and ) ( short ( production runs ) ) ) ) ) ) ) ) . ) ) ( ( ( Men 's ) clothing ) ( typically ( ( ( has ( the ( ( most stylistic ) diversity ) ) ) ( unlike ( ( the blandness ) ( of ( ( women 's ) fashion ) ) ) ) ) . ) ) ) (ROOT (S (NP (NP (NNP Women) (POS 's)) (NN clothing)) (VP (VBZ is) (VP (VBN characterized) (PP (IN by) (NP (NP (JJ great) (NN diversity)) (PP (IN in) (NP (NP (NNS styles)) (CC and) (NP (JJ short) (NN production) (NNS runs)))))))) (. .))) (ROOT (S (NP (NP (NNP Men) (POS 's)) (NN clothing)) (ADVP (RB typically)) (VP (VBZ has) (NP (DT the) (ADJP (RBS most) (JJ stylistic)) (NN diversity)) (PP (IN unlike) (NP (NP (DT the) (NN blandness)) (PP (IN of) (NP (NP (NNS women) (POS 's)) (NN fashion)))))) (. .))) Women's clothing is characterized by great diversity in styles and short production runs. Men's clothing typically has the most stylistic diversity unlike the blandness of women's fashion. +2 130938 130938 nineeleven ( ( ( ( ( Reports ( from ( ( two ( flight attendants ) ) ( in ( the ( coach cabin ) ) ) ) ) ) , ) ( ( ( Betty Ong ) and ) ( Madeline ( Amy Sweeney ) ) ) ) , ) ( ( ( tell us ) ( ( most ( of what ) ) ( we ( know ( about ( how ( ( the hijacking ) happened ) ) ) ) ) ) ) . ) ) ( ( ( The report ) ( on ( the hijacking ) ) ) ( ( ( was ( ( over ( five hundred ) ) pages ) ) long ) . ) ) (ROOT (S (NP (NP (NP (NNS Reports)) (PP (IN from) (NP (NP (CD two) (NN flight) (NNS attendants)) (PP (IN in) (NP (DT the) (NN coach) (NN cabin)))))) (, ,) (NP (NP (NNP Betty) (NNP Ong)) (CC and) (NP (NNP Madeline) (NNP Amy) (NNP Sweeney))) (, ,)) (VP (VBP tell) (NP (PRP us)) (SBAR (WHNP (JJS most) (WHPP (IN of) (WHNP (WP what)))) (S (NP (PRP we)) (VP (VBP know) (PP (IN about) (SBAR (WHADVP (WRB how)) (S (NP (DT the) (NN hijacking)) (VP (VBD happened))))))))) (. .))) (ROOT (S (NP (NP (DT The) (NN report)) (PP (IN on) (NP (DT the) (NN hijacking)))) (VP (VBD was) (NP (QP (RB over) (CD five) (CD hundred)) (NNS pages)) (ADVP (RB long))) (. .))) Reports from two flight attendants in the coach cabin, Betty Ong and Madeline Amy Sweeney, tell us most of what we know about how the hijacking happened. The report on the hijacking was over five hundred pages long. +3 40009 40009 nineeleven ( ( At ( about 9:20 ) ) ( , ( ( ( security personnel ) ( at ( FAA headquarters ) ) ) ( ( ( ( set up ) ( a ( hijacking teleconference ) ) ) ( with ( ( ( several agencies ) , ) ( including ( the ( Defense Department ) ) ) ) ) ) . ) ) ) ) ( ( The teleconference ) ( ( lasted ( for ( 13 ( straight hours ) ) ) ) . ) ) (ROOT (S (PP (IN At) (NP (QP (RB about) (CD 9:20)))) (, ,) (NP (NP (NN security) (NNS personnel)) (PP (IN at) (NP (NNP FAA) (NNS headquarters)))) (VP (VBD set) (PRT (RP up)) (NP (DT a) (VBG hijacking) (NN teleconference)) (PP (IN with) (NP (NP (JJ several) (NNS agencies)) (, ,) (PP (VBG including) (NP (DT the) (NNP Defense) (NNP Department)))))) (. .))) (ROOT (S (NP (DT The) (NN teleconference)) (VP (VBD lasted) (PP (IN for) (NP (CD 13) (JJ straight) (NNS hours)))) (. .))) At about 9:20, security personnel at FAA headquarters set up a hijacking teleconference with several agencies, including the Defense Department. The teleconference lasted for 13 straight hours. +4 105266 105266 nineeleven ( So ( we ( ( 've ( ( got ( ( a couple ) ( of aircraft ) ) ) ( ( up there ) ( that ( ( have ( those instructions ) ) ( at ( this ( present time ) ) ) ) ) ) ) ) ? ) ) ) ( ( At ( the ( present time ) ) ) ( , ( there ( ( ( ( ( were n't ) ( ( any aircraft ) ( in ( the air ) ) ) ) , ) right ) ? ) ) ) ) (ROOT (S (IN So) (NP (PRP we)) (VP (VBP 've) (VP (VBD got) (NP (NP (DT a) (NN couple)) (PP (IN of) (NP (NN aircraft)))) (ADVP (ADVP (RB up) (RB there)) (SBAR (WHNP (WDT that)) (S (VP (VBP have) (NP (DT those) (NNS instructions)) (PP (IN at) (NP (DT this) (JJ present) (NN time))))))))) (. ?))) (ROOT (S (PP (IN At) (NP (DT the) (JJ present) (NN time))) (, ,) (NP (EX there)) (VP (VBD were) (RB n't) (NP (NP (DT any) (NN aircraft)) (PP (IN in) (NP (DT the) (NN air)))) (, ,) (ADJP (JJ right))) (. ?))) So we've got a couple of aircraft up there that have those instructions at this present time? At the present time, there weren't any aircraft in the air, right? diff --git a/test/data_for_tests/io/MNLI/train.tsv b/test/data_for_tests/io/MNLI/train.tsv new file mode 100755 index 00000000..4ceebefd --- /dev/null +++ b/test/data_for_tests/io/MNLI/train.tsv @@ -0,0 +1,7 @@ +index promptID pairID genre sentence1_binary_parse sentence2_binary_parse sentence1_parse sentence2_parse sentence1 sentence2 label1 gold_label +0 31193 31193n government ( ( Conceptually ( cream skimming ) ) ( ( has ( ( ( two ( basic dimensions ) ) - ) ( ( product and ) geography ) ) ) . ) ) ( ( ( Product and ) geography ) ( ( are ( what ( make ( cream ( skimming work ) ) ) ) ) . ) ) (ROOT (S (NP (JJ Conceptually) (NN cream) (NN skimming)) (VP (VBZ has) (NP (NP (CD two) (JJ basic) (NNS dimensions)) (: -) (NP (NN product) (CC and) (NN geography)))) (. .))) (ROOT (S (NP (NN Product) (CC and) (NN geography)) (VP (VBP are) (SBAR (WHNP (WP what)) (S (VP (VBP make) (NP (NP (NN cream)) (VP (VBG skimming) (NP (NN work)))))))) (. .))) Conceptually cream skimming has two basic dimensions - product and geography. Product and geography are what make cream skimming work. neutral neutral +1 101457 101457e telephone ( you ( ( know ( during ( ( ( the season ) and ) ( i guess ) ) ) ) ( at ( at ( ( your level ) ( uh ( you ( ( ( lose them ) ( to ( the ( next level ) ) ) ) ( if ( ( if ( they ( decide ( to ( recall ( the ( the ( parent team ) ) ) ) ) ) ) ) ( ( the Braves ) ( decide ( to ( call ( to ( ( recall ( a guy ) ) ( from ( ( triple A ) ( ( ( then ( ( a ( double ( A guy ) ) ) ( ( goes up ) ( to ( replace him ) ) ) ) ) and ) ( ( a ( single ( A guy ) ) ) ( ( goes up ) ( to ( replace him ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ( You ( ( ( ( lose ( the things ) ) ( to ( the ( following level ) ) ) ) ( if ( ( the people ) recall ) ) ) . ) ) (ROOT (S (NP (PRP you)) (VP (VBP know) (PP (IN during) (NP (NP (DT the) (NN season)) (CC and) (NP (FW i) (FW guess)))) (PP (IN at) (IN at) (NP (NP (PRP$ your) (NN level)) (SBAR (S (INTJ (UH uh)) (NP (PRP you)) (VP (VBP lose) (NP (PRP them)) (PP (TO to) (NP (DT the) (JJ next) (NN level))) (SBAR (IN if) (S (SBAR (IN if) (S (NP (PRP they)) (VP (VBP decide) (S (VP (TO to) (VP (VB recall) (NP (DT the) (DT the) (NN parent) (NN team)))))))) (NP (DT the) (NNPS Braves)) (VP (VBP decide) (S (VP (TO to) (VP (VB call) (S (VP (TO to) (VP (VB recall) (NP (DT a) (NN guy)) (PP (IN from) (NP (NP (RB triple) (DT A)) (SBAR (S (S (ADVP (RB then)) (NP (DT a) (JJ double) (NNP A) (NN guy)) (VP (VBZ goes) (PRT (RP up)) (S (VP (TO to) (VP (VB replace) (NP (PRP him))))))) (CC and) (S (NP (DT a) (JJ single) (NNP A) (NN guy)) (VP (VBZ goes) (PRT (RP up)) (S (VP (TO to) (VP (VB replace) (NP (PRP him)))))))))))))))))))))))))))) (ROOT (S (NP (PRP You)) (VP (VBP lose) (NP (DT the) (NNS things)) (PP (TO to) (NP (DT the) (JJ following) (NN level))) (SBAR (IN if) (S (NP (DT the) (NNS people)) (VP (VBP recall))))) (. .))) you know during the season and i guess at at your level uh you lose them to the next level if if they decide to recall the the parent team the Braves decide to call to recall a guy from triple A then a double A guy goes up to replace him and a single A guy goes up to replace him You lose the things to the following level if the people recall. entailment entailment +2 134793 134793e fiction ( ( One ( of ( our number ) ) ) ( ( will ( ( ( carry out ) ( your instructions ) ) minutely ) ) . ) ) ( ( ( A member ) ( of ( my team ) ) ) ( ( will ( ( execute ( your orders ) ) ( with ( immense precision ) ) ) ) . ) ) (ROOT (S (NP (NP (CD One)) (PP (IN of) (NP (PRP$ our) (NN number)))) (VP (MD will) (VP (VB carry) (PRT (RP out)) (NP (PRP$ your) (NNS instructions)) (ADVP (RB minutely)))) (. .))) (ROOT (S (NP (NP (DT A) (NN member)) (PP (IN of) (NP (PRP$ my) (NN team)))) (VP (MD will) (VP (VB execute) (NP (PRP$ your) (NNS orders)) (PP (IN with) (NP (JJ immense) (NN precision))))) (. .))) One of our number will carry out your instructions minutely. A member of my team will execute your orders with immense precision. entailment entailment +3 37397 37397e fiction ( ( How ( ( ( do you ) know ) ? ) ) ( ( All this ) ( ( ( is ( their information ) ) again ) . ) ) ) ( ( This information ) ( ( belongs ( to them ) ) . ) ) (ROOT (S (SBARQ (WHADVP (WRB How)) (SQ (VBP do) (NP (PRP you)) (VP (VB know))) (. ?)) (NP (PDT All) (DT this)) (VP (VBZ is) (NP (PRP$ their) (NN information)) (ADVP (RB again))) (. .))) (ROOT (S (NP (DT This) (NN information)) (VP (VBZ belongs) (PP (TO to) (NP (PRP them)))) (. .))) How do you know? All this is their information again. This information belongs to them. entailment entailment +4 50563 50563n telephone ( yeah ( i ( ( tell you ) ( what ( ( though ( if ( you ( go ( price ( some ( of ( those ( tennis shoes ) ) ) ) ) ) ) ) ) ( i ( can ( see ( why ( now ( you ( know ( they ( 're ( ( getting up ) ( in ( the ( hundred ( dollar range ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ( ( The ( tennis shoes ) ) ( ( have ( ( a range ) ( of prices ) ) ) . ) ) (ROOT (S (VP (VB yeah) (S (NP (FW i)) (VP (VB tell) (NP (PRP you)) (SBAR (WHNP (WP what)) (S (SBAR (RB though) (IN if) (S (NP (PRP you)) (VP (VBP go) (VP (VB price) (NP (NP (DT some)) (PP (IN of) (NP (DT those) (NN tennis) (NNS shoes)))))))) (NP (FW i)) (VP (MD can) (VP (VB see) (SBAR (WHADVP (WRB why)) (S (ADVP (RB now)) (NP (PRP you)) (VP (VBP know) (SBAR (S (NP (PRP they)) (VP (VBP 're) (VP (VBG getting) (PRT (RP up)) (PP (IN in) (NP (DT the) (CD hundred) (NN dollar) (NN range))))))))))))))))))) (ROOT (S (NP (DT The) (NN tennis) (NNS shoes)) (VP (VBP have) (NP (NP (DT a) (NN range)) (PP (IN of) (NP (NNS prices))))) (. .))) yeah i tell you what though if you go price some of those tennis shoes i can see why now you know they're getting up in the hundred dollar range The tennis shoes have a range of prices. neutral neutral +11 11877 11877c travel ( ( Fun ( for ( ( adults and ) children ) ) ) . ) ( ( Fun ( for ( only children ) ) ) . ) (ROOT (S (VP (VB Fun) (PP (IN for) (NP (NNS adults) (CC and) (NNS children)))) (. .))) (ROOT (S (VP (VB Fun) (PP (IN for) (NP (JJ only) (NNS children)))) (. .))) Fun for adults and children. Fun for only children. contradiction contradiction diff --git a/test/data_for_tests/io/QNLI/dev.tsv b/test/data_for_tests/io/QNLI/dev.tsv new file mode 100755 index 00000000..ac4ecabe --- /dev/null +++ b/test/data_for_tests/io/QNLI/dev.tsv @@ -0,0 +1,6 @@ +index question sentence label +0 What came into force after the new constitution was herald? As of that day, the new constitution heralding the Second Republic came into force. entailment +1 What is the first major city in the stream of the Rhine? The most important tributaries in this area are the Ill below of Strasbourg, the Neckar in Mannheim and the Main across from Mainz. not_entailment +2 What is the minimum required if you want to teach in Canada? In most provinces a second Bachelor's Degree such as a Bachelor of Education is required to become a qualified teacher. not_entailment +3 How was Temüjin kept imprisoned by the Tayichi'ud? The Tayichi'ud enslaved Temüjin (reportedly with a cangue, a sort of portable stocks), but with the help of a sympathetic guard, the father of Chilaun (who later became a general of Genghis Khan), he was able to escape from the ger (yurt) in the middle of the night by hiding in a river crevice.[citation needed] entailment +4 What did Herr Gott, dich loben wir become known as ? He paraphrased the Te Deum as "Herr Gott, dich loben wir" with a simplified form of the melody. not_entailment diff --git a/test/data_for_tests/io/QNLI/test.tsv b/test/data_for_tests/io/QNLI/test.tsv new file mode 100755 index 00000000..55bfbeaa --- /dev/null +++ b/test/data_for_tests/io/QNLI/test.tsv @@ -0,0 +1,6 @@ +index question sentence +0 What organization is devoted to Jihad against Israel? For some decades prior to the First Palestine Intifada in 1987, the Muslim Brotherhood in Palestine took a "quiescent" stance towards Israel, focusing on preaching, education and social services, and benefiting from Israel's "indulgence" to build up a network of mosques and charitable organizations. +1 In what century was the Yarrow-Schlick-Tweedy balancing system used? In the late 19th century, the Yarrow-Schlick-Tweedy balancing 'system' was used on some marine triple expansion engines. +2 The largest brand of what store in the UK is located in Kingston Park? Close to Newcastle, the largest indoor shopping centre in Europe, the MetroCentre, is located in Gateshead. +3 What does the IPCC rely on for research? In principle, this means that any significant new evidence or events that change our understanding of climate science between this deadline and publication of an IPCC report cannot be included. +4 What is the principle about relating spin and space variables? Thus in the case of two fermions there is a strictly negative correlation between spatial and spin variables, whereas for two bosons (e.g. quanta of electromagnetic waves, photons) the correlation is strictly positive. diff --git a/test/data_for_tests/io/QNLI/train.tsv b/test/data_for_tests/io/QNLI/train.tsv new file mode 100755 index 00000000..fc0b966e --- /dev/null +++ b/test/data_for_tests/io/QNLI/train.tsv @@ -0,0 +1,6 @@ +index question sentence label +0 When did the third Digimon series begin? Unlike the two seasons before it and most of the seasons that followed, Digimon Tamers takes a darker and more realistic approach to its story featuring Digimon who do not reincarnate after their deaths and more complex character development in the original Japanese. not_entailment +1 Which missile batteries often have individual launchers several kilometres from one another? When MANPADS is operated by specialists, batteries may have several dozen teams deploying separately in small sections; self-propelled air defence guns may deploy in pairs. not_entailment +2 What two things does Popper argue Tarski's theory involves in an evaluation of truth? He bases this interpretation on the fact that examples such as the one described above refer to two things: assertions and the facts to which they refer. entailment +3 What is the name of the village 9 miles north of Calafat where the Ottoman forces attacked the Russians? On 31 December 1853, the Ottoman forces at Calafat moved against the Russian force at Chetatea or Cetate, a small village nine miles north of Calafat, and engaged them on 6 January 1854. entailment +4 What famous palace is located in London? London contains four World Heritage Sites: the Tower of London; Kew Gardens; the site comprising the Palace of Westminster, Westminster Abbey, and St Margaret's Church; and the historic settlement of Greenwich (in which the Royal Observatory, Greenwich marks the Prime Meridian, 0° longitude, and GMT). not_entailment diff --git a/test/data_for_tests/io/RTE/dev.tsv b/test/data_for_tests/io/RTE/dev.tsv new file mode 100644 index 00000000..f8f72536 --- /dev/null +++ b/test/data_for_tests/io/RTE/dev.tsv @@ -0,0 +1,6 @@ +index sentence1 sentence2 label +0 Dana Reeve, the widow of the actor Christopher Reeve, has died of lung cancer at age 44, according to the Christopher Reeve Foundation. Christopher Reeve had an accident. not_entailment +1 Yet, we now are discovering that antibiotics are losing their effectiveness against illness. Disease-causing bacteria are mutating faster than we can come up with new antibiotics to fight the new variations. Bacteria is winning the war against antibiotics. entailment +2 Cairo is now home to some 15 million people - a burgeoning population that produces approximately 10,000 tonnes of rubbish per day, putting an enormous strain on public services. In the past 10 years, the government has tried hard to encourage private investment in the refuse sector, but some estimate 4,000 tonnes of waste is left behind every day, festering in the heat as it waits for someone to clear it up. It is often the people in the poorest neighbourhoods that are worst affected. But in some areas they are fighting back. In Shubra, one of the northern districts of the city, the residents have taken to the streets armed with dustpans and brushes to clean up public areas which have been used as public dumps. 15 million tonnes of rubbish are produced daily in Cairo. not_entailment +3 The Amish community in Pennsylvania, which numbers about 55,000, lives an agrarian lifestyle, shunning technological advances like electricity and automobiles. And many say their insular lifestyle gives them a sense that they are protected from the violence of American society. But as residents gathered near the school, some wearing traditional garb and arriving in horse-drawn buggies, they said that sense of safety had been shattered. "If someone snaps and wants to do something stupid, there's no distance that's going to stop them," said Jake King, 56, an Amish lantern maker who knew several families whose children had been shot. Pennsylvania has the biggest Amish community in the U.S. not_entailment +4 Security forces were on high alert after an election campaign in which more than 1,000 people, including seven election candidates, have been killed. Security forces were on high alert after a campaign marred by violence. entailment diff --git a/test/data_for_tests/io/RTE/test.tsv b/test/data_for_tests/io/RTE/test.tsv new file mode 100644 index 00000000..e52dfac4 --- /dev/null +++ b/test/data_for_tests/io/RTE/test.tsv @@ -0,0 +1,6 @@ +index sentence1 sentence2 +0 Mangla was summoned after Madhumita's sister Nidhi Shukla, who was the first witness in the case. Shukla is related to Mangla. +1 Authorities in Brazil say that more than 200 people are being held hostage in a prison in the country's remote, Amazonian-jungle state of Rondonia. Authorities in Brazil hold 200 people as hostage. +2 A mercenary group faithful to the warmongering policy of former Somozist colonel Enrique Bermudez attacked an IFA truck belonging to the interior ministry at 0900 on 26 March in El Jicote, wounded and killed an interior ministry worker and wounded five others. An interior ministry worker was killed by a mercenary group. +3 The British ambassador to Egypt, Derek Plumbly, told Reuters on Monday that authorities had compiled the list of 10 based on lists from tour companies and from families whose relatives have not been in contact since the bombings. Derek Plumbly resides in Egypt. +4 Tibone estimated diamond production at four mines operated by Debswana -- Botswana's 50-50 joint venture with De Beers -- could reach 33 million carats this year. Botswana is a business partner of De Beers. diff --git a/test/data_for_tests/io/RTE/train.tsv b/test/data_for_tests/io/RTE/train.tsv new file mode 100644 index 00000000..70e5414f --- /dev/null +++ b/test/data_for_tests/io/RTE/train.tsv @@ -0,0 +1,6 @@ +index sentence1 sentence2 label +0 No Weapons of Mass Destruction Found in Iraq Yet. Weapons of Mass Destruction Found in Iraq. not_entailment +1 A place of sorrow, after Pope John Paul II died, became a place of celebration, as Roman Catholic faithful gathered in downtown Chicago to mark the installation of new Pope Benedict XVI. Pope Benedict XVI is the new leader of the Roman Catholic Church. entailment +2 Herceptin was already approved to treat the sickest breast cancer patients, and the company said, Monday, it will discuss with federal regulators the possibility of prescribing the drug for more breast cancer patients. Herceptin can be used to treat breast cancer. entailment +3 Judie Vivian, chief executive at ProMedica, a medical service company that helps sustain the 2-year-old Vietnam Heart Institute in Ho Chi Minh City (formerly Saigon), said that so far about 1,500 children have received treatment. The previous name of Ho Chi Minh City was Saigon. entailment +4 A man is due in court later charged with the murder 26 years ago of a teenager whose case was the first to be featured on BBC One's Crimewatch. Colette Aram, 16, was walking to her boyfriend's house in Keyworth, Nottinghamshire, on 30 October 1983 when she disappeared. Her body was later found in a field close to her home. Paul Stewart Hutchinson, 50, has been charged with murder and is due before Nottingham magistrates later. Paul Stewart Hutchinson is accused of having stabbed a girl. not_entailment diff --git a/test/data_for_tests/io/SNLI/snli_1.0_dev.jsonl b/test/data_for_tests/io/SNLI/snli_1.0_dev.jsonl new file mode 100755 index 00000000..2d091c73 --- /dev/null +++ b/test/data_for_tests/io/SNLI/snli_1.0_dev.jsonl @@ -0,0 +1,5 @@ +{"annotator_labels": ["neutral", "entailment", "neutral", "neutral", "neutral"], "captionID": "4705552913.jpg#2", "gold_label": "neutral", "pairID": "4705552913.jpg#2r1n", "sentence1": "Two women are embracing while holding to go packages.", "sentence1_binary_parse": "( ( Two women ) ( ( are ( embracing ( while ( holding ( to ( go packages ) ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (CD Two) (NNS women)) (VP (VBP are) (VP (VBG embracing) (SBAR (IN while) (S (NP (VBG holding)) (VP (TO to) (VP (VB go) (NP (NNS packages)))))))) (. .)))", "sentence2": "The sisters are hugging goodbye while holding to go packages after just eating lunch.", "sentence2_binary_parse": "( ( The sisters ) ( ( are ( ( hugging goodbye ) ( while ( holding ( to ( ( go packages ) ( after ( just ( eating lunch ) ) ) ) ) ) ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT The) (NNS sisters)) (VP (VBP are) (VP (VBG hugging) (NP (UH goodbye)) (PP (IN while) (S (VP (VBG holding) (S (VP (TO to) (VP (VB go) (NP (NNS packages)) (PP (IN after) (S (ADVP (RB just)) (VP (VBG eating) (NP (NN lunch))))))))))))) (. .)))"} +{"annotator_labels": ["entailment", "entailment", "entailment", "entailment", "entailment"], "captionID": "4705552913.jpg#2", "gold_label": "entailment", "pairID": "4705552913.jpg#2r1e", "sentence1": "Two women are embracing while holding to go packages.", "sentence1_binary_parse": "( ( Two women ) ( ( are ( embracing ( while ( holding ( to ( go packages ) ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (CD Two) (NNS women)) (VP (VBP are) (VP (VBG embracing) (SBAR (IN while) (S (NP (VBG holding)) (VP (TO to) (VP (VB go) (NP (NNS packages)))))))) (. .)))", "sentence2": "Two woman are holding packages.", "sentence2_binary_parse": "( ( Two woman ) ( ( are ( holding packages ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (CD Two) (NN woman)) (VP (VBP are) (VP (VBG holding) (NP (NNS packages)))) (. .)))"} +{"annotator_labels": ["contradiction", "contradiction", "contradiction", "contradiction", "contradiction"], "captionID": "4705552913.jpg#2", "gold_label": "contradiction", "pairID": "4705552913.jpg#2r1c", "sentence1": "Two women are embracing while holding to go packages.", "sentence1_binary_parse": "( ( Two women ) ( ( are ( embracing ( while ( holding ( to ( go packages ) ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (CD Two) (NNS women)) (VP (VBP are) (VP (VBG embracing) (SBAR (IN while) (S (NP (VBG holding)) (VP (TO to) (VP (VB go) (NP (NNS packages)))))))) (. .)))", "sentence2": "The men are fighting outside a deli.", "sentence2_binary_parse": "( ( The men ) ( ( are ( fighting ( outside ( a deli ) ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT The) (NNS men)) (VP (VBP are) (VP (VBG fighting) (PP (IN outside) (NP (DT a) (NNS deli))))) (. .)))"} +{"annotator_labels": ["entailment", "entailment", "entailment", "entailment", "entailment"], "captionID": "2407214681.jpg#0", "gold_label": "entailment", "pairID": "2407214681.jpg#0r1e", "sentence1": "Two young children in blue jerseys, one with the number 9 and one with the number 2 are standing on wooden steps in a bathroom and washing their hands in a sink.", "sentence1_binary_parse": "( ( ( Two ( young children ) ) ( in ( ( ( ( ( blue jerseys ) , ) ( one ( with ( the ( number 9 ) ) ) ) ) and ) ( one ( with ( the ( number 2 ) ) ) ) ) ) ) ( ( are ( ( ( standing ( on ( ( wooden steps ) ( in ( a bathroom ) ) ) ) ) and ) ( ( washing ( their hands ) ) ( in ( a sink ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (CD Two) (JJ young) (NNS children)) (PP (IN in) (NP (NP (JJ blue) (NNS jerseys)) (, ,) (NP (NP (CD one)) (PP (IN with) (NP (DT the) (NN number) (CD 9)))) (CC and) (NP (NP (CD one)) (PP (IN with) (NP (DT the) (NN number) (CD 2))))))) (VP (VBP are) (VP (VP (VBG standing) (PP (IN on) (NP (NP (JJ wooden) (NNS steps)) (PP (IN in) (NP (DT a) (NN bathroom)))))) (CC and) (VP (VBG washing) (NP (PRP$ their) (NNS hands)) (PP (IN in) (NP (DT a) (NN sink)))))) (. .)))", "sentence2": "Two kids in numbered jerseys wash their hands.", "sentence2_binary_parse": "( ( ( Two kids ) ( in ( numbered jerseys ) ) ) ( ( wash ( their hands ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (NP (CD Two) (NNS kids)) (PP (IN in) (NP (JJ numbered) (NNS jerseys)))) (VP (VBP wash) (NP (PRP$ their) (NNS hands))) (. .)))"} +{"annotator_labels": ["neutral", "neutral", "neutral", "entailment", "entailment"], "captionID": "2407214681.jpg#0", "gold_label": "neutral", "pairID": "2407214681.jpg#0r1n", "sentence1": "Two young children in blue jerseys, one with the number 9 and one with the number 2 are standing on wooden steps in a bathroom and washing their hands in a sink.", "sentence1_binary_parse": "( ( ( Two ( young children ) ) ( in ( ( ( ( ( blue jerseys ) , ) ( one ( with ( the ( number 9 ) ) ) ) ) and ) ( one ( with ( the ( number 2 ) ) ) ) ) ) ) ( ( are ( ( ( standing ( on ( ( wooden steps ) ( in ( a bathroom ) ) ) ) ) and ) ( ( washing ( their hands ) ) ( in ( a sink ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (CD Two) (JJ young) (NNS children)) (PP (IN in) (NP (NP (JJ blue) (NNS jerseys)) (, ,) (NP (NP (CD one)) (PP (IN with) (NP (DT the) (NN number) (CD 9)))) (CC and) (NP (NP (CD one)) (PP (IN with) (NP (DT the) (NN number) (CD 2))))))) (VP (VBP are) (VP (VP (VBG standing) (PP (IN on) (NP (NP (JJ wooden) (NNS steps)) (PP (IN in) (NP (DT a) (NN bathroom)))))) (CC and) (VP (VBG washing) (NP (PRP$ their) (NNS hands)) (PP (IN in) (NP (DT a) (NN sink)))))) (. .)))", "sentence2": "Two kids at a ballgame wash their hands.", "sentence2_binary_parse": "( ( ( Two kids ) ( at ( a ballgame ) ) ) ( ( wash ( their hands ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (NP (CD Two) (NNS kids)) (PP (IN at) (NP (DT a) (NN ballgame)))) (VP (VBP wash) (NP (PRP$ their) (NNS hands))) (. .)))"} diff --git a/test/data_for_tests/io/SNLI/snli_1.0_test.jsonl b/test/data_for_tests/io/SNLI/snli_1.0_test.jsonl new file mode 100755 index 00000000..49d40720 --- /dev/null +++ b/test/data_for_tests/io/SNLI/snli_1.0_test.jsonl @@ -0,0 +1,5 @@ +{"annotator_labels": ["neutral", "contradiction", "contradiction", "neutral", "neutral"], "captionID": "2677109430.jpg#1", "gold_label": "neutral", "pairID": "2677109430.jpg#1r1n", "sentence1": "This church choir sings to the masses as they sing joyous songs from the book at a church.", "sentence1_binary_parse": "( ( This ( church choir ) ) ( ( ( sings ( to ( the masses ) ) ) ( as ( they ( ( sing ( joyous songs ) ) ( from ( ( the book ) ( at ( a church ) ) ) ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (DT This) (NN church) (NN choir)) (VP (VBZ sings) (PP (TO to) (NP (DT the) (NNS masses))) (SBAR (IN as) (S (NP (PRP they)) (VP (VBP sing) (NP (JJ joyous) (NNS songs)) (PP (IN from) (NP (NP (DT the) (NN book)) (PP (IN at) (NP (DT a) (NN church))))))))) (. .)))", "sentence2": "The church has cracks in the ceiling.", "sentence2_binary_parse": "( ( The church ) ( ( has ( cracks ( in ( the ceiling ) ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT The) (NN church)) (VP (VBZ has) (NP (NP (NNS cracks)) (PP (IN in) (NP (DT the) (NN ceiling))))) (. .)))"} +{"annotator_labels": ["entailment", "entailment", "entailment", "neutral", "entailment"], "captionID": "2677109430.jpg#1", "gold_label": "entailment", "pairID": "2677109430.jpg#1r1e", "sentence1": "This church choir sings to the masses as they sing joyous songs from the book at a church.", "sentence1_binary_parse": "( ( This ( church choir ) ) ( ( ( sings ( to ( the masses ) ) ) ( as ( they ( ( sing ( joyous songs ) ) ( from ( ( the book ) ( at ( a church ) ) ) ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (DT This) (NN church) (NN choir)) (VP (VBZ sings) (PP (TO to) (NP (DT the) (NNS masses))) (SBAR (IN as) (S (NP (PRP they)) (VP (VBP sing) (NP (JJ joyous) (NNS songs)) (PP (IN from) (NP (NP (DT the) (NN book)) (PP (IN at) (NP (DT a) (NN church))))))))) (. .)))", "sentence2": "The church is filled with song.", "sentence2_binary_parse": "( ( The church ) ( ( is ( filled ( with song ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT The) (NN church)) (VP (VBZ is) (VP (VBN filled) (PP (IN with) (NP (NN song))))) (. .)))"} +{"annotator_labels": ["contradiction", "contradiction", "contradiction", "contradiction", "contradiction"], "captionID": "2677109430.jpg#1", "gold_label": "contradiction", "pairID": "2677109430.jpg#1r1c", "sentence1": "This church choir sings to the masses as they sing joyous songs from the book at a church.", "sentence1_binary_parse": "( ( This ( church choir ) ) ( ( ( sings ( to ( the masses ) ) ) ( as ( they ( ( sing ( joyous songs ) ) ( from ( ( the book ) ( at ( a church ) ) ) ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (DT This) (NN church) (NN choir)) (VP (VBZ sings) (PP (TO to) (NP (DT the) (NNS masses))) (SBAR (IN as) (S (NP (PRP they)) (VP (VBP sing) (NP (JJ joyous) (NNS songs)) (PP (IN from) (NP (NP (DT the) (NN book)) (PP (IN at) (NP (DT a) (NN church))))))))) (. .)))", "sentence2": "A choir singing at a baseball game.", "sentence2_binary_parse": "( ( ( A choir ) ( singing ( at ( a ( baseball game ) ) ) ) ) . )", "sentence2_parse": "(ROOT (NP (NP (DT A) (NN choir)) (VP (VBG singing) (PP (IN at) (NP (DT a) (NN baseball) (NN game)))) (. .)))"} +{"annotator_labels": ["neutral", "neutral", "neutral", "neutral", "neutral"], "captionID": "6160193920.jpg#4", "gold_label": "neutral", "pairID": "6160193920.jpg#4r1n", "sentence1": "A woman with a green headscarf, blue shirt and a very big grin.", "sentence1_binary_parse": "( ( ( A woman ) ( with ( ( ( ( ( a ( green headscarf ) ) , ) ( blue shirt ) ) and ) ( a ( ( very big ) grin ) ) ) ) ) . )", "sentence1_parse": "(ROOT (NP (NP (DT A) (NN woman)) (PP (IN with) (NP (NP (DT a) (JJ green) (NN headscarf)) (, ,) (NP (JJ blue) (NN shirt)) (CC and) (NP (DT a) (ADJP (RB very) (JJ big)) (NN grin)))) (. .)))", "sentence2": "The woman is young.", "sentence2_binary_parse": "( ( The woman ) ( ( is young ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT The) (NN woman)) (VP (VBZ is) (ADJP (JJ young))) (. .)))"} +{"annotator_labels": ["entailment", "entailment", "contradiction", "entailment", "neutral"], "captionID": "6160193920.jpg#4", "gold_label": "entailment", "pairID": "6160193920.jpg#4r1e", "sentence1": "A woman with a green headscarf, blue shirt and a very big grin.", "sentence1_binary_parse": "( ( ( A woman ) ( with ( ( ( ( ( a ( green headscarf ) ) , ) ( blue shirt ) ) and ) ( a ( ( very big ) grin ) ) ) ) ) . )", "sentence1_parse": "(ROOT (NP (NP (DT A) (NN woman)) (PP (IN with) (NP (NP (DT a) (JJ green) (NN headscarf)) (, ,) (NP (JJ blue) (NN shirt)) (CC and) (NP (DT a) (ADJP (RB very) (JJ big)) (NN grin)))) (. .)))", "sentence2": "The woman is very happy.", "sentence2_binary_parse": "( ( The woman ) ( ( is ( very happy ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT The) (NN woman)) (VP (VBZ is) (ADJP (RB very) (JJ happy))) (. .)))"} diff --git a/test/data_for_tests/io/SNLI/snli_1.0_train.jsonl b/test/data_for_tests/io/SNLI/snli_1.0_train.jsonl new file mode 100755 index 00000000..8be03c11 --- /dev/null +++ b/test/data_for_tests/io/SNLI/snli_1.0_train.jsonl @@ -0,0 +1,5 @@ +{"annotator_labels": ["neutral"], "captionID": "3416050480.jpg#4", "gold_label": "neutral", "pairID": "3416050480.jpg#4r1n", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is training his horse for a competition.", "sentence2_binary_parse": "( ( A person ) ( ( is ( ( training ( his horse ) ) ( for ( a competition ) ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (VP (VBG training) (NP (PRP$ his) (NN horse)) (PP (IN for) (NP (DT a) (NN competition))))) (. .)))"} +{"annotator_labels": ["contradiction"], "captionID": "3416050480.jpg#4", "gold_label": "contradiction", "pairID": "3416050480.jpg#4r1c", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is at a diner, ordering an omelette.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is ( at ( a diner ) ) ) , ) ( ordering ( an omelette ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (PP (IN at) (NP (DT a) (NN diner))) (, ,) (S (VP (VBG ordering) (NP (DT an) (NN omelette))))) (. .)))"} +{"annotator_labels": ["entailment"], "captionID": "3416050480.jpg#4", "gold_label": "entailment", "pairID": "3416050480.jpg#4r1e", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is outdoors, on a horse.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is outdoors ) , ) ( on ( a horse ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (ADVP (RB outdoors)) (, ,) (PP (IN on) (NP (DT a) (NN horse)))) (. .)))"} +{"annotator_labels": ["neutral"], "captionID": "2267923837.jpg#2", "gold_label": "neutral", "pairID": "2267923837.jpg#2r1n", "sentence1": "Children smiling and waving at camera", "sentence1_binary_parse": "( Children ( ( ( smiling and ) waving ) ( at camera ) ) )", "sentence1_parse": "(ROOT (NP (S (NP (NNP Children)) (VP (VBG smiling) (CC and) (VBG waving) (PP (IN at) (NP (NN camera)))))))", "sentence2": "They are smiling at their parents", "sentence2_binary_parse": "( They ( are ( smiling ( at ( their parents ) ) ) ) )", "sentence2_parse": "(ROOT (S (NP (PRP They)) (VP (VBP are) (VP (VBG smiling) (PP (IN at) (NP (PRP$ their) (NNS parents)))))))"} +{"annotator_labels": ["entailment"], "captionID": "2267923837.jpg#2", "gold_label": "entailment", "pairID": "2267923837.jpg#2r1e", "sentence1": "Children smiling and waving at camera", "sentence1_binary_parse": "( Children ( ( ( smiling and ) waving ) ( at camera ) ) )", "sentence1_parse": "(ROOT (NP (S (NP (NNP Children)) (VP (VBG smiling) (CC and) (VBG waving) (PP (IN at) (NP (NN camera)))))))", "sentence2": "There are children present", "sentence2_binary_parse": "( There ( ( are children ) present ) )", "sentence2_parse": "(ROOT (S (NP (EX There)) (VP (VBP are) (NP (NNS children)) (ADVP (RB present)))))"} diff --git a/test/data_for_tests/io/SST-2/dev.tsv b/test/data_for_tests/io/SST-2/dev.tsv new file mode 100755 index 00000000..3fec0fa6 --- /dev/null +++ b/test/data_for_tests/io/SST-2/dev.tsv @@ -0,0 +1,6 @@ +sentence label +it 's a charming and often affecting journey . 1 +unflinchingly bleak and desperate 0 +allows us to hope that nolan is poised to embark a major career as a commercial yet inventive filmmaker . 1 +the acting , costumes , music , cinematography and sound are all astounding given the production 's austere locales . 1 +it 's slow -- very , very slow . 0 diff --git a/test/data_for_tests/io/SST-2/test.tsv b/test/data_for_tests/io/SST-2/test.tsv new file mode 100755 index 00000000..6ad46368 --- /dev/null +++ b/test/data_for_tests/io/SST-2/test.tsv @@ -0,0 +1,6 @@ +index sentence +0 uneasy mishmash of styles and genres . +1 this film 's relationship to actual tension is the same as what christmas-tree flocking in a spray can is to actual snow : a poor -- if durable -- imitation . +2 by the end of no such thing the audience , like beatrice , has a watchful affection for the monster . +3 director rob marshall went out gunning to make a great one . +4 lathan and diggs have considerable personal charm , and their screen rapport makes the old story seem new . diff --git a/test/data_for_tests/io/SST-2/train.tsv b/test/data_for_tests/io/SST-2/train.tsv new file mode 100755 index 00000000..4d7ea56c --- /dev/null +++ b/test/data_for_tests/io/SST-2/train.tsv @@ -0,0 +1,6 @@ +sentence label +hide new secretions from the parental units 0 +contains no wit , only labored gags 0 +that loves its characters and communicates something rather beautiful about human nature 1 +remains utterly satisfied to remain the same throughout 0 +on the worst revenge-of-the-nerds clichés the filmmakers could dredge up 0 diff --git a/test/data_for_tests/io/SST/dev.txt b/test/data_for_tests/io/SST/dev.txt new file mode 100755 index 00000000..46fca6bf --- /dev/null +++ b/test/data_for_tests/io/SST/dev.txt @@ -0,0 +1,6 @@ +(3 (2 It) (4 (4 (2 's) (4 (3 (2 a) (4 (3 lovely) (2 film))) (3 (2 with) (4 (3 (3 lovely) (2 performances)) (2 (2 by) (2 (2 (2 Buy) (2 and)) (2 Accorsi))))))) (2 .))) +(2 (2 (1 No) (2 one)) (1 (1 (2 goes) (2 (1 (2 (2 unindicted) (2 here)) (2 ,)) (2 (2 which) (3 (2 (2 is) (2 probably)) (3 (2 for) (4 (2 the) (4 best))))))) (2 .))) +(3 (2 And) (4 (3 (2 if) (1 (2 you) (1 (2 (2 (2 're) (1 not)) (2 nearly)) (4 (3 (3 moved) (2 (2 to) (1 tears))) (2 (2 by) (2 (2 (2 a) (2 couple)) (2 (2 of) (2 scenes)))))))) (2 (2 ,) (2 (2 you) (2 (2 (2 've) (1 (2 got) (2 (3 (2 ice) (2 water)) (2 (2 in) (2 (2 your) (2 veins)))))) (2 .)))))) +(4 (4 (2 A) (4 (3 (3 warm) (2 ,)) (3 funny))) (3 (2 ,) (3 (4 (4 engaging) (2 film)) (2 .)))) +(4 (3 (2 Uses) (3 (3 (4 (3 sharp) (4 (3 (4 humor) (2 and)) (2 insight))) (2 (2 into) (3 (2 human) (2 nature)))) (2 (2 to) (2 (2 examine) (2 (2 class) (1 conflict)))))) (2 (2 ,) (2 (2 (2 adolescent) (2 (2 (2 yearning) (2 ,)) (3 (2 (2 the) (2 roots)) (3 (2 of) (2 (2 friendship) (2 (2 and) (2 (2 sexual) (2 identity)))))))) (2 .)))) +(2 (2 (2 Half) (1 (2 (2 (2 (2 (2 Submarine) (2 flick)) (2 ,)) (2 (2 Half) (2 (2 Ghost) (2 Story)))) (2 ,)) (2 (2 All) (2 (2 in) (2 (2 one) (2 criminally)))))) (1 (1 neglected) (2 film))) diff --git a/test/data_for_tests/io/SST/test.txt b/test/data_for_tests/io/SST/test.txt new file mode 100755 index 00000000..ebf325d8 --- /dev/null +++ b/test/data_for_tests/io/SST/test.txt @@ -0,0 +1,6 @@ +(2 (3 (3 Effective) (2 but)) (1 (1 too-tepid) (2 biopic))) +(3 (3 (2 If) (3 (2 you) (3 (2 sometimes) (2 (2 like) (3 (2 to) (3 (3 (2 go) (2 (2 to) (2 (2 the) (2 movies)))) (3 (2 to) (3 (2 have) (4 fun))))))))) (2 (2 ,) (2 (2 Wasabi) (3 (3 (2 is) (2 (2 a) (2 (3 good) (2 (2 place) (2 (2 to) (2 start)))))) (2 .))))) +(4 (4 (4 (3 (2 Emerges) (3 (2 as) (3 (2 something) (3 rare)))) (2 ,)) (4 (2 (2 an) (2 (2 issue) (2 movie))) (3 (2 that) (3 (3 (2 's) (4 (3 (3 (2 so) (4 honest)) (2 and)) (3 (2 keenly) (2 observed)))) (2 (2 that) (2 (2 it) (2 (1 (2 does) (2 n't)) (2 (2 feel) (2 (2 like) (2 one)))))))))) (2 .)) +(2 (2 (2 The) (2 film)) (3 (3 (3 (3 provides) (2 (2 some) (3 (4 great) (2 insight)))) (3 (2 into) (3 (2 (2 the) (2 (2 neurotic) (2 mindset))) (3 (2 of) (2 (2 (2 (2 (2 all) (2 comics)) (2 --)) (2 even)) (3 (2 those) (4 (2 who) (4 (2 have) (4 (2 reached) (4 (4 (2 the) (3 (2 absolute) (2 top))) (2 (2 of) (2 (2 the) (2 game))))))))))))) (2 .))) +(4 (4 (2 Offers) (3 (3 (2 that) (3 (3 rare) (2 combination))) (2 (2 of) (3 (3 (3 entertainment) (2 and)) (2 education))))) (2 .)) +(3 (2 Perhaps) (4 (2 (1 (1 no) (2 picture)) (2 (2 ever) (2 made))) (3 (3 (2 (2 has) (2 (2 more) (3 literally))) (3 (2 showed) (2 (2 that) (2 (1 (2 (2 the) (1 road)) (1 (2 to) (0 hell))) (3 (2 is) (3 (2 paved) (3 (2 with) (3 (3 good) (2 intentions))))))))) (2 .)))) diff --git a/test/data_for_tests/io/SST/train.txt b/test/data_for_tests/io/SST/train.txt new file mode 100755 index 00000000..d5296ab0 --- /dev/null +++ b/test/data_for_tests/io/SST/train.txt @@ -0,0 +1,6 @@ +(3 (2 (2 The) (2 Rock)) (4 (3 (2 is) (4 (2 destined) (2 (2 (2 (2 (2 to) (2 (2 be) (2 (2 the) (2 (2 21st) (2 (2 (2 Century) (2 's)) (2 (3 new) (2 (2 ``) (2 Conan)))))))) (2 '')) (2 and)) (3 (2 that) (3 (2 he) (3 (2 's) (3 (2 going) (3 (2 to) (4 (3 (2 make) (3 (3 (2 a) (3 splash)) (2 (2 even) (3 greater)))) (2 (2 than) (2 (2 (2 (2 (1 (2 Arnold) (2 Schwarzenegger)) (2 ,)) (2 (2 Jean-Claud) (2 (2 Van) (2 Damme)))) (2 or)) (2 (2 Steven) (2 Segal))))))))))))) (2 .))) +(4 (4 (4 (2 The) (4 (3 gorgeously) (3 (2 elaborate) (2 continuation)))) (2 (2 (2 of) (2 ``)) (2 (2 The) (2 (2 (2 Lord) (2 (2 of) (2 (2 the) (2 Rings)))) (2 (2 '') (2 trilogy)))))) (2 (3 (2 (2 is) (2 (2 so) (2 huge))) (2 (2 that) (3 (2 (2 (2 a) (2 column)) (2 (2 of) (2 words))) (2 (2 (2 (2 can) (1 not)) (3 adequately)) (2 (2 describe) (2 (3 (2 (2 co-writer\/director) (2 (2 Peter) (3 (2 Jackson) (2 's)))) (3 (2 expanded) (2 vision))) (2 (2 of) (2 (2 (2 J.R.R.) (2 (2 Tolkien) (2 's))) (2 Middle-earth))))))))) (2 .))) +(3 (3 (2 (2 (2 (2 (2 Singer\/composer) (2 (2 Bryan) (2 Adams))) (2 (2 contributes) (2 (2 (2 a) (2 slew)) (2 (2 of) (2 songs))))) (2 (2 --) (2 (2 (2 (2 a) (2 (2 few) (3 potential))) (2 (2 (2 hits) (2 ,)) (2 (2 (2 a) (2 few)) (1 (1 (2 more) (1 (2 simply) (2 intrusive))) (2 (2 to) (2 (2 the) (2 story))))))) (2 --)))) (2 but)) (3 (4 (2 the) (3 (2 whole) (2 package))) (2 (3 certainly) (3 (2 captures) (2 (1 (2 the) (2 (2 (2 intended) (2 (2 ,) (2 (2 er) (2 ,)))) (3 spirit))) (2 (2 of) (2 (2 the) (2 piece)))))))) (2 .)) +(2 (2 (2 You) (2 (2 'd) (2 (2 think) (2 (2 by) (2 now))))) (2 (2 America) (2 (2 (2 would) (1 (2 have) (2 (2 (2 had) (1 (2 enough) (2 (2 of) (2 (2 plucky) (2 (2 British) (1 eccentrics)))))) (4 (2 with) (4 (3 hearts) (3 (2 of) (3 gold))))))) (2 .)))) +(3 (2 ``) (3 (2 Frailty) (4 (2 '') (3 (4 (3 (2 has) (3 (2 been) (3 (4 (3 (3 written) (3 (2 so) (3 well))) (2 ,)) (2 (2 (2 that) (2 even)) (1 (2 (2 a) (2 simple)) (1 (2 ``) (0 Goddammit))))))) (2 !)) (2 ''))))) +(4 (2 (2 Whether) (2 (2 (2 (2 or) (1 not)) (3 (2 you) (2 (2 're) (3 (3 enlightened) (2 (2 by) (2 (2 any) (2 (2 of) (2 (2 Derrida) (2 's))))))))) (2 (2 lectures) (2 (2 on) (2 (2 ``) (2 (2 (2 (2 (2 (2 the) (2 other)) (2 '')) (2 and)) (2 ``)) (2 (2 the) (2 self)))))))) (3 (2 ,) (3 (2 '') (3 (2 Derrida) (3 (3 (2 is) (4 (2 an) (4 (4 (2 undeniably) (3 (4 (3 fascinating) (2 and)) (4 playful))) (2 fellow)))) (2 .)))))) diff --git a/test/data_for_tests/io/imdb/dev.txt b/test/data_for_tests/io/imdb/dev.txt index 6b548a0c..423e158b 100644 --- a/test/data_for_tests/io/imdb/dev.txt +++ b/test/data_for_tests/io/imdb/dev.txt @@ -1,2 +1,6 @@ -neg It, at all, you have seen when harry met sally, then avoid this one. It will not only make you bang your head on the table as why can't bollywood even make a good remake; but also annoy you with the so called funny moments in it. The charm of the movie is missing. Ranee looks terrible. Saif tries to act like he is one hell of an actor. The plots that have been picked up from the original, don't look effective either. The part where both of them bring their friends along and they hit a note, it just doesn't look appealing. What can be more disastrous? you wanna waste some money, this is what you can get. Otherwise, put some more bucks, and watch the original. Its too good to miss.. -neg The monster from Enemy Mine somehow made his way into a small mountain community, where he has taken up residence. He's being hunted by a female doctor-turned-vigilante who is out to exterminate him. This female assassin, who looks like a refugee from a Motley Crue video, rides around on a motorcycle and tries to save a bunch of kids who have chosen to have a Big Chill weekend right smack dab in the middle of the monster's turf. Decapitations and lots of blood are primarily in place to draw attention away from the story which limps along like a bad version of the Island of Dr. Moreau (and yes, it's worse than the one with Val Kilmer). +neg You can never have seen either film and still know that The Jerk Too is a disaster. The question is not, "How did it get made," because if you throw money at anyone and tell them to make a film, they will do so.

No. The question is "Why, oh why, did Steve Martin allow it to be made?" I think he needed the money to fight a nuisance lawsuit and was determined it not cost him anything. He knew the sequel was going to be so frightful, that out of pride, he wouldn't even count it's royalties as income.

The only way this sequel could not be an embarrassment is to have had Carl Gottlieb and Steve Martin revive the nation's favorite poor black family.

And "dcreasy2001" (aka Mark Blankfield?): It's just transparently obvious that you worked on this film in some sad capacity, and the only way you can feel better about your involvement is to be the sequel's lone cheerleader as an IMDb user comment. I was praying for you to veer over into satire, but alas, you were really making an effort at spin. Why not 10 stars? +neg The Hazing is confused mumbo-jumbo that wants so hard to be The Evil Dead that it even references Bruce Campbell several times. The problem is, it is simply not in the same league as that terrific movie. This movie is nowhere near as original. The plot has been used before, by Kevin Tenney in Night of the Demons, and that was a lot more fun. This flick wastes too much time with complicated exposition before getting the kids into the spooky mansion and starting the demonic happenings.

Brad Dourif is, as usual, not given much to do here, but when he is on screen he puts in another over-the-top performance that would make Christopher Walken jealous. As for the acting of the kids, it's passable but by no means good. The shaky camera work is more annoying than clever or atmospheric. There are a few good moments when the first guy gets possessed and throws around some deadly one liners while dispatching his victims, but it was never scary for a second. The gore level is mid-range to low, but the director tries to make up for it by showing the actresses topless a few times. All in all, just okay if you have 87 minutes to waste. +neg I have seen bad movies before, but this one takes the "Worst Movie of a Lifetime" award by far !! Anthony Hopkins has to be completely mentally ill to have his name attached to this one - anywhere ! I will never see another movie with him in it, directing it, etc., etc. ! I can't believe the other actors & actresses that I liked, (in this picture), that stooped so low to be a part of this disaster ! There must be some great drugs out there ! For anyone to not be embarrassed to be a part of such a film, is beyond me ! Save your money on this one ! HUGE FLOP from beginning to end ! Shame on you Mr. Hopkins ! Also, shame on Christian Slater ! I can't believe you put your reputations on the line for this one ! +neg You may want to know up front that I am not a Mormon, unlike a good number of those who have already reviewed this film. I mention this so you'll understand that the way I look at the film may differ greatly from those in the faith. For some, being critical of the film might be seen as being critical of the faith--and that is NOT my intention. So, my review is that of an outsider trying to look inside and learn more about who this man and his people were. Well, after seeing the film, I doubt if I have learned much at all. Since I have been a history teacher, I have a good basic understanding about Young as well as Joseph Smith as well as the teachings of the church. But anyone wanting to see this film to really learn anything will probably be disappointed because the film seems so gosh-darn nice--too nice and too unrealistic in its portrayal. Plus, you learn practically nothing about the church's beliefs other than they are nice people, work hard and some have many wives (and this latter part is only barely hinted at in the film). Instead, the people are almost cartoon-like in their simplistic portrayals. Joseph Smith and Brigham Young and their followers are angelic, the non-Mormons were all devils and Brian Donlevy (playing EXACTLY the same sort of role Edward G. Robinson later played in THE TEN COMMANDMENTS) is the trouble-maker who claims to be a Mormon but just comes along so the film can have a bad guy. It's all so very simple....too simple. Almost like an indoctrination film or infomercial.

Brigham Young especially was a very complex man--with many good points (an excellent organizer and visionary) as well as bad (don't even get me started on his views about Blacks within the church or intermarriage). To portray him in such vague terms is just plain silly. It's also a lot like how Gandhi was portrayed in the film with Ben Kingsley--only the facts that led to his being almost super-human were emphasized. Heck, now that I think about that, this is the trouble with most religious films--they often come off as one-dimensional, trite and bland. Let's have a full and more complete film of these men--one that will stick to facts and not emotional appeals.

Now if you can ignore the fact that you won't learn very much about the faith or its second leader, the film is enjoyable enough. It's obvious someone at 20th Century-Fox really cared about the film, as they had a wonderful cast of both premier actors (Tyrone Power), up and coming actors (Linda Darnell, Jane Darwell and Vincent Price) and wonderful character actors (Dean Jagger, John Carradine and Brian Donlevy). The film also had wonderful location shooting and lots of gloss. It just didn't have a lot to tell us other than they were all "swell". Plus, there were plenty of factual errors and a few just plain dumb scenes. A few of the mistakes include Young taking over the helm immediately after the death of Joseph Smith (it was three years later), no mention of the various Mormon denominations and splinter groups, talk of "gold in California"--even though it was 1847 and gold wouldn't be discovered until 1948, as well as no specific mention of polygamy or Smith's many wives. Just plain dumb scenes include Carradine pulling out a gun and waving it about in the courtroom scene--and no one seemed to care--even though it was a very hostile audience! Don't you think at least the judge would tell him to put it away and stop threatening people with it?!

One final comment. Do not, I repeat, do not watch this film when it's shown on American Movie Classics (a one great station that has sunk a lot in recent years). While I am critical of the film because of its simplistic message, I was horrified with the complete disrespect the station had for the church and its traditions. What I mean is this. The film was punctuated with ads for penis enlargement formulas as well as tons of pop-ups (some advertising a show that features the "sexiest cast"). Talk about disrespectful and gross and I would be just as offended if they did this for any other religious film. By doing this, they not only insult the faith but marginalize their market--after all, who is into hearing about these things AND the life of Brigham Young?! Is this a movie, in this form, that you can show to your kids or recommend to others?! +pos Fifteen years later and Paris Is Burning is still aflame. This is a classic in black gay films, right up there with the other honorary black gay films, The Color Purple and Mahoganoy. This seminal work captures underground and underclass (i.e."underserved) black and Latin gay culture and community like no other work before or since, including all the sentimental Harlem Rennaissance gay retrospectives and renderings. They're good, but this is the best (dare I say the only "real") film you'll find on the subject. It's Relentlessy Cunty (the classic house music invention)comes to Hollywood, non-stop, hilarious camp (like only we do it) and dead-on social critique. All this by a white female director (who obviously must have been a Sister Gurl or Mizz Thing in a former life.) I could go on, but I think you get the point by now: I love this movie! +pos I have been an admirer of Edward Burtynsky's work for years, and it was such a pleasure to be able to see the man at work, thanks to Jennifer Baichwal's documentary. The severe beauty of the ship-breaking yard in Bangladesh, the stone quarry in Vermont, the enormous assembly plant in China, the beleaguered old neighbourhoods in Shanghai that are just waiting to be torn down: these landscapes are captured so well by the photographer and the filmmaker.

At times I thought of old TV documentaries on abandoned coal mines and plastic-mold factories; the sort of stuff I grew up watching. Burtynsky's work has the great value of pointing out how the industrial activity has only shifted to Asia, it has not stopped. The strangest scene for me was the computer scrap-yard somewhere in China--the waste had a threatening air about it, while the workers were very jovial. diff --git a/test/data_for_tests/io/imdb/test.txt b/test/data_for_tests/io/imdb/test.txt index c9bfae74..68768ec6 100644 --- a/test/data_for_tests/io/imdb/test.txt +++ b/test/data_for_tests/io/imdb/test.txt @@ -1,2 +1,6 @@ neg Alan Rickman & Emma Thompson give good performances with southern/New Orleans accents in this detective flick. It's worth seeing for their scenes- and Rickman's scene with Hal Holbrook. These three actors mannage to entertain us no matter what the movie, it seems. The plot for the movie shows potential, but one gets the impression in watching the film that it was not pulled off as well as it could have been. The fact that it is cluttered by a rather uninteresting subplot and mostly uninteresting kidnappers really muddles things. The movie is worth a view- if for nothing more than entertaining performances by Rickman, Thompson, and Holbrook. neg I have seen this movie and I did not care for this movie anyhow. I would not think about going to Paris because I do not like this country and its national capital. I do not like to learn french anyhow because I do not understand their language. Why would I go to France when I rather go to Germany or the United Kingdom? Germany and the United Kingdom are the nations I tolerate. Apparently the Olsen Twins do not understand the French language just like me. Therefore I will not bother the France trip no matter what. I might as well stick to the United Kingdom and meet single women and play video games if there is a video arcade. That is all. +neg In Los Angeles, the alcoholic and lazy Hank Chinaski (Matt Dillon) performs a wide range of non-qualified functions just to get enough money to drink and gamble in horse races. His primary and only objective is writing and having sexy with dirty women.

"Factotum" is an uninteresting, pointless and extremely boring movie about an irresponsible drunken vagrant that works a couple of days or weeks just to get enough money to buy spirits and gamble, being immediately fired due to his reckless behavior. In accordance with IMDb, this character would be the fictional alter-ego of the author Charles Bukowski, and based on this story, I will certainly never read any of his novels. Honestly, if the viewer likes this theme of alcoholic couples, better off watching the touching and heartbreaking Hector Babenco's "Ironweed" or Marco Ferreri's "Storie di Ordinaria Follia" that is based on the life of the same writer. My vote is four.

Title (Brazil): "Factotum – Sem Destino" ("Factotum – Without Destiny") +neg This film is bundled along with "Gli fumavano le Colt... lo chiamavano Camposanto" and both films leave a lot to be desired in the way of their DVD prints. First, both films are very dark--occasionally making it hard to see exactly what's happening. Second, neither film has subtitles and you are forced to watch a dubbed film--though "Il Prezzo del Potere" does seem to have a better dub. Personally, I always prefer subtitles but for the non-purists out there this isn't a problem. These DVD problems, however, are not the fault of the original film makers--just the indifferent package being marketed four decades later.

As for the film, it's about the assassination of President Garfield. This is a MAJOR problem, as Van Johnson looks about as much like Garfield as Judy Garland. In no way whatsoever does he look like Garfield. He's missing the beard, has the wrong hair color and style and is just not even close in any way (trust me on this, I am an American History teacher and we are paid to know these sort of things!). The real life Garfield was a Civil War general and looked like the guys on the Smith Brothers cough drop boxes. Plus, using some other actor to provide the voice for Johnson in the dubbing is just surreal. Never before or since has Van Johnson sounded quite so macho!! He was a fine actor...but certainly not a convincing general or macho president.

In addition to the stupid casting, President Garfield's death was in no way like this film. It's obvious that the film makers are actually cashing in on the crazy speculation about conspiracies concerning the death of JFK, not Garfield. Garfield was shot in Washington, DC (not Dallas) by a lone gunman with severe mental problems--not a group of men with rifles. However, according to most experts, what actually killed Garfield (over two months later) were incompetent doctors--who probed and probed and probed to retrieve a bullet (to no avail) and never bothered cleaning their hands or implements in the process. In other words, like George Washington (who was basically killed by repeated bloodletting when suffering with pneumonia) he died due to malpractice. In the movie they got nothing right whatsoever...other than indeed President Garfield was shot.

Because the film bears almost no similarity to real history, it's like a history lesson as taught from someone from another planet or someone with a severe brain injury. Why not also include ninjas, fighting robots and the Greek gods while you're at it?!?! Aside from some decent acting and production values, because the script is utter cow crap, I don't recommend anyone watch it. It's just a complete and utter mess. +neg I only comment on really very good films and on utter rubbish. My aim is to help people who want to see great films to spend their time - and money - wisely.

I also want to stop people wasting their time on garbage, and want to publicize the fact that the director/producer of these garbage films can't get away with it for very long. We will find out who you are and will vote with out feet - and wallets.

This film clearly falls into the garbage category.

The director and writer is John Shiban. It's always a bad sign when the writer is also the director. Maybe he wants two pay cheques. He shouldn't get any. So remember the name - John SHIBAN. And if you see anything else by him, forget it.

I won't say anything about the plot - others have already. I am a little worried by how much the director likes to zoom in to the poor girl's face when she is crying and screaming. These long duration shots are a little worrying and may say something about the state of mind of Mr. Shiban. Maybe he should get psychiatric help.

Enough already. It's crap - don't waste your time on it. +neg When you look at the cover and read stuff about it an entirely different type of movie comes to mind than what you get here. Then again maybe I read the summary for the other movie called "Mausolem" instead as there were two movies of this title released about the same time with both featuring plots that had key elements in common. However, reading stuff about that movie here I know I saw this one and not that one and that movie is even less what one would imagine a movie with that title would be about. I will be honest, I expect more of a zombie type picture and you get that in this movie to some degree. However, there is more stuff involving the occult and strange powers as the opening scene of the people being taken away by the coroner at the beginning of the film will attest to. The movie also has the old theme of kids going somewhere they do not belong to have some crazy party, in this case it is in fact a mausoleum. The other movie I do not think really has that key feature playing that prominent role in the movie and I see the score for this one is higher too, still it was just not the movie I was expecting. diff --git a/test/data_for_tests/io/imdb/train.txt b/test/data_for_tests/io/imdb/train.txt index d6ac6b68..bbf4d799 100644 --- a/test/data_for_tests/io/imdb/train.txt +++ b/test/data_for_tests/io/imdb/train.txt @@ -1,2 +1,6 @@ +neg The monster from Enemy Mine somehow made his way into a small mountain community, where he has taken up residence. He's being hunted by a female doctor-turned-vigilante who is out to exterminate him. This female assassin, who looks like a refugee from a Motley Crue video, rides around on a motorcycle and tries to save a bunch of kids who have chosen to have a Big Chill weekend right smack dab in the middle of the monster's turf. Decapitations and lots of blood are primarily in place to draw attention away from the story which limps along like a bad version of the Island of Dr. Moreau (and yes, it's worse than the one with Val Kilmer). neg I'll try to use words to describe this on....

I saw the original, which was good in its own way, but back then I should have feared a sequel.

And I was 'afraid' when I picked this one up, but now that I've seen it, I have to say, it's even worse then I thought. Why these movies still get money still makes my mind spin.

Let's start with the actors;they aren't all that good, but it has to be said, some make heads turn by being just plain awful. But what can an actor do with a script like this one. It's trying to be a copy of the original only this time the places have changed, any form of story is gone and any attempt of actually coming up with something that hasn't been done before, fails miserably. In a futile attempt to get it up-to-date, they try to make it exciting by making use of the whole 'big-brother' theme , but that has been worn out ages ago and offers nothing but a filler for between the beginning and the end. An attempt was made to try to save the movie by making a ton of references to the '83 original, but it just ended up being plain funny and sometimes a bit sad. In conclusion, if you have nothing , and I mean nothing , to do... go watch it, or play Frisbee... with the DVD.... by yourself. It'll offer you the same amount of fun.. I promise -pos This movie is totally wicked! It's really great to see MJH in a different role than her Sabrina character! The plot is totally cool, and the characters are excellently written. Definitely one of the best movies!! +pos Most yeti pictures are fatally undermined by a grave paucity of energy and enthusiasm. Not so this gloriously bent, batty and berserk over-the-top Italian-made shot-in-Canada kitsch gut-buster: It's a wildly ripe and vigorously moronic ghastly marvel which reaches a stunning apotheosis of righteously over-baked "what the hell's going on?" crackpot excess and inanity.

A freighter ship crew discovers the body of a 30-foot yeti that resembles a hirsute 70's disco stud (complete with jumbo wavy afro) perfectly preserved in a large chunk of ice. They dethaw the beast, jolt him back to life with electric charges, grossly mistreat him, and keep the poor hairy Goliath in an enormous glass booth. Before you can say "Hey, the filmmakers are obviously ripping off 'King Kong'," our titanic abominable snowdude breaks free of his cage, grabs the first luscious nubile blonde Euro vixen (the gorgeous Pheonix Grant) he lays lustful eyes on, and storms away with his new lady love. The yeti gets recaptured and flown to Toronto to be showed off to a gawking audience. Of course, he breaks free again, nabs the vixen, and goes on the expected stomping around the city rampage.

The sublimely stupid dialogue (sample line: "Philosophy has no place in science, professor"), cheesy (far from) special effects (the horrendous transparent blue screen work and cruddy Tonka toy miniatures are especially uproarious in their very jaw-dropping awfulness), clunky (mis)direction, and a heavy-handed script that even attempts a clumsily sincere "Is the yeti a man or a beast?" ethical debate all combine together to create one of the single most delightfully ridiculous giant monster flicks to ever roar its absurd way across the big screen. Better still, we also have a few funky offbeat touches to add extra shoddy spice to the already succulently schlocky cinematic brew: the vixen accidentally brushes against one of the yeti's nipples, which causes it to harden and elicits a big, leering grin of approval from the lecherous behemoth (!); the vixen nurses the yeti's wounded hand while he makes goo-goo eyes at her, the yeti smashes windows with his feet while climbing a towering office building, and the furry fellow even breaks a man's neck with his toes (!!). Overall, this singularly screwball and shamefully unheralded should-be camp classic stands tall as a remarkable monolith of infectiously asinine celluloid lunacy that's eminently worthy of a substantial hardcore underground cult following. +pos One of the best movies I ever saw was an Irish movie titled Philadelphia,Here I Come. I read the play before I saw the movie and loved them both. It's the story of a young man preparing to leave Ireland to go to America because he can't earn a living in Ireland. It is told both from the perspective of the young man(whom the other characters in the film can see) and another young man representing his uncensored thoughts and feelings., but who cannot be seen by the other characters in the film. It is a very sad movie, but deeply touching, and I would recommend this film to anyone who wants something to think about. I love any Irish movie, or almost any movie about Ireland, and any film that has the late Irish actor Donal McCann in it gets my vote.I would watch that man chew gum for 2 hours on screen, and unfortunately,I have.Terrible shame to have lost him so young. +pos There is such rubbish on the cable movie channels that I hit a gem with this one. From beginning to end it had me gripped and deserves top marks.

Father of two sons hears messages from "God" to kill people who he is told are 'demons'.

When the opening credits showed the director as one of the cast that can often be a warning of a bad film; exceptionally it is the reverse here as the drama is non-stop from beginning to end.

And there is not one moment in the movie when one is not fully enthralled as there are no unnecessary or needless sub-plots, and the script is first class.

All the actors give wholly convincing performances especially the lead child actor who is exceptional.

This film is at least as good as the likes of 'Silence of the Lambs'. +pos This is a nice piece of work. Very sexy and engaging enough plot to keep my interest throughout. Its main disadvantage is that it seems like it was made-for-TV: Full screen, and though there were several sex scenes, there was absolutely no nudity (but boy did it come close!). Strange, too, since Netflix shows that it was rated R.

Nonetheless, very titillating, and I wish Alicia Silverstone made more movies like this.

One Netflix reviewer stated that it was part of a series, but I have been unable to find out what series that is. I'd like to find out, though, because this movie was THAT good.

Walt D in LV. 8/23/2005 diff --git a/test/data_for_tests/io/yelp_review_full/dev.csv b/test/data_for_tests/io/yelp_review_full/dev.csv new file mode 100755 index 00000000..ecc93b0b --- /dev/null +++ b/test/data_for_tests/io/yelp_review_full/dev.csv @@ -0,0 +1,6 @@ +"2","Two meals, on the recommendation of a friend who lives near the place, and after the second trip, I was compelled to write. 'Rocky' would definitely describe the experiences.\n\nOn the first trip, I went to try their (at that time)raved about Reuben. And YET to find a true good Reuben in da burgh, I tried it.\n\nWell, they were out of the proper bread, and the guy had to run to the store to buy the closest thing he could find, which was not the proper bread, and instead of one of their 'raved about' Reubens, I received two mini-Reubens, which basically took the guts from one Reuben, and spread it out onto two sandwiches on regular sized bread. I ate it. It wasn't great, but they swore it was because they'd run out of the bread. Bread or not, it still wasn't great. The atmosphere was pleasant in that 'blue collar bar' kind of way, and the staff was very nice, but not a winning pitch on the Reuben.\n\nThe second trip was after a long day of moving furniture with the same friend. Sat in the back room, instead of the bar, which felt more like a restaurant, of course, with the big screen TV covering the sports of the moment.\n\nI was in the mood for dinner this time, and after a scan, decided on fried chicken and mashed potatoes with the salad bar. My friend ordered one of her faves, the breaded pork chops.\n\nWe hit the salad bar, which was uber-basic. Three soups (mostly vegetable loaded, which left me out), basic iceberg lettuce mix (very probably out of a bag), a few veggie toppings, and three or four dressings. It was a basic salad, no big deal. More or less an appetizer filler before the meal.\n\nThe mind-blower in this trip was the ordering of the fried chicken dinner. Our waiter looked like a 19 year old gas station attendant, skinny little blonde guy with a sweet but incredibly naive face, and an air of vapidity, which was confirmed when I placed my order. I asked what chicken pieces came in the dinner, and asked if it was possible to only get dark meat. I never imagined how confusing a question that could possibly be. It literally took him two trips back to the kitchen to 'ask', and the child honestly had no clue what 'white meat' and 'dark meat' meant. The first answer he came back with was that the chicken came in a pre-portioned prepared bag, kind of Kentucky Fried Chicken style...which didn't answer my question, thus prompting the second trip. \n\nAfter the second trip back I heard the cook holler 'Tell him I'll fix him up'. \n\nWell, the chicken was prepackaged dreck like you'd find in the freezer case of Walmart, tiny and not good, and the potatoes had that slight tinge of chem-spuds flavor, laden with some kind of chopped up green (parsley?), and a side of that basic brown gravy served up in 5 gallon buckets.\n\nThank goodness for the basic salad bar.\n\nEven my friend admitted that her pork chops were different and not what she'd expected. They also appeared to be from a freezer bag.\n\nThe irony was that the boy who didn't know white meat from dark meat, was chatting with some other customers...about baseball...and he was a genius about the mindless sport of baseball. Ahhhh da burgh.\n\nThird base? Nah...why bother when there are so many other options around. Go on in a grab a beer and chat black and gold if you happen to be in Carnegie...they can help you out all types of ways in that area. Just don't go hungry if you actually have tastebuds.\n\nFrom what I understand it 'used to be' really good homecooked food. But apparently, mama has left the kitchen." +"4","I belong to this gym... I live in the South section of Pittsburgh, and I find that this gym is not too far from me. The staff is friendly, the equipment is quite good. You get two free personal training sessions when you join. They have lots of weights (which my boyfriend uses) and a decent cardio room. The only thing I would say is to increase some of the cardio equipment. Water is only $1 a bottle!" +"3","I've been to Papa J's twice and had mixed experiences.\n\nBoth times I had the banana pepper appetizer, which is great and goes really well with the FRESH and delicious bread and cheese they give you at the start of your meal.\n\nFor entrees, me and my girlfriend have had mixed experience. I've had the fish sandwich (very good) and the eggplant parm sandwich (okay). My girlfriend got the salad with bread and basil on it, but the basil was over powering and the bread was soggy with the dressing. \n\nThe service is also a mixed bag. The first time our server went out of her way to take care of us and even MADE me cocktail sauce for my fish sandwich. The second time, the server was lackluster, didn't know anything about the menu and wasn't able to take proper care of us. \n\nI would return to Papa J's, but I my terrible experience last time isn't enough to say it would be my first pick of places to eat around Carnegie/Robinson." +"4","Yay, I'm a fan but sometimes service is a little slow, it was very good for us this visit. Go to Papa j's every once in a while but mostly for the White Pizza. It is the best white pizza I have ever had. Order the white pizza on our visit this weekend... it has garlic, spinach, feta cheese and we usually add some veggie on top. It was delicious! Order fried calamari and it was OK...note to self next time try the calamari roman style.\n\nLike the dinning room with the hardwood floors and bright lighting. \n\nThe bar was jumping thou never go to the bar." +"3","Had dinner at Papa J's with a group of 6. I loved how the restaurant is in a old brick building with large windows. It felt like a neighborhood restaurant. On a Saturday night, the restaurant was full but not crowded. We were seated in a room with poor acoustics. It was difficult to hear people at our table and the waitress. While she tried, I can see the asperation in her face when she had to repeat the specials to both sides of the table.\n\nPeople ordered bourbon on the rocks before dinner which seemed watered down, while my lemon drop was made nice. The bread was delicious! Can you describe it to be creamy? The fried zucchini was lightly breaded and not too oily. It was a large portion made up of 2 sliced zucchinis.\n\nWe ordered a variety of dishes. The pasta dish was dry with more pasta than sauce or meat. Those who ordered the fish special thought it was delicious. The shrimp dish was enjoyed as well. I had the chicken marsala which was pretty good. The marsala sauce wasn't too thick, and the chicken moist.\n\nHard to tell if the deserts were \""homemade.\"" The tiramisu and spumoni were small in portion and meant for one. \n\nOn the whole, I was on the fence with my overall impression of Papa J's. \""A-ok\"" probably is the best way to describe it." +"2","Rather typical SnS. Had a good lunch crowd. Milkshake was good but not as good as EnP down the street. It took to long to get the burger for some reason, 25 minutes, I realized cooked to order but this is a little long for SnS. Ordered the Guacamole Steakburger and it only had a small portion of Gauc...not your usual amount..kitchen was not up to speed on portion sizing for some reason. Definitely did not look like the picture on the website. Oh well!" diff --git a/test/data_for_tests/io/yelp_review_full/test.csv b/test/data_for_tests/io/yelp_review_full/test.csv new file mode 100755 index 00000000..63d84891 --- /dev/null +++ b/test/data_for_tests/io/yelp_review_full/test.csv @@ -0,0 +1,6 @@ +"1","I got 'new' tires from them and within two weeks got a flat. I took my car to a local mechanic to see if i could get the hole patched, but they said the reason I had a flat was because the previous patch had blown - WAIT, WHAT? I just got the tire and never needed to have it patched? This was supposed to be a new tire. \nI took the tire over to Flynn's and they told me that someone punctured my tire, then tried to patch it. So there are resentful tire slashers? I find that very unlikely. After arguing with the guy and telling him that his logic was far fetched he said he'd give me a new tire \""this time\"". \nI will never go back to Flynn's b/c of the way this guy treated me and the simple fact that they gave me a used tire!" +"1","Don't waste your time. We had two different people come to our house to give us estimates for a deck (one of them the OWNER). Both times, we never heard from them. Not a call, not the estimate, nothing." +"1","All I can say is the worst! We were the only 2 people in the place for lunch, the place was freezing and loaded with kids toys! 2 bicycles, a scooter, and an electronic keyboard graced the dining room. A fish tank with filthy, slimy fingerprints smeared all over it is there for your enjoyment.\n\nOur food came... no water to drink, no tea, medium temperature food. Of course its cold, just like the room, I never took my jacket off! The plates are too small, you food spills over onto some semi-clean tables as you sit in your completely worn out booth seat. The fried noodles were out of a box and nasty, the shrimp was mushy, the fried rice was bright yellow.\n\nWe asked for water, they brought us 1 in a SOLO cup for 2 people. I asked for hot tea, they said 10 minutes. What Chinese restaurant does not have hot tea available upon request?\n\nOver all.... my first and last visit to this place. The only good point was that it was cheap, and deservingly so." +"1","I have been to this restaurant twice and was disappointed both times. I won't go back. The first time we were there almost 3 hours. It took forever to order and then forever for our food to come and the place was empty. When I complained the manager was very rude and tried to blame us for taking to long to order. It made no sense, how could we order when the waitress wasn't coming to the table? After arguing with me he ended up taking $6 off of our $200+ bill. Ridiculous. If it were up to me I would have never returned. Unfortunately my family decided to go here again tonight. Again it took a long time to get our food. My food was cold and bland, my kids food was cold. My husbands salmon was burnt to a crisp and my sister in law took one bite of her trout and refused to eat any more because she claims it was so disgusting. The wedding soup and bread were good, but that's it! My drink sat empty throughout my meal and never got refilled even when I asked. Bad food, slow service and rude managers. I'll pass on this place if my family decides to go again. Not worth it at all with all the other good Italian options around." +"1","Food was NOT GOOD at all! My husband & I ate here a couple weeks ago for the first time. I ordered a salad & basil pesto cream pasta & my husband ordered the spinach & feta pasta. The salad was just a huge plate of spring mix (nothing else in it) with WAY to much vinegar dressing. My lettuce was drowning in the vinegar. My pesto pasta had no flavor (did not taste like a cream sauce to me) & the pesto was so runny/watery & way too much sauce not enough noodles. My husband's pasta had even less flavor than mine. We ate about a quarter of the food & couldn't even finish it. We took it home & it was so bad I didn't even eat my leftovers. And I hate wasting food!! Plus the prices are expensive for the amount of food you get & of course the poor quality. Don't waste your time eating here. There are much better Italian restaurants in Pittsburgh." +"3","This is a tiny Starbucks and it locations like this (although cute) makes you wonder if your really meant to hang out or just grab your coffee and leave. Leaving is always a good idea at this location anyway since you have a nice fountain in the back with benches and it is a central part of the Waterfront Shopping. \n\nStarbuck isn't my favorite coffee chain by any means. Is it just me or do all Starbuck coffees taste a little burnt and bitter? No matter how trendy, cool and upscale their establishments are I can't get around the yicky tasting bitterness of Staryucks regular coffees. Talk about over roasting a bean...Maybe something has changed with their regular coffee but I have not drank it in about a year. I am not one for soy caramel latte foofy stuff. Still I'll give the establishment tres estrellas for the fact that their espresso is acceptable and doesn't taste half as bad as the regular coffee bean." diff --git a/test/data_for_tests/io/yelp_review_full/train.csv b/test/data_for_tests/io/yelp_review_full/train.csv new file mode 100755 index 00000000..032d423a --- /dev/null +++ b/test/data_for_tests/io/yelp_review_full/train.csv @@ -0,0 +1,6 @@ +"5","dr. goldberg offers everything i look for in a general practitioner. he's nice and easy to talk to without being patronizing; he's always on time in seeing his patients; he's affiliated with a top-notch hospital (nyu) which my parents have explained to me is very important in case something happens and you need surgery; and you can get referrals to see specialists without having to see him first. really, what more do you need? i'm sitting here trying to think of any complaints i have about him, but i'm really drawing a blank." +"2","Unfortunately, the frustration of being Dr. Goldberg's patient is a repeat of the experience I've had with so many other doctors in NYC -- good doctor, terrible staff. It seems that his staff simply never answers the phone. It usually takes 2 hours of repeated calling to get an answer. Who has time for that or wants to deal with it? I have run into this problem with many other doctors and I just don't get it. You have office workers, you have patients with medical needs, why isn't anyone answering the phone? It's incomprehensible and not work the aggravation. It's with regret that I feel that I have to give Dr. Goldberg 2 stars." +"4","Been going to Dr. Goldberg for over 10 years. I think I was one of his 1st patients when he started at MHMG. He's been great over the years and is really all about the big picture. It is because of him, not my now former gyn Dr. Markoff, that I found out I have fibroids. He explores all options with you and is very patient and understanding. He doesn't judge and asks all the right questions. Very thorough and wants to be kept in the loop on every aspect of your medical health and your life." +"3","Got a letter in the mail last week that said Dr. Goldberg is moving to Arizona to take a new position there in June. He will be missed very much. \n\nI think finding a new doctor in NYC that you actually like might almost be as awful as trying to find a date!" +"1","I don't know what Dr. Goldberg was like before moving to Arizona, but let me tell you, STAY AWAY from this doctor and this office. I was going to Dr. Johnson before he left and Goldberg took over when Johnson left. He is not a caring doctor. He is only interested in the co-pay and having you come in for medication refills every month. He will not give refills and could less about patients's financial situations. Trying to get your 90 days mail away pharmacy prescriptions through this guy is a joke. And to make matters even worse, his office staff is incompetent. 90% of the time when you call the office, they'll put you through to a voice mail, that NO ONE ever answers or returns your call. Both my adult children and husband have decided to leave this practice after experiencing such frustration. The entire office has an attitude like they are doing you a favor. Give me a break! Stay away from this doc and the practice. You deserve better and they will not be there when you really need them. I have never felt compelled to write a bad review about anyone until I met this pathetic excuse for a doctor who is all about the money." +"5","Top notch doctor in a top notch practice. Can't say I am surprised when I was referred to him by another doctor who I think is wonderful and because he went to one of the best medical schools in the country. \nIt is really easy to get an appointment. There is minimal wait to be seen and his bedside manner is great." diff --git a/test/data_for_tests/io/yelp_review_polarity/dev.csv b/test/data_for_tests/io/yelp_review_polarity/dev.csv new file mode 100755 index 00000000..09228213 --- /dev/null +++ b/test/data_for_tests/io/yelp_review_polarity/dev.csv @@ -0,0 +1,6 @@ +"1","Hoofah." +"1","Two meals, on the recommendation of a friend who lives near the place, and after the second trip, I was compelled to write. 'Rocky' would definitely describe the experiences.\n\nOn the first trip, I went to try their (at that time)raved about Reuben. And YET to find a true good Reuben in da burgh, I tried it.\n\nWell, they were out of the proper bread, and the guy had to run to the store to buy the closest thing he could find, which was not the proper bread, and instead of one of their 'raved about' Reubens, I received two mini-Reubens, which basically took the guts from one Reuben, and spread it out onto two sandwiches on regular sized bread. I ate it. It wasn't great, but they swore it was because they'd run out of the bread. Bread or not, it still wasn't great. The atmosphere was pleasant in that 'blue collar bar' kind of way, and the staff was very nice, but not a winning pitch on the Reuben.\n\nThe second trip was after a long day of moving furniture with the same friend. Sat in the back room, instead of the bar, which felt more like a restaurant, of course, with the big screen TV covering the sports of the moment.\n\nI was in the mood for dinner this time, and after a scan, decided on fried chicken and mashed potatoes with the salad bar. My friend ordered one of her faves, the breaded pork chops.\n\nWe hit the salad bar, which was uber-basic. Three soups (mostly vegetable loaded, which left me out), basic iceberg lettuce mix (very probably out of a bag), a few veggie toppings, and three or four dressings. It was a basic salad, no big deal. More or less an appetizer filler before the meal.\n\nThe mind-blower in this trip was the ordering of the fried chicken dinner. Our waiter looked like a 19 year old gas station attendant, skinny little blonde guy with a sweet but incredibly naive face, and an air of vapidity, which was confirmed when I placed my order. I asked what chicken pieces came in the dinner, and asked if it was possible to only get dark meat. I never imagined how confusing a question that could possibly be. It literally took him two trips back to the kitchen to 'ask', and the child honestly had no clue what 'white meat' and 'dark meat' meant. The first answer he came back with was that the chicken came in a pre-portioned prepared bag, kind of Kentucky Fried Chicken style...which didn't answer my question, thus prompting the second trip. \n\nAfter the second trip back I heard the cook holler 'Tell him I'll fix him up'. \n\nWell, the chicken was prepackaged dreck like you'd find in the freezer case of Walmart, tiny and not good, and the potatoes had that slight tinge of chem-spuds flavor, laden with some kind of chopped up green (parsley?), and a side of that basic brown gravy served up in 5 gallon buckets.\n\nThank goodness for the basic salad bar.\n\nEven my friend admitted that her pork chops were different and not what she'd expected. They also appeared to be from a freezer bag.\n\nThe irony was that the boy who didn't know white meat from dark meat, was chatting with some other customers...about baseball...and he was a genius about the mindless sport of baseball. Ahhhh da burgh.\n\nThird base? Nah...why bother when there are so many other options around. Go on in a grab a beer and chat black and gold if you happen to be in Carnegie...they can help you out all types of ways in that area. Just don't go hungry if you actually have tastebuds.\n\nFrom what I understand it 'used to be' really good homecooked food. But apparently, mama has left the kitchen." +"2","I've lived in Pittsburgh for 6 years, and in Carnegie for over 2 years, and by far, this is the best greasy spoon joint I've found. If you can stomach the wait (no reservations, naturally), you'll enjoy overflowing plates of goodness, thanks to the well-seasoned griddle where all of the food is made. \n\nHere are the highlights:\n\n-Cheap: Breakfast for two can be well under $10, with lunch around the same.\n-Crowded: Get there early and expect to wait. They close pretty early on the weekends too (oddly, at 12:45pm)\n-Cash only\n-Huge portions: When ordering fries or homefries, always get the half order, unless you're a lumberjack\n-About those homefries: They're often undercooked. I've had better, believe me. My favorite things to eat in life are potato products.\n-My favorite item: hot sausage sandwich on thick Italian toast, with cheese, lettuce, tomato and mayo" +"2","Classic breakfast joint. Grimy looking hole in the wall located on one end of a seedy looking strip mall. Window is opaque due to the grease so you can't hardly see inside. On the outside, there are about a dozen people waiting to get in. When you finally do get inside, you see that there are 15 tables and a counter, all occupied by people from all walks of life.\n\nWhat's the attraction behind this flea hole? The FOOD! Lots of it and dirt cheap. I sat at a vacant stool behind the formica counter and ordered the mixed grill. Potatoes, eggs, sausage, bacon and Italian toast. A giant mound of food guaranteed to sooth any hangover. I swear the full mixed grill had two pounds of food. Neat thing is that the grill is right in front of you so you can see your potatoes and eggs frying in a pool of fresh grease. All that food, plus coffee and tip for around ten bucks. Cash only, so put that plastic away.\n\nOnly bad thing that could happen is some douche bag from the Food Network or Travel Channel will make this place famous, and then I'll never be able to get in." +"1","Some of the worst pizza I've ever had. We used a coupon from the paper for a 2 topping 8 cut Sicilian. First of all the pizza wasn't even cut through, and the sad attempt at cutting was so uneven that 4 of the slices were about an inch wide, while the others were about 4\"" each. The toppings were scarce, they used mini pepperoni and put maybe 8 on the whole pizza. The onions were huge chunks and the mushrooms were straight from a can. The worst part though was the thick doughy crust that tasted more like a fishy sourdough roll. I'm serious... It was so noticeable that it made me wonder if the dough was bad or if they for some weird reason put fish sauce in it. It was gross. \n\nWe also ordered steak and Italian hoagies. The veggies were old and wilted, and there was no dressing on either. The Italian had deli meat that was clearly bottom of the line and not very generous. The \""steak\"" (if you an call it that) was greyish instead of brown and looked like it was a processed meat chopped into pieces. No flavor or seasoning and the texture was reminiscent of spam. It was so bad that I only ate 1/4 of it and tossed the rest. \n\nI have ordered from here in the past and always been disappointed. I thought I would give them another try since I'd never ordered a Sicilian pizza from there. What a mistake. I will never order from them again!" +"1","Terrible service. Food unremarkable. Waiter disappeared for 45 minutes to serve larger group due to staffing mismanagement. Saved his tip by discounting meal after I complained. All and all, a very crude and unpleasant dining experience for me and my guests. Not to be repeated, never again!" diff --git a/test/data_for_tests/io/yelp_review_polarity/test.csv b/test/data_for_tests/io/yelp_review_polarity/test.csv new file mode 100755 index 00000000..95ac34f3 --- /dev/null +++ b/test/data_for_tests/io/yelp_review_polarity/test.csv @@ -0,0 +1,6 @@ +"2","Contrary to other reviews, I have zero complaints about the service or the prices. I have been getting tire service here for the past 5 years now, and compared to my experience with places like Pep Boys, these guys are experienced and know what they're doing. \nAlso, this is one place that I do not feel like I am being taken advantage of, just because of my gender. Other auto mechanics have been notorious for capitalizing on my ignorance of cars, and have sucked my bank account dry. But here, my service and road coverage has all been well explained - and let up to me to decide. \nAnd they just renovated the waiting room. It looks a lot better than it did in previous years." +"1","Last summer I had an appointment to get new tires and had to wait a super long time. I also went in this week for them to fix a minor problem with a tire they put on. They \""fixed\"" it for free, and the very next morning I had the same issue. I called to complain, and the \""manager\"" didn't even apologize!!! So frustrated. Never going back. They seem overpriced, too." +"2","Friendly staff, same starbucks fair you get anywhere else. Sometimes the lines can get long." +"1","The food is good. Unfortunately the service is very hit or miss. The main issue seems to be with the kitchen, the waiters and waitresses are often very apologetic for the long waits and it's pretty obvious that some of them avoid the tables after taking the initial order to avoid hearing complaints." +"2","Even when we didn't have a car Filene's Basement was worth the bus trip to the Waterfront. I always find something (usually I find 3-4 things and spend about $60) and better still, I am always still wearing the clothes and shoes 3 months later. \n\nI kind of suspect this is the best shopping in Pittsburgh; it's much better than the usual department stores, better than Marshall's and TJ Maxx and better than the Saks downtown, even when it has a sale. Selection, bargains AND quality.\n\nI like this Filene's better than Gabriel Brothers, which are harder to get to. Gabriel Brothers are a real discount shopper's challenge and I'm afraid I didn't live in Pittsburgh long enough to develop the necessary skills . . . Filene's was still up and running in June 2007 when I left town." +"2","Picture Billy Joel's \""Piano Man\"" DOUBLED mixed with beer, a rowdy crowd, and comedy - Welcome to Sing Sing! A unique musical experience found in Homestead.\n\nIf you're looking to grab a bite to eat or a beer, come on in! Serving food and brews from Rock Bottom Brewery, Sing Sing keeps your tummy full while you listen to two (or more) amazingly talented pianists take your musical requests. They'll play anything you'd like, for tips of course. Wanting to hear Britney Spears? Toto? Duran Duran? Yep, they play that... new or old.\n\nThe crowd makes the show, so make sure you come ready for a good time. If the crowd is dead, it's harder for the Guys to get a reaction. If you're wanting to have some fun, it can be a GREAT time! It's the perfect place for Birthday parties - especially if you want to embarrass a friend. The guys will bring them up to the pianos and perform a little ditty. For being a good sport, you get the coveted Sing Sing bumper sticker. Now who wouldn't want that?\n\nDueling Pianos and brews... time to Shut Up & Sing Sing!" diff --git a/test/data_for_tests/io/yelp_review_polarity/train.csv b/test/data_for_tests/io/yelp_review_polarity/train.csv new file mode 100755 index 00000000..6b72a7d6 --- /dev/null +++ b/test/data_for_tests/io/yelp_review_polarity/train.csv @@ -0,0 +1,6 @@ +"1","Unfortunately, the frustration of being Dr. Goldberg's patient is a repeat of the experience I've had with so many other doctors in NYC -- good doctor, terrible staff. It seems that his staff simply never answers the phone. It usually takes 2 hours of repeated calling to get an answer. Who has time for that or wants to deal with it? I have run into this problem with many other doctors and I just don't get it. You have office workers, you have patients with medical needs, why isn't anyone answering the phone? It's incomprehensible and not work the aggravation. It's with regret that I feel that I have to give Dr. Goldberg 2 stars." +"2","Been going to Dr. Goldberg for over 10 years. I think I was one of his 1st patients when he started at MHMG. He's been great over the years and is really all about the big picture. It is because of him, not my now former gyn Dr. Markoff, that I found out I have fibroids. He explores all options with you and is very patient and understanding. He doesn't judge and asks all the right questions. Very thorough and wants to be kept in the loop on every aspect of your medical health and your life." +"1","I don't know what Dr. Goldberg was like before moving to Arizona, but let me tell you, STAY AWAY from this doctor and this office. I was going to Dr. Johnson before he left and Goldberg took over when Johnson left. He is not a caring doctor. He is only interested in the co-pay and having you come in for medication refills every month. He will not give refills and could less about patients's financial situations. Trying to get your 90 days mail away pharmacy prescriptions through this guy is a joke. And to make matters even worse, his office staff is incompetent. 90% of the time when you call the office, they'll put you through to a voice mail, that NO ONE ever answers or returns your call. Both my adult children and husband have decided to leave this practice after experiencing such frustration. The entire office has an attitude like they are doing you a favor. Give me a break! Stay away from this doc and the practice. You deserve better and they will not be there when you really need them. I have never felt compelled to write a bad review about anyone until I met this pathetic excuse for a doctor who is all about the money." +"1","I'm writing this review to give you a heads up before you see this Doctor. The office staff and administration are very unprofessional. I left a message with multiple people regarding my bill, and no one ever called me back. I had to hound them to get an answer about my bill. \n\nSecond, and most important, make sure your insurance is going to cover Dr. Goldberg's visits and blood work. He recommended to me that I get a physical, and he knew I was a student because I told him. I got the physical done. Later, I found out my health insurance doesn't pay for preventative visits. I received an $800.00 bill for the blood work. I can't pay for my bill because I'm a student and don't have any cash flow at this current time. I can't believe the Doctor wouldn't give me a heads up to make sure my insurance would cover work that wasn't necessary and was strictly preventative. The office can't do anything to help me cover the bill. In addition, the office staff said the onus is on me to make sure my insurance covers visits. Frustrating situation!" +"2","All the food is great here. But the best thing they have is their wings. Their wings are simply fantastic!! The \""Wet Cajun\"" are by the best & most popular. I also like the seasoned salt wings. Wing Night is Monday & Wednesday night, $0.75 whole wings!\n\nThe dining area is nice. Very family friendly! The bar is very nice is well. This place is truly a Yinzer's dream!! \""Pittsburgh Dad\"" would love this place n'at!!" +"1","Wing sauce is like water. Pretty much a lot of butter and some hot sauce (franks red hot maybe). The whole wings are good size and crispy, but for $1 a wing the sauce could be better. The hot and extra hot are about the same flavor/heat. The fish sandwich is good and is a large portion, sides are decent." diff --git a/test/io/loader/test_classification_loader.py b/test/io/loader/test_classification_loader.py index f099c1b2..fdfc9008 100644 --- a/test/io/loader/test_classification_loader.py +++ b/test/io/loader/test_classification_loader.py @@ -1,5 +1,7 @@ import unittest + +from fastNLP.io import DataBundle from fastNLP.io.loader.classification import YelpFullLoader from fastNLP.io.loader.classification import YelpPolarityLoader from fastNLP.io.loader.classification import IMDBLoader @@ -8,6 +10,7 @@ from fastNLP.io.loader.classification import SSTLoader from fastNLP.io.loader.classification import ChnSentiCorpLoader import os + @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") class TestDownload(unittest.TestCase): def test_download(self): @@ -21,7 +24,26 @@ class TestDownload(unittest.TestCase): class TestLoad(unittest.TestCase): - def test_load(self): - for loader in [IMDBLoader]: - data_bundle = loader().load('test/data_for_tests/io/imdb') - print(data_bundle) + def test_process_from_file(self): + data_set_dict = { + 'yelp.p': ('test/data_for_tests/io/yelp_review_polarity', YelpPolarityLoader, (6, 6, 6), False), + 'yelp.f': ('test/data_for_tests/io/yelp_review_full', YelpFullLoader, (6, 6, 6), False), + 'sst-2': ('test/data_for_tests/io/SST-2', SST2Loader, (5, 5, 5), True), + 'sst': ('test/data_for_tests/io/SST', SSTLoader, (6, 6, 6), False), + 'imdb': ('test/data_for_tests/io/imdb', IMDBLoader, (6, 6, 6), False), + } + for k, v in data_set_dict.items(): + path, loader, data_set, warns = v + with self.subTest(loader=loader): + if warns: + with self.assertWarns(Warning): + data_bundle = loader().load(path) + else: + data_bundle = loader().load(path) + + self.assertTrue(isinstance(data_bundle, DataBundle)) + self.assertEqual(len(data_set), data_bundle.num_dataset) + for x, y in zip(data_set, data_bundle.iter_datasets()): + name, dataset = y + self.assertEqual(x, len(dataset)) + diff --git a/test/io/loader/test_matching_loader.py b/test/io/loader/test_matching_loader.py index cb1334e0..8d6e182c 100644 --- a/test/io/loader/test_matching_loader.py +++ b/test/io/loader/test_matching_loader.py @@ -1,5 +1,7 @@ import unittest + +from fastNLP.io import DataBundle from fastNLP.io.loader.matching import RTELoader from fastNLP.io.loader.matching import QNLILoader from fastNLP.io.loader.matching import SNLILoader @@ -23,7 +25,23 @@ class TestMatchingDownload(unittest.TestCase): class TestMatchingLoad(unittest.TestCase): def test_load(self): - for loader in [RTELoader]: - data_bundle = loader().load('test/data_for_tests/io/rte') - print(data_bundle) + data_set_dict = { + 'RTE': ('test/data_for_tests/io/RTE', RTELoader, (5, 5, 5), True), + 'SNLI': ('test/data_for_tests/io/SNLI', SNLILoader, (5, 5, 5), False), + 'QNLI': ('test/data_for_tests/io/QNLI', QNLILoader, (5, 5, 5), True), + 'MNLI': ('test/data_for_tests/io/MNLI', MNLILoader, (5, 5, 5, 5, 6), True), + } + for k, v in data_set_dict.items(): + path, loader, instance, warns = v + if warns: + with self.assertWarns(Warning): + data_bundle = loader().load(path) + else: + data_bundle = loader().load(path) + + self.assertTrue(isinstance(data_bundle, DataBundle)) + self.assertEqual(len(instance), data_bundle.num_dataset) + for x, y in zip(instance, data_bundle.iter_datasets()): + name, dataset = y + self.assertEqual(x, len(dataset)) diff --git a/test/io/pipe/test_classification.py b/test/io/pipe/test_classification.py index 45c276a3..88bf6921 100644 --- a/test/io/pipe/test_classification.py +++ b/test/io/pipe/test_classification.py @@ -1,9 +1,11 @@ import unittest import os +from fastNLP.io import DataBundle from fastNLP.io.pipe.classification import SSTPipe, SST2Pipe, IMDBPipe, YelpFullPipe, YelpPolarityPipe from fastNLP.io.pipe.classification import ChnSentiCorpPipe + @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") class TestClassificationPipe(unittest.TestCase): def test_process_from_file(self): @@ -27,4 +29,35 @@ class TestCNClassificationPipe(unittest.TestCase): for pipe in [ChnSentiCorpPipe]: with self.subTest(pipe=pipe): data_bundle = pipe(bigrams=True, trigrams=True).process_from_file() - print(data_bundle) \ No newline at end of file + print(data_bundle) + + +class TestRunClassificationPipe(unittest.TestCase): + def test_process_from_file(self): + data_set_dict = { + 'yelp.p': ('test/data_for_tests/io/yelp_review_polarity', YelpPolarityPipe, (6, 6, 6), (1176, 2), False), + 'yelp.f': ('test/data_for_tests/io/yelp_review_full', YelpFullPipe, (6, 6, 6), (1023, 5), False), + 'sst-2': ('test/data_for_tests/io/SST-2', SST2Pipe, (5, 5, 5), (139, 2), True), + 'sst': ('test/data_for_tests/io/SST', SSTPipe, (6, 354, 6), (232, 5), False), + 'imdb': ('test/data_for_tests/io/imdb', IMDBPipe, (6, 6, 6), (1670, 2), False), + } + for k, v in data_set_dict.items(): + path, pipe, data_set, vocab, warns = v + with self.subTest(pipe=pipe): + if warns: + with self.assertWarns(Warning): + data_bundle = pipe(tokenizer='raw').process_from_file(path) + else: + data_bundle = pipe(tokenizer='raw').process_from_file(path) + + self.assertTrue(isinstance(data_bundle, DataBundle)) + self.assertEqual(len(data_set), data_bundle.num_dataset) + for x, y in zip(data_set, data_bundle.iter_datasets()): + name, dataset = y + self.assertEqual(x, len(dataset)) + + self.assertEqual(len(vocab), data_bundle.num_vocab) + for x, y in zip(vocab, data_bundle.iter_vocabs()): + name, vocabs = y + self.assertEqual(x, len(vocabs)) + diff --git a/test/io/pipe/test_matching.py b/test/io/pipe/test_matching.py index 932d8289..8b0076c2 100644 --- a/test/io/pipe/test_matching.py +++ b/test/io/pipe/test_matching.py @@ -2,6 +2,7 @@ import unittest import os +from fastNLP.io import DataBundle from fastNLP.io.pipe.matching import SNLIPipe, RTEPipe, QNLIPipe, MNLIPipe from fastNLP.io.pipe.matching import SNLIBertPipe, RTEBertPipe, QNLIBertPipe, MNLIBertPipe @@ -29,6 +30,37 @@ class TestMatchingBertPipe(unittest.TestCase): class TestRunMatchingPipe(unittest.TestCase): def test_load(self): - for pipe in [RTEPipe, RTEBertPipe]: - data_bundle = pipe(tokenizer='raw').process_from_file('test/data_for_tests/io/rte') - print(data_bundle) + data_set_dict = { + 'RTE': ('test/data_for_tests/io/RTE', RTEPipe, RTEBertPipe, (5, 5, 5), (449, 2), True), + 'SNLI': ('test/data_for_tests/io/SNLI', SNLIPipe, SNLIBertPipe, (5, 5, 5), (110, 3), False), + 'QNLI': ('test/data_for_tests/io/QNLI', QNLIPipe, QNLIBertPipe, (5, 5, 5), (372, 2), True), + 'MNLI': ('test/data_for_tests/io/MNLI', MNLIPipe, MNLIBertPipe, (5, 5, 5, 5, 6), (459, 3), True), + } + for k, v in data_set_dict.items(): + path, pipe1, pipe2, data_set, vocab, warns = v + if warns: + with self.assertWarns(Warning): + data_bundle1 = pipe1(tokenizer='raw').process_from_file(path) + data_bundle2 = pipe2(tokenizer='raw').process_from_file(path) + else: + data_bundle1 = pipe1(tokenizer='raw').process_from_file(path) + data_bundle2 = pipe2(tokenizer='raw').process_from_file(path) + + self.assertTrue(isinstance(data_bundle1, DataBundle)) + self.assertEqual(len(data_set), data_bundle1.num_dataset) + for x, y in zip(data_set, data_bundle1.iter_datasets()): + name, dataset = y + self.assertEqual(x, len(dataset)) + self.assertEqual(len(data_set), data_bundle2.num_dataset) + for x, y in zip(data_set, data_bundle2.iter_datasets()): + name, dataset = y + self.assertEqual(x, len(dataset)) + + self.assertEqual(len(vocab), data_bundle1.num_vocab) + for x, y in zip(vocab, data_bundle1.iter_vocabs()): + name, vocabs = y + self.assertEqual(x, len(vocabs)) + self.assertEqual(len(vocab), data_bundle2.num_vocab) + for x, y in zip(vocab, data_bundle1.iter_vocabs()): + name, vocabs = y + self.assertEqual(x + 1 if name == 'words' else x, len(vocabs)) From e314a367784d47aeb2082bfe6e2cb75f7b6a79b6 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Thu, 12 Sep 2019 03:10:17 +0800 Subject: [PATCH 77/92] fix a bug function _indexize in fastNLP/io/pipe/utils.py --- fastNLP/io/pipe/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fastNLP/io/pipe/utils.py b/fastNLP/io/pipe/utils.py index 3db9c4fe..92d61bfd 100644 --- a/fastNLP/io/pipe/utils.py +++ b/fastNLP/io/pipe/utils.py @@ -105,18 +105,20 @@ def _indexize(data_bundle, input_field_names=Const.INPUT, target_field_names=Con target_field_names = [target_field_names] for input_field_name in input_field_names: src_vocab = Vocabulary() - src_vocab.from_dataset(data_bundle.datasets['train'], field_name=input_field_name, - no_create_entry_dataset=[dataset for name, dataset in data_bundle.datasets.items() if - name != 'train']) + src_vocab.from_dataset(*[ds for name, ds in data_bundle.iter_datasets() if 'train' in name], + field_name=input_field_name, + no_create_entry_dataset=[ds for name, ds in data_bundle.iter_datasets() + if ('train' not in name) and (ds.has_field(input_field_name))] + ) src_vocab.index_dataset(*data_bundle.datasets.values(), field_name=input_field_name) data_bundle.set_vocab(src_vocab, input_field_name) for target_field_name in target_field_names: tgt_vocab = Vocabulary(unknown=None, padding=None) tgt_vocab.from_dataset(*[ds for name, ds in data_bundle.iter_datasets() if 'train' in name], - field_name=Const.TARGET, + field_name=target_field_name, no_create_entry_dataset=[ds for name, ds in data_bundle.iter_datasets() - if ('train' not in name) and (ds.has_field(Const.TARGET))] + if ('train' not in name) and (ds.has_field(target_field_name))] ) if len(tgt_vocab._no_create_word) > 0: warn_msg = f"There are {len(tgt_vocab._no_create_word)} target labels" \ From 8614b89a19719e3cb3fc9f0f9196a1dbe65a8e02 Mon Sep 17 00:00:00 2001 From: yh Date: Sun, 15 Sep 2019 22:34:22 +0800 Subject: [PATCH 78/92] =?UTF-8?q?=E5=A2=9E=E5=8A=A0conll=E7=9A=84pipe?= =?UTF-8?q?=E5=AF=B9bioes=E7=9A=84=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/io/pipe/conll.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/fastNLP/io/pipe/conll.py b/fastNLP/io/pipe/conll.py index 24ba2ba0..70af5acb 100644 --- a/fastNLP/io/pipe/conll.py +++ b/fastNLP/io/pipe/conll.py @@ -38,8 +38,10 @@ class _NERPipe(Pipe): """ if encoding_type == 'bio': self.convert_tag = iob2 - else: + elif encoding_type == 'bioes': self.convert_tag = lambda words: iob2bioes(iob2(words)) + else: + raise ValueError("encoding_type only supports `bio` and `bioes`.") self.lower = lower def process(self, data_bundle: DataBundle) -> DataBundle: @@ -132,12 +134,16 @@ class Conll2003Pipe(Pipe): """ if chunk_encoding_type == 'bio': self.chunk_convert_tag = iob2 - else: + elif chunk_encoding_type == 'bioes': self.chunk_convert_tag = lambda tags: iob2bioes(iob2(tags)) + else: + raise ValueError("chunk_encoding_type only supports `bio` and `bioes`.") if ner_encoding_type == 'bio': self.ner_convert_tag = iob2 - else: + elif ner_encoding_type == 'bioes': self.ner_convert_tag = lambda tags: iob2bioes(iob2(tags)) + else: + raise ValueError("ner_encoding_type only supports `bio` and `bioes`.") self.lower = lower def process(self, data_bundle) -> DataBundle: @@ -236,8 +242,10 @@ class _CNNERPipe(Pipe): """ if encoding_type == 'bio': self.convert_tag = iob2 - else: + elif encoding_type == 'bioes': self.convert_tag = lambda words: iob2bioes(iob2(words)) + else: + raise ValueError("encoding_type only supports `bio` and `bioes`.") self.bigrams = bigrams self.trigrams = trigrams From 6d8b4483f1d9d1ce5bf138920b0af9627d51852c Mon Sep 17 00:00:00 2001 From: yh Date: Mon, 16 Sep 2019 14:05:23 +0800 Subject: [PATCH 79/92] add tutorial --- tutorials/cn_cls_example.png | Bin 0 -> 161768 bytes tutorials/命名实体识别.ipynb | 41 ++ tutorials/文本分类.ipynb | 834 +++++++++++++++++++++++++++++ 3 files changed, 875 insertions(+) create mode 100644 tutorials/cn_cls_example.png create mode 100644 tutorials/命名实体识别.ipynb create mode 100644 tutorials/文本分类.ipynb diff --git a/tutorials/cn_cls_example.png b/tutorials/cn_cls_example.png new file mode 100644 index 0000000000000000000000000000000000000000..5055bb02baeaa24c16663862b25619dbca09e482 GIT binary patch literal 161768 zcmXtfb9i3C_VtO4Hny6^ZqV3fV>@YVv$1VAY>dXX`Nme`ys`P^-uwN|Kj%CL&+M5q zd+k|!%??+Tmq0MM_fi2LM2V004py9`fT(>D6b>j}w%Wu#^fsJpAgm!WICK z0#c$vD(+dQ*&d`S`p@0DJkQK`479Z5r7HE_tF!rX1Vu1PejQ@; z1>T)=RpM$#(jh1PxLL?W!_digx)61$&w9hkQAtXd^~E$bF~wfw&D{~ z612^iMUs;7+F=!Ux@*kSMmVytII2E}(X>Pd0OlVB3xw@=ynGA{M}~)Y9)Ir!i;8k` zge|-4!0*gQ#=!8p-Z6W(Yr4{z|N|JQj8C`Q8Sh zwpi${(g8s5P+-Wi)O9A1poBu&!rinZct2AfRlSCTFHU$jR@^(uI1b`uIT%VCDPv%s zN&4}9h~$2$CNzNjBV%CkSlymXAnXv@Zglt< zszSXjDI37#qlS)vJv2R}|68OMu;m@i=iB%15aiQj4yO0bedB)x2ObhI$#|+vegNEu zweO$bNWzKSQBJm|Q7l{B0@q&zQaG61aR-+5-pb=*?j6zuD5x(sNE!u_gmB=ERS({T zpQF#vQ}%7IWnQy1AXAq{O-$b zn-!Mn;utPwyIv(xTXR_Rz263Yu{NjHr$EG1EGd+MY1_D)@R9qctJCo)by59v3y<$- ze0;y}m8Prk5WUVQiyP0P@w(4$j+>l@ML=BKTDlrfZH%X>KC~drxWt<6E6AkKZKrr32r3f6{mm_6pknVN$o1X z!$r>Kx~DU+AvQuQFIfhMQjq?tZOzVnwTpxDp5x*=k|?1s)Hg1ypP=xkTQMnlU^TKN zQstH3bC{{%LXN+TV{A$B>%qvuXw;A?KAr5KB!`bQ&B*{TIqrw94~1`LbyL)NfJ-(q zoZyLB_9n*wkcvu4x8+5GB9KlX(Er7TJN^$J1Z0^Sh@i#5+m@Kf;uR!#Kmn_)pDaeK z48u#>YxM0{Vi1S7r!Mq~oZI&#J?Sa~d%U6}8kldy<%br+4Fh$$Lu-50CG!dMCFzVa zw)ugw)jQVm0t5;CZbK zT<210XtaX9Qp@L8^6F2U=4m~=m7J*i=2va0T5lidptO4VRQoBHq@U6(l{5Y+nzI%Z z(97l^My?wA5i&(zVyAg(p~z@a2nwhc@6-*4sbPSJg`4jO1Sg9mioKJuOGk^FeS>oY zo5;Yit=8o2r~$zDdLfQhk(nLeF($0%bM`lqhww<7+!fsmi>l;dbO8l=k8u)UK_w^2>ODw+F_xCdF zrJ^Y#y;6L94OuO5aet-s$A^c(Zlqkljzil2O*{Fx_Zf_z1A|0SJbYg#En_5Ff06|M zr~4lmY5ss@!{dvAOQ^3Wb*wBAR_nBZ`Tjeb_P=xDKQQD-HFho}J4MkOpJadTVfJ0Z zwV!-7S=uz!wW9(n+mPwMO^Cj*yqWi-x|$w`?z1CtPUwh0+pZtr^q~6wwvc;2Qh7y{ z?Bjg4BbGngp)Gk@7P^;#A*Yc4p9Lgae)EkY|8d{{*faFFSqnbIt~d>o;%4pAycJPo z*WpuF-szf=ff15-8gCZb=@PK2w2LF`x>m_c#x5;iwNLOd@oH+>j}1`9HRh7j7Vo}A z5|=i=vwrQ=HQQ7F)ngDB)8z|)15hPpvS)i+{dSA{W_$qW zKN79xo#OqYAb{_e!7-neNU8~U=Xs&niQ9{D6Yc1`Jt%1x&OA(rGbtpU@jIil6-|B( z^-H{RZS-#$M{^RQLVm3b>3?@h!ON05j@<4oilKTxuNrAMrnIJ_3uKo_%+mkP^0&uD z7YvVu8?OeuR@+psH>@N4z+zhvV03RZpkeQF)@{H2G|=zx7}qXm zs%tIWd&~A6+cdqw;b^-KLS#fA6qyM@ohUyDW~kLXe2)JWRvMFQNBU0oWXn-Ww@MsP z+&4i7%bCK+^IVi_t3zoXGH_mE!@Bg+T3*B>>iTp>(RsXRePt=5h25gakz8!hxVd?A zEk9PBReT=4&9UfR;CK+*VC+yyvKW@fU%s&bXyi^g~b| znyZhVl{YI+Nz?g;(8k?k6-~$01qfDcVEDNoHRk+B$u}63@80Gu!7_9&ahe0rq{m#F zF*?gUIF6NayHr|b~olq?c-4GcaF!ITr_ zHXTbpR`Wkxmi@7P$9Sh)6ZFV5YpV6WBH9wuW|<`t=s*5!mX zcMDUue)&E0R>2U`meBf&?3Z89B0U(7=DKK?)5!8`VcnC&SPoa+a zsE_k;KGiP0i$&X=tL=KMCL)BG3Jlk|*+v%#TWHvJE}Axq6y9(-*}Ckd9E_9yZ@h+z zae%|gZ}Uv*5F~KpemLrwJ37aqqc+E zXy#n+;HJGZwY!*Ypu}OuK1YPqO zNAM91Kcb!R1qCt9Vm{ z?ag9WRY{9#Dwf~2l}UYwYUL@uRAggkY_a%i zh_<+5p$Rb-K%Zj}VXP$R-b{Sl%vyLVKl=?1&^=EZ9qTi4v-&X>DXsZhuj(kjOrS4A z|7REd8|Qkn318g1(o@?yk~PU(GTEZV$8cvNaD~2FouWjy;Ai=w6v9glC)w1>Lf`t(^ z1MT0w?yeIF)FfkTJb^*pdkrJ^LVSR@j`1Y7`l^)j&}~7{T9+n`$apae0KjKtRyj+$ zB4rvSa3DbEVJ%jqoW9ezG2kwbXlIo~LTQAo)mqcVFx~IEJIw(~*KPcf=M1K)3{kKA zQt}&4D>grw3Z`zdY#jb(!1+tYthRgocx>rF3q9k)DXuYny}a-597UwXBvfjA&lA$I z)|Vp8DPgr&v=Nb$vJhbkrmfw!T0kjpvBlCZ0lU5Sn;0hcw+j|v;dD9rrxRFqe9;Os z0O2KxFxZ`>`PQ(FqVv zy83X5D6V%&5QL_#@0NoRvzgFZE(S%JgA)Bb2S&GeITXJVc{zL6#f5`9sVttMMQ}&F zu$MNLclpYF@vo93zfqtgO;m@JOk5fpIK>L4^Z!^N01-T9 zSsD47bW%(dDMBSBuX^IfzO^Xj+-EZVVa+@NsL?UrTJS??j4o!?xo~wy>kFA*aW-NJ zrx9G|R5RYL>N5L)SngHEuY6Vwk*S*Mm%o4DzU#_k`(b}rL!gG2XPWESIf`UHAI>d3MOT(recgj-I1y$uR*Tg_?|lLIe`%mWr?4q(c{pi;Isp{2<;D z0lVeuwm#zj?gpAD(PKhgpBz@!$83x)&Gea1On~o zs`Akt^ndOkEgJaEatD)%eJ*1G*-bqzKxC6_(!g3ckKORHl2NNfh>G~*Zy#}+ubjLE z-x^L&V%(18qJO_QCJ+|LaJQUTHitt!x(yaCq1hS@uKVsFf3{5mr_gl&sbX3wpvl4AUHmWcl zV<>vRxLmqCdeViGr^9F)ByKGWwSk`LblE^ELS|KPOtfYTYxAG;GK_ehry=X?+H;S@ zR#nL$)yn%io7!ZX9p2s67GK#Ry8XGl*yo)T_7jR5AS>ExoyCND90!vjJ#b|kvf=ry5f4RBr5%`gpD0bVxX zUu>P*dWO*O%B?wdVED&uRY1)~%8>7uX#d@G!Ec?ffdicAuZmW}A3wK0UlDFI$qa~^ z6HOJ2452_B;s-g5BDJ?ub*`RZD-S{d;SOFg71(xmK>}-LJOOcE!n%K&yokdM2z;>J zA%V}(BBYmfsCE}pHX*j%W%M_OZakk=Fl74ck5v$iD^3*AEG~agZ8(T7rA?&{B|&k>k#PZ3&Q8b(Y#bs z^CTAx0VOMN&sK1RaJ?e_Bo&o2yzoix5|h2>wG{_f-swukzyAK1`0aTK&fkzmL%9|u zy+_eYJ(ce_Tu&pZc9r=w32MA{MZZ?kK3#cKm;2QY2EO%dx|cb5P~;o>U1!{qXhD!$ zRPi~ui(80pT^`Qel-(1&c}(l3|K_!|!ZH{r_R^cSVZdq+qOWSEB&1@WqVo2Z;ZoDV z)_8>cR8y(~2--vQ!U7`w#^}f~f}}gO@BsrD2+*iNo(}f2*NkJ`9g_x;?uhy&B3qVg zl$3z;g_2n8UTob-S9~g*o)!RX;VoYc#J1XF0iBc}55Qj;_c1o6V$S|Mujy8E>Fo0y z)iaY&arYaT4Ha+$z7ilomMd?Za$Z|_4q+dGE))FV@@g!8}0?2$7!)Dz=Uovd)m@ZPu7cCYj- zb+iOTY0%D!mCju`aaP;5rm;+2sXl+P`5cu0Lr?=%6H5~=cS+ODs0m+gLEFD|WNWq+ z=f*F$$;6>JNU922&xB{_y>#Ks`mTS%^lnb)_S1%5D;nihD(n(+&T5(z7=}kcGLd1k zglokR1}C%zVc`XQ*FUFsuNHdu9C+rKq-L$(Qw&ku2vIxAPI$8^P_%px+yDCdUc?}? zOc?e7b&Q0u>66hs-!8wo_RgF~((?;64-4bg_pZAt7FYLObE~*dts&Ib!sTSK9$Ip$ z(gj#9<^Gh*dRn_l!lVVH+;YAot@&JX6vYK9&)V#NOEoDj#Iy$lwe$?I9}{g-&oMk(J3~e+F-f0b8|#=icKMOMR3tH`^gjBXTEuV>R#*r9Ey=As3Wvb9!3B|N zW7`)GhtvM}^z8wH@&1R7RuxL;%Vho@4%5L!!uBT8ep3mJ~i7hIJr(uF<)W0 zl|OccfIFn|Lb<*7^~-J?m_g?m31w-fye?{=+>d_*N7rIb3ZCjq z8M00N`qD2=uupU&GFw~fkwib0#l8zZThRDHi*MZanLoFD4_(i3n_NEnH)g4_{4hSB zO~I}8Sj(Tws_#@);{%i9X*c$=@YG#e+2LxVC&N{87qaesjtpDB{1F)e`R&{mkuj34 z2Vn^af7Z*NFP8{5MFR9ss*$Yk4}V1g;7^231M`-$`s-(M{tJ@8@VPBPf8tS+Pl*z_ z&$d1@1Ex6%khT7Ny(`b%T!_T6{uu>RqQt11j0INDn`!HJUK;KdPyn67>i8`JbfqB_ zwFVG39oPQ}GbCz3B+b9>Pa3go62kGSEtQScqg)4-J;L73x8kYAK4pT0UR1qdBLKKK zR_=47_hI%0^S?(MPjE2$cHBpdr63Hx%q~?2DeuTIJRm|2xR`HsSn%j2^&+heoxl*ZA}* z0l#D9&h>DCg#=@^aLGV>)iz~Jp1{X8X_}A4WxGw3xAuWs=Iz62hNg6{4fSlcIKz8{mlSw8RlYoKJpi))NwJqX7VnY}o z>~K<{_y{`J%+MdIDa*9N@hEPH%1)|%CWa}MNTcVOKe9HhooO@vU@=Y12;T1^3+x>Q zL$T2Qs`g-ixZinG$9ZA5$GRxMjhj~r9PvGM)t?W&=UaypC(Eu{9W@15o`~fa(uplj zV@!G!Q*3otW&_i|vwHZ&Y%2Zd490=^g@SH|m<`)-0pe|9$5x?m7vJ?|1!PhP`;#-x zDL1KNj8Jep5%;j$-IICr2ast9_~+7s?~N!>zLv$qm{27YSJ9(C%W2g7{9|w)#<54q zJ63T3p*gf-)qt49S78JxHMUgfH-j}*5PnN7Z}=3$H|vOc-ljAT3v4M}mvaG%=tuP! zfS`5SEkxTdJpNTsYwCAs(!1OahG}E9_2pX*PRtnkZx!yR;Dnm5-JDl1is!l#Bv|j3 zY-rF-<&^?4vmWt9`Z(C{T}|1D2j-ZFxa7g`-Gv~(K!+heXg!gA6sGH@Y9hk=7D8g6 z`epwq-Hkgok2-Z=v(}JuKe#hz2xpphbK5jkPm|I)K6Anxl?anTGL+CRN7spk&0XW~Ra_$B>HD1TUfCjB!To>mu|f%MQ<{P-Rb_ zOrM{`vE&TDQVX|OxLBm{()mSBQNg*4v&qcewCx5~SXU;q^|hqm+2|)(`n*SG^s?dt zsByu(d+bkb8pWLyKY0Pc+rGfdb%Bwp_HFQEO@suY(TVvjA04KL7$xP`xy@U*HYC9R z;!p=5rM0x6N&EzO?%>(+n%(a>6i9f#X{2e6h2TrT6LW8l@7Wo{EB$Cc z4rn<_hvWdlq6HofgUCM#*<+5ZP;DTo3sj=Ba*zSt<$gzPEa5>AwMuOwM4G+?P}-G0 zrTPd1*9#SvTBz8eMqkt(Up=D!krM6gzpT+3PT)~2sEc%xH8m&h<0`Q&Z?kJ_%nDtA zqy@1YKXiq7{I=Wi_#)|VnA94<-`~(0@BCW*)D7t0XZ*74>?jmEqoGNoZ<v^RBi0@6o1kQxeSYRJK0L zI}I*Ok;x>b1@%JBFCz*{rpR4p$|IuVhJ+~nhS3_n0`eoTi4p)nC9v`2ze)IFbx+sA z@LkA*nslkrIC`XSfuv5Zu@0VT=jB0rDVXy`HvxLm^ealOBfC1>mRa_F`1oEfZr;rf zlMOknl1l(;3kzkbO4zH}^>YoIg?QV2Rc8z94bJn_(S!tKccoMZ2@+r@ccXBRZa8wV zKtkP-Unpr6~Ov4=~E-zI`V)($k~dYz_jHIT_)78Uo+e#r$i#NS_;D^ z*g7tk*$#`*_MY0w>6@)cJQS-G*e5(1DcW>@ke|XqgJ>G<>oR`|S*IM%+fjwu6X35? z)1!?rvQev4tq^)?CpJvzZztJq7H07=YkX9#Ta1x52ZoSXsQYOqT?NeesM+kLWM6;g z4wW2?BAlycD)XhR^s8I}=$AsU`EgghZkt@Av<_%W2`A< zAS5^|(nJ02q}`MPH(O4k_0QrD!(adPJ*s}x_WVgji@c7hinF5cEAMDarm<*XUt|El zErO9?;&Qoa9vDZd)%2@9^r~v9Z)S*GS6;!9RN~0)qj5{)-Sa9n!o<$%?ELhx7W8x~ zS6cW?WoM34GF5t~0QifPmsVI$Cw-kO!Sb#=T^u)LS5Z|{V?KUFsbt12^wA8>4Ozrd z;N2k*@sJBvCCroMeRxg#rBLzpS)|3fbhd4D)z zynpwD*VBo**l-!lSkkJ0nq2Wa`^ef&^2R? zPw=_dVKu9$eeZjw;ViHn;**h_EH{P%&{Y`})^&!3ApnYE*$`QP^d`GywD1f64*Q`j zr&hbkjPVzbJIgHw#1nR)krdjafNTz!oIRFP=hvdZ?7 z(^PvSP^)>Jh8#BcK(Tz4`V9OBla;=w2ju6wP3ME6f&RK1scgsEMYub8Vkt;-VgBcA z!dfBbmQ5bCp)xWz4(*h4&ao0qbVt5RiMVYRC;i7Y6XxHb_#XNq^C9Ig6y_0*&CX6> zuxQ~0H%nzw(5w=x?XM`rQ4IEr_vf0hm0fl+vpo5aPC~UX05v23w|YD_GbRmQ4?&R^ zA*&{yN~D;q5GFC}`P)zTIP-rMyQOI9?9X+}_Fm?+n_bRx*Sw~38D6%25&X)?nEs}v z&HFB8jWak91o0O+DdY3GDUzzWy|f*icx6Yk>(~u}Hi^$BOaNA5eE;;Z#mjyTJgktm zeEI2{r3}dPF}_}iu$6(kx4rA4M|t`)A@F4o+~TI5SH-I zu6k0GxBRF_^3$eU&cX6)2y2|v{*gNBUl!2%(WhK-;!VHW)y8eR7qqpHH7vY7_sYZd8*21ToeW(t`Xltuf$_H=%)x{&M%|!fS}`gzCYB1jdQd z157SNY`TGf@pbuRBN0KhgxjGhgs&eD)+AaaQNQo5IdaJ~|GHf16H`3*h2>a>&cRI& zq@dGNJXm6r&0Q7EwNmoX1S`<-FdEWBd^yUDI~(<1JWqh@5ki{BEL*J2t?#^lT*hjy zpv5vLuR1TUc;2GJz&8Lu@9hr8)`DreSG`s==(CHoq`}QS$?~#At!4l1(5}!Q4pvNgP(elCg3xm5kZQN2OnXb?VTuiAvh9 z)}^sqHK9t|`q1Fbiy3itaUuoQ|I=hiR^)`Y{yai#ISM^$q$7`Z+Fz@4A}@Z&N|n6Q zRRUT>QMuxtct|}A*b1f@`bX;&?t3*bWB%|8Jq!!V|{%>9Vg+npOvw0jlRSGcQNiJf`VFy1K>7z-5)O8Tt{zNyw@r7Ok$EiZtbIv77(Fm1Mbz2~2Kl1D0WK99sO|H3?kA5CCxpE6=WzmzH16 zge$PIF(g+#GUa+-{{b+*1*M@Ae-@F$#;x_>?|Sp59l zj2ggV@i6(vxa=im3lNk;S#3J8z1zLfV$^in>p?%LeH>yqolLb{4b8diDE`wp|NYxz z=ksX5>Qxa8JrEVJ{fn*594$_`+14$wy#@e$#$7<@22g;e)uj86?Sg-ujguj0u4*rB zAZ$^iv~75u5in1U{p8iQ5D8DZ0kadMC%E~RUxaS>lmB9t*Sw9Ff>Hu3!#&z6U$Bba zgfgrkM6*kJbhYoJFk!b0h|LrKK49UrqP zGjG%p?603d&B0Z2aelr1fl3PFX6Ngwaj-AfEkkXbq2IU>gWiAHzF)j?V&oyqO>`m4 zIBGnTd5DY}22pb*Vxt~}b?Jln!5`p;g1tI=5{BK8N&nGwvcx_}{6Ln{!@iF{7r%_> zxWl(_LmZT6ihcMt|E`hTU?b{Q{thpcB}lmA*frsN(q;n}n}O%^_b%yD67*N1AqavE z`JZF2o_dolg;LsfO_GY_F2vR|0i%*Mf33q}q&PqteV1X!yp>3js(ZaXGk_M$c-4VG zB``Usl@bC8=;=Xi7Hj`w_lvN1j(t>|5QG4#x!vaDrdVRHG9k#M;+Hln_cAxQ%CSV=6=8t1|4dVF)J z#`sd}j;ITBYBVKo`C2J&o`z>__007t8xGezP|oe3!G3}zj&q#EI>}ATtU<;O_I=D^ zgh`NU7LDuPSK3Y8fXCjSi4*bD^ZaQq)fJ*K-`)tRe*tVb0p+Uu%{FORNuGZT=J`rq z=64-M<&gk@!V1^B&#od*t8Klo-tqm`h9=r+0cm{23dc-L` z>*pVRq%YOFVvv1NOgemJ%p17;d%|vyB&0WuU81J`Bo?-@^=gN&B_~6VQjd)e|GexE zP@S>ZK|R?K&ri{9uD14HcM`-iwVOJ>m!i+h2mBYhc=mq!A_DQ*esA0RjjNXl(k7Nl zd~jb-Ci`Jx_Re)}3KFeXeg7)!?rROXs$dz}vzC>8h7S5;OY+#C(Dg_LErn26@SBGG z_un<}OT82d#Kr{m>%iH-x&@EqN{ZM1qJE7RkDQ;SO+XjhXM9(0^Si1+#yS*$%fW-i zRX*(bNWq!ZQq$uKrKTc(N#@atqXPG3r!Hb%c;w5bIm5FnGixGY#U{@7c~6m6J3D=Hhp^#D`u&~#(PxW1+eFkPDq?H zfsv?w6bVUca%)P!8OweD;NZ&^OI;vYXvPe?5<G;xB?~(>CbZ|Qy?-CtUB28)QzKucvTeSqV1S(Cseqb^oSdNDH z4J|`>y&nO{)>;%Djema2{gR&9#K1D5PsiwRd)#hokrT$kXm2e{p<~Ra-s2TX&`sz+Ho))*T*s(ltGkSD+hyKE(T3aeOHRiPyg<{887uRB9Ulnn)Fy9np*i%gc)B2yR?NPqsgC$XUU*goDYgj?75LoA(9pb{Q_I8`%y-kN46kO^ zeCc|=em#eKWSA4bY*IO6V|)>c#Q&R1{RD;TjLF@5?S839of0`oHAjuB&*l3g=){ww zB)q_|F~iO1pr|GF_6sa-8_zLzj_zZ79WD;8vBzi+GXb1fw%+1LcniuiY&C@Bn5S#} zQ%^&0>lL>fbHwDzTXp2f8Uz9b+@FToT3z9`l&-qUzj~i`gR3j+@T!kw5IPSk(s3 zos$%#b^82Q|+EUNt%(c*6OoT#jUwzWFN$6I^t3bShekG(G+hw%gidZUWHe8Ij|%_U|^@G zTkTQ~D?f<_pYrC0*$NLU#|U`P{ah6yH(KSU7#)s=Tae~ovj3}Lh7aV;MdeNK@PB5_ z#CFyc_D@y%R4wE8M*l_uH7w_pM)z3dFlG#uzZm2WR!U8misXo_{?LX0*t!nesl~rv zIXjX9u}@@cOepYfDH8~S_OG&m!g6(P8ia3x61CDuvALY3*7F>B^jc8KBQT_vS}^Xi zn0<!jN(@W80CGSZo6XT>(e*52G+{8K_%z=176&L>I<9>V1+ zZ?{;9@l@X>5;P zw@Ofl)bUWlY?r6IM-f;v@t_Mod7Xt=z%tgiZ?@8lfm|X{2s}qc4KlwmFbpqWs$GuV z^9?^ly_`%&?u?A%<0C3waVxJGCh5=ufSyM^HpBLgI_fY4DHp3|`+N$K$RLtKY)qnx zTKu4sMRHLC-#2?vrY$kgl4;k(AEHLn%pRV4W*%&jFhH|!Ybz-2?LXok*oUUI-Kur8yDbMkQ*ZT;CwOkQEUJyK64mjBdWUhf z@jX5{Z$ncO_i-(F$(J-RSBpgb=hW48C`8`XzzR zOANSxB)`@;5@BD%mqp8Oxj_&9bIk_QnQI ze3s&y`PkT1d5W6ciWuO1yB#@CAd;1c(P*22^O(o3jLs?3)qc#ypjJj~3?(&HOK+WL znP=Ln)hhjqeFq&l2GZ(umg08`b_0aEIp_ji#v^r7uxkO}lp zz&|P#G^@y=d{fhU6DwYwhqe>|{J+f^%`Vkl?AhT@Uv^h@ zG!4C3>kU1L7M2jfv)H_f5a(6PJ|{g!BS8a6%K8bZnX5Ch(e&%XzZPIH5+)nRw)de} zZg!I3+;(?!`TcG?3&RU*yWjj*f1vP!NjBq*ML7g)q<^(zdA^1KwY5pSh#ZP!I}S}VPqyftIgFg7xzQBDhmJm z+!sBQ3+a|(3VWB_%S|L}H{CP|_?>Agp;Agz=14Vt| zyciB|-%#%2Dv+x|#WuKRQ(bgL&~>YK*}$-?t~?lzHLGy3M2P91*V$|FTUKpreo8p> zc+ls2ol!Stv!CIClc#Z zwAXWY+mu<%H!w%eT474uQ#eA@)D?XFoQQ^13qW*Bk8?TO!M?L$*}?2~qt(NQ*PX52 zOuH^j+7hbGvY}S7pkeR8A{3=I*j@>l4B;MK@Hg*&g8)cD?@MXrJ{@(lyFK@Ef*Q+5 zUQh@2L0%318;xH9)Q0>*-OI>tKE@Z#a}fzUJ9=PQFjyM^`a`u~AI5vz5hpnjTg{%0 zax2GfUlt`^c1j4j>{n`S*MIk`A@REQJT^q`Iur%-F-5XoSLO0%>1HigPE`?azx7Li zP<th09o;jY+;B27m+YMcBnBr)Hu;m?^{6C}>KtW~{+ zAFYQb1C(>G&pRRc6vU^MtOCf5$lg`WlqqpasU`8T*a<(WXQ3 zw5uQ3c3m>CpXrUh(jkB|DIuzjYDVMeZUh5UsKYR&q()K@^WuVY;{BD`mpcNk=8ilK zBh$L8p0zX-ryZJwO&Yo~RnVV~EsU;XeGP$|Gk8X`nQRelVqf_;l@URRiwFuF>vcoZ zCp3=je&Q}X-@EqKML1nMG3ZJHug#G;@CYX((0y>Bo09sMP5N=qxZ>s85Ew&a%B}rj zFwpg#@{3L6Eya&Xar1$kd&IUZVW_y{G{|n0W#1$I-1G%EnDD`$!L6|^D7^xPqQY)P zwkGsn%iSiiJKj=w&?q>{6^k}>Qch)e4#M?8J+s-z>g zta4rDdOVAiO#j4YCM|H;aJ`O>!RsBKY)JAsAk2axjZ9x_W^vDZ&VK6FwkcARl*7Fa zmWipZu#&uaGsh#Xiha}X%R{G>Z*)HQ+1|Fx;3qstgoC>8fWKt41N_prLN4pu+XTk1 znH5)2XaFF&fJe1j&IOHL+cW6Jn>sq+!4=|TOXnC+3`U3*QkwMA!s=Zsu3oQDbAhYo z*f2M@L@{I#PEyJn6tKuLG3{;FLS75;T5wwRelx&~rZBSM<_$Nn#{~>#+nU(+W+MLb zyShQeg;cFD=RB^F)uIA)1$JJ;Zt}`EA$iS7u6z6rON1=8s`BdOKdZH7netNYdT8WV zoDVND$Y2aP+5$Fr1-2#$ojSawS4Q7XeHNoF%_nWRJTm@{y!Ef9P`cV0E4jraJQ(b6 zqMq+nc0s8r%@Kz#e%ZFVTzj(iy)TYBb(&-Z1kZXJAxu@j=zULz_aC0r;wU@Y2p3NI z{Gk+g4m@+Ip=s{3;cNZ7Wqhm!y}gYuBuc1_eR0oxr1DkZEO|*ev0c9q9J-cJ8e{n6 zr*gHma8T&{Ee$|yiRmkQN=kY#9wK}F`j!Z;$1#Enz(^&B^zZloQ~-Aoyy$fO*RNse zS3F3!cIK`v!0vFTK?H)d?P;=-e&zg1JW4(Ng!uR`@ILLLWH1lfrJ^{W@N#}VyS_0) zh7Ig7aM1Y31orJ@&YrFHjK<`0wA}S&-e*U-!~3WO^Z)y+W{?_{VK`g?)lw@B&zTzt zX@GffS~p9B=;AY5_G5silGn+Pv0NF#Xu#w$ngjssUyJdF%|eTs`2EpU#pN^T7nFR& zLC4p+Q&AvCjFk!{A?vWx#3K1gO0NO}DXf3X%1Xf2Lf7}7VCW$qsDfu5Bz~{UyUhIJ zvYV*8RGB}=@)@SqrhInOi!!LsIgQAHyg?Ot6@R)_(pug3`k!Q!(wU~#gBZ8G`HjFY z6)|wY7owpW9|pK%GS(+??$f9mCc;sF!(&i_$TND2zLtVk_i&UB_}osQlE(XWZOn*3 z7)`&O`}9xf`M^KBIoSM&Cj$g5EC=M+P)Krq`-r0_8`qG}Oy1Tn8?67}(2|D?k`Ags z`zAR|CICRr(o4G)gtEP|axy^AfQ>`9tge~zL)!e`K+Y%>vXslt@kET{hSRB93ari* z^73r+D4m@7@)t+7Za7-uxYGjepVOq#WE4YkSzLxm-(F9iTfg%Dd^K*I)n?dM1%+m{ zzXUL0MKcWIxSIyrNW7}kwr_NCb0z+YpCha;5#A!ZYQ2K5Mh9z@|8&w+iXNZyh)iYV zBvz2+YTkIo-h>Vn{bPFhlyqQb>Y}Ie3c*~L*x0-(3&Q^<84ek(e}D^OWn2nSn8uj;C z7Du4E#SOsuu!@a)hl0Wy0&%!j`FUm$yq9=m|UiZ$AR0dWLVv zjO{_M?691OW)P-}ey=~)X|(yeVC)?!QkTwGI#nVs&-%GbYLICWFKF$-p5J&)lqCfd z(|42EGC5>Ivg&zV{2Wio&bzIST)=by?g$yiP-y}L%;=vUqv`P%o=Jyoy4I&rMz?tX zo9-SEKz*&PZFv;192wK%X#c_B6PS0_nl&2O<&ASoj?OJ36-(LJDF z3JvrldGQBn{(yk?3YSL*mV8x=6YTURalLaV+2KQ?_U+uNB`5z&cVjur?j+#_TI74t zMhi1*Ux@-CDJ-?`7Cc^1-?61;bYb3t`sdJqe*g`=)=a701B|3BY7{@S9^uEH4g@p_ z;*QgA6{K7aEYKB9gVq1?i+ciu{w%D9Y6h?RT+E#6w+N24yWU&W~q0W|Ge?_BC=!|d@C}& z=6stU`@>1sX7BpKss$N9Zric3{^@#iFT&hBr(SZePj_5DOkR4h#Twlw@{9TBq^$SL z<-xqoil1onlwc7re1I^t>ok*2-I3z zkMzv@g`eX$;7;@1btCZ4{WsCcrQH`bl=|)2y{%qs7^r|mH6gv$OyW*uLAVP|)08Ij z^WIXZsuJ%4BTQr;COf4fb)WXMmuM=U9y0 zb(ELvg|MUQ+riZLT)hzM+b{)&Vh0E^n%FEByUBV78IJ0OAK29qjqDq#qEMtLtXe#| zFT;Jkbj4vX{swSqj}Os()A+|$8+tA2s}>_L<=>p{FZqsJ(5h`YB0L{&n+SzQ3b4#6 z*WbOtvqR%qJFRzk5FW@ExZmBTBpRy@$$l7v-J%W%6xx2@)Cy31dkpO=aVx4(Yr zu#_{&uEY^Okb?Kn>ErYMF;ug1$WAn+2AvqNsmMpwc_)Z5yz9cc^S<{&2WE2VoZ*O=GCYm&%lf;`h2e z^A&62aEtsIeC6s8UfL7kJ*nw>l<(hTK|1>C*^aw<+f!f{Ft+$cG)Ac2{X!FG+Z9`& zs>`lI{Vbzk$j|q~v*vgH8h4$@1s!zgKq1AXd!EACE*yG@DPtFGn>TD|Z-f_eW-j{J zKA7&$4(+yRRia^-ER2<o^>kf~7XO$I zmE(uD#E*m~2Fs{+#32UXxC$B^KCG66O}Z1y8PrLrOOBP05Qh5Ci}tF z0#m|rp?c!Km?S@;%+29_H`}RSzQ)efvbJ~Rv8VqBs+6b(DR1z$;SjX=w9hdeg}h_> z8B+A2)Q87pU^Te!6@fTriNhUi)$i^RST+fjH)d8sc?VO^+r$4=D!YBY({&(WfX%j$ zzhqv=UnI3VBsdG>=~})$J}@KwY1`s4FWg?2@GBKe5m{MYMnyA+;+%EQvuF95ewqf^H=(jR^%1xHy_;Nh z@4Ac4#3y$Y-*#v#KQ-g-0Qb~uuUxFL>1P*z;G z6mJ|%d1TJzNZye*mp>=lQ0x!f)|i4Vx=SJ3Y-e`j<1JlY$HTmaRT}nXN$R#XAcMO= z3cE}rG6DSt$X_S4n|dAEWqONWk{QKC`@KX1^a72Cvr{^A2L1cqWo8H|L*=E2 znd&J6%kpts%ww5;-Eg@LUlg{fvIVT!%bTpgMg8lDg=@-?P1V!F^BrI~^^Z;5n3bx@ zWnATif;8%wdIh|T1LypE7)+))j+F%T{_v0 z+{fnbR#uIGM^C!!kRe(5<5*V70!wX*#c@)=rF88Q&+^1v0+RTrO^bt&K;Mw?rAiUQ zGa9pZOv;oPY~{z&t+=JJF=?WBB|YQ2(j~*arjw%L^kVJxOnC4H(aN3rB~~~=Ch@UG z8v`gpevVzQ`X2-UX_(v*-I=I12myH1p4K~fuC8FljK}Ixkt%x6n|_>rA(Zo(6M&%E zBu{1rD{k+$s^*4)=q~pmM7KshBg7%8Tex>Z1gZ~u?#UT&yg}6j@y_~$>~v%@JO*J_ zWsiB6Je7og!@W&k8nP%h@n&3Pn>;j-0IHClFZd{50c;#T);65Y34+$e&QVxKfgfOl zKKf!G+D#k)GRCvyHnZTzUfbNOI0h1!Aej5L5mI^&7_bIJz^ThB=LjfN>?o z{i4J90{NEBWO*iDKg4s5yG=v!N3C^L$LSGU#9fd1UsV-0;Cr!$4PVBi?BZZlgR5QE zNdY}SYoO>id^5er0ZrF>b2^SD9gw3%$6MW2KnMdbxlGu z;Q}VefI;w(`AT)@j8|j?qyD$N1-(k@=~*X?Am0qivg+FhcFT60L^-I>H@nYh_tSHK$GG>mu77A~e!PVu=-1h5j60VvaocpAQc}RJs-!Nmq=R=4$!+ZL z>f(2^>_wZNirpu_IJ9$F`jtHCX3xLFYq>1{TdM9qWS6wM@=Bb+Lu?Cpu>J~kmul(p zSEJP!NR(XKYP3zEZpj1iR{~OWQ^qG364rvDPflvqoik` zVa#tDb+oPer6fzbG6>M`ZlM?1+&JRT+(auZyu3wJ=8d-aDa=vIVAJ({NRlSRUSp&B ztW|LR0!O8aQM)|2Ge7AnUC$;!0e}FD*hA(x?qElWdP~G%5f%HL1kF)3MB7-R=1rw@ zzIPd`3C>zG!#QYDw94DvLH+s955lOsSi{|On+?zMyOeJ@0mjTdPSqG~SxVypslnsE6P?zb)-cA@|g||N_H<@Oo@3u{9n+v*f2ZgC$0ZU z$}BtmTX@6gB_T;qW6pf|8 z&j2OY%)Q2+I|Dh^5D4u&{j*r~>Mp~M+sNzA(N!`c0=Q7B)PpXe`z*(lZffQ+8F|v} ze9}MPWhw>^#PsA2Z+w5?Haw^;#U)YC=9IWD01D1lpGWZ@GsSK{8FVq~;~Ho|Pn*@( z_w-%PrLk-IpjnI1;DwIg-D9mqwGB->fxWde;x=UfI58Ov2~%fw4lTOU zN9;&N`xtvu8?TlmVm2+IsPN?Q;t!hnO^RFP)ys)tz7}U+)Tok?)+>s@cIu@{-s#g@cM`G1Ee)sKD6H> zBnalQNM)pX*)ec~Hv65NwLBc=_N+ui4m}quEOg?U({N_X*u7pI6;97nRr&B|9+@JF z5#uzjDr86xvM4zjna4c==+&!q*B&!PmQFMaCDY1wcGZxq zp?&bX4YmEn7phW~F#Lw4LI5Ns|BtvMU;5PFQiyT?pw?5ywgPhQ8cx|`WWE53&iU1o zudl=bHGv~=^FOT>`W#XRQ;u0xP5XXma3Z6rz82c|9Iuk*!W!E_a}RAMcx0hPRAtA8urZwbI-x`^8kMS2{qd_= zWp_+hoaR}!0wiQWKd}aS0N~$~r$LuR!>ZwsGO(j+8PufI$;<1DVsMEKYM>4FYW1c) zRp!S8{AP7jM{wx)CKq!L?$hIg@V8V74zFA)G8vpt%?OA9QXbD6c;VlUayK8ylVHB+ ztG|C#hPH!P3mxfs$V@o_R*rK-iyLl93akJVo#+v3s95T%=Oi{5L!^^Opxe#|ZF`66 zcAmmuj@_~X5X5Ne6T47^0#fYXmfr_N(aQCfADZV~b#CGM(!1ClVQ;be;Qv&rciUWy zF%BHQ4JD@2B11!Z5hhcr#7~+!fJqCY00sw{jYt_uTgX4}jJy>X&;a{i#wuZz?(TOV zzt>>}yvjGXJ4+|Q1bg~!>@QeNV^5t)I!VH%AKcDnapVd*I6((k$>QqFoL*}wXt;^8 zeZ>7|5AzE}=^7vl?r~&R&S_C+^!mZIoj` zj~~ms%B9`n^Y1}h*?e3LI%Ecb(6qMblx-%?AdJ=GdJ3M^0q21QAyIR8{6*7Wt<<#Q z*3sG-{m3Lh5}lL~(ts{#Ma(`g54$F&(}?`jaPAHlKQw9myf~&F#90gdH)LyTwV3s{ zruOCbn|dzeH->Ek_GI_y zIMtsUrSryayfNMBV@^SOBb$6RGeVhfa=XA;|7GQR8Buf*LVa-@7Wy4(lo)#Z>fOjh%`*!=`N;CFi4K%2rE=Pn4z1MX&|~*BE|k zLkRJT`~~m3xYDG)Hqfq*X|$WrD6ZdtEC+*WZ)5o2yV=*G>wEL2ZriESzdH*%lZEA! ziQZ$qZ4C*C{wmvWBH2>sBGM#Z<0f5o?QrlSkCC8#BT<-EK>=+rUw#VSpDDp^S9yB= z%95M{T0Sa^j2lpiYTl*`CzFLwwef-#hfR69z{tnF-4{Kv&2gFifX#DCS@1`fhKJvN ziq};Co}UB0iL=+4q_6g2ow*RqK`X^YhNJ#_UH^IrVUS)XWZY4SK}+^3q^eF=YRH{* zBYC@bX34-U?VzChm`3nW8hn^94pr^y#j~`dv}9nXySSFy7^c`utpVhX3~bfCh<9Gq zdPgWLDF*Mr5r)wAvBqsk5ZRPkQB-sK%;(K5vg-}AFR68OkG<&FHjQABmV56EobTwa zWTkF~SCuFuWeD)IJKVIAy7Qeua0vX&7kcIJa5K!2T4mf^y3K?Yi11A$*_yAc*5BAj zJ?OcUYdxjB!1yBZ8I|5O4;tZ(f%lq&_oK1tDB)q~nPe5^#L(5A2681r%b7_fvWB&{{Y0LPrMRPpw{da16%OSG#_ z^hqelL&@75C9r6c-=%TfScp3Uy-%BB}DH)w!K+NWzzub5Xnq=HK7@HSBojyOuchRRhU}}0k z;D4YFTiy7Q%}7vS)wNgn7et%rZcJVy7-s~tMDIM$ChN7u&eU3x<){0>#lsT;1_=J+ zum?!Fhkp(Zg-d_EFvoI<6N)_h!JfUo5Fa0^pIpCcZ=Mg-1l&#Pr(rZ2J={chgmJzr zj8L2Ug81Fp{$96E55I`t;>1;&6W)##q=M=V+NPW<+Lx3;a3Bg)0iDigltoH)rtEkE z&@9S>a(4yYIe9HfWDTc_0<%wywB?~scZF9v|CA@ndEp%IASsUoV8TC~yL*Yv-c}76 z!0A976#J3y`tChBugAJ@KgMHf{QNVl{G;sQTrO;XXiE0|rBJ)IYwXGr^N=BBn?V}{ z?S5n9UwQ5`{&%&>&&Oi)*mILi@ZEcB&%K(tS*aH*6)pB;zQ(adngU{IHsIo`y>${u z+8po+1l=_sL+HnB1y`}A|32(+&@u996(91C(4V`bX%OYp4^Rf37_W+a6BB{!;QRBSzkm;GwYr9_Cv- z6?04O(vW1%%(brtLDFR8s-@_|)DcPWaX8|~2A~=;;q`cl@KK!!7l+siYk80~#=IBc zm(+$b%p;#IqqUlXVruSwLi$6YM%IR4cc|JM_2%FEWfg`Zh5B<1YUr&_FI1n*X2ctr zfoS2t1UBumewDC`^TH|PX4c2xV=@(Z8`Xf!AtqsU?OSt~S$oTFNBD{dKX5bJ`w%*5 z#5R-f^hM6Suu4hjEOAno_p7+q+?XUR8GN*&$~3>*!3=poDxh(EeGA_?vPO=MC}*** zwqNv_Lvl=lKVYR{(f7f(hx#!5?}To#`;5py#Nv3c00qq9^gWdAGPG!^sj2h$PFvJz2JE=lB|A{8!$gjeJvFM zdjYCl8~`}KIiN{k;VUHX!~-1 zWx2Z08n_R^=19i22OAmtlkf&LKma3+vXkR*qh6&~XVyRV%_fMASIk3Eh?)TXRygK| zyDj4J_b6Mrh?d!3F0-}Y7AKRSCz<`9DI$iWV8mU6`a&|owL`44k~ywJ>s&%0yvfm#+XF+f zalIT_0%FKVrBFCA&EumWyQ{lS4e*7QcF>yR^p)o)z5)c&QCSNrbKHBr^)&a*^b2UB zd;7%8CW3Ge0*Jn-tDmvMgjm|*KVSyU$FUGZ90VS5!(V|wD!H8+KqAP6T{7IC{(QJ~ zHpPgYRP4w@)QsK_5U_l}c`2OX)WKW09g8BU$dmq-SpZGSH`-@?Wq=QKvGLq5H5d(? zO-UHm9bsUA{^8Szn+zFP*ABBB!f(TjjC0r2zp&yG)b8&%zg_UbWsmN_D)=Moi}->{ zH)HKHD*jpq{}`_vH1(#SN}~G~Zee4K5iK8WfGi~OCF&yiou3%hv4a60SM&C9;tnGH zrt~I~*UoL!M1os$gQ3sz=jyquRA{y{1MlnFn+an;^7Pz-ToyzRG4U20fZSfcvIG-n zR0|0SNXPc`@*AjHd;8h^Qkgb?;TfbYmpf7Z@=Iq+eY5s1{LA$UDM|rV+vS?eC&R&n zw`PTh{~ZNB9O^a4VpQFb2@s5Nb~#@CVs=ob3Uph&U+#%HEGmg&IC*eH9QBR)E~MxHCF~VX&n~+b-!g{)ur=L@>Ww* zvbCs75ofG0N#}HVFR(k1?ORzYxd^ru`y1aOf=+OYRf^0aP;)7AD2GZd?Y3F?Fi@U8 z*q>H%zpLSKCPZXqs0Wg*Yp@lNLJwD-x`t;?@Y-uV-W%NLeW%^MjFA#6I|Sd-D^di#8bj(kpS}ho0GTPf*}BG?{MGiQbI3OkQ6AGvfZsaw z5&aD+Z$4~TzpYD-8CBV}xg}*xnG68R&+f))!(8=!bCo3+ru7FRon#W0xa~{djwib~ zy8h%Gm%gZ#FOM$##0%1*nJXN~;=|_jm~Kv_X!|cS5QIM@ie)QscPXg*yc>c<#P1Q8 zq&^7Vy`!*I+X9YUQ&y|TOC6h>zAVJZL`4|WYXAkw8{WU;(bi?uuP)x9v+O7%f549A zZ5kSygcNq$)l;S4Ky@EgLW;fmhtR zGcSE~jlRRy84eT12>RnfRrzp}az#V2CuLDwZ=9eSD&p$MX`0zl6fcI-Eh!(Dw0+_9 zV6g>Wd^>Q}he?jXj81j3LnXvyIr5zj4zgmUJ%fash-s&cX0NE?8jK(=l_-CGqB?rn zrs?v1!;`P7Q)AcyvqwT^fMfI5+Kxj^izwb_GWUgMP3Pr~JqCEGTP*sj1?sxI!ut=` zp<;MW0yU7{J)|?Ou{$0gZQ7ONt1RZS^Sv}}uw~o}U~U-xb?e_q&0~!@VgUAeV{zgz zNy}IlxiSf_h0_Kxc0S=_LO+h>VZW6!XA)%n<@v0){BNQ2ol9$KV6V?-`w0WMjBN`& zIi!%z@yfZQ0^E4d=C|HvU?FOocmA4;u8bQ`?HGW4xHS7jPvU~3r9Quk+3OBpqOC7{L+3lUn~lL#MXupDz-xr;aLKri zyMG!Daj+69M_{GF?CCr1&oe|8orIUGsuM)MVXU5)>?D?z&H~dA3c8DdV5rg$Ft!>9=3=7P?!Rs1$WDr$ zJ=RXWi(^+ccuW=FgyeTNhEc_&*Zr!r@zawa)n~Y{=|<#ul{=P~Y*9=t9aU3o&H5se zNgRaiFq>Q*cX9$jBalbyk4n<;ooPGgy7;9DZQ^>ts+@i`ccGk~ElLvbzkZ2L7N8zm zT;@<8wve!Q!TTcLb7eU6rA?ERT$SuLsyVK=U!&SjqWWk0&K?I80KN_woHO$X_14+G zv37wH7tf~<)4_I*__oyuinTX`Ks-FV71iU)d>7x@H8X~Cn_TRnv1`dJ_{pHoAU4sd-^v6m3>olXKGD*1Z$zD zII-?BMn-Cs9$-ZE(Ef40kX%(~ORY)Kj8mc|=D4hZ!-PdJ*_RrE%Cauxx+{z31##S{ zT>$M>l+vJjdZ_p}Q3{}O#e90txmryy{r()-BCVnYfkftEd6Tx0wVa=9<$l)u_H;uN zo#gXC`vPkRV|nTn(!hr_HuQoY|MzBU!g<7kncTYw!g)@|)TO7*645VoATIXxfnl*D zYrw;qooQs-Y;2G9Ih_&ac=F0S${_Hfqypq{GUZ=%v*00-J@2x9l5EBpOhxg{Uxd5z zwx-rmw%$Z~f~Ui3+eaFP8C?>_+EMueTyFWT87}kVFQcu`uQeN{C!JR>`k_VII{oIYgbg$U~@ba+OSY)_E z5jUy9S5a?pQRYywo&G)hyIgBhTQmEK>~;F>vuM*6>c|;ueF65f$(NobExy`YWl_%i zh1v(}XS~wzOj3In^Xj=UH7c{sKU**U3qqY5OLO~f>ihdD7`wAf#@Gig9?lIDK9gA) zhsbqvRm`AW+t(-498z5d-P9;qRNxk8(@wrO%go1-;bCfKBXcm$T*IT}PPG>B!stBN z?A*Nn;=ChAMA^YrwgXJ5We_F{uB802U6ySY59HHk(8<*Iu#M3Bp_fw1+?e)V;QNT_ znP0S1Jr7UAD{oWUFR2tk>^CCs65mLHa1e1c3zO=H{r7f;5n&WX8xW#*(d zOxjCTv!bF2F^tj+ZAg$ZkeJ%|_NQr{F09dCgQv~Wy&#nbqJ4ouCwQX~c)l9t)% zq#{W!mPK6L6u_*<180Z``G(v-k9BId-sm1R!ZT`mU%jFLWn*5B0_=~BsK~e{%Vj?C z+s6wzdJg({>m&#M!#XuFSn9c0k#)8MM?Y`XDaOvbss zd?K|sv9zy$iJgkHn4Pwsd=M}W_WNU!kdSaxAVfmtR0#xfPuwoSjDBO}yURrLyxQ$z z@njCTB{}E3L(s3~qjC5A;QNe(8VBiTJ7U*HVpms%L3Nk6dhYyIUFz{;p7}&W;PtP! zQENK3jlMG&&s>Z5%T8nw_*(6AaJe!WvLPhgz3?u4B36I-k7W|Mx>>{yXl3*8Tx?*1 zdRsoJ7-c6<;#}i&0;ZN2S4j&mm2V@QuTmBtq<5@aj#`#l3HN&5?x9lPh?xuIc%t@s zDya5a7LfySNxzkq97n2}&s(P?%tL9cc~RyIn}k*2K;DfB&9(UgGrnB&6MmO zC1YDKsC&~d^76nyhWQvWL4n$bK?QoM&o*CYQb}w%b=62^Bo`3ye6M>4N6D-i zg*z<`k2)k+%=VaFI<13M+&IT-)P0n>dO$ZSDIo*vk%MU`KpM-Fz|w_aG^q!3Pssgv zFC9}~tozOpm}Sku#+4+A_NiDcuSs)HTD(5rn{xQRzs(Q{ciWbhx7t-7E>%jN48}vL zuDC6kdl0nkhPkb87b9o%F^&IS+Cb&Z$Rk5MCdD8d8HqpAlzBQXUo4A5aam7)tcL&X zH>+}A=od_0Qn6`X^twH>oW6{J4|a*G?eiLqeIbA5Jhru0kAkKAHxh|};+zwQ4d*A7 z#SVZ`W5j9`5HsuMhl~+yWYV7z{nMon{~IlJPNFdVRaq78q~t$vKwQ)@LcgZ;3y?J@ zjD;*UxqQKsF1rf>*y9k$i>95#d;EgI0o-J`qr_+HRwRq3e~y#ebs?JgM*f&s;!RE? zx-UrM09J+YO-;BMPRI5i)ERu1TRmr;!Um}?{t?*NCGdZ4$GZYkm!9*SkBkx9ts0ks zDJIuhHoe>VOM;m2@8O}SYgs)s%O<;FxR-?1Ecekdfx~mJ%4m+k0WOt@kx~1RS87V% zwGwZK%%440Uh9d70_Dlj|N`G0ohJHxw=(r|!!D8L!}{f*5zd5q|I7R{7~aeTZ!_o7S5i(PzDKP!U-|{Zfr@7KSXXggyXf# z(V0ACDMIT>;eeDZ>}mB5ki;>xeg*M~3 zTg@C!1?jWb+ zD6oTBQ9E&jD3dG4@xWp9EM8i2TRXoGo^MS;eqr;JCN?g?jg75Kn`mm(@5oco^f3LR zX{FtOGf4mGWT)k5Dg9V=EV{aXDxOCoSrm^uaGw@EXmBYD+G^d*=Xnu=$ct#qdb5Gn zS0zl_W7%bN-)@o=dzpFGoaR2s%D8V&0lcE-dp<><)AJ-M`T)(P)MR@&{CEi`3ir|^ zQ_>D{m;YqRtJ3fN#gQoeSlsxk(hiN=;YuMitGo*Y^cf9o{{{I6B8V*s`1rsiOI##? z2{kRg@N`DtKmN$#Yv1BOt^lx-okrrC0A#>ozY=V!U|;v zrJaVdJc4uIYcnAx@7M7-EdfP`^CA$-1M23!Kxb@=FoeWEVPlbn-QeFR58&06xiO*s zES8=*lc-}6@%oE_lqr07ERLPecl(MTgO1+b@=NR>Xt~p0MZ2^7g%l7u`y7i$GAUxI zXQ)55Y|!qi_P-LUWawOs-%lQa16!=PI7ff(e#ah=z^DcHR0plyA`dUOMPd+&>YNME z9d$tla$C5Dj$@O3(AeJAPrp7qnbPC8|3iH-ww(?C*OWbsmBk2d%M-$6w&1_KrWO5( zz^P%w6@4ba#roXB_&YT!i0-ZBi0=~^CJ)R=-{kpIklE8E^6!d=2ExMQV8PXFIu(}R zaIa`u%~&tNQ%sx8GqJ6DeZ2Q$rqx=v8BgE+whT6*NMCBU7vkUC<5*~{ngPKyr=#mw zasnwSL3?x-Si<4SOdfp(|NKA9_FF5g>CtF5IJVE8N?pq@OJdVJo`JsqSC`70*DYje zN-AK8`U)h-RCw7x-7PLU3sr*wqWq?338EebAC1x~BkZ0G4EMCSO`4xbB; zJKyC^qSnR+62-M``QKX&yAckN28jAxm>hd6`5iRDRbB_3zk&ysv1=BcI~%qMLIDxI zyD-pGEY+JHoB#0El_PTMSDv7^KI@Y19od|dL&kH)Jh#;V zV$XmpgNW^VM1=VboUg6!3B z$i5*5qj^`_g2k^;po~0!q$u-RQ-k zWF&-tpHZ3~gR-@#7_Jth6E-!!faNzTQN*@s{V+j1=fia$+cyeqj-5Z|VMrQ@HK&Wt z-a4;^^;{<|Fl6G|;oWz<6fY*JH13hm<+tGCwZW~N3SfERNf1)Ya;9(IansNe0? z?kZ-7c8!QUNF)_TxP1DQ;H2w9<;~qluQA3qGmmi;gLndJQ@!f?10Jxcm6rU{0Xx?I zgQsqk3LIKfc5zr&;^h6?>e0Ewr!pKwpcLln1!{yvL`Wnn~N)ZPTI?Vq5_@A5oG zd1wz=rOhUZqT*6&v(=!WwIVAsTAu=wC+Amb^jxCLvMP|~OPu~BcB4!7g-`N4ppY(1 zsjOh|FgU{kXXmyoJtA@OD{Kn9qz!b0ebqJW*iF>$0@!Q5`3M<^s+iI1RxLHmh2aKV zhF&H~W@28qvl=*BWG2DW^0cRo{)A~<(M-(T91{^E6TR}oV-N8|xrFiymHjT@v1X*w zBk33dyG|ugUUVKlikmxUgmcMi6TimS>a z7WgB0efDLcLi{AK>A(z3%qr9@JQ5`(`p?OERasTN;x&7cpaOXmG&KWHtA&1nmAa;@Glsn1=sQ2YTd6u8U0RU=P>c(~P zBM)dA4s{TiH{u_wvS=&5Y|ruXS+>)e1|4UugC*0SYz~BY5Va_lHd-Qn8+@BxR6H9G zppo}cSuG;Utb!$TNWSIwD8i z;^=7ek$i+y^GYFsSwSymOv#^HAhi;;55TO#UAwJ03;A_H#E+>dtd$lY@072l=iWyQ zA5OWG+d%d)KI`><$rXXr*HuI9`Ld^Y(i;+^*IsRX2vKhul1>M6VFBdAYa492&bx)d zK8#O;Tn_o`h0}tDN@wq{ILtC`@A+@)m3|yVf0G;;9=_{!`4R8gFC>BCq+>I+{$9e}_npw9=m>|4NvE3d1Cy-_Mr_ z`kg+fN$$6zU+auhoieV2w|=lLw|h3(u6N#4gn!Pc7~{h}X!lB>!{I~-YMCJ!=XyY0 zZss!kT4_F+%@rA&iHham{d<_E@=nV%A$XacopW)nV$9b))<6_Jgf`p&%&+2y*o%5? z&RH(0_)O|&Z?2q5fa!TjzalGLL)*29n(7)XPP)aH$Oiw9+}nY*00i1 zR&0)_J07oTvyXinlko|XE_C<;IqrhM^x37)U?AZIm z@j3(?E4q_deX=Gq~@F4zkl%2luIMLiS7>aUskwW%a+5!#V<=$2jNTORwc}rne*bh=Rbi z#6$ai{5w3vXptUetHI$H3ewx!@IZe@+-=?CS&a8f;&v$}rc-6TKD%mVqu9WXWzlxe z%_Y2+vgt`F$K57n(pp^k??Z~SMMrMysp3G@q#f^Lb-s&|y7ste#(db7)6tQUQQDF4 zED4aa=WA63={JW^N<}%x`yEs1WZRo%v-s1tiSM0u?QD@_BfVH1=2LouoJ65g$8+J0 zW+8U^55qH{KP?yUaSiOQt!Pl+9xe33HcxbUx_veY;t2(u3MZECJ)?^^S%I8m5SK*e zDJhl}V>X-nzckS4o~RgfN2B}!Ye?ckP!~ID*H>ug$c+iUd`XDKEj0=spQI3dC*nED zyPNgV1*@IxB9P(!Xd4st+p&cEN~C#MAUuGUW&Q1X zJ8V6KbZuhdv^YuGskloy>*1Pcsp3_{E9BD%I37)q3Ff>XBX=1R*g3OvxiVzflpPGybFlM;6NqW>GbeP| z6n=?-JCYs0otRpI4i{<85ZFrJ`+!EqC+s!+;3@Fd$Sa}32@!f6!_4K?rUKiqZ5I4P zR4hsD`BC`S_csR(vsXLp%kJvT{YO;LW1V2=%uLEa>pjt=ejptObZNQeE(9?&^!t&a zAhZgK5N+mLzFZ#3&QDKAUQk1eEhCXjibzt8{hoDbB471GJ1L0S$S@Ykz!Z&_9 z`C5%)BfoAEG<`saj@%?i{+yij_$W+b-craTxwgz7M3iv6;IDq%?bQ9+ojvLD+OPk= z^??G9p8L%BzVrVU$1daQ6_)S+*2lM3-(DBu%~94DN{DYrILtCTc%F-TF#)j|Hdyxd z0ne`1Lmm)3KWHDlrdE138>qx)chJnde)XretZ~n@}D!a*PMbD@=)w>c_1=z?ge~}(cYdJ+qvdnhZfCCD)b=Pj!=Lr$p$NOjwN+7E z9X3AnYWaW~4Cs0!`j`DZ_eZ?f`Ta1>6Tw`c?E>P+se>%CzRN*uwi#Yj&)1)_H+--! z;7QODfJM7)nKM;B0sx4hCnFyxUJ%A=;G=dCz24pP9@)T+8MAXZw@~Ev%2f9;hUvE* z3yUR#kvp&SNN)tl*)(*%%cfAxpm%+Z9c_){Gg}&44S8tYTfi9Tz*=ENvZ-&`bTl40 zn|dL1PNLF;Cw2-*i&$9|s{ z8$ZaIHLD05U%6F^T?ov5_5ynhnECbzQnQe+jt@M?v0th-UpHhg!{HX_(qG9MD+d8- zr}pzQ>iwp#?8DEPeOFb(iZmg~))xd{p{olTd_NN8bKhayGryH66e0{V*uB@72VCH_ ze$MO4200xZT}GtC%k6!Lviv1-5;h%Al2vBZ<(hWpeeSMYikey9A76}8!yh>@y^~MH zxjuSa0}C4!q;BE)#^NJCwhZb$i6`J_+o*oyLNlBpyYN(uud-W!9l;w)LqJKl-L(<8 z>wL;gPRw+v1V_{tntbZMkL#c=CHuyz#Gl@Ca*d4pmyK+|<;a*a;Ebls@<&qvbOwk4D}+U-6M z2{QnIqL!o{`WqNgAhZ^3xUWcsMQjUY7JmAD{MawaW?Oa27ce#3zs5Drh4V8V4G=5% zzbqP>k9+ukMpwCd95Vd=F|lb?Oz(aD8Z+x0$`I}2RxYO&;vu+G3_?)kunBpX{qzt%+~4sldyZ?YzO4fcx}vgZ zSOicyD?fB^db|KZWV!uOT8<62D=@#l#R{qa$5==gl2_KsRFE>M*b zL1gIo7H8Hk1C+QiFGtlX|m1VyBB1PEXG7Gf(C6d~C;?4wJ?{ z-J=!2eJD_To0sT5mwbWeUw6viZpY%6fZ(epQmI(_p)5twhDfKvc8e7(%Xx) zzSA6;er8`SrXwf!smI%$Oz!Vc`fZgO<&CfwhbHp5i4;qPI^uliMgtygs&$$?J4^wG zid~!+A4ZHzfj|&~_vV9zA{0H+qQhnjIZf9;dXu)j*^-DjZEyMS_1Hw-v#)5G5@KY- zg2?2EG6dAdu4mfcU~@cKRGUt9cI)<9amxJ4z7*4;SH0MNfakZ3`$@?yZxb2irsCE2 zgXGw-Na4CGqnFE^_mn4Jgg*UHQge9C#?W6vpcvY!%^fKp5n|S5rNk70ZiLmdHvPNC z_{H5Xp8`p?9;d=jjJP6MojIE6-*zi76Fk|@W>*+p*b=#*aUmOerNwTgB9DP3b|My7 zk4eaD(d4>ld^^&Qvvalyi>f_vhxVI9G%Hij4XCdB;3tm@Itu5#HN!Qe!)GKiTK+*& zuL})Sz&}~o^zL8PCnMZxtw$j-p(I88?GNoK6j0F~x<2u6^8Q@(+F!jZZ^Q#xS$MIT z(mEE3>gnQ#mqd5$>DRVT4Pd$q}Oeqab)`?`l zgW(*b!{-8niTL?N&13ab0(*853@z1-Sveds0%nf9E!O9xZ@`zH$SAaeF zT8-gw6<)qZ(xGm2&UwL`xcKI_wvbjkqm&CRM=}EPLXM7;QNd@Ne80{uax~v-O4^4L zOBF4NKYLvW?Lc6PdVWmj*SqXR$hZrAIFmDLf-?h$W&s_7juK-$<0jXsa=q5pCI@ZW zXSHES>HXc;>iU}=YK)rKXj zDUd1CFTtz5;3#SAzfE%i-d@l?RNPH}o(g}G(TgS9FZqjBSZ*oi8#o~RCqc01POFTM z2=LQ&vn>{rLBYb-uWmWQN|F$F&pMFo#5S+YU*alRIE<@k(<1uycwz|{n7aSG`0*-{ z3&u24F1O}54+$(eoNpldT`PNV503uT2$U`_Eh#YoEEMpXK8C5cz8{zt6yA|Dan6U? z_=7DYw(sCpvl8Bn{A;=pNDT4pjpBD;k#p)weOR|c-M9x6jw>({(s4(|$q`tQ)Q!puU?*&A{4;+u_K89TTfH^e%0elHG?@9a^TO%up6Uz$DMH~3f#ENU zgze}0mL%}v(Ou`)Q?wLZ!T<9DfN{~OcuZs&B$L}8PW4!IeX>pc{4~|qiOH9#6UDf^ zkS~|4F(aU;(kYQIIju3vMG7dkK0*3AfRCYn@!M?GB>T(~9~j;+6*w`!kSBO2bs~%H zJ$|n3)v-QHU0I6YWHk4lD9Q=}^0X!)r3|hYUCcc!ORrV+$%s~1oTyl6$4;4|m_0VU zkczNyp82SVpDfz7=SM`>(ZFY|c|Gl>Wmw<+D{syjd%shkuCIfu+rM?`U@Z4jg}sv7 z@gkT8VzGN^b!Wn2h1l0bYgl%B#lQ~*2eoB_A10UzWMN@|%TGV`^y+9VoSMu2$ z?0S+7jL8232z8g(%xQr6(b$GXY<&9Rn6{tm-lD=N)G*c)u~W}i;MUp_>B>cZG-81# z$p0VO&M7#u@N4%SXJXs7?TO8a?POxxnTc&rY}=aHMh6qy_UZqr^PT$6cYnHeRbO;p zbk*K(zwcVl^LrMd-`arYX^l8hql}HCLc!jGWW3h-zlNrZf@br7jabD~#<_)8yhJ5n z!I4O~H6>JkS>nUOjq>->IfkfdoqvH$jN)CMAgp=pW@XK(o*#hz-_fpKT4;{{PEh@M zZxeF8wp?>XTkpdn2~5fE1v-w7(2QJrRQr`Pwz9^70brq(tOE8Nu(uL6f0wnNh2tWH zsA<*v$=WnD+UcplsQe%9&0X9M^%^GzG;#OhLV*D+Fz}O%xm6o)9n-N3v{P%(VCG@Us3!^x6 zE&kkMOo-`(~?1{tBRoM?*Q-P zG8j=pp8M+lG5?rT5=Ie~0|$_+`{QK@nBD#=MYg_~Gsvfw>ei&}=aTo9{MXJQWF`%1 zzS;10e)TbBysI8fi$ta#@;tTUG?Plm5g$hng=HbwT|0c|uf@gF_V3=^-4~qR*Z4EO zX-@o29*Hj=;Uk1W<+{fO3xfupgasJ@PmJL$&JWd|Zp|0~@hRdeT}IhGaT!ER4P^%TQSLwxtMo9~o!ONlb@T@D_sgD>wLd^0VHb zbI<;-l~Z{=7E`4na5w?B!4tTe#YtimHlr>7NQZ2(MrJd^t1^8 z5PywG&=;=Aw$9qHJKz7d_A3VgL(-ihYsTL6VF6XB@ZL)`tSW}b|9#pnKG(N!n?ULP z@TvTSFs2$Idc9BvGL(ciL_*!SR`Fuh7|1Tr!q@vO@qkfLH$9VbYJ3Yy97Ty7Hc_CF zrm+MqO+3tBSl1f5Kmp}0#;2hK{cM00H+#|IvIJS{cpMJ|& z9qr|sdFBTDV$wKPkNFC|fOARQus;nqfvgWdwHBqQ0Fthe z#d`wZ1i%5&1=tQ|oEA3Kx}72^YOaC|G%b!z4;i7j*?0rj`N2m*-jCmg7+_`etgnu| zUU1YsEKMj|pAcot*nWkJ0iFfsv|oXx5hOSj;AB)KmOP0N?^cC>i4QH}xf=?exxh}A+l7W8nYz5=A+lqDIo5-DP<&&2D z%GanKY8GxoUwg7OjY^hwWEgxSJRXi$(#C7XgrNtKXZ-zVd?m^ za{XL@C2FE;uA@oa_F>~m0hL*eqj%J!chT9{fYQ$S7UR)HLRZ;qcfdF%I5%V({0=eLiY z_3I>`Rz}Yc8xGFPiA#r-r1#7zMaJ+{;f7VV$6*{UK&wUbPbdEciC2vCClk4h_V%@+)0W=Dg_UWgbli!U?Y zZ&1e!O!HVidbc#@jSf`NnFQ+t8>sBDu=EJHCxtO7^XV@kxfpr0{O#>E3UXPv|Fb_L zaZ68r?X!_6;>n9_VEnG71NMBelYz3GFT&;0 zr7+x9@Lq-slrF?S=Oeo$|Nn|Vb;NW?)?|x92kw+eA^C2&9?MV>15gThA_l<;FxS^8 zXyIDgsHHtkBi9gix{$*sbpT-giY(#*^>6D)x({Ah-=lp>qLAZ|1e4$v)Xp6tF8Ao$leS1 z?saG->h$->9q_}V2u&xAOwi|`B6bsw+pG;%!+CO7{a%s?0O;*LK1hmV^ab=KQi`)y zPgM`R6m>QzAZgLDxus22by%HAKl~9JU~Z=nVquUf3-erJv_Jf)G<@w15Nbui<=%IrGK&wZ`f(yqzFd&3C3X@@xF;2jh>!t3bQX4>)!%N5dvqE-?b#uxR4#rjgRSLee5ySC+ zKx9wp`yTHoQd$I8d*5hUS7f4d&It>S|MW`#ihl|EMa<`sD6Kv4b~p{(S(@52QTXw8 z>QmeDaF zga<^41x*~niS$*Bd*2j^8`xWnpbh`=6hI2WM7gHHiW=PcHCa-IeqVAoj_a%8t98If zT+4Y@owT~w()MzZvFQ$UdBcqU9_X@I{gM_On=Lb5VGco(SUD|pj|X(~S`MOA!yzjtFZf8$?zNQ2~(EEz_d1OW?q-3 zL`(TWBPa2goJhH@oLn>?Dck8zXC^~Km+ZG~kFa%%S5;XdiNzv$SUdu~J}+tjlEdemITBCR)LPCfH5v`LxIc6iL7`J^Xv zO}cOKfkQ$_v`6n+?L~mH7C=Q<1NnV=`#aDG{|LU_w}nXShSJi2`J^YtJ8tubo-{TV zBR;{UWl{qdNfI=yN;<53CZHvxtY^&QNZvp0>;WdO`s#U%vD>KS)ylspYLk&b1>|o< zY*+g`X+16%-8*RcU0M}8t7y$D5OtZOq2(aS=mSvA437raG}Ve%0T<)io*bB#np6Z5 zl{?+khp-jMJM{&l8tJb(Z4uv77%59%w{p#q!%fi0cIW4}>BF5f4RSVKRA(t8Q|HLq zQ}=Zp45ZD@1-tBfb_XZA*_w>^5y%|8;Osx=iaffNoaCl!w;dfOKh=CJO?sDt&>{;0 z)%4K)CwjKu5zklnEp(}$E&!tC0V*j$X!M=E%t1je{dCUY>{F1`0#j|w!E#S@QZ9}lr! zT}b`(Q-=HyhN>eKULRG~b8t45=BFcfw<$+636;L-ioUZyQ8xmbcb4Q-Un*{(~j0!J5j>cf}Engqrx#3{g=Hkf-T19 zKf}-fI$%(ep;faM1v5!A5flL9;18?{EOKCyVb@o3As)ve1b{>p9=z74Y@d3oooI+v zC%?t9Q75jn%=#Gok9ow7<{jn(LXIU}r#;fR{SZp%wH{8F#Y$5`Vg=eT{u+obTHgkQ z>1^@!KU_lsVDG^LGe$h}A7li4Afq?35ly3Ng_D3BHHQ%aeade)?v@u=F{m$SXFpLq zzwjaceN;Tjv|K9Ystk9!6uR3Ar>Jjc8JYpEUx!kyfN*YvaaOCh938RH#X!Ti!3`8a7w4=d=;ou67&MY5zPgWlSM_Eo6 zPR!c#UpP8|BHrr$tKwrX6(?+I>s-u_^vx@HX8!w28u&Cu3tVM%PQ6)ii13U{!oxv@ zsi1q?i$_g^He&e+Kr}#oi@0Ds@&onFg4f6H2D zS>~vWR;A~WS(8}aT=lOE1D!%(xvfp)!uhB>pbv?SC*>b65wf!T^xxxWsUAjP7`F_7 zV%-;7v;3^Msi1XC=GP`N1$$~!p>vLACxsRSCb~m7T_T5{LWeAO1BCyCuKe9X1yd66PLN~YJ+m%C$@vQGs}|5M zZdIhPhg;`8>av>-bU@RJ+nEUX80?S#OA&eln>GkiLB^+>^qAf*5^XftJ8PREC8brt zfs3nm;FTOVCQ?1>eLWpHS51Si?=Lpq6PUdQoob7`xjZ+<3Mv zYhFxwhE%t@AJ@9%*Dzno@91kt2+ky#(w%6L%tJ>W3Jl6Uf3-duU8x zOWMF?VdwO;(Vage92cQ`e_m(x^Sdd0C}OI?fDsaLg{JmWFs|8ya1-$-DNC3VwbM{r z4nKd|Ch7F}eABQE)sZ`#*m~nD`CZ|73G+y^z({@@`U|QlqIwZtpk$BED?BCsUwA{Jh=^Ac2p(hq}mf z*mmG;^?&@cU+P>E7HlhZ2xb4`Sd7B2r$nfP`YrMvNhm?cnvLmAWE$ChB09Uztwezk z%eU1;)Q>VOW;s4&ote02kwrQ5zPoZe!1+sLlx-XYPu-Fj6S&eLpd{4m#^$XHJwJe% zwzd`YOfZiBe9!z!I2sehvOhh9#33f@u06>UZaW*R5b455P+3yZg`{REEvk~oPB<~Z zMU6P&ujZ_%a@k_IEfPq<4r?04~4TIxuhN0XivmtirEDt|(6sM(TlslNQ-E`(W*C=BN8Y1IWp52eAEYsQ2g(6MnrD zz;zZ^z|l(%Kbrc5qyIF8$l97;T^)rK(QX>%k_>J2wKVTlO z-n-*I3~ZYWw)bww2)Y`n^J<-6GIJ=y2P=4c$7_Ab-_W!Xrruipkh ziQJ~=92Hs!>hzM4q=5m7j4SUT~u7w(x5LNOzWvS|v z073sn72(_M0tTLvTDx`Lt&GO{V6?P*-OPL%o4g)p>=1zdc#6}?1Ud8CSGqEH z;&u038b|;vfLLzRmF$S_Pf%G_=PW1CT^fM!Xx${pv0tHd_$nrJrpm~S@pL6UNylq$ z>zZk}&f5PpN89c{feeX5D#KRT)0{>YuKV!}N@gU%9u&{wxxDR89h77861+tBgg!XT z+rrNTt|Xafylrho;jC}zZBz1&M6@&ni8)kNR-T#g(|kz4klNU9SPGQwZ`1$cP^*6!$mHSGw6~u|*xnN07+r6z z<>B$P<*!WTx!o|a)%NI03a>Lit;eL@rBHkyJqXcb#Ku@Fu*?J@BHb_!H=uQltS4!w z4V7|LIVgQ*r%yWNo24=g=$n|?!aZ&^pk*K3xDoX?G`#FJ{?D_`4SKev-b$}yf?#l? zb8X;Dt9uHnK{xV53OB~cOiC%YV$sFEKYVN_yw0;TW%S_~Nm!&{7e_*Itety@dVWGF zN14ag|GU*p^N^nRP?|GCtE8gOP112|CjDVVk2DTP4d1N_Z^$1?aX$iA(mYb3?iKpA z#nao{+v##$E7cu(!>{MGSL9V~#?GadfOH~H*I&u^0u#}G^x z?MZ9K29Tl!zZHIfHe_P*Dn7S5^-d+`i2Ch8#9<9FFaJ^SF3b zlwV_l0KK&f^WS_6p~Iap_T>%2NlPPgI}Q+n{CykeO@XvX#hR|e%n%xv-;*|FeqaFF z?xFeY>r{-MHFnOp4!_g-B1MMpy8;y=;|6QsVtYrwY<2v1T>!za$}(0pKX>X0H(FNg z&$`*yK?A6&A8aGW9!U)f4S8$5R;_H{t$<;TZ&wV_uxoF2AN^^FISvxiy9JO05{bc3 zbbx;;t@<(z|M1-*mB>s2#MZ$8kQv5gjBAK~o`gEDtXGyRA&RdGq7PJ>5*ax!B+~n8 zZ=Y_7fF#vwjUURsH--QM>tcsviq+qn=IcU6Iv%^=P%bD^-(;tSvk z>E5YwLpq3yhK*@}72qJgJ9XV@m_5^A_N&@3fj1<6@DW;Ac}f=BJN&a42stFMHbB8r4iH2pf|!;C6*O z;^Z-&s;s6VQ&hxworxc!{*lusMqNmJy$a6nqZ@}g;2K8%8K!|Sut0v8d6v1A@gup@ zgNhIZl`(1%2Ae}4Ss-(^tke7|wI1!BInRc_4FUv-)SIXyPZY<*64dAZk>x>Ua?i&A zT+>OuA*;n267pvI*v5=$8?_?WJKeUde}r}MwK|byL^7>wmLVI4X87F#H9li92d$l95Z(u0F;cYBLj;gy92!Hwvq2^}~wo1G2|7Db3 zly?y+A~sF#Eep}5Z6F);CAGNP^z$%K!34fs^p_vkgAt|8#5t{>zO_X@dAR_!^+eX> zsghmIR50+xO53jAZ_5>qhB=%ds-a`bN?uG0_+ecQ_A zvFFV_<=O=(s#tGlr&#jdISEBGjwec78}_#a0k7L6to#j@*H#QlUG#h6=lTlSp_)=l z(hVgkmuYIolZ+{mOhcBifrm^D%nqs~k<{jqGV|q@M&o0d>1qO&OJ`JfBh;Vhqu40I zt^0#*ne{g!@z{Og5H*8?9&O&Q=iel(SB^?be}wL|F0z$Bv4^nUL%v|1LCqx|#MU=l zdtthz=k_yd13h7Ei;Dm90$7*b#;dSFqWS+0%7_iE%C0&m5#3xV+0iAlk434LfkSRj z(3mDeL-FQ^!^xnPQ^(&nQ+6_mtLgm#EJ#jOnNEfZW&1hB=3H^6IgG=QATCmW`e=|@ z%be`xFrOqNS7wLJqiGzfUdpIdwD{7UzyXbl^7$ZXHeatO=p%FCobh-*OUKpCJ38FK z*7+wQ1U(gn65xM~Kvoi#g`-hbDg~XF_$&30ORY4o=leCB3s=@u9xt367Mz?cRN&*Q z195JiE6{#K&}H|S@WN;0Bi6uAS7}aX@9?NA&~bZ=<4_IpD17a<@_8K;PBVqd=1EdP z4{xl@^e)J!9$LlbH?1{xHA~Q1P4i-HzdS|%PF=q%mNGh9!Qu6YU@pXj(2ty1SUdaN z)MWZ{Fu$#TFTBl}Q*e>e6^Ixfe{02wDrpLlQeer4qQUK1h&?+uH7Y`RAp2oAso?_sQ)`4G$)YLbkJWTxe!-btC2|iUG;MEMzz$ zsjIfB$uH#aqKJ7t$iO^eivtPRsMVb3^Mve^JPEAOC87%D_w|;dA@^mv>(BR-3;3BJY>mE#T1m+Z= zl42P@S#BWmIqtu9_Vm=_bcLt4(bR{8{e_#LA4`k?oj`D!l%QD22Hi$RcR=&0!(%l2 z3YyP*rGA7LN=wT%Zf$p~Q`Qy;!G@5>!-~9V#6LC!^7{@8q}kM-%dr~@_}t!~G-~o< z+UW%@pDRzi_}VXp&(5xugqdKvd)#ftPeNd3x~u-(DfY)x z9_;T!))CGp6SeU2NQf!HTy#v}*X;K=v~DaO@0#;}iA)v`F{?HRr7dwcb8vF10JQ@9 zNpKLLzo&DyUqxDhg1=6Q-|4SsO!?ckr`I|KE6(ziQR~a6dnQhDbUilyNL+rmP>oS{ z$klYBnSHvy|puTn#CxJ-9ZV?pp8T~62-aQtFY>A+C=~#s7n_-Uso(1VBE{#c} z*yAQGV|hV(jTkP}5rKP?Z+Sw%jrB)QTbQS$Ou_bx6tcU;9Nud++;6sJ$3_Ce3%(zH z4Y$u0wCnsz3xW{pmc?<2^cD~g%hlif(HDF zAZFG7{YNjEO+%w%w_^|KtCZ9BTEFwSJmF=X5}IFwE)s71WqeGq?fiwAM%R%Ik8KEB zY`zR2{y*QPIvVzG5zwKdwV5x8pLE&-9S?2o>e(tv= zo6=1MCfZBVA*(5Vo91hRw3yLarx@2cQJ_wy&Gug*dgAaXGu_V^bi-tBJw-A7_ zI%fWsU07A`i4gfU9hgrL+|YXdE9OT*Ek(q@mb5k>ao!k+jbZN3l8G(*hsP}|%R3>j zx8_(Y&Fsv#OD1W`6e8PzHI9F)iI0n>pe&d^I-H!Y4r^syF^SXSw3Kcf28_~%En!Eh zUlO3ygP0TS>E1W~0+R^&yLS8rWk|b9dPYWidS*sNp}pe=fy!jdJC}2SjH1hH*DrW1 zTb`zVp_LV*iROl_8$cuh@`8 zVUv7!A5jQBkcFo*M^$;Wv>-;IV`6nq$Xb zIwU*<14OIKL{THo(683-@`!a>LgcqKjTF|_@?v(tIGf+f`_QD#Dcw_UP*}fuWe&NO zNyJf&lO-?6+fNa*>v(FaWt8Xnvy)$0ipu`f)V@OGkBoY_c zCHhO)c7*u7|G%!Zo40oI&wt4O_ks@nf4>lR9d_{?(u;<(lg}=n^9Vd0gJaX6s+e5P z;}yLZ7I$s?-YCC3mH6dWX`TI!F!-ym7$$q{!U4?dzGV=mRUzf{lA~&a3|Tm94DG{h zFQ8<7f9JK*0vTNrg|r!p>UB_BxMy!0lHt;UTD9tY($1jsXyTE$_{{KIS3&L$8e#di z6{d>p_ux2B)`E)D@q&PqD+mqe^T{!CiqEJT3`kl`N@M?1(KdOY~@^#oa56hmMo*{n3Dvco#Xa7@;<{~?pnD476uv3? zpYo=9Kp0=^xg>oNCC7hNH<-{alNS`!nB z)P>h`%iSI6PdkuWw+c_@&)xu7ZQRf>iim#vC zTZqB8&NH(RHj;22Zb*t=Q#Y&Wg6I;SmLN)_=75Q&_KJ*gEJft|09-ItNNB*fH#$_l-2K7u}fe3a~nDI_%8Z~fqAweokDV6 zkJE^3GuR{bZ-z=%yMxCMOX4nX-!u27!Dl3Ak-sGuQE{8xQYA^tECqa7AENIou4u%P z_CY>@SE+M3FYRB|`-^uM5Hun27)_u7^EhFat<_a?@R|5r9H+N0mM9OxKxL) zY9nzAxTJD{6~}H9A$MOBxUdm;8Y43``%zZ{8+XwLGh`FU5==)QyQu~&x_`4 z&u?tNK|2-MO|H3x_{>s5$4LQ#uX@cMnz|gkO*-c(G1->~+z(JbwZ9ej^b+7{R#a6Y z6l>B!$MdS8)J49w$ca0H4{WJ|B`hN=@$eYTQjd=#v#~`np|S)s{LE@tLk&3WxKQ$> zMrkNH$e$WVZuP_W{KLHQr`+4w{M(JMQ0?yeYhk*>%wz1+z1$KAqpboS`|;Y(`_n%? z*FNtI3z6z*E>{cdj*iTeps8I55>SX42u{{q@zGY;vEXK1=FGtYv`OtXAEt@wb zdJgrDMiV$Zes_LB1}t^%%$t^2Js{kO;^m8gJ3W+pdp`*BFe#cE$hsp|a!#2J-PWA* ziJ~Km^K)DPJt@M#lASJ96stn;KN>o_Dzz0bZ2=;ozZLPAUoI3RDZe0YLM*3}tK0MD zfi9^l6{AGNNVW)3z}3QV{U2R~SJ*mEUW0VK8>!6gZsft_@}&UCQa7pZx+q&-5@$0zQQ!% zHhb~L=n9(JTlr{R`ph!WN%XTNkrCNnd+7O%+}Lx5v6yu)h(2};#$^s2>?>Qigh{o= z#}58tU&zy{eo(?b?_%H^Vx{qX-?M{$Q+rl5B|%%@!X0fkOCDF8&)QUKTtnn_*ji~s zWwlGXI4iOKSyovoy$jZcnGu&qElzo)(msH|W%LavIVLIZ$jLadpxMpJ0v=9ewyxKG zPP90Q{?IpyL*GZyd1c3q?fHA*NV&{Tph3djsZ-tFrc|xavRlUow`F#(79J+bugjKe zFO3N>`@tyk@4Ppi=Ox>~kFIY+h}u8dD!8_|o7qV)o(yFb1zAX1LKuYQwUhPLRIi(x z2`OdDqjA_RQxI-+M0+padJ%#v`ni&Lt%kya*~;fazIMaD@NhETuujkSK2o&^9!A@@ z9F&e1&Ih2W0RNJ?^dfXWY;%l9(+uQsRK8m#c4!?*MWxK5jHo6L&&(&C?iV=Eyg{B; ztrI1%aLz8i?t0AozvI9{M}4p3K>!E651MEZFgb-rY%<8l1~e9wq&D1l5wg!MhH!>R zS|f_CDHy_!>jV_5G*b>W`x(Zni-VORdoZ}k*k4MVhoT<(mufzakLF+D7BH8go-^kU zQ|g?#Fw;A6gB`;^S`>QsfSUKs$PDi^70y9cGrc$DwQi*}FCH>#ne{z}^~ zH<*VQdE0eN&m+F1;nA&4^A(*I3Ch#ZfvM4IOg^5S^ud|!Pgw0R{z1&A?SMXeCKMdB zOp01gv@9oXO}F#&H4f`<0unMl9iP`uF&-e*H)c6;Y`ppdj9+2?rVG=$Z{o7Rn{f&a z@YCK-XGdS4--=Bl6eYE(uG4nV&E+HH3Rl?B?y>a^W;x*1nPZ)OpnURLMAAxm-40o* z+Y?x-KSNc4dzG%|Wg*63C@T;>$#8daUg};R!-Kn-QHXOj4I_|i(13$UIz>PmmT0TD0WsZojZn!HHCSy(r+`5xT(sP;D(e zXh7iu>JnYLf>Upq=Hd4?W1_|fW(asD5_~z`OY>Dn(z@>6H*<27a0pO3F#zk^7l!q` zHJR|UnQrs!23N_wcQ#@(`#mznj$#NNlY2rNeq_y3d`u0?hnp_H02X}DGv}>q_bZR^ zwg$>`Ew8^{YiV0^4Vfwux17hJ9-3Vg3qTt@Coo-vWRJIb23dEtu257gR)yO(?sHOv z#gpyzC0XnrKcA%U@5(Cy0qFk73WXd}x2t`pFbM56VmYYdeG?3Qps5V6+s*aPMv6wg zIxHCQneDW{Bc?D}!eGG(ohzflSE%)f9vi+7$l?5%v%d%dVa(qagUdZ_=}KJcm)@}K zST&7v64wnHD#CTl`&nthtKLE~Xt!Id`?+V5(d^eokkBy5xS?db)X^xx08rcmx_nnK zck?5|Frp|~eesFDpvC>MdcvJ?Rky!b(M*4-gr}-~tLscY5T3(btK(`*DR9;MFe%(L zpLE`%f?GEpOFfd`RgU|+BUC{OZ1y+IGPEG9Q(sJhynyT7zQDInwfjjw8)UFQ=UhL1 z4572~Ov&Tc&^`RO;GhGkN-w*$!V<8f1W#4DN(Ov<}#W<%x*~#zDU|S z?9&oD(q+}mq<9)mr{pyoIZoLr%>WN_IAn1w4v(Z?lz9O+FkVe5BXEk@ZmT{W&#tZH z6Hy=CCTLy{B^)yzo1)`N{)wxEdEXLr2>g#`Ly_u5e?)?{zV8&r!J((X0&w;|jErr%)r{I1c&iLa9Kaxk7U#?QO$P=DK@66`}3UPRU0ns-fJfr zNcBO5y>?unv$Y6b@03HpL(8wrnh8TVgw{uVBeg3#-Q#8MPkdpTq&R_ZDJMLA^jQ+a z_G*Wn=fuLLXRQu;D>`cbTL13LnhP6C(CFA`(gcSWEU0u79cf=7*Q|OLqERYK_gU>J z@57j?#kGMNYCnFcdx=Fj8jScY8XA|qQ7{#by*n1AXSrQZIB*~VN)HpBuzpi9UXa(@ zp6Qz~Cy+{dw*bF~RG2EF5+7eQLtTjAwO%c+ZN2jW00_cQ7Us2Tli6bQ+y<5V?+-51 zN>9}Gqt1(uC{2^&;fjJw9=e=$w_jeZ0vQ_ zf{YaPZ7$Y_iQI@=*y>72ntUpZyL}pHt^Ex%)LvVwj&TK8tIQiZ-~_j5sDlIEJ*uAp zETUp1H5>-jx=xcXnRWWlVBLyuf}v!!up)*o0{)c5GFh(kFqI7MGt(}=82)Jt>Bz@k z_~)Kiz&};pah(#68v{U_BxB7mF2E7b#gGFWV0~Wm=OOzqz2=oUf|>d`n1xeQiZOW5 zpP#6^rCxTG3wu~z5Zg&QT7R}0^^Q{&VH#>?x<$-p(0XvQIV_%LxZ(Q0MUWXXy%_95 z5sn}mbd1P`&MK+i<@S~1@Z!pu6ZNRVnmH4@k&K(LJiFw~CweGmiI9C8b)R>GDb1P7 z;6|wJ;N|;yNEkJB6qODjJ59pWY1rmQHfZtobPFCS1lN|zgIa@m+O8J=9YZWi!}F!+ z*n!c58dtNt;oTSNIPoYBEoa}+n3kZkoZE2~3%8!_{ntGU$7{4h%!m2_nkVOLLoXps z4KEa2#&55j1``$Ods+T(35GbgHMeT=yfuI%f%&rE^t8Ve41fgvR67uLX|v7C>;*IK z?|DJ&6#+Sa;+T0$iwjt2Q$WMIzESc47DOro6I19Lmc99a?XWBz{0@t!@i&=K9xHp~ zYSH<546Nq-c0#D=dt3fH(!S4Ga={D3DVJ+dKI+oa{@4c!_;^+2u^>#n5RsmRcVQ;I z0Qq@@QuhPd69S(y6%6HT<2g28Z!yI8&7H%cF>qqgusR;7j~sf6e1?HhmSpPl!SUK| z*Od}aAX!=y7vQ~aY7UG?GYQA-tL>u@3Y1+df}kYGJLzFKeLKbNrQ)P3+$|3Q*q1vC zjo}+gW9qd#j%*ZIT{P(7b^2 zz)GCl5%Bg=0VYvD2yzye4Y6C2#U)ezizqCe2Jl@)G9Zzc^O`Rsw7UZWTP7Le@u^-I zpQK)B-oV8Yi71Bl@Y)n4I4=O60Hlv!VcLB@LWC8wX4`Ta-S!`8iE5UFFnVbHDxqUM z2;a5bSKDYC0L{e;Um;-l^(N%^2nzBm)r$#WWnQNeQrc5-%-NsPA^?_w!sQFrgejpT zpYC|lXR+NDtcMg|1i(fM)h&f?t=C~qajnB&y)Iw=;v&1gQ(g^IZ6^?76(f(--I2{Y zOx~x*DqJiocE{hE{H0T|B^f3y_C`)GS?$NJ^`|p)f1;55V)8mKQ(3Si7pp!E&!w?c+0sv<=8|BFl z{-3(@Q=+$ZcrKhg#5KyLYZ+rlL@~@JXi_!`1Zq;yBI04zM%(&ImpSx3GvP_D?ob(( zM&EhQ*JJ^KzdQN;@)iRERLr^eEsIPre-KIpE+b%Znbvu*Y zh?`QpM0buvh;X?OhrY1` z#|qu*iBOSWla}7GgTTCvRm~P-9BiM>5AcG~)#ELU?O6qMzDbL~{p5j7=yx zN>AzHD~L8~3!=05i9zzL44~!l`~`qdKEv$pA$f7(z*;te2lnDV)9 zod1yJpDCWWk}DLl0mOi}drN(AyHD^DGW8r0h(HDWIbU9nymb}=N2}T6CjK%cDGh2Q z+&6|-u=F_Q({>}yr8sPUXZ|%ZBxB9Q!p5oJs-=d2W0+*}{$!}+sV~0=-q-op=L*0w z@8U}!RRCJhVh)^e5h!or-pbx;YQpeO$o=oW?t9rhF>BDnHX5}~Ncu-Va)^jsXsJHU z^8?+UI*nGk+EKlD?iH6FLkk5}$KBp7iaME^J2VZy(m3JzlWs(mdYw=Gq3{C*elA?5 zFk*u{_%gk$8*9?Bx@p$sP(kzH=2mXbd)s}x7MmOpD}nQ6q37;18@%WcV#YS~yp%yc z{H6P~xU$zDv-iorP^B7Ovb%jT^P0MKe71C3mK~#lN=ZtC4JC3$%*){O*F(3LoQVM2 zsRU2gV4iPaz%NHSyE0%tkfDiXJ_#!pAcD148?%9<&Z-sAnwOUU>=K7)yta-C$UbZx zY4M57;dKQWQ(cIB#0=iw!(A%T2z<$c8rXQEeJwPt?C}94o^W96z;wOWqr94y+2p=X zQkNmnUFhEGo?JHag`7zc^LtB?%Sr(QL=LW03xBkW{hHbNlm6X>;i!>VpEjL-Rz&LB zMv_6F({wfvD*b{Ixt1c8RgTC2gvRb;^6d~}OY^-EXM;wfZOaMYwn&X5ze~5mWa#fZ z9X`z0(C^fCRvE)V_Tp04Uilg6ogG|Ux<1Vos}M9_~ovu`#&M+=9~(+!3p zl97YboLqRFu2AqW>B&U@sOe@l4(tKL8wz-;-ZL9j?(DT~<7&l!wEi8w<)P-7@-JD6 z+BeNYWV8GD7_?UmcXb!i?U{A!-nY3bd-q2_fQ{MQ?Lknu?k+g_c~5(sn%UUBcb+yK z7Z6S)%`3vZS3htnTJ{q?frN^fV-)` z$$rn@(LuO9>{>1^g98-Xcbw$<7K z5)LUJ;VTR`BIT}7q!rgu-EA2++S9vdaYCq3BDyKeJO!{&Nkl=l_0klb4?26d_9B*0 z3k*F4T_}{~LK?Cl`v5l;RNHnJs<(hG6WHA{ZFj_Gy4#x&C?7^z7Pgefz6=G!gI?30 zWw&W#g0vlD%I-)ng)NDk*Z04?0Ppe;sxY%JAl)y^m=}FsqaaO08qCp=z^+tRi70Je=<;D9y0i0tGWzrHV+_$ zLl2TMbq19|(eKjOow_^)>(IpbY)EZ-NTB(pUN1!GJCjY`gee|MPE_`i$Xrs`ZuWxh zOUQl%!pb*~jWCn$MxmTBRd&r#3gh5&uMH$j^;M=kuyk_KrE}=Dg6RL$=V7W6;j^%Y z@8n5PcVQ>O(W8XgGTw?8ydNOPf1=VPnXMG-cc*JP3Kom(e6L>1gbHymq)E%MdKxtgCTPb2tYSI=qhlv>!ZW}6-=ZQD_W$pEryGQ$U~LO)V>9xRDgMMA5_I6O=wUAN7;#V`}F5iB6>Gmv3EFihHg;nt z8|}unoou+VwXtp6w(VqN+qP}nKKV}7sav=1sk&8D{b#1;&rEl}{k#vRjd*q&;tY_- zT|Qj7qbM7}8?u`wv6Qw|!vY?aGTNJKR~(w{Y@r6dV>CTq9<(Ni1k+aA(dvwE(}-yn zi-gL_HXkL=wSFLP=cL+tlOH3H^PaQhy>AEaLeX+s>oKx#3{yui84d^tUl+1I%6D4v zL=d#-y+`{(5d=z7A-EJ<*1E+fM3|8HA^hQ36_lo)w5tg(&lMAs1t>0F*lbf2K9EGW zg28~s`-sU%6c;Xawof1XFORQoGzMdcr_W{9^e-tYg_c*vvy*)5%~}+ic(-zq z%OGI=W1e^*9BZPbX=BI0k=Z8aK?iF=LK|-SHBuBpt^KQq*N4^@lPJNDNaZ`7Xfre? z>0jP(TzoiH#@RkIzb0*Aa<%>~c2#3Euh5H<{l2d%Io?)16b`#^9GCeI#jC?PuT1|_ zt&ThNBijU>dZXT9%3xvu?M|g;t0(KedQl0p$rASFEKMzaiBpZpm!Y@}4?$7(WrNqe z8@ZjI=)U_KL*u@@=?(dZFQ%$0<-}S|xw1I>8yAtB9YvsE#esH>&CUsIds_GU!HBQim5!{k?Zb@sKi2r@I0`?ta2D)`XrR3gxSWJd$o zfHAd9*vk`L{ehc4AlS~JUG`+k4JQnX0dE_JlpLcMzoT84K$ zF}}^87=rcwJx{&?CwN@+P3h+r@GsX-T$`p^&QUnMe%i+z5-=$rWac#Nf2ELv8l@I6NJ4s`J%l8r zpta#S&?iiFpZn$F?{~UfCitlC){k-DBq(OLI}OcwKPE^v$Y}Djax|d){dZkn+Ka*G z6dXwrHBXc?{PN`(yYH3R`||YLMbE(q|n6p;k&`&_zgjW{^<9@@S!# zsGaD7EZ^4YWJe1eW(bVof>}`nKD&e_KPKSKqmp~x|1M9js(RU3FCf9yr0-vAL3?}MuTOT%bq_lwGw7uf`0pQ}Kk>pdT08x2R{m(hpJ&23CLY20 z)`hYaHTaG|@ckT#yNg&TXHMtVfIbSCq`7#+%~ppyj~J~<0KgrBAO>3859#+xw5Hb$ zQSdskjw^px;D1QusR^SAeN(fL=|Cqg;r^ZS{Ht9O*{qMo3)8crwfNtgtm3>@po=r9 zxToKDOTn7X^PmlU-FHv}7JIQ#dIq?44I>ub7P(@h9dl=WeaT-q0ITzGq4=37eg~#9 z(x|F#$|EPda0l*kIRUP| zIkwUiyV*{Ha#Ir){D8RXvm0vBVf7QY-PFO{(9;R!0c@5Z^)t!BSZ*1Q5xIuSyR})c zgpRi@C64tCr3{Irku?Y%iOF@@;o$>rWLdM=B-1{<&kZo)ZqI+8c@ABgt zPoLgqUMYz3S0$@LS`;23efyh}?1_hC{`U$$ZjE(&f76HlU%i08IdSJ*Ls&u^H)nrt zpLxb__9@wS^hR3n8*AWXsu&{;Lxb&pYiqXSB)$s|-JJ254|Vj4X9yseyEw8_O@C`W z{OfvV0Tji6bHJS?sWQV7sTgV_>1W{%|qT?eI4asP?g138IRII5On9t9b z`hbtTk;l$P(E|-&L=q#>#mwWHfBB;4aY7BiME zM9S=SqB24?CDHn=yPtN<@)>f(P>xKUQ<@kxqcHXgmn?!|V$9~-!*b@Tu!SV5cmDak zI&tL84N4Xm@?4OFfSn>7l=uDY`~j3eY!;Jt9@U7W(UO~5l#7q1&7uWZBXgI6>JxA4 zI3Ecawp2l#BJt>2!x{r;>Np4x-<#ba+gaNv@c)r2bc)oZm0^coLJV+u#>UHsM6TJr zzAa7s#tac(*eHYrvJG1d9>6a9q!rQ9R>TCCshRJ`=)p~0h;a;*H`b`pSkoz8cLl#S){BfCVXuJz zqdVA?N`)D_A;E3l88rJc6cAW`*w;4Ej9V|zcitVgEGs-hmfhldox&K@uwT{ zCu2(6^h!Qjvt@ZK8gDZbd=oF+s`$8jiS;Yz;+g~-gK2m2}f_N#b92SxQO>ltdWAHmEY3ZeCdT;m|*8Nyx$vee_H0XTn>Un9+nMbIw)0-4@V8_{XrB~s?HGo~?;Zn4z(<}0dTe1p9{y`L;^@*a(f0N?=SMIrNaUZvx#dK6 z*REpbquJAxj>#=i(Y-7RvZSaa$5kHGdOY_}>C*>uaj){6h%MyK;lTs2hkTO-Pf+-Q z7D-^RTo}%;-g&WoxrFl_wr0{G{ef*oR3Q%ujZZLAjY3JU*-?(hf)6E7ODydzSrIYnLF|-f}k|KIxIQ+8PbB3?$G6Wb>;*Aq*I(hfiC#!_UDC$R-Ina z4X^)ACBqjUSz`>AK=cSv%kiq{usN#&L27n!7aP8p?~@Y$qo)_$e4~C)5B7$^+0FZ; zn|PJwn2RHMA`z28B{3Aj(DNzKQjvn+VZDR8baSG${q5VLPu%Q6B7~gk`U;z}Uzc(X z3x=e@4XA!6-E&DSoxR7AxcDTgw28$P5_u7b4qy(Iq?VpR$)I5p9U91h4^r1&!o^ET z%>1Yvr|FJ_SzSG_S~16gBwDe?1Pk1lp;&t4zZ+F3r_?XQY%jlS4Z!>k-T|Q`65#W6 zq%pL~H!5ew6B*EzGv&YQK)GB2wnpc7wLP0F^ZBt)Y4Avj3;^i z>B(y(cwf9}<^9Z4rj4Tw^O(R$l{deUwi2|BJv}b>ImA3JO>Y`_l8eQpQ8H=5OdNfV zxDV&DKfq0i7d+K;wrQFwJcGizsl_D*T^!|cU`7}(00UN@cCv5h!*YD+_w;EtFc`p3 z)Hp@cnN~=NTqBr(da6gwh1FA}-dfmyWYsc<*X-B54PjB7^^*nLB7wRA?H!!{gcLr% z#Jxmu8BI{F>A|_$*{U2vW2~TwUe2CzXv2)p!7y!T-q(dcZ#9RNrye9SDJg&CbE!%B zPcQKm9Hm6|qQ53Ke04@RKh5hGd9e<^@3n|k?`%&CI@~%DG*=|OaM6hM|S+E+B*ic#d z^LP#yOb_Cnh&6!ol^X(zA#6LG*}@W66T`2pP-(-(cK8*KH$hI|%P6Rq9^;(V(yYoj1OB?s zg+Lrj!rU;sM-9hMdRfZY#pPqKQCGyrD7>@H^X9I9)%U~@yCR_*64ytJ@VnSwIdLp@ zcJ|>1)|kUTrZe~sOMBKwQukk3OSmZmpor}ym6efJ)uJX@ndI+HoDdW28GJ(X1Bjr- zl1(4g6L;I50{u1At& z)3?n*JC2fufLbuUTem#KCN;efZpiYZ+~9hWtjOm-Q+1?xOuTaJc^K*wn{7X7;Yj0! z_o0KaGNjP$)x4&JRWSKRn@Bm}&`TGew@+^y8rnLnoUM$twTwp6%@baGa$ey6JQIfpIWW#57X7Ws z@h930EZ4WRE40l|Rttf}RvuK=pno*)sLteZ+P$OSCpL+=YG;cv=x#^{jhz;MawX7= zwPM(do4W|_&l!Z$w!W-YJ>mI8GYnn)3&XGKs4RT?2q-eh=r^mooOpWVY^9iJTD{Sp zE!r&%LoS!HB|`cKgQTHGP(ROfrCP>|qx87q_3l{NnU!&Rqg>Wo+xjhKd6R}sRg-e~ zO=)2cyH}99mVuLaIRQpL$tm$3hQmx|P;D5)V|YyL^UF?n(+{b@9G-U3YTyI`$6fIC zoWVinDTK`>T>fVJIBueD1hX>9V5qtT^~()RdTu}zt!J~WVIcM+`roGgVu2)?em(+-+mcgYrysH0c|S1=UDLbY2(PW*m*GF%)^*Vf=JfCL&qyBjs*op6%_>qjpWCj zy+}=&Kd{1;Jw6?OnR`PR4c~90`nAF=OkTrDk7I0@{EmB5Ezi_k-jtMT>r=HKbQWjs zgv@9MkM)^yH<7~&Sv0ObEA{gdJ-7Mk8-JaTq|8J?^rN|_?Fg*9k=YXHk#${e~-dg?=3ah!;m zPBZns+6WY>&!Y5f<{hn5dS;i_S+icDFm&5SZ+C$b@&Je!X`k=QF26kB;|*pyPZa+J zBqJ*-QZ%>ek9PaR@|>yDbLt+aqS-|>QZ2+MFaZ!Io^YA@08mBv@$?b(-Ohs*EC;^M zmi~F(@&sruu^gmkUP7p!FG->`m^m^ zhFvsBA;km#vWuewGF8PVxL#-=LNfRZv-tf zTZ-gX_9}(Z&t#89O*66yIr=X#rpBPE<|H}$dRKzcY|_H;?QNH%czQ@Q6uiF zd}f?Ba*29+-Z*?Wct;1i7pr2^_dQWM`~SRnA>=O9LDY)yM0{7L`x``)!diNSe%kn= z>bY<{ctMI|6mo0u&2?O})xD>k^2-F@`=Ox)x3Sr&^-r@;SYp<%qow)chm~Zr(aeTI z96&(3wFxuN>^DcgC5C3XLCs^fOO-m#=n`L=>&|nG+;>xQ36h`ELHuc+J1)Rkavf2% zprcJqWc6%eN0v0C_NVv`Cvddp4w6*G)&HwpblvvQEde%cg8L3@Nmx8k#q;@rFCIPJ z?MO|RQ{O0dFLkQWpO=^8}gLok=|Ysa0GIvx4VcIQ8Z6RxpFUjaTx+8O}@^9t&MG<^Js{ zgaN4(7K_yB_>a)G+{MKXvl$z-dI5kC>AK0zb?8{QEEkqkItAA52f^8n>v(;$x*y0E z>cH}a+6c!v$}-P=uD9H89J2#{&)bE27o!X4^l*j<`T0Z#zwBB)nd0GA2Rn~HAN}v} zrOjt5`R~Z83?Y)na68>g;U}r7D=RChr8Bv2mnAOqB(V5PpkF*cQSK{-c;~XJpo5A~ z%YwZeDHxf%iW(D_OK8=4^7@8f?OBOiYL2MoKBT!ileA03rP>{F%X5p7t^z~)zKu8y&Iom?$N)p%e}7z{SO0ox(hU%2JYX#+d0<} z6}GaJp;f?&u3tJl!Sk|7s(u!ex_&1w<29sXKfN!)2H|UzAKCorn9V&PqevMVg+iPc zL%?ALJg%2iW$0)B>eY~W4o|Z1s4tfR!QuYynOu^-WVGRa-olzC);oak^um(YaP`)L zwWtmSEp6aAwH#1^;Uqm+u~xJ zTg$~$g4^OsC&JBDge&=r0J0i^V1LUU`k_mD`A2cLX)e4-Qy`LyPL1*LS!xu50I;=~ z-!642QBTU89vbvNTma+J{Xg-{e{I$nB#kVNRKcR8UQvX$8=dd>V;VK@u?n91*D~Lk zYXr^{H9X_gN6AmDI?yfVMY&?*?iU<7?%Pur^cCzP=fU}7?E*?OGh51A;4_Pt%Bt{_ z;P@6|$+t{WTV7lvK$ToP<-iE8ZXolZO0xN|@&)pdgQU_)R{MIS7|w6+%gqYMewOb9 zCMmna5I}QFy|o>lnEe3(RE~+zKe-rfuI#vN5<5(ft7!p39df*t?^F_mqB!SR4GsG0 z*C(G#V*=V1UTWfa=eByBo0~6JuPob8ic?>yFx@@uz0Hdn?UfjHe|_h1=cA$t66OaU zM2_0$ba#yd9DT}rR#@MWgJ9a--^?s0%s#Wojp%*U?SZ?5{9E*-g zN;{@^($j#^!*=XuLv>L|xLn_*ull&b5RCQ-@rO?yfJg{{07Y~?V++M6_FWmVEHM14 zq7eHYrN&Anus9tBipK;Y;qmniT&YxKF8^jGMf0gvEg1y1g$HHAvTGma_9Lw4M}KX3 zg5UsGrg`LM44daMbc*jVPo&qa3++M>?)XuIyVoYOk^QgPJhfDo3KDN~0nA7Zw31}R zY{Gf(MrQ#&iI!37pbhlW+(uV5_Z+H8RURjOk3SjB-R6cvqAHYe-Dy)Y|M6-P#*qC(w%Gk@+4$AG+OeOMW=~akbDE-#)NV-OsDc z-*(V8Zg-r86wsj$qi(B=5fhOj4YT5$(igw zvNofsN;!NCiAXvwDN{iHNBkoU0sOq+wf$x2n;fVdR~m-^2_dvEd33OlPxRovKENQT zlxDoZ?J<^e>d`cK!Oj_%y5ul#(TfwLf7RC^W;r2zc`hT|!&GXzx5x48{NLudVFRS^ zfjetZPSqoUv+M*I(4_(ap+RUmzKXWZis7`l{vfnUHENWIfsPt1Ra>VeC1A{I_Zw8J zVe>5F;-c_nGY>Bjl7|Le-tJTD)+fnn+Gdvg%nocID19=PP2^j`#!gBns$sw;4_%8q ztp8L}$xcvPxjyCC5A|3%owt@AEA7rNd#ob1dDgKGqrK23|A!aiPuJU)#41M+ZtvK|!Gs&-t z`s)wFPb;Yh3I5%3bQ{M8@X)GqK0jtCwy&JQzb+6VAzxBdXnMd>6D_TrN_4BE{k%T# z*k-|P@s0Lj@%IzpJ2Pq~GI;ZhIrNV5Jd}6jVOuG-9*jIB)(wH=B^T-vo7s=g$)H6w z@jZsXs4?d<38iD{z^*z@h@d!g)1{ie?RJb5kUqH(KCMcC3u&R$aFkK*A|g@cf+=r@ zcZQB6?3Gz_ZB1^(1(yoQii_*K4S6TKP}1Y(MP8g4^=+ve#ixv=F=PE}2mJjjHkF(a zlSmf(aAX>OFlzl%`e$fqUQ|OhqZpE-*P=IJij7mV5u&T7#ziW)cV5 Zyl{DV$+ zXWEx3zoLMOv|s6O-Z2!4R=lAx&Dj#Yhhz9DHm|BCw~ECFbj90E@onnM~y!E=%fzz)OU@tCDJ{?93gzDg=b?!YbwRC0A?aChVHc zolo2uoNvf97g7C~vx&z^${EfNMXsQVL7{H?9gZUFO#PC|?YS#PQ`-Zl5m#f0==yOR zoIdcP)VVAmbGUrXxN)7shv&8G?$WH4^p0EJ?nRHgSZY3%Nxj1(0@+{w>cHjYGL;V* zR9ITB!KIY&M0!T}B#w@{*XLY5L1!(st%_6%6B?Tm76$|sUgX}e^Pwe1&0BJ<+U#VI z+l@d8dWr{5d37w*&3RY8sGy0SbcoW@bcz2^nM{lJgu~(2%Ie=y_-&7%T`SB^b^qed z13u@$@RQp%-$QXzrcd=DiP>Pwc+_!ZmOXdpG{0DJ%(Ph(w&`GWaA^mF*{l`rrUdOj z6d(X;9;oB*KB`EeA`}-%NcFGq&WQb0d;hG50oK!Y)htA+6pyyw{1A$nb4X@jB<$XPp`}@b*$J!2|GqR#vG# zH0B+)NwcA)Jdp)~x3wQP>zl3}_%CqA-YHpPsk_eXlqQQH7cm%&3wib|u+|rk1P_BP zl^7v$GpxKtDG7BIDK#jEnsXzWkvrOh`Iqw;kE}uTCff|s8Zdnr=t21s$EI%JFJb){ zX%S~xbMH&_CN`c>`-ac=RJwP=Wg{X!GU(N2vLR4zECD1ik?}Mwzx(1a`~Qn-}{ty(Z=W zVKKrCDI+6eS>Hxw_KwdaWCPKy1zCyl?s+6TjLpdG+I#pYXomFPe{2b=xB%dIfgm&c z*0c4`A?>h#wLPUVpBA99TWu`2b4bOv%B;WLzcg?m#3|q0WmW293!70uE16>IvD}4~hgX%6aczIT^G@KUOK}3~>=4Juy4YT; zV$W8)el$2epKukv=k@CtJeRKjGSuoaX*KV!weVo&Jx9y}pOl`|wSx8kkn(rF52=9HdSx27Ex*Oe^ z(@4S|q7_-k4L)!C{kuVUCRa=B7>yNAZKUCzaOQ z%|6$NVtqaKKgn&sB4GAn(h$#RHkn3yEw&lAX={B5h3e%7a1LeO2947Z@c=^Va|Lq| z;s>V?Jiyzgti$synL^@0s{8;%+Oe>_~s z(~Z2J?MqW%#6HL5>pQ-kIgZOrZxre}G2nPkD7^I^5dTb&-pz&vN;`}{ryW|(p+O)N zqqtb1jO^{zwiMl+Z)DFf<$k2OQ5B0xJ1+%TiQXEpseLB!_)N?yx@iBfIG6D-gaxT$ z+Ta1<}?4275=sTY3pv~ zFs{XY)l`9TX$}G4*OunCm6x-%2m68r3UvCkn>Yj6j|svd6p+25gt%Qz@ET6A$}3mY zuj9-ybKyH#vkiGS?e=peUyqrN<_eh)jkGU}LPu%dqP{%%DD5Y#9nCWyXGLl~^*{nb zo@<7KzZ6*~``f?$%7yVrG|v!xb-z+H7)|oIZpQSfB&1vwRIYNG)XRT6Zr%-;mijWI z>;B7t1y2??jC^DnX`Q@A3-OJ?E|c3TksbI++@5D~i)&E{zG4|puTs0oPrWkz$qm+9 zHhy!1uXYPI!i*~y_*17E0uzYjStgZi%zMO%Ed3Ss>T+WWo)9D4p}FM5FN7p>#J)M5 zD*cvoikvkL7QbYkvF*`$hNEjcKd?;VsA*3`VC*6K;D2e<;pEI8u{^l}3fxrK%(dvc zV9<1wnV3+l*xj?-zXY}FXLVR4sX^=K2dYno^)A|pUNFwna9TAyS#xOr2N7tgbFgcZ zUk04-4U8<8leFHS+!&zvl>)So^~~@=0dM8x9gh{^?eO)6(}LJUN{);!LueE1cGrj0 zW<%6i6iFOrJC+2A0xxj`bQh?3Gn6PimI&g&P$kH@DF*awy+ZuqWR#NvUUpg{cfThr{_$?vY4H+0>oUzc zS39&cZ6to$SaITCUDKWw@6!E85Aca8I1W_g>0Y)XGpYGZz0%roZgB;@{g|>{T#j$a z(i$}gw7g>`-B@zxN-1KQc3J$>jyhPHH7eUfBQXlbC9I_dIUeQu*QYQW zdf3yffj=+2h9P0Fxa3gBTRyYFLCR{&sP6PrJDS~^fjdZ=C6(T7TZb#;pH1A`Xpj?< zrE@e1T^P)w#h%VpBbF)&P-I$XLbx4syWhs5h!aU72u@Qg%q4iGP=If0N<~O9D+>^n zQQg|bS@PP-xz=VVYc=zCRlMC0v8O=rEo>gOND809tW^<(1wO|iI1Oar+Rr$@&rcu* zSMZ^dS*&v;;$YmReC1Dx*H`&mKDQDZ6masK67rvA5#c7jTVnw zrQb&YXw@jz#>eh_DOeBNx=}m=^2hXh-bHYSaLfAe&INbixgnPXZJp<;Z+D8b(AYZ$ z;T$p9O!l+B*wnw=>Od#&zR$*nNzbP5)8AEvFZYcCgR!1`S`94~BhY=U|UfmxlY>wEV;;$&Kc(46jt98EYU5&xws&^10;36?!|;_RMJ zNXZx{K6R0HpLw5?t7Q^$iv96XkY?pxbqKv`P8vMod^okF0#IBMxXvh! z%hs#n2T3NR@j&;d_{U2*|J{e=-^+1lGP8#XDx%tYo5QXgr`F{X%(&1I)~i957-D= z9#Op?N%gc%q?Eeh><1J)99;Gn$YE5GSW-N;B%%9#gq%Fo5^mX4ilzCLuH;KWez&rL;-o3Uh~z_Hp7IwDtJOj>VM7s0b+LP37v z69zt}Oc-sbyhqHsS-Y%P;NL^z4=WxTHh+Qwd~?jJt>eg1V2@3Q7vD`|oy@CA$loh{ zl1}+C7UCTxq$OpKeg(}t^3Z~~MmBD8HgNhSXw<3L^8NiMbR?KadzyjfMSky+Z#Q$! zwks|LfeRV|jteRJWez}m?{8O{{~GGUW&PWGGSp@TjnyNGpe^QIK1Xgm8-5k*Um*s5 zogfMZ7%;ciu=)5z{)ej0d&+Elxh$=2HKUfzWia>rsP-_N`I`AAu7Oh~7kCqMJps?n zZ2^s;>wCU|yT?cM6kbW|5z*d{^od{Hbm%LZhGaccqh+9z`PZQ+9zRv$ zDK&QI;IVtiW8=xF8z0(709bfU{?|(C4vYZ5gZdHVK9oDq-7yd(h_ZSIWn+_RDTs%i z48D%OvIqwec?pBOCJs9M4lEk&rn7L_q)lhSTduoEX-N3+?K~7S%g+keRIH5s$~;Cb zL+8R%s^_&zoaAHIk;a?8vC!Taztnzykw`e257Xt-06r0~k63YT&zUf6Z+UCp1Kl)T zU!ES)hrP3?)o{%$$5YlwE=QFo5HoXQV;xwggXN0ob`Po-N~o`vVbVTk_2{(Nnc2xX zZzD>9Z^|UmwP5!twY)_ZqgN~RSAXP^t7aY<%J*-T!G#rPu#Nx5!ME<+R+DrR!a4j@ zRFNWua_=}fc_N064~2hj`x%0wS&iyL0aBZA>OADWzJUvv>Pgg`yqY(UaG^4x?VyB= z{2>&@Rj)|e{pT+l~*PhIUkVxI|jPcIN{F9U=nywa1?!Gp#M_wUH-4^mX9m$wYu7FLZLaUwu)|GVfsu zyU&%iJS{dB7j@-`u+k3eL-5#^<iRJsDaSA2aaG0h ziV7ub_4rak{iHb=R-z-^kRp%sAt3-M%1uGMNgLj0YAvUNFA@c^uX`1dADLvbIg{0d zbwCe!jMgg;GI_FEAeV~f-OKjwA+5H@R=uQ8MsI)+rZr7TMvD9Fd}ErBI!9K#QxVt( zh=X|%!_zk&8>IBqBqm&8Mf8=T@IkpFh4?9}^_<%aL9{HDw6oVSNUi;(A0IFLY6?#j zm?pIjzE`HQDJoIB3WXJ)qWAzLX8;FU^10WqVI*a6&*R!9l>RCj^l}wYpslogj)IWF z_%1rW9LD*+py0lOGHd52YA&65A`>#y3Cs)!gccQwyu66kPKa zzo<2nKI2H3pM2b~QJvi9?1VOd}@;};+Ss$ZivYD zMAG#Z*K+~$DsH)1U0*ro&}Hr%<9U%}B@XePmGr?%sWnZ8i)EzaZkw`Yg1fclW=Lu6i*onPt#8W3V8ySStg-)guEJ>4D!5|1 zL4fj-wtuQ+5^`8(H5$~+Okaz}6NHW1+cXc(g5x#Q0;f}6PHa3_==oZ?5 zIShgZNaKW8di{4%!dNjpAqY^AsJAKig>T^dJQBq-Gj>%;5-;;tW2gOJK?aCw9WM%6 z${g`6cjS+k4rqLEYrd@?I0Tv4Ei?V2uIEQd_%QPV>E(qKYDpUJVQ-=cq|BXh?HMVK z^lTBPO~-X|A4D>d9IMrKuafbbSpooK1>`0z4335XA2OwoERN;YDjVAswj|V`&c62ywLe9 z&48AkR(E*nJNURq$Ob?6KCxtbG!CNMb&S7_P3}uS7_#T7ExnmtrgoFAc3NXGoh5-2 z5<&OL5!5JN$O%wKIR&h5z{u`$vb*-+N_lx)zyW}%+mQ9(W_asrw~Oe5F-% zAtrtQ@}C7l&ub8_v5z?{oD+|>#G}dP?O%i7pHDoSY4rLa^zx9zvA`)vTkYGthuiAC zKGAQSd0*9V4Xt);M~CM(j8bmX7b@aB_Jh#y1+Ks~t7$ED{IIlB8`!vX_k%-%>$YU_ zN^E{b!zSm*@^i<$H)_+zk4Fn^$#C^wp#vgrV{0dd>}KUY6d>6!%GW=7eqEc~J|rYO zRtD?PS63BIIdAH1*CWWO>e=~?O(8gX$nV&R%6z#zCfi=A1SB{pBypfhU)2B8AUp|k z@b_7NuKi8&?KUA@OBXkJ9HOmf%7ByGX>67h`gCLz&B)?51p->UIY_VN^e#eMt2Ax6bwfvb z$BM88W}0yJe-QGa)$PRu)>NV$S;SBQ-bH0YHk4*;fVU}i9ewPx?NMwK97!VwZ5j5@ zIS%{Si&+ak z0u@a=P^4>;eE_CYhG$Jh7f{r@V*oA zOnQB>V8!tQg!vRVUTh&ssJ>qwFCv>xf`u@A{XKa8#nVFl@S-4o%l8e#1BahaOjO>r~nL>F*gDdBp)WinITP+ZMMJ38qVWH&**H*9f z01*Dc6`a9rB6jD|3(BmOw3Yy7nMVsZSiTA2Ftz{T0$>r9n5I{;Ub3XKq%G_U`>vd{ zgQUF{y;=_UCCe)P|lA;atk!~OWwq+2zmq_gvGruZ<`Paeg>7uGk zXB00`ZwAxOurKzJ;6ob5c~h}X`uzffb0JGr z11u#2;?C+o&A9KoIMZyalOSFvvP-oMVNd&3T1i%#d)O>hP8Rw5-)zu9Z9nTRZ%-yx z$fWSt3l5(7$@l*5*&Nc+AY4&yc~~b;i&TIC&Iq$%tV*n`rH}5XXe*mp7JPV=fi#|b=eFdjknEdxLn?ahPR|aI%iZTH*0@Z?Z?Dn> zLOb6oaxcO|*lk*OS3H67}5A0C)%1g-~23b(sfMD{(5f)ZQc8)u3y3 zP}~rzDqj79=73|+4AeAL#3~%Qap4QccOcaDK6f{?#fwectHOt)D6i5=U6;Zm3BnA4SbAe%bLp(u_L- zM!_nnpu7im8jTn(J4?QpG{$4Ac&3ZQxq7Yk{@!=eJqFc zz!rw49nxg>Vo5v<%7wHSuH&O^f0!rGE=q(no^Xv+N9wJMVGp5*7&}8# zmJi#NYZl}Lw_!Mn-z}#MeN)T_jzM<0e5xY0|D67dtG5cP8*J8w7gC_OQ{3H(ySuwv z@#5|d#ogWA-B~!ry|8e1cbAX%-FqMW{~RR;$(4DoWRlF>!VEC|_cWyrE`J@0w{dbE zF%3)&_O*lrBt_jUd*>lEFPl?)*{SFT^s54Te-MhC;m?O1N8kFDZ*W@1@^f<=L2-9w zFt-nRr6!`8TH%S|dp%v>wNvI!qL)`AgM$Uj%UgPje?!C1r;de=C&sO1@>^AYZqg&P zWCB!i&kAh4W7Pgi?RN?7hId7D!o9gqI}Xlz$YD+MpFX?=1E7Rmx9lFzBdujthS5q! zAgwg*S}IdRe@U8n{(jGBY&MO6%LW2teuM9J%UM3&VD?;{JS=E{4WJcT%gIAXt0*d; z7T1c~%67JyYdEvOK}b^`<}_5R)6?>h@z_ufn>y|kIwd_hb;3sX z?E4o}(j$v_cJ)4~1@N`kt_0&geJ0?`*^=X#7&&2{+QF7Vb1U~-hgRShjD3l#dV3|K zO=tID*qWX2s)()VzSaLh*YWDgat)&tV;D7F&B>k1P+iZ0XC;OUJD{=z8XTY~e%4M{ za=_M_Jox>5ewl6=R{2uP%qrB4US-ErHNSA_npW&-=#kWD)lyF7vaoQT7&9iXSTf;k zfx*IhYwsl*r)Wh^5$$i~(;IuGQh6(E4g0;Dg$kzyv>Aexxh{BDh{NwJo@Kwjj`3o@ zg90)zRPC|a_m|_Lps|RviHWF*2}T1rE5B%Y0!og-{>P&S2MkJjA2h2c?t;etk|sqS zg>yJBd4rh;r!_D7Q;jZNKUi<0o^*=c@~Q~h&X^Yapf|>Fe1E1MUT<*0Gmr$z$+$Lz z^p|eRKyFh0l}Z9k zVjlZbfzZPv=(*CecKvzI#J$ByPNKa!B}1YygJX-L$zq8&)XR>3ayIkpJ(NJwI`5`+Krq9e>DRXYTozVRH)7R zF?1-I<%W~fNtZ)@9BSr#bN+>=n-rYMw>>03)h9z>3^xucGQV*1?oRkKuA}G#^6a)M zIoKiyOkhm@OgRoJ=$RXKP@PQ1zhA7^z{nL3$?i;Drgnd9C9W?%yF;^`GQ!#(VP(!; zzDo>57gi@^na?|#FngU*m~e#?=-F%O46eJLv9wuX?ELrk;|CAv;3A!?9Xy0-gP!l!7H= z(uZ?79~ZF`iLt|##BXKMf6{;h&+Aczcwwg&R&9pxHdaYU(!@~@OJi3Ome1?-ZegA} zygTD?3;9ghr2W#u^!0khI-;cP1PPDL^(>8PKuy`FHD{@%VfKD$V_oC#Bn%Qvd(u_$ zkj3j`<}_nOeU4Y^lC62bcbv)&=CwjTEL)D0PwzA7_^g%~?quO&<#J|bRoGyf#9LOl zR%nBHnLA6QW0h!;Y|*jphrZ^&iBqeV^$KM0xO)&D7iaF>h{bg}fh`K65(@hG zFF~rrM+3i^1j_FcYUID@4QMNDc8qG`z0NF{H1;S193lt^8+3DKIzM<8&7ai^t*7a+ zcw<8J<{X)vIvM{mg>EV00j@N6)FBHN+XDXWwAt-|L;187t9@pys@IagJirFnE@fU{ zg}tX&GImF@+l$^y;dc2{i_bHYax6jlSq~ENSY~F=EhlCM0b$kMd8^O?c{GI@+EK-5 zlC&kmP`0}9nJd(+f!R(AqkF)PygFR~AiB7nIyEcim6Fx3uk57ArY{r=^ruE3ZV&+Y zn>j!Lnudn{_RnU`f#STC90{yZdg6zsJMAyfWQ^fQSm10xSTa5wiD-tS0s40=(|4z5fYad`xlno9ouo*Z}kcVK_TF= z))^oj6eApLZ;G-I2g?fslH(IKkdGmm_$veYFu~iWd7(jEj#P&0QB*v;Bf=$QOu;(t zAg_I(y6^txHwt8u0^P43$OKl5Q!0$*kUH9*5)Rk3rsIt@1;p9GmrA!z^Vb zXC#tBKFQdrnrcJE(i(!8=~d8K*k9@{ERPZfyb{yH&k@vhF;D(GCw7Uy$!R53!P(EU<4c{GCe6 zN>`pl0|y8nG5HuWn!SJ?j2MMy(z-Xi5B9*^BV>@zeJ-ImhF#?i&Js^WsO^-{o7cDt~1GZydAfLR&?e%$emFEsJbzw|19Ok79M#j;-NJFW}7*DmEIP=HR=V&XO@pjWZvPowJ!)aVqb23_dc=Z@HgDCCy{j z#^uNAsMorJ;&;!lv780q&u<@eC+Oap5IF;6$IXbG(nXna@l z$ll(ilxNDYe>1&b< z3Xa(Bjo)Rn8pD=ngo1rRI0oJQt=F+rkSVR&{w+^?{eA&P zq7b*VB*B$FK>0E@z{}bB=7v>m#1R0BuTzImTNR<{b@KF`@1-&g!QyePE{%mfE--UQ zJcv=$2Ujx&ISAueDKnRC8`5|LTaM>FBp&l5=;f<4MQ$9t1l~0teIy#4=owXHA^KZl zEZtq%sSMo8KSz@{_4(x8Ul=O8az?F9Zo$YC!pKO{Nh-?6Ww>3M{t*gQWsRPH7;4!-q^IRyN zwTQg7Yz3uVR7N2xD!e8)Wp9CRffHDEjmlP#{u+3^7JqQ`iIVWErI3@@6%h9f^;3fm zXxxTZgGl31@_$UO+MyrAx_bo0L!dhdcc~k2Noi%0$3$NUsvd(3f_jaeQ1-7|;M8bp zB9A^$in&m}v@R3!IxhxSzwFoLA9R1YRKh~=0uOtY(99W_bhv~XU+#~ySB24}-2vpa zgJD1X3-N234r8Yc@FZ6quy-w+)>IgKZS z{SdP#r@my^jFlYi&(xPo6GEeHKYlC3WKZo6D>jP_?SyKZT%>FFbGBQM4ISVpE#dQs zPsoXnPc}d(2KCpnyhP08y|ax7GW{ze^1g=G>G~4;YI`=b zI$Y$Rsy2rcjZR9$x_%@ri&}{@)e-;B-|lPtpSjdu>gTIFslzOn{~z6T(h$Gv?dfU? z$_Ri=!k2RPJxdB2pl0YtpDL;>rmilqW!vJ3?8E>6rby`J#t^WysWj%CB{Zy zgYc@3)3bVH$zhy!)2v<(L0tauCm|LI4Gk3yHOG$*{V>S-#nrfH|Ge%){1!ffF!>s6 zOCggH>}ya1X0B0jL!-j5hAEcHB4EJT^-0ajge&%oojDT+H+|yoS5Vl>yu);;&!&R3 z<#{`Ig_@B8_J2NP*jHR~v#|3kb{*EQ6Ip|D{bqefho%luc)#E5zI7oQOrmbe9Bgu7 zzm?z9Z>dq(HJuTkICUM|>@ry;;fy)jLYp-LAafbD79YXCz@#}IuCW|^)mHOV1&wQc#x+ahqA)jnaWP)^HPLja zf<=NJC+&@?H|}>1z95l<(FH%R=clH~!(s_71aQEJtF|MDS$ul=H3^6`Wi_Xhe#Nzw zEtsZeShq(7b1W+2pU&fL+sxCo>L41{?qm+!%mHAfnhWP@eo-HpI8d$?z})A5oG z1&DI^dVX~&hz<6m^l9N7m!8R>h;HZurZ57@AKKSrC*XdYi3<-vcGMp&0J=;+r1o_+-FX`8c|P>wP;T zS6d~7J6(PPW>=&=Y^>*9vj~o=?>1X2z9nw6ehLgami$a*M7@?5Q8{AJEtbS2n?9%8 z!@=`n0(dTyyzkvW9(w^(u*It#mFWDL|cp!idthP%N<=mAf5dz@Pq)f`ISNm}`H(dKM(mN9Ja5$xXvdu-Ft50^(%IozmV%mA~h0n=x z^>BK9!9lW{PP9A|(-|B)bMNsV2MC(fRpJ z_`lS3u5Sc?85nmg=#G)TzskAkjhPr>5v*qejb``yL=F|S4NE1H6v-&vrHoROmTidc z>S17BL)R2_-pLt$%E_K#p>JOCbIw=+&7W|>A%uJBq}sQd50#RUnT~2-ccI@gdJ|1e z%OUYdOh6#^4<6N4dchA(AlbJsjai}OCBT^MEm`ksVws z+?C8d6g+}H$bP~_D2bB!@48ThkzeWz0L~6qMr99u4U{?U| zgM>}*_{m)&^%T`R{$D8StsXx1D_AHIwo?`s>CR1YFk}qX@Anwde#{PUk@R9xzp>l^rc0f`iAEGZxexXJ_s}PW-CAvUi^PE(0^Hlg1V!JGgkqF-Z-B7fd~cz# z0Vj`yc|E|tir@V=q4r@Wj3rAV%WA>Z==%i3w9V!Fpw0cT@=HP4YYqe%Zy~2+@b&fC z1BzJL3f;6ZK@h>}dPn$h)mjlk2^ch9>b&E0)-Q947i z=6rE93u+F7q#J*t+7c4d2vTH6D_WXNNPz#?sa%#_gL{RtwCrLRu+gsidUqN>$LHkk zmAf3R>VMeZpH@$mgE4Pw z)UG0O)}|WPoY+PGdXA5NnaZRS-szAFUz7Oa$OxJp&yOOY*0w*M&7rxd^v(E;I=a~OVUHOge zD|f^wJuUU>B1yFKC#4+ZpJ1uu+xCEc*q^e!yN;X%Pb^U&#f6!MlN%XZFHQf=cD~_t zW~`1L&Q|ZGc0`Vns3k7}S@fQ0jf`K1e!Y%mTlZfxbhu4z@T3_6+L$3ZEaOx0u!xMO zl~N;)zEK3GN4Pb1p`1tRi4C6aIx$h=yPqo#zlY1#df(kvM)At#uJ%{gTP9Cs{NLhe zN248qA@DfN)?+_Rv!+sH+(3IkzxxxhwrV>g>GCpE6M(Hl(s9DTH`3V6js6;Qb_xUj zyS85Q)`wyh4uPLS@dMC!1QzaI|K1<=k#kLJIdtEYpfvyQ=B>Qa_yV9q+T4y*W)3EH z5kzHfynJ?BCF*=ka{mOeme4~W)$X;tR^ZvlyS~TDBK^>x6}&&C#tn@t;wCS&am}bD zVeo}UR^7Z`^}f-`Y=Nk4XQgT)hlUnD?HM!##SCRE9|BuaWu4bNX7KN?xUZ`qK~{go zYxS&NeZSW)I}l6oa+SOk^j7j0M%dzAdjsdZm;v>VsOF)wRhP{~fBF+k#shWIf0Yfi zbi{ahkyY_oV7}^j-?4xx_4v_tQss^h+fyOt9jnwuSBr~C=qvwUVyoAwUx@*_WP83< z)+WJOBnlH=mbYY|$4`KNF-St8Jyy-QdW^y9Hy=(9Go7eV)n2eIBk{O?+M2qq#7b%Z z!Dv0hT0giAE?b=A>#j6a*s%P9to$|S__y${={q7MHq#1Gl-{(7H;iuM@z^jo*qUc< z$JnFhmd$Qe?Qa3Tsd>^m#e|htVj6`ps?D6wb{?hu# zfmn8x(Pl|YipFE6yyLyk-q?68bAiwQWd&Sl5u3(uzJZR90}uj2gA2}Pwm9M&92@RL zHYwL9VfEHNr$A(Lr)j($jWu%N@=i@yLS_%-X&Wqn3Gsj#HbIp;UfPsPEthN`$R^xg>q zX=>@}&-cAPf@`6hoJRbF{lD^x<9^vI@&~;?srx$rr9^{w7mlgX|zSF>iaY~Pxt_7!u z9{?=q5;z#^q(WZ5Y3P(J`w<$RTNXa?br$fSi+`DP*|Cf-2?I>mKgIz4{TvR%n`>dJ z^XGP)d8v3>LOPMk3mMoDrL5~Lz0H0c^iy}4z!&M~=^rvrQ4QjOF?3R{|5|_q)}>j&ZJ*&q@*i`O+wm=&E)8db3uHn)PsDzl7@YG?8i9 z>HPBw3j8C!)&IKuR$oX04ogF%=SeAus{HUO1c#jAd3oq{$?r$h+4vuwAQtnfEUr-h zuh>F5P(bC8WaVN{j0zJY0RU7@mv*RWEQB%(q=V(i;S-|3ap|cUQf<}4Z|8D1yB@mb z@%_s&l%!1(o&Z#xpg+F-p_%8lh+Hn37=S;=0W8rFxcc&^z;_}4ivR~WTP*cRp1#GP zAcq6RP4m!0L_8i4L59$i*?yR5Esg{hI+k8zb{lO7w0}c%$V0r8=RHFZv?8$QuR-ZP zNxrgT#9z+WYLz<-{%mipkP!>qC_>hvyOY4QkXMu7WITc^>GnTMLV{m$1$_%0j!%|6 z+WpAE_ON1#W=(@C=adoBD%Apy8L9TI!r<_f#(rQOYpB|DU zP3?)pnq>2vyP!)|H<|F3y=GA#9ji-=t3kXj4&R5y+Wh)v_RCHg98kdde3oca%?l+4 zGAa$-sE6{h#|fO<+grY&gAK2+b0S)5GT>l?0w`$6UG04=V2$do!`@rqs@D}f*h*TU z^XaI*z3VWOH=iLPeMtZJvNsQqP;nAifW2NdKK9z@PZ2vIyc_?A z1t^!}xA!PL2yRd5z81hh{6S3I_5)hqmKn)k-w9cbsacCnrZPX*=}(O`M8mQ&rAmC; z({sxRcRhq8>o*KFHEe*O$uc^sw1%RZS~Vmfp@6q|fs#lG+XVS0cyi^JBKO3N&rdi2 z5!cV--Jn_!a4h7)b^gI+{jmxU@YRh}$Sla+Yd>-DHLiyGTxD84wxNRo|4#&dQhpHC z7h?WdFyGH#;EQ{zG${lV-~WboI><`)^LpVkMeSZWAFXp98`fIWQD190HSaB$A@FtuzMmhN@@!t^M&1P zdeT4pn3w&n1=^vNB14wZLnwE}dT8D-3o6*`l=l-lW%Vv%htY>#lf{}#Vk;X;FTS@6 ztiSKDm>)~%E^!ER!Tj$f3nU9Dp8fjmlKkwsf3(nm$X8O+v{q0XfobZnbOO7^;bp@> zCW*m~oB@wv9R0{I!0=de8gwEOo$$+=-UY$MbKVL}Fb%_#ePZphDmDwK^2^%zR_*%= zaD;^6Yn~uwB!*h{klW*Vd%~1eX{C>FJ!g8s7AH(*YAs6fw1RThtS6%l23jWMZd zKEo~N1gE))^zG-5DhWRorq|xTyrtwCmRMHE&xWhhs8h4wVZP$S{^MsogmGPR5@hbh z#)Q&5`;lNj6)S8~=}?a!3gZQ@^WER^S`Bk=ZvxDhJBy{$X|3dDQdT*?j9?T&+a>+s za}CrhFw!S@kwyz+s!>xlHWElFakn}6^9GJwxT4iObzd%^5tI3>U~-H$J{L(w2k*DO zVktr@JuN+v!Y^h}u_S3eyr~NU0s|m!$(jaGs^l0Es9q|+4wYW>WdJP~(_QDXIQzOu zOXPhLiejchq0`Q|0X{m-kjsPSEkKis^aklS-0YG%r^4lmyvM+7i4E8zL;bJsxpl>TAWtU++qoiA%3_ zmd56anX0OeW5LKR%sAh*)f*4T9F-$?g1krf2^WMR#mr+W=`qh7h*N?f00Z{~F7TEaLpEuk6vd^z!0YMgljJPnkC* zBWNO3Of<$r)n@h#BUz8ChCh!SnoLAguq3l~*po~x-C(pl{Mqx;XXq6THLx7yg~83) z6miJNXsG|YTzGyd|5FUV${}OO-RC>tzvuosI|fXB(0{7^SG^|;MCSSMIO02dY+puF zuVVC4b$MHSABS%E^s|C6J#{%2wClQkax?3VD{j7hFlp`%x-->j*tNm;#8I5*Ix#2u5yTXT8qgmF)4;x(PN zbUB<&!Ot^76GD-_zrs!uCXFWWcVFYx*;E0>yQ9=?{Fv(zGj{bY0M37AY0MG*&)`YV z^_e^{a~u!6()cL6eaO@#nxO-q?cV}hQ|FHdcBYFd_h#Fe)5?UF^v1DR9{dh2D+?C} zzy$vku|yOhHv<>CQUuilj%Agwt?e5+x@b?t7#8@<{bV-cvdg58r^)a|BVmMS@7w9% zc+fkpuErw?s^Z@xEJU}helgi$N=F71dm`5M1ArFxw-x;x*H_w)} z86xO{Qe7u`x@bV}WWpuUK4Y-_g{0YFRG%;%4QJEw-1A*~n*MQWLh(1fR?;}kYAAV6 z3dX|B^P*N_Vj>-*Oh`c-q1(%Khp0p;jLrGos@pDk7M@ZE_d2h@*4Z`t-v#cloxR3bjsyBihmU{iM?Px=P)GRL4JH3JoIH=kYIohblizn9BK+#~ z0V|BIaZ3~|hj$wO{U9OYTrR%>TUgPS(jlnBU}JcU(X|OAj@WZ9XO8_=m%F(>Ztd*$ zj+##NfQ6r1lSSg8PqRFw3>}MtDZOYh+!wKyOY4d}tDe@1Nf|n!eFJKKYuya9O5=|? zOF)_(6QkWnvieY4?(SxXq>O{GRi?!kPdz(^gAgy9-)#_WAKlWVt!;8;@8lfr5_PKp z%xGyS;hj(hNmX?XAu(;VlD)cbpW#q>jQ2QVc*N(T{xXyb0Y5IMy z*?D|^Ck03Jd6*P;noz*u?$1cy1*^|2L2SOV#24Mm_9^3fJSPRrem6!d)`!U_dsY0C zrOJZ^$qxd6a#=k*R=5Dc*!a1)%Pv=2dL)~{)AATPj)ww}QrF3(mW~|TgYf^!b2j%X zCLw>u*rJP@p#KdZ$fE1IHDVweKCx-%^a5T$X3%+$baUFZ(3~z%rpD&G2DW&`HE+WfLST^^@-7||Z>N~tA9$3i^c*dU^=IA9Q{V)vwshGh(c;5qtDF*a=B-5s1rRK|v7||w zJL`&AX~=UkaJAP9>i$8TU$lts-KMDuY(+{0ot8|JfC0obft}oX1QW zlQN;KYR=n-jWQRa;CVOA)rj*w6e&YELu6dvYhPqYCdW4mp@+zK)MD2SMSMHF;j%a^ z*MAzTrsL0{i%34M?FLcQ?`+G3diTP|9WOT?xaW%{gKD{tvW+%0!_5k3f@v=$6x9); zJ%AD$56)kDwPUdw3c6IsoPRBAbn;02LdFvVd**jre_qbk6dRaI0*L0l_5(1+Kpf}p zX&O4pCe>QDtK#PP27X5(*oZ_sMHpv>UpIcg4JkPvyy|He`?~EO-Lw>DHMOy_u4MIt`2@^-3vhy-B|ZaWrPHL)vcX5 z4KMucO^)~>C2TpqA?I2)-)mkXtI4%iIfHM(4KpNs0rvh&c z&ibI3Gd&Fcb%#0!J!#qToquW)8w5_Fr*f1y1qn;EItN|A0=1N0=(zZNVX^+yr6o-Q z06?bM)roN3;?quM5{TR7&qxm!ya^m3}F;`f!LDNG)pl5$Bd7VGT8 zI`L4;ZZ%P#jMrt;m1y~?Rl=XO$9k4({O5MYynrscIrC$j_DKF=U9?hLcqFH0IkUxk z%Cw4Q#8!uEFHo*r1voPLA|xeI<=vS%QK<080Hp_m8T-6|p298!K={(~{P!T7K+;P7 zry_=}nX^g5;FNubz8}5n^48mG`$7*biR+|e+v3vbSHds}Z)Oj-5C`_3r8@JN3TyA- z&`}>3L^RXWV-tHunik1nnYqd7T8I;G+z2skoO5>RY99eak1E1mI@86A%xtv*G|?_R zy!YekrH!M_KJ8`9od+V~Bdkxp-uoA=cmSf&veU91t@33y4)R$wo^oT4^7MTYfRGRg zAwca%$COo2K-{zyvR*i4Qf7~-#=G4g;lhN1tWWKywj{HL_k+>D&6kdhU#dHq!-P4; zE;Rd_vbf8D3wHl$va~CygKpR~*`xZsP6$nSd7{2zT(L{%k8YwuB zm4@qf56>wV5)2?5ucEdEyy3kmX)DfwRKwrj`%R@&_+4BRN<@yQ5r>9w6eojKv>tR6 z9_-Oi82k1{V-|hNV{gKLvt!}B&Da)atpSAbU0c4y(*Vhh-05#y1_Yn+-*x9RQX5`_ zjHqM(G1LbwI&1-o*QT+h`M6sS8(IcV%zK)!zWkL?Uw6aR0I8d|b~!)YGO(`tDf78T zFWq{lUon$4-Y+?Fc`(65xKlkhVr!?ATt}k%Uc(Kd)!}qEYo2{DW^>r_uc&eLOvuVdWLSEuncAFdM5FQufjb_N&P4 zL0z_;u#kD*N7*nIGdgT9Ancp3{19yBY%zHzmtg9c{S%lJ&?KVDSFy)mMNe`)Kz3*@?hs2hnfW40} z5c=z4K5jBOH@$317Q~IdWr^QB zCT)AU@lcY7u6Xa!=LR|j6*+a^PGJ+BFGZ#M)}Bzj_=IpSCkP0%ducR;l2AuGy5b|` zPZ@&*Quyg9-1L3lcgm!{%R%Ymp?O}~Mn-a`#k}+^l5u-Mk{bg^ z@mmow{^9>~I<86opy{ZixALBxfAH)Z?+F!n*ME&PnpyuIvAN#w6PXu>myC_A*I zT1s*vPUB@1+hVXl@LP`p-mLEt&CRleEGsSU)#D(gP~7=WPV$APM^w}ya5J0l{bVhJ zq3g-lFbJ|mXuAaf=r2OMJs|kp_<6$n_hQ8jP+RX1po%NrRYAl=L6HFm5G+N9^(R{$ zc2#Pfh}0n*CIhV{7tK{m>#Fso&Lt=f^<)84u>qp|pM%6Il4@#`vHRbZKj+E|`7eAmqdtmU>SYMO6_k~_?n=9NB~&~p{~-qc2x<=4y& z?JfqWBm%#OUV30{vw;5^=q0k?bn|c$$(&hU}C;GLQ%W)Wu*TivP|k1tpuaF{Gpe zvACi~;qJP#ukBLV$hIgA)|xL+u)!i~nn4d!sk(XwDO>(xXJWhx zEgK>1tj}h*NO!we>m}+eQY|J)0=kDy8&5uAYc&<_+AfWr!-WctzI2Y5yLOxImAd0H zZwuKdV>_H4JEIfFVLbX42st$kcB0CjtoA@g#DF|jOoZ}^G7G7ln}2CYaR$LO_c+!? zM0!!EnBEaoBQfvDiAqY08C%%CjU}%XQt;eo*QFwEN!57mCvMPNmJ&c#LGjX3924&e z>%?fBCXq9uQCvip{+&x-YD^@HQ?_$RwHZ+Ax2f@3mkg4R*b&zHVu#uoXUT+$h)p75-Ih!(O$sV$dLswHPPh<4LYo153ddxIM!70Y6>Rwrhe4lT_~vs36&C zpRnN0RbQZ`899$9arpg{#4q(zCcoU^FBFn+_G&@ z1;a|rM7?s9hSPF75nZ@((-}sr_(+}Id=J3t7%&^>2x*`Tyt~gMgSOK_OGC`YG-zj~ zS#oLq6HFJ#jLuV5;TTXbI0zVA%c{+#y@&rTu;W;vliQ`7UPkk|Iy@rDpm=vaO_*6K z2Gu)m&^fW9X(Jp=^Re$ZI2uO@MN`3S*b_>+zqhkbxog=?j;gFibpqUZS{u52({D^C{o@{2a8%}LMY#SQ6(*d4QRl<`^lv40AO&SPlnkP+Y6?l! zT*qq1R3e<)I-xcZ6j451w0wp~5<8p-eG5~I3`xZN=4~V(gacnd^yUh zVhV+ncoXBEyiE}qvkuy}G);IK@G!kR4lelzl|S;2?5G7uDJ89DBZ;xepY22ZtBD6~ zYttBDfH;$TPv>}Zmz3eMF5AC8ach6b01l(sYGyhl?P(AF<3c{b$rKFo27o`!)FB*CkzaUpwaUS#C*}_f zDFM^g8%r0N5*#|7c7yCk+xI6XbW|<3ZR#E+J9f_Oh_T&BFwr&C9Kp1=;vfDb6T5IO zzTT11HVZ{bD;6QI<@Xd8sqe57Hg|D9KSiimX{a67>rT>eVdPRmVA|NYhbk`3vr_;7 zt0hbjFvz56qwQ@W!{YS9TFbGt?=_Cf87f2n_u)g|E z@9?@qY`#PNsn|2$@@`SskWfxBsV+I zR1Vg|3t?4*18-{YH-M7EUa-xrtzKGvpNpl!a~wz!vt_ma*0SM`AcbttZ-F-&6@4fg znQ^P+bYa_SBJtU5b_v3pi#)!N=p%JElt7}jq_9WoeIo!M;Vtf7!tI9&7=WAW6@I_l zDd#{fHANI$myV5QYntT1%r81d%96%)CJvmxX6=b;(eO^Zb|`x;sH=ym$;Aasa*T#G zX=3r;?EWGqBm>px1eK+{(yS3K8rHgjF@Y2*nAYmt@IaZ5UB8Eq_hI_w0&~gPR&}UN zg!~0Jjz;0RVtU=YkE)MR!dO^vp~+A$&awv6Px{mXE z>p4yY=zZU5y|*`7)rTO?CqurKVNMwP(n3imMZcNvp`=)(YZys^3Dz`IHZpoAiL4`| zN4aD(Em(4h@zR4!jzext zH2C5LTx03-{oK5mDO>*%Z@Kq~9jf1qgZo;P(Xk11aKOyczA&D>ke1SSKw)D8UYYEs z?d$I()@D^UvW%>#vZVPtv}!Tc`1gB{Ut!bnso%VCc^QoJG>(+oR}84+Vj`TlqVLMH zu;dG&n@??6t*7`}%ioopUHS%r-=u<{knzRf8}bLrh7zOG_yv3&{3KpagN?S_$w$=t z%l^?kWf#o?i9W8Z$LHr-1DoZ}@_yeDR%u;R?|FUu;IM5cH)Lf7^hp1qc|2lJFngRy z72#L?4)zl#9rDU85+6pcc-6p%a+OOKRbSVYOASVlW_(2C+!fMb@=zto<%h!~xhNs3 z%6Oj20EUE+lA|!DMoD2n3myd*)=)cFxYRebFZb#$tzO*57nLlp{)8kl29<}Qw5+?aRuVQFhtchV-S~v&$o72R3RIuYo=QPdW+-;qB+QXHjK@j$aNqAJ8zkA z&>Pk!%Gx~d!$;0lC(xgtj2{S6hW`6xYJfUk=TYSy_fns}-BZ8)Z(4HKVU@G?Ew$0# z#~YQ$!Te=uEZNYNNeAD$>~f7Pv9$hi83`#F$vVrPlgJ>2kA%XkI^SpTV^iwtN4eqK z^><@duzJS{Pwj?vy(nqxqMROj+h6Ugc~uQyL6Y6jlD5drRd_~ zZj=sw^njF586$e^-cDc|=kqKWWH_|G+3D`y-(9l+GU_w%^fUBU)9|Vx0f^(&+*LGwP1OD(C zr=tuHy5RjucM!x$9pmo(km3; zrO}hN|4lzDr3AE)@c1(W;&ZEC2#UEft`)gy++*1*Be zxwtMpa#pGfm!8P#HT(V=S?xsmwvZg6xTnTbmOwtHq^hK;$N-6K#?S2n4k%_iHqm|# zJ!F*Pg^CO=NS?()^6|V6nlOq76Jy@w^+=pb#!IuWkJA|4zY!d%Znj3a2)3)K*`w){78Y3FTtHT9rjyrChIwvrS*LA4Z*#~l0c z_xWwLD>(+1TFPQ=MN4b+J8Rlp)-7Wx=pSFWECtvT@Q2nduF&WYWyQ{sz=Zn#uttq( z0b26vEIXERnUFWl`>dJJas)P3wNb2A{M;YB!(&ecr>iCU$TXWgNKCF!06@id2#>j( zyn<~TS<=VW9upwL0s7VfA1v>!`gVy)6;-jx%;VdVix`0D_M{`3(umXO>!oJJh6p1U zLqS!khhGk3V8xwR4??Zai>LX1IEZJ!OKx->5VQ*N3)6mW`VqfyPNx%}LE&TdvxL_> zyB36Z?}!Jm2Sjw=>u*w{W>m+>Z$MOjT3N(AG5S~J$xWXt{5{bzxuzV4;`#zG?%<{io>4 zFCRP%o9ucKC1X0UTi#Ap~xl_7i2I*%JZ;g_G{Y#M==Wa^Ui77h-Bv*fpqYx35* zoqN}v*>;twAW6rU5b3w1rLGRVxSCUjH3Lcy(+e)R@0U_h{5*s?W!FQjzH1E~s#d4t zhjLFF>a;pZK3=JhMOIU|rR#M*GrA=D{l{vqjAMmHaG4J^81)~BP6p#9lCE8M1hn$c zzX_QQKCbl4O3t3RIw>vjn-HY2z<=R1rbMaB-x*-4FCER6Wz)B=xYBhs)B6KA98Q}Q zU*qVA0gDbV2f^HBQormW7Tu4UG4u`*~+_@xz3HTj6y*0m5vq6WRDx|uK2}*ju z`nyV@vvR=!u7{2d(d$#^X`X+dbv83K^VEEhbgf?q$9B@OZKsoTY}>YN+qP|0oL%4g*7{HWeX`#=siUg* zn%A6T&N;@opXYkmZN7qvpk60^-byvYdS^4D^5@~ejnl;xQQe%))H=B?R@bo0e7+Y8 zASUqvsZoO(Ao|)`6vNX->5bYpTfq2hRH8{CTL7tz_8S!gW3H&<*d0wndwPZ5GFpS$HvT!&H?X!x=1Qq9+{$v2zQ-4 z5ZsI4cBchXzqRc*6u)nvQJhPW?lZ~M!xf&-)paF+uGwaBaFxv8q1j@jrw`|9GHTG5 z*&=E9RE@qmWEQf=Gub^+w-7 zAD@`Y?gp8p%d<3}Qr#3CXr}=s@VPY$Ie2R{zLGj=45K&c#)OSD=}WfBI=SBRobuZ4 zY15PvYL;3=#D{j;yx7#mB_X__HS&?L(dR~M;Y%=K>Vq)T@=31F>MSIM$7H>S^QDM+ zaj@z*E$d%?w_<3ABo%y&JMe2bd0B5S&v?fu{&JWNv-Ao0YNX&hVuX$^ZKM5M&&!=`$%0ADynSNYvi(8dkGj+a#OFG? ziH3x#se;XsqDf)Ja}}u#a^xwvJYjvcwQMm%sZ8NhC)40}HX1@!j%35eyM}+altU_e z&z+^-)m9OQ+XFh7C+5#ZMw;>&ahr&?J#4tUO4auC!*dFP>J^mT7qGT5p<9=e2wU#6 zkc*a^h_%wqJ$K5x=Jk(>%s*943)>Y$M(%h&D4#fXezZM*uf&mm40U{DBl21xc^FII? zirfhot#33omB|5#%?%o&L6v|`OP%B#p5-hKdQs)B<&4br&p5rB*FInql7`}%x)Qa; zM^H^S|J&SMFvH%v*`LG7D{WG&Bx7<)%BJT^_<37GxcYlssh1Wq(J*fWznFRDBOz+| z*MN3rH)t(z0$wS0j^lRX49ia$gcP>7U-3Fm?3cHeC>J=%rzWRT_c-%ZpA%>EVqg+- zgz?U+ZhI;=&QC^S{1gS`WK1tFs&C9?6{BMT=L=U>Yd&O+9_C27W)aS}>II)?y}^BW zazq!s<@}rv2@8vAwyt0`4uYJWdA*vLTqf0nV|+BnmqpOYNAW>&wxw2|5HlQzD*rTS z27k&!?D($yaFjty z2}Dq;X-#NON#8xNSP2qPSRUWmvRQ_ukWef`Hk=vZ)1K{>LX7~Be|7udqru-|c@*e8 z7!&Zdx%gFe+CJiRB2@kytHXaNifaYkM^}vo28&KCVPA}Roq%iPN#6zzFdQ?)r0B@L zO!Fz!HCuytn7P1*yZi>JiOn~V5oij$2NfIS|7DTAmfv@Ey2UGI=)Jr-fi%UWTE3Fl z%r7QFGpn8{RKS=cO{C@jP4t+4)x>h?GI9Eayvfnu{6{5tn@A8M2j_%Z$XqrR79jd> zb{c-&skS7qrmIZLV3=%f)M)eF9OTCI9y*+f zsS#vOaXNK|(jsdfp0h|Yj#*}Ty3rsPx^h-6@Ll>Icumu*h?u05C+cbU$HtQ9&(1r& zr$nXvtnTlf&!Im%Ss!O|kc9WGY*O;wx6d0fS1vvablRU3M)*#kx;X~Q)M*Ts(4hmw zCr+okT-$Sy^uU+whq5x~`LuO1l()#8jm5ocpI-aH77h{k`zhYxVlh57o4vuW>n8sd z426BhHN#wJ8%?`sI2>zMbut>;N-GsK)Iy__teAEMXYS+7YJ_v?b+L1K#uAsa-`?^A7Y&+Df zGJnZ9Pmoxc;@d7)H80zA=~WW+xx1etg$I+b8*UAGPZ?_L!X#UFWsYXR0Z16(fZBTz zME1Z|w)eFtXChZccUYM(c7vN8yoX~9jMn{Jr z+1nLZwS){d^jGuL+(Qt)qT)y4W%rkL4x#uITNSJhn%Z9==rr`+a){nEvT5^Y#C#t! zpeALLbQ?(=0tC!$*iT?wwyamxer z-O))lu=?vb){@{oni{f`aTd>%Un~^;%qv%esn!>Td{uvaO7}py868bWCv!tQ{dr0F za1wcwP@OG0k7gw7ibE3A?p8KnZ4=ufQ}epM4y5nw8rZEmW%=4K9$9w@YNn|OPXSs~ zuHu!hA*cS$0{*~n#8hx!X%d66q=&cH)i*q@j1T(~YKfKSdkPz8 zi;tr#uzWY)wBnL2>~`Y$s+4WWDFresqrwZzX2K1!{DTcZl=oyPo3Xv!f_VC<4GG|s zJ&v=0mhE6?p$uT(lt&NyNj8V)b6&Dm(bJypK_Z{@aQ1tq(-S3I%+$d?2VdQ@4JmA$ ze_t#%Qb|X%&EKEkdzJh2nCm)&s+aR>cf?jvCZlrYDBnBh)AO12DLTzzfA4sH6mLb6 z;I-pQvDxRI-`TR5I^}pYVaiCmmG2L6&18}>&${_ux}7~$+ji0O{MtoHpQ=cO6+-ot ztGvu|pN(xcTe)9Sxl*(OZA>~F)QYHmcrBxz!nMiswL|cF3L1);Lele@jbDKQK8ho= zDMV7sFi^NecMqL+MI=mjqv+(-y9@pnH5X1haaZg;_j%VFEiNBN@zENOMPkZ7vYD<@ zUKlu=W?l4@yNX{rz#LL89!^Ez`AnYqtORbGQL@p^TzvHXH~cwjM_; z;_LWlsc#E*0DA^e6qiufQHoY9A40lu1kq#(ydY(|V!ul3S;gnYsSC0YrqRLR043g? zy1zBAeyS?xzcIgFcRS`yKA~pR9HK8#BHFuLVEGzY5};W8NZ&hDD2~!4sOmIY29I2b zpK`62Y?Pu`4_0uIGScnpiUmqRXvC)`)UeYvnhXpy4B0-VKZ0yY|3tn?=A5Qo-(-|y zc=j;Wr51`Rn=d@&NRl35C5JnGp3$Xye|aOPaIa1-lOjlEJ!gf9v}?7zRdH3mqw1`= zE@|F}yY!xxa?~t*vr2w(FAXI}sHinJ*W%+nYrR(qm+hJid=nJVTqVm4()GeYj}#1HK|eR(9HAp)oGS<3pO=+;6{RjD$luMYFRbgf@Bw?S!MYXV~CG#b|FS zAUTUE8mSYwb9g*qfE?^-e(b|r|5bA9;7~`ep}_Jkb4-o33t zgh4BJ6f{JlEix+b+jFk0q2*Z_wC+iiIWOMQ#K<5G%(gUS7injRKY$EX)6PWMkUIne zs5svIG8DrY9!-N#{-&QefR|Tt;OT^3U1^obYOf;P7>{Kha}Z%tv3d1F<60snAai-$ z;J5n55dV~{#DwaBmk<6o!t@#E4Wr55Swe`g;2p%@cA?;6o4o_BHsK>jcXx#pW@fKd zi5HXd%70O?EVMaYQmT~)l#GmhN6|WLGA{yq{TQu`5J0%_Ea*fc6qr`zC*`S6C{j<- zATi`NOMwS9L8*3i-JvbZe>b&%n_w;UeSDt~rmiDwaB_C$bG_vMGZND~bhkmA9Y#0u zhk3-JpYbxenoMBWSMIm$BuzO-YN0bT9*cEXw~z@LXSaZR;!4ot3=@_9_K8RulD4X-vift=9{o{MnG9aI~B5zrJ=l4Mt0hmM9%v- z1F+@|Ttw2%;}Vz zD!-aW1a0{`Ny5nKHO;AD*C1n7Dj9CCsQl1Cmj;_7lijoIS<2>HacC-;KZ~-*RMS#r z#LY+rI3TQUHl!3w>XPcjWLv$RvSn$6(2i71$l&zpY*s~?w(l?bvTsTe30o(kDT*$) zOyP}>x`q2%^Wi)r2s`p_$pl{h^ENh+W7Kf}k^43@E|Cd=QkwC;a>}H7HoL$Z+D)i4 zKfNHOWIyv_Q{WiS$JG9$j|;Hd=8n{Qs_5kuDX`?~T*^A%M)?mS<^6`DdK8QnSO*;t zNn>9vu-qPJ8hzJ#Z2tSm;rE>YRBMIcMOZZ!JlaLW&|wq4%b&OZt1w{o88e#)`|!jssW(226L;Jwj@n=D zpoakbd2sdhGN6x~j_(@rpDppnrW@jJy26U)82GrFWuXV`98q=f?w{d3RHFIKIxcEd zGaesRx1}wcENYydT1MB@r=yN8%dRiN;^C`|c3HnJvf}PvG*W|0yShl?6~f-8L`5 zzz3?Nj2S`@BoTdR!vR91lWER@HUz`B_4eGD+im7h-{%#Qv*MLW068aoLE0eeG?h`B z$uAG*^;h&bpJo<{{a!r5K0}ZqtlyaH<)m!1bg8OgO^^pa?+<_E#MIql&I&-OwqF346dW@QLK6#O`=j{B@fe z?vG`~Ry4d3tv~5}Di`zSxSSZ82`+BfC{NV77guQVqe_{xQcxJq${coDIZpdcD<~K~M`AIM$wg)qL ztgoAmP;}vGMLZ@>C63`Lo*L7pqxH(8@~Z_7=A&O~m*Ii*HxX5kLMd$!(nj6gp^PQ&a-!yYp;|VSu*D4x)Q6$R(ACNR1O?X%PMEqK z^m8rTr;bUAF+br#KEH1pqVMS>w{$aNL44=xIo+Ak7@}<`>uWjZBLsF+t7Hk(x#~RT`wIn-Pj#@p{=K}O8c7SKMA+()ypMryt;#m!K`saV+#cU<2c@KJls8+Ggrvys)kBb^!8 zcX7H?Uw`ZweF;|W2$}$;NP1n~9~)q2{w-{l$%J9lgSnmmi2w*=R{}N37LG^Dk>4q_d(_55Q_rFki_ zly6_W^y_tyLJJFJCS_>MOt5~bEGZlo6-r&W<(*11bJ4otzU~HNK#)u8R)4`NO@eYh zdH8ZLW@{qSzlgUB8uHl5OCvTL+Q#|1O zM6a$l!Z#ozg5caQ-`#%U&|-xF$A-2b5h%+|=nEsjMy-qN=H75&S$ zmIKl7Kjk>11qnypBEOuLU>jzex`_g5>86DO-^cEL_g4GD8ex7!sf>L( z)2^@Qt#5LC>_7et`7g``0O}VvGw`*wIk`=XpnZzcqja(Gpmbnh!j}K} z;hpaUehUj!;p7;Zl8(bqCRmQ50CEj$>mq6f+ISQ|-$jkowMugKil9UFRBtry79q6%A!E~oQa{AnWSiVWrk+kjt!|BIM)_YgPs29KGJ&h zX*u?eEJl!?HENnKT14lIC%shvW%Mt@rJATS;tMZFbWX?p^wU29#dfQMx7JEcU754? zCGx*m8}ODYl(4~uZVGBjKE~y@>vCakB(k?-`X6@+2}Gg!^j&tH(EF-ByFDiep~Rpk zL!s8bp3!;FMM}(nFEF7T!?L;FZV%sON;n^9770ImHXgR{1YF;k0l$lJ@s}Qti?9N4 zSi$K{2tPFGte7LztO*{r-rU?QeY*b#&xD0t>&Yf}x!FUW!o6Om_JWnPuW+3i>yvSG z@u`3;;ba=LC%7q_17`sMzUK>y5)a01-6!Gi0*o{?cmd88n`JDK*Lwfco!H=i(XLf# zI9Muqm=TGAUWrGAmEeAms0cgNd@&><(|u>bIWpF-rRP7E&lfvJ`vs} z4Pan-pjspOeaQL@1Q?H!4~~vh!Nn~1mPE!Au39UAU)tyLcuZaw?0YQ9bdZ3ah9j6w z&Him;8%K_24WxfHC!T2R?#{wT@iExwNmg(f@}^?6)KTqqFE32or~f;jV|Jp^_zM1p z4|4zf`3+6Qu^=FQ9Xvn7GM$*5J!F&WU`f+k-l=$UHn@Y&ELxweZ81{8^;%%(5JHJd69d}vPDOHc~0LNQ_9maUf9V13jdY?H>`hDi#F%@F$E*=%Qed1cbMgN0?gTPIxca1e>N=lzy=N9x7H zC8C1EoWYX25{521M2BhyKD%RnLg{L2!6wf6rye!$;{GZAx%bK`pK1#VL)rkuKCFFR z^#6@4-+g0Ngt;`mZ#TfLOMZD0dsuGYwxetpC53~jlMM>}l$`XW5yOP31Sva{V;^!% z^LJnb&+*}_on<}=I&_&Pm65VdERHc%@fgP7p}KsTQ#Mps-X@Zxg=p0r<(z2P=x$}l zdISBI^NYecyB(7LcYu0`$zANaGhCWH`$9kB1``Qh>{)jF{>%*`E9Y+Sb;4Y{8GeJY z&X}UOwPd7`i5Ml3;aUjHn+ah*WesjQd8wiZ#gxoh^Zh)VEFZS3s&eNT`H}&gE1d+u zpk!xe2sU=)&qv?=cNU2cp9e3!~5Os!jbjp1#982Mi#iCJ#v@EWJn5068xU zrNu#VXeCi0tOZQIfAIdlUiF;S6fqijozFN&oC=dQY#ko{Rn|vsed&VjDhWm~r?ak( zdEa9F@^N|DNZol;XnptH+x*|v2|=RMe){1n7{Jaf6YkiY*$(cPLw@-&xT*Cnknl_A zqX$RXoW1>A*0KsO+h;5`zPlT`zUjE!h`5D z!9HcS!=c0zMFQsk!;Zt)nEw|$&R}N$Zzy;seDD4L?QGrQf{V8Qx6O4U|GfCW-7lDA zAj+4a(oNghfAIrf(%E8inBYfWiG3Uguf1oNVd!>H9XTvNELwWKd{JFk` zpn^@w7_;ZHv!U`Qw_fw57qfILkYr!BBHI@UZD}G17_>T4I>J?Ap78r)F|R*C2D>_a zJ@GcY`6U}li7S&^G<+*)&h`__f()B6ldYit`jT5di2hP7>x?SLB07azTCYh)grGC{ zLxAh8H|#M~VewWA$V5EySxH7tyWU!@pYIbgz{$=rYgT3V`2-^}105Yu+m3veuc-%w z9sko(Djcs6Sn#K9$@|>O`*qs*RgjbVTLZ3z+WJTEaCbxM-V>X?Q)>T*p4+Z$|3y|Z zS9*}aj?=!^mr^{vh=U|yCDWF}89l$w!%%D!Fx=92_U*#mP1|{RL@bv809c(Z{@HB2 z8oiD-3I%dMU2}K~Oj|bt5wvmF1}9$U6{Dk)Ny!xXFQd_XA8tB%`AUIJF0YsEx0pz* zgAJywgmp4585eQG!y4lCYVBGUlv^vu9y_niNiu<}5J?s46$g&{Kkp@fT?mhBn>xr{ zb7Lw|x-)$%PT`rcK)Bp|i)4~2)E>rhBUx}YqSLd`jTyw4W)f_+d^;wJAkQB-z)o0f zM60{X*zB4ovzl}fZJh3o*jVwCI7h>tg1tIAvK)W4W1v2nn}-Z-nO5cx-%M{8*zCCX zX+oubm70c(bhqHUwzFz9CDa5OV}ln{v`nl$H2gwbOeA8LE0p1Jyka0~eu2OiQl?OT z9e;5Pj$6eiI;bzs^6uy_X_T@ct|GLKC@LtVTu(Flw{w_U>fUjLhcD*47>OXlls3(u ze*4y^ZAkbkr%~%YarBH&w2r(n!hB~{)cL;$g0m$d@#(>I0*MX{EU#9=7uMic0Y2&V zA9=f`boO1*jw9dpgQKL7#S*@Ubq3*;$E4Ho3kyG#mw1r^07acQW08pr!$YqP#tZ)K zie!4sG6XIy&H@~VGCwuSZ-3AbN=Oqe17BLhUK8p%O{Fh@`^Z4oA@D(VO6x~w43Ehs z6xnL$BhY3o3|z1V6=Rq07OYyXF^_}V_Z+S}9|86^cY(L8|h4z}+fXD0X7@hieh1eW}g z&AS-v_r0a-rIjrNwt^{R^5IbEFhCBLCAD0ksa1ESj_!{qY0aQKPP3-ULnEd9D0*QZ z&G$Dgu&(92aLp@RRFmXegbFHT_NyYwsaG^YpRbElm~A>uf=!=2G=Q{h`Erfqk%_l) z{^oM_^HSeRU`;SRyBb6Iggh>@mnfTgpjdXlASOJR-`4C(0lJc+R}nrf2`mm05pB{# zQ;^!BPgu+oCPMwsiy{7mt;8Lb!nwD+rJe(jU`PEgfY7W;ZZ4~4P^U*}7{JcP<>ybE z$s&^sJ;}@C1Hs>0CSvjq-hW4)w0`JO-9EhDC=vj;iGLi+hSh90dTQO|^m?kRYVmsoBf zn-)(1KZc}!T)(�gq0(j$jHA{+j%0cg%e){wUYOT4ZPPG3YV4k1+|RlC&c|{#xN1 z_bX|t0;^`@??=>8{2uaG9lO09;hXSJg`Z-=zfmC}gcW;0-!SAcYWaE@Au;6J+V%f< z9K8u6F;nk!0v|uN+UN3PR7xUGiSDn)*951KnY$k7bts0-6Xg_md2+hJL$a+}v z@BwRoE}B0B&QGGaU-LnUK}Y+|KTPOu(}iBE>;-9wWuD}P>o3q7>{qlsUD1i4soq)N zbrGs}+^R166+b9+XV``EdLJF&PPvTVi`MslvUxU2%xN=)%0!vP}S)O!AwJGV+fWkZeKY^(*6{oS_VzDTp*X6ecbR?yKe(Qxj6gUFuxo0MVVYIQx3$2Rnr z7|&e@dZ$K!fc5tGUi-h10+Crp!P1Z}Ek3u$)KMOpOXvPl;t`zJ~bC2pdh+MPvER^*U-OsDF zO9Z(;Hw%kc!D^HEO{)*;iq)A~Uvjp33N9|QdUVFP2_MYaT4!&PBGPDiZBN2IlE=aSVmTb1Vd=pS-t8zo(aU1R*UAL$1i zhlP|>&1*ieB^H$FDcNZmb7jf9nkFAb3HyjSe_E5Z`4^&y9$GmaN)jJSrzD@ey<^pI8{aZ z&l1+*;y$d~0~0$6QLwP73%s@J*5T%Q*IgIK-Ac@TrECveNQ8ybdOlJ~D zANxkoc4a)!kCe`#LTziF5~>veZJN2AIV~n)X=0smclVL2J1&p%q?!4#rGRN=A*P_O zFLjJG5}?&`6HiojZ2?n%=ebv$9SQb@Y_|RGWXhMw@m<7qBZT9tnO`Vy7vJS<#0!zt zAxzrx@{>t)qp{ngcVOwPTEbJ16G6ayLA5&a?4LpsrEoT03jo#mwATB;TuCmRs!jM5 z93;6e3+8f&K0T=yN=;&>8QcKM43+ImZ8cruRnIfYTtBo@ETIwhBAR`TceZyoZdi^NKU-H0p3HHo3 zy}Bz0)Ctda?&#nwf!~WVM>Nf!h?6a8-I)42Iz3N4b&0m>8wPZ6O{#SqbgvZc4!+cR zJ9~8-l~ysFhkPJOCHF5esMr{)+}Q--G1|yS*B`*nN}J`8(6n?En}m-8f~Q0_gXzg% z-lsOl>B0Q$wjYab!&b9@D}Je!54%AEP-ADcI-5$np4MF}g=!)r*E6=C;OuCLi5qgF z^L*Z;n6^Anwku6VC8syf3zd8pPOW$&6%u>ZjzbETB3d7obWsGAWBplWWd%q0 z6KW^Vzj+Sj^0KhbWXiwpE+jvG1>^X(=8I3=TJO8ei7t--E}O_;h!J%b?PV&JEZ2SU zUW0JKg|=!r&ifHo077X8AC}D++3liMpQC)9XN0Pk9}nTiDQmdu_=X%RD(rAwB>E$1 z0e@`}G`I+x-2}?t|Fs+}6nC}wUfVj=Iq7n9Vf#V)VQINsUDgqOeVx?8{3DAK6@^R; z{MF+$Tf=N*53lDM4am<`X;wbzyq#GTWg zFm;s}XL4;M(jh4FRLhwH15mHJuWO6LJE`Q3l#wgD3{!rL{~g>%X-IURxH`wiB2THh zm3(m*5t8Tj8kf)uj=5$B;k+f8!b$2Fs`jO)qGunSE(-3I4uEf&SN4j@Q?E$T2nugE z`Hth%)>^&Os;-~Lguqd}oWsz=pT*Ia-ar8Sk}b=Yu6-)DE3P*sW|I9#{d5O>*21d| z6Qz2?hb=a6sbuj!{MW?VWmCyZ-Q<&>8WL>~6|yqb9_VoMOejw;(Di^$KW;X(!*Wi8`DM@(LRFg$9vYCaaW)@~CBU)e)!6z7y-pU5 zjHG9ZySzvkK-3_xS2S)$@FlPo91yHg6>QSL5v%{Hwq9T%b1`%n+|4pjv;w19BI4*m zm?<&+qto8m$Up9dwn8sM~d6aMBd|W94Ok9v?TH zy2?-y0oz7aV`?_#_3{l6b8!@!K71^u_o=+DyNyFZwl0h$_oIsPVL?U3$>maG#e*b_ zIsm4?5x-Y5q)Pj-#jJ*-;@(+$rEh z*yEe4_1K7&`UH!V^SflfdVjzU|Ya{$4GaWWal$4Db7GN@Ck-al{rjv$T{95`p z6~DP>0zEV2fqU~kA=aa+Yf1{PI@)H{m$Xzu)i@*K2&XYhNT;IiJDiLvA(wsK&*8rp zK@|za-F7FRkQrkgm?GiXA7m$DYI*|v zTOOB40MyJ{4~*0MVyqt8DsGH3<|B!a)X73GAN|29x~jwV0Thm>Rla{cUa7iLs<@&| zwE2$!vt7)QvXa66_(h0em|q>)iW`JGgW!ex*3ECYrsrhZFG z+8{CXFV+Ak$~srAj3gu8;uAKS=JJgsSX(DdhOS+_1{Iz>iB%r8tECi%mt7^J{gDg) ze4y@A`!{qbpSJ{RWowNBdcf_>;I1e?-#H0mX|`sQtSQQ*xEnO0lVas4%r#!tEeWcplG$4;{X86Wj-CzzW1F+#hu%}RN9(4TrWX6^3(w6 z%H-fd8d)NVzZb>ZZK7w3zTRnY^TRT#13qJC_Im)?Q$8Hb0HJQojPxg;)U$f*C=24E z9CC?}Tv6rX39XVD#s#;Bxq)VLXToei^_9xa=ykKpWs3deCy(!Tu!44@C;#r&Rm?zE z9x+4rf`0siR`mF)v^-RgyfQ~)!y2D+LIGN;tulETheYYwV=~^B9`zp;iq7)-CA~Z+ ze$ZJZ%lXmEq#^N8FZ#XRX9X5I8Y4(cb2!lf0J!O|?DN7rA$DQX2ocd=YzhpX@;$f0 z&Qc!T0gct8)1KXY4++1bFySwdHdB37Ma{m(2rUBvV{u+djeN`|Jm`Ili&a%FlC%ad zRRU3=r!_k1&!29uEMVqp$yuZOg}+fi%hWh5rW2kWpuKjbAqn9WS#sySW*w`60e2~g z=PXfbC?bGvIAs@|Rs5(ZTobQDiZYu9X6bm!e_hc)Q3E0&Uu3H;(PSP^( zlSHXTxGrA-nHSoT!~g5!is+BG124{}qrd)k#SRpI5-67a-Jm0y!iGO~UH z83z~mt9lDt<(^LDpYq&$PhGyy-Jh7WhcuV~02?*dyQKKaHGm&y>HHpy*IEF;jAA`7 zqXu8>7@UZqePMYU8r!6cjJ~p*bHd7mHeF&c9r!x20KB*=t*}2E3bvNWRj2_Dm3M;6 zo;DQ1K~wPgrei&oI$G%#6NvAM+4U5Bx}@kkjz%vmWCamQ-!BNLAHOQsbcNfCdVR*e zWf^L6v3e!r4kdsAqBpfwG!^^Q2pl-{9%#$YBm{O;oV{f=LbmJgyfh~}uzwhuL<>T7tnC?($-C?t z{@Za0OPr1QJ9H3NMT*-|v}Ztzh))?1ki|RjpY?Omx3Xz+3L+Zd--w{-v5n_WE308; zNB|?&>w;Y()5ofto3&YTmw=rFeqKU8W5$2}yTlHL2b2-}{I_r(TH7Lc1_?_Z7r5y@ zqCj&@Z03oy_n)!OS6k=LC^Qpg#&@8e%TXFjaSq=f3Ill|e^wig@~F7(E0-n|T6Dkx zJ&c~HcOz#dgBSP5ZQFsvWlT&A=8xWJi!q`LTVZbjr#{)L#E$(33=m+i10*x}eCU5-9h#sSJJ*n8m2eMNc!H5TG9 zGAa(T*InBdl>Ixkena}G0Fo$Hc7^fIlZ*{o%^Fs`J9(mIZYR$Nn#;U_AGFwFs@Xa3 zqYkHA)oMiS;yiqHU2avMBiF%ZFHS2?_~jX9l%$Hzn;#?XK{k013_Om&UJ4_dtxz5O z>jfuX$8UhaRo?SP-V2U7!=ZB4x-O#zeDpGbj`0#~tEK&gkLTwj>vS*~$OouuD!2~| z!%n~0nT!Hed+FsHud8WF(pvbRh#qh;4DkOsN9PpoIk_FP%~QtkZDCkNVV;yWD2 z(KjpR`2|YW`uuT2Y;jz<@G~{S9P%O_$VwH~njXSi>v9%K!$HEOM8Kr@ z{;F-Zx20L}S4JY449j$^C+Qy^k^X7d>QQ$MED<#eJpW@YGJ~ zHk>$EtQ2LF=IWH;0#*RkPmF>aq@9G%9(p-Ra1vU8U7Su*15(Ah0iQI~pi2zp%NBpV zdYTi z3pJHsGQal-b&V5JegkFGlw7W3tSwH45_8|^{c}XA6PjxAqD3ri z^zQ+vwWCj@HcGGy!Oh+5Bp8p~4F@-{fSdvLftf&I!(v%oIeJ4@-x#`c7&HnH(}<1mDk7cKt0wexVCa5yl#sHVPAHci$#XER z53?oSaA!^bhu=iUD2fG`5weE%yD%j#;|2sOBuyy^LI+WB+2c7X&HrUa_EY-$*c?~m z_(2k}jeSn%ENZLf1p)XiI~CUq>DEw3TuN96Ew2t9lqZhpQTi=phu2Jmvo zhK(gK)bT=D5j%j8}u!asz$+D;U6IhTKw`fiE)O9{0Oud zfV#`E%%FHUaG1~O$2R9pCxVUl*?xMdP#U{?aThm0+$5~E__|5bIl0oRuiH`%3P9BU zz6-`yn{&dyE1`~mY3Oho*bNEk0sMUlF^Ou>*1uVjFbGXekG#!~h8ZqK z#=jRoyQkBymv&Wue(zvrG;2^OzfE}k_|Yv7)*F8d+E<#^STbwh^}C(b``{AJ&ydcebnP|0^-?8XG)`PnuW zUu{fdaDFQ5HM=fLgKGOFa4Eo)RI7jO2o$B8*Q}sjll7x2|_0|65Q5EbVw4a^}z`t0C0A>e-4LOa4 z7F*sW6L`{+X?&4C0k0de2U;pJt3VMxe_#!*)FV4B(G{IR7=)_*;Z{CbDFKF<2v~X< z_x5bBHNS)4)xC5(x~IjZTv2o;*LC$o4-q9j03ftW9jBPL8KCM*Fs{Jwl-K`3jHU+{D0PqNjl{PSH^g;>;Xli@6C@b0MQe+~iV?y{=-$oMxx2`Qbk802FPjOxjw-mAv zPMb3)AWEkYB9B{!H_+7mJ8afBj$E^rhPBjismw0nS7+PO)13Nv3@iehFQxdaQKaLz z|1OGtWY&O-o1|neZZ0iF?O+r!;Hy59KYD+2a*2j`ramy6q>_eU<4j zoq@^krOgMUvIt>PuR^nH1df{-0RDuwbWN0gT|-i^3^G0xWI3JD)O$yJyafXMRMgaA zlrW890q@p9)cMr#&SKzHxF|abz8E&%LvPEtms%;pQk_q7)-Cu-t$lvz-r+S{W>fs} z-g3{~jtL9&$&P>9GFo=l9bGKQvo{_a1|=j2=52&1Blzr?P|rKOu1)bmr1@SK@Iild z%W_2v@LP%L#^4?lq}33b?(*iq7v*a(aS_{lgEuJ71P5>%b6G9@FjZcVvh8rN9!=yZ znhvoDoO|#9kXTkehs)p7K(W0Sqb$cA)a!qQs@BrxU1D90JhJg^aLOOe(U!I19IEiC ze!FV5@yhq?MFQPyT6||)mZ-%0+M_!WKlt&ywiCSFR4qfXBM6fgH*X!lnk z-qx)7k|ke@vP=HV7!br)7R7+^qmYu6ntv%v9xhs1oPS9t@b3k*tD*E>Vq zSG&bs7lQ;Cu=ljhTH7@rIy(j!k^oF|f$TdQv%`>P_MbTM?@T=q~H%G^5ocK)CP7T&7hG}DTjUT`LO70PhzUb_>QHb2ftxQCO#z!AU$gq*_W zal1 zr|H|gdWNY@B?@*DupcJw+v)Wb;}tDEx`0Ou_3KDmAsC1CU@L{V=7oTjT-7bcay|U& zI~hYjl7g)1{BJfzC8*NVh=XFa&U^y{B|J%`zr$YE{L8P-TvQ0+ldM<2fip5h<&0m$ zz$G{`QC7VU0e=4Xoy9}!4uY&x=s$%w4|hr1xM&XP4rtqlQW0qaSYP{wT>}oskLL-g zvhDupn*7jaq57J6+-3*m4WF15+XmY5xTR31r6jaeKh36IQ=T9BszN8rCMBh;)f8G9 z;kKIK06K+Ed=VAvD0tKO(}}(x+EXWXJ5#adU|6RtJpPD&5Q==v=H?mA~@Hr1`vn!ds{mlID$rlUzf@99Mpv<#ljiKhp!gy~HhJRO;S`Pf^);wuy5 z&~sGJtK8Dc#Z$laA*d)LoUilexs2%*f@5`oXM}xZ%C5@q2?JpAh-++W?P>`gtITt`zsF9d3QR<~+|HDVShSm0@gsYWdT9 zlTF!oG&|-{w|31SacIXawyL4=K1a+C~PE%^?4yj;$FK3uB#L8 zDhu-biM^jsxCf=a7*~y99-cs{%O<(1cHvks?BHSfyXllm&x+|bTwVcQ#}5FxRlo8 z0+3H26{qN-OCx}}jVHbn4!ZwJjtW{gQs&dYXIyl?~rTa@I1suR)Q{yfMIJWn6dp8c_Wf zM{{)yb)JrM4egK5F5-)O{q!Q|V5l;4s5bWYCC1`tJ9vQ0M-bUSBrW-)&HOR3Vas3e zEzji-jls!LP}D89rojND3O)M)_ScW`2Va^3{a0O5&!as-$aHSqUvGzGOTJyNWfX3? zFMUch&FkR1%mG-bOsIvF+L9Ovq@F^>512Na>m)GgS>vcVg~wN`&D2z8zS2a#mV1pD z4bwZ~BmK@9AKGevu6XbXJ+#GBAOIjA&@D#}gVyZ(P7I#Dex4m0WF$Zi&tae;JfO-E zbz~pxk8v`Ohn&eBY94_YWbNI9qkj@^8|dDx5^N9iGY zQ6##`y>(QL!q{2Y;VAE)xQHhV?brGmPkyz1%EEEriZB$Pw(qj#i@DLXz4uYaV2kYc^Q*+je?dSU^?^0dOPh586iTXjBqV0It#q$KBXGBMaTNxLM?O=^wJ?^>ib` z-NR;5?sfV?M6BK)&jpe54*^GD1LHoO$rS~*mVqo8%C9th9jC$g=*}b#?l#uX%gJdd zVv!AGjzd;3VbULrD4=vmD4m3TcWYx2xx_%RW(p!>l)b=Bjp0O4Qbzmr2%3v+KXIR1 zHuk817at_I#%CrcG{7$eF|+>v;p!`c;@YA$8$z()?oMzB?(PmDxVyVM!JQD?-QC^Y z-QC@tMyBt*^J?nNpRQBgRbAD6*4}%aZ+&7X}6)?_WJ+R7<_01gG{kWV{%3Xf< z8@5_P0s^<`_AL|lY<;i5CZCkY<>bR`m`{?=^gtVTt_M_6?JQ}()LWtKHWYOz08n(B zGPL70^~2K{yd+H-x2m?EYFRqu*5dQ<8#pb___~Cp^k_<*O${ciRz@`6bNyX6P<~t>>pfH7(^a+*wnvOa#>3;sQ z+@cV!5V;tx_N$D_zNzxFE^0O3ezze`N&!R-ukF*zFj(xOE$uSh8i~%^8K48(Ha8-& zB92h$XvOP5aFU=%*Iy7Z7;A`c`ft0XLdSI{?289hz;arZXr${UyPlK|_bC7J^KK7{1OHF~`S zkI6N`1DMonEfL|HLMH)Z?uLr?te?8pa~C3tsOzN-QOYcZ63(6XVLehb$EOD~^9XiwXOfnqD`RS;wlX3P zWOjs{)!65PLbJJ}lO=?$FngC_C1#RtFP|r@`B#L1mFmrCJiP}agpKMjgmZ&NxWSSS z>*Xa8gQRM6z5LDfmwx0pSwNH$rCR$(dzpCC>Ctynr>E!{$KCWrAA)|WQ>57ORR43F z58>hPUjQpn9T$}=^x4}Tzn#e|Pp=9P6&m(~_p2rSm~88=6VJB0CL37fxy)Le_~Lvf z)1CJg1VENBx_Kj#%bld!r_5W#VhzeKRDQ#x@b!K9Ov(k=4p3S7#=rMfyoCkQCx0Q9 z4xWOJhgOJpk}&tAji=($OksU++_efB`N6nS2WnBe%Z|B4rCq40LM^E33LPBaei$3- zZ0o|QRfTQSYz<4=^D2xF&@k{f&Yt>vXt^eoc}?y;WJyPCPt8pI^%(4Y7i-KI(M}u9 zj-aTe^-S$b*UC_YuFgBr5k)4Z78%(ZsJ8}JhpwZ0Vm6>W83I$Dz)iUD%VZIrZ+kDL zX|s^o)Yx~6FEp~DIzz0Z;l=(dX1}jD2OxX6Pjv;~Nr&`t*KZ*9DPwM-xxQSQ&7+`E zaI@%=&Jz#A=V9_L|1So;%Ea>LwMK7%cQApgha|y>r4+(SYr#n0)a273(?&MkB4*vn z&$tN$0u6!QYHOFX_?NA5tE7A_3rRVfI#B4Doc0#TTrMf@ z*}j&Dx=h=7Lz!Yb>O5M4G^O6J+3Nds34Sy2ATb5m6Wy7N=^{Rwj*z6Xb#bu9NkKD@ zI4Ef|yvdE?eq3Qz474%{DrmWNjgxan?L=7sF)Ip>?-N6aB?q4((`-^!0BEciC3C-8eX`Pn$NTFW zt*Y7&!*Trayt-27ugfx_h@ZOju9)dBt(l>-2gK;tPET9)MFf{s${a7S(VCrY-&~$4 zwO3KGnVGKj1jdV`>RUULo@#^5_QcVxPy(Wm(CP=Hiz}?sD5J3J^)|^So(_l4E~=_w-s;CetY;{aGR_Hdvl7 zt<4lhFOP3cBvDto6h{ z0Q51r72$^%jNBTGRVQnrZLCvAR%+v($y13bXBf!M?XS%{;~YjUp4SP1_$5=9{9X?g z`IRLlC*dIsu{`DH?@Jk7zp$rxz1H7Hndc&L z+~I8c>r<2*&(g512>i2=(1@U6^;L}XF`qJ(2{0qs#BUA{egAZNOs0Rzk#b2CP zt=%GPL2)ktl3XT@0HLZfYmy1Z*KZj|+VeAejNb?b+mmgw-PG|tf4MQ&SGQTmz&Pj)cl8)Zo98--#mXLX#|0bTZEYTRgtR+dF^ zf4&UFZ7Zgdh~cks1=3%(?De<~ZSWuBdL1Yw_~D#JtO_!`ygk1j=@LG(cT`%>`=N=w= zj$`IiUop#HFIE5`;1b!>JOZzQhl;Bp`deVm*rI%SOwFdtAvZpN%ID>I4V<^iP;5N4 z9+zOFP7j%7q8;2c!lmIt0}tW!d19Q?(h94WpEI?%;moSZYQ2=+)WpdOE%3PVn`umH zr6PXa#{82vFJdmeyjEeI#me6d8d@sw&hG?y+p}?~TDxe#Iq)TOh-pqcts-FOW4&T% z=wRu_CVd31kgV;8fjc5VK>D#u?;9?36wgSeKWyepm{SSf#JPg)iV;9RM+Ts)1pL( z75qHo6W5Wr93Z3N8n_w8sPSyXs6gdemQDC2qA7?6-(aNyIum8b4x^Hqe`-vON8NO` zew5clzrO#GfxN%F_@bBJFer(kwffoY|6^x=VsGvTVB6sYL~1kBw#ZmwB;FmA15I3_ zIO$5tTwe^bCsLkl9n#zZSP5T43@<_)&7z$739_tpkK)#nedbKZD)PejC}Fp@H*a)V;cbcoER!t(k@T5W+~rJAM;JOey1=A)W8dvF9YJno)>3`trBHdWeN9 zG&oXL2dwYx9l1+#cyy&~TKQxhR(d9K_4$h;1Eu-49lD?LyH_@b5Gy_L!Tv5=9Kj@{ zqZMV=HrkeW;sT7^g-rSq5IWyF=!OfkYzYAh938L_+^;2^72hdjA$Zv!f>T7XZCx>a z=em%^h=T{g40Eaa@hwN4=v#Q36v|DOq&zC$5;n0pLqjL-+jpw$?DrN!WjaF!5t||- z8R?xhJMIW{VR8KHM~M{2862L7cq}A+$U8Lalb?6*X#V}vZijA7OYk9rBx$Uyo=yhvmodfF4D;#2i6GG7%ggHenLu>YOmyE)I1-xOp7 zggz>3I_Z)3r?5~gB#b_;Sq0Y-lZ1-^*Jz#6h&xNrg7AYHu1l(<<-!G#oNyVmWd`A8OUA+irU+t(R00?}o~pjlq~dhu>fDF$=qv`L;f^#Ujn; zUvn`*HiGP8#u;meJt;YI@@@GYq|voeTuoYNK3{20s-q7UP}l%OxvM5BsI$UT1y4R4 zg6553Gt=?YcZ6pjaWhB%*3iEE;nYv=;}p#DMKAWJo5l?dJNke2^nnE`)y$2Fw2NZJo_?x9fvPZ1*7wN}XSl`cAr9=?h_p32bIzbZT8?aH_ z5b_F`UR18@EHRaEoBueyZi|+z+>)fn56TrmlJGgZ4ueI7`!nHG6;zvtZdQH?NFZBL z4jPzPuYI!!THt<^fLGDeV$?Z_TP@wHPA#Qwr9<9C_+MmV)b?cnY*mMG6=cLvW&4p= zHq3PCT3+QW>?VJx({0NKNuJIz$TIq$ji}>JyaYc`u+JEWDMG`|3ex_thvv7^8Y$Hv z#K5Tw-oad|a@WZ~bb*U4rx{O}L@x+xr#O3Ep#{)}1zPjg-s>N80?@*iTm8r){9Y!G zrEFg9xvv<&(J=#=7I=-Mff#jMXB#gUMO%rZ4C{^Tx6$81y5F~L3DU_pRWubX{5uZ@ zspzU0wu~p3pt%+t?5@Hnf}8*<`ssBy&S9LJ_F$w}w#=MQdVBBUVILJsYc^x9Wdp&$ ztyQ|l@T*_=%M0xIaH|clVFk$YPb%lEb|3Q=4?UFFx}9|=NmIcJVy7*yTnyyxe67Zx zLHVznJmlxWYbqHv_<#h}Cfw(aGD~K=>>22gyR3-=^8Yqwtq-DxFM0WUm>2oaH{*OU z)k_Bqjqbv!IA800X%ekdE7x!dxMMkm)unNDaqt87l7ijz1tSWOPe87psxl-}wP zE;>d|n0C*cjp2DCXmg6qG_>lBLU#W#7G;X&)aF9nHSro+A6gHbRgxz$yX3hHylQQQ z64Rh~UcGO8mS+)#A@99uacSB(2}n>7G7IDPr2qekeQqU9I;HsqwdExnCH^@1j*s*4 zWf+r)K|UD&ooRmj5*K=yx-0tO&A>}OWU2<^2G_)OjtW5t2tN*uOo^Wa{Kgt{cH~T9 ztS+bZf_U3(m}Euq^t%ys?rUV=Az4GXVQcV_qc1i@KsFU@pO~S@0XcEep9w+af_#bNH!%c2w#Y)m z0$P6yzicf1e+3Zo5-^VdurWL>*l4X9Ecw~huq~Dbkv@xTZjnz)rpRLw8n7f3NGa7D z?R1?EC88*`V!q*Q>cDll0~fY``%yX={{{V6gfbnYRlKDU302#6zJ)<_*OGEUI7NM(8Y6>>>p?5 z1+EvID!ttuP3e+(J?kvY>^PpF4}>?e6DA{ETFhmG^4dEuYO)uwn8wPeTwiCGL)u{eZF4?FKPVeOGQ?xi5q<>#2SAUarDFLnk*(uHa{uoIC?#Wd z(Gd7M-3Ip9W}e*~fOtTc@-=->$%Nw@M^X*k(?ip*jC{bD}+J{N~ zB49NHq4T+ZYEM1UywSuTjvj z()(mBmG$l$hcm=xtJ<7=`D?u=_fVo}WmfQ7>z$_NCJJsxA zVKPQea9d^Ix+r_cTu2~F#)w!R9F!?R~NB>(t!G_bD# ztnf(if;(RiZDsc}S&oT?gRZz(*?)V~H{YWeCGH>LG#AbZXB%Z$Aj&6J%L>X1D{)hzHfdNSh*bJ2(lKEJ%Btth=eF5_PVc8hDXlGw;?@pU%cwQp?vj{! zpfl6B=q_EJFNw@bTby_GYA?9SrSTh{<5M^Nm8Yo8E@j!Zz(rE!qbUk5SH-8=fG9WK zBK5u&0!JJX2frxAdg`OFkCwF73>tyzGg)iu_Szfl!5uWKTMXMEfmtJz3z>97B@!{LGj=p zefi~q;T(gZ=bfd1tEr%IEa1NAZ_q!F4a1+{zKN@X)9dH0D8PBP?`bfGUo0T|d)Ayr0_`+XH zc`R`Fru$kCVc4#>GWF$oC8l1^mPawTrM6w>l|17KQ_Y`+V}V+N)uo*jD$ZL}Qs&#W zr=7m9)86oj1a_KySuWhqL6Fd*WAj6nir!WN*BM;a5xf|NKXPTN z4w=zX@X#b}{-Cwwgeg5dp5jCBtNBJ!d2bgc@VV%0F(%8E=l{Fwl&PAF()T zlls2az`&a6ptmI{D8w&Ucd-=s+8iGTYR6?JD^C6PE&0E<+85RL?`jXdr_+3zq^BBD zam?3FOYwN04@_UAy0@=|zso zp&t9#crGg8-By%G&fBZQzDwwh7x{+LB!F7ddh2>~fQQ0@<~8xGrlYQt{=nOWVKhiP ztIp?G^ForQU-}b%si~6t{x?5gKmw{s<<@}(v~elWxJoh^!$QgXa3Z^YYvg7V7Lj>* znL!UDOf>IjiE|by1KE?oH{!^37ak*moR}ozu7*s@1un27-D0Ev8gSjOOa}R{zO8Vd zdqiq@;R9$U%P9YLm7kw4+zzg=FLZ7~8N&Z9>KLM5-2Xi}BN2b12sQloHZ<}r!|%kz z*f;s49bfp}W9!Ve@Adg!RJjeZdV*tg_G?YaDamTIW{TLv{9nuJt)4Pj|F-BdopBq) zD*k0YN3{7#{~WJ!cgD#=8J=3mQgQmJ?xUQZ3=u7A+SEXSCP1i(0V_-t5hguC4HA`B z@R|>%2T7}*o#NUuY3FI&-U=^0O5(Me{gzS^bay;Lqpa^g9vBm6ti8ka%^)Tuj#0Cz z{aM4p+SSq&fb1X761+KkF_GP0pJ=N_sq;tWHtS96bz#uwOHMP=?PnCf`S&qjxbpMj zuHf*8#%)Pe4tRCl*-|yoCcF@33NAKiwFb;@)wVMDHDOpq-(I`=qcD;p57m8Xf93UI zXT#)hJN~Fri-(x_ul!8P!OJRMQX(5lsCK?`AS`Jp$6$~gB>-?qJf#4E|&Y_=CgP4BlnZ+*D> z9MaTKIJFO`E}iR2@Gs7XhV1!UkYAwALj{Y`^M|~kpd>Q!wKdV@cT8*+n|S5(WYoCa zZ>s-H2Gka}7Wy2Iuih)<3unNlYo$BhO!%Cgd^IM(x`*lRZqqWh_dUA&g>kODEY7x^ zx_o}C>eh%kiJ$c7C;CLnw-L5HI4kj&vNSr*RAi(VYHjjUj!d2!-aM++u1`OzG5=G z?mK?5OI~ncBIcN<2n|p$`Krt4)jDz-o4n%g`$8HV^4IJiC<iojz6K<+c3ou(?$Y>EPM0kH0ouIfgRmxz10#d55)2DIB-+An)Dv6U>ii z(=6)eK#RPxzhyf)@G@xnisO5fSl(PRNt|{2gqTUNRZT~-hUgI>QD*2P-Wk~g5+eqQQIM3zuv2(m|NZ%uCR$1%qjpNkT)~x2-|h z3Cjpoo&Rh$?U>xtCe5F>wLv?t5jLQ0}%|$F8)g4sM%y%hD1U zECxXt8wDL}0;C^48|LzYxg=Eyh`wq(4kB`b!KiAN?L<&O-G6#t8%}0+2Wv@KwF)vZi?)0ik0=6P2Aa+D{ zRK907b`br4YJAUs)%e6)eyKdoLEkbozEqJjPcBrmtf`STB}(FP$s_`ZNQC>5^5Y@P z{tVb*)wYL+!{YlXr)B6Mc2m1Mu|1JgSr&azDG<@H1)8_Zs~|{Um||uq)NY+jM^19q?|v0b>1kE;ph3z-#Hn>i zQk(0HolX)OXUBe;WLCsP8}%EpRD@%xIhiC$(&{uFIrUD}m!{suVc6#W>AR~%r?r7M z7q^MSGJmTFZ}7Z&ha;gw-WYsWsTDxzv5{%lL_mF^IoUe>$&&N5(Li&XJ13O2m9H*xx^t7!~~1i?ud zq1HD_pRG7wo`NKVj|Wl4=pB*V0D_ zIvxbr_Kn+mV(nnG)FfZ+AW>d;vz*VBFFyXvWmQ*sF={8M!YxX>g*L(R3PqQVxJZ%%ZBALXhl1obyAm5gY=%pKICB~+yrs#$1(##rz?t4U8Tj=T0>>dgShK_g_tgF9@~`gGIF`X zU4DH#)t6sf_AGy<60uz)-!B`D>#v(x&f>=3MSQ%RtknZ(Z|a;hu!FVt{&Y(q8!x8v zx=*jUS$5?{*_#$ytc*#=CbuUwVB3p6NuCJxY>9T*h$Jas2Ox0XH96yg0ngVdUj_CH z`#1CPc1F?*Kr*P`~R9szJ;#}p#z+(7%`Yi+Yxr;W6bdG*fXnZ9Q;B{^8 zEIV~e7s6>?{iRC3ej*bP6nqV#v=n80r1gbgbq|6VTOEc87T~wE$RcBk46LwsP*?O8 zJae;RMd;^qWSi>I-SSRZkO~=Z(JykP&+9l)Uv*nsqtKYhLm{KEY;!+B2z6Pt@udm) zN-3)$7pbbDHu>r^AW}E}mJFhD5Hu59w`cr}+B@hdHiQ_txih7n)|iJZo#l|szR#+0 zm>5okhpl?~ICcN|mGC3|pSq2i#cv3po|A{5MsW}SwophDjtZzAB_zzva)s2Vgu6*`&m^`}B~J8GjuF_B>wXiJ3ack_E`}rXCFe zN$WyoS$Q43bnN?Mg)f&-&c>M(!i$>&Au<9{f(FR@2h2p`NvMg5>4gMi*|gdGe+CAc z*Dt;ZXUDdg=B&f4-=1F`p#8)SS)#ZCHo*m-2WN{nk}}d`LX#Ox2=iW$&6O2~V(Hfj6yw_tQ=fvx*I6+v%R+oP zGv-y#`msB}su0b&_~-`1Ai7h%iHzs@Y%h8!RmTrqKnC5|z6udvi32(&y4S?dN^6r z!B%?|SMhxqNFn=h=dmOcWWTeC(d?thofKz8RqcNPP&eE!d(W+JM9>XPEJ>nYSR<`)+Y<|PcSYJ1 z<3p5U!TT{;^!(CMS+9Fi7Fj`78kE;=3G`&;sll~z_@t8X#!St!Q$qK3PyryRIBPrUHuvW0SO@^y`{!XeRDDUz@n+L*~6|oOW*x^BpwcKU~569 zQ}+Px+fop#SNnI+>Q>nQ+r?G&?HkyRJr|Vf&Q7JO82<9}1aR_)G-;eW16@}9hk+of z(IGyMqnU@S%7DX`o?V?oN9H(wt%3ShJ9ib+Uk4f9A#L6t_!_17^D`Li^wzOeV>5NO zs%#&2rxO&kqypL#4Dc8HzFA3U7^@ZU+`PI;H(-85oU&KRQ_H-xLKVo&X78U2R(H^BVNE4_wyZU?@F zvsFIQ=^t_$bmn(>EnU@Z0SLe^Ic74kI!0ET@F%eeX?RfdjQ^1i_kttOqz05$R-CMA zE+2QXzAFPJVe`u8Y&2}vMq^XmDYuZzW8JkK4(A!S<9Q`7^KCS@))Zz&siEtzUbNYR zsZnOjk_VL}6$}+j3ZK!;PKXQ%-)cumKd1*p>VEN(JCR>iGi+cyLKs|(@8=t7++=YD zP%%b*6Chx*Z8e&i18%7Ux*1@XG;jY#0=$F8lNkX-D0drGgz98p{VtmC#j=DziDW)b zTW&1}<&cT^<&31wp4j1|^n6~kR>Qbg7ptnN4Ht_)K+$r`*E=sdP`DMcwgm0Kk&N{` zF39fiCf!_vEfU$VfsYv8LX;fJUxrcQ2JEW49u)Kt(9cOT@C{#x|G)r@c6LpPr@9ul z0odvV{d8wyF}PH*cuKFI6qW3P4yOcHd8>Ogy>u*i1a%mo`OP+lDDXlDHSb_sAH&Tu zHR-qVu&Nl)8>WfnJ!F2r#-YO1;y}4@@H^&0j2!gt*i@%ATYE`#H#O5-iqp*c2iPSf zV;b)swx7Z=+V&5wSWRnRm5erL0_EHC8G1 zeepKe?4NfCZkDrz*jaA;(};c#yu-vsJE9a#&n)lL<+`A}2g&}__%X3cKEnK^H3Ci~ z57W&cnsQc$lOsYSLqB8PCKf!0i^$dNuXDi8Fxp~I6z;px-y00Y_hA^Dnz!o}45kjh zxmK@-ZO-#sbPd$O4Chr`#p?!yY`vWkb)FR_p#Znt@HW4pj&8QZ$Yiya-&QVU{pz+B zO@FAMd9UhC^x1ol-u~!T@>;;mAZ;KNMI2J21x0}fVI-iDlai}GmFZmE*PYDH_NJHC zmg>&(hdR``;H8g6NeV*$lH}y%Y%DkPCMSiK92ps@Dd<0oRv*E{M-24!l5AK6>rT;? zH>cwrGh|KBTu@R?s~&XNXROs|390SWb#=;eA39*zFAOU@t0RE}I1pyG)b(ufiD(He zk^Kn!{YBs*B+iBKcrS_p_m3_?E!@cdZ2=jD%%4*F5r(|c&|4-ZKJp>uM9lmmlQ>m8 zPJ_;eSTh|vAM5Oz zd*yDx8IQ)wHBX8riCu#R97uHyKdX(_MZKuM65064=6vtdEFVlv+&dH zj7f%3GL|!)r@E?{nj7Lo3Yoe__jc={gz7fCJ5pX*QZRs2FD?6kxTfdI8I4`8HNpG} z=I+7>qk8jdyfOhhWF|A~T@3voplaj;FD+{H!PIf=ui;jQ^pXFMn#i#OvZ2lhQ3?(1 z74EmXJB%gVC>vToMyYUA`b$TyA*PnD?2(FiRd%pRTw|BBdBD^HFiRDXy-R9}P>B3n zap`84RbBVi3<3a`pl(n7Rt;N_VY__m|Auy0Y5tCSe1WhT0gj2shea)b;*nul)z`SHdh8zk(5-U+28BITz z+Vc4Z(;2Lj0mCY2GSfq<(Tj+vJ+7*g(-8hhC300&vXA`Tl|k0W`W!_iXZLp4cX8*X zE%W741(8;&Om!h}xVz-w`sOzI>?W}9Y$&4BO)=q#FQ|ggG1*Rnz_q`X{AUT~rPHxR zBsveT<2e=4eXOSTyNf(SmqGCbzEWAX{duVQCsZ(^Zs8Yh@@|DaE*LQXX9eSu)5viW z3_^)73p^Rh1ZgWGZ&o=*J8A=n7~5{JHN3}odmz>M7L-*kt)axVtB8nn_bPXGI z!werP`kr~A{51b;&B9K?7u${&%K$WtvJ(Amt?y8NBILP%Pj9+3z7v7v6!_edrHfxN zo6=?Gu6m;kF}x*p1=BC}4JC5>+##;~l^Pe}xkv{~tsHW0dM?lI*eYpCBy5?w55ry! z##Vh=H0mqQF4O%cBirk@gj6HWg(oZ?b%GdYXD@yFH1u3)d>h-Me{-v0Eq%Rl!b+RX zs9*9TjJvT2i;PDxUm{Fql%k=y4u$`$sU)l+YU+-M4-lBiTE9jVjU>=Wy2|m<*eH7cl_QwYu{?ir8*$`E zyvkN>p#}WiC!#7IwY7#Cd)}_`AsE%y2LM8sS;_=}U^%V9`k$XDhiCyzMi@*3H%R;50>`3{=keix*oH~j*j*;_VQ z=(d7W80i|nNHW52==wYI)qlMGU5vRFoX5tVyP^N(d?O&)-BFt-sxz%Erpa+jKS*i){B< zUXgtUQd29&Ek6U{&|lQiWf0OP3~HDS_k5`3m>&7xlr|0-O<#ZC5lEARCF1inMaCv! zc{0F=0-&ALxpb7*c|cAP!yk)3qA9{hVQM1AmH`SkH-bzvUM~_mn2h7&$yT|SPG46| zSMa4vY!Suxop%rYr_qy6lT)nHM%}lwPwmdQHNC=XnogS6yR8YTMqdC5NBR(CqTZ@! ztW~TkZ4@=X`{|AJ$KU4r`!n6Aod8|uqPAyhq(%c1o+e1U_?M;!3+XW|cg$>APqXbE z&g1;l|CFbalWAjsx=PPiDq><3fq(}V^I zWBYwd-30R}^xTGWqs|f|4#>Cpq zab91pWC$Yxf*yvh*kEJ|Y`ErqCIw zb7t#H04+0pQ%w^Eb!zZX3R@cl@Up0o;LiuJ2%4+J5dIHj$*Kk7E4ImJ^&hC~kLIB? zcR9B2%=~y>3&vVYCT}kPJcJSEac9!5|Gj!yp8lN|9O_e5M@&9O^*|~pYGd!YxcDjP zR*;q3dL+_QMs>j=fM1#YLFePE5)O4WRB`P4)}p!{7%);Ii++ zX2CFX-xE5Z8*<=cQ2It9Ac>p<&{P}$?LL?$XfP!Z!s1c7!y@N0t%iEP_0n%pWdDNx z4TQA?CCx8c9vgGF9J_BR*lSmzg2F!lk}mglT!8U49#ImK_9#?yDCn4#y|dcl{a(2$ z7jK%I)GOpmZjw{+%VLlx<*3@|%Q=Dt4hhfCGf+@>vPc!ajkdHkEwI(HSfkn`Xo>4= z`usb7(6s4FXG1>}(6*@P2bqYxRNzK=+Uv2(%if-(rURl9f_I;H5F^ zgy@|<8OD6N?2HRapCeqpbOTKbB#Xw+_H0RhwanBN(xCB2NeGFJEBE^b9#p5T$)Oq# z$|FgCUU#I59XD9Ir0UKV0Thssvt!?nbQjT==1@b`mn**_SS{A@+2 zR{8l;)+pNd35_-xTH>-)vN9IAu)#^U_lDgjc^9JlnX~!NQ(-bG*;sHQnbmgjF04Ha z9>k7>w>8q-4+JC#B>&J^lrMKL2R?Eo-Blh&&Gm7j82-<&0=h*6>G2|F04VahdzgcM*dHu)fk>f88M)Ogrn z%Tb=1WrtfOsKOTK`FAnAW(T0iqT_q>qYNoD3bp2iR@6~!ZaQm4m9f($oZnBfS45m# z4l4tcF^ClhP0eJKxw8`> z9dk_B8ao--dH{m}O891Ab6h1!*+x!W?9>s#s_wC%*^8e`XkC`$ovOib7dmBDUKn=S zntx*!gUoFl63d)#VH?=nPna>qiwSHWxvU-PtvStlad)@giYMW>-9mnz;Pq==ADW-4ro6 z;Ca0FifW|GIJN*4w4*sA8Jl8f+^xUKe7)~hq6#G z!6DuhTQQ;IRbO{5wXF{88A*%ai2y}RPh)dSSF@JBzRL)|tkP36FRO$5Vl7>xA^hTZ zuenFMyg!q4p%Sq8IuRvThL_0b=ckI|#lHs5y7s2K_reSPVbMeC2vUN`dZAj*qpGEv zq%S_JLfZLl7#vnTI#wEVXp@=bOh3MxP_+^WX&Cpe&>^cGfun4uLy!v#S&#Ci1 zsVp6ku1jHF(|-0{<7T-pMd<^a!!3tZ+ValdxZZFp+a&lO> zk?WMkB%K>+OG;BH2y$wz)HJ)!%B9^#^KERSy=m!bH_?*^JPn=ovY^R?+N$}&e(4<6u0MuAmK{pq<4GRBHV z0*)-_ZKm|%Z$D$?mjGU29O_GQg1=B#e>@mdiB}~Oj@lrR%-?311 z>ho34?zE1BHI!8+y>z<2B}o1cXKxu4SJ-yxHtz1hC3qmXOK^AB;O;Js6C8rO1b2eF z1b2c<;~Ly)oauMIGc)y7ojE_wzg^YUQoA3ypM76zEh>Pow$bUr5faZ^Y}#R_I}4zI zT|J$lNVKxY&eTKQI(0S5p|TM^MT&HOgA`XkM3q{N1m>*Hh|t#5`^eOAq_1w2f!kKe?o<{x0*PG}(zKh?qhOS)gzJ9hrRU=R3XB}G0)c@JX1H&(d$wzdb z^7A%d@9vy{L;#~r3(eGtz+3SEg7TDcW?zHfZ^yUb-V_r+Q&fOFGc@=R(7WzYX#~`KqrSlpRdZV0P0j;aP{(| z;N-=oI8qX|R16I%Pv=#Q2R%*5N~Gijyk*X&mHU&2V?@-BAh)p?_KWY+zw;)iEw$c~ zqoqz$=(ft5m<r`NGS9pM*mUt&?wJM*DlAo_XDl<$WwOJ|5ID zFo4&}gtGe<|6ICky)p5ULvr3rw&wkfM#Koi zn}!UA8*1duX+p$B%BY`*U`e4wE$?31ePY$jT-6gm)UoY{@jKg!>wLF7idEvKN1MJa z-F7ov#dSirlOYmNdv5G}@^YDZ!F53^^0j=>dp!~+JO8KzZ{d7EiP*|#$qKk=n0{`= zb0U(Ge9K{=^(l@cVBu6bvizpnWN4>F zYLhRUb!(L>9Zb96aa8g`9&=I&%XC}1m}a_{JpIxtHT*&oRqN`x)rZE$*^v|nRcy$n zOUmW3ad^;3J3BJTb89qf%ZmR%d~4z77C41d=jXwUo_+2*;K#;wUK?|mNF0@VPCuse zN4#@Na%?F=(LEs#YomVpT>Z#C z4}6u?A%xf;-6*^SPkTL+1a(t!8wU2dh$J(Z>EFRiDzMF4zjNX3CM8^9~|U)Q2RL0j>n z=a>|tLH~L6l@iu=q!&5lC;dGExBTU7QDkqIbY+mED1LGQS2O!JzZc?yxYfPGE~_6J z>WA3-X=@$C3dxT(t8r?_3%&KP@83Zm;;+A4v99uqIvue;2(c%i^H*Ni8_O98rL=#h zK5U_90xfdaN>~cD-m$4rrh9J0_*Wf6DLA%<9bk@ThvdDsjvDp0US9vs2+7;)t(MQz z=^21Eeay?FXy@^&Rd!m8R>?eFHKW1+PTE4T2!;)H z8b6o6F(>JGzF$+uZ@6|NkV?g!kyJ)+)CAoF*Sd7?tAG<|ThSH99ZlP--!5D4gN-4Z zXe{{w+B?O93qmdb?0V~3PtA<3hNaxE?U5du~F>nrs<^9J4nxo*&SHHxowMvGnMTbXg|D-F7e5Ky0P<9oOwDhxe! zZ0%DVqQ-LXsw$^y^y?&-+g}2cx&TvAO&4!(%b!QP4q!iB#0_YyTvMMhJC5tl=Zb%i zXiT(6n`WLPEh*fy2eYfc0t|<|;p%mjymD1KHECHHPstmuUfm*GL(^V1z-eTyKvY0b z`S07cb6}`|BHzw-$PY$|BvP(#s?~l2Xhs%#{%ULlfFNF()28J}CO>;B;?NFE1RG-% zeRq$z=Tv9|$gL8ZDyyu7{C`V%#>@mzAAwJ;6{IOxFD%ba7h89F^qPd9`^M-Rv)X^% z$dM6N@_Et5Ac}({gcltdByB}_9oXW|WC6RzA=!;NN9)v`60Q?!N87(1 zsd=qE2a~}UJ%pEhVV|tBGQZ%Ki6OTZ73p;LBE_^5k-Y@A8v)^NP$NEh&EdH>#9)(m zco#ZR4APD`YR-K~l2g@i!=&%g#Q-^<0&~_fJ&pHm7Q<&|s%U2lY3h@69?}anhj?WJ@jG^Mr4> z-&FgzzAhBUsV!^Ndhg~zj;|f#OU-YAd2tRr1vYP;E@7IU{0>PAn-|%AFEHZ9bA8Q9 zEc2Tg_U$w_E)T^nsTYYE#w(X~aDrN~M?J~1%z%(HcB_O{qM^5Vb^b7 z%% z)j_(&&BTv})Tr8{Dg%Rp8<_AQF+R{)WHo&V37AuyQ zljm4z#yg1X9JnhtfPGcIc2XDM<=Y|CBF~~F#!6(scRaCpN0=$*lpOrSCrM-&>nEQN zPZFrQCe~wCBHNvSCXxN`Zu?BGeyOb6$lu<@z{l8c_0KRLs#bu)=&u^AKLa+an*Kgi zsZx|JCvup-f;pC1(v%Q8vvr0L`l>|o*6f&FR+l{j?|cV{7~__)5GwO~O-5aH%OF~} zpJ%Cm7G!h5O0Yi76JG8~&Dvz|gM&Wz;9Uc5cnM@EsBI3raE%3$5iLu5fm}6K^ilTc zIhDJbM}mIoBK8;(e@~1u0M&xBQkXfpe|Hu_*X{^YGkT8iZ=oR7Bl-AXPm5EG zp329UJC)r&56j9q;{>BuH55^jwqfURogufK%u`MM9=^Jp)A`XsWMRNnG`T=lEHV;r z)A`tvD0DbL@a)prrzVu&yhuMt`1-7N&*i}0`Rl3Q+5YDdGkEkJ((uAdEZ$aTAqKWY zBvG|21RdGAk(sDefT-yVl#$I%S=gXj4$Hgzx9$o?Pp|2^WJp_-(&Vxks^H)BG*<)% zol_1ijhOI#pYIfWxLGz_hA;{<4tRjYc;-S0t+}LeG=Ak zA9vhPh11?pt+H~Nkkl*Yy}1E7OreWpyfvTNI9W-w`Rs5*s;S4ZR5hj9HM{-S}5q(RRbiYT~U6 zW1x}U<6zYHm|j47@S3hkb)Z8ULYcMG&)xNR(e0$J4vHu+mpk|KQt#~$(yJ(XAY1(W zi{{A!M8`p89~*i~H3vh*KfkjNuQkwA!;t1= z5!zg>8!DORA?G3;^lCU+;}Wuexi4tA1?wbyZ`LMNd8VMe^NW#gN+ZztjCS-^#54f* z>1A*tj{HI2J3H@S^DE&bD`u*cK62q@8U7NngZE=Njdc0OWp&Z2&LL`Ut|Mb;r-2vn zw-Ztf^$zsPofjy2_IO#gn+x>9b~!W?@ZEOr!M|lcc{oHebGV*iCjLnLY7^9j87yr5 zx?l13iHTDy^MS7<2FY6X2(l%jG7Q43W$Acqy&o&&*%^F?d8| zTa}lzh!`0y-VGuN|OhX<(q`SD$)9_6M?Q8i;`V-(05HzQ3UfP28IZh`D|(U{X54 z5P=TlE5(bix3#m)Fc2m;TV7gVZ}<0HP1xT@6cM=yR^d3Ws2#nihg;?n;Y8Lo!4m?d z?*7!!$?@L$zKq5WnYg34^FT z7|g)N{J#h-eKn)i)HN0>zqoSLb?Oaqa#YaB%&@T=GD2)8V+GDV-=sOXBDE7WHorQJ zFvz~VKB;dkUG1x}6cplkoqbC58sffjlNtzEZJ_4$S3>Ko@Dnd@7$10^1MwY2+tNj+`qvifnT`43qO5CAZmMic0EJ6HOcgE*lJ&-U(> zVHF?hkSuD$KVDJ&%94=c%h6&4XuZB`ecz^tN_`%_w_4E!&uqO8S6jF~_U$b!v1fXJ ziHNx6o7AIitWfdOW`TZ}e$=iYtsdOt&xS? zm0m=w$7JVY#Zx1&s=6HJri&H>aU}1XYTNa23y!xJ;ZIVh6!d?0=H%hrNymS3six|* z4w7|lOX*@p>$@GU$#3>|JPgf_cuiUrbrq@eT-?fk<*;9V`Z2^I;op&XnY$3a=SCtyHizw!2jn)x5OH3%h@79;Y z<6cV*i@+*q30E1k1pU0sq?}1;4NGsde043azXF*5i;N>xi70Ou_|o4GS#)X!1xOjZzy|}AX1PgOkwdjEe>%Jo-0zB`@;0F_6-yu zRnPaV$L8WfGOb==dtbZIqd(K$gNOg+ca#hHi($qHQ zL>|62?W10Jo-od~s1!1gNw0#m>t?WAdg{RzKQmy(0N!j{fw%@VYP0PBYKv^K^lY@C{*OMfA)B9nhc14uXK5P|SY4Yi zfgK=4Gz!?(QMAEQ_Q`Ix9U@v5h|`}?l_tt)tQZk5f||*xiC>?w7RDHS!oqqm$-Yp7OT~+Mrjr(w zt1Do#M?z*NQG^Hlj(z;D8@N*rs{he914&KYWU9!G@X&;e`Lh3vdEQ$qsWoMQhVOLN z=a^)J;D5giyK}(r>~1)MLv3X63z5 z=YTrR>TCV$6Nl>MNuP&q~0(gr7OUnDcLjgP{$s*#4 zBYM0vJejyZi<32&fc}QX{K5WlDqhFbfhAE+G%|s_23E7VFm=umCE2Q$Go(hO62c zHiKH(G($EI5r2Yhk#AuEYi!}8>cK>+m2G4V`Zh@V{O*Edq8XP1WQ-(u^zR33-Yh z9xqjDfR8;uBMqTkmayR6sNx=!Ddo0`Xc4~ZH_P*I@vdFE3+_3V3nd<_X>fxps)Lrm z<3T;ax(NDGCa;xaTf*%Wm9Q+M9-UG78xgsu3p_$W?i5oeBjlj1{#pgGS`aO^cVLus=$p zojn8<$v)qdyAD=hz*yuF%olXwhs@_4@fSl=X;%GQk^V*Y_PHUW%Uq!B29>-3i*Mwk zd9zPpI1lw);8gX`1*>87Y;gfxBLSV_3z78^(Bmqn-N#GSbY>1?c11;}2mfXL#B}J2 zl=cX*$?eQbZVyRj7Rj!q+mYGu?l$c*yoP5~RL5WOfn^KJAANF2Xye`X4Q`Xfbzv-% z{)|b5WJ31KtvGdGWRzhFqO`uZc{kqDM+5S~ov}Rs@1R?Si>9)nLx01kc%RtnmVm1A z$-9#1oIwuVhV1wb#XvI~AE9g%K#-aweP6BR9)KRWAobS0$FKpc%1W=`iDTC49+G3~ zSfbZmO*9fH^hYw#l3tS?$y~pwkkt+qx%A8|X|=I7_|D(Ses4yp42*s^S)||iNq)W= zEH@>vT_YQ4?!WscDgsynQ$+j>C&*pjoxc-r)a&K@=Kr(`n=P;UzgU3QiR}0)>fh0m z?zD!St${(E-zF_*yx}Qu`QdnD4>yQ)P1@HKEX`MLL#8$1i~=5m4P8p|QAQ@M1xKb5 ze8;8lw0Xv;7&AKFk)O0hv4ic4b z{la!a5OC+|zqmgJ4S>ZWVpDPl`~g3mZd zF!DHYi1@-PHlnob&Z#cfX5V#q6Jw9xSEiVs=LE9)ZaXBi*q!pk2|X$olZ}m)Uq9n~ zXLTuVY-@3@5&biNmC!Lz5I0zY|B`hF1|+7p`i zvUR@_4a?;HQ~XaAe_>e#ATxucxLcv(ALOuX=dicege#aYF7gINzP-En> z$SYQgg2jv1EC|^C=xPS8)#UZ)DT2|=U060EqxZ8SrdP590v3Poew`&y zIvvDyxeMFDH+Sb$79SsX6!LqQ-NvvZDFOfiQ&z@BW_#ky${Cgpx85Mv^MwCm3b7$W zncuh9xZ4bFuc*#q`%YB5C_Ou8%Oz~EV%=7%k&5EMu)wV8={$lM%@#on>yn8zfxAg8 zM_XS_2NHey-7;_-V7)V`m;KJ4+~t)9ec(Yw3@8t^V#J<%>(zLpt!d775MlcNV%3@q zF86yJd#d0@t&i61{`krUgetCHoZAdOTJDfyFB z96cBT6;WvX#+Auh%j?UTFX-Mw_3I}5=kCARDJFU&{i`oUP1mTh-#(#myyzr=ak+4pt&^9C;A(d)6R_udP#{bO?)4=9eR%zOEB_;WNT888@q$$5bNDs5mNFQ0yUSltyQrSirJllQFa$QL{QWO8c z!`fYK{08=$pPbbRsGDi%R?L6?@1t6Qv@EA$b*k8rAA+;hlyvJ3m#4@A_GP6cVL?=0tetLu(1B5Kd7}yGSSbqOLj(f)t<_xFj0o1{TWXcQBm#ZP3q6ZFeX;qk)=kh z>O#J(H9$YN0G~72+%MtNWi^T(R9k#qFD{F=+PLBLy-Pzb{VKUn`{# ztFpy-lTzcH5ESjx6r9FO=&+^w0*S-tgB@l4HSETqZ?RC9mTovZe$eb!xMCj)p4 za^IHhw{7S$cip?w$B)6Hf7IN<(f2*3LZ_Iw3La#4yIm{hOLaL{NPB^^`878DVCojLO|QI)?)X z8SpanS?8FCc}Ze+r396vfa1Un3X`eqD_hCgpmKi9Bcy8$_-~kFoa3@l`<+}r z;^WMXlbV3e^@F^|nSM@T0uOr6Z^{YZ(1T zsK0n0Ss)P(cE)VQPi^Dwm&~q*t=blfpP>I6ZB{%1h1C1}5|L50T(hyM!P$ELTW@Vn z5hBp``q$L{bC2{>Q?lw(v}0?54=6uEoJ{xd+3rfyi%O2IS?(s)!ZkypEHaO$Us@r(w+03 z*t@yuV-ru8P$y^!8@eC+%JD}%5~R}YjD@)RNhv* zj5Kk+d(BJznH@Q(T2Kd@5rr5gnG#!_dm6LyEvmXy zUr6z?954K8U82z`qoOedG*b0uOAUOPOkiKNpCB7=4aLOF*6Cq2!^8TtfSbpaK3c$J zsNW8ltV@B}v7J$gxO+m$>mufSvwvb650?Yn(wg)d?ere+l? zDs6_65z@KN{@b?@dx3`WNWpHa;<2wf&_`SsxS-gW^fJ9v@3oL8rYu}n6nNu?hQ1XN zi(1T*x8XO(;&=OJ)V96;COp9iGBXQy)%OY3$lyNRY0*1lhYH05ZTz(zWtBi-SCuO` z!<|>f37?z5Owf^tjNNzt+Nbj4yz*zZw_n%hLMmH7%~&6IFYy8J)CuXuNO~0Y?-CnT zbxeAjr*9dMYU|drX`PDyGw__NpREFnjPyb1g?Ff-MVp|JeS>w6DhW7hW3rTATY`&CRefbaRkkwq+PT$%|qJKuSJc{g~V(<2V0!WI9>`7z!xN!F2LYks6in!8-`r8voq|Rzg~wo!Qly1pp1@( zwth_C`G>LpE7o@YrQsDC5gPHEgiTcN)2pjUnZ~xZ5WB(J7?pg&c=)Tp=h&bRz=v3K zs7yLJht2e9IQq z^V)-f*q80soJ&npIhe4Yuv!N%1tZN?EvzRm`*Hy%aHq+)wYp?_cpJFJgmq84a;c&F zzY|islXljlLl`q!zu8gKyqYhS$rfPfHyZWa43*^*79_b%F}8l76oqKWFRhR*asfa%sw$x+D%!swcuy4{C=+ntY$sO#BJ)z;QF z_HZtnEo0VUq@`t~Vq~Ocq?LQ}*<$6bhX*iu2OOGLwHDpw3mZ!!L;S=fo@{`aXX>E~ zy9D8{0+O5&}%=-0oB&e>m845KR>WpPqRhHAZAk_M5+HD3*bB4cOWZ@Ka)@9w+e|BPWw z;i71mPQDVosnRCPt8X?{kn^w|w!*SrUijTJCPViAx`3m~l5f*>r%?Yta6v5%=i%0J z?Ko-w!?N!QrHEO)otuORSY*7G8HiO~)q2(!OGse~%N><*a?|Xdi%BoLCFHr;q&3~G zYnz)K5~P4i%PJenRSUe-R4#im3Xdzfo9PJK|9N6Z?AD|v0Gv|D*w^-U#AJ?ELLwKZG30DE8rOvym}lrx=ZTqGhzZU=ci!bv&wIZ{WDx%EQwv)fTitiqqpYCm*x5VOaQ%6GMO94 z&E3dzJ*m+{KOM!i1<1s>MvI|7a5$&lUqyzbkpIWc@al;vdnkhTly+}coB#ZwqMp`C zgO~$K)J%;Mzemm9W<&(zxjUyCETF5D{(dL(x2Ko!F8h-j3SE}44d3l{-_;NCR==67fZMi@uBm&; zU!ST3llY%WNZOiX^^~%|Q5(H6^esJ0c~yPYdTy`fsFUUOTVdWGm`|}K!OHCVj(;Se}wM~N5~AGzU@u}B+?})7ySBfmLTvbzNzNeffj-c z`b&BG&O5E|?k_wCetvJ@Z!Z(6YAED=zG+@CcK$K4`gF}&xF*Fvq8gqXFmf~pQ8ylc z>&(Z!h7z^Txd>KlLX0ugZeRn@2b?y+wH3HFg$~Q|H*N~|MVO(_gGl)$WEWhk|5{U zn)H)oI`aN62+q&uN1R;6WZ?)LC|@}6L!Jm;G#e4oVvyf3ls5BQ5(WEQR;DDl+JJHE zP6JY4Du<$yv)$zfM9_w%;i?nSCb0SfsU zUqt$xs=p%2i8^m?wf@Nt;z!g_Ri(s^zS2#d%At-d1WIPW9RzmHTPiatu&xPzQ}~vb zpKSO>j)Qof9uxGz@NW2jShx>1p_WqY{|A3V{6`AiEdD=9I0(=efmu%49Zt_74^nVw zQ(4ybghVv=BbK1I?m8Cpun7eqNw}Ytb_$&DN#~)To`PR|A+95(&r2m{G#!z*YqHRV;!+7}1s{hb;-gJzS|Ny%Z6K_yjv#sg zr{1E?E!|N70G3kKHo-XKuT?YnfZFcG!*(tj6V~*MT-^K4Q@TuuqvXAm3X`mH!1VIu zS8pA#HB8WXY{^27D$t_(Y4X!Zw|uXr==NKih6@ha0WVm>b!L<ZFk4==u1*K7l!EtvB~MY$ z@syCx^$LF7{b^jXb6nOD?dXx_p~#di#%9Au{fpY&e)taG+-I*L0b5sl(Wes6J}xNe zZ4|bqmyhnhQ?YkWZlp8+n@ex?-a)Z0xY2gd^4JG`T+i;h>5PY{E{d7=e}aD=K4QdH z-s94DSFT0riw;Kex<~f^JUt^zj_Z%$3;=-fp9glDqZGnAvbwUevOaAgG>Q5t*e@4T z^l&?q3lBv?Fu(GB{{i0=HY62I;Q50&y^F%zS>|6BlUWU=jEiC$P(`Y?+hjg8Z&h&n z)zOVyEzfeD9$T1LMW5F>>f+VVSynuInYM5xY!jCatz1GzVw@yI~m!FdPWLzFIc{$#?s zp5Iayfhu3Enl*9fd7tA?gB*zg^wB!DxJgg)4;h?qU$o;<1GlY#kK&|M9QM1;Pmb7F z+@bnU#~!BghmdXcx@du+Rj-;f?fjOMdV?WMv|Y7q&^S>7EgGp&zl16S zpxAFD{sihLQojRlIH6Q5!^bi!l|`=iTP(4G9DgYy`W%Axc?XU+oXQYI@zSMYcyqZp zs_tfgVZe;2@oRqRwW)UokDw2R$|<|@wYj{GOa*g J`w0{_S*2eKI0DBdY9Z3Mhb zx~{gIHuFp#UA*5-SZ=}|AfY%Ij;v)G-JrdGWms`~_VP-UOq_@d;po`=1^N1UjNIEA z(BmrxB^P)>Je_$WkMP|oXvOO!WR==D$Htl*%SFg6R5jd7=YwNuq#IMR-MU(M9SaMyG}nKi9t=RMY<^VCDC_6c?47;w^5qFRf*vP# zL>#mjNzCz`0iMf4*+k5k^II;n&4aTixkFlt(W*e-2s_2AHK%@DMoA3fyRPUTi zS{=Nvd3-FMFBA(~ZbPIHmMj_9?l|G(W#n?|D^YdH60Zm;FRd;3=(%DA7FLwDW`RMK-&jR4nNUa!ow zZ{VJdyXWC@9UtDA$T6-MS(kl1zy ztA4WFkTcczUZf`!EziWf)lFR>^zB~axm5xg3g57?lUtH$`<9bw0!iNev`zfRJj7hj z!1E_<1?QnY3aeA(M&o+Z04KFg#^Z&dO5ocZooYoY_>22f(jMOi8l_L$(VKH~OD`-} zW04_jqz~2?dBYwBvE(th@Ldab2Dv<+72=r)*iGrPd)I0?H&L`Y=J@@=3$mmE%O=R9z3T z_MN~szO>VZZ)eI@ULzVcc1B%SYyO0T#U-xLkthfBTCikcQV~f4Jc}A2gZR6O5(Glj|>rzC6Og6&yPc2DMODGf#5c0!|%nl4Rhw}NCmdw(-E(pos zaVz2>EUjH>- z4RD`TC-l84%3llBF+9=2hesY74L4i<5;6)fT|s;}cd3U<=^FGSVX?@2!An>$QVZiH z5o*_CBSxgpW%EgT9zM-~-e=B?qDVuALyEn7%HPS;$vCS;+L_Cq6JCDpyiJBs8T{Z1 z=<;RmXt|tBW$IEI<0NwOGkCfjx5+sPS=Xb|H5Z=~3VWq(k;$`kyzp7}80(+87b7$G z>g1E!U+5uF#l})!QZY;9Pr}!A7&nM3lvd`@js6i}a#)dyOPM5`qVw*w?_!*s{QTn&tkL)JmP+UGES zqx&U_O=m{0YM>?;4G?1~N_)DDWTdHC^kaSwh*`gpk!bv*fVlc$J^bgbbBo2rHYO}? z8`s5j(+2z(7hl(EWAi|R1j=CJ#`P|?v(%D6?*cb}8|Xpj2HFLFsc^w$vzQVN>0cJk z7L?*LU(n)Qz=;)`$?|W~^Zsg-R+n(tR?iAZ6ea?ZA|zT!yqZ-7j<-EWg(@d z-?7MxNNk?I1F%V5(NqwN$8Gn|L;svCWROYFcz&Eu%oT*h>bAqzLkrsww z>-{Wrk}~4$5p)FN+(XpaEW*-1i>&;KcdTf?GGQi&`q(+rOX{Mgji}Tl#bhJ!!{}M65 zIypXhR-i$3$F)h`QmVXM`s|4MuQ+-y@LIo#y#tfnIF@-Sz(*sP501#CnHD2W>_5cS|8nu2^Jtq*5p|gY(Av99I3$#pafgWcw=MzI%Z)bqSL*jIN9Vu z?`5fI%Ix!{^hDH^WVn{3f|FR(75{OKtd8>G-9h$T?$I)U=rdBiOggm>Fo?G6@R_u3 zlpUP7(9{paUI;`NM9N+YJzGXj+{2DKrfh}qEyNx7Qf?RkdPPHH(K~dV-o@%>6rj2fa@7Ck-^Y)%LsPcbm`4_LjylfO?k*Ma!nez0_XhO_KW#&w`q8 zGTl#xg17T;HTb#}ctR}5yoy?=pBOVlQb)yA-u`}wDj-2}jgu4txqKZiT5aPogA8-- z%Z5YB+b07IHrf~G(NilcuGu*tKU$oUgTZ`i-C1d42JsVYvju4C*b767);&Ki=dV*# z716oR;A-oEptS;j(Nz-Mfx^H?9>SN3oV=qBOyrYVVj{6-;Qln-WOC_>3f*BsIUq zvD35F_G?P1lw_mCX4F*q9#ucSCI!o?a=6aZ@06+SK|B*F;8!U{Gzo@jT2PM)O79SlTvB{tD8hJLY@NTpkWEgCRe$ zTdn5DH|{fAT3YxDr6X6x`V#H4+B=v1`QhRwc^K|b8gPJTkXg7ny>rOg>8&@p6ai~K zgHux*8i%-4Y&)GBjLjl4Hf@&>ZZwrEx>YKf-O6t+&?W_}JbN{cxmW%`y1|ZB|sdM@#c*w42HMQXZ+_zsU&>8=XyAA%GT@B9*xfRfp z-0OB5BGgIq=*CSG9(qI>YjWy?!4GAj5$v~72(_82i<r^eB=4Z&!r_t~{5GzNnjB5uGuxa%ROzDPJXl)U2VCEI^ zlQYprT0_4f^wXmWsUR)mN}4)y|2M?c zMpO9V)TC$bzJ`Z6_xX6Bo*n@BW+){_0w_TKarmJ!FZPfZ@E(VdqPm%oQ1Lip1njO7 zTz)+6OA5zQ+b`{aCj-zH2^iG#f4+7T*g@PmMM^NM__Ec{GX19cEw0#cH*mq=!0l>$ zufni2`}ad?-Tc^}va^1SrMh*0%CgR$ozC|R~`$YIwrb>(x9nTkVG3~5%dVBEwq$BdR)8A z;CsOnqPh05$0#2fKjYK;=fd3+2hhCb{Mqs1h@B-NKW)>xY%AaQMqdcP$YpMy|E!Gk znd&Zqe$s1= zK{d>j0DPFT9MR-4+@DI5ytG|B!a)q#A|WpPF<*nBG}TK)np}qX)OM%=12H?k5HB{? zXxyH#tFD>pY@4ONkwil1ek%Dpf3*MR!-kda`ohYcUzP;iw2K;eJ2Q?O*hFe;NNh#e zm_OlzEX$^$mj6VD4-VZs#mKUiQ?wTeX=+8(=$^C{`dr%U-4N0R12RXjp#cuBJ4b)^ z^t2^5n4Cx;?|amRP|7L!99(zGr?M$xgU<=M+<7i%6rx}=mTmS*Mkdh0gFn1^t2UPXL&tiwDz{ks~=txE>N1HORAGW}-;jNZrkqa2R+RaHXlt4@Q5qsfWQj z<&l7M8dqX^+l!+7YV+lb%lEElwMBks#c-r8Sj&Nk`x4QIOZqckq>$Ih3XU-$8TEx< z4ZJBHUg0!lv*dY~cQ1dFx}&PiE*s^d)?Dfal&-l5cR*vs8m{=!6Vtaby0sL;aNpOr zz6OHqe&p<6p1MF&0=5}{IQmV}p&zq~1HPj#K0gZ|>-rD0b$dO`(9v!PnjW}Q6Hsn; zcv-|gX=x`29UL#sKCQ3Y^*ECAd!znD$h|8D%wpZt|sT@+!V4mMNoh~kanSR@T5S*qziiI$ftKqTc{Le#Dxr>BS4dRzT}J-|4@H@ zGzQH78|PqR1yc=?l&KEi{bcR%xPG6pmr&XQnVEHj|3OGx;ys9B&G-1qFvP%#ylLZ8 zpy~{W6)T%>Cq1hCIpb)(FaJ;fnY4vi*UuqzuOjxMA}JGhce~X0kDb1I+iPhYWj?<# zT_tgr^|Fva#w4tD>-6p+M6|Kkzoa9;T=Uix&2B&OboIne#dDh5(OL17P9b7DRXRuR zKJF#DphrMtTPO#>WZ$ya?|1`J0|2xfKA^-YetW2&U!U5A1L6m9v>xcxvi%hMCKcb; ztl~7pcQ@!RaWXi^KZ%K(=z>60upKn!d4j~tiz%w3Hrsd2{4eX3Bg-Z|N6nIJN2xvW zW3G3oYZ)#5ta(+!E|9Pi2^N%~{98sG527-ufV@d5njbSngNnj=i_hM$->?F-ey-j$ zhIgy6V?IweeqBmiI+ITMT<3{Pvz73^ZpZ~&5rFSW0XA$^ljiu!*6e~qpp^PfLaN8! ze_Rmy@nrnaW{c2zwzesRF%DP!ja&H8LAh{joQakq?IB1_K2nRy~Zj2VLkI4(|2;hQ{VWh?P=c-4qV zy9t}*9`1V=8JJ3KXHLt0>3}}%JX|@4qoUmPk<`vTqkhlud^`#sS2iw__wKrPb?g+z z{2YZ9JaD!lpU7ny|7*G1CsR8Y%&Yj-VXaMggdIe^%pvl_S0sDSsoU*V^x!zfoc!`- zOR;PPTbUO`M3x}c|+OqE;M)l>h205XkF#xsKCnf|IgE{6iB z0prPWX~!*|BrF2%ZF=g5BX>(18j@$7cWW1&sm8Zl7gHd7PfGZSK=;E)OwFTeka>@L zB|h{}KDF1{@J}H}M~fxMQTPugAXo*F;`{#Rb-w359{v97$#}hwbH81Pxb5-eYV}B; zGBg3)*ye7r1SkGK$oj_M$f9oTj+2RP+Y{S%W}=CmOl&(7+qP}nwr$(S?f0wi{c-D_ z+UG}i)u~fmUA6aG>ruUo5bjpiz#ljQzyg^-Ycky%RiE$Ix!>)rN+37gvNZ!Dw0zM#6{o+9A`=>Bd`yD*j!Q$yH@q!ubz6lSqomDJ^!-@2du6R5 zxHVR_kcZMQU}J)Wd+x(w)J6shm}q9Wzsy?DH>hLXYCaEs%APs~0T?GdPt@8ce?0SD z*`f(5a@4&*y*8YCsob~dJ3??P8W&0eA^{--e}duN6OShuk3(z3WKHtsvjSg8m&4tl z1BF2VG^xHsSfSE|iW|V&&Oa^mOb}5wVMpc z)*L!*P7LGREj=_|mmuxPg~=UP=a76dCvF1c(c4|l%{w{uUwT{yl(k^7IHr8H+s%KE z59H5GyKdLF9=ShnyoEJW&h+f*OVoPg~Hb6Vg}8?knymTq%u|JM0q zLJD9gt_)M#NY4B@owIr#kdr6XUl2U;Q&0IG1Q?&uYNK1Q2m^3^HC2n^c}KlgwKrXk zicS6p%B!R*j(|g{je@G%3k48g8wVXNL{)|!U5(pMg2SZnC}=9JN6J{SEZD5KM+UrX z=S?Kxzm0(iO8)jrNHj7U5zL{bq8**J1s4i&hsvTL_*=PQzTPYS;N>Au?<+C;+=0TY z86`|2*t734F>%xAZ0Czq&`2q#Bqz{ztnZw^<2HpP=YOFGddCyI3GMG+!%xY?&=jM9 zjMZ2v4^I70gae*?$(YY6+Yb%fz74nzDf)oIBxoFS_z06|=jgOZSP%I1_CUJ9 z*ZDrXKY(G5u{0da%>euPXmgCNz1^@A9(%t1{ab%3Bb1=eoM&BLlic1T!(|7<@NlO4siG@ zO$s0ar$J7`;p$XZ2>mRIOg5ADjvK^aCleSApsyDn&m3hmYgbYH{LoX}8@Qx35mrt4 zT+~_L{VrJXlm2@)A|xWM5zRBV4G4>e0m$L;Q*ap7kO`TGB!Qz$?}F?Yf&RF;4#TJN zi*P{k&&br^q9@^}zwpy^^R1TK&&_V*93P{j5Uq{DN z>kn3?^;Z}BPJo+Luc5Sc6(7MXB=$rf`uTKT52N%^5<0!+8-fdYd=2bKGLpd2!si(_ZxNU5BAoA>7lAi4sv$G00*_7)Xx@w(6q}h z?^~8FMFgB~RFzY@rX&gUigYI?5GJkKZ_PE=vt*lCb111+7TN0OHI519o86zST-4OcsfrT z#{xk)?1#INXgOajYx=EZ00xdB^X%z1AM>eoE^+T!LRf&X>i75{ln(lNiB4m2+E6 zKctrTz~9umW9mjwKqv?)Iuld76N#EJ@5eIh-Tju(l~2MeOD zh;PW^^A3gKTXU3HEwfOg^xn6)dY9Oo&7rw?E4{@_x5jA`CQW8>F^0UcEPq`l24N3s zyUMFSzq}ts>v89Y%C2(22AiKz;n1!<&6b|K&aqU8AyERDJ)z*?CQ%z$atKESFNJpyGfWt~Wqyqu`44X^y zHsm+pq&VPk+Wot6<{`xRKu3na;Nmlk1S&QD<@r*xtR6X|tf@d&f48;mij)`|d&}0I z(hF9r?@Um|UHp{U=Kk=wnD2p<*J7Z{#@6WliPmD5eTEcjCUb%J(3LD)4Thsrv@O4^ zY1sZpTh`;fbsbzC!i)wI0IIpaN0&*Co5`bkoH$_F{P_K1Va+Riv~1y_$_q_WXa1rL zq~`~$Y}-EeWN5zAfcNohi3(dBP<^751&O{Vf%HOS*{*8W8{VF?=<>B>Z_-MHw_$2V z#I+pytSExSXJFns>%996@#$DuvxigYl-O?!e|_3_N6QLIvZC0gbRzTE7KSP%?3~xK zLi*Nx^p}&=r0@x!jcFr)P1{g0SOH@6(2u>M8C|SjO!x;_d9G4wtNg3x2=@a?1mB_$joi06J{W?2lYKW*YdBW0Uv^ja{#Cb!_s z=OV!h?^;Yc$FPIGy>;lG&d=A^QLjWI4IG!NtXt2A{5Co;M{tKLK>rd?OrmC>T2w5;V{lsLIuSzT2m|K~0`@vwSk)K{kot5liLe(*vN@fl51S zT0(@szl0+{{u_M^)LKCBs$?g;R^z4P4-u*@=UnRNyzbwRdmAv$gBN zlRE=Y6MPveOFDVr*yxpCaC8scWNq?hVmFILrHpZ#3Kb9oIi0%J{E{S74-)8`nx3Gz z^0O$G7HBp{91FuKhgI+K=>_Ofk&6PCbMF9dJQgpYU05(_-j_4D#9_!%z&KqL4OVNb z2$>$qPWw*oi9gXF7y4|!tln5tA4ZzdL6H&!?l!+lL~unDOCjqec*87>=Y-|Gq@3-g zobTF?zAw%#hPA#+oBLEScY5|=!gVvkCw7-9D0h>F@$566w687rG+d;=TI;jVfCHnD z=$ehcf>f`Qi|N~n>kD6Tj@vS1Gw(T()tY6bqD=ld zwY$$7oDh+R-DyZK9+m7J>&?V1t8f*Y_&6z{5=~h+9%ns8$ugY=T*^|+$-NhY=Z1HN zrX@o9%1lx^LLk2Bw~3+h863(VSM^P;g7_wv?7*ObaxKre@it_e&OHyx&(DMvzwy5o zp(0+w?vLFD?32~BWnhYDq){*|_j@{EfDoCHd0#c{Cyy?VPJob(j82S)(dyWdtt=W} zGp+2V;1~MN?EnZs(i()ELrjzhrHII>*PAwD9;toeBR(YWB z;APHC`sUluUnDPcIM#5)-*)3Yj_BYzb^!pb#WMR7nC$0 z(_5@vX-sTw1hnLog>EaSW~a|D7NGUo?R*B|;%3`TdpBwFo?J)6^k^Mhe)f?3UDNI6 ztWPg9qVIq80!F+uz7yQumha17M4+}xtuL6{?n2y?Y7m3q!bNFn(Y(1n%P+5kJDDjM z%!$fS#Xa)ox=J~lm^3p8e+-pD=8Vq%wAbZ(T?0lHfhvOTP_)QmB1`=WBsabOrdGmM zmbNhv7~xEwovKhQLqq@Lgr5&UMjxJFkI{>VqCt*@Q*~qd7d?gxzPZhU2T{`EQ>SHc zRuyP}Dq-VdVGF}6{vY^=X1iw}l_oqqEG=g38n5(o?xdbY-nzkUR95!XGn|C!$)TQ&4Nc6hl@{*FeR6lPkTxkIWp%TRSU*|5rzMAn=gjSu zTb`r!jq^PG0;;kQrtRZyH>5-#yMLQ4nC-eO2}d>>H0bAEwNNquEi8EL%<9ZHwML8H=*hZ}8)`9FdH%~6u&Q;-D-E>Ab6hHnw z*JGH3P*-v~d8ISd8p>EO3{{;f0?{{jZyWqiZog!ocSUHcg_l}J9c6k%oXn30^>~2b z2(_E{SKFxp3!VOSqZYxAIR*s}S1mDOIFI+mbR4-^&Ze1-nNK?=kr8IMz+s~RdoVFL zK;KGhnAN?^%B6_tLN*jqpXf2G6d>ztZq^K?W3 z+R-7rI9;Dm*Xim41ut~dsWLRz-6F62!LRH#Y2m>k;y)_qvdJ#=^42p>@dLq9ReHeB z1J*yYdCGI&zFRop3@HvAzHIJymq}9mfJLitPsyB-yo$=Re8DAANi`QvqZ|15>eL~F zOE)dt2S^O7>c!7Qxu>?wXVftEx+QZ&RQrhi#Rb>UngxsW4a`ovmO1JLP+^t1GsaCZ z1^Hje6hsYOYi(^mNVI7Wlub;N>H-0Bi(-R~zOe^R@2F*QMneo(6~xarK_BD~x((up zalG3h16MsKVS7DPhK?uh%3h;Gm2$#>658;6Xb?d~?`gXsT0Nk3Ny!G@cKJT8&6>`2 z;O+&Y&PlR_?#SB{aba<||64@@kayzjzCh4?j{XsvzTppl`tgDf)2ULw8)jJaaCkU` z*>w4BCwIJlo7d*1qk8gVexKybHt(noUMY0-Rt(4DS!06(UJ7Uf-_8A~#PwIEL*TyU zp9`aqst87@TgLa>f`$?koNu0$*NB&fQ$zfFCJF%%Lh>?dF*XL)uid$l4~@u7k*nt& z>?4PZGyN1| zb66+LP$IMY8C%bNjOAL65#UR)CunK2n;CLH?o#lJVCaRkkkhjIWAW3TGqf4d8y+dE zI#vYqw1ggQkg#bexPH3xWO%TKSsXNdTi4ZO}<*Sqs2HN1r(A>E?6Z{ z10yc6Wr>jvj;>+SyNlP|59v*MQd;8Be&5%n!TRoDoj($+_iJ}dP>HYyUw_QMB6}lN zOiN@o4LHQ(jSV-IXEO{^NCEg?)peRM3 z+10S%Y4Wx?ehMIBullT|s@y)|5|@#cEs_~_*R<-4`6q^LeY@>+n^ASny?snF7qGI^ z`1Nj@XA@=22;FIUZ=W5xzIf3b?3ty&ZM$=Fgfs5D!oC|zw5}Vct+i3~OIXsLW??O1 zjBr#9+-*~c6_C>QL0Ib1O#4ktOw<8%kh0?(%M%eAd`*P)WH!SQ2`V+}W*}%i zuUHK6Hd?_t$wE)>8F0VJeD{iBmF25gEVt1RK>75Jt3c6KRH7kqhJM8xl8J7O%Ivsy zn>fP(Xz8CN8E(y+PpLx;JPt9!5s$&m;FK{$Io}*v4^~I;-=PMx`M*^wT!HKaHe=8c z@atF}$orhn$svb23E=tM<)G6-0T>#7UVPul)^VP=bXinP|PB_xrkcw5jBY<|(8 z1hvh;#rw}?xIYQ0;lEq}2IfjHtHVWfqvghjL8|0M)YoYCKzI5y=}F7FwTu zB7dr4p=yOQoY**QAJ^&naS#blaoFMrYQ9DqlFI(H?|CCBf{wM_?7jK%twoq&K=a!c z!aJ%qntaGX0@&`j=9l7v_>+E7q3WF9p@RL6v&?oDtf&U}qp_QYttX>Uwj5*6P z^eS=dh(nOTvdvm{J{-Q54Un=RnGU$-f>&D#aUm~F1u$H$)FVLC`^xAfLMC;&TmsCb~D%Wz|%4O`B1 zsW;dH1P%9|DFv;cp(&Z?@Av|(3+0R!2VOWuA$%`$|9TSstsv3T`sdU$zaev9hZ3j|C>XntJ zrV|}$svTDLU2_XCk|pY-VoD0n4npVM{o#Rx3~6?=l=n?)bAemCP6+8`rcEEC`k$ctb(qXFJ5ejkY`csu2J& zbADt|oDN?2#{xh*;T?l;b ztkzkFF1KsI*Fk0U=HcR_CR;t8m}thrGb_GzCNRk5&MH^|DkW#)+@} z72IoP?X&5#J0K;f+1jm@3)9nv9(KL)>go5lLL_iH1ClgMstU8EjktE;eD36Is5>r0 z`@=v7;~pUCRh&k*Kny^nB9*H=DkkViok9Zz2wxT#OT9ep?6hsxhOVl}32QlBo+Os_ z^IZqaq8chsZ`b-*Iyq?*+bpz|>f)x0h#{c|(IoS`s!Txmt|>(GD=j_d_e2jMq#Z%G zrX@@e1aJlLPYjroM2-iW0iF8d6bhYVOpwdoHt794;BoRUo?PIT1(jbVf`2>m9b;x_ zXbPQi$cH~4T5>^^QnV`mH*aMA}7t&5a7~QI4XhY1er{T$bK1ZkRCn zX)2|p$Iq|ILdGsx^NfXt}w+jsH2wdWdPZD?LPF?o<`z1wI~ zYRK54Ul}jWK>)t#F3$fNZO<_r41;}aou&H7pFe{Mp74C`qkT(ShvtRJ+5VcY#u&?o zM@AMuGIlm?j&U^UK>PkSJ~W#dAcxNqP_H@?vu^%K1KP%)x?jk~%+9D3jao<&c`_(K z_qDAkc27N(b+p+hC46O^lD6uWuNPr3u-U&-N$cC%;odsX*3p=at5s?UMoo`IZ`wTS zP=N~`yp^%*4wd|%LCXLnVi#KW7(YZ@G`G?bj70GmoZ_VpAc1yp{ne-C#lJ$j+ODy` z58OwH+S?_z=r^F)f?x5txuZ;a6NOimc5U@7%UM$piwn68j*K0P{1PYRl@5*q1r(Lg zi071^M&eY`0YKWx=()PxCNbC(RASM0#tR49bdVru^_z2%eg6Vr01*rZ?p_~O<*!#9 z0Hk?#KHC8&6fXXOC&kx2U%oFPhpY|v8+P*Mf;PWkJ*=?ZYEVyG&xHqUtRVfqv;s^! zay&rZ$4KE60Ym+JL3UF;&qtT`@FZCLY211t&&Tw zUV85+r@8g!haRv5V>ixtv>hybD;>8mr=}_eDp>qVpqJifhYyObc+vF~u`Ri8X$*F} zPxfHn6DXgry5F>v`}KY@xW6g>{g%xpB_@`~#7ss?MxspIgpZuM?w*)5J*P96z^Ffo zu>a~+3BrUye^f@tr{@BVfh-Hl6$WwiWM&XpeVr-tGxL^_7f%+zyg+pBelfCgPb87Y zW3}E}$yf`b$U+iQ1cv~^7V={58pLp{_dMo>c<1$Re}2FM%CcM3(XqI*J31xvk8;~v zkM~I!_KuieEJw0viyGUO&zXWsCp66I1j5i5-}>2vrZ1K23dnTT?LOwBmW@ zb~(*7-N+&s~?SGim=gHB9lhNFPx2jLq*d)&U_CF&sv&eG_S*(7^q2qkq?l z`<&hF{>jjq!v%Lg{gYhTdy1VW(*~JmyK$th#eeiRr8R?r!$8?g_|p`q7%;wLJ*|$- z*02_oly*m@_|0>Vcqb@kV8iA?rLE*sLV_HUhU1tpeV(IE*L8&YR$IJtQu5FOKc&Qt z2JP@_nTo;)=k=14aiD{7-?BpWN9MUOl6`Jl@us0emM(T`3JY3}Vy#N75kn7bZ zKDFFbA)8snev~)#3|CVjEI{7GD+ z){3m$JRa3ncUFAxAT4-t*Y`h6M1Al4qTe~tzTb-d{Ze7@az*}j!#*>%D=u8_^Ha!b zadzI~O`oVlz8xJc^9yxp?@u;4$S>N1R`k&0D`(_v%^qn!rwF=2~y7~c+x53L+q^*C^ znO=!X+>R}pcK%&v@g3n_#_RG>L=Xm2z8Tfpp|97RG|s8JOozk!-&aG#|Lbc;u=EMI z-TG|<-`Tk$k~SSpVqW@pyLsiiw4ZJ=$&A{alge3Z zi6Ag75eqfras&K^$M9bm28S4_ia?4Dae0F;1QAycH>)0Vtcos>fyrAXQbi$a=a(GK z#Ia^KALvOScmzS&f1DqgG@br4j@>9t##e~)J+p|T%7Fu>sC^Alj_+^ntMEDA;5&|6 z3w9g<;h)xTdm0e{pf8O>#{^?=D;V#|MnU~Rq&Y+OrR7AmO@^O%ajXGgl!!oTWA zP~T&}BL_p1ia&;3mXjSX64&?b@b{~=5d~&@1bTEjx;K5rzF@*jxHivU1Oyy9RC#uaBO-PuB+7+zk(3T1_1(Mb`kf9k$13I=Ed2gH%VD-9B$13m% zg5npeC`jTxtKB2@J-u=E>zCeCf2V&!mKouB&USDCbG$A$nv}X@XF!v2$&x+T$)Z36 zaawlIS`+6QQ=6@K>YG|7D&6e$_xot8@70AlefIYx!S=;qWwjhF8j6TTjhbC$c`5=`%- zvw}dl)hdh|BeaIYr}Gj1{q7rMY&?4wHbfZDE3Bd@N+XQ}&Z(W$S7y{O{TqB3Sx2q; zAxiMj^m2uj=;w=dY+J!IS&X76+EaDjm^|Md+-)$d`!V?6CGnDdcl#RN z{qq0Uck->;#eY8lzx{?zsmOSUw)pIZ$UD10->>PoDElU3Au=9m^TL&2r|_ct3*Q|` zJ+XSqlpY_WIP#Uv-+zf93PVkjym@96`m~3ZwqSaoY-R0j7wq6w7M0qWE4;d#v6dB$ z{VPKj$V-YJ90Tn4(@^sD5ym0{k#ePx4H{q+8# zvbPgzEScq}DYUUH*-S`Qo1bMk>ds{B21gqfn!yc^`iAW~j0WwsD^SC~?+^N`F;7u$ zCoS#Ka0>h-0svyEQgcK?7PliuZL)WnQ^vYoAC{UuM&G>F8E1E^hfvJR(08O$w^+oC z?p3S{c_T0yQ^%t&!=l5D%}eZdkZV)B*P)=VZ7WW<;|Df+ir)VtspNk2;J^@L;P`R1 zM1dQt>H3!I(B-80XU|&bg2$=%B9?I~Bw!tNnS~uo#Wi@5lp{wD)(0IRk6M}SboZO6 z9M?*{I28}sKA%sN-FU$d%f~jgo-YeR5Z~b@m&0RTtJ&4j^QN8@mFLCNbB%wJw+ONg z5TAMFHtLh*_!anQ@ww$RNw$U1ab0J~QBi*hzp=&QSI~KCU-Cq))NZj2D-%2}r`0?S z1FK<;`dR73){kb&W5X(fpc8qfsAAdTaB_t$?Z|`^DSmq5+*8FsGcrBRIHKT&trjz! ziA7GVN5g)=vD|hem^xgf4jGRUu6+ z7l<6k23{fyH|qY)V4t3@{WGL}$w!}|+Q!4kqz^PWoZ!g*T>!Lr&o2N-^f)F7IH-=B z-6808!V~+*KUELcDK|so6YT9P!_ad2)z z#s$m$wvjPzCYanqn4gGU5WrX7&xjPSV~Ewk0NKKKwWc1l%i*lT3u~_X72YcwN0#CL z(Yr=FpNDNsd$jA9ZCW66(A3`cMt7XCPi&_4EUy8zyMZWYh>4bKK@r^Y@^Z(m}FFQ;1 z(U*qSjt+t^90CI&7YW}f-SVc{^Y@4JT)}nw9PRm zR57>1OJ;DPUp@sS`TlaTDvHXAz)SfeKc^&TK}}mmDfR~-BJbgWk(r#ZYh&uNFZIS7 zBe@!Dh)&m@HyCP^o(p*W%z&9mQ4EFgo#lvdI-3I?O=8-v7u8 zR(p%@8WKo~d*3y`{BCWX#f4K+yV$t?%ksGzq%t050PvNRfBL}|=498(sE^;KeNj{C z!wd@$Tv1M(1_1<=aa)%BP;n#{Dr_J_itCGE+7H1`5(FWoymc@u=gHb@q5curTy5GA z9aWIkt2Y&lnkrys9yuP>^aR_0oq*$5*^(%;i}KkBSt5fRnp0IiZUUmXKn^1sHM9c) zv5mCD(RJG-a90QekknY}z52b`L|bG?;HpP(zO?(7zC8yisL^VrI16u(n!BhRa*6** z$^`g#x2%#R)`cJn*h1{0>)gX>*gEwurAFgjv9)K;N>n<`Ou}f}RVv-A4ps2{8tz8o zqQeNE+zW|x=G80BL^iKIb(DrkM+CoNMIo&;C-cuaCbX^ftyaz~|nQV!Lm$)!dW8iDm!`8PY4Z?6GO^9rd}{nhME{|_ganc#MMdv<_uu9F=`Rr@!MLcZ z3<#rp2GT{xHgv7sj_L;-1XjW$q|s5;wEX6kxmrh|RI_v)AqRhGmDm{mbvuslg=5T; zbR~bEdcKnDv8vOsE`^-UE23nA%SBB63jB_@DRpgG{plBxGdJItlbDi?;oZ#AlFMS*&2d)YB0%t5AKI4P7>d0pOZ&VKFqFr+;`@Y-F z?l1{;inm|y-dsMN7ee*C6jzJA8qm%*5o#wfOUD@NrBR!xUu9XRrKw>)QlK}!9T1;= zxmzz@zwWD*(II0^z!4sr-$4*4cb&-iwP8VZ(F3FO6TDrjB5J$~Mvp93ZO?O!vz}hH4XgBJM~3CE#dzh2eju zn@bxuRaB1*xwqM{jJz@MDbAj(kr_1@#1-In4swHAK0Y=!{{Qaq-@3yBoaw1wg3Vz$D%JlS7Rh@?MxPFB&4ay%ROxDAp zpZOzz8a_`)kI&2DPc%z^!I8?>q^%6h5ZRKWAMIiIB?naRGf1$B&siq#RhYBPwMr85 zSxUAJD?ZifSjsJBvE$6E4Lv2(k${#nVF|F%HB*CqOvdw(wO1fYPf!?AaJhT9&Fi?> zqpIQuhaL$KBxKFiwH~Ot*G4(|fqklQT?7UI{F-ESOhhC8;thTrbm@*!u zOvc5x-VDyi3IY&eg#r8xu!3NI;t2tIf8zBA{5j4Yw*O!}8n=N?NJ)W%$0-+^m(4MvLFTNe^2`2;x1nbI3y8sCm5%(J z+CM7xH1eOX0%j0A3D6wxf5J~LO?Q2rnAm$=FA?Y~sk__%)<6Y7R3H2x_oy0WkdmP@ z`;wZ~a_-r-RNOtBZ0edG7bAR#hr@`+wzr`cVeHiB`Ls*kkjE!JVDcg1^Q64N_4cd!Gs-{Y;P0`}zE_9J0eO)D!_>Z6?E>f6 zXF5=k-kI-``azZ7aTPF)+PL(x^o3d|i|BD*dRblbP}e#OJYhfp8+q&2MQ$4Yj84q= zq}P|YTb}Yi9}al>n`pQxx9LHzg6Zok2ph?~Ae6n=+Ah~NvsiOk*=W)hC;wr=0R(MH zKp+IAz|n;4NMc2Mkibb}LPQAwK)A~qmu27UHqFLajEtO~f@5>lX`vX<9bBlLN`Xus zhqLdQA(DO%66b^mY6VRtTY48rxvv$hjJ9fUM!TqOo3+Pl*5f z*NJ;F%QepplDdIxl&<5t~D9pBaHn|zYE zeG_5(dgm>QdSYuYN7JMK%$*-mF_k*1>7So^p=p;DO{kjMugxic*X=1vek)`iq*H8u zO6_2_Ye~?maBE?IYiuyuQPaz%;ynFCDf9cvVirOl?9+2@y7!jDDV{=JfT(?iMag>0 zm_w6V{kQLgpQM~EyhiaWy?Ld&-keQ<%@;p(Svg5{l@dDjZ6|-_<+GoK71PAL*HNkt z)owkCvn9zHPyQWVnIg7DB!hP`zQ%sSgq`cP>3_15Lo`Nu(kp20( zl}F8Zb#N^oV?m7I039K5%@Jl%`;2|}O2%AkMvFnp3m6W@&2y|LG0!dQZt?-BDj^#O z^RBB*=h5$xtG-1ZLuu}tTpzct4@lcKR`e9~i6`-2Wn>V(GZLU+CN9x$&)pEQ`0Yln z^RUOwi^3pDw-6mquBOH-S&U~-U`VO0!`H0WgaE)3|5poU&f6CEeT#KDa9dGvNZQ46 zV~X@f!|`Yu9rH7maxvszhVd<7^-}U}4n2UisozZQazs^nGsUW~ihCe>oj^U;7T#VS zlifC|{&;$M(c<)P*3<9&>82VYUR@*02ZzA~D~`V`nQcz> z-$^$#$uwn+H#YnR6%4xB_5gyJI~e=?U8l4so9?Z^pzV!pn7x!_2y8~~$HIIFJ<+1$ z7Bke7ttujZD6FZfm$Zxzp_rf)22NHm@8Z_ZY;Dg{OSo}>En!K0M;GvhM? zz4=SS8)Cz1k5YWfYvK_(E?cgUDJY_mry`WR^ets;KivyU1OxzdY=}_!fRr+Ry?gfj zl2k$SR?<@!jEbDg(K3a*X4S;1-QCIUjhO>qR945EFwzC|yoSBex2Tz|SF>319am)C^ZA|m}0S!Lr79G>IX0?y{lcQF2t3Nm8W%&Y*8 zrg;wD(pa1Z+Pgj9dWYrBYRF&YGUw?^5|)lBs&=an&VhS>%QwRl;+m3XzG@zw(^Tz7 zCvK(433Eu+X2<%U&uRthK9)Y0yJu!~mvy{R63T=F-)zW9?D&`2Y2WUa@Nj*Wvr!$`5x^L&i-QJpiL#qdZMIMiuNsp?LelRtR5 zMM#=%X}_GNVn~iQV-#>75M8>q<^!sU8S1Nw%E}7yk&FkhOk7$OjuN+Be!fZ)$GE3u za5JA&O@2unZVbgm{XS{%rGeAzFiCX|2f>vR!>3|~RT~feGiUdMw0_Iv2fY66oBZ<& zkfK5akP;PAjsu$O*V;t|Mi(_t>1(_mHZt+?!SV3I{KB%}lqv$A>dgk0hhNm*9Xd{M z!nidC(eHCO-6+(-aB zHQ_HXQ4+v-faD%3YwL*P6NPo2xn-PXG%P4nASpYZzh2PvinDOGXH|Cc|1|Lmem3bi z`eDo=Yn8Vf1*YUo0nv*0-0_VL;7n|`n&7*C`=h%t%Gp94(srLWn2f(pGIn5+(c0EF zUo76Z?7FQ^RhKeO54Wj*NU4f0sF-P8D&c8B7&o7%OC&iO<2#AO6K-m(yZVzhLQUja zG(L!hCGX>NUL9bx`4}}>TJ9EUGj(+*Zwdk^ola9~Vg&%=mK?|Vo-Bq-le(i^%j*B; zFV9C_+Wf&j$CcD+TK$Yt*I0?gr_^y;-)nMdAY$MVzA`@XbfkEmTAHsmzeQN44*cn0 z*q64n4t@4wFdmDo&}Z#jBEOe7$XWe%Kgq6UI8Kl|tMs$GibYNw2J_K-6O z^qDbA4V;>}`e5+YFxYSyDg_gAm2;y~IknYTI~vhX2uM)h2T*TLVm|%mw2EIsh6V=} znjU);hH7T`wossWfnUuvsnXF5K7z2`va2L7;IF=z?Mf6glegw!3`THSHS(bs;Y>nF zbS$Xd9VY1H6C%q4_!7Fk4E>6M3j%!h?Qz`N0s;gFX6lIc#fq6dtZbPVi2M&Dlam07 z9@|%{&0rk(6gBs_P7`1{?3C++wAn_tasU(0N<@O#-=gO%m*q_>` z*Z8qc?t_c67$vqiE9ODTS&jG!>bk2WucF=s3Xt+nPs!#0l;()R zOZAwiSBGR-X}@&BqZ*0|%YL869vf>l2EXKq3S7H`1 zthx2lygUlj*V5yb`hWUiQ9H(}(gabW6TX zc_?pg*XT7X-Uro(ZNrk>Dy6`x_gw*q0qt~rx1T`9N(`5SIw$b);n?5tUk8n!4d3Cv z!=~5-!mt_%r>fqPwB1jK4i=MG%I8no(xWR?qXjNk^lH^$h@aCsa}=VI7jIi|@O}b_ zN-BSNcb2_s$giCqmREgb%mOi%n(*vQW{xSB+x1Ha3u%Hz!q-#&c`z4PTYRu;S__LQ z>kY!#cgzXI)Nji<){Z`j^hK9o7Hnv&!{z<8Rmvj5)mg=o9$v7y(9D4z$ za30d_n*^@uTDMv8^*~#Ca6of5E;7`GS4>N@+4%LM{}Cy@)HPA(DPi1@k<2jZQ9kr_ zM}_gYSq5B)s{pqiI1*Es3 zod)}KHwR;7#N_98V=@EDW@R%86#`aCgfYybFxg}-UkG5vOIBfQGKn=3c=GH&oM?xm z+`hH7aYHINbS3&(F0}4c)AcweaaIgS{e54D?roFPt%3r8HD9YI=;8kAA=Wb@nKC=c zLxp5HoXhPe98E-jiLn}vw3xbVjY`d|*>S(>Tp!=<*=ZBsns%y5StE4LDpora%)7kA zLy-O@itX}o zYq>u}^*^&*kK9C6uv9SG5#iy7SIM^K1g(r}Nu;aLDA*DYo(l28V%i}mwn*kA|E}SO zto|5vU&N5r&+)FFa~O|cbV;Cv>y^Zs8gWHMoo^f_E1^g9Y;o}&$%?Y#p|MV3;Jw8S z#dj64TT%mReIj#GnzfOtclHgH#Y?ja#8a+1><6bzXY6e0r6x6WGsI(=V^ z8gRinyx&ir7raX?fu^(}aA&moFuZ1TGO z>cccCLjJ=Oe&M3;1_k))i6$_bSox~C+gt+!GK^HM-=3GCVS&~4ASeLC)@VAj;Ln{k zETH2y{P>K^o7ZH#e?g3J+oOJ!D`|Mdia_jVp&#k{IER<0aw%e@;1V9QGjHST=&O^u zRJP2zinqy8CFvBZHDP*0z~vvO4>N2`zz>3#EuAALsy{mV{RrPrup`<*zIo6{2Mw%+ z=5gbz08jC^O{H^#D{KPpI^)-Rcz$d;&`^K7^H%7vJm34YAYA1zaY&5J#X-WO18uf! ze1@OLq6Kf2&raF*#@9H?ti#|tVq0Kj?I-0EV-FqDl9)b?|OO0PbK*b}TJf!Z#> z*PMZOrz7~Rbz)?Cvd@2iw`;)FbEO!{m^UDg#W~lnbQi9WQ78#=8C81Sxgw?Mc91AN zmz8dqwlxf~XCmTeQZ>c#W@w=;{5*PC^&TWkmk|>1qPTj6&R+~oc&pZ!oc*tYPZ&zj zWKO>sp38(F8y5gAEi&}io>?d1X(@d{Ux21rtlG*uGx)Cv}S697f z5>~+d$GMsQQm=>Pqjr-p)2a62P1m89RVO};8rL8ya9>;uuMJ0sEv8~8K?&iTsBll( z)bL>W)q87e7$I^(`$mzWtNUyEJrqH;E!(d?x!;Jj8MMI;52|8aI(vNyzi57^YmM9K zzTI13mJ-0}qSCtFCLwDS*wRm-88qvO83RCrr5Y+qgALdD=h+%l!~PNiKSYV!55U z*bNTAW!{$;xLvTvZ0k{c&?fLGM!0~?P=6Zr+y+9=%neqrXuPzaooYzwN*H3hj_ejz~WQ97{!jXkfhb-rI%uR zXkO#v87{P!GkzNmp+O$3z`rhQgN!VA=<%$UXL_8#^eVW(#Cyv6F=^l|;tMr=`V0@5^3)(JY;RF_7QMB)x# z#O?m}cM%~xdwRt4l|;J+WrRMeQx|S-Qv6-3=)`n~1H;QbooK1)AG^Ujb~01tC1u+s z@4P_?#6`4x`0^Ib5ePu2>-#UC3N;!zlA?69+j^5+(o`pI8VSel^r0x42oqet`Tl;n zs=|u*Xr@gY^^OF742WP2oQF?zm92&1jK#icaPhX}%ECkUad;{q#)xC^!;T&RC|XcL z2081I@}X)fhe**N{GX=YGN`Stec#>`T1v6DxKp6Gy9S3+yf_poP~6>u7kBsIL5sUX zi@Q4%cMGm>?q_~8?|*;VGs%a{%D&dRrM!gkN=w1-tHiA!DFBr4yFXFPEN2VJ z)ERDx_($Zv(pDMiM_+BF!etsQ-9}IiYF^d@dd7pKm&6=+eH}lt zq{!d=W8>o`pZ@gb(1flMhfB@ba~`hjHFw4OZG@GD(w5FV4OGL11F`u@wvP;pLyCU%`{-(h9iUB?wx0n@SeQbe10qz;k_d<#tcLp!C}>xIOMDG~7dtA{ zhGhkXoK~?;yFZjSwOvHX87dp?oZjhf zkvL3_rd;svGOf4XK@yLDeq`r#Cu#?@-16@_PA_KzRfX;G; z{t~Q{&p)gJ*Jw9JN6RgDiLW2RNFEm(vL0L+vQv7$FC8Q5XY#{Z{V|c6*mOvCl5Tld z`hV1fcT}GRA@U#}(9QR0_06Q1v2u1Pcb_qOd0V&~I4yS^OZ zdJh65vi*gdfbU%Ed9BtqiZAWxjZb6wzEf^l8mCZ05Nh?57ir5u_E&`s-j00QqSD5NCg|oGnI>Ri<7;711>$Myn3v{`0*fm=MvFz>zm@7R@h%U!z zsg(~uw9;X3X~S~gt}}YbJ3Au+;axk32$CpNeCT{dU`Jt!;v`H!VuWBPTt}Bq$Zaz; zPrNkz>!Rsv$uD6ms{0y|`p`LN(M-Q=_IF^XU~Kiba@%CE)@iZv4QY`@`?rF
X%tOfTe-gYx_vHNWj{EatGSLVm@kUVEi0k!7e!vdHf4K}5`Ja$WDrH!oPIkSj39&{NbJePk{CvQkL`la2ARm)?mN*Hez!s_U?
z1^~W;W_z?S*$Y{sJAjrGv{4xg$n3U)t+ypEc6&KJ+&3Ec
z<`wv1`5A6k=Xoxn+pT|}+A@UeNXeh~UAv}fDi&K+u@He=#y6Fn-4@gsolKE((dalY
zgwr2zXDJ;8buT;^fpu)DBiC$s`f`rby3D&!xKc%$SrE|JH@*|tiBX&B2aG?zL-3`A<`yn0
zsVR?GwIejWO#WKQ?z~Ze6aT(x_i*N{i`pcMZ#dVT-WPjZIlfQy=&uQXcW^6$v_30j
zzwA;R$tZR7!|U3tL%oHyBu(LTwas&1A+03J+@{sZXMd&K#19aO$sK_E3hA`3w@Tfg
zGPnTIS?3!+xZ?SLK7(9(1D0;GrV!rJPg1-1Z1oCzk6S#UtB^eebMx%#io#e!uNK^Q
zR}^AZ8dFJwdO`B2xk%gQOA*uTE~cz@9RJY-S7WCvZRcge*t03-P
zlGVLgX0wD#1EMwXG_zEp)#FxO&k4z{k&ycvbZfcd7d;M3IgqW7*4j&;hD_o@Y6HbT
zgM4W+&kYWpH$b);{`*tB9&tsVZ^qC}*cJFIGC+UM8~7YKh7du5|<1l~@DE-ifJOypGd9m@$#bORv7=
z?_zVUXS2?|6dUz;2aMRGNoeZ^e@oRF=;GWV739`cON~fiw*$zpwKs7LIQ(y{AYH$U
zCKc`({4^CNb!mrK+8;hQRQl?ixNA?@RD_Q;4!O9r+m^VABuIs}9#jq`bwO4v^y
zfzGz)1*c**u5~qumN2|!Le9_L13=6{TklQU=iP{$?K%@7?82(?tHM0>%A8Q>EeaN3
z7fx|~Oz^>>_h}X_rW|vrnx^(lz=*9Qhf|oPf)x1jRNkhbh@o45VMvMx6!Dkme{H>n
zS?f2~S=pApzv!1|I$D;qsfuR{ADA`@p2j**
z55vg*+-t
zZ?daQh~&?@>Co(-|L}f(s!p!&61E5!|EpUmZBp4qjML9Yb|wFKfG6o11uCba&!H^9
zpoS_~(wcu2J`)GeMbDK&0*l!41=I=k{=6c8wSF{sbg9lv42W!~vEGSQAx^P5!4|ni
z#R&o5rOsa37|(~2wW&Wh?pEUn{?ii4MWz&UNxB?Q=zlSg05#&eUgw5Tg^-*Hrt-U<
zkkk``V=>N9ky$RzV^n;b_9D|GHNWJy9js>1HZ#$S)a|
z%?*7Ah5{-c63@|_?Qgo#_et|O@%6vayNSH&Y~q5$Z#wFaaSOH&8%2IG`*Tc^_5m^X
zH>!b)aSG~N2k}B>nNGGxSM72F#;)w_vbSu6!p&umQvqLyYTfKlGm1RP_=IQ_d`jtUmjkxt}A^#^&*;xPMni{H-CiWt4{
zh=I6`+s-#JRhaTxX_r;V+w_M;VxXh65b0A*Jy1|mR%G#eEe%9ED06$$$pPfW*Xluo
zg0WlD!mC}}T?6ga?&g&{-FY@L
z6q$ntFc-7om?q??YIiWf7pI^ii`rR!2LNm)R(fXXm~76zf#ooDBFiG3_z5x-Cy9Rm
z#?tr-@d0QZ(i=MXC2UI$?S@hMZqi|d{uhjMF_N)CmbTaq|DgB3F-Y~T+V1(3W+vh}
zEmqqO!}#Dm*9O6J(X)k?waKX|4%x&xq}N1+5Pp*P8t}~?wbQhZjGZrSe%aU#qwS~x
zOW>`;inCeDN!nF4W@edfk{Pp08uiurZFv3S>kn{SkRy%3&w9LC)rQcyiTm=X8^Cu>
zGcGlWzTL)vf5?^S8{`CSaUA|>%s1rzJJqoL}5N?|=>Wd=a5ZmsFJ#L>3x@s9Cact$X
z3A!mEzrSuXy^-$IpgjcTzU|
z9co8jlxSkFZJXV-;#SvhBQ|%n0fige=4KLxO1s9a)%w!-18(Qi(wb0my32n|xlnZV)1
zA<`$|K^@&w+7rolrv6=9x!5Suz_z+;5D~ZkTWF4Qce!!hn~lA=RX22_EUdNlUJ4Y$
zMWdBrm!;OJT87{-!rzuPC*)SkaEuGT*UKWvc&sk|ro
zK5l`DkY|obSWr<>Asl#p5K%76Le|3IO@G}o+Ly%1Ll{`n@W-v1KZtIh0UGb!*s_?AeVR6jN{1
z%-j@kxWZTE+_?ADI&MQ`r!C$^dQaFXvt42)q>&ncv>m4=Zvc=EfYnTEb(=kTS54
z0^RJ^^WaI-h5cz(1b~#;1oJTSWYAj{k$lcz`3wnj9r~^3gZx=-MeogEk!0K@=?t~U
zM7Z+2e-G_M77lBhpz}$I5OIv!4Zrp*3VXiP1Hx#?gpdFi`wQ6KgA9(K!S;%<@gtY{
zQkFGQW`%88l#o?5Wo~nD?9e{WqfYuLoEJ0N*fM6LSrj}KG`wUK%(@9yAp~M42&VNV
z!`MaA{nxF%0F*78Jm`q3*<4c|{gZJA*=EB~_66eZ@4D3{4^CLl=6%luj+gHsRJuX;
z?5w7PRIvZ9|JT_~w-xbQ=gvX!C`L^p==-9@9s$iLw3_X3R5WjTOkwr+zaIZ1*NjLy
z>o5dWZ5j1|>2w)gG#V)pqHkHq#4Z?Lu$aV=B>not)e1?kg|iPV8`*Lb`id*I5t}N`
zt9qu!8=s&X+JEDuiokHYa``TIK~PoS7o5PU{ttW+%Ge2sd}Mhx$!1Qc6$+hSj6bqE
zOs)MXggwZmRXyy;9zOor9m|D)z}FY?t4trfA0
z?T1qlsWrO`6%#5yj0Mx{%mg=A4{gyvqBg_V0qPg#C>42NTVhYx^=1Ph^B45AxAW_i
zm{=H`YM8HLLng!1xK({bJ%4QX6%>{0fVQ^(MiU=vN%agGdhth+g_jIF%=5@o4{dtM
z;b8dEE7mrxn@kFw-uF(SjHG>W*_jbfHqqx2-kqtmhAqs^v$)|IVq)>!Tv66-yqb(*+`ROYGQK6v`$(}O;>
z0XWh<$wNA`Q*)7OGqV$gu>Q0QDha3Q$8%6qwsN!18mR;OQ?2n^FK<}e`_!yFDYJNc
zkOIMC_&YC*j$^1;KYd&9-WYMN$X?T()FU?qCocG_+tY=)iKJ9mr08E#GQ0-laL`Ko
zk>|epcN;_uq^6vpbnZ**X0LTam>&?M?!D_mp9gc6^B*+MW21Twj!AL3e=jF$TUEE2
zC3$eFnBGJc@}~-pInqQze;P0va!BhQ@U5=B2x4$)CHtT4x-u>$~Z>wj7RvC^ad
za?h9b3w!dbG>!oL%~iou5gE$a2e
z;Wu{o=~K8S2aREO!x`zxs~o`FgdBlK;$W<^#fbjRf$}RtqzxrXOLP;@_;0zbA0Myw
zJVTZ>gBh|IS~%JzhSe^nw%Ysw?Mik)H2)0xZ$~jnairu>RK;bA~{=w-eFOO07MeZ
z9ow3!zC&kZ``I>;w(z`ky<$C;VwW|Hy@WNCh~TTEam#lprm5gak}J7M!P)l
zw3;`kH4?_-GEh2%an1whJg0)daHG*f%(8ij=+(OU>lG2G-A&Uw5tgq{7kq5}_nK~A
ze&pEBM)ntZr32BwvnYzuvYMvH?KXeSP$#3IFHe2#8R-oU+qXMgmUWTV=f3iqj-6GI
zO_66#?8t3fuVYa>I(1(c`Qyk{%Y0mactVsZ^h^_U|q*H*dvl^Pj%`_98hZoD6(
zdeQA(zSK%H62hW+qI$7ZOdI)}=K4>)wZpRjl*`M)!@|;0l$)~>p8f4(Ei3vxj9
zPPGaG!;G)i?PYD>!%otR5s6hWXa0)A*t8b-I7*((U@Cm=`&UbgRpBWbyvi-=Bh1xu
zWOVll8{71c83<8DOwJDJvWS3Lf0OH@^UuSy@cpuP5w-gx%<}S$;Qgz^xr#L^Q?IH;s74fCyCJ1Jgw)g(7VZmr$fi9G?$7yiTfh(2hx@nW(1xK%F3fhq9=h&&Ja
zTr_H)>y(H|5%sIzvya9k0If`aWeE9s~YiG-A!op
zSnQ{m6SC9ZuP0j|@o+T~0+D{{n<9K^%Qn{1dKc?^mRcul+`#fi!irp6?%jhYHaPQ%
zA3@SiuSF{aE>X!R(B@Or#|x}(P`zXO#SEt77i0V-<36dOuOB~|u6~kIg6>uD)lCcMv&nnbR~-RbN5cLF
zewEh84@3{f5?|G`{Oz>RBZ97TNdbzy946gN{ZHJdvZ(lK808hm7Jo7!{zci;@+`-O
z9v#%)l51C`o%wX*^Lp3^zCJDUpG5tUP(D(9xt$xA^illpE{0sxKd>WA;^@Dd|2})9
zpfU_bdKD^1Ekpl*U*5ed7t|_>Z_9!ZP5nh+oikc+fM7i3y^t!Eq!s2)-(pKZmm}ldHYh?Vf7xw8#Q$w)EX#w+q#@2O-W#!Md3
ziey6cZPmiQU!IU_&&7|ic<&cJ^ZR93b2TOxUQ&J=W>
zU62ey5Ki6K^{A}tYqHlJ)mkQYC7wyCGv~e@+;QiVT;*j)#>!z+C3EcBM5R
zPjp>i5%=0_R?g2)8;68{Ru2Qj%kMkq;=5~OGS}`PEFS;+(-bdsLP>aQTJyij#L|Dk
zi?YAABz|IJ2wgyNXXHNwua2doL{*22YemJ=d+!?pSpj-
zEUOH~mg}!Nq27ITLAU;bc7^eKBFo%jZ>!>u_Rg8a`&Y0BX+MpRRrdE-hBPXA%z5`v
zWYS*LQntKyAD`OvALG{H(^se%K4yPj)5-f4b_{&w+3sF#aDE#{=nXkOtrm9QKA8iM
z7QZk=3p5ANHnETizXjdxxr2}a)~2kd+Ml#-IH4&{V}2gaSC_4qlQx!^wxslkK+%SQ
zmbuemF-+>a$Xi+-n#=iA8eL3n+E-%hGeY*Y81K)|sV(I7L0M0XN58)0>t1w@oLo=B
zAtWWdhO-h(JG+fbV?%phzr5pjQi;(CX&VfUV#otzXkJf$R*HOmefUtj@*w>GE+Zh3
zsg^9pBw2&d6sTzV{NgshRp^sG39EDzCyBTgQpD-?kz(m
zbN<+_;wI_oF91Ail{TI%RcsgG&SM6yF4(~B@ZzA)e4cg+@sog^J#pWiAaisdBJiyv
z5_D1;T>K5VUh{0}xGNJ{>egrjIxRuEjZ
zy!v^x-^WaRxNvPtlfWz&(m(9ODF$N$O3ZHrD
zLVxsED0blOyLXb{=ybRV1n9V`oz;AEEafW0o>PZ+B}sMNIM7l@8enS@=%c?Gw4CbaX45U42juI7_O*kv2FFSMjg!
z!>0!e_XSGV!OUyMw@TLL{T8<;6CO>^MnRTeozG2Ee}&qzA9%Afk`+V$RdEbqFC8&m
z>QrrY&$+k|JT}*+@LKhfJ#lU&>bP)&syXG2<{N#$22S4u{ph}Qy>!>tO@1kt<9z)}R9j0vRxVDN
zh~?RQPcu5v6D|M524mpgLg>lMbEeOa|7S(WG3Fo0?QN%$)=FQG
zNJ~(U>~B;Q?3HWIl6>_SYiC7(AJ){!N*dpTo#0KU@tj>4Ng<7;EP*dwi84y({fV-I
zZJxL(ZmG+0?0H;Fooq(jOLwl|P0xvz=sbe%$r>GGEbO6ibDa
z{rmpFeafHw%cA)M%4-dZ<8;OHC}*$ngo_V3k&E85VIC_RBgS)%LgLJ^tc!vj8n8f9
zy^seu|5<*ytMUx3(14TrDuqwJ9Q02wO>+$2=12aaJWK?3tkP3P#`UwTJGDT|ydZ`p
zet(^Kes
zyVWVg|JEavu4bD&@h!2s9+&r*mjq@(>VlSigCjpxI7W{!22BtFWz##9KDO6
zPq?Eq`F;G<;fjPvopR`x3Y@%^BaODhcWB)2zZlc0X6yYkU5hSv$>Nn`Dk)6E1d0|f
z6MHNu%8d4@%yYOy$1JeZBG#QzOs%aFX&w0Uc1FoGbyc-k-26}R-aApdj9)e!;cIx$
zoQV>U^_OElm4@s6m#aLFJSrF>`!PaAK2wNM%BnhF+)I`;iF(FaG#!^Sqp*j)Gn23T
zGLBTNCk2JO!rt5UZsoI*F`PBYVE|HMsRJ}4JfPSSncH3xv2LJI4x=MJFE`s^g=^p_
z?y-4|CK=m&dI3*{DgDwDK8-x(aI!1rOkS9O>#OuI%$+6IL2_2@FFl=N;@E4RBgX~N
z$9W${s1TQw6qf{D|2+08ZdY@!da64SMnaY>oF@!5p`Urk{cq4+ZMe-D_6H)T~ocJnyw
zIm^%bI_++S1Ix$Td8@ez8S8ZbRUvoZ#GH$h`S_c|QYI81_qtzoKpJ5+?eo=AIxhfd
z@u|_g=4h~x
zkB#Ri&Ewmir>OEh+o2msmDNnMGpZBcfS*34ejhi6maPESY2zKb5a!q_T|2UZg$IEg
zST{FB<7X?zr{6d8ys*jHhk1YN@NF!`)nV+YNp|s(avGyk%-2vK4WG_iyJMS$)}N()
znP1%=yUgz(rb~i6*13+^&MP@;3NqFp(ose%tjq|VGfws?m(#EUFrp&(pNpPINcS+u
z4KXrmg-ygNRfMdRQf8ZDqC!%K5r&ppB$|kN?t37_f!?A7T@7a@!>4c$#n_!M;ao%I
z-TyZF7t{aNQU9Z%pddU}@;xff+e`-djLh7b8V1rnOlj_UOer(WkN>F|`4R%Rui6KD
zai3usI=3t{nYw=a>*$k2qJZRzJ@lC8;uLgUdU6GG!{N_7LWQEX8JBGXZjbvK;tL0=+p!ap4uckPWH#2bku&Ar(tRRi6^tfTd#f2tn&2G
z5D)*a$m{kAfoCMydwYZVLEqn&tB^hJs>g$$rpHStn*9q6H%m87x(KWTF&07}gm?}=
zgc~``>{hHT*CuN`UwD`~Fp=%q3W0^?m#PMCS`EjC2s0N-@#v=Vj?*rm)`aQZQV=Yo
zj!AWiOx)XgE?H
z)@Qa&oR5_?wPG~_q17;(WA13maEasW$c0@SQP-7&YnjU{&7XW$0FXQTagN+*3pVs8
z!uB&-u?=P6*tAOEBx?h07oh~+yDx%n;wMYHO-xH$i^X#ITItURT2{%YkBE}dCFuL=
z7iW?mkO2#j;4_u=f-d2IWih-AtKH?n5?~GUuV&vhnm+y*dGa)af*&gPO3LOwecs6%
zBE#Xb=4GAb!u_hR$-_#CnsyFY9s49%NkI
z4@_(PE)2pnr)GxUtfVwvuByHG!e@k%aIuAiioru%E_eiV8M4lw~_U`s$c>3#?jB`@=^?D*vI0~#IFSt!Vp?;Q2
zDOkrdOD9hn%jRNe<2#e?F5q~?G+Cih#@ZqP3Ct7
z1ph|;O?7zO?pQn)N|2;)N+Q62^gLSAICi|1zGM
z#sqcqvOPr7ysWged@#~}4NTvBNTBl^Nni#)dOj?^pF%_)?k&l2HgC|qX%Rl|CzSZ#
z8ZAAl8zPf`jT4

literal 0
HcmV?d00001

diff --git a/tutorials/命名实体识别.ipynb b/tutorials/命名实体识别.ipynb
new file mode 100644
index 00000000..95975f2c
--- /dev/null
+++ b/tutorials/命名实体识别.ipynb
@@ -0,0 +1,41 @@
+{
+ "cells": [
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "##1. 命名实体识别(name entity recognition, NER)\n",
+    "命名实体识别任务是从文本中抽取出具有特殊意义或者指代性非常强的实体，通常包括人名、地名、机构名和时间等。\n",
+    "如下面的例子中\n",
+    "\n",
+    "我来自复旦大学。\n",
+    "\n",
+    "其中“复旦大学”就是一个机构名，命名实体识别就是要从中识别出“复旦大学”这四个字是一个整体，且属于机构名这个类别。这个问题现在一般被转换为了\n",
+    "在本tutorial中我们将通过fastNLP尝试写出一个\n",
+    "\n",
+    "##2. 数据\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/tutorials/文本分类.ipynb b/tutorials/文本分类.ipynb
new file mode 100644
index 00000000..de29f632
--- /dev/null
+++ b/tutorials/文本分类.ipynb
@@ -0,0 +1,834 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 文本分类(Text classification)\n",
+    "文本分类任务是将一句话或一段话划分到某个具体的类别。比如垃圾邮件识别，文本情绪分类等。\n",
+    "\n",
+    "Example::   \n",
+    "1,商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!\n",
+    "\n",
+    "\n",
+    "其中开头的1是只这条评论的标签，表示是正面的情绪。我们将使用到的数据可以通过http://dbcloud.irocn.cn:8989/api/public/dl/dataset/chn_senti_corp.zip 下载并解压，当然也可以通过fastNLP自动下载该数据。\n",
+    "\n",
+    "数据中的内容如下图所示。接下来，我们将用fastNLP在这个数据上训练一个分类网络。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "![jupyter](./cn_cls_example.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 步骤\n",
+    "一共有以下的几个步骤  \n",
+    "(1) 读取数据  \n",
+    "(2) 预处理数据  \n",
+    "(3) 选择预训练词向量  \n",
+    "(4) 创建模型  \n",
+    "(5) 训练模型  "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### (1) 读取数据\n",
+    "fastNLP提供多种数据的自动下载与自动加载功能，对于这里我们要用到的数据，我们可以用\\ref{Loader}自动下载并加载该数据。更多有关Loader的使用可以参考\\ref{Loader}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from fastNLP.io import ChnSentiCorpLoader\n",
+    "\n",
+    "loader = ChnSentiCorpLoader()  # 初始化一个中文情感分类的loader\n",
+    "data_dir = loader.download()  # 这一行代码将自动下载数据到默认的缓存地址, 并将该地址返回\n",
+    "data_bundle = loader.load(data_dir)  # 这一行代码将从{data_dir}处读取数据至DataBundle"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "DataBundle的相关介绍，可以参考\\ref{}。我们可以打印该data_bundle的基本信息。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "In total 3 datasets:\n",
+      "\tdev has 1200 instances.\n",
+      "\ttrain has 9600 instances.\n",
+      "\ttest has 1200 instances.\n",
+      "In total 0 vocabs:\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(data_bundle)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "可以看出，该data_bundle中一个含有三个\\ref{DataSet}。通过下面的代码，我们可以查看DataSet的基本情况"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DataSet({'raw_chars': 选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。酒店装修一般，但还算整洁。 泳池在大堂的屋顶，因此很小，不过女儿倒是喜欢。 包的早餐是西式的，还算丰富。 服务吗，一般 type=str,\n",
+      "'target': 1 type=str},\n",
+      "{'raw_chars': 15.4寸笔记本的键盘确实爽，基本跟台式机差不多了，蛮喜欢数字小键盘，输数字特方便，样子也很美观，做工也相当不错 type=str,\n",
+      "'target': 1 type=str})\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(data_bundle.get_dataset('train')[:2])  # 查看Train集前两个sample"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### (2) 预处理数据\n",
+    "在NLP任务中，预处理一般包括: (a)将一整句话切分成汉字或者词; (b)将文本转换为index  \n",
+    "\n",
+    "fastNLP中也提供了多种数据集的处理类，这里我们直接使用fastNLP的ChnSentiCorpPipe。更多关于Pipe的说明可以参考\\ref{Pipe}。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from fastNLP.io import ChnSentiCorpPipe\n",
+    "\n",
+    "pipe = ChnSentiCorpPipe()\n",
+    "data_bundle = pipe.process(data_bundle)  # 所有的Pipe都实现了process()方法，且输入输出都为DataBundle类型"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "In total 3 datasets:\n",
+      "\tdev has 1200 instances.\n",
+      "\ttrain has 9600 instances.\n",
+      "\ttest has 1200 instances.\n",
+      "In total 2 vocabs:\n",
+      "\tchars has 4409 entries.\n",
+      "\ttarget has 2 entries.\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(data_bundle)  # 打印data_bundle，查看其变化"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "可以看到除了之前已经包含的3个\\ref{DataSet}, 还新增了两个\\ref{Vocabulary}。我们可以打印DataSet中的内容"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DataSet({'raw_chars': 选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。酒店装修一般，但还算整洁。 泳池在大堂的屋顶，因此很小，不过女儿倒是喜欢。 包的早餐是西式的，还算丰富。 服务吗，一般 type=str,\n",
+      "'target': 1 type=int,\n",
+      "'chars': [338, 464, 1400, 784, 468, 739, 3, 289, 151, 21, 5, 88, 143, 2, 9, 81, 134, 2573, 766, 233, 196, 23, 536, 342, 297, 2, 405, 698, 132, 281, 74, 744, 1048, 74, 420, 387, 74, 412, 433, 74, 2021, 180, 8, 219, 1929, 213, 4, 34, 31, 96, 363, 8, 230, 2, 66, 18, 229, 331, 768, 4, 11, 1094, 479, 17, 35, 593, 3, 1126, 967, 2, 151, 245, 12, 44, 2, 6, 52, 260, 263, 635, 5, 152, 162, 4, 11, 336, 3, 154, 132, 5, 236, 443, 3, 2, 18, 229, 761, 700, 4, 11, 48, 59, 653, 2, 8, 230] type=list,\n",
+      "'seq_len': 106 type=int},\n",
+      "{'raw_chars': 15.4寸笔记本的键盘确实爽，基本跟台式机差不多了，蛮喜欢数字小键盘，输数字特方便，样子也很美观，做工也相当不错 type=str,\n",
+      "'target': 1 type=int,\n",
+      "'chars': [50, 133, 20, 135, 945, 520, 343, 24, 3, 301, 176, 350, 86, 785, 2, 456, 24, 461, 163, 443, 128, 109, 6, 47, 7, 2, 916, 152, 162, 524, 296, 44, 301, 176, 2, 1384, 524, 296, 259, 88, 143, 2, 92, 67, 26, 12, 277, 269, 2, 188, 223, 26, 228, 83, 6, 63] type=list,\n",
+      "'seq_len': 56 type=int})\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(data_bundle.get_dataset('train')[:2])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "新增了一列为数字列表的chars，以及变为数字的target列。可以看出这两列的名称和刚好与data_bundle中两个Vocabulary的名称是一致的，我们可以打印一下Vocabulary看一下里面的内容。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Vocabulary(['选', '择', '珠', '江', '花']...)\n"
+     ]
+    }
+   ],
+   "source": [
+    "char_vocab = data_bundle.get_vocab('chars')\n",
+    "print(char_vocab)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Vocabulary是一个记录着词语与index之间映射关系的类，比如"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "'选'的index是338\n",
+      "index:338对应的汉字是选\n"
+     ]
+    }
+   ],
+   "source": [
+    "index = char_vocab.to_index('选')\n",
+    "print(\"'选'的index是{}\".format(index))  # 这个值与上面打印出来的第一个instance的chars的第一个index是一致的\n",
+    "print(\"index:{}对应的汉字是{}\".format(index, char_vocab.to_word(index))) "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### (3) 选择预训练词向量  \n",
+    "由于Word2vec, Glove, Elmo, Bert等预训练模型可以增强模型的性能，所以在训练具体任务前，选择合适的预训练词向量非常重要。在fastNLP中我们提供了多种Embedding使得加载这些预训练模型的过程变得更加便捷。更多关于Embedding的说明可以参考\\ref{Embedding}。这里我们先给出一个使用word2vec的中文汉字预训练的示例，之后再给出一个使用Bert的文本分类。这里使用的预训练词向量为'cn-fastnlp-100d'，fastNLP将自动下载该embedding至本地缓存，fastNLP支持使用名字指定的Embedding以及相关说明可以参见\\ref{Embedding}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Found 4321 out of 4409 words in the pre-training embedding.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from fastNLP.embeddings import StaticEmbedding\n",
+    "\n",
+    "word2vec_embed = StaticEmbedding(char_vocab, model_dir_or_name='cn-char-fastnlp-100d')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### (4) 创建模型\n",
+    "这里我们使用到的模型结构如下所示，补图"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torch import nn\n",
+    "from fastNLP.modules import LSTM\n",
+    "import torch\n",
+    "\n",
+    "# 定义模型\n",
+    "class BiLSTMMaxPoolCls(nn.Module):\n",
+    "    def __init__(self, embed, num_classes, hidden_size=400, num_layers=1, dropout=0.3):\n",
+    "        super().__init__()\n",
+    "        self.embed = embed\n",
+    "        \n",
+    "        self.lstm = LSTM(self.embed.embedding_dim, hidden_size=hidden_size//2, num_layers=num_layers, \n",
+    "                         batch_first=True, bidirectional=True)\n",
+    "        self.dropout_layer = nn.Dropout(dropout)\n",
+    "        self.fc = nn.Linear(hidden_size, num_classes)\n",
+    "        \n",
+    "    def forward(self, chars, seq_len):  # 这里的名称必须和DataSet中相应的field对应，比如之前我们DataSet中有chars，这里就必须为chars\n",
+    "        # chars:[batch_size, max_len]\n",
+    "        # seq_len: [batch_size, ]\n",
+    "        chars = self.embed(chars)\n",
+    "        outputs, _ = self.lstm(chars, seq_len)\n",
+    "        outputs = self.dropout_layer(outputs)\n",
+    "        outputs, _ = torch.max(outputs, dim=1)\n",
+    "        outputs = self.fc(outputs)\n",
+    "        \n",
+    "        return {'pred':outputs}  # [batch_size,], 返回值必须是dict类型，且预测值的key建议设为pred\n",
+    "\n",
+    "# 初始化模型\n",
+    "model = BiLSTMMaxPoolCls(word2vec_embed, len(data_bundle.get_vocab('target')))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### (5) 训练模型\n",
+    "fastNLP提供了Trainer对象来组织训练过程，包括完成loss计算(所以在初始化Trainer的时候需要指定loss类型)，梯度更新(所以在初始化Trainer的时候需要提供优化器optimizer)以及在验证集上的性能验证(所以在初始化时需要提供一个Metric)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "input fields after batch(if batch size is 2):\n",
+      "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "\tchars: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 106]) \n",
+      "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "target fields after batch(if batch size is 2):\n",
+      "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "\n",
+      "Evaluate data in 0.01 seconds!\n",
+      "training epochs started 2019-09-03-23-57-10\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3000), HTML(value='')), layout=Layout(display…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.43 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 1/10. Step:300/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.81\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.44 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 2/10. Step:600/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.8675\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.44 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 3/10. Step:900/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.878333\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.43 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 4/10. Step:1200/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.873333\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.44 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 5/10. Step:1500/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.878333\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.42 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 6/10. Step:1800/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.895833\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.44 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 7/10. Step:2100/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.8975\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.43 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 8/10. Step:2400/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.894167\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.48 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 9/10. Step:2700/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.8875\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=38), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.43 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 10/10. Step:3000/3000: \n",
+      "\r",
+      "AccuracyMetric: acc=0.895833\n",
+      "\n",
+      "\r\n",
+      "In Epoch:7/Step:2100, got best dev performance:\n",
+      "AccuracyMetric: acc=0.8975\n",
+      "Reloaded the best model.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=19), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 0.34 seconds!\n",
+      "[tester] \n",
+      "AccuracyMetric: acc=0.8975\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'AccuracyMetric': {'acc': 0.8975}}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from fastNLP import Trainer\n",
+    "from fastNLP import CrossEntropyLoss\n",
+    "from torch.optim import Adam\n",
+    "from fastNLP import AccuracyMetric\n",
+    "\n",
+    "loss = CrossEntropyLoss()\n",
+    "optimizer = Adam(model.parameters(), lr=0.001)\n",
+    "metric = AccuracyMetric()\n",
+    "device = 0 if torch.cuda.is_available() else 'cpu'  # 如果有gpu的话在gpu上运行，训练速度会更快\n",
+    "\n",
+    "trainer = Trainer(train_data=data_bundle.get_dataset('train'), model=model, loss=loss, \n",
+    "                  optimizer=optimizer, batch_size=32, dev_data=data_bundle.get_dataset('dev'),\n",
+    "                  metrics=metric, device=device)\n",
+    "trainer.train()  # 开始训练，训练完成之后默认会加载在dev上表现最好的模型\n",
+    "\n",
+    "# 在测试集上测试一下模型的性能\n",
+    "from fastNLP import Tester\n",
+    "print(\"Performance on test is:\")\n",
+    "tester = Tester(data=data_bundle.get_dataset('test'), model=model, metrics=metric, batch_size=64, device=device)\n",
+    "tester.test()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 使用Bert进行文本分类"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading vocabulary file /home/yh/.fastNLP/embedding/bert-chinese-wwm/vocab.txt\n",
+      "Load pre-trained BERT parameters from file /home/yh/.fastNLP/embedding/bert-chinese-wwm/chinese_wwm_pytorch.bin.\n",
+      "Start to generating word pieces for word.\n",
+      "Found(Or segment into word pieces) 4286 words out of 4409.\n",
+      "input fields after batch(if batch size is 2):\n",
+      "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "\tchars: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 106]) \n",
+      "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "target fields after batch(if batch size is 2):\n",
+      "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
+      "\n",
+      "Evaluate data in 0.05 seconds!\n",
+      "training epochs started 2019-09-04-00-02-37\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=3600), HTML(value='')), layout=Layout(display…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=150), HTML(value='')), layout=Layout(display=…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 15.89 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 1/3. Step:1200/3600: \n",
+      "\r",
+      "AccuracyMetric: acc=0.9\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=150), HTML(value='')), layout=Layout(display=…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 15.92 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 2/3. Step:2400/3600: \n",
+      "\r",
+      "AccuracyMetric: acc=0.904167\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=150), HTML(value='')), layout=Layout(display=…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 15.91 seconds!\n",
+      "\r",
+      "Evaluation on dev at Epoch 3/3. Step:3600/3600: \n",
+      "\r",
+      "AccuracyMetric: acc=0.918333\n",
+      "\n",
+      "\r\n",
+      "In Epoch:3/Step:3600, got best dev performance:\n",
+      "AccuracyMetric: acc=0.918333\n",
+      "Reloaded the best model.\n",
+      "Performance on test is:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=19), HTML(value='')), layout=Layout(display='…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Evaluate data in 29.24 seconds!\n",
+      "[tester] \n",
+      "AccuracyMetric: acc=0.919167\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'AccuracyMetric': {'acc': 0.919167}}"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 只需要切换一下Embedding即可\n",
+    "from fastNLP.embeddings import BertEmbedding\n",
+    "\n",
+    "# 这里为了演示一下效果，所以默认Bert不更新权重\n",
+    "bert_embed = BertEmbedding(char_vocab, model_dir_or_name='cn', auto_truncate=True, requires_grad=False)\n",
+    "model = BiLSTMMaxPoolCls(bert_embed, len(data_bundle.get_vocab('target')), )\n",
+    "\n",
+    "\n",
+    "import torch\n",
+    "from fastNLP import Trainer\n",
+    "from fastNLP import CrossEntropyLoss\n",
+    "from torch.optim import Adam\n",
+    "from fastNLP import AccuracyMetric\n",
+    "\n",
+    "loss = CrossEntropyLoss()\n",
+    "optimizer = Adam(model.parameters(), lr=2e-5)\n",
+    "metric = AccuracyMetric()\n",
+    "device = 0 if torch.cuda.is_available() else 'cpu'  # 如果有gpu的话在gpu上运行，训练速度会更快\n",
+    "\n",
+    "trainer = Trainer(train_data=data_bundle.get_dataset('train'), model=model, loss=loss, \n",
+    "                  optimizer=optimizer, batch_size=16, dev_data=data_bundle.get_dataset('test'),\n",
+    "                  metrics=metric, device=device, n_epochs=3)\n",
+    "trainer.train()  # 开始训练，训练完成之后默认会加载在dev上表现最好的模型\n",
+    "\n",
+    "# 在测试集上测试一下模型的性能\n",
+    "from fastNLP import Tester\n",
+    "print(\"Performance on test is:\")\n",
+    "tester = Tester(data=data_bundle.get_dataset('test'), model=model, metrics=metric, batch_size=64, device=device)\n",
+    "tester.test()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

From bf2920cba98ff6c534ccbc5c16fe0942411cb360 Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 15:09:20 +0800
Subject: [PATCH 80/92] update reproduction/matching/matching_bert.py

---
 reproduction/matching/matching_bert.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/reproduction/matching/matching_bert.py b/reproduction/matching/matching_bert.py
index 323d81a3..05377dff 100644
--- a/reproduction/matching/matching_bert.py
+++ b/reproduction/matching/matching_bert.py
@@ -8,8 +8,7 @@ from fastNLP.core.optimizer import AdamW
 from fastNLP.embeddings import BertEmbedding
 from fastNLP.io.pipe.matching import SNLIBertPipe, RTEBertPipe, MNLIBertPipe,\
     QNLIBertPipe, QuoraBertPipe
-
-from reproduction.matching.model.bert import BertForNLI
+from fastNLP.models.bert import BertForSentenceMatching
 
 
 # define hyper-parameters
@@ -65,7 +64,7 @@ print(data_bundle)  # print details in data_bundle
 embed = BertEmbedding(data_bundle.vocabs[Const.INPUT], model_dir_or_name=arg.bert_model_dir_or_name)
 
 # define model
-model = BertForNLI(embed, class_num=len(data_bundle.vocabs[Const.TARGET]))
+model = BertForSentenceMatching(embed, num_labels=len(data_bundle.vocabs[Const.TARGET]))
 
 # define optimizer and callback
 optimizer = AdamW(lr=arg.lr, params=model.parameters())
@@ -76,11 +75,11 @@ if arg.task in ['snli']:
     # evaluate test set in every epoch if task is snli.
 
 # define trainer
-trainer = Trainer(train_data=data_bundle.datasets[arg.train_dataset_name], model=model,
+trainer = Trainer(train_data=data_bundle.get_dataset(arg.train_dataset_name), model=model,
                   optimizer=optimizer,
                   batch_size=torch.cuda.device_count() * arg.batch_size_per_gpu,
                   n_epochs=arg.n_epochs, print_every=-1,
-                  dev_data=data_bundle.datasets[arg.dev_dataset_name],
+                  dev_data=data_bundle.get_dataset(arg.dev_dataset_name),
                   metrics=AccuracyMetric(), metric_key='acc',
                   device=[i for i in range(torch.cuda.device_count())],
                   check_code_level=-1,
@@ -92,7 +91,7 @@ trainer.train(load_best_model=True)
 
 # define tester
 tester = Tester(
-    data=data_bundle.datasets[arg.test_dataset_name],
+    data=data_bundle.get_dataset(arg.test_dataset_name),
     model=model,
     metrics=AccuracyMetric(),
     batch_size=torch.cuda.device_count() * arg.batch_size_per_gpu,

From 9b21071c8d01087e37c953c7e3e37e20b29b901d Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 15:36:59 +0800
Subject: [PATCH 81/92] delete online Data Getter and Iter

---
 fastNLP/core/batch.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py
index b14b21de..1a31e92a 100644
--- a/fastNLP/core/batch.py
+++ b/fastNLP/core/batch.py
@@ -201,19 +201,6 @@ class TorchLoaderIter(BatchIter):
         self.batch_size = dataset.batch_size
 
 
-class OnlineDataGettter:
-    # TODO
-    pass
-
-
-class OnlineDataIter(BatchIter):
-    # TODO
-    def __init__(self, dataset, batch_size=1, buffer_size=10000, sampler=None, as_numpy=False,
-                 num_workers=0, pin_memory=False, drop_last=False,
-                 timeout=0, worker_init_fn=None, **kwargs):
-        super().__init__()
-
-
 def _to_tensor(batch, field_dtype):
     """
 

From 7b08c777bc88f37db3b23c0e1460354580a56e5c Mon Sep 17 00:00:00 2001
From: ChenXin 
Date: Mon, 16 Sep 2019 15:39:43 +0800
Subject: [PATCH 82/92] Delete dev.tsv

---
 test/data_for_tests/io/rte/dev.tsv | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 test/data_for_tests/io/rte/dev.tsv

diff --git a/test/data_for_tests/io/rte/dev.tsv b/test/data_for_tests/io/rte/dev.tsv
deleted file mode 100644
index 725d7542..00000000
--- a/test/data_for_tests/io/rte/dev.tsv
+++ /dev/null
@@ -1,3 +0,0 @@
-index	sentence1	sentence2	label
-0	Dana Reeve, the widow of the actor Christopher Reeve, has died of lung cancer at age 44, according to the Christopher Reeve Foundation.	Christopher Reeve had an accident.	not_entailment
-1	Yet, we now are discovering that antibiotics are losing their effectiveness against illness. Disease-causing bacteria are mutating faster than we can come up with new antibiotics to fight the new variations.	Bacteria is winning the war against antibiotics.	entailment

From f2face3b406cb3ad4424e8ee4f7c099c97d5f04a Mon Sep 17 00:00:00 2001
From: ChenXin 
Date: Mon, 16 Sep 2019 15:39:58 +0800
Subject: [PATCH 83/92] Delete test.tsv

---
 test/data_for_tests/io/rte/test.tsv | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 test/data_for_tests/io/rte/test.tsv

diff --git a/test/data_for_tests/io/rte/test.tsv b/test/data_for_tests/io/rte/test.tsv
deleted file mode 100644
index aeceb467..00000000
--- a/test/data_for_tests/io/rte/test.tsv
+++ /dev/null
@@ -1,3 +0,0 @@
-index	sentence1	sentence2
-0	Mangla was summoned after Madhumita's sister Nidhi Shukla, who was the first witness in the case.	Shukla is related to Mangla.
-1	Authorities in Brazil say that more than 200 people are being held hostage in a prison in the country's remote, Amazonian-jungle state of Rondonia.	Authorities in Brazil hold 200 people as hostage.

From d582bd3e15945c2927155a2d2f8b996c8cb0c16c Mon Sep 17 00:00:00 2001
From: ChenXin 
Date: Mon, 16 Sep 2019 15:40:18 +0800
Subject: [PATCH 84/92] Delete train.tsv

---
 test/data_for_tests/io/rte/train.tsv | 4 ----
 1 file changed, 4 deletions(-)
 delete mode 100644 test/data_for_tests/io/rte/train.tsv

diff --git a/test/data_for_tests/io/rte/train.tsv b/test/data_for_tests/io/rte/train.tsv
deleted file mode 100644
index 9f3dab6e..00000000
--- a/test/data_for_tests/io/rte/train.tsv
+++ /dev/null
@@ -1,4 +0,0 @@
-index	sentence1	sentence2	label
-0	No Weapons of Mass Destruction Found in Iraq Yet.	Weapons of Mass Destruction Found in Iraq.	not_entailment
-1	A place of sorrow, after Pope John Paul II died, became a place of celebration, as Roman Catholic faithful gathered in downtown Chicago to mark the installation of new Pope Benedict XVI.	Pope Benedict XVI is the new leader of the Roman Catholic Church.	entailment
-2	Herceptin was already approved to treat the sickest breast cancer patients, and the company said, Monday, it will discuss with federal regulators the possibility of prescribing the drug for more breast cancer patients.	Herceptin can be used to treat breast cancer.	entailment

From 30180b91e12325a4be4e168d86ddcd3ab1d6751e Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 15:45:02 +0800
Subject: [PATCH 85/92] Revert "delete online Data Getter and Iter"

This reverts commit 9b21071c8d01087e37c953c7e3e37e20b29b901d.
---
 fastNLP/core/batch.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py
index 1a31e92a..b14b21de 100644
--- a/fastNLP/core/batch.py
+++ b/fastNLP/core/batch.py
@@ -201,6 +201,19 @@ class TorchLoaderIter(BatchIter):
         self.batch_size = dataset.batch_size
 
 
+class OnlineDataGettter:
+    # TODO
+    pass
+
+
+class OnlineDataIter(BatchIter):
+    # TODO
+    def __init__(self, dataset, batch_size=1, buffer_size=10000, sampler=None, as_numpy=False,
+                 num_workers=0, pin_memory=False, drop_last=False,
+                 timeout=0, worker_init_fn=None, **kwargs):
+        super().__init__()
+
+
 def _to_tensor(batch, field_dtype):
     """
 

From a298021a5671dad68abe760e7460ebafeb48b77a Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 15:46:42 +0800
Subject: [PATCH 86/92] delete Online Data Getter and Iter

---
 fastNLP/core/batch.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py
index b14b21de..1a31e92a 100644
--- a/fastNLP/core/batch.py
+++ b/fastNLP/core/batch.py
@@ -201,19 +201,6 @@ class TorchLoaderIter(BatchIter):
         self.batch_size = dataset.batch_size
 
 
-class OnlineDataGettter:
-    # TODO
-    pass
-
-
-class OnlineDataIter(BatchIter):
-    # TODO
-    def __init__(self, dataset, batch_size=1, buffer_size=10000, sampler=None, as_numpy=False,
-                 num_workers=0, pin_memory=False, drop_last=False,
-                 timeout=0, worker_init_fn=None, **kwargs):
-        super().__init__()
-
-
 def _to_tensor(batch, field_dtype):
     """
 

From e0c86346619606f926e22c13140d56b36182817c Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 16:16:23 +0800
Subject: [PATCH 87/92] add chinese char-level tokenizer

---
 fastNLP/io/pipe/matching.py |  4 ++--
 fastNLP/io/pipe/utils.py    | 21 +++++++++++++++------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/fastNLP/io/pipe/matching.py b/fastNLP/io/pipe/matching.py
index 7620a556..d6506f66 100644
--- a/fastNLP/io/pipe/matching.py
+++ b/fastNLP/io/pipe/matching.py
@@ -51,7 +51,7 @@ class MatchingBertPipe(Pipe):
         super().__init__()
         
         self.lower = bool(lower)
-        self.tokenizer = get_tokenizer(tokenizer=tokenizer)
+        self.tokenizer = get_tokenizer(tokenize_method=tokenizer)
     
     def _tokenize(self, data_bundle, field_names, new_field_names):
         """
@@ -191,7 +191,7 @@ class MatchingPipe(Pipe):
         super().__init__()
         
         self.lower = bool(lower)
-        self.tokenizer = get_tokenizer(tokenizer=tokenizer)
+        self.tokenizer = get_tokenizer(tokenize_method=tokenizer)
     
     def _tokenize(self, data_bundle, field_names, new_field_names):
         """
diff --git a/fastNLP/io/pipe/utils.py b/fastNLP/io/pipe/utils.py
index 92d61bfd..4925853f 100644
--- a/fastNLP/io/pipe/utils.py
+++ b/fastNLP/io/pipe/utils.py
@@ -65,27 +65,36 @@ def iob2bioes(tags: List[str]) -> List[str]:
     return new_tags
 
 
-def get_tokenizer(tokenizer: str, lang='en'):
+def get_tokenizer(tokenize_method: str, lang='en'):
     """
 
-    :param str tokenizer: 获取tokenzier方法
+    :param str tokenize_method: 获取tokenzier方法
     :param str lang: 语言，当前仅支持en
     :return: 返回tokenize函数
     """
-    if tokenizer == 'spacy':
+    tokenizer_dict = {
+        'spacy': None,
+        'raw': _raw_split,
+        'cn-char': _cn_char_split,
+    }
+    if tokenize_method == 'spacy':
         import spacy
         spacy.prefer_gpu()
         if lang != 'en':
             raise RuntimeError("Spacy only supports en right right.")
         en = spacy.load(lang)
         tokenizer = lambda x: [w.text for w in en.tokenizer(x)]
-    elif tokenizer == 'raw':
-        tokenizer = _raw_split
+    elif tokenize_method in tokenizer_dict:
+        tokenizer = tokenizer_dict[tokenize_method]
     else:
-        raise RuntimeError("Only support `spacy`, `raw` tokenizer.")
+        raise RuntimeError(f"Only support {tokenizer_dict.keys()} tokenizer.")
     return tokenizer
 
 
+def _cn_char_split(sent):
+    return [chars for chars in sent]
+
+
 def _raw_split(sent):
     return sent.split()
 

From e37e1e2a0ef70348c9b56217ae722e2a953a6cc2 Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 16:30:43 +0800
Subject: [PATCH 88/92] update document in fastNLP/core/const.py

---
 fastNLP/core/const.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/fastNLP/core/const.py b/fastNLP/core/const.py
index ad5d1f1e..9bcea2d6 100644
--- a/fastNLP/core/const.py
+++ b/fastNLP/core/const.py
@@ -1,6 +1,5 @@
-"""
-.. todo::
-    doc
+r"""
+fastNLP包当中的field命名均符合一定的规范，该规范由fastNLP.Const类进行定义。
 """
 
 __all__ = [
@@ -50,11 +49,13 @@ class Const:
     
     @staticmethod
     def RAW_WORDS(i):
+        """得到第 i 个 ``RAW_WORDS`` 的命名"""
         i = int(i) + 1
         return Const.RAW_WORD + str(i)
     
     @staticmethod
     def RAW_CHARS(i):
+        """得到第 i 个 ``RAW_CHARS`` 的命名"""
         i = int(i) + 1
         return Const.RAW_CHAR + str(i)
     

From 9b2317e3e8dcc82bf1eae1f93885e682895039d1 Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 16:44:37 +0800
Subject: [PATCH 89/92] delete util functions and test cases in
 fastNLP/core/losses.py

---
 fastNLP/core/losses.py | 79 ------------------------------------------
 test/core/test_loss.py | 13 -------
 2 files changed, 92 deletions(-)

diff --git a/fastNLP/core/losses.py b/fastNLP/core/losses.py
index 9b32babb..92f2f364 100644
--- a/fastNLP/core/losses.py
+++ b/fastNLP/core/losses.py
@@ -352,82 +352,3 @@ def _prepare_losser(losser):
         return losser
     else:
         raise TypeError(f"Type of loss should be `fastNLP.LossBase`, got {type(losser)}")
-
-
-def squash(predict, truth, **kwargs):
-    """To reshape tensors in order to fit loss functions in PyTorch.
-
-    :param predict: Tensor, model output
-    :param truth: Tensor, truth from dataset
-    :param kwargs: extra arguments
-    :return predict , truth: predict & truth after processing
-    """
-    return predict.view(-1, predict.size()[-1]), truth.view(-1, )
-
-
-def unpad(predict, truth, **kwargs):
-    """To process padded sequence output to get true loss.
-
-    :param predict: Tensor, [batch_size , max_len , tag_size]
-    :param truth: Tensor, [batch_size , max_len]
-    :param kwargs: kwargs["lens"] is a list or LongTensor, with size [batch_size]. The i-th element is true lengths of i-th sequence.
-
-    :return predict , truth: predict & truth after processing
-    """
-    if kwargs.get("lens") is None:
-        return predict, truth
-    lens = torch.LongTensor(kwargs["lens"])
-    lens, idx = torch.sort(lens, descending=True)
-    predict = torch.nn.utils.rnn.pack_padded_sequence(predict[idx], lens, batch_first=True).data
-    truth = torch.nn.utils.rnn.pack_padded_sequence(truth[idx], lens, batch_first=True).data
-    return predict, truth
-
-
-def unpad_mask(predict, truth, **kwargs):
-    """To process padded sequence output to get true loss.
-
-    :param predict: Tensor, [batch_size , max_len , tag_size]
-    :param truth: Tensor, [batch_size , max_len]
-    :param kwargs: kwargs["lens"] is a list or LongTensor, with size [batch_size]. The i-th element is true lengths of i-th sequence.
-
-    :return predict , truth: predict & truth after processing
-    """
-    if kwargs.get("lens") is None:
-        return predict, truth
-    mas = make_mask(kwargs["lens"], truth.size()[1])
-    return mask(predict, truth, mask=mas)
-
-
-def mask(predict, truth, **kwargs):
-    """To select specific elements from Tensor. This method calls ``squash()``.
-
-    :param predict: Tensor, [batch_size , max_len , tag_size]
-    :param truth: Tensor, [batch_size , max_len]
-    :param kwargs: extra arguments, kwargs["mask"]: ByteTensor, [batch_size , max_len], the mask Tensor. The position that is 1 will be selected.
-
-    :return predict , truth: predict & truth after processing
-    """
-    if kwargs.get("mask") is None:
-        return predict, truth
-    mask = kwargs["mask"]
-    
-    predict, truth = squash(predict, truth)
-    mask = mask.view(-1, )
-    
-    predict = torch.masked_select(predict.permute(1, 0), mask).view(predict.size()[-1], -1).permute(1, 0)
-    truth = torch.masked_select(truth, mask)
-    
-    return predict, truth
-
-
-def make_mask(lens, tar_len):
-    """To generate a mask over a sequence.
-
-    :param lens: list or LongTensor, [batch_size]
-    :param tar_len: int
-    :return mask: ByteTensor
-    """
-    lens = torch.LongTensor(lens)
-    mask = [torch.ge(lens, i + 1) for i in range(tar_len)]
-    mask = torch.stack(mask, 1)
-    return mask
diff --git a/test/core/test_loss.py b/test/core/test_loss.py
index 8db54615..9ba8159f 100644
--- a/test/core/test_loss.py
+++ b/test/core/test_loss.py
@@ -4,7 +4,6 @@ import torch
 import torch.nn.functional as F
 
 import fastNLP as loss
-from fastNLP.core.losses import squash, unpad
 
 
 class TestLoss(unittest.TestCase):
@@ -73,15 +72,3 @@ class TestLosserError(unittest.TestCase):
         
         with self.assertRaises(Exception):
             ans = l1({"my_predict": a}, {"truth": b, "my": a})
-
-
-class TestLossUtils(unittest.TestCase):
-    def test_squash(self):
-        a, b = squash(torch.randn(3, 5), torch.randn(3, 5))
-        self.assertEqual(tuple(a.size()), (3, 5))
-        self.assertEqual(tuple(b.size()), (15,))
-    
-    def test_unpad(self):
-        a, b = unpad(torch.randn(5, 8, 3), torch.randn(5, 8))
-        self.assertEqual(tuple(a.size()), (5, 8, 3))
-        self.assertEqual(tuple(b.size()), (5, 8))

From 5768cbbfeff6f17c78d581e99755f0bc6c3f8e2f Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Mon, 16 Sep 2019 17:02:16 +0800
Subject: [PATCH 90/92] add test code in AdamW

---
 fastNLP/core/optimizer.py   |  9 ++++++---
 test/core/test_optimizer.py | 11 ++++++++++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/fastNLP/core/optimizer.py b/fastNLP/core/optimizer.py
index b534a72a..4d76c24e 100644
--- a/fastNLP/core/optimizer.py
+++ b/fastNLP/core/optimizer.py
@@ -33,8 +33,9 @@ class Optimizer(object):
     
     def construct_from_pytorch(self, model_params):
         raise NotImplementedError
-    
-    def _get_require_grads_param(self, params):
+
+    @staticmethod
+    def _get_require_grads_param(params):
         """
         将params中不需要gradient的删除
         
@@ -43,6 +44,7 @@ class Optimizer(object):
         """
         return [param for param in params if param.requires_grad]
 
+
 class NullOptimizer(Optimizer):
     """
     当不希望Trainer更新optimizer时，传入本optimizer，但请确保通过callback的方式对参数进行了更新。
@@ -113,7 +115,8 @@ class Adam(Optimizer):
 
 class AdamW(TorchOptimizer):
     r"""
-    对AdamW的实现，该实现应该会在pytorch更高版本中出现，https://github.com/pytorch/pytorch/pull/21250。这里提前加入
+    对AdamW的实现，该实现在pytorch 1.2.0版本中已经出现，https://github.com/pytorch/pytorch/pull/21250。
+    这里加入以适配低版本的pytorch
     
     .. todo::
         翻译成中文
diff --git a/test/core/test_optimizer.py b/test/core/test_optimizer.py
index b9a1c271..2f2487c7 100644
--- a/test/core/test_optimizer.py
+++ b/test/core/test_optimizer.py
@@ -2,7 +2,7 @@ import unittest
 
 import torch
 
-from fastNLP import SGD, Adam
+from fastNLP import SGD, Adam, AdamW
 
 
 class TestOptim(unittest.TestCase):
@@ -52,3 +52,12 @@ class TestOptim(unittest.TestCase):
         self.assertEqual(optim.__dict__["settings"]["lr"], 0.001)
         res = optim.construct_from_pytorch(torch.nn.Linear(10, 3).parameters())
         self.assertTrue(isinstance(res, torch.optim.Adam))
+
+    def test_AdamW(self):
+        optim = AdamW(params=torch.nn.Linear(10, 3).parameters())
+        self.assertTrue('lr' in optim.defaults)
+        self.assertTrue('weight_decay' in optim.defaults)
+
+        optim = AdamW(params=torch.nn.Linear(10, 3).parameters(), lr=0.002, weight_decay=0.989)
+        self.assertEqual(optim.defaults['lr'], 0.002)
+        self.assertTrue(optim.defaults['weight_decay'], 0.989)

From 3d726b887eb3158f78d722b3f538b35e6fecfcb6 Mon Sep 17 00:00:00 2001
From: yh 
Date: Mon, 16 Sep 2019 18:43:55 +0800
Subject: [PATCH 91/92] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=83=A8=E5=88=86tutor?=
 =?UTF-8?q?ial?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../tutorials/tutorial_1_data_preprocess.rst  | 29 ++++++-----
 .../tutorials/tutorial_2_vocabulary.rst       | 22 ++++----
 .../source/tutorials/tutorial_3_embedding.rst | 43 +++++++--------
 .../tutorials/tutorial_4_load_dataset.rst     | 52 +++++++++----------
 fastNLP/core/dataset.py                       |  1 +
 fastNLP/io/loader/__init__.py                 | 15 +++---
 fastNLP/io/loader/classification.py           |  4 +-
 fastNLP/io/loader/loader.py                   |  5 ++
 fastNLP/io/pipe/cws.py                        |  2 +-
 fastNLP/io/pipe/pipe.py                       | 12 ++++-
 10 files changed, 101 insertions(+), 84 deletions(-)

diff --git a/docs/source/tutorials/tutorial_1_data_preprocess.rst b/docs/source/tutorials/tutorial_1_data_preprocess.rst
index 47ad3b3f..59b42571 100644
--- a/docs/source/tutorials/tutorial_1_data_preprocess.rst
+++ b/docs/source/tutorials/tutorial_1_data_preprocess.rst
@@ -133,7 +133,7 @@ FastNLP 同样提供了多种删除数据的方法 :func:`~fastNLP.DataSet.drop`
         return words
     dataset.apply(get_words, new_field_name='words')
 
-除了手动处理数据集之外，你还可以使用 fastNLP 提供的各种 :class:`~fastNLP.io.Loader`和:class:`~fastNLP.io.Pipe` 来进行数据处理。
+除了手动处理数据集之外，你还可以使用 fastNLP 提供的各种 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` 来进行数据处理。
 详细请参考这篇教程  :doc:`使用Loader和Pipe处理数据 ` 。
 
 -----------------------------
@@ -142,27 +142,28 @@ fastNLP中field的命名习惯
 
 在英文任务中，fastNLP常用的field名称有:
 
-    - raw_words: 表示的是原始的str。例如"This is a demo sentence ."。存在多个raw_words的情况，例如matching任务，它们会被定义为raw_words0, raw_words1。但在conll格式下，raw_words列也可能为["This", "is", "a", "demo", "sentence", "."]的形式。
-    - words: 表示的是已经tokenize后的词语。例如["This", "is", "a", "demo", "sentence"], 但由于str并不能直接被神经网络所使用，所以words中的内容往往被转换为int，如[3, 10, 4, 2, 7, ...]等。多列words的情况，会被命名为words0, words1
-    - target: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列。
-    - seq_len: 一般用于表示words列的长度
+    - **raw_words**: 表示的是原始的str。例如"This is a demo sentence ."。存在多个raw_words的情况，例如matching任务，它们会被定义为raw_words0, raw_words1。但在conll格式下，raw_words列也可能为["This", "is", "a", "demo", "sentence", "."]的形式。
+    - **words**: 表示的是已经tokenize后的词语。例如["This", "is", "a", "demo", "sentence"], 但由于str并不能直接被神经网络所使用，所以words中的内容往往被转换为int，如[3, 10, 4, 2, 7, ...]等。多列words的情况，会被命名为words0, words1
+    - **target**: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列。
+    - **seq_len**: 一般用于表示words列的长度
 
 在中文任务中，fastNLP常用的field名称有:
 
-    - raw_chars: 表示的是原始的连续汉字序列。例如"这是一个示例。"
-    - chars: 表示已经切分为单独的汉字的序列。例如["这", "是", "一", "个", "示", "例", "。"]。但由于神经网络不能识别汉字，所以一般该列会被转为int形式，如[3, 4, 5, 6, ...]。
-    - raw_words: 如果原始汉字序列中已经包含了词语的边界，则该列称为raw_words。如"上海 浦东 开发 与 法制 建设 同步"。
-    - words: 表示单独的汉字词语序列。例如["上海", "", "浦东", "开发", "与", "法制", "建设", ...]或[2, 3, 4, ...]
-    - target: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列。
-    - seq_len: 表示输入序列的长度
-
-.. todo::
-    这一段移动到datasetiter那里
+    - **raw_words**: 如果原始汉字序列中已经包含了词语的边界，则该列称为raw_words。如"上海 浦东 开发 与 法制 建设 同步"。
+    - **words**: 表示单独的汉字词语序列。例如["上海", "", "浦东", "开发", "与", "法制", "建设", ...]或[2, 3, 4, ...]
+    - **raw_chars**: 表示的是原始的连续汉字序列。例如"这是一个示例。"
+    - **chars**: 表示已经切分为单独的汉字的序列。例如["这", "是", "一", "个", "示", "例", "。"]。但由于神经网络不能识别汉字，所以一般该列会被转为int形式，如[3, 4, 5, 6, ...]。
+    - **target**: 表示目标值。分类场景下，只有一个值；序列标注场景下是一个序列
+    - **seq_len**: 表示输入序列的长度
 
 -----------------------------
 DataSet与pad
 -----------------------------
 
+
+.. todo::
+    这一段移动到datasetiter那里
+
 在fastNLP里，pad是与一个 :mod:`~fastNLP.core.field` 绑定的。即不同的 :mod:`~fastNLP.core.field` 可以使用不同的pad方式，比如在英文任务中word需要的pad和
 character的pad方式往往是不同的。fastNLP是通过一个叫做 :class:`~fastNLP.Padder` 的子类来完成的。
 默认情况下，所有field使用 :class:`~fastNLP.AutoPadder`
diff --git a/docs/source/tutorials/tutorial_2_vocabulary.rst b/docs/source/tutorials/tutorial_2_vocabulary.rst
index d5bb9b7f..fffb94c6 100644
--- a/docs/source/tutorials/tutorial_2_vocabulary.rst
+++ b/docs/source/tutorials/tutorial_2_vocabulary.rst
@@ -19,10 +19,10 @@ Vocabulary
     vocab.to_index('我')  # 会输出1，Vocabulary中默认pad的index为0, unk(没有找到的词)的index为1
 
     #  在构建target的Vocabulary时，词表中应该用不上pad和unk，可以通过以下的初始化
-    vocab = Vocabulary(unknown=None, pad=None)
+    vocab = Vocabulary(unknown=None, padding=None)
     vocab.add_word_lst(['positive', 'negative'])
     vocab.to_index('positive')  # 输出0
-    vocab.to_index('neutral')  # 会报错
+    vocab.to_index('neutral')  # 会报错，因为没有unk这种情况
 
 除了通过以上的方式建立词表，Vocabulary还可以通过使用下面的函数直从 :class:`~fastNLP.DataSet` 中的某一列建立词表以及将该列转换为index
 
@@ -86,7 +86,7 @@ Vocabulary
     vocab.from_dataset(tr_data, field_name='chars', no_create_entry_dataset=[dev_data])
 
 
-:class:`~fastNLP.Vocabulary` 中的 `no_create_entry` , 建议在添加来自于测试集和验证集的词的时候将该参数置为True, 或将验证集和测试集
+ :class:`~fastNLP.Vocabulary` 中的 `no_create_entry` , 建议在添加来自于测试集和验证集的词的时候将该参数置为True, 或将验证集和测试集
 传入 `no_create_entry_dataset` 参数。它们的意义是在接下来的模型会使用pretrain的embedding(包括glove, word2vec, elmo与bert)且会finetune的
 情况下，如果仅使用来自于train的数据建立vocabulary，会导致只出现在test与dev中的词语无法充分利用到来自于预训练embedding的信息(因为他们
 会被认为是unk)，所以在建立词表的时候将test与dev考虑进来会使得最终的结果更好。通过与fastNLP中的各种Embedding配合使用，会有如下的效果，
@@ -96,7 +96,7 @@ Vocabulary
 果找到了，就使用该表示; 如果没有找到，则认为该词的表示应该为unk的表示。
 
 下面我们结合部分 :class:`~fastNLP.embeddings.StaticEmbedding` 的例子来说明下该值造成的影响，如果您对
-:class:`~fastNLP.embeddings.StaticEmbedding` 不太了解，您可以先参考 :doc:`tutorial_3_embedding` 部分再来阅读该部分
+ :class:`~fastNLP.embeddings.StaticEmbedding` 不太了解，您可以先参考 :doc:`使用Embedding模块将文本转成向量 ` 部分再来阅读该部分
 
 .. code-block:: python
 
@@ -108,7 +108,7 @@ Vocabulary
     vocab.add_word('train')
     vocab.add_word('only_in_train')  # 仅在train出现，但肯定在预训练词表中不存在
     vocab.add_word('test', no_create_entry=True)  # 该词只在dev或test中出现
-    vocab.add_word('only_in_test', no_create_entry=True)  # 这个词肯定在预训练中找不到
+    vocab.add_word('only_in_test', no_create_entry=True)  # 这个词在预训练的词表中找不到
 
     embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d')
     print(embed(torch.LongTensor([vocab.to_index('train')])))
@@ -119,12 +119,12 @@ Vocabulary
 
 输出结果(只截取了部分vector)::
 
-    tensor([[ 0.9497,  0.3433,  0.8450, -0.8852, ...]], grad_fn=)  # train
-    tensor([[ 0.0540, -0.0557, -0.0514, -0.1688, ...]], grad_fn=)  # only_in_train
-    tensor([[ 0.1318, -0.2552, -0.0679,  0.2619, ...]], grad_fn=)  # test
-    tensor([[0., 0., 0., 0., 0., ...]], grad_fn=)   # only_in_test
-    tensor([[0., 0., 0., 0., 0., ...]], grad_fn=)   # unk
+    tensor([[ 0.9497,  0.3433,  0.8450, -0.8852, ...]], grad_fn=)  # train，en-glove-6b-50d，找到了该词
+    tensor([[ 0.0540, -0.0557, -0.0514, -0.1688, ...]], grad_fn=)  # only_in_train，en-glove-6b-50d，使用了随机初始化
+    tensor([[ 0.1318, -0.2552, -0.0679,  0.2619, ...]], grad_fn=)  # test，在en-glove-6b-50d中找到了这个词
+    tensor([[0., 0., 0., 0., 0., ...]], grad_fn=)   # only_in_test, en-glove-6b-50d中找不到这个词，使用unk的vector
+    tensor([[0., 0., 0., 0., 0., ...]], grad_fn=)   # unk，使用zero初始化
 
 首先train和test都能够从预训练中找到对应的vector，所以它们是各自的vector表示; only_in_train在预训练中找不到，StaticEmbedding为它
-新建了一个entry，所以它有一个单独的vector; 而only_in_dev在预训练中找不到被指向了unk的值(fastNLP用零向量初始化unk)，与最后一行unk的
+新建了一个entry，所以它有一个单独的vector; 而only_in_test在预训练中找不到改词，因此被指向了unk的值(fastNLP用零向量初始化unk)，与最后一行unk的
 表示相同。
\ No newline at end of file
diff --git a/docs/source/tutorials/tutorial_3_embedding.rst b/docs/source/tutorials/tutorial_3_embedding.rst
index 9a6d00d2..7de2bb1b 100644
--- a/docs/source/tutorials/tutorial_3_embedding.rst
+++ b/docs/source/tutorials/tutorial_3_embedding.rst
@@ -24,7 +24,7 @@ Part I: embedding介绍
 Embedding是一种词嵌入技术，可以将字或者词转换为实向量。目前使用较多的预训练词嵌入有word2vec, fasttext, glove, character embedding,
 elmo以及bert。
 但使用这些词嵌入方式的时候都需要做一些加载上的处理，比如预训练的word2vec, fasttext以及glove都有着超过几十万个词语的表示，但一般任务大概
-只会用到其中几万个词，如果直接加载所有的词汇，会导致内存占用变大以及运行速度变慢，需要从预训练文件中抽取本次实验的用到的词汇；而对于英文的
+只会用到其中的几万个词，如果直接加载所有的词汇，会导致内存占用变大以及训练速度变慢，需要从预训练文件中抽取本次实验的用到的词汇；而对于英文的
 elmo和character embedding, 需要将word拆分成character才能使用；Bert的使用更是涉及到了Byte pair encoding(BPE)相关的内容。为了方便
 大家的使用，fastNLP通过 :class:`~fastNLP.Vocabulary` 统一了不同embedding的使用。下面我们将讲述一些例子来说明一下
 
@@ -35,7 +35,7 @@ Part II: 使用预训练的静态embedding
 
 在fastNLP中，加载预训练的word2vec, glove以及fasttext都使用的是 :class:`~fastNLP.embeddings.StaticEmbedding` 。另外，为了方便大家的
 使用，fastNLP提供了多种静态词向量的自动下载并缓存(默认缓存到~/.fastNLP/embeddings文件夹下)的功能，支持自动下载的预训练向量可以在
-`此处 `_ 查看。
+`下载文档 `_ 查看。
 
 .. code-block:: python
 
@@ -46,10 +46,10 @@ Part II: 使用预训练的静态embedding
     vocab = Vocabulary()
     vocab.add_word_lst("this is a demo .".split())
 
-    embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d', requires_grad=True)
+    embed = StaticEmbedding(vocab, model_dir_or_name='en-glove-6b-50d')
 
-    words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]])
-    print(embed(words).size())
+    words = torch.LongTensor([[vocab.to_index(word) for word in "this is a demo .".split()]])  # 将文本转为index
+    print(embed(words).size())  # StaticEmbedding的使用和pytorch的nn.Embedding是类似的
 
 输出为::
 
@@ -92,7 +92,7 @@ Part IV: ELMo Embedding
 
 在fastNLP中，我们提供了ELMo和BERT的embedding： :class:`~fastNLP.embeddings.ElmoEmbedding`
 和 :class:`~fastNLP.embeddings.BertEmbedding` 。可自动下载的ElmoEmbedding可以
-从 `此处 `_ 找到。
+从 `下载文档 `_ 找到。
 
 与静态embedding类似，ELMo的使用方法如下：
 
@@ -123,13 +123,13 @@ Part IV: ELMo Embedding
 
     torch.Size([1, 5, 512])
 
-另外，根据 `这篇文章 `_ ，不同层之间使用可学习的权重可以使得ELMo的效果更好，在fastNLP中可以通过以下的初始化
+另外，根据 `Deep contextualized word representations `_ ，不同层之间使用可学习的权重可以使得ELMo的效果更好，在fastNLP中可以通过以下的初始化
 实现3层输出的结果通过可学习的权重进行加法融合。
 
 .. code-block:: python
 
     embed = ElmoEmbedding(vocab, model_dir_or_name='en-small', requires_grad=True, layers='mix')
-    print(embed(words).size())
+    print(embed(words).size())  # 三层输出按照权重element-wise的加起来
 
 输出为::
 
@@ -141,7 +141,7 @@ Part V: Bert Embedding
 -----------------------------------------------------------
 
 虽然Bert并不算严格意义上的Embedding，但通过将Bert封装成Embedding的形式将极大减轻使用的复杂程度。可自动下载的Bert Embedding可以
-从 `此处 `_ 找到。我们将使用下面的例子讲述一下
+从 `下载文档 `_ 找到。我们将使用下面的例子讲述一下
 BertEmbedding的使用
 
 .. code-block:: python
@@ -187,7 +187,7 @@ BertEmbedding的使用
     torch.Size([1, 7, 768])
 
 在英文Bert模型中，一个英文单词可能会被切分为多个subword，例如"fairness"会被拆分为 ``["fair", "##ness"]`` ，这样一个word对应的将有两个输出，
-:class:`~fastNLP.embeddings.BertEmbedding` 会使用pooling方法将一个word的subword的表示合并成一个vector，通过pool_method可以控制
+ :class:`~fastNLP.embeddings.BertEmbedding` 会使用pooling方法将一个word的subword的表示合并成一个vector，通过pool_method可以控制
 该pooling方法，支持的有"first"(即使用fair的表示作为fairness的表示), "last"(使用##ness的表示作为fairness的表示), "max"(对fair和
 ##ness在每一维上做max),"avg"(对fair和##ness每一维做average)。
 
@@ -200,7 +200,8 @@ BertEmbedding的使用
 
     torch.Size([1, 5, 768])
 
-另外，根据 `文章 `_ ，Bert的还存在一种用法，句子之间通过[SEP]拼接起来，前一句话的token embedding为0，
+另外，根据 `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding
+ `_ ，Bert在针对具有两句话的任务时（如matching，Q&A任务），句子之间通过[SEP]拼接起来，前一句话的token embedding为0，
 后一句话的token embedding为1。BertEmbedding能够自动识别句子中间的[SEP]来正确设置对应的token_type_id的。
 
 .. code-block:: python
@@ -229,7 +230,7 @@ Part VI: 使用character-level的embedding
 -----------------------------------------------------
 
 除了预训练的embedding以外，fastNLP还提供了两种Character Embedding： :class:`~fastNLP.embeddings.CNNCharEmbedding` 和
-:class:`~fastNLP.embeddings.LSTMCharEmbedding` 。一般在使用character embedding时，需要在预处理的时候将word拆分成character，这
+ :class:`~fastNLP.embeddings.LSTMCharEmbedding` 。一般在使用character embedding时，需要在预处理的时候将word拆分成character，这
 会使得预处理过程变得非常繁琐。在fastNLP中，使用character embedding也只需要传入 :class:`~fastNLP.Vocabulary` 即可，而且该
 Vocabulary与其它Embedding使用的Vocabulary是一致的，下面我们看两个例子。
 
@@ -297,9 +298,9 @@ Part VII: 叠加使用多个embedding
 
     torch.Size([1, 5, 114])
 
-:class:`~fastNLP.embeddings.StaticEmbedding` , :class:`~fastNLP.embeddings.ElmoEmbedding` ,
-:class:`~fastNLP.embeddings.CNNCharEmbedding` , :class:`~fastNLP.embeddings.BertEmbedding` 等都可以互相拼接。
-:class:`~fastNLP.embeddings.StackEmbedding` 的使用也是和其它Embedding是一致的，即输出index返回对应的表示。但能够拼接起来的Embedding
+ :class:`~fastNLP.embeddings.StaticEmbedding` , :class:`~fastNLP.embeddings.ElmoEmbedding` ,
+ :class:`~fastNLP.embeddings.CNNCharEmbedding` , :class:`~fastNLP.embeddings.BertEmbedding` 等都可以互相拼接。
+ :class:`~fastNLP.embeddings.StackEmbedding` 的使用也是和其它Embedding是一致的，即输出index返回对应的表示。但能够拼接起来的Embedding
 必须使用同样的 :class:`~fastNLP.Vocabulary` ，因为只有使用同样的 :class:`~fastNLP.Vocabulary` 才能保证同一个index指向的是同一个词或字
 
 -----------------------------------------------------------
@@ -339,27 +340,27 @@ Part VIII: Embedding的其它说明
     vocab = Vocabulary()
     vocab.add_word_lst("this is a demo .".split())
 
-    embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased')
-    embed.requires_grad = False  # BertEmbedding不更新
+    embed = BertEmbedding(vocab, model_dir_or_name='en-base-cased', requires_grad=True)  # 初始化时设定为需要更新
+    embed.requires_grad = False  # 修改BertEmbedding的权重为不更新
 
 (3) 各种Embedding中word_dropout与dropout的说明
 
 fastNLP中所有的Embedding都支持传入word_dropout和dropout参数，word_dropout指示的是以多大概率将输入的word置为unk的index，这样既可以
 是的unk得到训练，也可以有一定的regularize效果; dropout参数是在获取到word的表示之后，以多大概率将一些维度的表示置为0。
 
-如果使用 :class:`~fastNLP.embeddings.StackEmbedding` 且需要用到word_dropout，建议将word_dropout设置在 :class:`~fastNLP.embeddings.StackEmbedding` 。
+如果使用 :class:`~fastNLP.embeddings.StackEmbedding` 且需要用到word_dropout，建议将word_dropout设置在 :class:`~fastNLP.embeddings.StackEmbedding` 上。
 
 
 -----------------------------------------------------------
 Part IX: StaticEmbedding的使用建议
 -----------------------------------------------------------
 
-在英文的命名实体识别(NER)任务中，由 `论文 `_ 指出，同时使用cnn character embedding和word embedding
+在英文的命名实体识别(NER)任务中，由 `Named Entity Recognition with Bidirectional LSTM-CNNs `_ 指出，同时使用cnn character embedding和word embedding
 会使得NER的效果有比较大的提升。正如你在上节中看到的那样，fastNLP支持将 :class:`~fastNLP.embeddings.CNNCharEmbedding`
 与 :class:`~fastNLP.embeddings.StaticEmbedding` 拼成一个 :class:`~fastNLP.embeddings.StackEmbedding` 。如果通过这种方式使用，需要
 在预处理文本时，不要将词汇小写化(因为Character Embedding需要利用词语中的大小写信息)且不要将出现频次低于某个阈值的word设置为unk(因为
 Character embedding需要利用字形信息)；但 :class:`~fastNLP.embeddings.StaticEmbedding` 使用的某些预训练词嵌入的词汇表中只有小写的词
-语, 且某些低频词并未在预训练中出现需要被剔除。即(1) character embedding需要保留大小写，而某些static embedding不需要保留大小写。(2)
+语, 且某些低频词并未在预训练中出现需要被剔除。即(1) character embedding需要保留大小写，而预训练词向量不需要保留大小写。(2)
 character embedding需要保留所有的字形, 而static embedding需要设置一个最低阈值以学到更好的表示。
 
 (1) fastNLP如何解决关于大小写的问题
@@ -372,7 +373,7 @@ fastNLP通过在 :class:`~fastNLP.embeddings.StaticEmbedding` 增加了一个low
     from fastNLP import Vocabulary
 
     vocab = Vocabulary().add_word_lst("The the a A".split())
-    #  下面用随机的StaticEmbedding演示，但与使用预训练时效果是一致的
+    #  下面用随机的StaticEmbedding演示，但与使用预训练词向量时效果是一致的
     embed = StaticEmbedding(vocab, model_name_or_dir=None, embedding_dim=5)
     print(embed(torch.LongTensor([vocab.to_index('The')])))
     print(embed(torch.LongTensor([vocab.to_index('the')])))
diff --git a/docs/source/tutorials/tutorial_4_load_dataset.rst b/docs/source/tutorials/tutorial_4_load_dataset.rst
index f5f8dbd4..525ab961 100644
--- a/docs/source/tutorials/tutorial_4_load_dataset.rst
+++ b/docs/source/tutorials/tutorial_4_load_dataset.rst
@@ -2,7 +2,7 @@
 使用Loader和Pipe加载并处理数据集
 =======================================
 
-这一部分是一个关于如何加载数据集的教程
+这一部分是关于如何加载数据集的教程
 
 教程目录：
 
@@ -18,26 +18,26 @@ Part I: 数据集容器DataBundle
 ------------------------------------
 
 而由于对于同一个任务，训练集，验证集和测试集会共用同一个词表以及具有相同的目标值，所以在fastNLP中我们使用了 :class:`~fastNLP.io.DataBundle`
-来承载同一个任务的多个数据集 :class:`~fastNLP.DataSet` 以及它们的词表 :class:`~fastNLP.Vocabulary`。下面会有例子介绍 :class:`~fastNLP.io.DataBundle`
+来承载同一个任务的多个数据集 :class:`~fastNLP.DataSet` 以及它们的词表 :class:`~fastNLP.Vocabulary` 。下面会有例子介绍 :class:`~fastNLP.io.DataBundle`
 的相关使用。
 
-:class:`~fastNLP.io.DataBundle` 在fastNLP中主要在各个 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` 中被使用。
-下面我们将先介绍一下 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` , 之后我们将给出相应的例子。
+ :class:`~fastNLP.io.DataBundle` 在fastNLP中主要在各个 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` 中被使用。
+下面我们先介绍一下 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` 。
 
 -------------------------------------
 Part II: 加载的各种数据集的Loader
 -------------------------------------
 
-在fastNLP中，所有的数据Loader都可以通过其文档判断其支持读取的数据格式，以及读取之后返回的 :class:`~fastNLP.DataSet` 的格式。例如
-\ref 加个引用。
+在fastNLP中，所有的 :class:`~fastNLP.io.Loader` 都可以通过其文档判断其支持读取的数据格式，以及读取之后返回的 :class:`~fastNLP.DataSet` 的格式,
+例如 :class:`~fastNLP.io.ChnSentiCorpLoader` 。
 
-    - download 函数：自动将该数据集下载到缓存地址，默认缓存地址为~/.fastNLP/datasets/。由于版权等原因，不是所有的Loader都实现了该方法。该方法会返回下载后文件所处的缓存地址。可以查看对应Loader的download的方法的文档来判断该Loader加载的数据。
-    - _load 函数：从一个数据文件中读取数据，返回一个 :class:`~fastNLP.DataSet` 。返回的DataSet的格式可从Loader文档判断。
-    - load 函数：从文件或者文件夹中读取数据并组装成 :class:`~fastNLP.io.DataBundle`。支持接受的参数类型有以下的几种
+    - **download()** 函数：自动将该数据集下载到缓存地址，默认缓存地址为~/.fastNLP/datasets/。由于版权等原因，不是所有的Loader都实现了该方法。该方法会返回下载后文件所处的缓存地址。
+    - **_load()** 函数：从一个数据文件中读取数据，返回一个 :class:`~fastNLP.DataSet` 。返回的DataSet的格式可从Loader文档判断。
+    - **load()** 函数：从文件或者文件夹中读取数据为 :class:`~fastNLP.DataSet` 并将它们组装成 :class:`~fastNLP.io.DataBundle`。支持接受的参数类型有以下的几种
 
         - None, 将尝试读取自动缓存的数据，仅支持提供了自动下载数据的Loader
-        - 文件夹路径, 默认将尝试在该路径下匹配文件名中含有 `train` , `test` , `dev` 的python文件，如果有多个文件含有这相同的关键字，将无法通过该方式读取
-        - dict, 例如{'train':"/path/to/tr.conll", 'dev':"/to/validate.conll", "test":"/to/te.conll"}
+        - 文件夹路径, 默认将尝试在该文件夹下匹配文件名中含有 `train` , `test` , `dev` 的文件，如果有多个文件含有相同的关键字，将无法通过该方式读取
+        - dict, 例如{'train':"/path/to/tr.conll", 'dev':"/to/validate.conll", "test":"/to/te.conll"}。
 
 .. code-block:: python
 
@@ -56,9 +56,9 @@ Part II: 加载的各种数据集的Loader
 
 这里表示一共有3个数据集。其中：
 
-    - 3个数据集分别为train、dev、test数据集，分别有17223、1831、1944个instance
+    - 3个数据集的名称分别为train、dev、test，分别有17223、1831、1944个instance
 
-也可以取出DataSet并DataSet中的具体内容
+也可以取出DataSet，并打印DataSet中的具体内容
 
 .. code-block:: python
 
@@ -77,21 +77,22 @@ Part II: 加载的各种数据集的Loader
 ------------------------------------------
 Part III: 使用Pipe对数据集进行预处理
 ------------------------------------------
-通过:class:`~fastNLP.io.Loader` 可以将文本数据读入，但并不能直接被神经网络使用，还需要进行一定的预处理。
+通过 :class:`~fastNLP.io.Loader` 可以将文本数据读入，但并不能直接被神经网络使用，还需要进行一定的预处理。
 
-在fastNLP中，我们使用 :class:`~fastNLP.io.Pipe`的子类作为数据预处理的类，Pipe和Loader一般具备一一对应的关系，该关系可以从其名称判断，
+在fastNLP中，我们使用 :class:`~fastNLP.io.Pipe` 的子类作为数据预处理的类， :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` 一般具备一一对应的关系，该关系可以从其名称判断，
 例如 :class:`~fastNLP.io.CWSLoader` 与 :class:`~fastNLP.io.CWSPipe` 是一一对应的。一般情况下Pipe处理包含以下的几个过程，(1)将raw_words或
 raw_chars进行tokenize以切分成不同的词或字; (2) 再建立词或字的 :class:`~fastNLP.Vocabulary` , 并将词或字转换为index; (3)将target
 列建立词表并将target列转为index;
 
-所有的Pipe都可通过其文档查看通过该Pipe之后DataSet中的field的情况; 如 \ref{TODO 添加对例子的引用}
+所有的Pipe都可通过其文档查看该Pipe支持处理的 :class:`~fastNLP.DataSet` 以及返回的 :class:`~fastNLP.io.DataSet` 中的field的情况;
+如 :class:`~fastNLP.io.`
 
 各种数据集的Pipe当中，都包含了以下的两个函数:
 
-    - process 函数：对输入的 :class:`~fastNLP.io.DataBundle` 进行处理, 然后返回处理之后的 :class:`~fastNLP.io.DataBundle` 。process函数的文档中包含了该Pipe支持处理的DataSet的格式。
-    - process_from_file 函数：输入数据集所在文件夹，使用对应的Loader读取数据(所以该函数支持的参数类型是由于其对应的Loader的load函数决定的)，然后调用相对应的process函数对数据进行预处理。相当于是把Load和process放在一个函数中执行。
+    - process() 函数：对输入的 :class:`~fastNLP.io.DataBundle` 进行处理, 然后返回处理之后的 :class:`~fastNLP.io.DataBundle` 。process函数的文档中包含了该Pipe支持处理的DataSet的格式。
+    - process_from_file() 函数：输入数据集所在文件夹，使用对应的Loader读取数据(所以该函数支持的参数类型是由于其对应的Loader的load函数决定的)，然后调用相对应的process函数对数据进行预处理。相当于是把Load和process放在一个函数中执行。
 
-接着上面CWSLoader的例子，我们展示一下CWSPipe的功能：
+接着上面 :class:`~fastNLP.io.CWSLoader` 的例子，我们展示一下 :class:`~fastNLP.io.CWSPipe` 的功能：
 
 .. code-block:: python
 
@@ -112,8 +113,8 @@ raw_chars进行tokenize以切分成不同的词或字; (2) 再建立词或字的
 
 表示一共有3个数据集和2个词表。其中：
 
-    - 3个数据集分别为train、dev、test数据集，分别有17223、1831、1944个instance
-    - 2个词表分别为chars词表与target词表。其中chars词表为句子文本所构建的词表，一共有4777个字；target词表为目标标签所构建的词表，一共有4种标签。
+    - 3个数据集的名称分别为train、dev、test，分别有17223、1831、1944个instance
+    - 2个词表分别为chars词表与target词表。其中chars词表为句子文本所构建的词表，一共有4777个不同的字；target词表为目标标签所构建的词表，一共有4种标签。
 
 相较于之前CWSLoader读取的DataBundle，新增了两个Vocabulary。 我们可以打印一下处理之后的DataSet
 
@@ -147,9 +148,8 @@ raw_chars进行tokenize以切分成不同的词或字; (2) 再建立词或字的
 Part IV: fastNLP封装好的Loader和Pipe
 ------------------------------------------
 
-fastNLP封装了多种任务/数据集的Loader和Pipe并提供自动下载功能，具体参见文档
-
-`fastNLP可加载数据集 `_
+fastNLP封装了多种任务/数据集的 :class:`~fastNLP.io.Loader` 和 :class:`~fastNLP.io.Pipe` 并提供自动下载功能，具体参见文档
+`数据集 `_
 
 --------------------------------------------------------
 Part V: 不同格式类型的基础Loader
@@ -165,12 +165,12 @@ Part V: 不同格式类型的基础Loader
         data_set_loader = CSVLoader(
             headers=('raw_words', 'target'), sep='\t'
         )
-        # 表示将CSV文件中每一行的第一项填入'words' field，第二项填入'target' field。
+        # 表示将CSV文件中每一行的第一项将填入'raw_words' field，第二项填入'target' field。
         # 其中项之间由'\t'分割开来
 
         data_set = data_set_loader._load('path/to/your/file')
 
-    数据集内容样例如下 ::
+    文件内容样例如下 ::
 
         But it does not leave you with much .	1
         You could hate it for the same reason .	1
diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py
index 38395e57..840f3417 100644
--- a/fastNLP/core/dataset.py
+++ b/fastNLP/core/dataset.py
@@ -480,6 +480,7 @@ class DataSet(object):
             for field in fields:
                 table.add_row(field)
             logger.info(table)
+            return table
 
     def append(self, instance):
         """
diff --git a/fastNLP/io/loader/__init__.py b/fastNLP/io/loader/__init__.py
index 06ad57c3..4ad228b0 100644
--- a/fastNLP/io/loader/__init__.py
+++ b/fastNLP/io/loader/__init__.py
@@ -8,7 +8,7 @@ Loader用于读取数据，并将内容读取到 :class:`~fastNLP.DataSet` 或
     将尝试自动下载数据集并缓存。但不是所有的数据都可以直接下载。
 
 1.传入一个文件的 path
-    返回的 `data_bundle` 包含一个名为 `train` 的 dataset ,可以通过 ``data_bundle.datasets['train']`` 获取
+    返回的 `data_bundle` 包含一个名为 `train` 的 dataset ,可以通过 ``data_bundle.get_dataset('train')`` 获取
 
 2.传入一个文件夹目录
     将读取的是这个文件夹下文件名中包含 `train` , `test` , `dev` 的文件，其它文件会被忽略。假设某个目录下的文件为::
@@ -19,23 +19,24 @@ Loader用于读取数据，并将内容读取到 :class:`~fastNLP.DataSet` 或
         +-test.txt
         +-other.txt
 
-    在 Loader().load('/path/to/dir') 返回的 `data_bundle` 中可以用 ``data_bundle.datasets['train']`` , ``data_bundle.datasets['dev']`` ,
-    ``data_bundle.datasets['test']`` 获取对应的 `dataset` ，其中 `other.txt` 的内容会被忽略。假设某个目录下的文件为::
+    在 Loader().load('/path/to/dir') 返回的 `data_bundle` 中可以用 ``data_bundle.get_dataset('train')`` ,
+    ``data_bundle.get_dataset('dev')`` ,
+    ``data_bundle.get_dataset('test')`` 获取对应的 `dataset` ，其中 `other.txt` 的内容会被忽略。假设某个目录下的文件为::
 
         |
         +-train.txt
         +-dev.txt
 
-    在 Loader().load('/path/to/dir') 返回的 `data_bundle` 中可以用 ``data_bundle.datasets['train']`` ,
-    ``data_bundle.datasets['dev']`` 获取对应的 dataset。
+    在 Loader().load('/path/to/dir') 返回的 `data_bundle` 中可以用 ``data_bundle.get_dataset('train')`` ,
+    ``data_bundle.get_dataset('dev')`` 获取对应的 dataset。
 
 3.传入一个字典
     字典的的 key 为 `dataset` 的名称，value 是该 `dataset` 的文件路径::
 
         paths = {'train':'/path/to/train', 'dev': '/path/to/dev', 'test':'/path/to/test'}
     
-    在 Loader().load(paths)  返回的 `data_bundle` 中可以用 ``data_bundle.datasets['train']`` , ``data_bundle.datasets['dev']`` ,
-    ``data_bundle.datasets['test']`` 来获取对应的 `dataset`
+    在 Loader().load(paths)  返回的 `data_bundle` 中可以用 ``data_bundle.get_dataset('train')`` , ``data_bundle.get_dataset('dev')`` ,
+    ``data_bundle.get_dataset('test')`` 来获取对应的 `dataset`
 
 fastNLP 目前提供了如下的 Loader
 
diff --git a/fastNLP/io/loader/classification.py b/fastNLP/io/loader/classification.py
index 53bc6789..ac1bba22 100644
--- a/fastNLP/io/loader/classification.py
+++ b/fastNLP/io/loader/classification.py
@@ -287,7 +287,7 @@ class SST2Loader(Loader):
     数据SST2的Loader
     读取之后DataSet将如下所示
 
-    .. csv-table:: 下面是使用SSTLoader读取的DataSet所具备的field
+    .. csv-table::
         :header: "raw_words", "target"
 
         "it 's a charming and often affecting...", "1"
@@ -345,7 +345,7 @@ class SST2Loader(Loader):
 class ChnSentiCorpLoader(Loader):
     """
     支持读取的数据的格式为，第一行为标题(具体内容会被忽略)，之后一行为一个sample，第一个制表符之前被认为是label，第
-    一个制表符及之后认为是句子
+    一个制表符之后认为是句子
 
     Example::
 
diff --git a/fastNLP/io/loader/loader.py b/fastNLP/io/loader/loader.py
index 384125a8..fa1128ed 100644
--- a/fastNLP/io/loader/loader.py
+++ b/fastNLP/io/loader/loader.py
@@ -15,6 +15,11 @@ from ...core.dataset import DataSet
 class Loader:
     """
     各种数据 Loader 的基类，提供了 API 的参考.
+    Loader支持以下的三个函数
+
+    - download() 函数：自动将该数据集下载到缓存地址，默认缓存地址为~/.fastNLP/datasets/。由于版权等原因，不是所有的Loader都实现了该方法。该方法会返回下载后文件所处的缓存地址。
+    - _load() 函数：从一个数据文件中读取数据，返回一个 :class:`~fastNLP.DataSet` 。返回的DataSet的内容可以通过每个Loader的文档判断出。
+    - load() 函数：将文件分别读取为DataSet，然后将多个DataSet放入到一个DataBundle中并返回
     
     """
     
diff --git a/fastNLP/io/pipe/cws.py b/fastNLP/io/pipe/cws.py
index 1fc64785..97bda896 100644
--- a/fastNLP/io/pipe/cws.py
+++ b/fastNLP/io/pipe/cws.py
@@ -144,7 +144,7 @@ class CWSPipe(Pipe):
        "2001年  新年  钟声...", "[8, 9, 9, 7, ...]", "[0, 1, 1, 1, 2...]", "[11, 12, ...]","[3, 9, ...]", 20
        "...", "[...]","[...]", "[...]","[...]", .
 
-    其中bigrams仅当bigrams列为True的时候为真
+    其中bigrams仅当bigrams列为True的时候存在
 
     """
     
diff --git a/fastNLP/io/pipe/pipe.py b/fastNLP/io/pipe/pipe.py
index 20bafe01..ab3c9120 100644
--- a/fastNLP/io/pipe/pipe.py
+++ b/fastNLP/io/pipe/pipe.py
@@ -9,8 +9,16 @@ from .. import DataBundle
 
 class Pipe:
     """
-    .. todo::
-        doc
+    Pipe是fastNLP中用于处理DataBundle的类，但实际是处理DataBundle中的DataSet。所有Pipe都会在其process()函数的文档中指出该Pipe可处理的DataSet应该具备怎样的格式；在Pipe
+    文档中说明该Pipe返回后DataSet的格式以及其field的信息；以及新增的Vocabulary的信息。
+
+    一般情况下Pipe处理包含以下的几个过程，(1)将raw_words或raw_chars进行tokenize以切分成不同的词或字;
+    (2) 再建立词或字的 :class:`~fastNLP.Vocabulary` , 并将词或字转换为index; (3)将target列建立词表并将target列转为index;
+
+    Pipe中提供了两个方法
+
+    -process()函数，输入为DataBundle
+    -process_from_file()函数，输入为对应Loader的load函数可接受的类型。
 
     """
     

From 86ba31b4cf437273eb8501b7f43ebb4b32c23f3b Mon Sep 17 00:00:00 2001
From: Yige Xu 
Date: Tue, 17 Sep 2019 10:42:38 +0800
Subject: [PATCH 92/92] delete get_tokenizer in fastNLP/io/utils.py

---
 fastNLP/io/utils.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/fastNLP/io/utils.py b/fastNLP/io/utils.py
index e1de2ae7..496aee77 100644
--- a/fastNLP/io/utils.py
+++ b/fastNLP/io/utils.py
@@ -76,15 +76,3 @@ def check_loader_paths(paths: Union[str, Dict[str, str]]) -> Dict[str, str]:
             raise ValueError("Empty paths is not allowed.")
     else:
         raise TypeError(f"paths only supports str and dict. not {type(paths)}.")
-
-
-def get_tokenizer():
-    try:
-        import spacy
-        spacy.prefer_gpu()
-        en = spacy.load('en')
-        logger.info('use spacy tokenizer')
-        return lambda x: [w.text for w in en.tokenizer(x)]
-    except Exception as e:
-        logger.error('use raw tokenizer')
-        return lambda x: x.split()