diff --git a/README.md b/README.md index 65d713e6..8d638c0b 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,10 @@ For example: ## Resources - [Documentation](https://fastnlp.readthedocs.io/en/latest/) +- [Tutorials](https://github.com/fastnlp/fastNLP/tutorials) - [Source Code](https://github.com/fastnlp/fastNLP) + ## Installation Run the following commands to install fastNLP package. ```shell @@ -70,7 +72,7 @@ pip install fastNLP fastNLP.core - data representation & train/test presedure + data representation & train/test procedure fastNLP.models diff --git a/fastNLP/api/README.md b/fastNLP/api/README.md new file mode 100644 index 00000000..3604bd07 --- /dev/null +++ b/fastNLP/api/README.md @@ -0,0 +1,43 @@ +# fastNLP 高级接口 + +### 环境与配置 +1. 系统环境：linux/ubuntu(推荐) +2. 编程语言：Python>=3.6 +3. Python包依赖 + - **torch==1.0** + - numpy>=1.14.2 + +### 中文分词 +```python +text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', + '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', + '那么这款无人机到底有多厉害？'] +from fastNLP.api import CWS +cws = CWS(device='cpu') +print(cws.predict(text)) +# ['编者 按 : 7月 12日 , 英国 航空 航天 系统 公司 公布 了 该 公司 研制 的 第一 款 高 科技 隐形 无人 机雷电 之 神 。', '这 款 飞行 从 外型 上 来 看 酷似 电影 中 的 太空 飞行器 , 据 英国 方面 介绍 , 可以 实现 洲际 远程 打击 。', '那么 这 款 无人 机 到底 有 多 厉害 ?'] +``` + +### 中文分词+词性标注 +```python +text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', + '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', + '那么这款无人机到底有多厉害？'] +from fastNLP.api import POS +pos = POS(device='cpu') +print(pos.predict(text)) +# [['编者/NN', '按/P', '：/PU', '7月/NT', '12日/NR', '，/PU', '英国/NR', '航空/NN', '航天/NN', '系统/NN', '公司/NN', '公布/VV', '了/AS', '该/DT', '公司/NN', '研制/VV', '的/DEC', '第一/OD', '款高/NN', '科技/NN', '隐形/NN', '无/VE', '人机/NN', '雷电/NN', '之/DEG', '神/NN', '。/PU'], ['这/DT', '款/NN', '飞行/VV', '从/P', '外型/NN', '上/LC', '来/MSP', '看/VV', '酷似/VV', '电影/NN', '中/LC', '的/DEG', '太空/NN', '飞行器/NN', '，/PU', '据/P', '英国/NR', '方面/NN', '介绍/VV', '，/PU', '可以/VV', '实现/VV', '洲际/NN', '远程/NN', '打击/NN', '。/PU'], ['那么/AD', '这/DT', '款/NN', '无/VE', '人机/NN', '到底/AD', '有/VE', '多/CD', '厉害/NN', '？/PU']] +``` + +### 中文分词+词性标注+句法分析 +```python +text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', + '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', + '那么这款无人机到底有多厉害？'] +from fastNLP.api import Parser +parser = Parser(device='cpu') +print(parser.predict(text)) +# [['12/nsubj', '12/prep', '2/punct', '5/nn', '2/pobj', '12/punct', '11/nn', '11/nn', '11/nn', '11/nn', '2/pobj', '0/root', '12/asp', '15/det', '16/nsubj', '21/rcmod', '16/cpm', '21/nummod', '21/nn', '21/nn', '22/top', '12/ccomp', '24/nn', '26/assmod', '24/assm', '22/dobj', '12/punct'], ['2/det', '8/xsubj', '8/mmod', '8/prep', '6/lobj', '4/plmod', '8/prtmod', '0/root', '8/ccomp', '11/lobj', '14/assmod', '11/assm', '14/nn', '9/dobj', '8/punct', '22/prep', '18/nn', '19/nsubj', '16/pccomp', '22/punct', '22/mmod', '8/dep', '25/nn', '25/nn', '22/dobj', '8/punct'], ['4/advmod', '3/det', '4/nsubj', '0/root', '4/dobj', '7/advmod', '4/conj', '9/nummod', '7/dobj', '4/punct']] +``` + +完整样例见`examples.py` \ No newline at end of file diff --git a/fastNLP/api/__init__.py b/fastNLP/api/__init__.py index e69de29b..a21a4c42 100644 --- a/fastNLP/api/__init__.py +++ b/fastNLP/api/__init__.py @@ -0,0 +1 @@ +from .api import CWS, POS, Parser diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index 5ae05dac..8368dcc9 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -7,28 +7,28 @@ import os from fastNLP.core.dataset import DataSet -from fastNLP.api.model_zoo import load_url +from fastNLP.api.utils import load_url from fastNLP.api.processor import ModelProcessor -from reproduction.chinese_word_segment.cws_io.cws_reader import ConlluCWSReader -from reproduction.pos_tag_model.pos_io.pos_reader import ConlluPOSReader +from reproduction.chinese_word_segment.cws_io.cws_reader import ConllCWSReader +from reproduction.pos_tag_model.pos_reader import ZhConllPOSReader from reproduction.Biaffine_parser.util import ConllxDataLoader, add_seg_tag from fastNLP.core.instance import Instance -from fastNLP.core.sampler import SequentialSampler -from fastNLP.core.batch import Batch -from reproduction.chinese_word_segment.utils import calculate_pre_rec_f1 from fastNLP.api.pipeline import Pipeline -from fastNLP.core.metrics import SeqLabelEvaluator2 -from fastNLP.core.tester import Tester +from fastNLP.core.metrics import SpanFPreRecMetric +from fastNLP.api.processor import IndexerProcessor # TODO add pretrain urls model_urls = { - + "cws": "http://123.206.98.91:8888/download/cws_crf_1_11-457fc899.pkl", + "pos": "http://123.206.98.91:8888/download/pos_tag_model_20190108-f3c60ee5.pkl", + "parser": "http://123.206.98.91:8888/download/biaffine_parser-3a2f052c.pkl" } class API: def __init__(self): self.pipeline = None + self._dict = None def predict(self, *args, **kwargs): raise NotImplementedError @@ -38,8 +38,8 @@ class API: _dict = torch.load(path, map_location='cpu') else: _dict = load_url(path, map_location='cpu') - self.pipeline = _dict['pipeline'] self._dict = _dict + self.pipeline = _dict['pipeline'] for processor in self.pipeline.pipeline: if isinstance(processor, ModelProcessor): processor.set_model_device(device) @@ -48,6 +48,9 @@ class API: class POS(API): """FastNLP API for Part-Of-Speech tagging. + :param str model_path: the path to the model. + :param str device: device name such as "cpu" or "cuda:0". Use the same notation as PyTorch. + """ def __init__(self, model_path=None, device='cpu'): @@ -63,7 +66,7 @@ class POS(API): :param content: list of list of str. Each string is a token(word). :return answer: list of list of str. Each string is a tag. """ - if not hasattr(self, 'pipeline'): + if not hasattr(self, "pipeline"): raise ValueError("You have to load model first.") sentence_list = [] @@ -75,59 +78,72 @@ class POS(API): # 2. 组建dataset dataset = DataSet() - dataset.add_field('words', sentence_list) + dataset.add_field("words", sentence_list) # 3. 使用pipeline self.pipeline(dataset) - output = dataset['word_pos_output'].content + def decode_tags(ins): + pred_tags = ins["tag"] + chars = ins["words"] + words = [] + start_idx = 0 + for idx, tag in enumerate(pred_tags): + if tag[0] == "S": + words.append(chars[start_idx:idx + 1] + "/" + tag[2:]) + start_idx = idx + 1 + elif tag[0] == "E": + words.append("".join(chars[start_idx:idx + 1]) + "/" + tag[2:]) + start_idx = idx + 1 + return words + + dataset.apply(decode_tags, new_field_name="tag_output") + + output = dataset.field_arrays["tag_output"].content if isinstance(content, str): return output[0] elif isinstance(content, list): return output - def test(self, filepath): - - tag_proc = self._dict['tag_indexer'] - - model = self.pipeline.pipeline[2].model - pipeline = self.pipeline.pipeline[0:2] - pipeline.append(tag_proc) - pp = Pipeline(pipeline) - - reader = ConlluPOSReader() - te_dataset = reader.load(filepath) - - evaluator = SeqLabelEvaluator2('word_seq_origin_len') - end_tagidx_set = set() - tag_proc.vocab.build_vocab() - for key, value in tag_proc.vocab.word2idx.items(): - if key.startswith('E-'): - end_tagidx_set.add(value) - if key.startswith('S-'): - end_tagidx_set.add(value) - evaluator.end_tagidx_set = end_tagidx_set - - default_valid_args = {"batch_size": 64, - "use_cuda": True, "evaluator": evaluator} - - pp(te_dataset) - te_dataset.set_target(truth=True) - - tester = Tester(**default_valid_args) - - test_result = tester.test(model, te_dataset) - - f1 = round(test_result['F'] * 100, 2) - pre = round(test_result['P'] * 100, 2) - rec = round(test_result['R'] * 100, 2) - # print("f1:{:.2f}, pre:{:.2f}, rec:{:.2f}".format(f1, pre, rec)) - - return f1, pre, rec + def test(self, file_path): + test_data = ZhConllPOSReader().load(file_path) + + tag_vocab = self._dict["tag_vocab"] + pipeline = self._dict["pipeline"] + index_tag = IndexerProcessor(vocab=tag_vocab, field_name="tag", new_added_field_name="truth", is_input=False) + pipeline.pipeline = [index_tag] + pipeline.pipeline + + pipeline(test_data) + test_data.set_target("truth") + prediction = test_data.field_arrays["predict"].content + truth = test_data.field_arrays["truth"].content + seq_len = test_data.field_arrays["word_seq_origin_len"].content + + # padding by hand + max_length = max([len(seq) for seq in prediction]) + for idx in range(len(prediction)): + prediction[idx] = list(prediction[idx]) + ([0] * (max_length - len(prediction[idx]))) + truth[idx] = list(truth[idx]) + ([0] * (max_length - len(truth[idx]))) + evaluator = SpanFPreRecMetric(tag_vocab=tag_vocab, pred="predict", target="truth", + seq_lens="word_seq_origin_len") + evaluator({"predict": torch.Tensor(prediction), "word_seq_origin_len": torch.Tensor(seq_len)}, + {"truth": torch.Tensor(truth)}) + test_result = evaluator.get_metric() + f1 = round(test_result['f'] * 100, 2) + pre = round(test_result['pre'] * 100, 2) + rec = round(test_result['rec'] * 100, 2) + + return {"F1": f1, "precision": pre, "recall": rec} class CWS(API): def __init__(self, model_path=None, device='cpu'): + """ + 中文分词高级接口。 + + :param model_path: 当model_path为None，使用默认位置的model。如果默认位置不存在，则自动下载模型 + :param device: str，可以为'cpu', 'cuda'或'cuda:0'等。会将模型load到相应device进行推断。 + """ super(CWS, self).__init__() if model_path is None: model_path = model_urls['cws'] @@ -135,7 +151,13 @@ class CWS(API): self.load(model_path, device) def predict(self, content): + """ + 分词接口。 + :param content: str或List[str], 例如: "中文分词很重要！"， 返回的结果是"中文 分词 很 重要 !"。 如果传入的为List[str]，比如 + [ "中文分词很重要！", ...], 返回的结果["中文 分词 很 重要 !", ...]。 + :return: str或List[str], 根据输入的的类型决定。 + """ if not hasattr(self, 'pipeline'): raise ValueError("You have to load model first.") @@ -153,33 +175,55 @@ class CWS(API): # 3. 使用pipeline self.pipeline(dataset) - output = dataset['output'].content + output = dataset.get_field('output').content if isinstance(content, str): return output[0] elif isinstance(content, list): return output def test(self, filepath): - - tag_proc = self._dict['tag_indexer'] + """ + 传入一个分词文件路径，返回该数据集上分词f1, precision, recall。 + 分词文件应该为: + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + 以空行分割两个句子，有内容的每行有7列。 + + :param filepath: str, 文件路径路径。 + :return: float, float, float. 分别f1, precision, recall. + """ + tag_proc = self._dict['tag_proc'] cws_model = self.pipeline.pipeline[-2].model - pipeline = self.pipeline.pipeline[:5] + pipeline = self.pipeline.pipeline[:-2] pipeline.insert(1, tag_proc) pp = Pipeline(pipeline) - reader = ConlluCWSReader() + reader = ConllCWSReader() # te_filename = '/home/hyan/ctb3/test.conllx' te_dataset = reader.load(filepath) pp(te_dataset) - batch_size = 64 - te_batcher = Batch(te_dataset, batch_size, SequentialSampler(), use_cuda=False) - pre, rec, f1 = calculate_pre_rec_f1(cws_model, te_batcher, type='bmes') - f1 = round(f1 * 100, 2) - pre = round(pre * 100, 2) - rec = round(rec * 100, 2) + from fastNLP.core.tester import Tester + from fastNLP.core.metrics import BMESF1PreRecMetric + + tester = Tester(data=te_dataset, model=cws_model, metrics=BMESF1PreRecMetric(target='target'), batch_size=64, + verbose=0) + eval_res = tester.test() + + f1 = eval_res['BMESF1PreRecMetric']['f'] + pre = eval_res['BMESF1PreRecMetric']['pre'] + rec = eval_res['BMESF1PreRecMetric']['rec'] # print("f1:{:.2f}, pre:{:.2f}, rec:{:.2f}".format(f1, pre, rec)) return f1, pre, rec @@ -191,30 +235,30 @@ class Parser(API): if model_path is None: model_path = model_urls['parser'] + self.pos_tagger = POS(device=device) self.load(model_path, device) def predict(self, content): if not hasattr(self, 'pipeline'): raise ValueError("You have to load model first.") - sentence_list = [] - # 1. 检查sentence的类型 - if isinstance(content, str): - sentence_list.append(content) - elif isinstance(content, list): - sentence_list = content + # 1. 利用POS得到分词和pos tagging结果 + pos_out = self.pos_tagger.predict(content) + # pos_out = ['这里/NN 是/VB 分词/NN 结果/NN'.split()] # 2. 组建dataset dataset = DataSet() - dataset.add_field('words', sentence_list) - # dataset.add_field('tag', sentence_list) + dataset.add_field('wp', pos_out) + dataset.apply(lambda x: [''] + [w.split('/')[0] for w in x['wp']], new_field_name='words') + dataset.apply(lambda x: [''] + [w.split('/')[1] for w in x['wp']], new_field_name='pos') # 3. 使用pipeline self.pipeline(dataset) - for ins in dataset: - ins['heads'] = ins['heads'].tolist() - - return dataset['heads'], dataset['labels'] + dataset.apply(lambda x: [str(arc) for arc in x['arc_pred']], new_field_name='arc_pred') + dataset.apply(lambda x: [arc + '/' + label for arc, label in + zip(x['arc_pred'], x['label_pred_seq'])][1:], new_field_name='output') + # output like: [['2/top', '0/root', '4/nn', '2/dep']] + return dataset.field_arrays['output'].content def test(self, filepath): data = ConllxDataLoader().load(filepath) @@ -276,7 +320,7 @@ class Analyzer: def test(self, filepath): output_dict = {} - if self.seg: + if self.cws: seg_output = self.cws.test(filepath) output_dict['seg'] = seg_output if self.pos: @@ -287,28 +331,3 @@ class Analyzer: output_dict['parser'] = parser_output return output_dict - - -if __name__ == "__main__": - # pos_model_path = '../../reproduction/pos_tag_model/pos_crf.pkl' - # pos = POS(device='cpu') - # s = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。' , - # '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', - # '那么这款无人机到底有多厉害？'] - # print(pos.test('/Users/yh/Desktop/test_data/pos_test.conll')) - # print(pos.predict(s)) - - # cws_model_path = '../../reproduction/chinese_word_segment/models/cws_crf.pkl' - # cws = CWS(device='cpu') - # s = ['本品是一个抗酸抗胆汁的胃黏膜保护剂' , - # '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', - # '那么这款无人机到底有多厉害？'] - # print(cws.test('/Users/yh/Desktop/test_data/cws_test.conll')) - # print(cws.predict(s)) - - parser = Parser(device='cpu') - # print(parser.test('/Users/yh/Desktop/test_data/parser_test2.conll')) - s = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', - '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', - '那么这款无人机到底有多厉害？'] - print(parser.predict(s)) diff --git a/fastNLP/api/examples.py b/fastNLP/api/examples.py new file mode 100644 index 00000000..10cc6edc --- /dev/null +++ b/fastNLP/api/examples.py @@ -0,0 +1,29 @@ +""" +api/example.py contains all API examples provided by fastNLP. +It is used as a tutorial for API or a test script since it is difficult to test APIs in travis. + +""" +from fastNLP.api import CWS, POS, Parser + +text = ['编者按：7月12日，英国航空航天系统公司公布了该公司研制的第一款高科技隐形无人机雷电之神。', + '这款飞行从外型上来看酷似电影中的太空飞行器，据英国方面介绍，可以实现洲际远程打击。', + '那么这款无人机到底有多厉害？'] + + +def chinese_word_segmentation(): + cws = CWS(device='cpu') + print(cws.predict(text)) + + +def pos_tagging(): + pos = POS(device='cpu') + print(pos.predict(text)) + + +def syntactic_parsing(): + parser = Parser(device='cpu') + print(parser.predict(text)) + + +if __name__ == "__main__": + syntactic_parsing() diff --git a/fastNLP/api/processor.py b/fastNLP/api/processor.py index b495ea70..7354fe0f 100644 --- a/fastNLP/api/processor.py +++ b/fastNLP/api/processor.py @@ -11,6 +11,11 @@ from fastNLP.core.vocabulary import Vocabulary class Processor(object): def __init__(self, field_name, new_added_field_name): + """ + + :param field_name: 处理哪个field + :param new_added_field_name: 如果为None，则认为是field_name，即覆盖原有的field + """ self.field_name = field_name if new_added_field_name is None: self.new_added_field_name = field_name @@ -92,6 +97,11 @@ class FullSpaceToHalfSpaceProcessor(Processor): class PreAppendProcessor(Processor): + """ + 向某个field的起始增加data(应该为str类型)。该field需要为list类型。即新增的field为 + [data] + instance[field_name] + + """ def __init__(self, data, field_name, new_added_field_name=None): super(PreAppendProcessor, self).__init__(field_name, new_added_field_name) self.data = data @@ -102,6 +112,10 @@ class PreAppendProcessor(Processor): class SliceProcessor(Processor): + """ + 从某个field中只取部分内容。等价于instance[field_name][start:end:step] + + """ def __init__(self, start, end, step, field_name, new_added_field_name=None): super(SliceProcessor, self).__init__(field_name, new_added_field_name) for o in (start, end, step): @@ -114,7 +128,17 @@ class SliceProcessor(Processor): class Num2TagProcessor(Processor): + """ + 将一句话中的数字转换为某个tag。 + + """ def __init__(self, tag, field_name, new_added_field_name=None): + """ + + :param tag: str, 将数字转换为该tag + :param field_name: + :param new_added_field_name: + """ super(Num2TagProcessor, self).__init__(field_name, new_added_field_name) self.tag = tag self.pattern = r'[-+]?([0-9]+[.]?[0-9]*)+[/eE]?[-+]?([0-9]+[.]?[0-9]*)' @@ -135,6 +159,10 @@ class Num2TagProcessor(Processor): class IndexerProcessor(Processor): + """ + 给定一个vocabulary , 将指定field转换为index形式。指定field应该是一维的list，比如 + ['我', '是', xxx] + """ def __init__(self, vocab, field_name, new_added_field_name, delete_old_field=False, is_input=True): assert isinstance(vocab, Vocabulary), "Only Vocabulary class is allowed, not {}.".format(type(vocab)) @@ -163,19 +191,19 @@ class IndexerProcessor(Processor): class VocabProcessor(Processor): - """Build vocabulary with a field in the data set. + """ + 传入若干个DataSet以建立vocabulary。 """ - def __init__(self, field_name): + def __init__(self, field_name, min_freq=1, max_size=None): super(VocabProcessor, self).__init__(field_name, None) - self.vocab = Vocabulary() + self.vocab = Vocabulary(min_freq=min_freq, max_size=max_size) def process(self, *datasets): for dataset in datasets: assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: - self.vocab.update(ins[self.field_name]) + dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) def get_vocab(self): self.vocab.build_vocab() @@ -183,6 +211,10 @@ class VocabProcessor(Processor): class SeqLenProcessor(Processor): + """ + 根据某个field新增一个sequence length的field。取该field的第一维 + + """ def __init__(self, field_name, new_added_field_name='seq_lens', is_input=True): super(SeqLenProcessor, self).__init__(field_name, new_added_field_name) self.is_input = is_input @@ -195,10 +227,15 @@ class SeqLenProcessor(Processor): return dataset +from fastNLP.core.utils import _build_args + class ModelProcessor(Processor): def __init__(self, model, seq_len_field_name='seq_lens', batch_size=32): """ - 迭代模型并将结果的padding drop掉 + 传入一个model，在process()时传入一个dataset，该processor会通过Batch将DataSet的内容输出给model.predict或者model.forward. + model输出的内容会被增加到dataset中，field_name由model输出决定。如果生成的内容维度不是(Batch_size, )与 + (Batch_size, 1)，则使用seqence length这个field进行unpad + TODO 这个类需要删除对seq_lens的依赖。 :param seq_len_field_name: :param batch_size: @@ -211,13 +248,18 @@ class ModelProcessor(Processor): def process(self, dataset): self.model.eval() assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - data_iterator = Batch(dataset, batch_size=self.batch_size, sampler=SequentialSampler(), use_cuda=False) + data_iterator = Batch(dataset, batch_size=self.batch_size, sampler=SequentialSampler()) batch_output = defaultdict(list) + if hasattr(self.model, "predict"): + predict_func = self.model.predict + else: + predict_func = self.model.forward with torch.no_grad(): for batch_x, _ in data_iterator: - prediction = self.model.predict(**batch_x) - seq_lens = batch_x[self.seq_len_field_name].cpu().numpy().tolist() + refined_batch_x = _build_args(predict_func, **batch_x) + prediction = predict_func(**refined_batch_x) + seq_lens = batch_x[self.seq_len_field_name].tolist() for key, value in prediction.items(): tmp_batch = [] @@ -228,8 +270,8 @@ class ModelProcessor(Processor): for idx, seq_len in enumerate(seq_lens): tmp_batch.append(value[idx, :seq_len]) batch_output[key].extend(tmp_batch) - - batch_output[self.seq_len_field_name].extend(seq_lens) + if not self.seq_len_field_name in prediction: + batch_output[self.seq_len_field_name].extend(seq_lens) # TODO 当前的实现会导致之后的processor需要知道model输出的output的key是什么 for field_name, fields in batch_output.items(): @@ -246,6 +288,10 @@ class ModelProcessor(Processor): class Index2WordProcessor(Processor): + """ + 将DataSet中某个为index的field根据vocab转换为str + + """ def __init__(self, vocab, field_name, new_added_field_name): super(Index2WordProcessor, self).__init__(field_name, new_added_field_name) self.vocab = vocab @@ -256,15 +302,23 @@ class Index2WordProcessor(Processor): return dataset -class SetIsTargetProcessor(Processor): +class SetTargetProcessor(Processor): # TODO; remove it. - def __init__(self, field_dict, default=False): - super(SetIsTargetProcessor, self).__init__(None, None) - self.field_dict = field_dict - self.default = default + def __init__(self, *fields, flag=True): + super(SetTargetProcessor, self).__init__(None, None) + self.fields = fields + self.flag = flag + + def process(self, dataset): + dataset.set_target(*self.fields, flag=self.flag) + return dataset + +class SetInputProcessor(Processor): + def __init__(self, *fields, flag=True): + super(SetInputProcessor, self).__init__(None, None) + self.fields = fields + self.flag = flag def process(self, dataset): - set_dict = {name: self.default for name in dataset.get_all_fields().keys()} - set_dict.update(self.field_dict) - dataset.set_target(**set_dict) + dataset.set_input(*self.fields, flag=self.flag) return dataset diff --git a/fastNLP/api/model_zoo.py b/fastNLP/api/utils.py similarity index 100% rename from fastNLP/api/model_zoo.py rename to fastNLP/api/utils.py diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py index 7b3b6d11..9ba8dca8 100644 --- a/fastNLP/core/batch.py +++ b/fastNLP/core/batch.py @@ -1,6 +1,8 @@ import numpy as np import torch +from fastNLP.core.sampler import RandomSampler + class Batch(object): """Batch is an iterable object which iterates over mini-batches. @@ -17,14 +19,15 @@ class Batch(object): """ - def __init__(self, dataset, batch_size, sampler, as_numpy=False): + def __init__(self, dataset, batch_size, sampler=RandomSampler(), as_numpy=False): self.dataset = dataset self.batch_size = batch_size self.sampler = sampler self.as_numpy = as_numpy self.idx_list = None self.curidx = 0 - self.num_batches = len(dataset)//batch_size + int(len(dataset)%batch_size!=0) + self.num_batches = len(dataset) // batch_size + int(len(dataset) % batch_size != 0) + self.cur_batch_indices = None def __iter__(self): self.idx_list = self.sampler(self.dataset) @@ -40,6 +43,7 @@ class Batch(object): batch_x, batch_y = {}, {} indices = self.idx_list[self.curidx:endidx] + self.cur_batch_indices = indices for field_name, field in self.dataset.get_all_fields().items(): if field.is_target or field.is_input: @@ -58,6 +62,9 @@ class Batch(object): def __len__(self): return self.num_batches + def get_batch_indices(self): + return self.cur_batch_indices + def to_tensor(batch, dtype): if dtype in (int, np.int8, np.int16, np.int32, np.int64): diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py new file mode 100644 index 00000000..ce9627ea --- /dev/null +++ b/fastNLP/core/callback.py @@ -0,0 +1,242 @@ +class Callback(object): + """An Interface for all callbacks. + + Any customized callback should implement at least one of the following methods. + + """ + + def __init__(self): + super(Callback, self).__init__() + + def before_train(self): + # before the main training loop + pass + + def before_epoch(self, cur_epoch, total_epoch): + # at the beginning of each epoch + pass + + def before_batch(self, batch_x, batch_y, indices): + # at the beginning of each step/mini-batch + pass + + def before_loss(self, batch_y, predict_y): + # after data_forward, and before loss computation + pass + + def before_backward(self, loss, model): + # after loss computation, and before gradient backward + pass + + def after_backward(self, model): + pass + + def after_step(self, optimizer): + pass + + def after_batch(self, *args): + # at the end of each step/mini-batch + pass + + def after_valid(self, eval_result, metric_key, optimizer): + """ + 每次执行验证机的evaluation后会调用。传入eval_result + + :param eval_result: Dict[str: Dict[str: float]], evaluation的结果 + :param metric_key: str + :param optimizer: + :return: + """ + pass + + def after_epoch(self, cur_epoch, n_epoch, optimizer): + """ + 每个epoch结束将会调用该方法 + + :param cur_epoch: int, 当前的batch。从1开始。 + :param n_epoch: int, 总的batch数 + :param optimizer: 传入Trainer的optimizer。 + :return: + """ + pass + + def after_train(self, model): + """ + 训练结束，调用该方法 + + :param model: nn.Module, 传入Trainer的模型 + :return: + """ + pass + + def on_exception(self, exception, model, indices): + """ + 当训练过程出现异常，会触发该方法 + :param exception: 某种类型的Exception，比如KeyboardInterrupt等 + :param model: 传入Trainer的模型 + :param indices: 当前batch的index + :return: + """ + pass + +def transfer(func): + """装饰器，将对CallbackManager的调用转发到各个Callback子类. + + :param func: + :return: + """ + + def wrapper(manager, *arg): + returns = [] + for callback in manager.callbacks: + for env_name, env_value in manager.env.items(): + setattr(callback, env_name, env_value) + returns.append(getattr(callback, func.__name__)(*arg)) + return returns + + return wrapper + + +class CallbackManager(Callback): + """A manager for all callbacks passed into Trainer. + It collects resources inside Trainer and raise callbacks. + + """ + + def __init__(self, env, callbacks=None): + """ + + :param dict env: The key is the name of the Trainer attribute(str). The value is the attribute itself. + :param Callback callbacks: + """ + super(CallbackManager, self).__init__() + # set attribute of trainer environment + self.env = env + + self.callbacks = [] + if callbacks is not None: + if isinstance(callbacks, list): + if all([isinstance(cb, Callback) for cb in callbacks]) is True: + self.callbacks.extend(callbacks) + else: + obj = [not isinstance(cb, Callback) for cb in callbacks][0] + raise TypeError(f"Expect sub-classes of Callback. Got {type(obj)}") + else: + raise TypeError(f"Expect callbacks in CallbackManager(callbacks) to be list. Got {type(callbacks)}.") + + @transfer + def before_train(self): + pass + + @transfer + def before_epoch(self, cur_epoch, total_epoch): + pass + + @transfer + def before_batch(self, batch_x, batch_y, indices): + pass + + @transfer + def before_loss(self, batch_y, predict_y): + pass + + @transfer + def before_backward(self, loss, model): + pass + + @transfer + def after_backward(self, model): + pass + + @transfer + def after_step(self, optimizer): + pass + + @transfer + def after_batch(self): + pass + + @transfer + def after_valid(self, eval_result, metric_key, optimizer): + pass + + @transfer + def after_epoch(self, cur_epoch, n_epoch, optimizer): + pass + + @transfer + def after_train(self, model): + pass + + @transfer + def on_exception(self, exception, model, indices): + pass + + +class DummyCallback(Callback): + def before_train(self, *arg): + print(arg) + + def after_epoch(self, cur_epoch, n_epoch, optimizer): + print(cur_epoch, n_epoch, optimizer) + + +class EchoCallback(Callback): + def before_train(self): + print("before_train") + + def before_epoch(self, cur_epoch, total_epoch): + print("before_epoch") + + def before_batch(self, batch_x, batch_y, indices): + print("before_batch") + + def before_loss(self, batch_y, predict_y): + print("before_loss") + + def before_backward(self, loss, model): + print("before_backward") + + def after_batch(self): + print("after_batch") + + def after_epoch(self, cur_epoch, n_epoch, optimizer): + print("after_epoch") + + def after_train(self, model): + print("after_train") + +class GradientClipCallback(Callback): + def __init__(self, parameters=None, clip_value=1, clip_type='norm'): + """ + 每次backward前，将parameter的gradient clip到某个范围。 + + :param parameters: None, torch.Tensor或List[torch.Tensor], 一般通过model.parameters()获得。如果为None则默认对Trainer + 的model中所有参数进行clip + :param clip_value: float, 将gradient 限制到[-clip_value, clip_value]。clip_value应该为正数 + :param clip_type: str, 支持'norm', 'value'两种。 + (1) 'norm', 将gradient的norm rescale到[-clip_value, clip_value] + (2) 'value', 将gradient限制在[-clip_value, clip_value], 小于-clip_value的gradient被赋值为-clip_value; 大于 + clip_value的gradient被赋值为clip_value. + """ + super().__init__() + + from torch import nn + if clip_type == 'norm': + self.clip_fun = nn.utils.clip_grad_norm_ + elif clip_type == 'value': + self.clip_fun = nn.utils.clip_grad_value_ + else: + raise ValueError("Only supports `norm` or `value` right now.") + self.parameters = parameters + self.clip_value = clip_value + + def after_backward(self, model): + self.clip_fun(model.parameters(), self.clip_value) + + + +if __name__ == "__main__": + manager = CallbackManager(env={"n_epoch": 3}, callbacks=[DummyCallback(), DummyCallback()]) + manager.before_train(10, 11, 12) + # print(manager.after_epoch()) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 844492dd..64aa2934 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -254,6 +254,8 @@ class DataSet(object): :return results: if new_field_name is not passed, returned values of the function over all instances. """ results = [func(ins) for ins in self._inner_iter()] + if len(list(filter(lambda x: x is not None, results))) == 0 and not (new_field_name is None): # all None + raise ValueError("{} always return None.".format(get_func_signature(func=func))) extra_param = {} if 'is_input' in kwargs: @@ -261,8 +263,6 @@ class DataSet(object): if 'is_target' in kwargs: extra_param['is_target'] = kwargs['is_target'] if new_field_name is not None: - if len(list(filter(lambda x: x is not None, results))) == 0: # all None - raise ValueError("{} always return None.".format(get_func_signature(func=func))) if new_field_name in self.field_arrays: # overwrite the field, keep same attributes old_field = self.field_arrays[new_field_name] diff --git a/fastNLP/core/instance.py b/fastNLP/core/instance.py index a2877a8f..a102b51c 100644 --- a/fastNLP/core/instance.py +++ b/fastNLP/core/instance.py @@ -30,5 +30,7 @@ class Instance(object): return self.add_field(name, field) def __repr__(self): + s = '\'' return "{" + ",\n".join( - "\'" + field_name + "\': " + str(self.fields[field_name]) for field_name in self.fields) + "}" + "\'" + field_name + "\': " + str(self.fields[field_name]) +\ + f" type={(str(type(self.fields[field_name]))).split(s)[1]}" for field_name in self.fields) + "}" diff --git a/fastNLP/core/losses.py b/fastNLP/core/losses.py index 04f8b73e..9b8b8d8f 100644 --- a/fastNLP/core/losses.py +++ b/fastNLP/core/losses.py @@ -195,7 +195,7 @@ class CrossEntropyLoss(LossBase): def get_loss(self, pred, target): return F.cross_entropy(input=pred, target=target, - ignore_index=self.padding_idx) + ignore_index=self.padding_idx) class L1Loss(LossBase): @@ -250,7 +250,7 @@ class LossInForward(LossBase): if not (isinstance(loss, torch.Tensor) and len(loss.size()) == 0): if not isinstance(loss, torch.Tensor): - raise TypeError(f"loss excepts to be a torch.Tensor, got {type(loss)}") + raise TypeError(f"Loss excepted to be a torch.Tensor, got {type(loss)}") raise RuntimeError(f"The size of loss excepts to be torch.Size([]), got {loss.size()}") return loss diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index 388f3e84..dfb20480 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -10,6 +10,7 @@ from fastNLP.core.utils import _build_args from fastNLP.core.utils import _check_arg_dict_list from fastNLP.core.utils import get_func_signature from fastNLP.core.utils import seq_lens_to_masks +from fastNLP.core.vocabulary import Vocabulary class MetricBase(object): @@ -80,11 +81,6 @@ class MetricBase(object): f"Parameter `{func_param}` is not in {get_func_signature(self.evaluate)}. Please check the " f"initialization parameters, or change its signature.") - # evaluate should not have varargs. - # if func_spect.varargs: - # raise NameError(f"Delete `*{func_spect.varargs}` in {get_func_signature(self.evaluate)}(Do not use " - # f"positional argument.).") - def get_metric(self, reset=True): raise NotImplemented @@ -108,10 +104,9 @@ class MetricBase(object): This method will call self.evaluate method. Before calling self.evaluate, it will first check the validity of output_dict, target_dict - (1) whether self.evaluate has varargs, which is not supported. - (2) whether params needed by self.evaluate is not included in output_dict,target_dict. - (3) whether params needed by self.evaluate duplicate in pred_dict, target_dict - (4) whether params in output_dict, target_dict are not used by evaluate.(Might cause warning) + (1) whether params needed by self.evaluate is not included in output_dict,target_dict. + (2) whether params needed by self.evaluate duplicate in pred_dict, target_dict + (3) whether params in output_dict, target_dict are not used by evaluate.(Might cause warning) Besides, before passing params into self.evaluate, this function will filter out params from output_dict and target_dict which are not used in self.evaluate. (but if **kwargs presented in self.evaluate, no filtering will be conducted.) @@ -299,6 +294,368 @@ class AccuracyMetric(MetricBase): self.total = 0 return evaluate_result +def bmes_tag_to_spans(tags, ignore_labels=None): + """ + + :param tags: List[str], + :param ignore_labels: List[str], 在该list中的label将被忽略 + :return: List[Tuple[str, List[int, int]]]. [(label，[start, end])] + """ + ignore_labels = set(ignore_labels) if ignore_labels else set() + + spans = [] + prev_bmes_tag = None + for idx, tag in enumerate(tags): + tag = tag.lower() + bmes_tag, label = tag[:1], tag[2:] + if bmes_tag in ('b', 's'): + spans.append((label, [idx, idx])) + elif bmes_tag in ('m', 'e') and prev_bmes_tag in ('b', 'm') and label==spans[-1][0]: + spans[-1][1][1] = idx + else: + spans.append((label, [idx, idx])) + prev_bmes_tag = bmes_tag + return [(span[0], (span[1][0], span[1][1])) + for span in spans + if span[0] not in ignore_labels + ] + +def bio_tag_to_spans(tags, ignore_labels=None): + """ + + :param tags: List[str], + :param ignore_labels: List[str], 在该list中的label将被忽略 + :return: List[Tuple[str, List[int, int]]]. [(label，[start, end])] + """ + ignore_labels = set(ignore_labels) if ignore_labels else set() + + spans = [] + prev_bio_tag = None + for idx, tag in enumerate(tags): + tag = tag.lower() + bio_tag, label = tag[:1], tag[2:] + if bio_tag == 'b': + spans.append((label, [idx, idx])) + elif bio_tag == 'i' and prev_bio_tag in ('b', 'i') and label==spans[-1][0]: + spans[-1][1][1] = idx + elif bio_tag == 'o': # o tag does not count + pass + else: + spans.append((label, [idx, idx])) + prev_bio_tag = bio_tag + return [(span[0], (span[1][0], span[1][1])) + for span in spans + if span[0] not in ignore_labels + ] + + +class SpanFPreRecMetric(MetricBase): + """ + 在序列标注问题中，以span的方式计算F, pre, rec. + 最后得到的metric结果为 + { + 'f': xxx, # 这里使用f考虑以后可以计算f_beta值 + 'pre': xxx, + 'rec':xxx + } + 若only_gross=False, 即还会返回各个label的metric统计值 + { + 'f': xxx, + 'pre': xxx, + 'rec':xxx, + 'f-label': xxx, + 'pre-label': xxx, + 'rec-label':xxx, + ... + } + + """ + def __init__(self, tag_vocab, pred=None, target=None, seq_lens=None, encoding_type='bio', ignore_labels=None, + only_gross=True, f_type='micro', beta=1): + """ + + :param tag_vocab: Vocabulary, 标签的vocabulary。支持的标签为"B"(没有label)；或"B-xxx"(xxx为某种label，比如POS中的NN)， + 在解码时，会将相同xxx的认为是同一个label，比如['B-NN', 'E-NN']会被合并为一个'NN'. + :param pred: str, 用该key在evaluate()时从传入dict中取出prediction数据。 为None，则使用'pred'取数据 + :param target: str, 用该key在evaluate()时从传入dict中取出target数据。 为None，则使用'target'取数据 + :param seq_lens: str, 用该key在evaluate()时从传入dict中取出sequence length数据。为None，则使用'seq_lens'取数据。 + :param encoding_type: str, 目前支持bio, bmes + :param ignore_labels, List[str]. 这个list中的class不会被用于计算。例如在POS tagging时传入['NN']，则不会计算'NN'这 + 个label + :param only_gross, bool. 是否只计算总的f1, precision, recall的值；如果为False，不仅返回总的f1, pre, rec, 还会返回每个 + label的f1, pre, rec + :param f_type, str. 'micro'或'macro'. 'micro':通过先计算总体的TP，FN和FP的数量，再计算f, precision, recall; 'macro': + 分布计算每个类别的f, precision, recall，然后做平均（各类别f的权重相同） + :param beta, float. f_beta分数，f_beta = (1 + beta^2)*(pre*rec)/(beta^2*pre + rec). 常用为beta=0.5, 1, 2. 若为0.5 + 则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 + """ + encoding_type = encoding_type.lower() + if encoding_type not in ('bio', 'bmes'): + raise ValueError("Only support 'bio' or 'bmes' type.") + if not isinstance(tag_vocab, Vocabulary): + raise TypeError("tag_vocab can only be fastNLP.Vocabulary, not {}.".format(type(tag_vocab))) + if f_type not in ('micro', 'macro'): + raise ValueError("f_type only supports `micro` or `macro`', got {}.".format(f_type)) + + self.encoding_type = encoding_type + if self.encoding_type == 'bmes': + self.tag_to_span_func = bmes_tag_to_spans + elif self.encoding_type == 'bio': + self.tag_to_span_func = bio_tag_to_spans + self.ignore_labels = ignore_labels + self.f_type = f_type + self.beta = beta + self.beta_square = self.beta**2 + self.only_gross = only_gross + + super().__init__() + self._init_param_map(pred=pred, target=target, seq_lens=seq_lens) + + self.tag_vocab = tag_vocab + + self._true_positives = defaultdict(int) + self._false_positives = defaultdict(int) + self._false_negatives = defaultdict(int) + + def evaluate(self, pred, target, seq_lens): + """ + A lot of design idea comes from allennlp's measure + :param pred: + :param target: + :param seq_lens: + :return: + """ + if not isinstance(pred, torch.Tensor): + raise TypeError(f"`pred` in {get_func_signature(self.evaluate)} must be torch.Tensor," + f"got {type(pred)}.") + if not isinstance(target, torch.Tensor): + raise TypeError(f"`target` in {get_func_signature(self.evaluate)} must be torch.Tensor," + f"got {type(target)}.") + + if not isinstance(seq_lens, torch.Tensor): + raise TypeError(f"`seq_lens` in {get_func_signature(self.evaluate)} must be torch.Tensor," + f"got {type(seq_lens)}.") + + if pred.size() == target.size() and len(target.size()) == 2: + pass + elif len(pred.size()) == len(target.size()) + 1 and len(target.size()) == 2: + pred = pred.argmax(dim=-1) + num_classes = pred.size(-1) + if (target >= num_classes).any(): + raise ValueError("A gold label passed to SpanBasedF1Metric contains an " + "id >= {}, the number of classes.".format(num_classes)) + else: + raise RuntimeError(f"In {get_func_signature(self.evaluate)}, when pred have " + f"size:{pred.size()}, target should have size: {pred.size()} or " + f"{pred.size()[:-1]}, got {target.size()}.") + + batch_size = pred.size(0) + for i in range(batch_size): + pred_tags = pred[i, :int(seq_lens[i])].tolist() + gold_tags = target[i, :int(seq_lens[i])].tolist() + + pred_str_tags = [self.tag_vocab.to_word(tag) for tag in pred_tags] + gold_str_tags = [self.tag_vocab.to_word(tag) for tag in gold_tags] + + pred_spans = self.tag_to_span_func(pred_str_tags, ignore_labels=self.ignore_labels) + gold_spans = self.tag_to_span_func(gold_str_tags, ignore_labels=self.ignore_labels) + + for span in pred_spans: + if span in gold_spans: + self._true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + self._false_positives[span[0]] += 1 + for span in gold_spans: + self._false_negatives[span[0]] += 1 + + def get_metric(self, reset=True): + evaluate_result = {} + if not self.only_gross or self.f_type=='macro': + tags = set(self._false_negatives.keys()) + tags.update(set(self._false_positives.keys())) + tags.update(set(self._true_positives.keys())) + f_sum = 0 + pre_sum = 0 + rec_sum = 0 + for tag in tags: + tp = self._true_positives[tag] + fn = self._false_negatives[tag] + fp = self._false_positives[tag] + f, pre, rec = self._compute_f_pre_rec(tp, fn, fp) + f_sum += f + pre_sum += pre + rec_sum + rec + if not self.only_gross and tag!='': # tag!=''防止无tag的情况 + f_key = 'f-{}'.format(tag) + pre_key = 'pre-{}'.format(tag) + rec_key = 'rec-{}'.format(tag) + evaluate_result[f_key] = f + evaluate_result[pre_key] = pre + evaluate_result[rec_key] = rec + + if self.f_type == 'macro': + evaluate_result['f'] = f_sum/len(tags) + evaluate_result['pre'] = pre_sum/len(tags) + evaluate_result['rec'] = rec_sum/len(tags) + + if self.f_type == 'micro': + f, pre, rec = self._compute_f_pre_rec(sum(self._true_positives.values()), + sum(self._false_negatives.values()), + sum(self._false_positives.values())) + evaluate_result['f'] = round(f, 6) + evaluate_result['pre'] = round(pre, 6) + evaluate_result['rec'] = round(rec, 6) + + if reset: + self._true_positives = defaultdict(int) + self._false_positives = defaultdict(int) + self._false_negatives = defaultdict(int) + + return evaluate_result + + def _compute_f_pre_rec(self, tp, fn, fp): + """ + + :param tp: int, true positive + :param fn: int, false negative + :param fp: int, false positive + :return: (f, pre, rec) + """ + pre = tp / (fp + tp + 1e-13) + rec = tp / (fn + tp + 1e-13) + f = (1 + self.beta_square) * pre * rec / (self.beta_square * pre + rec + 1e-13) + + return f, pre, rec + +class BMESF1PreRecMetric(MetricBase): + """ + 按照BMES标注方式计算f1, precision, recall。由于可能存在非法tag，比如"BS"，所以需要用以下的表格做转换，cur_B意思是当前tag是B， + next_B意思是后一个tag是B。则cur_B=S，即将当前被predict是B的tag标为S；next_M=B, 即将后一个被predict是M的tag标为B + | | next_B | next_M | next_E | next_S | end | + |:-----:|:-------:|:--------:|:--------:|:-------:|:-------:| + | start | 合法 | next_M=B | next_E=S | 合法 | - | + | cur_B | cur_B=S | 合法 | 合法 | cur_B=S | cur_B=S | + | cur_M | cur_M=E | 合法 | 合法 | cur_M=E | cur_M=E | + | cur_E | 合法 | next_M=B | next_E=S | 合法 | 合法 | + | cur_S | 合法 | next_M=B | next_E=S | 合法 | 合法 | + 举例： + prediction为BSEMS，会被认为是SSSSS. + + 本Metric不检验target的合法性，请务必保证target的合法性。 + pred的形状应该为(batch_size, max_len) 或 (batch_size, max_len, 4)。 + target形状为 (batch_size, max_len) + seq_lens形状为 (batch_size, ) + + """ + + def __init__(self, b_idx=0, m_idx=1, e_idx=2, s_idx=3, pred=None, target=None, seq_lens=None): + """ + 需要申明BMES这四种tag中，各种tag对应的idx。所有不为b_idx, m_idx, e_idx, s_idx的数字都认为是s_idx。 + + :param b_idx: int, Begin标签所对应的tag idx. + :param m_idx: int, Middle标签所对应的tag idx. + :param e_idx: int, End标签所对应的tag idx. + :param s_idx: int, Single标签所对应的tag idx + :param pred: str, 用该key在evaluate()时从传入dict中取出prediction数据。 为None，则使用'pred'取数据 + :param target: str, 用该key在evaluate()时从传入dict中取出target数据。 为None，则使用'target'取数据 + :param seq_lens: str, 用该key在evaluate()时从传入dict中取出seqence length数据。为None，则使用'seq_lens'取数据。 + """ + super().__init__() + + self._init_param_map(pred=pred, target=target, seq_lens=seq_lens) + + self.yt_wordnum = 0 + self.yp_wordnum = 0 + self.corr_num = 0 + + self.b_idx = b_idx + self.m_idx = m_idx + self.e_idx = e_idx + self.s_idx = s_idx + # 还原init处介绍的矩阵 + self._valida_matrix = { + -1: [(-1, -1), (1, self.b_idx), (1, self.s_idx), (-1, -1)], # magic start idx + self.b_idx:[(0, self.s_idx), (-1, -1), (-1, -1), (0, self.s_idx), (0, self.s_idx)], + self.m_idx:[(0, self.e_idx), (-1, -1), (-1, -1), (0, self.e_idx), (0, self.e_idx)], + self.e_idx:[(-1, -1), (1, self.b_idx), (1, self.s_idx), (-1, -1), (-1, -1)], + self.s_idx:[(-1, -1), (1, self.b_idx), (1, self.s_idx), (-1, -1), (-1, -1)], + } + + def _validate_tags(self, tags): + """ + 给定一个tag的Tensor，返回合法tag + + :param tags: Tensor, shape: (seq_len, ) + :return: 返回修改为合法tag的list + """ + assert len(tags)!=0 + assert isinstance(tags, torch.Tensor) and len(tags.size())==1 + padded_tags = [-1, *tags.tolist(), -1] + for idx in range(len(padded_tags)-1): + cur_tag = padded_tags[idx] + if cur_tag not in self._valida_matrix: + cur_tag = self.s_idx + if padded_tags[idx+1] not in self._valida_matrix: + padded_tags[idx+1] = self.s_idx + next_tag = padded_tags[idx+1] + shift_tag = self._valida_matrix[cur_tag][next_tag] + if shift_tag[0]!=-1: + padded_tags[idx+shift_tag[0]] = shift_tag[1] + + return padded_tags[1:-1] + + def evaluate(self, pred, target, seq_lens): + if not isinstance(pred, torch.Tensor): + raise TypeError(f"`pred` in {get_func_signature(self.evaluate)} must be torch.Tensor," + f"got {type(pred)}.") + if not isinstance(target, torch.Tensor): + raise TypeError(f"`target` in {get_func_signature(self.evaluate)} must be torch.Tensor," + f"got {type(target)}.") + + if not isinstance(seq_lens, torch.Tensor): + raise TypeError(f"`seq_lens` in {get_func_signature(self.evaluate)} must be torch.Tensor," + f"got {type(seq_lens)}.") + + if pred.size() == target.size() and len(target.size()) == 2: + pass + elif len(pred.size()) == len(target.size()) + 1 and len(target.size()) == 2: + pred = pred.argmax(dim=-1) + else: + raise RuntimeError(f"In {get_func_signature(self.evaluate)}, when pred have " + f"size:{pred.size()}, target should have size: {pred.size()} or " + f"{pred.size()[:-1]}, got {target.size()}.") + + for idx in range(len(pred)): + seq_len = seq_lens[idx] + target_tags = target[idx][:seq_len].tolist() + pred_tags = pred[idx][:seq_len] + pred_tags = self._validate_tags(pred_tags) + start_idx = 0 + for t_idx, (t_tag, p_tag) in enumerate(zip(target_tags, pred_tags)): + if t_tag in (self.s_idx, self.e_idx): + self.yt_wordnum += 1 + corr_flag = True + for i in range(start_idx, t_idx+1): + if target_tags[i]!=pred_tags[i]: + corr_flag = False + if corr_flag: + self.corr_num += 1 + start_idx = t_idx + 1 + if p_tag in (self.s_idx, self.e_idx): + self.yp_wordnum += 1 + + def get_metric(self, reset=True): + P = self.corr_num / (self.yp_wordnum + 1e-12) + R = self.corr_num / (self.yt_wordnum + 1e-12) + F = 2 * P * R / (P + R + 1e-12) + evaluate_result = {'f': round(F, 6), 'pre':round(P, 6), 'rec': round(R, 6)} + if reset: + self.yp_wordnum = 0 + self.yt_wordnum = 0 + self.corr_num = 0 + return evaluate_result + def _prepare_metrics(metrics): """ diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 3e8360d2..109315a3 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -7,9 +7,14 @@ import numpy as np import torch from tensorboardX import SummaryWriter from torch import nn -from tqdm.autonotebook import tqdm + +try: + from tqdm.autonotebook import tqdm +except: + from fastNLP.core.utils import pseudo_tqdm as tqdm from fastNLP.core.batch import Batch +from fastNLP.core.callback import CallbackManager from fastNLP.core.dataset import DataSet from fastNLP.core.losses import _prepare_losser from fastNLP.core.metrics import _prepare_metrics @@ -27,7 +32,11 @@ from fastNLP.core.utils import get_func_signature class Trainer(object): - """ + def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch_size=32, print_every=50, + validate_every=-1, dev_data=None, save_path=None, optimizer=Adam(lr=0.01, weight_decay=0), + check_code_level=0, metric_key=None, sampler=RandomSampler(), use_tqdm=True, use_cuda=False, + callbacks=None): + """ :param DataSet train_data: the training data :param torch.nn.modules.module model: a PyTorch model :param LossBase loss: a loss object @@ -48,16 +57,10 @@ class Trainer(object): smaller, add "-" in front of the string. For example:: metric_key="-PPL" # language model gets better as perplexity gets smaller - :param BaseSampler sampler: method used to generate batch data. :param bool use_tqdm: whether to use tqdm to show train progress. - """ - - def __init__(self, train_data, model, loss=None, metrics=None, n_epochs=3, batch_size=32, print_every=50, - validate_every=-1, dev_data=None, use_cuda=False, save_path=None, - optimizer=Adam(lr=0.01, weight_decay=0), check_code_level=0, - metric_key=None, sampler=RandomSampler(), use_tqdm=True): + """ super(Trainer, self).__init__() if not isinstance(train_data, DataSet): @@ -109,9 +112,10 @@ class Trainer(object): self.use_cuda = bool(use_cuda) self.save_path = save_path self.print_every = int(print_every) - self.validate_every = int(validate_every) + self.validate_every = int(validate_every) if validate_every!=0 else -1 self.best_metric_indicator = None self.sampler = sampler + self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) if isinstance(optimizer, torch.optim.Optimizer): self.optimizer = optimizer @@ -119,11 +123,7 @@ class Trainer(object): self.optimizer = optimizer.construct_from_pytorch(self.model.parameters()) self.use_tqdm = use_tqdm - if self.use_tqdm: - tester_verbose = 0 - self.print_every = abs(self.print_every) - else: - tester_verbose = 1 + self.print_every = abs(self.print_every) if self.dev_data is not None: self.tester = Tester(model=self.model, @@ -131,7 +131,7 @@ class Trainer(object): metrics=self.metrics, batch_size=self.batch_size, use_cuda=self.use_cuda, - verbose=tester_verbose) + verbose=0) self.step = 0 self.start_time = None # start timestamp @@ -141,20 +141,30 @@ class Trainer(object): 开始训练过程。主要有以下几个步骤:: - 对于每次循环 - 1. 使用Batch从DataSet中按批取出数据，并自动对DataSet中dtype为float, int的fields进行padding。并转换为Tensor。 + for epoch in range(num_epochs): + # 使用Batch从DataSet中按批取出数据，并自动对DataSet中dtype为(float, int)的fields进行padding。并转换为Tensor。 非float，int类型的参数将不会被转换为Tensor，且不进行padding。 for batch_x, batch_y in Batch(DataSet) - # batch_x中为设置为input的field - # batch_y中为设置为target的field - 2. 将batch_x的数据送入到model.forward函数中，并获取结果 - 3. 将batch_y与model.forward的结果一并送入loss中计算loss + # batch_x是一个dict, 被设为input的field会出现在这个dict中， + key为DataSet中的field_name, value为该field的value + # batch_y也是一个dict，被设为target的field会出现在这个dict中， + key为DataSet中的field_name, value为该field的value + 2. 将batch_x的数据送入到model.forward函数中，并获取结果。这里我们就是通过匹配batch_x中的key与forward函数的形 + 参完成参数传递。例如， + forward(self, x, seq_lens) # fastNLP会在batch_x中找到key为"x"的value传递给x，key为"seq_lens"的 + value传递给seq_lens。若在batch_x中没有找到所有必须要传递的参数，就会报错。如果forward存在默认参数 + 而且默认参数这个key没有在batch_x中，则使用默认参数。 + 3. 将batch_y与model.forward的结果一并送入loss中计算loss。loss计算时一般都涉及到pred与target。但是在不同情况 + 中，可能pred称为output或prediction, target称为y或label。fastNLP通过初始化loss时传入的映射找到pred或 + target。比如在初始化Trainer时初始化loss为CrossEntropyLoss(pred='output', target='y'), 那么fastNLP计 + 算loss时，就会使用"output"在batch_y与forward的结果中找到pred;使用"y"在batch_y与forward的结果中找target + , 并完成loss的计算。 4. 获取到loss之后，进行反向求导并更新梯度 - 如果测试集不为空 - 根据metrics进行evaluation，并根据是否提供了save_path判断是否存储模型 + 根据需要适时进行验证机测试 + 根据metrics进行evaluation，并根据是否提供了save_path判断是否存储模型 - :param bool load_best_model: 该参数只有在初始化提供了dev_data的情况下有效，如果True, trainer将在返回之前重新加载dev表现最好的 - 模型参数。 + :param bool load_best_model: 该参数只有在初始化提供了dev_data的情况下有效，如果True, trainer将在返回之前重新加载dev表现 + 最好的模型参数。 :return results: 返回一个字典类型的数据, 内含以下内容:: seconds: float, 表示训练时长 @@ -187,10 +197,11 @@ class Trainer(object): else: path = os.path.join(self.save_path, 'tensorboard_logs_{}'.format(self.start_time)) self._summary_writer = SummaryWriter(path) - if self.use_tqdm: - self._tqdm_train() - else: - self._print_train() + + self.callback_manager.before_train() + self._train() + self.callback_manager.after_train(self.model) + if self.dev_data is not None: print("\nIn Epoch:{}/Step:{}, got best dev performance:".format(self.best_dev_epoch, self.best_dev_step) + self.tester._format_eval_results(self.best_dev_perf),) @@ -199,8 +210,11 @@ class Trainer(object): results['best_step'] = self.best_dev_step if load_best_model: model_name = "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time]) - # self._load_model(self.model, model_name) - print("Reloaded the best model.") + load_succeed = self._load_model(self.model, model_name) + if load_succeed: + print("Reloaded the best model.") + else: + print("Fail to reload best model.") finally: self._summary_writer.close() del self._summary_writer @@ -208,22 +222,43 @@ class Trainer(object): return results - def _tqdm_train(self): + def _train(self): + if not self.use_tqdm: + from fastNLP.core.utils import pseudo_tqdm as inner_tqdm + else: + inner_tqdm = tqdm self.step = 0 - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, - as_numpy=False) - total_steps = data_iterator.num_batches*self.n_epochs - with tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: + start = time.time() + data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False) + total_steps = data_iterator.num_batches * self.n_epochs + with inner_tqdm(total=total_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: avg_loss = 0 for epoch in range(1, self.n_epochs+1): pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) + # early stopping + self.callback_manager.before_epoch(epoch, self.n_epochs) for batch_x, batch_y in data_iterator: + indices = data_iterator.get_batch_indices() + # negative sampling; replace unknown; re-weight batch_y + self.callback_manager.before_batch(batch_x, batch_y, indices) _move_dict_value_to_device(batch_x, batch_y, device=self._model_device) prediction = self._data_forward(self.model, batch_x) + + # edit prediction + self.callback_manager.before_loss(batch_y, prediction) loss = self._compute_loss(prediction, batch_y) avg_loss += loss.item() + + # Is loss NaN or inf? requires_grad = False + self.callback_manager.before_backward(loss, self.model) self._grad_backward(loss) + # gradient clipping + self.callback_manager.after_backward(self.model) + self._update() + # lr scheduler; lr_finder; one_cycle + self.callback_manager.after_step(self.optimizer) + self._summary_writer.add_scalar("loss", loss.item(), global_step=self.step) for name, param in self.model.named_parameters(): if param.requires_grad: @@ -231,65 +266,41 @@ class Trainer(object): # self._summary_writer.add_scalar(name + "_std", param.std(), global_step=self.step) # self._summary_writer.add_scalar(name + "_grad_sum", param.sum(), global_step=self.step) if (self.step+1) % self.print_every == 0: - pbar.set_postfix_str("loss:{0:<6.5f}".format(avg_loss / self.print_every)) + if self.use_tqdm: + print_output = "loss:{0:<6.5f}".format(avg_loss / self.print_every) + pbar.update(self.print_every) + else: + end = time.time() + diff = timedelta(seconds=round(end - start)) + print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format( + epoch, self.step, avg_loss, diff) + pbar.set_postfix_str(print_output) avg_loss = 0 - pbar.update(self.print_every) self.step += 1 - if self.validate_every > 0 and self.step % self.validate_every == 0 \ + # do nothing + self.callback_manager.after_batch() + + if ((self.validate_every > 0 and self.step % self.validate_every == 0) or + (self.validate_every < 0 and self.step % len(data_iterator)) == 0) \ and self.dev_data is not None: eval_res = self._do_validation(epoch=epoch, step=self.step) - eval_str = "Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, total_steps) + \ + eval_str = "Evaluation at Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, + total_steps) + \ self.tester._format_eval_results(eval_res) pbar.write(eval_str) - if self.validate_every < 0 and self.dev_data: - eval_res = self._do_validation(epoch=epoch, step=self.step) - eval_str = "Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, total_steps) + \ - self.tester._format_eval_results(eval_res) - pbar.write(eval_str) - if epoch!=self.n_epochs: + + # if self.validate_every < 0 and self.dev_data: + # eval_res = self._do_validation(epoch=epoch, step=self.step) + # eval_str = "Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, total_steps) + \ + # self.tester._format_eval_results(eval_res) + # pbar.write(eval_str) + if epoch != self.n_epochs: data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False) + # lr decay; early stopping + self.callback_manager.after_epoch(epoch, self.n_epochs, self.optimizer) pbar.close() - def _print_train(self): - epoch = 1 - start = time.time() - while epoch <= self.n_epochs: - - data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, - as_numpy=False) - - for batch_x, batch_y in data_iterator: - # TODO 这里可能会遇到问题，万一用户在model内部修改了prediction的device就会有问题 - _move_dict_value_to_device(batch_x, batch_y, device=self._model_device) - prediction = self._data_forward(self.model, batch_x) - loss = self._compute_loss(prediction, batch_y) - self._grad_backward(loss) - self._update() - self._summary_writer.add_scalar("loss", loss.item(), global_step=self.step) - for name, param in self.model.named_parameters(): - if param.requires_grad: - self._summary_writer.add_scalar(name + "_mean", param.mean(), global_step=self.step) - # self._summary_writer.add_scalar(name + "_std", param.std(), global_step=self.step) - # self._summary_writer.add_scalar(name + "_grad_sum", param.sum(), global_step=self.step) - if self.print_every > 0 and self.step % self.print_every == 0: - end = time.time() - diff = timedelta(seconds=round(end - start)) - print_output = "[epoch: {:>3} step: {:>4}] train loss: {:>4.6} time: {}".format( - epoch, self.step, loss.data, diff) - print(print_output) - - if (self.validate_every > 0 and self.step % self.validate_every == 0 and - self.dev_data is not None): - self._do_validation(epoch=epoch, step=self.step) - - self.step += 1 - - # validate_every override validation at end of epochs - if self.dev_data and self.validate_every <= 0: - self._do_validation(epoch=epoch, step=self.step) - epoch += 1 - def _do_validation(self, epoch, step): res = self.tester.test() for name, metric in res.items(): @@ -300,10 +311,13 @@ class Trainer(object): if self.save_path is not None: self._save_model(self.model, "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time])) - + else: + self._best_model_states = {name:param.cpu().clone() for name, param in self.model.named_parameters()} self.best_dev_perf = res self.best_dev_epoch = epoch self.best_dev_step = step + # get validation results; adjust optimizer + self.callback_manager.after_valid(res, self.metric_key, self.optimizer) return res def _mode(self, model, is_test=False): @@ -359,7 +373,7 @@ class Trainer(object): torch.save(model, model_name) def _load_model(self, model, model_name, only_param=False): - # TODO: 这个是不是有问题？ + # 返回bool值指示是否成功reload模型 if self.save_path is not None: model_path = os.path.join(self.save_path, model_name) if only_param: @@ -367,6 +381,11 @@ class Trainer(object): else: states = torch.load(model_path).state_dict() model.load_state_dict(states) + elif hasattr(self, "_best_model_states"): + model.load_state_dict(self._best_model_states) + else: + return False + return True def _better_eval_result(self, metrics): """Check if the current epoch yields better validation results. @@ -472,7 +491,7 @@ def _check_code(dataset, model, losser, metrics, batch_size=DEFAULT_CHECK_BATCH_ break if dev_data is not None: - tester = Tester(data=dataset[:batch_size * DEFAULT_CHECK_NUM_BATCH], model=model, metrics=metrics, + tester = Tester(data=dev_data[:batch_size * DEFAULT_CHECK_NUM_BATCH], model=model, metrics=metrics, batch_size=batch_size, verbose=-1) evaluate_results = tester.test() _check_eval_results(metrics=evaluate_results, metric_key=metric_key, metric_list=metrics) diff --git a/fastNLP/core/utils.py b/fastNLP/core/utils.py index e93c95f4..2e0f383e 100644 --- a/fastNLP/core/utils.py +++ b/fastNLP/core/utils.py @@ -9,7 +9,7 @@ import numpy as np import torch CheckRes = namedtuple('CheckRes', ['missing', 'unused', 'duplicated', 'required', 'all_needed', - 'varargs'], verbose=False) + 'varargs']) def save_pickle(obj, pickle_path, file_name): @@ -400,7 +400,7 @@ def seq_lens_to_masks(seq_lens, float=False): assert len(np.shape(seq_lens)) == 1, f"seq_lens can only have one dimension, got {len(np.shape(seq_lens))}." assert seq_lens.dtype in (int, np.int32, np.int64), f"seq_lens can only be integer, not {seq_lens.dtype}." raise NotImplemented - elif isinstance(seq_lens, torch.LongTensor): + elif isinstance(seq_lens, torch.Tensor): assert len(seq_lens.size()) == 1, f"seq_lens can only have one dimension, got {len(seq_lens.size())==1}." batch_size = seq_lens.size(0) max_len = seq_lens.max() @@ -430,3 +430,30 @@ def seq_mask(seq_len, max_len): seq_len = seq_len.view(-1, 1).long() # [batch_size, 1] seq_range = torch.arange(start=0, end=max_len, dtype=torch.long, device=seq_len.device).view(1, -1) # [1, max_len] return torch.gt(seq_len, seq_range) # [batch_size, max_len] + + +class pseudo_tqdm: + """ + 当无法引入tqdm，或者Trainer中设置use_tqdm为false的时候，用该方法打印数据 + """ + + def __init__(self, **kwargs): + pass + + def write(self, info): + print(info) + + def set_postfix_str(self, info): + print(info) + + def __getattr__(self, item): + def pass_func(*args, **kwargs): + pass + + return pass_func + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + del self diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index 87682751..27d8a360 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -270,8 +270,8 @@ class ClassDataSetLoader(DataSetLoader): def parse(lines): """ - :param list lines: lines from dataset - :return: a 3-D list, indicating words, sentence, and dataset respectively. + :param lines: lines from dataset + :return: list(list(list())): the three level of lists are words, sentence, and dataset """ dataset = list() for line in lines: diff --git a/fastNLP/io/embed_loader.py b/fastNLP/io/embed_loader.py index aae29ab5..e55fc55b 100644 --- a/fastNLP/io/embed_loader.py +++ b/fastNLP/io/embed_loader.py @@ -96,7 +96,7 @@ class EmbedLoader(BaseLoader): :param int emb_dim: the dimension of the embedding. Should be the same as pre-trained embedding. :param str emb_file: the pre-trained embedding file path. :param Vocabulary vocab: a mapping from word to index, can be provided by user or built from pre-trained embedding - :return: the embedding matrix, numpy.ndarray + :return embedding_matrix: numpy.ndarray """ if vocab is None: diff --git a/fastNLP/models/__init__.py b/fastNLP/models/__init__.py index 5bb2bc3d..a83c3936 100644 --- a/fastNLP/models/__init__.py +++ b/fastNLP/models/__init__.py @@ -3,4 +3,4 @@ from .biaffine_parser import BiaffineParser, GraphParser from .char_language_model import CharLM from .cnn_text_classification import CNNText from .sequence_modeling import SeqLabeling, AdvSeqLabel -from .snli import SNLI +from .snli import ESIM diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index efb07f34..fb687301 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -134,17 +134,13 @@ class GraphParser(BaseModel): def _mst_decoder(self, arc_matrix, mask=None): batch_size, seq_len, _ = arc_matrix.shape - matrix = torch.zeros_like(arc_matrix).copy_(arc_matrix) + matrix = arc_matrix.clone() ans = matrix.new_zeros(batch_size, seq_len).long() lens = (mask.long()).sum(1) if mask is not None else torch.zeros(batch_size) + seq_len batch_idx = torch.arange(batch_size, dtype=torch.long, device=lens.device) - mask[batch_idx, lens-1] = 0 for i, graph in enumerate(matrix): len_i = lens[i] - if len_i == seq_len: - ans[i] = torch.as_tensor(mst(graph.cpu().numpy()), device=ans.device) - else: - ans[i, :len_i] = torch.as_tensor(mst(graph[:len_i, :len_i].cpu().numpy()), device=ans.device) + ans[i, :len_i] = torch.as_tensor(mst(graph.detach()[:len_i, :len_i].cpu().numpy()), device=ans.device) if mask is not None: ans *= mask.long() return ans @@ -219,6 +215,7 @@ class BiaffineParser(GraphParser): self.pos_fc = nn.Linear(pos_emb_dim, pos_hid_dim) self.word_norm = nn.LayerNorm(word_hid_dim) self.pos_norm = nn.LayerNorm(pos_hid_dim) + self.use_var_lstm = use_var_lstm if use_var_lstm: self.lstm = VarLSTM(input_size=word_hid_dim + pos_hid_dim, hidden_size=rnn_hidden_size, @@ -249,10 +246,9 @@ class BiaffineParser(GraphParser): self.label_dep_mlp = copy.deepcopy(self.label_head_mlp) self.arc_predictor = ArcBiaffine(arc_mlp_size, bias=True) self.label_predictor = LabelBilinear(label_mlp_size, label_mlp_size, num_label, bias=True) - self.normal_dropout = nn.Dropout(p=dropout) self.use_greedy_infer = use_greedy_infer self.reset_parameters() - self.explore_p = 0.2 + self.dropout = dropout def reset_parameters(self): for m in self.modules(): @@ -278,18 +274,15 @@ class BiaffineParser(GraphParser): head_pred: [batch_size, seq_len] if gold_heads is not provided, predicting the heads """ # prepare embeddings - device = self.parameters().__next__().device - word_seq = word_seq.long().to(device) - pos_seq = pos_seq.long().to(device) - seq_lens = seq_lens.long().to(device).view(-1) batch_size, seq_len = word_seq.shape # print('forward {} {}'.format(batch_size, seq_len)) # get sequence mask mask = seq_mask(seq_lens, seq_len).long() - word = self.normal_dropout(self.word_embedding(word_seq)) # [N,L] -> [N,L,C_0] - pos = self.normal_dropout(self.pos_embedding(pos_seq)) # [N,L] -> [N,L,C_1] + word = self.word_embedding(word_seq) # [N,L] -> [N,L,C_0] + pos = self.pos_embedding(pos_seq) # [N,L] -> [N,L,C_1] + word, pos = self.word_fc(word), self.pos_fc(pos) word, pos = self.word_norm(word), self.pos_norm(pos) x = torch.cat([word, pos], dim=2) # -> [N,L,C] @@ -325,7 +318,7 @@ class BiaffineParser(GraphParser): head_pred = heads else: assert self.training # must be training mode - if torch.rand(1).item() < self.explore_p: + if gold_heads is None: heads = self._greedy_decoder(arc_pred, mask) head_pred = heads else: @@ -355,7 +348,7 @@ class BiaffineParser(GraphParser): batch_size, seq_len, _ = arc_pred.shape flip_mask = (mask == 0) - _arc_pred = arc_pred.new_empty((batch_size, seq_len, seq_len)).copy_(arc_pred) + _arc_pred = arc_pred.clone() _arc_pred.masked_fill_(flip_mask.unsqueeze(1), -np.inf) arc_logits = F.log_softmax(_arc_pred, dim=2) label_logits = F.log_softmax(label_pred, dim=2) @@ -421,7 +414,9 @@ class ParserMetric(MetricBase): if seq_lens is None: seq_mask = arc_pred.new_ones(arc_pred.size(), dtype=torch.long) else: - seq_mask = seq_lens_to_masks(seq_lens, float=False).long() + seq_mask = seq_lens_to_masks(seq_lens.long(), float=False).long() + # mask out tag + seq_mask[:,0] = 0 head_pred_correct = (arc_pred == arc_true).long() * seq_mask label_pred_correct = (label_pred == label_true).long() * head_pred_correct self.num_arc += head_pred_correct.sum().item() diff --git a/fastNLP/models/sequence_modeling.py b/fastNLP/models/sequence_modeling.py index e911598c..cb9e9478 100644 --- a/fastNLP/models/sequence_modeling.py +++ b/fastNLP/models/sequence_modeling.py @@ -1,8 +1,8 @@ import torch -import numpy as np from fastNLP.models.base_model import BaseModel from fastNLP.modules import decoder, encoder +from fastNLP.modules.decoder.CRF import allowed_transitions from fastNLP.modules.utils import seq_mask @@ -93,7 +93,7 @@ class AdvSeqLabel(SeqLabeling): Advanced Sequence Labeling Model """ - def __init__(self, args, emb=None): + def __init__(self, args, emb=None, id2words=None): super(AdvSeqLabel, self).__init__(args) vocab_size = args["vocab_size"] @@ -105,7 +105,8 @@ class AdvSeqLabel(SeqLabeling): self.Embedding = encoder.embedding.Embedding(vocab_size, word_emb_dim, init_emb=emb) self.norm1 = torch.nn.LayerNorm(word_emb_dim) # self.Rnn = encoder.lstm.LSTM(word_emb_dim, hidden_dim, num_layers=2, dropout=dropout, bidirectional=True) - self.Rnn = torch.nn.LSTM(input_size=word_emb_dim, hidden_size=hidden_dim, num_layers=2, dropout=dropout, bidirectional=True, batch_first=True) + self.Rnn = torch.nn.LSTM(input_size=word_emb_dim, hidden_size=hidden_dim, num_layers=2, dropout=dropout, + bidirectional=True, batch_first=True) self.Linear1 = encoder.Linear(hidden_dim * 2, hidden_dim * 2 // 3) self.norm2 = torch.nn.LayerNorm(hidden_dim * 2 // 3) # self.batch_norm = torch.nn.BatchNorm1d(hidden_dim * 2 // 3) @@ -113,7 +114,12 @@ class AdvSeqLabel(SeqLabeling): self.drop = torch.nn.Dropout(dropout) self.Linear2 = encoder.Linear(hidden_dim * 2 // 3, num_classes) - self.Crf = decoder.CRF.ConditionalRandomField(num_classes, include_start_end_trans=False) + if id2words is None: + self.Crf = decoder.CRF.ConditionalRandomField(num_classes, include_start_end_trans=False) + else: + self.Crf = decoder.CRF.ConditionalRandomField(num_classes, include_start_end_trans=False, + allowed_transitions=allowed_transitions(id2words, + encoding_type="bmes")) def forward(self, word_seq, word_seq_origin_len, truth=None): """ @@ -178,6 +184,7 @@ class AdvSeqLabel(SeqLabeling): assert 'loss' in kwargs return kwargs['loss'] + if __name__ == '__main__': args = { 'vocab_size': 20, @@ -208,11 +215,11 @@ if __name__ == '__main__': res = model(word_seq, word_seq_len, truth) loss = res['loss'] pred = res['predict'] - print('loss: {} acc {}'.format(loss.item(), ((pred.data == truth).long().sum().float() / word_seq_len.sum().float()))) + print('loss: {} acc {}'.format(loss.item(), + ((pred.data == truth).long().sum().float() / word_seq_len.sum().float()))) optimizer.zero_grad() loss.backward() optimizer.step() curidx = endidx if curidx == len(data): curidx = 0 - diff --git a/fastNLP/models/snli.py b/fastNLP/models/snli.py index 9be7f43d..6a7d8d84 100644 --- a/fastNLP/models/snli.py +++ b/fastNLP/models/snli.py @@ -3,29 +3,35 @@ import torch.nn as nn import torch.nn.functional as F from fastNLP.models.base_model import BaseModel -from fastNLP.modules import decoder as Decoder, encoder as Encoder +from fastNLP.modules import decoder as Decoder +from fastNLP.modules import encoder as Encoder +from fastNLP.modules import aggregator as Aggregator my_inf = 10e12 -class SNLI(BaseModel): +class ESIM(BaseModel): """ - PyTorch Network for SNLI. + PyTorch Network for SNLI task using ESIM model. """ - def __init__(self, args, init_embedding=None): - super(SNLI, self).__init__() - self.vocab_size = args["vocab_size"] - self.embed_dim = args["embed_dim"] - self.hidden_size = args["hidden_size"] - self.batch_first = args["batch_first"] - self.dropout = args["dropout"] - self.n_labels = args["num_classes"] - self.gpu = args["gpu"] and torch.cuda.is_available() - - self.embedding = Encoder.embedding.Embedding(self.vocab_size, self.embed_dim, init_emb=init_embedding, - dropout=self.dropout) + def __init__(self, **kwargs): + super(ESIM, self).__init__() + self.vocab_size = kwargs["vocab_size"] + self.embed_dim = kwargs["embed_dim"] + self.hidden_size = kwargs["hidden_size"] + self.batch_first = kwargs["batch_first"] + self.dropout = kwargs["dropout"] + self.n_labels = kwargs["num_classes"] + self.gpu = kwargs["gpu"] and torch.cuda.is_available() + + self.drop = nn.Dropout(self.dropout) + + self.embedding = Encoder.Embedding( + self.vocab_size, self.embed_dim, dropout=self.dropout, + init_emb=kwargs["init_embedding"] if "inin_embedding" in kwargs.keys() else None, + ) self.embedding_layer = Encoder.Linear(self.embed_dim, self.hidden_size) @@ -34,6 +40,10 @@ class SNLI(BaseModel): batch_first=self.batch_first, bidirectional=True ) + self.bi_attention = Aggregator.Bi_Attention() + self.mean_pooling = Aggregator.MeanPoolWithMask() + self.max_pooling = Aggregator.MaxPoolWithMask() + self.inference_layer = Encoder.Linear(self.hidden_size * 4, self.hidden_size) self.decoder = Encoder.LSTM( @@ -41,16 +51,16 @@ class SNLI(BaseModel): batch_first=self.batch_first, bidirectional=True ) - self.output = Decoder.MLP([4 * self.hidden_size, self.hidden_size, self.n_labels], 'tanh') + self.output = Decoder.MLP([4 * self.hidden_size, self.hidden_size, self.n_labels], 'tanh', dropout=self.dropout) def forward(self, premise, hypothesis, premise_len, hypothesis_len): """ Forward function - :param premise: A Tensor represents premise: [batch size(B), premise seq len(PL), hidden size(H)]. - :param hypothesis: A Tensor represents hypothesis: [B, hypothesis seq len(HL), H]. + :param premise: A Tensor represents premise: [batch size(B), premise seq len(PL)]. + :param hypothesis: A Tensor represents hypothesis: [B, hypothesis seq len(HL)]. :param premise_len: A Tensor record which is a real word and which is a padding word in premise: [B, PL]. :param hypothesis_len: A Tensor record which is a real word and which is a padding word in hypothesis: [B, HL]. - :return: prediction: A Tensor of classification result: [B, n_labels(N)]. + :return: prediction: A Dict with Tensor of classification result: [B, n_labels(N)]. """ premise0 = self.embedding_layer(self.embedding(premise)) @@ -68,16 +78,13 @@ class SNLI(BaseModel): B, PL, H = premise0.size() B, HL, H = hypothesis0.size() - # a0, (ah0, ac0) = self.encoder(premise) # a0: [B, PL, H * 2], ah0: [2, B, H] - # b0, (bh0, bc0) = self.encoder(hypothesis) # b0: [B, HL, H * 2] - - a0 = self.encoder(premise0) # a0: [B, PL, H * 2] - b0 = self.encoder(hypothesis0) # b0: [B, HL, H * 2] + a0 = self.encoder(self.drop(premise0)) # a0: [B, PL, H * 2] + b0 = self.encoder(self.drop(hypothesis0)) # b0: [B, HL, H * 2] a = torch.mean(a0.view(B, PL, -1, H), dim=2) # a: [B, PL, H] b = torch.mean(b0.view(B, HL, -1, H), dim=2) # b: [B, HL, H] - ai, bi = self.calc_bi_attention(a, b, premise_len, hypothesis_len) + ai, bi = self.bi_attention(a, b, premise_len, hypothesis_len) ma = torch.cat((a, ai, a - ai, a * ai), dim=2) # ma: [B, PL, 4 * H] mb = torch.cat((b, bi, b - bi, b * bi), dim=2) # mb: [B, HL, 4 * H] @@ -85,17 +92,12 @@ class SNLI(BaseModel): f_ma = self.inference_layer(ma) f_mb = self.inference_layer(mb) - vat = self.decoder(f_ma) - vbt = self.decoder(f_mb) + vat = self.decoder(self.drop(f_ma)) + vbt = self.decoder(self.drop(f_mb)) va = torch.mean(vat.view(B, PL, -1, H), dim=2) # va: [B, PL, H] vb = torch.mean(vbt.view(B, HL, -1, H), dim=2) # vb: [B, HL, H] - # va_ave = torch.mean(va, dim=1) # va_ave: [B, H] - # va_max, va_arg_max = torch.max(va, dim=1) # va_max: [B, H] - # vb_ave = torch.mean(vb, dim=1) # vb_ave: [B, H] - # vb_max, vb_arg_max = torch.max(vb, dim=1) # vb_max: [B, H] - va_ave = self.mean_pooling(va, premise_len, dim=1) # va_ave: [B, H] va_max, va_arg_max = self.max_pooling(va, premise_len, dim=1) # va_max: [B, H] vb_ave = self.mean_pooling(vb, hypothesis_len, dim=1) # vb_ave: [B, H] @@ -103,59 +105,10 @@ class SNLI(BaseModel): v = torch.cat((va_ave, va_max, vb_ave, vb_max), dim=1) # v: [B, 4 * H] - # v_mlp = F.tanh(self.mlp_layer1(v)) # v_mlp: [B, H] - # prediction = self.mlp_layer2(v_mlp) # prediction: [B, N] - prediction = F.tanh(self.output(v)) # prediction: [B, N] - return prediction - - @staticmethod - def calc_bi_attention(in_x1, in_x2, x1_len, x2_len): - - # in_x1: [batch_size, x1_seq_len, hidden_size] - # in_x2: [batch_size, x2_seq_len, hidden_size] - # x1_len: [batch_size, x1_seq_len] - # x2_len: [batch_size, x2_seq_len] - - assert in_x1.size()[0] == in_x2.size()[0] - assert in_x1.size()[2] == in_x2.size()[2] - # The batch size and hidden size must be equal. - assert in_x1.size()[1] == x1_len.size()[1] and in_x2.size()[1] == x2_len.size()[1] - # The seq len in in_x and x_len must be equal. - assert in_x1.size()[0] == x1_len.size()[0] and x1_len.size()[0] == x2_len.size()[0] - - batch_size = in_x1.size()[0] - x1_max_len = in_x1.size()[1] - x2_max_len = in_x2.size()[1] - - in_x2_t = torch.transpose(in_x2, 1, 2) # [batch_size, hidden_size, x2_seq_len] - - attention_matrix = torch.bmm(in_x1, in_x2_t) # [batch_size, x1_seq_len, x2_seq_len] - - a_mask = x1_len.le(0.5).float() * -my_inf # [batch_size, x1_seq_len] - a_mask = a_mask.view(batch_size, x1_max_len, -1) - a_mask = a_mask.expand(-1, -1, x2_max_len) # [batch_size, x1_seq_len, x2_seq_len] - b_mask = x2_len.le(0.5).float() * -my_inf - b_mask = b_mask.view(batch_size, -1, x2_max_len) - b_mask = b_mask.expand(-1, x1_max_len, -1) # [batch_size, x1_seq_len, x2_seq_len] - - attention_a = F.softmax(attention_matrix + a_mask, dim=2) # [batch_size, x1_seq_len, x2_seq_len] - attention_b = F.softmax(attention_matrix + b_mask, dim=1) # [batch_size, x1_seq_len, x2_seq_len] - - out_x1 = torch.bmm(attention_a, in_x2) # [batch_size, x1_seq_len, hidden_size] - attention_b_t = torch.transpose(attention_b, 1, 2) - out_x2 = torch.bmm(attention_b_t, in_x1) # [batch_size, x2_seq_len, hidden_size] - - return out_x1, out_x2 + return {'pred': prediction} - @staticmethod - def mean_pooling(tensor, mask, dim=0): - masks = mask.view(mask.size(0), mask.size(1), -1).float() - return torch.sum(tensor * masks, dim=dim) / torch.sum(masks, dim=1) + def predict(self, premise, hypothesis, premise_len, hypothesis_len): + return self.forward(premise, hypothesis, premise_len, hypothesis_len) - @staticmethod - def max_pooling(tensor, mask, dim=0): - masks = mask.view(mask.size(0), mask.size(1), -1) - masks = masks.expand(-1, -1, tensor.size(2)).float() - return torch.max(tensor + masks.le(0.5).float() * -my_inf, dim=dim) diff --git a/fastNLP/modules/__init__.py b/fastNLP/modules/__init__.py index f0f0404a..37223394 100644 --- a/fastNLP/modules/__init__.py +++ b/fastNLP/modules/__init__.py @@ -3,12 +3,11 @@ from . import decoder from . import encoder from .aggregator import * from .decoder import * -from .encoder import * from .dropout import TimestepDropout +from .encoder import * __version__ = '0.0.0' __all__ = ['encoder', 'decoder', - 'aggregator', - 'TimestepDropout'] + 'aggregator'] diff --git a/fastNLP/modules/aggregator/__init__.py b/fastNLP/modules/aggregator/__init__.py index dbc36abc..2fabb89e 100644 --- a/fastNLP/modules/aggregator/__init__.py +++ b/fastNLP/modules/aggregator/__init__.py @@ -1,7 +1,10 @@ from .max_pool import MaxPool +from .max_pool import MaxPoolWithMask from .avg_pool import AvgPool +from .avg_pool import MeanPoolWithMask from .kmax_pool import KMaxPool from .attention import Attention +from .attention import Bi_Attention from .self_attention import SelfAttention diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index edd82f31..3fea1b10 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -1,6 +1,7 @@ import math import torch +import torch.nn.functional as F from torch import nn from fastNLP.modules.utils import mask_softmax @@ -62,3 +63,46 @@ class MultiHeadAtte(nn.Module): heads.append(headi) output = torch.cat(heads, dim=2) return self.out_linear(output) + + +class Bi_Attention(nn.Module): + def __init__(self): + super(Bi_Attention, self).__init__() + self.inf = 10e12 + + def forward(self, in_x1, in_x2, x1_len, x2_len): + # in_x1: [batch_size, x1_seq_len, hidden_size] + # in_x2: [batch_size, x2_seq_len, hidden_size] + # x1_len: [batch_size, x1_seq_len] + # x2_len: [batch_size, x2_seq_len] + + assert in_x1.size()[0] == in_x2.size()[0] + assert in_x1.size()[2] == in_x2.size()[2] + # The batch size and hidden size must be equal. + assert in_x1.size()[1] == x1_len.size()[1] and in_x2.size()[1] == x2_len.size()[1] + # The seq len in in_x and x_len must be equal. + assert in_x1.size()[0] == x1_len.size()[0] and x1_len.size()[0] == x2_len.size()[0] + + batch_size = in_x1.size()[0] + x1_max_len = in_x1.size()[1] + x2_max_len = in_x2.size()[1] + + in_x2_t = torch.transpose(in_x2, 1, 2) # [batch_size, hidden_size, x2_seq_len] + + attention_matrix = torch.bmm(in_x1, in_x2_t) # [batch_size, x1_seq_len, x2_seq_len] + + a_mask = x1_len.le(0.5).float() * -self.inf # [batch_size, x1_seq_len] + a_mask = a_mask.view(batch_size, x1_max_len, -1) + a_mask = a_mask.expand(-1, -1, x2_max_len) # [batch_size, x1_seq_len, x2_seq_len] + b_mask = x2_len.le(0.5).float() * -self.inf + b_mask = b_mask.view(batch_size, -1, x2_max_len) + b_mask = b_mask.expand(-1, x1_max_len, -1) # [batch_size, x1_seq_len, x2_seq_len] + + attention_a = F.softmax(attention_matrix + a_mask, dim=2) # [batch_size, x1_seq_len, x2_seq_len] + attention_b = F.softmax(attention_matrix + b_mask, dim=1) # [batch_size, x1_seq_len, x2_seq_len] + + out_x1 = torch.bmm(attention_a, in_x2) # [batch_size, x1_seq_len, hidden_size] + attention_b_t = torch.transpose(attention_b, 1, 2) + out_x2 = torch.bmm(attention_b_t, in_x1) # [batch_size, x2_seq_len, hidden_size] + + return out_x1, out_x2 diff --git a/fastNLP/modules/aggregator/avg_pool.py b/fastNLP/modules/aggregator/avg_pool.py index 70e473df..e6f3fd4b 100644 --- a/fastNLP/modules/aggregator/avg_pool.py +++ b/fastNLP/modules/aggregator/avg_pool.py @@ -1,6 +1,7 @@ # python: 3.6 # encoding: utf-8 +import torch import torch.nn as nn import torch.nn.functional as F @@ -22,3 +23,14 @@ class AvgPool(nn.Module): stride=self.stride, padding=self.padding) return x.squeeze(dim=-1) + + +class MeanPoolWithMask(nn.Module): + def __init__(self): + super(MeanPoolWithMask, self).__init__() + self.inf = 10e12 + + def forward(self, tensor, mask, dim=0): + masks = mask.view(mask.size(0), mask.size(1), -1).float() + return torch.sum(tensor * masks, dim=dim) / torch.sum(masks, dim=1) + diff --git a/fastNLP/modules/aggregator/max_pool.py b/fastNLP/modules/aggregator/max_pool.py index 2e47a4f5..60d68497 100644 --- a/fastNLP/modules/aggregator/max_pool.py +++ b/fastNLP/modules/aggregator/max_pool.py @@ -25,3 +25,14 @@ class MaxPool(nn.Module): padding=self.padding, dilation=self.dilation) return x.squeeze(dim=-1) # [N,C,1] -> [N,C] + + +class MaxPoolWithMask(nn.Module): + def __init__(self): + super(MaxPoolWithMask, self).__init__() + self.inf = 10e12 + + def forward(self, tensor, mask, dim=0): + masks = mask.view(mask.size(0), mask.size(1), -1) + masks = masks.expand(-1, -1, tensor.size(2)).float() + return torch.max(tensor + masks.le(0.5).float() * -self.inf, dim=dim) diff --git a/fastNLP/modules/decoder/CRF.py b/fastNLP/modules/decoder/CRF.py index 9973275b..d7db3bf9 100644 --- a/fastNLP/modules/decoder/CRF.py +++ b/fastNLP/modules/decoder/CRF.py @@ -15,33 +15,154 @@ def seq_len_to_byte_mask(seq_lens): # return value: ByteTensor, batch_size x max_len batch_size = seq_lens.size(0) max_len = seq_lens.max() - broadcast_arange = torch.arange(max_len).view(1, -1).repeat(batch_size, 1) - mask = broadcast_arange.lt(seq_lens.float().view(-1, 1)) + broadcast_arange = torch.arange(max_len).view(1, -1).repeat(batch_size, 1).to(seq_lens.device) + mask = broadcast_arange.float().lt(seq_lens.float().view(-1, 1)) return mask +def allowed_transitions(id2label, encoding_type='bio'): + """ + + :param dict id2label: key是label的indices，value是str类型的tag或tag-label。value可以是只有tag的, 比如"B", "M"; 也可以是 + "B-NN", "M-NN", tag和label之间一定要用"-"隔开。一般可以通过Vocabulary.get_id2word()id2label。 + :param encoding_type: str, 支持"bio", "bmes"。 + :return: List[Tuple(int, int)]], 内部的Tuple是(from_tag_id, to_tag_id)。 返回的结果考虑了start和end，比如"BIO"中，B、O可以 + 位于序列的开端，而I不行。所以返回的结果中会包含(start_idx, B_idx), (start_idx, O_idx), 但是不包含(start_idx, I_idx). + start_idx=len(id2label), end_idx=len(id2label)+1。 + """ + num_tags = len(id2label) + start_idx = num_tags + end_idx = num_tags + 1 + encoding_type = encoding_type.lower() + allowed_trans = [] + id_label_lst = list(id2label.items()) + [(start_idx, 'start'), (end_idx, 'end')] + def split_tag_label(from_label): + from_label = from_label.lower() + if from_label in ['start', 'end']: + from_tag = from_label + from_label = '' + else: + from_tag = from_label[:1] + from_label = from_label[2:] + return from_tag, from_label + + for from_id, from_label in id_label_lst: + if from_label in ['', '']: + continue + from_tag, from_label = split_tag_label(from_label) + for to_id, to_label in id_label_lst: + if to_label in ['', '']: + continue + to_tag, to_label = split_tag_label(to_label) + if is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label): + allowed_trans.append((from_id, to_id)) + return allowed_trans + + +def is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label): + """ + + :param encoding_type: str, 支持"BIO", "BMES"。 + :param from_tag: str, 比如"B", "M"之类的标注tag. 还包括start, end等两种特殊tag + :param from_label: str, 比如"PER", "LOC"等label + :param to_tag: str, 比如"B", "M"之类的标注tag. 还包括start, end等两种特殊tag + :param to_label: str, 比如"PER", "LOC"等label + :return: bool，能否跃迁 + """ + if to_tag=='start' or from_tag=='end': + return False + encoding_type = encoding_type.lower() + if encoding_type == 'bio': + """ + 第一行是to_tag, 第一列是from_tag. y任意条件下可转，-只有在label相同时可转，n不可转 + +-------+---+---+---+-------+-----+ + | | B | I | O | start | end | + +-------+---+---+---+-------+-----+ + | B | y | - | y | n | y | + +-------+---+---+---+-------+-----+ + | I | y | - | y | n | y | + +-------+---+---+---+-------+-----+ + | O | y | n | y | n | y | + +-------+---+---+---+-------+-----+ + | start | y | n | y | n | n | + +-------+---+---+---+-------+-----+ + | end | n | n | n | n | n | + +-------+---+---+---+-------+-----+ + """ + if from_tag == 'start': + return to_tag in ('b', 'o') + elif from_tag in ['b', 'i']: + return any([to_tag in ['end', 'b', 'o'], to_tag=='i' and from_label==to_label]) + elif from_tag == 'o': + return to_tag in ['end', 'b', 'o'] + else: + raise ValueError("Unexpect tag {}. Expect only 'B', 'I', 'O'.".format(from_tag)) + + elif encoding_type == 'bmes': + """ + 第一行是to_tag, 第一列是from_tag，y任意条件下可转，-只有在label相同时可转，n不可转 + +-------+---+---+---+---+-------+-----+ + | | B | M | E | S | start | end | + +-------+---+---+---+---+-------+-----+ + | B | n | - | - | n | n | n | + +-------+---+---+---+---+-------+-----+ + | M | n | - | - | n | n | n | + +-------+---+---+---+---+-------+-----+ + | E | y | n | n | y | n | y | + +-------+---+---+---+---+-------+-----+ + | S | y | n | n | y | n | y | + +-------+---+---+---+---+-------+-----+ + | start | y | n | n | y | n | n | + +-------+---+---+---+---+-------+-----+ + | end | n | n | n | n | n | n | + +-------+---+---+---+---+-------+-----+ + """ + if from_tag == 'start': + return to_tag in ['b', 's'] + elif from_tag == 'b': + return to_tag in ['m', 'e'] and from_label==to_label + elif from_tag == 'm': + return to_tag in ['m', 'e'] and from_label==to_label + elif from_tag in ['e', 's']: + return to_tag in ['b', 's', 'end'] + else: + raise ValueError("Unexpect tag type {}. Expect only 'B', 'M', 'E', 'S'.".format(from_tag)) + else: + raise ValueError("Only support BIO, BMES encoding type, got {}.".format(encoding_type)) + + class ConditionalRandomField(nn.Module): """ - :param int tag_size: num of tags - :param bool include_start_end_trans: whether to include start/end tag - :param str initial_method: method for initialization + + :param int num_tags: 标签的数量。 + :param bool include_start_end_trans: 是否包含起始tag + :param list allowed_transitions: ``List[Tuple[from_tag_id(int), to_tag_id(int)]]``. 允许的跃迁，可以通过allowed_transitions()得到。 + 如果为None，则所有跃迁均为合法 + :param str initial_method: """ - def __init__(self, tag_size, include_start_end_trans=False, initial_method=None): + def __init__(self, num_tags, include_start_end_trans=False, allowed_transitions=None, initial_method=None): super(ConditionalRandomField, self).__init__() self.include_start_end_trans = include_start_end_trans - self.tag_size = tag_size + self.num_tags = num_tags # the meaning of entry in this matrix is (from_tag_id, to_tag_id) score - self.trans_m = nn.Parameter(torch.randn(tag_size, tag_size)) + self.trans_m = nn.Parameter(torch.randn(num_tags, num_tags)) if self.include_start_end_trans: - self.start_scores = nn.Parameter(torch.randn(tag_size)) - self.end_scores = nn.Parameter(torch.randn(tag_size)) + self.start_scores = nn.Parameter(torch.randn(num_tags)) + self.end_scores = nn.Parameter(torch.randn(num_tags)) + + if allowed_transitions is None: + constrain = torch.zeros(num_tags + 2, num_tags + 2) + else: + constrain = torch.ones(num_tags + 2, num_tags + 2) * -1000 + for from_tag_id, to_tag_id in allowed_transitions: + constrain[from_tag_id, to_tag_id] = 0 + self._constrain = nn.Parameter(constrain, requires_grad=False) # self.reset_parameter() initial_parameter(self, initial_method) - def reset_parameter(self): nn.init.xavier_normal_(self.trans_m) if self.include_start_end_trans: @@ -52,9 +173,9 @@ class ConditionalRandomField(nn.Module): """Computes the (batch_size,) denominator term for the log-likelihood, which is the sum of the likelihoods across all possible state sequences. - :param FloatTensor logits: [max_len, batch_size, tag_size] - :param ByteTensor mask: [max_len, batch_size] - :return: FloatTensor, [batch_size,] + :param logits:FloatTensor, max_len x batch_size x num_tags + :param mask:ByteTensor, max_len x batch_size + :return:FloatTensor, batch_size """ seq_len, batch_size, n_tags = logits.size() alpha = logits[0] @@ -73,9 +194,9 @@ class ConditionalRandomField(nn.Module): return log_sum_exp(alpha, 1) def _glod_score(self, logits, tags, mask): - """Compute the score for the gold path. - - :param logits: FloatTensor, max_len x batch_size x tag_size + """ + Compute the score for the gold path. + :param logits: FloatTensor, max_len x batch_size x num_tags :param tags: LongTensor, max_len x batch_size :param mask: ByteTensor, max_len x batch_size :return:FloatTensor, batch_size @@ -100,12 +221,12 @@ class ConditionalRandomField(nn.Module): return score def forward(self, feats, tags, mask): - """Calculate the neg log likelihood - - :param FloatTensor feats: [batch_size, max_len, tag_size] - :param LongTensor tags: [batch_size, max_len] - :param ByteTensor mask: [batch_size, max_len] - :return: FloatTensor, [batch_size,] + """ + Calculate the neg log likelihood + :param feats:FloatTensor, batch_size x max_len x num_tags + :param tags:LongTensor, batch_size x max_len + :param mask:ByteTensor batch_size x max_len + :return:FloatTensor, batch_size """ feats = feats.transpose(0, 1) tags = tags.transpose(0, 1).long() @@ -115,13 +236,20 @@ class ConditionalRandomField(nn.Module): return all_path_score - gold_path_score - def viterbi_decode(self, data, mask, get_score=False): + def viterbi_decode(self, data, mask, get_score=False, unpad=False): """Given a feats matrix, return best decode path and best score. - :param FloatTensor data: [batch_size, max_len, tag_size] - :param ByteTensor mask: [batch_size, max_len] - :param bool get_score: whether to output the decode score. - :return: scores, paths + :param data:FloatTensor, batch_size x max_len x num_tags + :param mask:ByteTensor batch_size x max_len + :param get_score: bool, whether to output the decode score. + :param unpad: bool, 是否将结果unpad, + 如果False, 返回的是batch_size x max_len的tensor， + 如果True，返回的是List[List[int]], List[int]为每个sequence的label，已经unpadding了，即每个 + List[int]的长度是这个sample的有效长度 + :return: 如果get_score为False，返回结果根据unpadding变动 + 如果get_score为True, 返回 (paths, List[float], )。第一个仍然是解码后的路径(根据unpad变化)，第二个List[Float] + 为每个seqence的解码分数。 + """ batch_size, seq_len, n_tags = data.size() data = data.transpose(0, 1).data # L, B, H @@ -130,19 +258,23 @@ class ConditionalRandomField(nn.Module): # dp vpath = data.new_zeros((seq_len, batch_size, n_tags), dtype=torch.long) vscore = data[0] + transitions = self._constrain.data.clone() + transitions[:n_tags, :n_tags] += self.trans_m.data if self.include_start_end_trans: - vscore += self.start_scores.view(1, -1) + transitions[n_tags, :n_tags] += self.start_scores.data + transitions[:n_tags, n_tags+1] += self.end_scores.data + + vscore += transitions[n_tags, :n_tags] + trans_score = transitions[:n_tags, :n_tags].view(1, n_tags, n_tags).data for i in range(1, seq_len): prev_score = vscore.view(batch_size, n_tags, 1) cur_score = data[i].view(batch_size, 1, n_tags) - trans_score = self.trans_m.view(1, n_tags, n_tags).data score = prev_score + trans_score + cur_score best_score, best_dst = score.max(1) vpath[i] = best_dst vscore = best_score * mask[i].view(batch_size, 1) + vscore * (1 - mask[i]).view(batch_size, 1) - if self.include_start_end_trans: - vscore += self.end_scores.view(1, -1) + vscore += transitions[:n_tags, n_tags+1].view(1, -1) # backtrace batch_idx = torch.arange(batch_size, dtype=torch.long, device=data.device) @@ -157,7 +289,13 @@ class ConditionalRandomField(nn.Module): for i in range(seq_len - 1): last_tags = vpath[idxes[i], batch_idx, last_tags] ans[idxes[i+1], batch_idx] = last_tags - + ans = ans.transpose(0, 1) + if unpad: + paths = [] + for idx, seq_len in enumerate(lens): + paths.append(ans[idx, :seq_len+1].tolist()) + else: + paths = ans if get_score: - return ans_score, ans.transpose(0, 1) - return ans.transpose(0, 1) + return paths, ans_score.tolist() + return paths diff --git a/fastNLP/modules/decoder/MLP.py b/fastNLP/modules/decoder/MLP.py index 2b518a12..c9198859 100644 --- a/fastNLP/modules/decoder/MLP.py +++ b/fastNLP/modules/decoder/MLP.py @@ -35,7 +35,7 @@ class MLP(nn.Module): } if activation in actives: self.hidden_active = actives[activation] - elif isinstance(activation, callable): + elif callable(activation): self.hidden_active = activation else: raise ValueError("should set activation correctly: {}".format(activation)) diff --git a/fastNLP/modules/decoder/__init__.py b/fastNLP/modules/decoder/__init__.py index 7b8b2814..a72b7cd0 100644 --- a/fastNLP/modules/decoder/__init__.py +++ b/fastNLP/modules/decoder/__init__.py @@ -1,4 +1,2 @@ from .CRF import ConditionalRandomField from .MLP import MLP - -__all__ = ["ConditionalRandomField", "MLP"] diff --git a/fastNLP/modules/encoder/variational_rnn.py b/fastNLP/modules/encoder/variational_rnn.py index 6c1b0d46..a7902813 100644 --- a/fastNLP/modules/encoder/variational_rnn.py +++ b/fastNLP/modules/encoder/variational_rnn.py @@ -1,7 +1,6 @@ import torch import torch.nn as nn -from torch.nn.utils.rnn import PackedSequence - +from torch.nn.utils.rnn import PackedSequence, pack_padded_sequence, pad_packed_sequence from fastNLP.modules.utils import initial_parameter try: @@ -25,30 +24,63 @@ class VarRnnCellWrapper(nn.Module): self.input_p = input_p self.hidden_p = hidden_p - def forward(self, input, hidden, mask_x=None, mask_h=None): + def forward(self, input_x, hidden, mask_x, mask_h, is_reversed=False): """ - :param input: [seq_len, batch_size, input_size] + :param PackedSequence input_x: [seq_len, batch_size, input_size] :param hidden: for LSTM, tuple of (h_0, c_0), [batch_size, hidden_size] for other RNN, h_0, [batch_size, hidden_size] :param mask_x: [batch_size, input_size] dropout mask for input :param mask_h: [batch_size, hidden_size] dropout mask for hidden - :return: (output, hidden) - **output**: [seq_len, bacth_size, hidden_size]. - **hidden**: for LSTM, tuple of (h_n, c_n), [batch_size, hidden_size]; For other RNN, h_n, [batch_size, hidden_size]. + :return PackedSequence output: [seq_len, bacth_size, hidden_size] + hidden: for LSTM, tuple of (h_n, c_n), [batch_size, hidden_size] + for other RNN, h_n, [batch_size, hidden_size] """ + def get_hi(hi, h0, size): + h0_size = size - hi.size(0) + if h0_size > 0: + return torch.cat([hi, h0[:h0_size]], dim=0) + return hi[:size] is_lstm = isinstance(hidden, tuple) - input = input * mask_x.unsqueeze(0) if mask_x is not None else input - output_list = [] - for x in input: + input, batch_sizes = input_x + output = [] + cell = self.cell + if is_reversed: + batch_iter = flip(batch_sizes, [0]) + idx = input.size(0) + else: + batch_iter = batch_sizes + idx = 0 + + if is_lstm: + hn = (hidden[0].clone(), hidden[1].clone()) + else: + hn = hidden.clone() + hi = hidden + for size in batch_iter: + if is_reversed: + input_i = input[idx-size: idx] * mask_x[:size] + idx -= size + else: + input_i = input[idx: idx+size] * mask_x[:size] + idx += size + mask_hi = mask_h[:size] if is_lstm: - hx, cx = hidden - hidden = (hx * mask_h, cx) if mask_h is not None else (hx, cx) + hx, cx = hi + hi = (get_hi(hx, hidden[0], size) * mask_hi, get_hi(cx, hidden[1], size)) + hi = cell(input_i, hi) + hn[0][:size] = hi[0] + hn[1][:size] = hi[1] + output.append(hi[0]) else: - hidden *= mask_h if mask_h is not None else hidden - hidden = self.cell(x, hidden) - output_list.append(hidden[0] if is_lstm else hidden) - output = torch.stack(output_list, dim=0) - return output, hidden + hi = get_hi(hi, hidden, size) * mask_hi + hi = cell(input_i, hi) + hn[:size] = hi + output.append(hi) + + if is_reversed: + output = list(reversed(output)) + output = torch.cat(output, dim=0) + return PackedSequence(output, batch_sizes), hn class VarRNNBase(nn.Module): @@ -78,60 +110,67 @@ class VarRNNBase(nn.Module): cell = Cell(input_size, self.hidden_size, bias) self._all_cells.append(VarRnnCellWrapper(cell, self.hidden_size, input_dropout, hidden_dropout)) initial_parameter(self) + self.is_lstm = (self.mode == "LSTM") + + def _forward_one(self, n_layer, n_direction, input, hx, mask_x, mask_h): + is_lstm = self.is_lstm + idx = self.num_directions * n_layer + n_direction + cell = self._all_cells[idx] + hi = (hx[0][idx], hx[1][idx]) if is_lstm else hx[idx] + output_x, hidden_x = cell(input, hi, mask_x, mask_h, is_reversed=(n_direction == 1)) + return output_x, hidden_x def forward(self, input, hx=None): + is_lstm = self.is_lstm is_packed = isinstance(input, PackedSequence) - is_lstm = (self.mode == "LSTM") - if is_packed: - input, batch_sizes = input - max_batch_size = int(batch_sizes[0]) - else: - batch_sizes = None + if not is_packed: + seq_len = input.size(1) if self.batch_first else input.size(0) max_batch_size = input.size(0) if self.batch_first else input.size(1) + seq_lens = torch.LongTensor([seq_len for _ in range(max_batch_size)]) + input, batch_sizes = pack_padded_sequence(input, seq_lens, batch_first=self.batch_first) + else: + max_batch_size = int(input.batch_sizes[0]) + input, batch_sizes = input if hx is None: hx = input.new_zeros(self.num_layers * self.num_directions, - max_batch_size, self.hidden_size, - requires_grad=False) + max_batch_size, self.hidden_size, requires_grad=True) if is_lstm: - hx = (hx, hx) - - if self.batch_first: - input = input.transpose(0, 1) - batch_size = input.shape[1] + hx = (hx, hx.new_zeros(hx.size(), requires_grad=True)) - mask_x = input.new_ones((batch_size, self.input_size)) - mask_out = input.new_ones((batch_size, self.hidden_size * self.num_directions)) - mask_h_ones = input.new_ones((batch_size, self.hidden_size)) + mask_x = input.new_ones((max_batch_size, self.input_size)) + mask_out = input.new_ones((max_batch_size, self.hidden_size * self.num_directions)) + mask_h_ones = input.new_ones((max_batch_size, self.hidden_size)) nn.functional.dropout(mask_x, p=self.input_dropout, training=self.training, inplace=True) nn.functional.dropout(mask_out, p=self.hidden_dropout, training=self.training, inplace=True) - hidden_list = [] + hidden = input.new_zeros((self.num_layers*self.num_directions, max_batch_size, self.hidden_size)) + if is_lstm: + cellstate = input.new_zeros((self.num_layers*self.num_directions, max_batch_size, self.hidden_size)) for layer in range(self.num_layers): output_list = [] + input_seq = PackedSequence(input, batch_sizes) mask_h = nn.functional.dropout(mask_h_ones, p=self.hidden_dropout, training=self.training, inplace=False) for direction in range(self.num_directions): - input_x = input if direction == 0 else flip(input, [0]) + output_x, hidden_x = self._forward_one(layer, direction, input_seq, hx, + mask_x if layer == 0 else mask_out, mask_h) + output_list.append(output_x.data) idx = self.num_directions * layer + direction - cell = self._all_cells[idx] - hi = (hx[0][idx], hx[1][idx]) if is_lstm else hx[idx] - mask_xi = mask_x if layer == 0 else mask_out - output_x, hidden_x = cell(input_x, hi, mask_xi, mask_h) - output_list.append(output_x if direction == 0 else flip(output_x, [0])) - hidden_list.append(hidden_x) + if is_lstm: + hidden[idx] = hidden_x[0] + cellstate[idx] = hidden_x[1] + else: + hidden[idx] = hidden_x input = torch.cat(output_list, dim=-1) - output = input.transpose(0, 1) if self.batch_first else input if is_lstm: - h_list, c_list = zip(*hidden_list) - hn = torch.stack(h_list, dim=0) - cn = torch.stack(c_list, dim=0) - hidden = (hn, cn) - else: - hidden = torch.stack(hidden_list, dim=0) + hidden = (hidden, cellstate) if is_packed: - output = PackedSequence(output, batch_sizes) + output = PackedSequence(input, batch_sizes) + else: + input = PackedSequence(input, batch_sizes) + output, _ = pad_packed_sequence(input, batch_first=self.batch_first) return output, hidden @@ -158,3 +197,36 @@ class VarGRU(VarRNNBase): def __init__(self, *args, **kwargs): super(VarGRU, self).__init__(mode="GRU", Cell=nn.GRUCell, *args, **kwargs) + +# if __name__ == '__main__': +# x = torch.Tensor([[1,2,3], [4,5,0], [6,0,0]])[:,:,None] * 0.1 +# mask = (x != 0).float().view(3, -1) +# seq_lens = torch.LongTensor([3,2,1]) +# y = torch.Tensor([[0,1,1], [1,1,0], [0,0,0]]) +# # rev = _reverse_packed_sequence(pack) +# # # print(rev) +# lstm = VarLSTM(input_size=1, num_layers=2, hidden_size=2, +# batch_first=True, bidirectional=True, +# input_dropout=0.0, hidden_dropout=0.0,) +# # lstm = nn.LSTM(input_size=1, num_layers=2, hidden_size=2, +# # batch_first=True, bidirectional=True,) +# loss = nn.BCELoss() +# m = nn.Sigmoid() +# optim = torch.optim.SGD(lstm.parameters(), lr=1e-3) +# for i in range(2000): +# optim.zero_grad() +# pack = pack_padded_sequence(x, seq_lens, batch_first=True) +# out, hidden = lstm(pack) +# out, lens = pad_packed_sequence(out, batch_first=True) +# # print(lens) +# # print(out) +# # print(hidden[0]) +# # print(hidden[0].size()) +# # print(hidden[1]) +# out = out.sum(-1) +# out = m(out) * mask +# l = loss(out, y) +# l.backward() +# optim.step() +# if i % 50 == 0: +# print(out) diff --git a/reproduction/Biaffine_parser/cfg.cfg b/reproduction/Biaffine_parser/cfg.cfg index 8ee6f5fe..9b00c209 100644 --- a/reproduction/Biaffine_parser/cfg.cfg +++ b/reproduction/Biaffine_parser/cfg.cfg @@ -1,13 +1,8 @@ [train] -epochs = -1 -batch_size = 16 -pickle_path = "./save/" -validate = true -save_best_dev = true -eval_sort_key = "UAS" +n_epochs = 40 +batch_size = 32 use_cuda = true -model_saved_path = "./save/" -print_every_step = 20 +validate_every = 500 use_golden_train=true [test] @@ -32,9 +27,9 @@ arc_mlp_size = 500 label_mlp_size = 100 num_label = -1 dropout = 0.33 -use_var_lstm=false +use_var_lstm=true use_greedy_infer=false [optim] -lr = 2e-3 -weight_decay = 5e-5 +lr = 3e-4 +;weight_decay = 3e-5 diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index 0519201a..656da201 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -3,24 +3,26 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) +import fastNLP import torch -import re from fastNLP.core.trainer import Trainer -from fastNLP.core.metrics import Evaluator from fastNLP.core.instance import Instance +from fastNLP.api.pipeline import Pipeline +from fastNLP.models.biaffine_parser import BiaffineParser, ParserMetric, ParserLoss from fastNLP.core.vocabulary import Vocabulary from fastNLP.core.dataset import DataSet -from fastNLP.core.field import TextField, SeqLabelField from fastNLP.core.tester import Tester from fastNLP.io.config_io import ConfigLoader, ConfigSection -from fastNLP.io.model_io import ModelLoader, ModelSaver +from fastNLP.io.model_io import ModelLoader from fastNLP.io.embed_loader import EmbedLoader -from fastNLP.models.biaffine_parser import BiaffineParser +from fastNLP.io.model_io import ModelSaver +from reproduction.Biaffine_parser.util import ConllxDataLoader, MyDataloader +from fastNLP.api.processor import * BOS = '' EOS = '' -UNK = '' +UNK = '' NUM = '' ENG = '' @@ -28,85 +30,25 @@ ENG = '' if len(os.path.dirname(__file__)) != 0: os.chdir(os.path.dirname(__file__)) -class ConlluDataLoader(object): - def load(self, path): - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet(name='conll') - for sample in datalist: - # print(sample) - res = self.get_one(sample) - ds.append(Instance(word_seq=TextField(res[0], is_target=False), - pos_seq=TextField(res[1], is_target=False), - head_indices=SeqLabelField(res[2], is_target=True), - head_labels=TextField(res[3], is_target=True))) - - return ds - - def get_one(self, sample): - text = [] - pos_tags = [] - heads = [] - head_tags = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - continue - text.append(t1) - pos_tags.append(t2) - heads.append(int(t3)) - head_tags.append(t4) - return (text, pos_tags, heads, head_tags) - -class CTBDataLoader(object): - def load(self, data_path): - with open(data_path, "r", encoding="utf-8") as f: - lines = f.readlines() - data = self.parse(lines) - return self.convert(data) - - def parse(self, lines): - """ - [ - [word], [pos], [head_index], [head_tag] - ] - """ - sample = [] - data = [] - for i, line in enumerate(lines): - line = line.strip() - if len(line) == 0 or i+1 == len(lines): - data.append(list(map(list, zip(*sample)))) - sample = [] - else: - sample.append(line.split()) - return data - - def convert(self, data): - dataset = DataSet() - for sample in data: - word_seq = [BOS] + sample[0] + [EOS] - pos_seq = [BOS] + sample[1] + [EOS] - heads = [0] + list(map(int, sample[2])) + [0] - head_tags = [BOS] + sample[3] + [EOS] - dataset.append(Instance(word_seq=TextField(word_seq, is_target=False), - pos_seq=TextField(pos_seq, is_target=False), - gold_heads=SeqLabelField(heads, is_target=False), - head_indices=SeqLabelField(heads, is_target=True), - head_labels=TextField(head_tags, is_target=True))) - return dataset +def convert(data): + dataset = DataSet() + for sample in data: + word_seq = [BOS] + sample[0] + pos_seq = [BOS] + sample[1] + heads = [0] + list(map(int, sample[2])) + head_tags = [BOS] + sample[3] + dataset.append(Instance(words=word_seq, + pos=pos_seq, + gold_heads=heads, + arc_true=heads, + tags=head_tags)) + return dataset + + +def load(path): + data = ConllxDataLoader().load(path) + return convert(data) + # datadir = "/mnt/c/Me/Dev/release-2.2-st-train-dev-data/ud-treebanks-v2.2/UD_English-EWT" # datadir = "/home/yfshao/UD_English-EWT" @@ -115,26 +57,29 @@ class CTBDataLoader(object): # emb_file_name = '/home/yfshao/glove.6B.100d.txt' # loader = ConlluDataLoader() -datadir = '/home/yfshao/workdir/parser-data/' -train_data_name = "train_ctb5.txt" -dev_data_name = "dev_ctb5.txt" -test_data_name = "test_ctb5.txt" -emb_file_name = "/home/yfshao/workdir/parser-data/word_OOVthr_30_100v.txt" -# emb_file_name = "/home/yfshao/workdir/word_vector/cc.zh.300.vec" -loader = CTBDataLoader() +# datadir = '/home/yfshao/workdir/parser-data/' +# train_data_name = "train_ctb5.txt" +# dev_data_name = "dev_ctb5.txt" +# test_data_name = "test_ctb5.txt" + +datadir = "/home/yfshao/workdir/ctb7.0/" +train_data_name = "train.conllx" +dev_data_name = "dev.conllx" +test_data_name = "test.conllx" +# emb_file_name = "/home/yfshao/workdir/parser-data/word_OOVthr_30_100v.txt" +emb_file_name = "/home/yfshao/workdir/word_vector/cc.zh.300.vec" cfgfile = './cfg.cfg' processed_datadir = './save' # Config Loader train_args = ConfigSection() -test_args = ConfigSection() model_args = ConfigSection() optim_args = ConfigSection() -ConfigLoader.load_config(cfgfile, {"train": train_args, "test": test_args, "model": model_args, "optim": optim_args}) +ConfigLoader.load_config(cfgfile, {"train": train_args, "model": model_args, "optim": optim_args}) print('trainre Args:', train_args.data) -print('test Args:', test_args.data) -print('optim Args:', optim_args.data) +print('model Args:', model_args.data) +print('optim_args', optim_args.data) # Pickle Loader @@ -159,84 +104,36 @@ def load_data(dirpath): return datas def P2(data, field, length): - ds = [ins for ins in data if ins[field].get_length() >= length] + ds = [ins for ins in data if len(ins[field]) >= length] data.clear() data.extend(ds) return ds -def P1(data, field): - def reeng(w): - return w if w == BOS or w == EOS or re.search(r'^([a-zA-Z]+[\.\-]*)+$', w) is None else ENG - def renum(w): - return w if re.search(r'^[0-9]+\.?[0-9]*$', w) is None else NUM - for ins in data: - ori = ins[field].contents() - s = list(map(renum, map(reeng, ori))) - if s != ori: - # print(ori) - # print(s) - # print() - ins[field] = ins[field].new(s) - return data - -class ParserEvaluator(Evaluator): - def __init__(self, ignore_label): - super(ParserEvaluator, self).__init__() - self.ignore = ignore_label - - def __call__(self, predict_list, truth_list): - head_all, label_all, total_all = 0, 0, 0 - for pred, truth in zip(predict_list, truth_list): - head, label, total = self.evaluate(**pred, **truth) - head_all += head - label_all += label - total_all += total - - return {'UAS': head_all*1.0 / total_all, 'LAS': label_all*1.0 / total_all} - - def evaluate(self, head_pred, label_pred, head_indices, head_labels, seq_mask, **_): - """ - Evaluate the performance of prediction. - - :return : performance results. - head_pred_corrct: number of correct predicted heads. - label_pred_correct: number of correct predicted labels. - total_tokens: number of predicted tokens - """ - seq_mask *= (head_labels != self.ignore).long() - head_pred_correct = (head_pred == head_indices).long() * seq_mask - _, label_preds = torch.max(label_pred, dim=2) - label_pred_correct = (label_preds == head_labels).long() * head_pred_correct - return head_pred_correct.sum().item(), label_pred_correct.sum().item(), seq_mask.sum().item() - -try: - data_dict = load_data(processed_datadir) - word_v = data_dict['word_v'] - pos_v = data_dict['pos_v'] - tag_v = data_dict['tag_v'] - train_data = data_dict['train_data'] - dev_data = data_dict['dev_data'] - test_data = data_dict['test_data'] - print('use saved pickles') - -except Exception as _: - print('load raw data and preprocess') - # use pretrain embedding - word_v = Vocabulary(need_default=True, min_freq=2) - word_v.unknown_label = UNK - pos_v = Vocabulary(need_default=True) - tag_v = Vocabulary(need_default=False) - train_data = loader.load(os.path.join(datadir, train_data_name)) - dev_data = loader.load(os.path.join(datadir, dev_data_name)) - test_data = loader.load(os.path.join(datadir, test_data_name)) - train_data.update_vocab(word_seq=word_v, pos_seq=pos_v, head_labels=tag_v) - datasets = (train_data, dev_data, test_data) - save_data(processed_datadir, word_v=word_v, pos_v=pos_v, tag_v=tag_v, train_data=train_data, dev_data=dev_data, test_data=test_data) - -embed, _ = EmbedLoader.load_embedding(model_args['word_emb_dim'], emb_file_name, 'glove', word_v, os.path.join(processed_datadir, 'word_emb.pkl')) - -print(len(word_v)) -print(embed.size()) +def update_v(vocab, data, field): + data.apply(lambda x: vocab.add_word_lst(x[field]), new_field_name=None) + + +print('load raw data and preprocess') +# use pretrain embedding +word_v = Vocabulary() +word_v.unknown_label = UNK +pos_v = Vocabulary() +tag_v = Vocabulary(unknown=None, padding=None) +train_data = load(os.path.join(datadir, train_data_name)) +dev_data = load(os.path.join(datadir, dev_data_name)) +test_data = load(os.path.join(datadir, test_data_name)) +print(train_data[0]) +num_p = Num2TagProcessor('words', 'words') +for ds in (train_data, dev_data, test_data): + num_p(ds) + +update_v(word_v, train_data, 'words') +update_v(pos_v, train_data, 'pos') +update_v(tag_v, train_data, 'tags') + +print('vocab build success {}, {}, {}'.format(len(word_v), len(pos_v), len(tag_v))) +# embed, _ = EmbedLoader.fast_load_embedding(model_args['word_emb_dim'], emb_file_name, word_v) +# print(embed.size()) # Model model_args['word_vocab_size'] = len(word_v) @@ -245,50 +142,49 @@ model_args['num_label'] = len(tag_v) model = BiaffineParser(**model_args.data) model.reset_parameters() -datasets = (train_data, dev_data, test_data) -for ds in datasets: - ds.index_field("word_seq", word_v).index_field("pos_seq", pos_v).index_field("head_labels", tag_v) - ds.set_origin_len('word_seq') + +word_idxp = IndexerProcessor(word_v, 'words', 'word_seq') +pos_idxp = IndexerProcessor(pos_v, 'pos', 'pos_seq') +tag_idxp = IndexerProcessor(tag_v, 'tags', 'label_true') +seq_p = SeqLenProcessor('word_seq', 'seq_lens') + +set_input_p = SetInputProcessor('word_seq', 'pos_seq', 'seq_lens', flag=True) +set_target_p = SetTargetProcessor('arc_true', 'label_true', 'seq_lens', flag=True) + +label_toword_p = Index2WordProcessor(vocab=tag_v, field_name='label_pred', new_added_field_name='label_pred_seq') + +for ds in (train_data, dev_data, test_data): + word_idxp(ds) + pos_idxp(ds) + tag_idxp(ds) + seq_p(ds) + set_input_p(ds) + set_target_p(ds) + if train_args['use_golden_train']: - train_data.set_target(gold_heads=False) -else: - train_data.set_target(gold_heads=None) + train_data.set_input('gold_heads', flag=True) train_args.data.pop('use_golden_train') -ignore_label = pos_v['P'] +ignore_label = pos_v['punct'] print(test_data[0]) -print(len(train_data)) -print(len(dev_data)) -print(len(test_data)) +print('train len {}'.format(len(train_data))) +print('dev len {}'.format(len(dev_data))) +print('test len {}'.format(len(test_data))) def train(path): + # test saving pipeline + save_pipe(path) + # Trainer - trainer = Trainer(**train_args.data) - - def _define_optim(obj): - lr = optim_args.data['lr'] - embed_params = set(obj._model.word_embedding.parameters()) - decay_params = set(obj._model.arc_predictor.parameters()) | set(obj._model.label_predictor.parameters()) - params = [p for p in obj._model.parameters() if p not in decay_params and p not in embed_params] - obj._optimizer = torch.optim.Adam([ - {'params': list(embed_params), 'lr':lr*0.1}, - {'params': list(decay_params), **optim_args.data}, - {'params': params} - ], lr=lr, betas=(0.9, 0.9)) - obj._scheduler = torch.optim.lr_scheduler.LambdaLR(obj._optimizer, lambda ep: max(.75 ** (ep / 5e4), 0.05)) - - def _update(obj): - # torch.nn.utils.clip_grad_norm_(obj._model.parameters(), 5.0) - obj._scheduler.step() - obj._optimizer.step() - - trainer.define_optimizer = lambda: _define_optim(trainer) - trainer.update = lambda: _update(trainer) - trainer.set_validator(Tester(**test_args.data, evaluator=ParserEvaluator(ignore_label))) - - model.word_embedding = torch.nn.Embedding.from_pretrained(embed, freeze=False) + trainer = Trainer(model=model, train_data=train_data, dev_data=dev_data, + loss=ParserLoss(), metrics=ParserMetric(), metric_key='UAS', + **train_args.data, + optimizer=fastNLP.Adam(**optim_args.data), + save_path=path) + + # model.word_embedding = torch.nn.Embedding.from_pretrained(embed, freeze=False) model.word_embedding.padding_idx = word_v.padding_idx model.word_embedding.weight.data[word_v.padding_idx].fill_(0) model.pos_embedding.padding_idx = pos_v.padding_idx @@ -302,18 +198,23 @@ def train(path): # pass # Start training - trainer.train(model, train_data, dev_data) + trainer.train() print("Training finished!") - # Saver - saver = ModelSaver("./save/saved_model.pkl") - saver.save_pytorch(model) - print("Model saved!") + # save pipeline + save_pipe(path) + print('pipe saved') + +def save_pipe(path): + pipe = Pipeline(processors=[num_p, word_idxp, pos_idxp, seq_p, set_input_p]) + pipe.add_processor(ModelProcessor(model=model, batch_size=32)) + pipe.add_processor(label_toword_p) + torch.save(pipe, os.path.join(path, 'pipe.pkl')) def test(path): # Tester - tester = Tester(**test_args.data, evaluator=ParserEvaluator(ignore_label)) + tester = Tester(**test_args.data) # Model model = BiaffineParser(**model_args.data) @@ -333,13 +234,18 @@ def test(path): print("Testing Test data") tester.test(model, test_data) +def build_pipe(parser_pipe_path): + parser_pipe = torch.load(parser_pipe_path) + + if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description='Run a chinese word segmentation model') - parser.add_argument('--mode', help='set the model\'s model', choices=['train', 'test', 'infer']) + parser.add_argument('--mode', help='set the model\'s model', choices=['train', 'test', 'infer', 'save']) parser.add_argument('--path', type=str, default='') + # parser.add_argument('--dst', type=str, default='') args = parser.parse_args() if args.mode == 'train': train(args.path) @@ -347,6 +253,12 @@ if __name__ == "__main__": test(args.path) elif args.mode == 'infer': pass + # elif args.mode == 'save': + # print(f'save model from {args.path} to {args.dst}') + # save_model(args.path, args.dst) + # load_path = os.path.dirname(args.dst) + # print(f'save pipeline in {load_path}') + # build(load_path) else: print('no mode specified for model!') parser.print_help() diff --git a/reproduction/chinese_word_segment/cws_io/cws_reader.py b/reproduction/chinese_word_segment/cws_io/cws_reader.py index 56a73351..34bcf7dd 100644 --- a/reproduction/chinese_word_segment/cws_io/cws_reader.py +++ b/reproduction/chinese_word_segment/cws_io/cws_reader.py @@ -6,6 +6,13 @@ from fastNLP.io.dataset_loader import DataSetLoader def cut_long_sentence(sent, max_sample_length=200): + """ + 将长于max_sample_length的sentence截成多段，只会在有空格的地方发生截断。所以截取的句子可能长于或者短于max_sample_length + + :param sent: str. + :param max_sample_length: int. + :return: list of str. + """ sent_no_space = sent.replace(' ', '') cutted_sentence = [] if len(sent_no_space) > max_sample_length: @@ -127,12 +134,26 @@ class POSCWSReader(DataSetLoader): return dataset -class ConlluCWSReader(object): - # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BMES的tag)。 +class ConllCWSReader(object): def __init__(self): pass def load(self, path, cut_long_sent=False): + """ + 返回的DataSet只包含raw_sentence这个field，内容为str。 + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + """ datalist = [] with open(path, 'r', encoding='utf-8') as f: sample = [] @@ -150,10 +171,10 @@ class ConlluCWSReader(object): ds = DataSet() for sample in datalist: # print(sample) - res = self.get_one(sample) + res = self.get_char_lst(sample) if res is None: continue - line = ' '.join(res) + line = ' '.join(res) if cut_long_sent: sents = cut_long_sentence(line) else: @@ -163,7 +184,7 @@ class ConlluCWSReader(object): return ds - def get_one(self, sample): + def get_char_lst(self, sample): if len(sample)==0: return None text = [] diff --git a/reproduction/chinese_word_segment/models/cws_model.py b/reproduction/chinese_word_segment/models/cws_model.py index 4f81fea3..c6cf6746 100644 --- a/reproduction/chinese_word_segment/models/cws_model.py +++ b/reproduction/chinese_word_segment/models/cws_model.py @@ -9,7 +9,7 @@ from reproduction.chinese_word_segment.utils import seq_lens_to_mask class CWSBiLSTMEncoder(BaseModel): def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, - hidden_size=200, bidirectional=True, embed_drop_p=None, num_layers=1): + hidden_size=200, bidirectional=True, embed_drop_p=0.2, num_layers=1): super().__init__() self.input_size = 0 @@ -65,9 +65,10 @@ class CWSBiLSTMEncoder(BaseModel): x_tensor = self.char_embedding(chars) - if not bigrams is None: + if hasattr(self, 'bigram_embedding'): bigram_tensor = self.bigram_embedding(bigrams).view(batch_size, max_len, -1) x_tensor = torch.cat([x_tensor, bigram_tensor], dim=2) + x_tensor = self.embedding_drop(x_tensor) sorted_lens, sorted_indices = torch.sort(seq_lens, descending=True) packed_x = nn.utils.rnn.pack_padded_sequence(x_tensor[sorted_indices], sorted_lens, batch_first=True) @@ -120,10 +121,24 @@ class CWSBiLSTMSegApp(BaseModel): from fastNLP.modules.decoder.CRF import ConditionalRandomField +from fastNLP.modules.decoder.CRF import allowed_transitions class CWSBiLSTMCRF(BaseModel): def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, - hidden_size=200, bidirectional=True, embed_drop_p=None, num_layers=1, tag_size=4): + hidden_size=200, bidirectional=True, embed_drop_p=0.2, num_layers=1, tag_size=4): + """ + 默认使用BMES的标注方式 + :param vocab_num: + :param embed_dim: + :param bigram_vocab_num: + :param bigram_embed_dim: + :param num_bigram_per_char: + :param hidden_size: + :param bidirectional: + :param embed_drop_p: + :param num_layers: + :param tag_size: + """ super(CWSBiLSTMCRF, self).__init__() self.tag_size = tag_size @@ -133,10 +148,12 @@ class CWSBiLSTMCRF(BaseModel): size_layer = [hidden_size, 200, tag_size] self.decoder_model = MLP(size_layer) - self.crf = ConditionalRandomField(tag_size=tag_size, include_start_end_trans=False) + allowed_trans = allowed_transitions({0:'b', 1:'m', 2:'e', 3:'s'}, encoding_type='bmes') + self.crf = ConditionalRandomField(num_tags=tag_size, include_start_end_trans=False, + allowed_transitions=allowed_trans) - def forward(self, chars, tags, seq_lens, bigrams=None): + def forward(self, chars, target, seq_lens, bigrams=None): device = self.parameters().__next__().device chars = chars.to(device).long() if not bigrams is None: @@ -147,7 +164,7 @@ class CWSBiLSTMCRF(BaseModel): masks = seq_lens_to_mask(seq_lens) feats = self.encoder_model(chars, bigrams, seq_lens) feats = self.decoder_model(feats) - losses = self.crf(feats, tags, masks) + losses = self.crf(feats, target, masks) pred_dict = {} pred_dict['seq_lens'] = seq_lens @@ -168,5 +185,5 @@ class CWSBiLSTMCRF(BaseModel): feats = self.decoder_model(feats) probs = self.crf.viterbi_decode(feats, masks, get_score=False) - return {'pred_tags': probs} + return {'pred': probs, 'seq_lens':seq_lens} diff --git a/reproduction/chinese_word_segment/process/cws_processor.py b/reproduction/chinese_word_segment/process/cws_processor.py index e7c069f1..9e57d35a 100644 --- a/reproduction/chinese_word_segment/process/cws_processor.py +++ b/reproduction/chinese_word_segment/process/cws_processor.py @@ -1,17 +1,18 @@ import re - -from fastNLP.core.field import SeqLabelField -from fastNLP.core.vocabulary import Vocabulary -from fastNLP.core.dataset import DataSet from fastNLP.api.processor import Processor +from fastNLP.core.dataset import DataSet +from fastNLP.core.vocabulary import Vocabulary from reproduction.chinese_word_segment.process.span_converter import SpanConverter _SPECIAL_TAG_PATTERN = '<[a-zA-Z]+>' class SpeicalSpanProcessor(Processor): - # 这个类会将句子中的special span转换为对应的内容。 + """ + 将DataSet中field_name使用span_converter替换掉。 + + """ def __init__(self, field_name, new_added_field_name=None): super(SpeicalSpanProcessor, self).__init__(field_name, new_added_field_name) @@ -20,11 +21,12 @@ class SpeicalSpanProcessor(Processor): def process(self, dataset): assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: + def inner_proc(ins): sentence = ins[self.field_name] for span_converter in self.span_converters: sentence = span_converter.find_certain_span_and_replace(sentence) - ins[self.new_added_field_name] = sentence + return sentence + dataset.apply(func=inner_proc, new_field_name=self.new_added_field_name) return dataset @@ -34,17 +36,22 @@ class SpeicalSpanProcessor(Processor): self.span_converters.append(converter) - class CWSCharSegProcessor(Processor): + """ + 将DataSet中field_name这个field分成一个个的汉字，即原来可能为"复旦大学 fudan", 分成['复', '旦', '大', '学', + ' ', 'f', 'u', ...] + + """ def __init__(self, field_name, new_added_field_name): super(CWSCharSegProcessor, self).__init__(field_name, new_added_field_name) def process(self, dataset): assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: + def inner_proc(ins): sentence = ins[self.field_name] chars = self._split_sent_into_chars(sentence) - ins[self.new_added_field_name] = chars + return chars + dataset.apply(func=inner_proc, new_field_name=self.new_added_field_name) return dataset @@ -73,6 +80,10 @@ class CWSCharSegProcessor(Processor): class CWSTagProcessor(Processor): + """ + 为分词生成tag。该class为Base class。 + + """ def __init__(self, field_name, new_added_field_name=None): super(CWSTagProcessor, self).__init__(field_name, new_added_field_name) @@ -107,18 +118,22 @@ class CWSTagProcessor(Processor): def process(self, dataset): assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: + def inner_proc(ins): sentence = ins[self.field_name] tag_list = self._generate_tag(sentence) - ins[self.new_added_field_name] = tag_list - dataset.set_target(**{self.new_added_field_name:True}) - dataset._set_need_tensor(**{self.new_added_field_name:True}) + return tag_list + dataset.apply(func=inner_proc, new_field_name=self.new_added_field_name) + dataset.set_target(self.new_added_field_name) return dataset def _tags_from_word_len(self, word_len): raise NotImplementedError class CWSBMESTagProcessor(CWSTagProcessor): + """ + 通过DataSet中的field_name这个field生成相应的BMES的tag。 + + """ def __init__(self, field_name, new_added_field_name=None): super(CWSBMESTagProcessor, self).__init__(field_name, new_added_field_name) @@ -137,6 +152,10 @@ class CWSBMESTagProcessor(CWSTagProcessor): return tag_list class CWSSegAppTagProcessor(CWSTagProcessor): + """ + 通过DataSet中的field_name这个field生成相应的SegApp的tag。 + + """ def __init__(self, field_name, new_added_field_name=None): super(CWSSegAppTagProcessor, self).__init__(field_name, new_added_field_name) @@ -151,6 +170,10 @@ class CWSSegAppTagProcessor(CWSTagProcessor): class BigramProcessor(Processor): + """ + 这是生成bigram的基类。 + + """ def __init__(self, field_name, new_added_fielf_name=None): super(BigramProcessor, self).__init__(field_name, new_added_fielf_name) @@ -158,22 +181,31 @@ class BigramProcessor(Processor): def process(self, dataset): assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: + def inner_proc(ins): characters = ins[self.field_name] bigrams = self._generate_bigram(characters) - ins[self.new_added_field_name] = bigrams + return bigrams + dataset.apply(func=inner_proc, new_field_name=self.new_added_field_name) return dataset - def _generate_bigram(self, characters): pass class Pre2Post2BigramProcessor(BigramProcessor): - def __init__(self, field_name, new_added_fielf_name=None): + """ + 该bigram processor生成bigram的方式如下 + 原汉字list为l = ['a', 'b', 'c']，会被padding为L=['SOS', 'SOS', 'a', 'b', 'c', 'EOS', 'EOS']，生成bigram list为 + [L[idx-2], L[idx-1], L[idx+1], L[idx+2], L[idx-2]L[idx], L[idx-1]L[idx], L[idx]L[idx+1], L[idx]L[idx+2], ....] + 即每个汉字，会有八个bigram, 对于上例中'a'的bigram为 + ['SOS', 'SOS', 'b', 'c', 'SOSa', 'SOSa', 'ab', 'ac'] + 返回的bigram是一个list，但其实每8个元素是一个汉字的bigram信息。 + + """ + def __init__(self, field_name, new_added_field_name=None): - super(BigramProcessor, self).__init__(field_name, new_added_fielf_name) + super(BigramProcessor, self).__init__(field_name, new_added_field_name) def _generate_bigram(self, characters): bigrams = [] @@ -197,20 +229,116 @@ class Pre2Post2BigramProcessor(BigramProcessor): # 这里需要建立vocabulary了，但是遇到了以下的问题 # (1) 如果使用Processor的方式的话，但是在这种情况返回的不是dataset。所以建立vocabulary的工作用另外的方式实现，不借用 # Processor了 +# TODO 如何将建立vocab和index这两步统一了？ + +class VocabIndexerProcessor(Processor): + """ + 根据DataSet创建Vocabulary，并将其用数字index。新生成的index的field会被放在new_added_filed_name, 如果没有提供 + new_added_field_name, 则覆盖原有的field_name. + + """ + def __init__(self, field_name, new_added_filed_name=None, min_freq=1, max_size=None, + verbose=0, is_input=True): + """ + + :param field_name: 从哪个field_name创建词表，以及对哪个field_name进行index操作 + :param new_added_filed_name: index时，生成的index field的名称，如果不传入，则覆盖field_name. + :param min_freq: 创建的Vocabulary允许的单词最少出现次数. + :param max_size: 创建的Vocabulary允许的最大的单词数量 + :param verbose: 0, 不输出任何信息；1，输出信息 + :param bool is_input: + """ + super(VocabIndexerProcessor, self).__init__(field_name, new_added_filed_name) + self.min_freq = min_freq + self.max_size = max_size + + self.verbose =verbose + self.is_input = is_input + + def construct_vocab(self, *datasets): + """ + 使用传入的DataSet创建vocabulary + + :param datasets: DataSet类型的数据，用于构建vocabulary + :return: + """ + self.vocab = Vocabulary(min_freq=self.min_freq, max_size=self.max_size) + for dataset in datasets: + assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) + dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) + self.vocab.build_vocab() + if self.verbose: + print("Vocabulary Constructed, has {} items.".format(len(self.vocab))) + + def process(self, *datasets, only_index_dataset=None): + """ + 若还未建立Vocabulary，则使用dataset中的DataSet建立vocabulary；若已经有了vocabulary则使用已有的vocabulary。得到vocabulary + 后，则会index datasets与only_index_dataset。 + + :param datasets: DataSet类型的数据 + :param only_index_dataset: DataSet, or list of DataSet. 该参数中的内容只会被用于index，不会被用于生成vocabulary。 + :return: + """ + if len(datasets)==0 and not hasattr(self,'vocab'): + raise RuntimeError("You have to construct vocabulary first. Or you have to pass datasets to construct it.") + if not hasattr(self, 'vocab'): + self.construct_vocab(*datasets) + else: + if self.verbose: + print("Using constructed vocabulary with {} items.".format(len(self.vocab))) + to_index_datasets = [] + if len(datasets)!=0: + for dataset in datasets: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + to_index_datasets.append(dataset) + + if not (only_index_dataset is None): + if isinstance(only_index_dataset, list): + for dataset in only_index_dataset: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + to_index_datasets.append(dataset) + elif isinstance(only_index_dataset, DataSet): + to_index_datasets.append(only_index_dataset) + else: + raise TypeError('Only DataSet or list of DataSet is allowed, not {}.'.format(type(only_index_dataset))) + + for dataset in to_index_datasets: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + dataset.apply(lambda ins: [self.vocab.to_index(token) for token in ins[self.field_name]], + new_field_name=self.new_added_field_name, is_input=self.is_input) + # 只返回一个，infer时为了跟其他processor保持一致 + if len(to_index_datasets) == 1: + return to_index_datasets[0] + + def set_vocab(self, vocab): + assert isinstance(vocab, Vocabulary), "Only fastNLP.core.Vocabulary is allowed, not {}.".format(type(vocab)) + self.vocab = vocab + + def delete_vocab(self): + del self.vocab + + def get_vocab_size(self): + return len(self.vocab) + + def set_verbose(self, verbose): + """ + 设置processor verbose状态。 + + :param verbose: int, 0，不输出任何信息；1，输出vocab 信息。 + :return: + """ + self.verbose = verbose class VocabProcessor(Processor): - def __init__(self, field_name, min_count=1, max_vocab_size=None): + def __init__(self, field_name, min_freq=1, max_size=None): super(VocabProcessor, self).__init__(field_name, None) - self.vocab = Vocabulary(min_freq=min_count, max_size=max_vocab_size) + self.vocab = Vocabulary(min_freq=min_freq, max_size=max_size) def process(self, *datasets): for dataset in datasets: assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: - tokens = ins[self.field_name] - self.vocab.update(tokens) - + dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) def get_vocab(self): self.vocab.build_vocab() @@ -220,19 +348,6 @@ class VocabProcessor(Processor): return len(self.vocab) -class SeqLenProcessor(Processor): - def __init__(self, field_name, new_added_field_name='seq_lens'): - - super(SeqLenProcessor, self).__init__(field_name, new_added_field_name) - - def process(self, dataset): - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: - length = len(ins[self.field_name]) - ins[self.new_added_field_name] = length - dataset._set_need_tensor(**{self.new_added_field_name:True}) - return dataset - class SegApp2OutputProcessor(Processor): def __init__(self, chars_field_name='chars_list', tag_field_name='pred_tags', new_added_field_name='output'): super(SegApp2OutputProcessor, self).__init__(None, None) @@ -258,7 +373,32 @@ class SegApp2OutputProcessor(Processor): class BMES2OutputProcessor(Processor): - def __init__(self, chars_field_name='chars_list', tag_field_name='pred_tags', new_added_field_name='output'): + """ + 按照BMES标注方式推测生成的tag。由于可能存在非法tag，比如"BS"，所以需要用以下的表格做转换，cur_B意思是当前tag是B， + next_B意思是后一个tag是B。则cur_B=S，即将当前被predict是B的tag标为S；next_M=B, 即将后一个被predict是M的tag标为B + | | next_B | next_M | next_E | next_S | end | + |:-----:|:-------:|:--------:|:--------:|:-------:|:-------:| + | start | 合法 | next_M=B | next_E=S | 合法 | - | + | cur_B | cur_B=S | 合法 | 合法 | cur_B=S | cur_B=S | + | cur_M | cur_M=E | 合法 | 合法 | cur_M=E | cur_M=E | + | cur_E | 合法 | next_M=B | next_E=S | 合法 | 合法 | + | cur_S | 合法 | next_M=B | next_E=S | 合法 | 合法 | + 举例： + prediction为BSEMS，会被认为是SSSSS. + + """ + def __init__(self, chars_field_name='chars_list', tag_field_name='pred', new_added_field_name='output', + b_idx = 0, m_idx = 1, e_idx = 2, s_idx = 3): + """ + + :param chars_field_name: character所对应的field + :param tag_field_name: 预测对应的field + :param new_added_field_name: 转换后的内容所在field + :param b_idx: int, Begin标签所对应的tag idx. + :param m_idx: int, Middle标签所对应的tag idx. + :param e_idx: int, End标签所对应的tag idx. + :param s_idx: int, Single标签所对应的tag idx + """ super(BMES2OutputProcessor, self).__init__(None, None) self.chars_field_name = chars_field_name @@ -266,19 +406,84 @@ class BMES2OutputProcessor(Processor): self.new_added_field_name = new_added_field_name + self.b_idx = b_idx + self.m_idx = m_idx + self.e_idx = e_idx + self.s_idx = s_idx + # 还原init处介绍的矩阵 + self._valida_matrix = { + -1: [(-1, -1), (1, self.b_idx), (1, self.s_idx), (-1, -1)], # magic start idx + self.b_idx:[(0, self.s_idx), (-1, -1), (-1, -1), (0, self.s_idx), (0, self.s_idx)], + self.m_idx:[(0, self.e_idx), (-1, -1), (-1, -1), (0, self.e_idx), (0, self.e_idx)], + self.e_idx:[(-1, -1), (1, self.b_idx), (1, self.s_idx), (-1, -1), (-1, -1)], + self.s_idx:[(-1, -1), (1, self.b_idx), (1, self.s_idx), (-1, -1), (-1, -1)], + } + + def _validate_tags(self, tags): + """ + 给定一个tag的List，返回合法tag + + :param tags: Tensor, shape: (seq_len, ) + :return: 返回修改为合法tag的list + """ + assert len(tags)!=0 + padded_tags = [-1, *tags, -1] + for idx in range(len(padded_tags)-1): + cur_tag = padded_tags[idx] + if cur_tag not in self._valida_matrix: + cur_tag = self.s_idx + if padded_tags[idx+1] not in self._valida_matrix: + padded_tags[idx+1] = self.s_idx + next_tag = padded_tags[idx+1] + shift_tag = self._valida_matrix[cur_tag][next_tag] + if shift_tag[0]!=-1: + padded_tags[idx+shift_tag[0]] = shift_tag[1] + + return padded_tags[1:-1] + def process(self, dataset): assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - for ins in dataset: + def inner_proc(ins): pred_tags = ins[self.tag_field_name] + pred_tags = self._validate_tags(pred_tags) chars = ins[self.chars_field_name] words = [] start_idx = 0 for idx, tag in enumerate(pred_tags): - if tag==3: - # 当前没有考虑将原文替换回去 + if tag==self.s_idx: words.extend(chars[start_idx:idx+1]) start_idx = idx + 1 - elif tag==2: + elif tag==self.e_idx: words.append(''.join(chars[start_idx:idx+1])) start_idx = idx + 1 - ins[self.new_added_field_name] = ' '.join(words) \ No newline at end of file + return ' '.join(words) + dataset.apply(func=inner_proc, new_field_name=self.new_added_field_name) + + +class InputTargetProcessor(Processor): + def __init__(self, input_fields, target_fields): + """ + 对DataSet操作，将input_fields中的field设置为input，target_fields的中field设置为target + + :param input_fields: List[str], 设置为input_field的field_name。如果为None，则不将任何field设置为target。 + :param target_fields: List[str], 设置为target_field的field_name。 如果为None，则不将任何field设置为target。 + """ + super(InputTargetProcessor, self).__init__(None, None) + + if input_fields is not None and not isinstance(input_fields, list): + raise TypeError("input_fields should be List[str], not {}.".format(type(input_fields))) + else: + self.input_fields = input_fields + if target_fields is not None and not isinstance(target_fields, list): + raise TypeError("target_fiels should be List[str], not{}.".format(type(target_fields))) + else: + self.target_fields = target_fields + + def process(self, dataset): + assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) + if self.input_fields is not None: + for field in self.input_fields: + dataset.set_input(field) + if self.target_fields is not None: + for field in self.target_fields: + dataset.set_target(field) \ No newline at end of file diff --git a/reproduction/pos_tag_model/process/pos_processor.py b/reproduction/pos_tag_model/pos_processor.py similarity index 99% rename from reproduction/pos_tag_model/process/pos_processor.py rename to reproduction/pos_tag_model/pos_processor.py index 5c03f9cd..7a1b8e01 100644 --- a/reproduction/pos_tag_model/process/pos_processor.py +++ b/reproduction/pos_tag_model/pos_processor.py @@ -4,6 +4,7 @@ from collections import Counter from fastNLP.api.processor import Processor from fastNLP.core.dataset import DataSet + class CombineWordAndPosProcessor(Processor): def __init__(self, word_field_name, pos_field_name): super(CombineWordAndPosProcessor, self).__init__(None, None) @@ -60,6 +61,7 @@ class CombineWordAndPosProcessor(Processor): return dataset + class PosOutputStrProcessor(Processor): def __init__(self, word_field_name, pos_field_name): super(PosOutputStrProcessor, self).__init__(None, None) diff --git a/reproduction/pos_tag_model/pos_io/pos_reader.py b/reproduction/pos_tag_model/pos_reader.py similarity index 53% rename from reproduction/pos_tag_model/pos_io/pos_reader.py rename to reproduction/pos_tag_model/pos_reader.py index 2ff07815..c0a8c4cd 100644 --- a/reproduction/pos_tag_model/pos_io/pos_reader.py +++ b/reproduction/pos_tag_model/pos_reader.py @@ -24,8 +24,8 @@ def cut_long_sentence(sent, max_sample_length=200): return cutted_sentence -class ConlluPOSReader(object): - # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BMES的tag)。 +class ConllPOSReader(object): + # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BIO的tag)。 def __init__(self): pass @@ -70,6 +70,70 @@ class ConlluPOSReader(object): return ds + + +class ZhConllPOSReader(object): + # 中文colln格式reader + def __init__(self): + pass + + def load(self, path): + """ + 返回的DataSet, 包含以下的field + words：list of str, + tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_one(sample) + if res is None: + continue + char_seq = [] + pos_seq = [] + for word, tag in zip(res[0], res[1]): + char_seq.extend(list(word)) + if len(word)==1: + pos_seq.append('S-{}'.format(tag)) + elif len(word)>1: + pos_seq.append('B-{}'.format(tag)) + for _ in range(len(word)-2): + pos_seq.append('M-{}'.format(tag)) + pos_seq.append('E-{}'.format(tag)) + else: + raise ValueError("Zero length of word detected.") + + ds.append(Instance(words=char_seq, + tag=pos_seq)) + + return ds + def get_one(self, sample): if len(sample)==0: return None @@ -84,6 +148,6 @@ class ConlluPOSReader(object): return text, pos_tags if __name__ == '__main__': - reader = ConlluPOSReader() + reader = ZhConllPOSReader() d = reader.load('/home/hyan/train.conllx') - print('reader') \ No newline at end of file + print(d) \ No newline at end of file diff --git a/reproduction/pos_tag_model/pos_tag.cfg b/reproduction/pos_tag_model/pos_tag.cfg index f8224234..c9ee8320 100644 --- a/reproduction/pos_tag_model/pos_tag.cfg +++ b/reproduction/pos_tag_model/pos_tag.cfg @@ -10,7 +10,7 @@ eval_sort_key = 'accuracy' [model] rnn_hidden_units = 300 -word_emb_dim = 300 +word_emb_dim = 100 dropout = 0.5 use_crf = true print_every_step = 10 diff --git a/reproduction/pos_tag_model/train_pos_tag.py b/reproduction/pos_tag_model/train_pos_tag.py new file mode 100644 index 00000000..adc9359c --- /dev/null +++ b/reproduction/pos_tag_model/train_pos_tag.py @@ -0,0 +1,113 @@ +import argparse +import os +import pickle +import sys + +import torch + +# in order to run fastNLP without installation +sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) + + +from fastNLP.api.pipeline import Pipeline +from fastNLP.api.processor import SeqLenProcessor +from fastNLP.core.metrics import SpanFPreRecMetric +from fastNLP.core.trainer import Trainer +from fastNLP.io.config_io import ConfigLoader, ConfigSection +from fastNLP.models.sequence_modeling import AdvSeqLabel +from reproduction.chinese_word_segment.process.cws_processor import VocabIndexerProcessor +from reproduction.pos_tag_model.pos_reader import ZhConllPOSReader +from fastNLP.api.processor import ModelProcessor, Index2WordProcessor + +cfgfile = './pos_tag.cfg' +pickle_path = "save" + + +def load_tencent_embed(embed_path, word2id): + hit = 0 + with open(embed_path, "rb") as f: + embed_dict = pickle.load(f) + embedding_tensor = torch.randn(len(word2id), 200) + for key in word2id: + if key in embed_dict: + embedding_tensor[word2id[key]] = torch.Tensor(embed_dict[key]) + hit += 1 + print("vocab_size={} hit={} hit/vocab_size={}".format(len(word2id), hit, hit / len(word2id))) + return embedding_tensor + + +def train(checkpoint=None): + # load config + train_param = ConfigSection() + model_param = ConfigSection() + ConfigLoader().load_config(cfgfile, {"train": train_param, "model": model_param}) + print("config loaded") + + # Data Loader + dataset = ZhConllPOSReader().load("/home/hyan/train.conllx") + print(dataset) + print("dataset transformed") + + dataset.rename_field("tag", "truth") + + vocab_proc = VocabIndexerProcessor("words", new_added_filed_name="word_seq") + tag_proc = VocabIndexerProcessor("truth") + seq_len_proc = SeqLenProcessor(field_name="word_seq", new_added_field_name="word_seq_origin_len", is_input=True) + + vocab_proc(dataset) + tag_proc(dataset) + seq_len_proc(dataset) + + dataset.set_input("word_seq", "word_seq_origin_len", "truth") + dataset.set_target("truth", "word_seq_origin_len") + + print("processors defined") + + # dataset.set_is_target(tag_ids=True) + model_param["vocab_size"] = vocab_proc.get_vocab_size() + model_param["num_classes"] = tag_proc.get_vocab_size() + print("vocab_size={} num_classes={}".format(model_param["vocab_size"], model_param["num_classes"])) + + # define a model + if checkpoint is None: + # pre_trained = load_tencent_embed("/home/zyfeng/data/char_tencent_embedding.pkl", vocab_proc.vocab.word2idx) + pre_trained = None + model = AdvSeqLabel(model_param, id2words=tag_proc.vocab.idx2word, emb=pre_trained) + print(model) + else: + model = torch.load(checkpoint) + + # call trainer to train + trainer = Trainer(dataset, model, loss=None, metrics=SpanFPreRecMetric(tag_proc.vocab, pred="predict", + target="truth", + seq_lens="word_seq_origin_len"), + dev_data=dataset, metric_key="f", + use_tqdm=True, use_cuda=True, print_every=5, n_epochs=6, save_path="./save") + trainer.train(load_best_model=True) + + # save model & pipeline + model_proc = ModelProcessor(model, seq_len_field_name="word_seq_origin_len") + id2tag = Index2WordProcessor(tag_proc.vocab, "predict", "tag") + + pp = Pipeline([vocab_proc, seq_len_proc, model_proc, id2tag]) + save_dict = {"pipeline": pp, "model": model, "tag_vocab": tag_proc.vocab} + torch.save(save_dict, "model_pp.pkl") + print("pipeline saved") + + torch.save(model, "./save/best_model.pkl") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--restart", action="store_true", help="whether to continue training") + parser.add_argument("-cp", "--checkpoint", type=str, help="checkpoint of the trained model") + args = parser.parse_args() + + if args.restart is True: + # 继续训练 python train_pos_tag.py -c -cp ./save/best_model.pkl + if args.checkpoint is None: + raise RuntimeError("Please provide the checkpoint. -cp ") + train(args.checkpoint) + else: + # 一次训练 python train_pos_tag.py + train() diff --git a/reproduction/pos_tag_model/utils.py b/reproduction/pos_tag_model/utils.py new file mode 100644 index 00000000..bf10bf47 --- /dev/null +++ b/reproduction/pos_tag_model/utils.py @@ -0,0 +1,25 @@ +import pickle + + +def load_embed(embed_path): + embed_dict = {} + with open(embed_path, "r", encoding="utf-8") as f: + for line in f: + tokens = line.split(" ") + if len(tokens) <= 5: + continue + key = tokens[0] + if len(key) == 1: + value = [float(x) for x in tokens[1:]] + embed_dict[key] = value + return embed_dict + + +if __name__ == "__main__": + embed_dict = load_embed("/home/zyfeng/data/small.txt") + + print(embed_dict.keys()) + + with open("./char_tencent_embedding.pkl", "wb") as f: + pickle.dump(embed_dict, f) + print("finished") diff --git a/test/core/test_callbacks.py b/test/core/test_callbacks.py new file mode 100644 index 00000000..20822cde --- /dev/null +++ b/test/core/test_callbacks.py @@ -0,0 +1,44 @@ +import unittest + +import numpy as np + +from fastNLP.core.callback import EchoCallback +from fastNLP.core.dataset import DataSet +from fastNLP.core.instance import Instance +from fastNLP.core.losses import BCELoss +from fastNLP.core.optimizer import SGD +from fastNLP.core.trainer import Trainer +from fastNLP.models.base_model import NaiveClassifier + + +class TestCallback(unittest.TestCase): + def test_case(self): + def prepare_fake_dataset(): + mean = np.array([-3, -3]) + cov = np.array([[1, 0], [0, 1]]) + class_A = np.random.multivariate_normal(mean, cov, size=(1000,)) + + mean = np.array([3, 3]) + cov = np.array([[1, 0], [0, 1]]) + class_B = np.random.multivariate_normal(mean, cov, size=(1000,)) + + data_set = DataSet([Instance(x=[float(item[0]), float(item[1])], y=[0.0]) for item in class_A] + + [Instance(x=[float(item[0]), float(item[1])], y=[1.0]) for item in class_B]) + return data_set + + data_set = prepare_fake_dataset() + data_set.set_input("x") + data_set.set_target("y") + + model = NaiveClassifier(2, 1) + + trainer = Trainer(data_set, model, + loss=BCELoss(pred="predict", target="y"), + n_epochs=1, + batch_size=32, + print_every=50, + optimizer=SGD(lr=0.1), + check_code_level=2, + use_tqdm=False, + callbacks=[EchoCallback()]) + trainer.train() diff --git a/test/core/test_dataset.py b/test/core/test_dataset.py index 01963af6..261d42b3 100644 --- a/test/core/test_dataset.py +++ b/test/core/test_dataset.py @@ -197,4 +197,4 @@ class TestDataSetIter(unittest.TestCase): def test__repr__(self): ds = DataSet({"x": [[1, 2, 3, 4]] * 10, "y": [[5, 6]] * 10}) for iter in ds: - self.assertEqual(iter.__repr__(), "{'x': [1, 2, 3, 4],\n'y': [5, 6]}") + self.assertEqual(iter.__repr__(), "{'x': [1, 2, 3, 4] type=list,\n'y': [5, 6] type=list}") diff --git a/test/core/test_metrics.py b/test/core/test_metrics.py index 125b9156..80ed54e2 100644 --- a/test/core/test_metrics.py +++ b/test/core/test_metrics.py @@ -4,6 +4,7 @@ import numpy as np import torch from fastNLP.core.metrics import AccuracyMetric +from fastNLP.core.metrics import BMESF1PreRecMetric from fastNLP.core.metrics import pred_topk, accuracy_topk @@ -132,6 +133,235 @@ class TestAccuracyMetric(unittest.TestCase): return self.assertTrue(True, False), "No exception catches." +class SpanF1PreRecMetric(unittest.TestCase): + def test_case1(self): + from fastNLP.core.metrics import bmes_tag_to_spans + from fastNLP.core.metrics import bio_tag_to_spans + + bmes_lst = ['M-8', 'S-2', 'S-0', 'B-9', 'B-6', 'E-5', 'B-7', 'S-2', 'E-7', 'S-8'] + bio_lst = ['O-8', 'O-2', 'B-0', 'O-9', 'I-6', 'I-5', 'I-7', 'I-2', 'I-7', 'O-8'] + expect_bmes_res = set() + expect_bmes_res.update([('8', (0, 0)), ('2', (1, 1)), ('0', (2, 2)), ('9', (3, 3)), ('6', (4, 4)), + ('5', (5, 5)), ('7', (6, 6)), ('2', (7, 7)), ('7', (8, 8)), ('8', (9, 9))]) + expect_bio_res = set() + expect_bio_res.update([('7', (8, 8)), ('0', (2, 2)), ('2', (7, 7)), ('5', (5, 5)), + ('6', (4, 4)), ('7', (6, 6))]) + self.assertSetEqual(expect_bmes_res,set(bmes_tag_to_spans(bmes_lst))) + self.assertSetEqual(expect_bio_res, set(bio_tag_to_spans(bio_lst))) + # 已与allennlp对应函数做过验证，但由于测试不能依赖allennlp，所以这里只是截取上面的例子做固定测试 + # from allennlp.data.dataset_readers.dataset_utils import bio_tags_to_spans as allen_bio_tags_to_spans + # from allennlp.data.dataset_readers.dataset_utils import bmes_tags_to_spans as allen_bmes_tags_to_spans + # for i in range(1000): + # strs = list(map(str, np.random.randint(100, size=1000))) + # bmes = list('bmes'.upper()) + # bmes_strs = [str_ + '-' + tag for tag, str_ in zip(strs, np.random.choice(bmes, size=len(strs)))] + # bio = list('bio'.upper()) + # bio_strs = [str_ + '-' + tag for tag, str_ in zip(strs, np.random.choice(bio, size=len(strs)))] + # self.assertSetEqual(set(allen_bmes_tags_to_spans(bmes_strs)),set(bmes_tag_to_spans(bmes_strs))) + # self.assertSetEqual(set(allen_bio_tags_to_spans(bio_strs)), set(bio_tag_to_spans(bio_strs))) + + def test_case2(self): + # 测试不带label的 + from fastNLP.core.metrics import bmes_tag_to_spans + from fastNLP.core.metrics import bio_tag_to_spans + + bmes_lst = ['B', 'E', 'B', 'S', 'B', 'M', 'E', 'M', 'B', 'E'] + bio_lst = ['I', 'B', 'O', 'O', 'I', 'O', 'I', 'B', 'O', 'O'] + expect_bmes_res = set() + expect_bmes_res.update([('', (0, 1)), ('', (2, 2)), ('', (3, 3)), ('', (4, 6)), ('', (7, 7)), ('', (8, 9))]) + expect_bio_res = set() + expect_bio_res.update([('', (7, 7)), ('', (6, 6)), ('', (4, 4)), ('', (0, 0)), ('', (1, 1))]) + self.assertSetEqual(expect_bmes_res,set(bmes_tag_to_spans(bmes_lst))) + self.assertSetEqual(expect_bio_res, set(bio_tag_to_spans(bio_lst))) + # 已与allennlp对应函数做过验证，但由于测试不能依赖allennlp，所以这里只是截取上面的例子做固定测试 + # from allennlp.data.dataset_readers.dataset_utils import bio_tags_to_spans as allen_bio_tags_to_spans + # from allennlp.data.dataset_readers.dataset_utils import bmes_tags_to_spans as allen_bmes_tags_to_spans + # for i in range(1000): + # bmes = list('bmes'.upper()) + # bmes_strs = np.random.choice(bmes, size=1000) + # bio = list('bio'.upper()) + # bio_strs = np.random.choice(bio, size=100) + # self.assertSetEqual(set(allen_bmes_tags_to_spans(bmes_strs)),set(bmes_tag_to_spans(bmes_strs))) + # self.assertSetEqual(set(allen_bio_tags_to_spans(bio_strs)), set(bio_tag_to_spans(bio_strs))) + + def tese_case3(self): + from fastNLP.core.vocabulary import Vocabulary + from collections import Counter + from fastNLP.core.metrics import SpanFPreRecMetric + # 与allennlp测试能否正确计算f metric + # + def generate_allen_tags(encoding_type, number_labels=4): + vocab = {} + for i in range(number_labels): + label = str(i) + for tag in encoding_type: + if tag == 'O': + if tag not in vocab: + vocab['O'] = len(vocab) + 1 + continue + vocab['{}-{}'.format(tag, label)] = len(vocab) + 1 # 其实表达的是这个的count + return vocab + + number_labels = 4 + # bio tag + fastnlp_bio_vocab = Vocabulary(unknown=None, padding=None) + fastnlp_bio_vocab.word_count = Counter(generate_allen_tags('BIO', number_labels)) + fastnlp_bio_metric = SpanFPreRecMetric(tag_vocab=fastnlp_bio_vocab, only_gross=False) + bio_sequence = torch.FloatTensor( + [[[-0.9543, -1.4357, -0.2365, 0.2438, 1.0312, -1.4302, 0.3011, + 0.0470, 0.0971], + [-0.6638, -0.7116, -1.9804, 0.2787, -0.2732, -0.9501, -1.4523, + 0.7987, -0.3970], + [0.2939, 0.8132, -0.0903, -2.8296, 0.2080, -0.9823, -0.1898, + 0.6880, 1.4348], + [-0.1886, 0.0067, -0.6862, -0.4635, 2.2776, 0.0710, -1.6793, + -1.6876, -0.8917], + [-0.7663, 0.6377, 0.8669, 0.1237, 1.7628, 0.0313, -1.0824, + 1.4217, 0.2622]], + + [[0.1529, 0.7474, -0.9037, 1.5287, 0.2771, 0.2223, 0.8136, + 1.3592, -0.8973], + [0.4515, -0.5235, 0.3265, -1.1947, 0.8308, 1.8754, -0.4887, + -0.4025, -0.3417], + [-0.7855, 0.1615, -0.1272, -1.9289, -0.5181, 1.9742, -0.9698, + 0.2861, -0.3966], + [-0.8291, -0.8823, -1.1496, 0.2164, 1.3390, -0.3964, -0.5275, + 0.0213, 1.4777], + [-1.1299, 0.0627, -0.1358, -1.5951, 0.4484, -0.6081, -1.9566, + 1.3024, 0.2001]]] + ) + bio_target = torch.LongTensor([[5., 0., 3., 3., 3.], + [5., 6., 8., 6., 0.]]) + fastnlp_bio_metric({'pred': bio_sequence, 'seq_lens': torch.LongTensor([5, 5])}, {'target': bio_target}) + expect_bio_res = {'pre-1': 0.24999999999999373, 'rec-1': 0.499999999999975, 'f-1': 0.33333333333327775, + 'pre-2': 0.0, 'rec-2': 0.0, 'f-2': 0.0, 'pre-3': 0.0, 'rec-3': 0.0, 'f-3': 0.0, 'pre-0': 0.0, + 'rec-0': 0.0, 'f-0': 0.0, 'pre': 0.12499999999999845, 'rec': 0.12499999999999845, + 'f': 0.12499999999994846} + self.assertDictEqual(expect_bio_res, fastnlp_bio_metric.get_metric()) + + #bmes tag + bmes_sequence = torch.FloatTensor( + [[[0.6536, -0.7179, 0.6579, 1.2503, 0.4176, 0.6696, 0.2352, + -0.4085, 0.4084, -0.4185, 1.4172, -0.9162, -0.2679, 0.3332, + -0.3505, -0.6002], + [0.3238, -1.2378, -1.3304, -0.4903, 1.4518, -0.1868, -0.7641, + 1.6199, -0.8877, 0.1449, 0.8995, -0.5810, 0.1041, 0.1002, + 0.4439, 0.2514], + [-0.8362, 2.9526, 0.8008, 0.1193, 1.0488, 0.6670, 1.1696, + -1.1006, -0.8540, -0.1600, -0.9519, -0.2749, -0.4948, -1.4753, + 0.5802, -0.0516], + [-0.8383, -1.7292, -1.4079, -1.5023, 0.5383, 0.6653, 0.3121, + 4.1249, -0.4173, -0.2043, 1.7755, 1.1110, -1.7069, -0.0390, + -0.9242, -0.0333], + [0.9088, -0.4955, -0.5076, 0.3732, 0.0283, -0.0263, -1.0393, + 0.7734, 1.0968, 0.4132, -1.3647, -0.5762, 0.6678, 0.8809, + -0.3779, -0.3195]], + + [[-0.4638, -0.5939, -0.1052, -0.5573, 0.4600, -1.3484, 0.1753, + 0.0685, 0.3663, -0.6789, 0.0097, 1.0327, -0.0212, -0.9957, + -0.1103, 0.4417], + [-0.2903, 0.9205, -1.5758, -1.0421, 0.2921, -0.2142, -0.3049, + -0.0879, -0.4412, -1.3195, -0.0657, -0.2986, 0.7214, 0.0631, + -0.6386, 0.2797], + [0.6440, -0.3748, 1.2912, -0.0170, 0.7447, 1.4075, -0.4947, + 0.4123, -0.8447, -0.5502, 0.3520, -0.2832, 0.5019, -0.1522, + 1.1237, -1.5385], + [0.2839, -0.7649, 0.9067, -0.1163, -1.3789, 0.2571, -1.3977, + -0.3680, -0.8902, -0.6983, -1.1583, 1.2779, 0.2197, 0.1376, + -0.0591, -0.2461], + [-0.2977, -1.8564, -0.5347, 1.0011, -1.1260, 0.4252, -2.0097, + 2.6973, -0.8308, -1.4939, 0.9865, -0.3935, 0.2743, 0.1142, + -0.7344, -1.2046]]] + ) + bmes_target = torch.LongTensor([[ 9., 6., 1., 9., 15.], + [ 6., 15., 6., 15., 5.]]) + + fastnlp_bmes_vocab = Vocabulary(unknown=None, padding=None) + fastnlp_bmes_vocab.word_count = Counter(generate_allen_tags('BMES', number_labels)) + fastnlp_bmes_metric = SpanFPreRecMetric(tag_vocab=fastnlp_bmes_vocab, only_gross=False, encoding_type='bmes') + fastnlp_bmes_metric({'pred': bmes_sequence, 'seq_lens': torch.LongTensor([20, 20])}, {'target': bmes_target}) + + expect_bmes_res = {'f-3': 0.6666666666665778, 'pre-3': 0.499999999999975, 'rec-3': 0.9999999999999001, + 'f-0': 0.0, 'pre-0': 0.0, 'rec-0': 0.0, 'f-1': 0.33333333333327775, + 'pre-1': 0.24999999999999373, 'rec-1': 0.499999999999975, 'f-2': 0.7499999999999314, + 'pre-2': 0.7499999999999812, 'rec-2': 0.7499999999999812, 'f': 0.49999999999994504, + 'pre': 0.499999999999995, 'rec': 0.499999999999995} + + self.assertDictEqual(fastnlp_bmes_metric.get_metric(), expect_bmes_res) + + # 已经和allennlp做过验证，但由于不能依赖allennlp，所以注释了以下代码 + # from allennlp.data.vocabulary import Vocabulary as allen_Vocabulary + # from allennlp.training.metrics import SpanBasedF1Measure + # allen_bio_vocab = allen_Vocabulary({"tags": generate_allen_tags('BIO', number_labels)}, + # non_padded_namespaces=['tags']) + # allen_bio_metric = SpanBasedF1Measure(allen_bio_vocab, 'tags') + # bio_sequence = torch.randn(size=(2, 20, 2 * number_labels + 1)) + # bio_target = torch.randint(2 * number_labels + 1, size=(2, 20)) + # allen_bio_metric(bio_sequence, bio_target, torch.ones(2, 20)) + # fastnlp_bio_vocab = Vocabulary(unknown=None, padding=None) + # fastnlp_bio_vocab.word_count = Counter(generate_allen_tags('BIO', number_labels)) + # fastnlp_bio_metric = SpanFPreRecMetric(tag_vocab=fastnlp_bio_vocab, only_gross=False) + # + # def convert_allen_res_to_fastnlp_res(metric_result): + # allen_result = {} + # key_map = {'f1-measure-overall': "f", "recall-overall": "rec", "precision-overall": "pre"} + # for key, value in metric_result.items(): + # if key in key_map: + # key = key_map[key] + # else: + # label = key.split('-')[-1] + # if key.startswith('f1'): + # key = 'f-{}'.format(label) + # else: + # key = '{}-{}'.format(key[:3], label) + # allen_result[key] = value + # return allen_result + # + # # print(convert_allen_res_to_fastnlp_res(allen_bio_metric.get_metric())) + # # print(fastnlp_bio_metric.get_metric()) + # self.assertDictEqual(convert_allen_res_to_fastnlp_res(allen_bio_metric.get_metric()), + # fastnlp_bio_metric.get_metric()) + # + # allen_bmes_vocab = allen_Vocabulary({"tags": generate_allen_tags('BMES', number_labels)}) + # allen_bmes_metric = SpanBasedF1Measure(allen_bmes_vocab, 'tags', label_encoding='BMES') + # fastnlp_bmes_vocab = Vocabulary(unknown=None, padding=None) + # fastnlp_bmes_vocab.word_count = Counter(generate_allen_tags('BMES', number_labels)) + # fastnlp_bmes_metric = SpanFPreRecMetric(tag_vocab=fastnlp_bmes_vocab, only_gross=False, encoding_type='bmes') + # bmes_sequence = torch.randn(size=(2, 20, 4 * number_labels)) + # bmes_target = torch.randint(4 * number_labels, size=(2, 20)) + # allen_bmes_metric(bmes_sequence, bmes_target, torch.ones(2, 20)) + # fastnlp_bmes_metric({'pred': bmes_sequence, 'seq_lens': torch.LongTensor([20, 20])}, {'target': bmes_target}) + # + # # print(convert_allen_res_to_fastnlp_res(allen_bmes_metric.get_metric())) + # # print(fastnlp_bmes_metric.get_metric()) + # self.assertDictEqual(convert_allen_res_to_fastnlp_res(allen_bmes_metric.get_metric()), + # fastnlp_bmes_metric.get_metric()) + +class TestBMESF1PreRecMetric(unittest.TestCase): + def test_case1(self): + seq_lens = torch.LongTensor([4, 2]) + pred = torch.randn(2, 4, 4) + target = torch.LongTensor([[0, 1, 2, 3], + [3, 3, 0, 0]]) + pred_dict = {'pred': pred} + target_dict = {'target': target, 'seq_lens': seq_lens} + + metric = BMESF1PreRecMetric() + metric(pred_dict, target_dict) + metric.get_metric() + + def test_case2(self): + # 测试相同两个seqence，应该给出{f1: 1, precision:1, recall:1} + seq_lens = torch.LongTensor([4, 2]) + target = torch.LongTensor([[0, 1, 2, 3], + [3, 3, 0, 0]]) + pred_dict = {'pred': target} + target_dict = {'target': target, 'seq_lens': seq_lens} + + metric = BMESF1PreRecMetric() + metric(pred_dict, target_dict) + self.assertDictEqual(metric.get_metric(), {'f': 1.0, 'pre': 1.0, 'rec': 1.0}) + class TestUsefulFunctions(unittest.TestCase): # 测试metrics.py中一些看上去挺有用的函数 diff --git a/test/models/test_biaffine_parser.py b/test/models/test_biaffine_parser.py index 8fafd00b..54935f76 100644 --- a/test/models/test_biaffine_parser.py +++ b/test/models/test_biaffine_parser.py @@ -10,6 +10,8 @@ data_file = """ 4 will _ AUX MD _ 6 aux _ _ 5 be _ VERB VB _ 6 cop _ _ 6 payable _ ADJ JJ _ 0 root _ _ +7 mask _ ADJ JJ _ 6 punct _ _ +8 mask _ ADJ JJ _ 6 punct _ _ 9 cents _ NOUN NNS _ 4 nmod _ _ 10 from _ ADP IN _ 12 case _ _ 11 seven _ NUM CD _ 12 nummod _ _ @@ -58,13 +60,13 @@ def init_data(): data.append(line) for name in ['word_seq', 'pos_seq', 'label_true']: - ds.apply(lambda x: ['']+list(x[name])+[''], new_field_name=name) + ds.apply(lambda x: ['']+list(x[name]), new_field_name=name) ds.apply(lambda x: v[name].add_word_lst(x[name])) for name in ['word_seq', 'pos_seq', 'label_true']: ds.apply(lambda x: [v[name].to_index(w) for w in x[name]], new_field_name=name) - ds.apply(lambda x: [0]+list(map(int, x['arc_true']))+[1], new_field_name='arc_true') + ds.apply(lambda x: [0]+list(map(int, x['arc_true'])), new_field_name='arc_true') ds.apply(lambda x: len(x['word_seq']), new_field_name='seq_lens') ds.set_input('word_seq', 'pos_seq', 'seq_lens', flag=True) ds.set_target('arc_true', 'label_true', 'seq_lens', flag=True) @@ -75,8 +77,11 @@ class TestBiaffineParser(unittest.TestCase): ds, v1, v2, v3 = init_data() model = BiaffineParser(word_vocab_size=len(v1), word_emb_dim=30, pos_vocab_size=len(v2), pos_emb_dim=30, - num_label=len(v3)) + num_label=len(v3), use_var_lstm=True) trainer = fastNLP.Trainer(model=model, train_data=ds, dev_data=ds, loss=ParserLoss(), metrics=ParserMetric(), metric_key='UAS', n_epochs=10, use_cuda=False, use_tqdm=False) trainer.train(load_best_model=False) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/test/modules/decoder/test_CRF.py b/test/modules/decoder/test_CRF.py new file mode 100644 index 00000000..0fc331dc --- /dev/null +++ b/test/modules/decoder/test_CRF.py @@ -0,0 +1,104 @@ + +import unittest + + +class TestCRF(unittest.TestCase): + def test_case1(self): + # 检查allowed_transitions()能否正确使用 + from fastNLP.modules.decoder.CRF import allowed_transitions + + id2label = {0: 'B', 1: 'I', 2:'O'} + expected_res = {(0, 0), (0, 1), (0, 2), (0, 4), (1, 0), (1, 1), (1, 2), (1, 4), (2, 0), (2, 2), + (2, 4), (3, 0), (3, 2)} + self.assertSetEqual(expected_res, set(allowed_transitions(id2label))) + + id2label = {0: 'B', 1:'M', 2:'E', 3:'S'} + expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 5), (3, 0), (3, 3), (3, 5), (4, 0), (4, 3)} + self.assertSetEqual(expected_res, set(allowed_transitions(id2label, encoding_type='BMES'))) + + id2label = {0: 'B', 1: 'I', 2:'O', 3: '', 4:""} + allowed_transitions(id2label) + + labels = ['O'] + for label in ['X', 'Y']: + for tag in 'BI': + labels.append('{}-{}'.format(tag, label)) + id2label = {idx:label for idx, label in enumerate(labels)} + expected_res = {(0, 0), (0, 1), (0, 3), (0, 6), (1, 0), (1, 1), (1, 2), (1, 3), (1, 6), (2, 0), (2, 1), + (2, 2), (2, 3), (2, 6), (3, 0), (3, 1), (3, 3), (3, 4), (3, 6), (4, 0), (4, 1), (4, 3), + (4, 4), (4, 6), (5, 0), (5, 1), (5, 3)} + self.assertSetEqual(expected_res, set(allowed_transitions(id2label))) + + labels = [] + for label in ['X', 'Y']: + for tag in 'BMES': + labels.append('{}-{}'.format(tag, label)) + id2label = {idx: label for idx, label in enumerate(labels)} + expected_res = {(0, 1), (0, 2), (1, 1), (1, 2), (2, 0), (2, 3), (2, 4), (2, 7), (2, 9), (3, 0), (3, 3), (3, 4), + (3, 7), (3, 9), (4, 5), (4, 6), (5, 5), (5, 6), (6, 0), (6, 3), (6, 4), (6, 7), (6, 9), (7, 0), + (7, 3), (7, 4), (7, 7), (7, 9), (8, 0), (8, 3), (8, 4), (8, 7)} + self.assertSetEqual(expected_res, set(allowed_transitions(id2label, encoding_type='BMES'))) + + def test_case2(self): + # 测试CRF能否避免解码出非法跃迁, 使用allennlp做了验证。 + pass + # import torch + # from fastNLP.modules.decoder.CRF import seq_len_to_byte_mask + # + # labels = ['O'] + # for label in ['X', 'Y']: + # for tag in 'BI': + # labels.append('{}-{}'.format(tag, label)) + # id2label = {idx: label for idx, label in enumerate(labels)} + # num_tags = len(id2label) + # + # from allennlp.modules.conditional_random_field import ConditionalRandomField, allowed_transitions + # allen_CRF = ConditionalRandomField(num_tags=num_tags, constraints=allowed_transitions('BIO', id2label), + # include_start_end_transitions=False) + # batch_size = 3 + # logits = torch.nn.functional.softmax(torch.rand(size=(batch_size, 20, num_tags))).log() + # trans_m = allen_CRF.transitions + # seq_lens = torch.randint(1, 20, size=(batch_size,)) + # seq_lens[-1] = 20 + # mask = seq_len_to_byte_mask(seq_lens) + # allen_res = allen_CRF.viterbi_tags(logits, mask) + # + # from fastNLP.modules.decoder.CRF import ConditionalRandomField, allowed_transitions + # fast_CRF = ConditionalRandomField(num_tags=num_tags, allowed_transitions=allowed_transitions(id2label)) + # fast_CRF.trans_m = trans_m + # fast_res = fast_CRF.viterbi_decode(logits, mask, get_score=True) + # # score equal + # self.assertListEqual([score for _, score in allen_res], fast_res[1]) + # # seq equal + # self.assertListEqual([_ for _, score in allen_res], fast_res[0]) + # + # + # labels = [] + # for label in ['X', 'Y']: + # for tag in 'BMES': + # labels.append('{}-{}'.format(tag, label)) + # id2label = {idx: label for idx, label in enumerate(labels)} + # num_tags = len(id2label) + # + # from allennlp.modules.conditional_random_field import ConditionalRandomField, allowed_transitions + # allen_CRF = ConditionalRandomField(num_tags=num_tags, constraints=allowed_transitions('BMES', id2label), + # include_start_end_transitions=False) + # batch_size = 3 + # logits = torch.nn.functional.softmax(torch.rand(size=(batch_size, 20, num_tags))).log() + # trans_m = allen_CRF.transitions + # seq_lens = torch.randint(1, 20, size=(batch_size,)) + # seq_lens[-1] = 20 + # mask = seq_len_to_byte_mask(seq_lens) + # allen_res = allen_CRF.viterbi_tags(logits, mask) + # + # from fastNLP.modules.decoder.CRF import ConditionalRandomField, allowed_transitions + # fast_CRF = ConditionalRandomField(num_tags=num_tags, allowed_transitions=allowed_transitions(id2label, + # encoding_type='BMES')) + # fast_CRF.trans_m = trans_m + # fast_res = fast_CRF.viterbi_decode(logits, mask, get_score=True) + # # score equal + # self.assertListEqual([score for _, score in allen_res], fast_res[1]) + # # seq equal + # self.assertListEqual([_ for _, score in allen_res], fast_res[0]) + + diff --git a/tutorials/README.md b/tutorials/README.md new file mode 100644 index 00000000..1de342e6 --- /dev/null +++ b/tutorials/README.md @@ -0,0 +1,12 @@ +# fastNLP 教程 + +### 上手教程 Quick Start +- 一分钟上手：`fastnlp_1min_tutorial.ipynb` [Click Here](https://github.com/fastnlp/fastNLP/tree/master/tutorials/fastnlp_1min_tutorial.ipynb) +- 十分钟上手：`fastnlp_10min_tutorial.ipynb` [Click Here](https://github.com/fastnlp/fastNLP/tree/master/tutorials/fastnlp_10min_tutorial.ipynb) + +### 进阶教程 Advanced Tutorial +- `fastnlp_advanced_tutorial/advance_tutorial.ipynb` [Click Here](https://github.com/fastnlp/fastNLP/tree/master/tutorials/fastnlp_advanced_tutorial/advance_tutorial.ipynb) + + +### 开发者指南 Developer Guide +- `tutorial_for_developer.md` [Click Here](https://github.com/fastnlp/fastNLP/tree/master/tutorials/tutorial_for_developer.md) diff --git a/tutorials/fastnlp_10min_tutorial_v2.ipynb b/tutorials/fastnlp_10min_tutorial.ipynb similarity index 59% rename from tutorials/fastnlp_10min_tutorial_v2.ipynb rename to tutorials/fastnlp_10min_tutorial.ipynb index f86e5bf3..534c4e49 100644 --- a/tutorials/fastnlp_10min_tutorial_v2.ipynb +++ b/tutorials/fastnlp_10min_tutorial.ipynb @@ -4,12 +4,29 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "fastNLP上手教程\n", + "fastNLP10 分钟上手教程\n", "-------\n", "\n", "fastNLP提供方便的数据预处理，训练和测试模型的功能" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "如果您还没有通过pip安装fastNLP，可以执行下面的操作加载当前模块" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append(\"../\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -24,21 +41,14 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "8529" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "77\n" ] } ], @@ -47,27 +57,23 @@ "from fastNLP import Instance\n", "\n", "# 从csv读取数据到DataSet\n", - "dataset = DataSet.read_csv('../sentence.csv', headers=('raw_sentence', 'label'), sep='\\t')\n", + "dataset = DataSet.read_csv('sample_data/tutorial_sample_dataset.csv', headers=('raw_sentence', 'label'), sep='\\t')\n", "print(len(dataset))" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'raw_sentence': A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .,\n'label': 1}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "{'raw_sentence': A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . type=str,\n", + "'label': 1 type=str}\n", + "{'raw_sentence': The plot is romantic comedy boilerplate from start to finish . type=str,\n", + "'label': 2 type=str}\n" ] } ], @@ -91,16 +97,17 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'raw_sentence': fake data,\n'label': 0}" + "{'raw_sentence': fake data type=str,\n", + "'label': 0 type=str}" ] }, - "execution_count": 11, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -121,21 +128,15 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .,\n'label': 1}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . type=str,\n", + "'label': 1 type=str}\n" ] } ], @@ -147,21 +148,15 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .,\n'label': 1}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . type=str,\n", + "'label': 1 type=int}\n" ] } ], @@ -173,21 +168,16 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .,\n'label': 1,\n'words': ['a', 'series', 'of', 'escapades', 'demonstrating', 'the', 'adage', 'that', 'what', 'is', 'good', 'for', 'the', 'goose', 'is', 'also', 'good', 'for', 'the', 'gander', ',', 'some', 'of', 'which', 'occasionally', 'amuses', 'but', 'none', 'of', 'which', 'amounts', 'to', 'much', 'of', 'a', 'story', '.']}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . type=str,\n", + "'label': 1 type=int,\n", + "'words': ['a', 'series', 'of', 'escapades', 'demonstrating', 'the', 'adage', 'that', 'what', 'is', 'good', 'for', 'the', 'goose', 'is', 'also', 'good', 'for', 'the', 'gander', ',', 'some', 'of', 'which', 'occasionally', 'amuses', 'but', 'none', 'of', 'which', 'amounts', 'to', 'much', 'of', 'a', 'story', '.'] type=list}\n" ] } ], @@ -201,21 +191,17 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .,\n'label': 1,\n'words': ['a', 'series', 'of', 'escapades', 'demonstrating', 'the', 'adage', 'that', 'what', 'is', 'good', 'for', 'the', 'goose', 'is', 'also', 'good', 'for', 'the', 'gander', ',', 'some', 'of', 'which', 'occasionally', 'amuses', 'but', 'none', 'of', 'which', 'amounts', 'to', 'much', 'of', 'a', 'story', '.'],\n'seq_len': 37}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "{'raw_sentence': a series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . type=str,\n", + "'label': 1 type=int,\n", + "'words': ['a', 'series', 'of', 'escapades', 'demonstrating', 'the', 'adage', 'that', 'what', 'is', 'good', 'for', 'the', 'goose', 'is', 'also', 'good', 'for', 'the', 'gander', ',', 'some', 'of', 'which', 'occasionally', 'amuses', 'but', 'none', 'of', 'which', 'amounts', 'to', 'much', 'of', 'a', 'story', '.'] type=list,\n", + "'seq_len': 37 type=int}\n" ] } ], @@ -235,25 +221,19 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "8358" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "77\n" ] } ], "source": [ + "# 删除低于某个长度的词语\n", "dataset.drop(lambda x: x['seq_len'] <= 3)\n", "print(len(dataset))" ] @@ -269,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -283,35 +263,15 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "5851" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2507" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "54\n", + "23\n" ] } ], @@ -335,21 +295,17 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'raw_sentence': the project 's filmmakers forgot to include anything even halfway scary as they poorly rejigger fatal attraction into a high school setting .,\n'label': 0,\n'words': [4, 423, 9, 316, 1, 8, 1, 312, 72, 1478, 885, 14, 86, 725, 1, 1913, 1431, 53, 5, 455, 736, 1, 2],\n'seq_len': 23}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "{'raw_sentence': a welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . type=str,\n", + "'label': 3 type=int,\n", + "'words': [4, 1, 1, 18, 1, 1, 13, 1, 1, 1, 8, 26, 1, 5, 35, 1, 11, 4, 1, 10, 1, 10, 1, 1, 1, 2] type=list,\n", + "'seq_len': 26 type=int}\n" ] } ], @@ -369,6 +325,23 @@ "print(test_data[0])" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 如果你们需要做强化学习或者GAN之类的项目，你们也可以使用这些数据预处理的工具\n", + "from fastNLP.core.batch import Batch\n", + "from fastNLP.core.sampler import RandomSampler\n", + "\n", + "batch_iterator = Batch(dataset=train_data, batch_size=2, sampler=RandomSampler())\n", + "for batch_x, batch_y in batch_iterator:\n", + " print(\"batch_x has: \", batch_x)\n", + " print(\"batch_y has: \", batch_y)\n", + " break" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -379,16 +352,32 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "CNNText(\n (embed): Embedding(\n (embed): Embedding(3459, 50, padding_idx=0)\n (dropout): Dropout(p=0.0)\n )\n (conv_pool): ConvMaxpool(\n (convs): ModuleList(\n (0): Conv1d(50, 3, kernel_size=(3,), stride=(1,), padding=(2,))\n (1): Conv1d(50, 4, kernel_size=(4,), stride=(1,), padding=(2,))\n (2): Conv1d(50, 5, kernel_size=(5,), stride=(1,), padding=(2,))\n )\n )\n (dropout): Dropout(p=0.1)\n (fc): Linear(\n (linear): Linear(in_features=12, out_features=5, bias=True)\n )\n)" + "CNNText(\n", + " (embed): Embedding(\n", + " (embed): Embedding(59, 50, padding_idx=0)\n", + " (dropout): Dropout(p=0.0)\n", + " )\n", + " (conv_pool): ConvMaxpool(\n", + " (convs): ModuleList(\n", + " (0): Conv1d(50, 3, kernel_size=(3,), stride=(1,), padding=(2,))\n", + " (1): Conv1d(50, 4, kernel_size=(4,), stride=(1,), padding=(2,))\n", + " (2): Conv1d(50, 5, kernel_size=(5,), stride=(1,), padding=(2,))\n", + " )\n", + " )\n", + " (dropout): Dropout(p=0.1)\n", + " (fc): Linear(\n", + " (linear): Linear(in_features=12, out_features=5, bias=True)\n", + " )\n", + ")" ] }, - "execution_count": 20, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -459,7 +448,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -496,7 +485,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -519,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -528,149 +517,61 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "training epochs started 2018-12-07 14:11:31" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "input fields after batch(if batch size is 2):\n", + "\tword_seq: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 26]) \n", + "target fields after batch(if batch size is 2):\n", + "\tlabel_seq: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2019-01-12 17-07-51\n" ] }, { "data": { "text/plain": [ - "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=915), HTML(value='')), layout=Layout(display=…" + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=10), HTML(value='')), layout=Layout(display='…" ] }, - "execution_count": 0, "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/5. Step:183/915. AccuracyMetric: acc=0.350367" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] + "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ - "\r" + "Evaluation at Epoch 1/5. Step:2/10. AccuracyMetric: acc=0.425926\n", + "Evaluation at Epoch 2/5. Step:4/10. AccuracyMetric: acc=0.425926\n", + "Evaluation at Epoch 3/5. Step:6/10. AccuracyMetric: acc=0.611111\n", + "Evaluation at Epoch 4/5. Step:8/10. AccuracyMetric: acc=0.648148\n", + "Evaluation at Epoch 5/5. Step:10/10. AccuracyMetric: acc=0.703704\n", + "\n", + "In Epoch:5/Step:10, got best dev performance:AccuracyMetric: acc=0.703704\n", + "Reloaded the best model.\n" ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 2/5. Step:366/915. AccuracyMetric: acc=0.409332" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 3/5. Step:549/915. AccuracyMetric: acc=0.572552" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 4/5. Step:732/915. AccuracyMetric: acc=0.711331" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 5/5. Step:915/915. AccuracyMetric: acc=0.801572" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] + "data": { + "text/plain": [ + "{'best_eval': {'AccuracyMetric': {'acc': 0.703704}},\n", + " 'best_epoch': 5,\n", + " 'best_step': 10,\n", + " 'seconds': 0.62}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "# 实例化Trainer，传入模型和数据，进行训练\n", - "# 先在test_data拟合\n", + "# 先在test_data拟合（确保模型的实现是正确的）\n", "copy_model = deepcopy(model)\n", "overfit_trainer = Trainer(model=copy_model, train_data=test_data, dev_data=test_data,\n", " loss=loss,\n", @@ -683,143 +584,43 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "training epochs started 2018-12-07 14:12:21" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "input fields after batch(if batch size is 2):\n", + "\tword_seq: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 20]) \n", + "target fields after batch(if batch size is 2):\n", + "\tlabel_seq: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2019-01-12 17-09-05\n" ] }, { "data": { "text/plain": [ - "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=395), HTML(value='')), layout=Layout(display=…" + "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=5), HTML(value='')), layout=Layout(display='i…" ] }, - "execution_count": 0, "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/5. Step:79/395. AccuracyMetric: acc=0.250043" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 2/5. Step:158/395. AccuracyMetric: acc=0.280807" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 3/5. Step:237/395. AccuracyMetric: acc=0.280978" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 4/5. Step:316/395. AccuracyMetric: acc=0.285592" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 5/5. Step:395/395. AccuracyMetric: acc=0.278927" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] + "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ - "\r" + "Evaluation at Epoch 1/5. Step:1/5. AccuracyMetric: acc=0.37037\n", + "Evaluation at Epoch 2/5. Step:2/5. AccuracyMetric: acc=0.37037\n", + "Evaluation at Epoch 3/5. Step:3/5. AccuracyMetric: acc=0.462963\n", + "Evaluation at Epoch 4/5. Step:4/5. AccuracyMetric: acc=0.425926\n", + "Evaluation at Epoch 5/5. Step:5/5. AccuracyMetric: acc=0.481481\n", + "\n", + "In Epoch:5/Step:5, got best dev performance:AccuracyMetric: acc=0.481481\n", + "Reloaded the best model.\n", + "Train finished!\n" ] } ], @@ -837,35 +638,16 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[tester] \nAccuracyMetric: acc=0.280636" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'AccuracyMetric': {'acc': 0.280636}}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" + "[tester] \n", + "AccuracyMetric: acc=0.481481\n", + "{'AccuracyMetric': {'acc': 0.481481}}\n" ] } ], @@ -879,6 +661,75 @@ "print(acc)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# In summary\n", + "\n", + "## fastNLP Trainer的伪代码逻辑\n", + "### 1. 准备DataSet，假设DataSet中共有如下的fields\n", + " ['raw_sentence', 'word_seq1', 'word_seq2', 'raw_label','label']\n", + " 通过\n", + " DataSet.set_input('word_seq1', word_seq2', flag=True)将'word_seq1', 'word_seq2'设置为input\n", + " 通过\n", + " DataSet.set_target('label', flag=True)将'label'设置为target\n", + "### 2. 初始化模型\n", + " class Model(nn.Module):\n", + " def __init__(self):\n", + " xxx\n", + " def forward(self, word_seq1, word_seq2):\n", + " # (1) 这里使用的形参名必须和DataSet中的input field的名称对应。因为我们是通过形参名, 进行赋值的\n", + " # (2) input field的数量可以多于这里的形参数量。但是不能少于。\n", + " xxxx\n", + " # 输出必须是一个dict\n", + "### 3. Trainer的训练过程\n", + " (1) 从DataSet中按照batch_size取出一个batch，调用Model.forward\n", + " (2) 将 Model.forward的结果 与 标记为target的field 传入Losser当中。\n", + " 由于每个人写的Model.forward的output的dict可能key并不一样，比如有人是{'pred':xxx}, {'output': xxx}; \n", + " 另外每个人将target可能也会设置为不同的名称, 比如有人是label, 有人设置为target；\n", + " 为了解决以上的问题，我们的loss提供映射机制\n", + " 比如CrossEntropyLosser的需要的输入是(prediction, target)。但是forward的output是{'output': xxx}; 'label'是target\n", + " 那么初始化losser的时候写为CrossEntropyLosser(prediction='output', target='label')即可\n", + " (3) 对于Metric是同理的\n", + " Metric计算也是从 forward的结果中取值 与 设置target的field中取值。 也是可以通过映射找到对应的值 \n", + " \n", + " \n", + "\n", + "## 一些问题.\n", + "### 1. DataSet中为什么需要设置input和target\n", + " 只有被设置为input或者target的数据才会在train的过程中被取出来\n", + " (1.1) 我们只会在设置为input的field中寻找传递给Model.forward的参数。\n", + " (1.2) 我们在传递值给losser或者metric的时候会使用来自: \n", + " (a)Model.forward的output\n", + " (b)被设置为target的field\n", + " \n", + "\n", + "### 2. 我们是通过forwad中的形参名将DataSet中的field赋值给对应的参数\n", + " (1.1) 构建模型过程中，\n", + " 例如:\n", + " DataSet中x，seq_lens是input，那么forward就应该是\n", + " def forward(self, x, seq_lens):\n", + " pass\n", + " 我们是通过形参名称进行匹配的field的\n", + " \n", + "\n", + "\n", + "### 1. 加载数据到DataSet\n", + "### 2. 使用apply操作对DataSet进行预处理\n", + " (2.1) 处理过程中将某些field设置为input，某些field设置为target\n", + "### 3. 构建模型\n", + " (3.1) 构建模型过程中，需要注意forward函数的形参名需要和DataSet中设置为input的field名称是一致的。\n", + " 例如:\n", + " DataSet中x，seq_lens是input，那么forward就应该是\n", + " def forward(self, x, seq_lens):\n", + " pass\n", + " 我们是通过形参名称进行匹配的field的\n", + " (3.2) 模型的forward的output需要是dict类型的。\n", + " 建议将输出设置为{\"pred\": xx}.\n", + " \n" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/tutorials/fastnlp_10tmin_tutorial.ipynb b/tutorials/fastnlp_10tmin_tutorial.ipynb deleted file mode 100644 index bad29f55..00000000 --- a/tutorials/fastnlp_10tmin_tutorial.ipynb +++ /dev/null @@ -1,860 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "fastNLP上手教程\n", - "-------\n", - "\n", - "fastNLP提供方便的数据预处理，训练和测试模型的功能" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "DataSet & Instance\n", - "------\n", - "\n", - "fastNLP用DataSet和Instance保存和处理数据。每个DataSet表示一个数据集，每个Instance表示一个数据样本。一个DataSet存有多个Instance，每个Instance可以自定义存哪些内容。\n", - "\n", - "有一些read_*方法，可以轻松从文件读取数据，存成DataSet。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'raw_sentence': A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story .,\n'label': 1}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "from fastNLP import DataSet\n", - "from fastNLP import Instance\n", - "\n", - "# 从csv读取数据到DataSet\n", - "win_path = \"C:\\\\Users\\zyfeng\\Desktop\\FudanNLP\\\\fastNLP\\\\test\\\\data_for_tests\\\\tutorial_sample_dataset.csv\"\n", - "dataset = DataSet.read_csv(win_path, headers=('raw_sentence', 'label'), sep='\\t')\n", - "print(dataset[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'raw_sentence': fake data,\n'label': 0}" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# DataSet.append(Instance)加入新数据\n", - "\n", - "dataset.append(Instance(raw_sentence='fake data', label='0'))\n", - "dataset[-1]" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# DataSet.apply(func, new_field_name)对数据预处理\n", - "\n", - "# 将所有数字转为小写\n", - "dataset.apply(lambda x: x['raw_sentence'].lower(), new_field_name='raw_sentence')\n", - "# label转int\n", - "dataset.apply(lambda x: int(x['label']), new_field_name='label_seq', is_target=True)\n", - "# 使用空格分割句子\n", - "dataset.drop(lambda x: len(x['raw_sentence'].split()) == 0)\n", - "def split_sent(ins):\n", - " return ins['raw_sentence'].split()\n", - "dataset.apply(split_sent, new_field_name='words', is_input=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# DataSet.drop(func)筛除数据\n", - "# 删除低于某个长度的词语\n", - "dataset.drop(lambda x: len(x['words']) <= 3)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Train size: " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "54" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test size: " - ] - } - ], - "source": [ - "# 分出测试集、训练集\n", - "\n", - "test_data, train_data = dataset.split(0.3)\n", - "print(\"Train size: \", len(test_data))\n", - "print(\"Test size: \", len(train_data))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Vocabulary\n", - "------\n", - "\n", - "fastNLP中的Vocabulary轻松构建词表，将词转成数字" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'raw_sentence': the plot is romantic comedy boilerplate from start to finish .,\n'label': 2,\n'label_seq': 2,\n'words': ['the', 'plot', 'is', 'romantic', 'comedy', 'boilerplate', 'from', 'start', 'to', 'finish', '.'],\n'word_seq': [2, 13, 9, 24, 25, 26, 15, 27, 11, 28, 3]}" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "from fastNLP import Vocabulary\n", - "\n", - "# 构建词表, Vocabulary.add(word)\n", - "vocab = Vocabulary(min_freq=2)\n", - "train_data.apply(lambda x: [vocab.add(word) for word in x['words']])\n", - "vocab.build_vocab()\n", - "\n", - "# index句子, Vocabulary.to_index(word)\n", - "train_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='word_seq', is_input=True)\n", - "test_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='word_seq', is_input=True)\n", - "\n", - "\n", - "print(test_data[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "batch_x has: {'words': array([list(['this', 'kind', 'of', 'hands-on', 'storytelling', 'is', 'ultimately', 'what', 'makes', 'shanghai', 'ghetto', 'move', 'beyond', 'a', 'good', ',', 'dry', ',', 'reliable', 'textbook', 'and', 'what', 'allows', 'it', 'to', 'rank', 'with', 'its', 'worthy', 'predecessors', '.']),\n", - " list(['the', 'entire', 'movie', 'is', 'filled', 'with', 'deja', 'vu', 'moments', '.'])],\n", - " dtype=object), 'word_seq': tensor([[ 19, 184, 6, 1, 481, 9, 206, 50, 91, 1210, 1609, 1330,\n", - " 495, 5, 63, 4, 1269, 4, 1, 1184, 7, 50, 1050, 10,\n", - " 8, 1611, 16, 21, 1039, 1, 2],\n", - " [ 3, 711, 22, 9, 1282, 16, 2482, 2483, 200, 2, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0]])}\n", - "batch_y has: {'label_seq': tensor([3, 2])}\n" - ] - } - ], - "source": [ - "# 假设你们需要做强化学习或者gan之类的项目，也许你们可以使用这里的dataset\n", - "from fastNLP.core.batch import Batch\n", - "from fastNLP.core.sampler import RandomSampler\n", - "\n", - "batch_iterator = Batch(dataset=train_data, batch_size=2, sampler=RandomSampler())\n", - "for batch_x, batch_y in batch_iterator:\n", - " print(\"batch_x has: \", batch_x)\n", - " print(\"batch_y has: \", batch_y)\n", - " break" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Model\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "CNNText(\n (embed): Embedding(\n (embed): Embedding(77, 50, padding_idx=0)\n (dropout): Dropout(p=0.0)\n )\n (conv_pool): ConvMaxpool(\n (convs): ModuleList(\n (0): Conv1d(50, 3, kernel_size=(3,), stride=(1,), padding=(2,))\n (1): Conv1d(50, 4, kernel_size=(4,), stride=(1,), padding=(2,))\n (2): Conv1d(50, 5, kernel_size=(5,), stride=(1,), padding=(2,))\n )\n )\n (dropout): Dropout(p=0.1)\n (fc): Linear(\n (linear): Linear(in_features=12, out_features=5, bias=True)\n )\n)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# 定义一个简单的Pytorch模型\n", - "\n", - "from fastNLP.models import CNNText\n", - "model = CNNText(embed_num=len(vocab), embed_dim=50, num_classes=5, padding=2, dropout=0.1)\n", - "model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Trainer & Tester\n", - "------\n", - "\n", - "使用fastNLP的Trainer训练模型" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "from fastNLP import Trainer\n", - "from copy import deepcopy\n", - "from fastNLP import CrossEntropyLoss\n", - "from fastNLP import AccuracyMetric" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "training epochs started 2018-12-07 14:07:20" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/plain": [ - "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=20), HTML(value='')), layout=Layout(display='…" - ] - }, - "execution_count": 0, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/10. Step:2/20. AccuracyMetric: acc=0.037037" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 2/10. Step:4/20. AccuracyMetric: acc=0.296296" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 3/10. Step:6/20. AccuracyMetric: acc=0.333333" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 4/10. Step:8/20. AccuracyMetric: acc=0.555556" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 5/10. Step:10/20. AccuracyMetric: acc=0.611111" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 6/10. Step:12/20. AccuracyMetric: acc=0.481481" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 7/10. Step:14/20. AccuracyMetric: acc=0.62963" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 8/10. Step:16/20. AccuracyMetric: acc=0.685185" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 9/10. Step:18/20. AccuracyMetric: acc=0.722222" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 10/10. Step:20/20. AccuracyMetric: acc=0.777778" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - } - ], - "source": [ - "# 进行overfitting测试\n", - "copy_model = deepcopy(model)\n", - "overfit_trainer = Trainer(model=copy_model, \n", - " train_data=test_data, \n", - " dev_data=test_data,\n", - " loss=CrossEntropyLoss(pred=\"output\", target=\"label_seq\"),\n", - " metrics=AccuracyMetric(),\n", - " n_epochs=10,\n", - " save_path=None)\n", - "overfit_trainer.train()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "training epochs started 2018-12-07 14:08:10" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/plain": [ - "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=5), HTML(value='')), layout=Layout(display='i…" - ] - }, - "execution_count": 0, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/5. Step:1/5. AccuracyMetric: acc=0.037037" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 2/5. Step:2/5. AccuracyMetric: acc=0.037037" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 3/5. Step:3/5. AccuracyMetric: acc=0.037037" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 4/5. Step:4/5. AccuracyMetric: acc=0.185185" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 5/5. Step:5/5. AccuracyMetric: acc=0.240741" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Train finished!" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# 实例化Trainer，传入模型和数据，进行训练\n", - "trainer = Trainer(model=model, \n", - " train_data=train_data, \n", - " dev_data=test_data,\n", - " loss=CrossEntropyLoss(pred=\"output\", target=\"label_seq\"),\n", - " metrics=AccuracyMetric(),\n", - " n_epochs=5)\n", - "trainer.train()\n", - "print('Train finished!')" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[tester] \nAccuracyMetric: acc=0.240741" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "from fastNLP import Tester\n", - "\n", - "tester = Tester(data=test_data, model=model, metrics=AccuracyMetric())\n", - "acc = tester.test()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# In summary\n", - "\n", - "## fastNLP Trainer的伪代码逻辑\n", - "### 1. 准备DataSet，假设DataSet中共有如下的fields\n", - " ['raw_sentence', 'word_seq1', 'word_seq2', 'raw_label','label']\n", - " 通过\n", - " DataSet.set_input('word_seq1', word_seq2', flag=True)将'word_seq1', 'word_seq2'设置为input\n", - " 通过\n", - " DataSet.set_target('label', flag=True)将'label'设置为target\n", - "### 2. 初始化模型\n", - " class Model(nn.Module):\n", - " def __init__(self):\n", - " xxx\n", - " def forward(self, word_seq1, word_seq2):\n", - " # (1) 这里使用的形参名必须和DataSet中的input field的名称对应。因为我们是通过形参名, 进行赋值的\n", - " # (2) input field的数量可以多于这里的形参数量。但是不能少于。\n", - " xxxx\n", - " # 输出必须是一个dict\n", - "### 3. Trainer的训练过程\n", - " (1) 从DataSet中按照batch_size取出一个batch，调用Model.forward\n", - " (2) 将 Model.forward的结果 与 标记为target的field 传入Losser当中。\n", - " 由于每个人写的Model.forward的output的dict可能key并不一样，比如有人是{'pred':xxx}, {'output': xxx}; \n", - " 另外每个人将target可能也会设置为不同的名称, 比如有人是label, 有人设置为target；\n", - " 为了解决以上的问题，我们的loss提供映射机制\n", - " 比如CrossEntropyLosser的需要的输入是(prediction, target)。但是forward的output是{'output': xxx}; 'label'是target\n", - " 那么初始化losser的时候写为CrossEntropyLosser(prediction='output', target='label')即可\n", - " (3) 对于Metric是同理的\n", - " Metric计算也是从 forward的结果中取值 与 设置target的field中取值。 也是可以通过映射找到对应的值 \n", - " \n", - " \n", - "\n", - "## 一些问题.\n", - "### 1. DataSet中为什么需要设置input和target\n", - " 只有被设置为input或者target的数据才会在train的过程中被取出来\n", - " (1.1) 我们只会在设置为input的field中寻找传递给Model.forward的参数。\n", - " (1.2) 我们在传递值给losser或者metric的时候会使用来自: \n", - " (a)Model.forward的output\n", - " (b)被设置为target的field\n", - " \n", - "\n", - "### 2. 我们是通过forwad中的形参名将DataSet中的field赋值给对应的参数\n", - " (1.1) 构建模型过程中，\n", - " 例如:\n", - " DataSet中x，seq_lens是input，那么forward就应该是\n", - " def forward(self, x, seq_lens):\n", - " pass\n", - " 我们是通过形参名称进行匹配的field的\n", - " \n", - "\n", - "\n", - "### 1. 加载数据到DataSet\n", - "### 2. 使用apply操作对DataSet进行预处理\n", - " (2.1) 处理过程中将某些field设置为input，某些field设置为target\n", - "### 3. 构建模型\n", - " (3.1) 构建模型过程中，需要注意forward函数的形参名需要和DataSet中设置为input的field名称是一致的。\n", - " 例如:\n", - " DataSet中x，seq_lens是input，那么forward就应该是\n", - " def forward(self, x, seq_lens):\n", - " pass\n", - " 我们是通过形参名称进行匹配的field的\n", - " (3.2) 模型的forward的output需要是dict类型的。\n", - " 建议将输出设置为{\"pred\": xx}.\n", - " \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tutorials/fastnlp_1_minute_tutorial.ipynb b/tutorials/fastnlp_1min_tutorial.ipynb similarity index 73% rename from tutorials/fastnlp_1_minute_tutorial.ipynb rename to tutorials/fastnlp_1min_tutorial.ipynb index 24b569f6..7a35d992 100644 --- a/tutorials/fastnlp_1_minute_tutorial.ipynb +++ b/tutorials/fastnlp_1min_tutorial.ipynb @@ -6,7 +6,7 @@ "collapsed": true }, "source": [ - "# FastNLP 1分钟上手教程" + "# fastNLP 1分钟上手教程" ] }, { @@ -19,14 +19,14 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/Users/yh/miniconda2/envs/python3/lib/python3.6/site-packages/tqdm/autonotebook/__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", + "c:\\users\\zyfeng\\miniconda3\\envs\\fastnlp\\lib\\site-packages\\tqdm\\autonotebook\\__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", " \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n" ] } @@ -37,26 +37,23 @@ "\n", "from fastNLP import DataSet\n", "\n", - "# linux_path = \"../test/data_for_tests/tutorial_sample_dataset.csv\"\n", - "win_path = \"../test/data_for_tests/tutorial_sample_dataset.csv\"\n", - "ds = DataSet.read_csv(win_path, headers=('raw_sentence', 'label'), sep='\\t')" + "data_path = \"./sample_data/tutorial_sample_dataset.csv\"\n", + "ds = DataSet.read_csv(data_path, headers=('raw_sentence', 'label'), sep='\\t')" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'raw_sentence': this quiet , introspective and entertaining independent is worth seeking .,\n", - "'label': 4,\n", - "'label_seq': 4,\n", - "'words': ['this', 'quiet', ',', 'introspective', 'and', 'entertaining', 'independent', 'is', 'worth', 'seeking', '.']}" + "{'raw_sentence': This quiet , introspective and entertaining independent is worth seeking . type=str,\n", + "'label': 4 type=str}" ] }, - "execution_count": 8, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -78,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -94,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -115,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -138,7 +135,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -156,33 +153,46 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "training epochs started 2018-12-07 14:03:41\n" + "input fields after batch(if batch size is 2):\n", + "\twords: (1)type:numpy.ndarray (2)dtype:object, (3)shape:(2,) \n", + "\tword_seq: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 25]) \n", + "target fields after batch(if batch size is 2):\n", + "\tlabel_seq: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2019-01-12 17-00-48\n" ] }, { "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "23979df0f63e446fbb0406b919b91dd3", + "version_major": 2, + "version_minor": 0 + }, "text/plain": [ "HBox(children=(IntProgress(value=0, layout=Layout(flex='2'), max=6), HTML(value='')), layout=Layout(display='i…" ] }, - "execution_count": 0, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch 1/3. Step:2/6. AccuracyMetric: acc=0.26087\n", - "Epoch 2/3. Step:4/6. AccuracyMetric: acc=0.347826\n", - "Epoch 3/3. Step:6/6. AccuracyMetric: acc=0.608696\n", + "Evaluation at Epoch 1/3. Step:2/6. AccuracyMetric: acc=0.173913\n", + "Evaluation at Epoch 2/3. Step:4/6. AccuracyMetric: acc=0.26087\n", + "Evaluation at Epoch 3/3. Step:6/6. AccuracyMetric: acc=0.304348\n", + "\n", + "In Epoch:3/Step:6, got best dev performance:AccuracyMetric: acc=0.304348\n", + "Reloaded the best model.\n", "Train finished!\n" ] } diff --git a/tutorials/fastnlp_advanced_tutorial.ipynb b/tutorials/fastnlp_advanced_tutorial.ipynb deleted file mode 100644 index c1322ab8..00000000 --- a/tutorials/fastnlp_advanced_tutorial.ipynb +++ /dev/null @@ -1,101 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "## FastNLP 进阶教程\n", - "本教程阅读时间平均30分钟" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 数据部分\n", - "### DataSet\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Instance" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Vocabulary" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 模型部分\n", - "### model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 训练测试部分\n", - "### Loss" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Metric" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Trainer" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Tester" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tutorials/fastnlp_advanced_tutorial/advance_tutorial.ipynb b/tutorials/fastnlp_advanced_tutorial/advance_tutorial.ipynb new file mode 100644 index 00000000..a787eeaf --- /dev/null +++ b/tutorials/fastnlp_advanced_tutorial/advance_tutorial.ipynb @@ -0,0 +1,1169 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# fastNLP开发进阶教程\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 组织数据部分\n", + "## DataSet & Instance\n", + "fastNLP用DataSet和Instance保存和处理数据。每个DataSet表示一个数据集，每个Instance表示一个数据样本。一个DataSet存有多个Instance，每个Instance可以自定义存哪些内容。" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/remote-home/ygxu/anaconda3/envs/no-fastnlp/lib/python3.7/site-packages/tqdm/autonotebook/__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", + " \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n" + ] + } + ], + "source": [ + "# 声明部件\n", + "import torch\n", + "import fastNLP\n", + "from fastNLP import DataSet\n", + "from fastNLP import Instance\n", + "from fastNLP import Vocabulary\n", + "from fastNLP import Trainer\n", + "from fastNLP import Tester" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Instance\n", + "Instance表示一个样本，由一个或者多个field（域、属性、特征）组成，每个field具有自己的名字以及值\n", + "在初始化Instance的时候可以定义它包含的field，使用\"field_name=field_value\"的写法" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'premise': an premise example . type=str,\n", + "'hypothesis': an hypothesis example. type=str,\n", + "'label': 1 type=int}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 组织一个Instance，这个Instance由premise、hypothesis、label三个field组成\n", + "instance = Instance(premise='an premise example .', hypothesis='an hypothesis example.', label=1)\n", + "instance" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'premise': an premise example . type=str,\n", + "'hypothesis': an hypothesis example. type=str,\n", + "'label': 1 type=int},\n", + "{'premise': an premise example . type=str,\n", + "'hypothesis': an hypothesis example. type=str,\n", + "'label': 1 type=int})" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_set = DataSet([instance] * 5)\n", + "data_set.append(instance)\n", + "data_set[-2: ]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'premise': an premise example . type=str,\n", + "'hypothesis': an hypothesis example. type=str,\n", + "'label': 1 type=int},\n", + "{'premise': the second premise example . type=str,\n", + "'hypothesis': the second hypothesis example. type=str,\n", + "'label': 1 type=str})" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 如果某一个field的类型与dataset对应的field类型不一样仍可被加入dataset中\n", + "instance2 = Instance(premise='the second premise example .', hypothesis='the second hypothesis example.', label='1')\n", + "try:\n", + " data_set.append(instance2)\n", + "except:\n", + " pass\n", + "data_set[-2: ]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cannot append instance\n" + ] + }, + { + "data": { + "text/plain": [ + "DataSet({'premise': an premise example . type=str,\n", + "'hypothesis': an hypothesis example. type=str,\n", + "'label': 1 type=int},\n", + "{'premise': the second premise example . type=str,\n", + "'hypothesis': the second hypothesis example. type=str,\n", + "'label': 1 type=str})" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 如果某一个field的名字不对，则该instance不能被append到dataset中\n", + "instance3 = Instance(premises='the third premise example .', hypothesis='the third hypothesis example.', label=1)\n", + "try:\n", + " data_set.append(instance3)\n", + "except:\n", + " print('cannot append instance')\n", + " pass\n", + "data_set[-2: ]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'image': tensor([[ 2.1747, -1.0147, -1.3853, 0.0216, -0.4957],\n", + " [ 0.8138, -0.2933, -0.1217, -0.6027, 0.3932],\n", + " [ 0.6750, -1.1136, -1.3371, -0.0185, -0.3206],\n", + " [-0.5076, -0.3822, 0.1719, -0.6447, -0.5702],\n", + " [ 0.3804, 0.0889, 0.8027, -0.7121, -0.7320]]) type=torch.Tensor,\n", + "'label': 0 type=int})" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 除了文本以外，还可以将tensor作为其中一个field的value\n", + "import torch\n", + "tensor_ins = Instance(image=torch.randn(5, 5), label=0)\n", + "ds = DataSet()\n", + "ds.append(tensor_ins)\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### DataSet\n", + "### 使用现有代码读取并组织DataSet\n", + "在DataSet类当中有一些read_* 方法，可以从文件中读取数据并组织DataSet" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "77" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import fastNLP\n", + "from fastNLP import DataSet\n", + "from fastNLP import Instance\n", + "\n", + "# 从csv读取数据到DataSet\n", + "# 类csv文件，即每一行为一个example的文件，都可以使用这种方法进行数据读取\n", + "dataset = DataSet.read_csv('tutorial_sample_dataset.csv', headers=('raw_sentence', 'label'), sep='\\t')\n", + "# 查看DataSet的大小\n", + "len(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'raw_sentence': A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . type=str,\n", + "'label': 1 type=str}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 使用数字索引[k]，获取第k个样本\n", + "dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "fastNLP.core.instance.Instance" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 获取的样本是一个Instance\n", + "type(dataset[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'raw_sentence': A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . type=str,\n", + "'label': 1 type=str},\n", + "{'raw_sentence': This quiet , introspective and entertaining independent is worth seeking . type=str,\n", + "'label': 4 type=str},\n", + "{'raw_sentence': Even fans of Ismail Merchant 's work , I suspect , would have a hard time sitting through this one . type=str,\n", + "'label': 1 type=str})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 使用数字索引[a: b]，获取第a到第b个样本\n", + "dataset[0: 3]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'raw_sentence': A film that clearly means to preach exclusively to the converted . type=str,\n", + "'label': 2 type=str}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 索引也可以是负数\n", + "dataset[-1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 自行读取并组织DataSet\n", + "以SNLI数据集为例，\n", + "SNLI数据集的训练、验证、测试集分别三个文件组成：第一个文件每一行是一句话，代表一个example当中的premise；第二个文件每一行也是一句话，代表一个example当中的hypothesis；第三个文件每一行是一个label" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'premise': A person on a horse jumps over a broken down airplane . type=str,\n", + "'hypothesis': A person is training his horse for a competition . type=str,\n", + "'truth': 1\n", + " type=str}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_path = ['premise', 'hypothesis', 'label']\n", + "\n", + "# 读入文件\n", + "with open(data_path[0]) as f:\n", + " premise = f.readlines()\n", + "\n", + "with open(data_path[1]) as f:\n", + " hypothesis = f.readlines()\n", + "\n", + "with open(data_path[2]) as f:\n", + " label = f.readlines()\n", + "\n", + "assert len(premise) == len(hypothesis) and len(hypothesis) == len(label)\n", + "\n", + "# 组织DataSet\n", + "data_set = DataSet()\n", + "for p, h, l in zip(premise, hypothesis, label):\n", + " p = p.strip() # 将行末空格去除\n", + " h = h.strip() # 将行末空格去除\n", + " data_set.append(Instance(premise=p, hypothesis=h, truth=l))\n", + "\n", + "data_set[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### DataSet的其他操作\n", + "在构建完毕DataSet后，仍然可以对DataSet的内容进行操作，函数接口为DataSet.apply()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'premise': a woman is walking across the street eating a banana , while a man is following with his briefcase . type=str,\n", + "'hypothesis': An actress and her favorite assistant talk a walk in the city . type=str,\n", + "'truth': 1\n", + " type=str},\n", + "{'premise': a woman is walking across the street eating a banana , while a man is following with his briefcase . type=str,\n", + "'hypothesis': a woman eating a banana crosses a street type=str,\n", + "'truth': 0\n", + " type=str})" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 将premise域的所有文本转成小写\n", + "data_set.apply(lambda x: x['premise'].lower(), new_field_name='premise')\n", + "data_set[-2: ]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'premise': a woman is walking across the street eating a banana , while a man is following with his briefcase . type=str,\n", + "'hypothesis': An actress and her favorite assistant talk a walk in the city . type=str,\n", + "'truth': 1 type=int},\n", + "{'premise': a woman is walking across the street eating a banana , while a man is following with his briefcase . type=str,\n", + "'hypothesis': a woman eating a banana crosses a street type=str,\n", + "'truth': 0 type=int})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# label转int\n", + "data_set.apply(lambda x: int(x['truth']), new_field_name='truth')\n", + "data_set[-2: ]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DataSet({'premise': ['a', 'woman', 'is', 'walking', 'across', 'the', 'street', 'eating', 'a', 'banana', ',', 'while', 'a', 'man', 'is', 'following', 'with', 'his', 'briefcase', '.'] type=list,\n", + "'hypothesis': ['An', 'actress', 'and', 'her', 'favorite', 'assistant', 'talk', 'a', 'walk', 'in', 'the', 'city', '.'] type=list,\n", + "'truth': 1 type=int},\n", + "{'premise': ['a', 'woman', 'is', 'walking', 'across', 'the', 'street', 'eating', 'a', 'banana', ',', 'while', 'a', 'man', 'is', 'following', 'with', 'his', 'briefcase', '.'] type=list,\n", + "'hypothesis': ['a', 'woman', 'eating', 'a', 'banana', 'crosses', 'a', 'street'] type=list,\n", + "'truth': 0 type=int})" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 使用空格分割句子\n", + "def split_sent(ins):\n", + " return ins['premise'].split()\n", + "data_set.apply(split_sent, new_field_name='premise')\n", + "data_set.apply(lambda x: x['hypothesis'].split(), new_field_name='hypothesis')\n", + "data_set[-2:]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(100, 97)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 筛选数据\n", + "origin_data_set_len = len(data_set)\n", + "data_set.drop(lambda x: len(x['premise']) <= 6)\n", + "origin_data_set_len, len(data_set)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'premise': ['a', 'woman', 'is', 'walking', 'across', 'the', 'street', 'eating', 'a', 'banana', ',', 'while', 'a', 'man', 'is', 'following', 'with', 'his', 'briefcase', '.'] type=list,\n", + "'hypothesis': ['a', 'woman', 'eating', 'a', 'banana', 'crosses', 'a', 'street'] type=list,\n", + "'truth': 0 type=int,\n", + "'premise_len': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + "'hypothesis_len': [1, 1, 1, 1, 1, 1, 1, 1] type=list}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 增加长度信息\n", + "data_set.apply(lambda x: [1] * len(x['premise']), new_field_name='premise_len')\n", + "data_set.apply(lambda x: [1] * len(x['hypothesis']), new_field_name='hypothesis_len')\n", + "data_set[-1]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "# 设定特征域、标签域\n", + "data_set.set_input(\"premise\", \"premise_len\", \"hypothesis\", \"hypothesis_len\")\n", + "data_set.set_target(\"truth\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'premise': ['a', 'woman', 'is', 'walking', 'across', 'the', 'street', 'eating', 'a', 'banana', ',', 'while', 'a', 'man', 'is', 'following', 'with', 'his', 'briefcase', '.'] type=list,\n", + "'hypothesis': ['a', 'woman', 'eating', 'a', 'banana', 'crosses', 'a', 'street'] type=list,\n", + "'premise_len': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + "'hypothesis_len': [1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + "'label': 0 type=int}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 重命名field\n", + "data_set.rename_field('truth', 'label')\n", + "data_set[-1]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(49, 29, 19)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 切分训练、验证集、测试集\n", + "train_data, vad_data = data_set.split(0.5)\n", + "dev_data, test_data = vad_data.split(0.4)\n", + "len(train_data), len(dev_data), len(test_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# 深拷贝一个数据集\n", + "import copy\n", + "train_data_2, dev_data_2 = copy.deepcopy(train_data), copy.deepcopy(dev_data)\n", + "del copy" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### DataSet的总结：\n", + "将DataSet的ield设置为input和target后，这些field在接下来的代码中将被使用。其中被设置为input的field会被传递给Model.forward，这个过程是通过键匹配的方式进行的。举例如下： \n", + "假设DataSet中有'x1', 'x2', 'x3'被设置为了input，而 \n", + "   (1)函数是Model.forward(self, x1, x3), 那么DataSet中'x1', 'x3'会被传递给forward函数。多余的'x2'会被忽略 \n", + "   (2)函数是Model.forward(self, x1, x4), 这里多需要了一个'x4', 但是DataSet的input field中没有这个field，会报错。 \n", + "   (3)函数是Model.forward(self, x1, kwargs), 会把'x1', 'x2', 'x3'都传入。但如果是Model.forward(self, x4, kwargs)就会发生报错，因为没有'x4'。 \n", + "   (4)对于设置为target的field的名称，我们建议取名为'target'（如果只有一个需要predict的值），但是不强制。如果这个名称不是target，那么在加载loss函数的时候需要手动添加名称转换map" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Vocabulary\n", + "fastNLP中的Vocabulary轻松构建词表，并将词转成数字。构建词表有两种方式：根据数据集构建词表；载入现有词表\n", + "### 根据数据集构建词表" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# 初始化词表，该词表最大的vocab_size为10000，词表中每个词出现的最低频率为2，''表示未知词语，''表示padding词语\n", + "# Vocabulary默认初始化参数为max_size=None, min_freq=None, unknown='', padding=''\n", + "vocab = Vocabulary(max_size=10000, min_freq=2, unknown='', padding='')\n", + "\n", + "# 构建词表\n", + "train_data.apply(lambda x: [vocab.add(word) for word in x['premise']])\n", + "train_data.apply(lambda x: [vocab.add(word) for word in x['hypothesis']])\n", + "vocab.build_vocab()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "({'premise': [2, 145, 146, 80, 147, 26, 148, 2, 104, 149, 150, 2, 151, 5, 55, 152, 105, 3] type=list,\n", + " 'hypothesis': [22, 80, 8, 1, 1, 20, 1, 3] type=list,\n", + " 'premise_len': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + " 'hypothesis_len': [1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + " 'label': 2 type=int},\n", + " {'premise': [11, 5, 18, 5, 24, 6, 2, 10, 59, 52, 14, 9, 2, 53, 29, 60, 54, 45, 6, 46, 5, 7, 61, 3] type=list,\n", + " 'hypothesis': [22, 11, 1, 45, 3] type=list,\n", + " 'premise_len': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + " 'hypothesis_len': [1, 1, 1, 1, 1] type=list,\n", + " 'label': 1 type=int},\n", + " {'premise': [2, 11, 8, 14, 16, 7, 15, 50, 2, 66, 4, 76, 2, 10, 8, 98, 9, 58, 67, 3] type=list,\n", + " 'hypothesis': [22, 27, 50, 3] type=list,\n", + " 'premise_len': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + " 'hypothesis_len': [1, 1, 1, 1] type=list,\n", + " 'label': 0 type=int})" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 根据词表index句子\n", + "train_data.apply(lambda x: [vocab.to_index(word) for word in x['premise']], new_field_name='premise')\n", + "train_data.apply(lambda x: [vocab.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis')\n", + "dev_data.apply(lambda x: [vocab.to_index(word) for word in x['premise']], new_field_name='premise')\n", + "dev_data.apply(lambda x: [vocab.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis')\n", + "test_data.apply(lambda x: [vocab.to_index(word) for word in x['premise']], new_field_name='premise')\n", + "test_data.apply(lambda x: [vocab.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis')\n", + "train_data[-1], dev_data[-1], test_data[-1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 载入现有词表\n", + "以BERT pretrained model为例，词表由一个vocab.txt文件来保存\n", + "用以下方法可以载入现有词表，并保证词表顺序不变" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "# 读入vocab文件\n", + "with open('vocab.txt') as f:\n", + " lines = f.readlines()\n", + "vocabs = []\n", + "for line in lines:\n", + " vocabs.append(line.strip())\n", + "\n", + "# 实例化Vocabulary\n", + "vocab_bert = Vocabulary(unknown=None, padding=None)\n", + "# 将vocabs列表加入Vocabulary\n", + "vocab_bert.add_word_lst(vocabs)\n", + "# 构建词表\n", + "vocab_bert.build_vocab()\n", + "# 更新unknown与padding的token文本\n", + "vocab_bert.unknown = '[UNK]'\n", + "vocab_bert.padding = '[PAD]'" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "({'premise': [1037, 2210, 2223, 2136, 5363, 2000, 4608, 1037, 5479, 8058, 2046, 1037, 2918, 1999, 2019, 5027, 2208, 1012] type=list,\n", + " 'hypothesis': [100, 2136, 2003, 2652, 3598, 2006, 100, 1012] type=list,\n", + " 'premise_len': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + " 'hypothesis_len': [1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + " 'label': 2 type=int},\n", + " {'premise': [2450, 1999, 2317, 1999, 100, 1998, 1037, 2158, 3621, 2369, 3788, 2007, 1037, 3696, 2005, 2198, 100, 10733, 1998, 100, 1999, 1996, 4281, 1012] type=list,\n", + " 'hypothesis': [100, 2450, 13063, 10733, 1012] type=list,\n", + " 'premise_len': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] type=list,\n", + " 'hypothesis_len': [1, 1, 1, 1, 1] type=list,\n", + " 'label': 1 type=int})" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 根据词表index句子\n", + "train_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['premise']], new_field_name='premise')\n", + "train_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis')\n", + "dev_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['premise']], new_field_name='premise')\n", + "dev_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis')\n", + "train_data_2[-1], dev_data_2[-1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 模型部分\n", + "## Model\n", + "模型部分fastNLP提供两种使用方式：调用fastNLP现有模型；开发者自行搭建模型\n", + "### 调用fastNLP现有模型" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'embed_dim': 300,\n", + " 'hidden_size': 300,\n", + " 'batch_first': True,\n", + " 'dropout': 0.3,\n", + " 'num_classes': 3,\n", + " 'gpu': True,\n", + " 'batch_size': 32,\n", + " 'vocab_size': 165}" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# step 1：加载模型参数（非必选）\n", + "from fastNLP.io.config_io import ConfigSection, ConfigLoader\n", + "args = ConfigSection()\n", + "ConfigLoader().load_config(\"./data/config\", {\"esim_model\": args})\n", + "args[\"vocab_size\"] = len(vocab)\n", + "args.data" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ESIM(\n", + " (drop): Dropout(p=0.3)\n", + " (embedding): Embedding(\n", + " (embed): Embedding(165, 300, padding_idx=0)\n", + " (dropout): Dropout(p=0.3)\n", + " )\n", + " (embedding_layer): Linear(\n", + " (linear): Linear(in_features=300, out_features=300, bias=True)\n", + " )\n", + " (encoder): LSTM(\n", + " (lstm): LSTM(300, 300, batch_first=True, bidirectional=True)\n", + " )\n", + " (bi_attention): Bi_Attention()\n", + " (mean_pooling): MeanPoolWithMask()\n", + " (max_pooling): MaxPoolWithMask()\n", + " (inference_layer): Linear(\n", + " (linear): Linear(in_features=1200, out_features=300, bias=True)\n", + " )\n", + " (decoder): LSTM(\n", + " (lstm): LSTM(300, 300, batch_first=True, bidirectional=True)\n", + " )\n", + " (output): MLP(\n", + " (hiddens): ModuleList(\n", + " (0): Linear(in_features=1200, out_features=300, bias=True)\n", + " )\n", + " (output): Linear(in_features=300, out_features=3, bias=True)\n", + " (dropout): Dropout(p=0.3)\n", + " (hidden_active): Tanh()\n", + " )\n", + ")" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# step 2：加载ESIM模型\n", + "from fastNLP.models import ESIM\n", + "model = ESIM(**args.data)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "CNNText(\n", + " (embed): Embedding(\n", + " (embed): Embedding(165, 50, padding_idx=0)\n", + " (dropout): Dropout(p=0.0)\n", + " )\n", + " (conv_pool): ConvMaxpool(\n", + " (convs): ModuleList(\n", + " (0): Conv1d(50, 3, kernel_size=(3,), stride=(1,), padding=(2,))\n", + " (1): Conv1d(50, 4, kernel_size=(4,), stride=(1,), padding=(2,))\n", + " (2): Conv1d(50, 5, kernel_size=(5,), stride=(1,), padding=(2,))\n", + " )\n", + " )\n", + " (dropout): Dropout(p=0.1)\n", + " (fc): Linear(\n", + " (linear): Linear(in_features=12, out_features=5, bias=True)\n", + " )\n", + ")" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 另一个例子：加载CNN文本分类模型\n", + "from fastNLP.models import CNNText\n", + "cnn_text_model = CNNText(embed_num=len(vocab), embed_dim=50, num_classes=5, padding=2, dropout=0.1)\n", + "cnn_text_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这是上述模型的forward方法。如果你不知道什么是forward方法，请参考我们的PyTorch教程。\n", + "\n", + "注意两点：\n", + "1. forward参数名字叫**word_seq**，请记住。\n", + "2. forward的返回值是一个**dict**，其中有个key的名字叫**pred**。\n", + "\n", + "```Python\n", + " def forward(self, word_seq):\n", + " \"\"\"\n", + "\n", + " :param word_seq: torch.LongTensor, [batch_size, seq_len]\n", + " :return output: dict of torch.LongTensor, [batch_size, num_classes]\n", + " \"\"\"\n", + " x = self.embed(word_seq) # [N,L] -> [N,L,C]\n", + " x = self.conv_pool(x) # [N,L,C] -> [N,C]\n", + " x = self.dropout(x)\n", + " x = self.fc(x) # [N,C] -> [N, N_class]\n", + " return {'pred': x}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 自行搭载模型\n", + "自行搭载的模型必须是nn.Module的子类， \n", + "(1)必须实现forward方法，并且forward方法不能出现**\\*arg**这种参数，例如\n", + "```Python\n", + " def forword(self, word_seq, *args): # 这是不允许的\n", + " xxx\n", + "```\n", + "forward函数的返回值必须是一个**dict**。 \n", + "dict当中模型预测的值所对应的key建议用**'pred'**，这里不做强制限制，但是如果不是pred的话，在加载loss函数的时候需要手动添加名称转换map \n", + "(2)如果实现了predict方法，在做evaluation的时候将调用predict方法而不是forward。如果没有predict方法，则在evaluation时调用forward方法。predict方法也不能使用\\*args这种参数形式，同时结果也必须返回一个dict，同样推荐key为'pred'。 \n", + "(3)forward函数可以计算loss并返回结果，在dict中的key为'loss'，如： \n", + "```Python\n", + " def forword(self, word_seq, *args): \n", + " xxx\n", + " return {'pred': pred, 'loss': loss}\n", + "```\n", + "当loss函数没有在trainer里被定义的时候，trainer将会根据forward函数返回的dict中key为'loss'的值来进行反向传播，具体见loss部分" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 训练测试部分\n", + "## Loss\n", + "### 键映射\n", + "根据上文DataSet与Model部分可以知道，fastNLP并不限制Model.forward()的返回值，也不限制DataSet中target field的key。因此在计算loss的时候，需要通过键映射的方式来完成取值。 \n", + "这里以CrossEntropyLoss为例，我们的交叉熵函数部分如下：\n", + "```Python\n", + " def get_loss(self, pred, target):\n", + " return F.cross_entropy(input=pred, target=target,\n", + " ignore_index=self.padding_idx)\n", + "```\n", + "这里接收的两个参数名字分别为pred和target，其中pred是从model的forward函数返回值中取得，target是从DataSet的is_target的field当中取得。在没有设置键映射的基础上，pred从model的forward函数返回的dict中取'pred'键得到；target从DataSet的'target'field中得到。\n", + "。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 修改键映射\n", + "在初始化CrossEntropyLoss的时候，可以传入两个参数(pred=None, target=None), 这两个参数接受的类型是str，假设(pred='output', target='label')，那么CrossEntropyLoss会使用'output'这个key在forward的output与batch_y中寻找值;'label'也是在forward的output与d ataset的is target field中寻找值。注意这里pred或target的来源并不一定非要来自于model.forward与dataset的is target field，也可以只来自于forward的结果" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 创建一个自己的loss\n", + "    (1)采用fastNLP.LossInForward，在model.forward()的结果中包含一个'loss'的key，具体见**自行搭载模型**部分 \n", + "    (2)自己定义一个继承于fastNLP.core.loss.LossBase的class。重写get_loss方法。 \n", + "      (2.1)在初始化自己的loss class的时候，需要初始化需要映射的值 \n", + "      (2.2)在get_loss函数中，参数的名字需要与初始化时的映射是一致的 \n", + "以L1Loss为例子：\n", + "```Python\n", + "class L1Loss(LossBase):\n", + " def __init__(self, pred=None, target=None):\n", + " super(L1Loss, self).__init__()\n", + " \"\"\"\n", + " 这里传入_init_param_map以使得pred和target被正确注册，但这一步不是必须的, 建议调用。传入_init_param_map的是用于\n", + " \"键映射\"的键值对。假设初始化__init__(pred=None, target=None, threshold=0.1)中threshold是用于控制loss计算的，\n", + " 则\\不要将threshold传入_init_param_map.\n", + " \"\"\"\n", + " self._init_param_map(pred=pred, target=target)\n", + "\n", + " def get_loss(self, pred, target):\n", + " # 这里'pred', 'target'必须和初始化的映射是一致的。\n", + " return F.l1_loss(input=pred, target=target)\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Trainer\n", + "trainer的作用是训练模型，是一个对训练过程的封装。trainer当中比较重要的函数是trainer.train()，train函数的主要步骤包括： \n", + "(1)创建batch \n", + "```Python\n", + "batch = Batch(dataset, batch_size, sampler=sampler)\n", + "```\n", + "(2)for batch_x, batch_y in batch: (batch_x, batch_y的内容分别为dataset中is input和is target的部分，这两个dict的key就是DataSet中的key，value会根据情况做好padding及tensor) \n", + "  (2.1)将batch_x, batch_y中的tensor移动到model所在的device \n", + "  (2.2)根据model.forward的参数列表，从batch_x中取出需要传递给forward的数据 \n", + "  (2.3)获取model.forward返回的dict，并与batch_y一起传递给loss函数，求得loss \n", + "  (2.4)对loss进行反向梯度传播并更新参数 \n", + "(3)如果有验证集，则进行验证 \n", + "(4)如果验证集的结果是当前最佳结果，则保存模型" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "除了以上的内容， Trainer中还提供了\"预跑\"的功能。该功能通过check_code_level管理，如果check_code_level为-1，则不进行\"预跑\"。 check_code_level=0，1，2代表不同的提醒级别。目前不同提醒级别对应的是对DataSet中设置为input或target但又没有使用的field的提醒级别。 0是忽略(默认)；1是会warning发生了未使用field的情况；2是出现了unused会直接报错并退出运行 \"预跑\"的主要目的有两个: \n", + "(1)防止train完了之后进行evaluation的时候出现错误。之前的train就白费了 \n", + "(2)由于存在\"键映射\"，直接运行导致的报错可能不太容易debug，通过\"预跑\"过程的报错会有一些debug提示 \"预跑\"会进行以下的操作： \n", + "  (i) 使用很小的batch_size, 检查batch_x中是否包含Model.forward所需要的参数。只会运行两个循环。 \n", + "  (ii)将Model.foward的输出pred_dict与batch_y输入到loss中，并尝试backward。不会更新参数，而且grad会被清零。如果传入了dev_data，还将进行metric的测试 \n", + "  (iii)创建Tester，并传入少量数据，检测是否可以正常运行 \n", + "\"预跑\"操作是在Trainer初始化的时候执行的。正常情况下，应该不需要改动\"预跑\"的代码。但如果遇到bug或者有什么好的建议，欢迎在开发群或者github提交issue。" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "training epochs started 2019-01-09 00-08-17\n", + "[tester] \n", + "AccuracyMetric: acc=0.206897\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/remote-home/ygxu/anaconda3/envs/no-fastnlp/lib/python3.7/site-packages/torch/nn/functional.py:1320: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n", + " warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tester] \n", + "AccuracyMetric: acc=0.206897\n", + "[tester] \n", + "AccuracyMetric: acc=0.206897\n", + "[tester] \n", + "AccuracyMetric: acc=0.206897\n", + "[tester] \n", + "AccuracyMetric: acc=0.206897\n", + "\n", + "In Epoch:1/Step:4, got best dev performance:AccuracyMetric: acc=0.206897\n", + "Reloaded the best model.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'best_eval': {'AccuracyMetric': {'acc': 0.206897}},\n", + " 'best_epoch': 1,\n", + " 'best_step': 4,\n", + " 'seconds': 0.79}" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from fastNLP import CrossEntropyLoss\n", + "from fastNLP import Adam\n", + "from fastNLP import AccuracyMetric\n", + "trainer = Trainer(\n", + " train_data=train_data,\n", + " model=model,\n", + " loss=CrossEntropyLoss(pred='pred', target='label'),\n", + " metrics=AccuracyMetric(),\n", + " n_epochs=5,\n", + " batch_size=16,\n", + " print_every=-1,\n", + " validate_every=-1,\n", + " dev_data=dev_data,\n", + " use_cuda=True,\n", + " optimizer=Adam(lr=1e-3, weight_decay=0),\n", + " check_code_level=-1,\n", + " metric_key='acc',\n", + " use_tqdm=False,\n", + ")\n", + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tester\n", + "tester的作用是训练模型，是一个对训练过程的封装。tester当中比较重要的函数是tester.test()，test函数的主要步骤包括：\n", + "(1)创建batch\n", + "```Python\n", + "batch = Batch(dataset, batch_size, sampler=sampler)\n", + "```\n", + "(2)for batch_x, batch_y in batch: (batch_x, batch_y的内容分别为dataset中is input和is target的部分，这两个dict的key就是DataSet中的key，value会根据情况做好padding及tensor) \n", + "  (2.1)同步数据与model将batch_x, batch_y中的tensor移动到model所在的device \n", + "  (2.2)根据predict_func的参数列表，从batch_x中取出需要传递给predict_func的数据，得到结果pred_dict \n", + "  (2.3)调用metric(pred_dict, batch_y) \n", + "  (2.4)当所有batch都运行完毕，会调用metric的get_metric方法，并且以返回的值作为evaluation的结果 " + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[tester] \n", + "AccuracyMetric: acc=0.263158\n" + ] + }, + { + "data": { + "text/plain": [ + "{'AccuracyMetric': {'acc': 0.263158}}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tester = Tester(\n", + " data=test_data,\n", + " model=model,\n", + " metrics=AccuracyMetric(),\n", + " batch_size=args[\"batch_size\"],\n", + ")\n", + "tester.test()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/fastnlp_advanced_tutorial/data/config b/tutorials/fastnlp_advanced_tutorial/data/config new file mode 100644 index 00000000..87348b72 --- /dev/null +++ b/tutorials/fastnlp_advanced_tutorial/data/config @@ -0,0 +1,8 @@ +[esim_model] +embed_dim = 300 +hidden_size = 300 +batch_first = true +dropout = 0.3 +num_classes = 3 +gpu = true +batch_size = 32 diff --git a/tutorials/fastnlp_advanced_tutorial/hypothesis b/tutorials/fastnlp_advanced_tutorial/hypothesis new file mode 100644 index 00000000..07aec3c9 --- /dev/null +++ b/tutorials/fastnlp_advanced_tutorial/hypothesis @@ -0,0 +1,100 @@ +A person is training his horse for a competition . +A person is at a diner , ordering an omelette . +A person is outdoors , on a horse . +They are smiling at their parents +There are children present +The kids are frowning +The boy skates down the sidewalk . +The boy does a skateboarding trick . +The boy is wearing safety equipment . +An older man drinks his juice as he waits for his daughter to get off work . +A boy flips a burger . +An elderly man sits in a small shop . +Some women are hugging on vacation . +The women are sleeping . +There are women showing affection . +The people are eating omelettes . +The people are sitting at desks in school . +The diners are at a restaurant . +A man is drinking juice . +Two women are at a restaurant drinking wine . +A man in a restaurant is waiting for his meal to arrive . +A blond man getting a drink of water from a fountain in the park . +A blond man wearing a brown shirt is reading a book on a bench in the park +A blond man drinking water from a fountain . +The friends scowl at each other over a full dinner table . +There are two woman in this picture . +The friends have just met for the first time in 20 years , and have had a great time catching up . +The two sisters saw each other across the crowded diner and shared a hug , both clutching their doggie bags . +Two groups of rival gang members flipped each other off . +Two women hug each other . +A team is trying to score the games winning out . +A team is trying to tag a runner out . +A team is playing baseball on Saturn . +A school hosts a basketball game . +A high school is hosting an event . +A school is hosting an event . +The women do not care what clothes they wear . +Women are waiting by a tram . +The women enjoy having a good fashion sense . +A child with mom and dad , on summer vacation at the beach . +A family of three is at the beach . +A family of three is at the mall shopping . +The people waiting on the train are sitting . +There are people just getting on a train +There are people waiting on a train . +A couple are playing with a young child outside . +A couple are playing frisbee with a young child at the beach . +A couple watch a little girl play by herself on the beach . +The family is sitting down for dinner . +The family is outside . +The family is on vacation . +The people are standing still on the curb . +Near a couple of restaurants , two people walk across the street . +The couple are walking across the street together . +The woman is nake . +The woman is cold . +The woman is wearing green . +The man with the sign is caucasian . +They are protesting outside the capital . +A woman in white . +A man is advertising for a restaurant . +The woman is wearing black . +A man and a woman walk down a crowded city street . +The woman is wearing white . +They are working for John 's Pizza . +Olympic swimming . +A man and a soman are eating together at John 's Pizza and Gyro . +They are walking with a sign . +The woman is waiting for a friend . +The man is sitting down while he has a sign for John 's Pizza and Gyro in his arms . +The woman and man are outdoors . +A woman ordering pizza . +The people are related . +Two adults run across the street to get away from a red shirted person chasing them . +The adults are both male and female . +Two people walk home after a tasty steak dinner . +Two adults swimming in water +Two adults walk across a street . +Two people ride bicycles into a tunnel . +Two people walk away from a restaurant across a street . +Two adults walking across a road near the convicted prisoner dressed in red +Two friends cross a street . +Some people board a train . +Two adults walk across the street . +Two adults walking across a road +There are no women in the picture . +Two adults walk across the street to get away from a red shirted person who is chasing them . +A married couple is sleeping . +A female is next to a man . +A married couple is walking next to each other . +Nobody has food . +A woman eats a banana and walks across a street , and there is a man trailing behind her . +The woman and man are playing baseball together . +two coworkers cross pathes on a street +A woman eats ice cream walking down the sidewalk , and there is another woman in front of her with a purse . +The mans briefcase is for work . +A person eating . +A person that is hungry . +An actress and her favorite assistant talk a walk in the city . +a woman eating a banana crosses a street diff --git a/tutorials/fastnlp_advanced_tutorial/label b/tutorials/fastnlp_advanced_tutorial/label new file mode 100644 index 00000000..e28836df --- /dev/null +++ b/tutorials/fastnlp_advanced_tutorial/label @@ -0,0 +1,100 @@ +1 +2 +0 +1 +0 +2 +2 +0 +1 +1 +2 +1 +1 +2 +0 +1 +2 +0 +0 +2 +1 +1 +2 +0 +2 +0 +1 +1 +2 +0 +1 +0 +2 +2 +1 +0 +2 +0 +1 +1 +0 +2 +1 +0 +0 +0 +1 +2 +2 +0 +1 +2 +0 +1 +2 +1 +0 +1 +2 +0 +0 +2 +1 +0 +1 +2 +2 +0 +1 +2 +0 +1 +1 +2 +0 +1 +2 +0 +2 +0 +1 +1 +2 +0 +0 +2 +1 +2 +0 +1 +2 +0 +2 +1 +2 +1 +0 +1 +1 +0 diff --git a/tutorials/fastnlp_advanced_tutorial/premise b/tutorials/fastnlp_advanced_tutorial/premise new file mode 100644 index 00000000..0c9af30e --- /dev/null +++ b/tutorials/fastnlp_advanced_tutorial/premise @@ -0,0 +1,100 @@ +A person on a horse jumps over a broken down airplane . +A person on a horse jumps over a broken down airplane . +A person on a horse jumps over a broken down airplane . +Children smiling and waving at camera +Children smiling and waving at camera +Children smiling and waving at camera +A boy is jumping on skateboard in the middle of a red bridge . +A boy is jumping on skateboard in the middle of a red bridge . +A boy is jumping on skateboard in the middle of a red bridge . +An older man sits with his orange juice at a small table in a coffee shop while employees in bright colored shirts smile in the background . +An older man sits with his orange juice at a small table in a coffee shop while employees in bright colored shirts smile in the background . +An older man sits with his orange juice at a small table in a coffee shop while employees in bright colored shirts smile in the background . +Two blond women are hugging one another . +Two blond women are hugging one another . +Two blond women are hugging one another . +A few people in a restaurant setting , one of them is drinking orange juice . +A few people in a restaurant setting , one of them is drinking orange juice . +A few people in a restaurant setting , one of them is drinking orange juice . +An older man is drinking orange juice at a restaurant . +An older man is drinking orange juice at a restaurant . +An older man is drinking orange juice at a restaurant . +A man with blond-hair , and a brown shirt drinking out of a public water fountain . +A man with blond-hair , and a brown shirt drinking out of a public water fountain . +A man with blond-hair , and a brown shirt drinking out of a public water fountain . +Two women who just had lunch hugging and saying goodbye . +Two women who just had lunch hugging and saying goodbye . +Two women who just had lunch hugging and saying goodbye . +Two women , holding food carryout containers , hug . +Two women , holding food carryout containers , hug . +Two women , holding food carryout containers , hug . +A Little League team tries to catch a runner sliding into a base in an afternoon game . +A Little League team tries to catch a runner sliding into a base in an afternoon game . +A Little League team tries to catch a runner sliding into a base in an afternoon game . +The school is having a special event in order to show the american culture on how other cultures are dealt with in parties . +The school is having a special event in order to show the american culture on how other cultures are dealt with in parties . +The school is having a special event in order to show the american culture on how other cultures are dealt with in parties . +High fashion ladies wait outside a tram beside a crowd of people in the city . +High fashion ladies wait outside a tram beside a crowd of people in the city . +High fashion ladies wait outside a tram beside a crowd of people in the city . +A man , woman , and child enjoying themselves on a beach . +A man , woman , and child enjoying themselves on a beach . +A man , woman , and child enjoying themselves on a beach . +People waiting to get on a train or just getting off . +People waiting to get on a train or just getting off . +People waiting to get on a train or just getting off . +A couple playing with a little boy on the beach . +A couple playing with a little boy on the beach . +A couple playing with a little boy on the beach . +A couple play in the tide with their young son . +A couple play in the tide with their young son . +A couple play in the tide with their young son . +A man and a woman cross the street in front of a pizza and gyro restaurant . +A man and a woman cross the street in front of a pizza and gyro restaurant . +A man and a woman cross the street in front of a pizza and gyro restaurant . +A woman in a green jacket and hood over her head looking towards a valley . +A woman in a green jacket and hood over her head looking towards a valley . +A woman in a green jacket and hood over her head looking towards a valley . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Woman in white in foreground and a man slightly behind walking with a sign for John 's Pizza and Gyro in the background . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +Two adults , one female in white , with shades and one male , gray clothes , walking across a street , away from a eatery with a blurred image of a dark colored red shirted person in the foreground . +A woman wearing all white and eating , walks next to a man holding a briefcase . +A woman wearing all white and eating , walks next to a man holding a briefcase . +A woman wearing all white and eating , walks next to a man holding a briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . +A woman is walking across the street eating a banana , while a man is following with his briefcase . diff --git a/tutorials/fastnlp_advanced_tutorial/tutorial_sample_dataset.csv b/tutorials/fastnlp_advanced_tutorial/tutorial_sample_dataset.csv new file mode 100644 index 00000000..e5c0a74f --- /dev/null +++ b/tutorials/fastnlp_advanced_tutorial/tutorial_sample_dataset.csv @@ -0,0 +1,77 @@ +A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . 1 +This quiet , introspective and entertaining independent is worth seeking . 4 +Even fans of Ismail Merchant 's work , I suspect , would have a hard time sitting through this one . 1 +A positively thrilling combination of ethnography and all the intrigue , betrayal , deceit and murder of a Shakespearean tragedy or a juicy soap opera . 3 +Aggressive self-glorification and a manipulative whitewash . 1 +A comedy-drama of nearly epic proportions rooted in a sincere performance by the title character undergoing midlife crisis . 4 +Narratively , Trouble Every Day is a plodding mess . 1 +The Importance of Being Earnest , so thick with wit it plays like a reading from Bartlett 's Familiar Quotations 3 +But it does n't leave you with much . 1 +You could hate it for the same reason . 1 +There 's little to recommend Snow Dogs , unless one considers cliched dialogue and perverse escapism a source of high hilarity . 1 +Kung Pow is Oedekerk 's realization of his childhood dream to be in a martial-arts flick , and proves that sometimes the dreams of youth should remain just that . 1 +The performances are an absolute joy . 4 +Fresnadillo has something serious to say about the ways in which extravagant chance can distort our perspective and throw us off the path of good sense . 3 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 +While The Importance of Being Earnest offers opportunities for occasional smiles and chuckles , it does n't give us a reason to be in the theater beyond Wilde 's wit and the actors ' performances . 1 +The latest vapid actor 's exercise to appropriate the structure of Arthur Schnitzler 's Reigen . 1 +More vaudeville show than well-constructed narrative , but on those terms it 's inoffensive and actually rather sweet . 2 +Nothing more than a run-of-the-mill action flick . 2 +Hampered -- no , paralyzed -- by a self-indulgent script ... that aims for poetry and ends up sounding like satire . 0 +Ice Age is the first computer-generated feature cartoon to feel like other movies , and that makes for some glacial pacing early on . 2 +There 's very little sense to what 's going on here , but the makers serve up the cliches with considerable dash . 2 +Cattaneo should have followed the runaway success of his first film , The Full Monty , with something different . 2 +They 're the unnamed , easily substitutable forces that serve as whatever terror the heroes of horror movies try to avoid . 1 +It almost feels as if the movie is more interested in entertaining itself than in amusing us . 1 +The movie 's progression into rambling incoherence gives new meaning to the phrase ` fatal script error . ' 0 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 \ No newline at end of file diff --git a/tutorials/fastnlp_advanced_tutorial/vocab.txt b/tutorials/fastnlp_advanced_tutorial/vocab.txt new file mode 100644 index 00000000..fb140275 --- /dev/null +++ b/tutorials/fastnlp_advanced_tutorial/vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/tutorials/sample_data/tutorial_sample_dataset.csv b/tutorials/sample_data/tutorial_sample_dataset.csv new file mode 100644 index 00000000..e5c0a74f --- /dev/null +++ b/tutorials/sample_data/tutorial_sample_dataset.csv @@ -0,0 +1,77 @@ +A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . 1 +This quiet , introspective and entertaining independent is worth seeking . 4 +Even fans of Ismail Merchant 's work , I suspect , would have a hard time sitting through this one . 1 +A positively thrilling combination of ethnography and all the intrigue , betrayal , deceit and murder of a Shakespearean tragedy or a juicy soap opera . 3 +Aggressive self-glorification and a manipulative whitewash . 1 +A comedy-drama of nearly epic proportions rooted in a sincere performance by the title character undergoing midlife crisis . 4 +Narratively , Trouble Every Day is a plodding mess . 1 +The Importance of Being Earnest , so thick with wit it plays like a reading from Bartlett 's Familiar Quotations 3 +But it does n't leave you with much . 1 +You could hate it for the same reason . 1 +There 's little to recommend Snow Dogs , unless one considers cliched dialogue and perverse escapism a source of high hilarity . 1 +Kung Pow is Oedekerk 's realization of his childhood dream to be in a martial-arts flick , and proves that sometimes the dreams of youth should remain just that . 1 +The performances are an absolute joy . 4 +Fresnadillo has something serious to say about the ways in which extravagant chance can distort our perspective and throw us off the path of good sense . 3 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 +While The Importance of Being Earnest offers opportunities for occasional smiles and chuckles , it does n't give us a reason to be in the theater beyond Wilde 's wit and the actors ' performances . 1 +The latest vapid actor 's exercise to appropriate the structure of Arthur Schnitzler 's Reigen . 1 +More vaudeville show than well-constructed narrative , but on those terms it 's inoffensive and actually rather sweet . 2 +Nothing more than a run-of-the-mill action flick . 2 +Hampered -- no , paralyzed -- by a self-indulgent script ... that aims for poetry and ends up sounding like satire . 0 +Ice Age is the first computer-generated feature cartoon to feel like other movies , and that makes for some glacial pacing early on . 2 +There 's very little sense to what 's going on here , but the makers serve up the cliches with considerable dash . 2 +Cattaneo should have followed the runaway success of his first film , The Full Monty , with something different . 2 +They 're the unnamed , easily substitutable forces that serve as whatever terror the heroes of horror movies try to avoid . 1 +It almost feels as if the movie is more interested in entertaining itself than in amusing us . 1 +The movie 's progression into rambling incoherence gives new meaning to the phrase ` fatal script error . ' 0 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 +I still like Moonlight Mile , better judgment be damned . 3 +A welcome relief from baseball movies that try too hard to be mythic , this one is a sweet and modest and ultimately winning story . 3 +a bilingual charmer , just like the woman who inspired it 3 +Like a less dizzily gorgeous companion to Mr. Wong 's In the Mood for Love -- very much a Hong Kong movie despite its mainland setting . 2 +As inept as big-screen remakes of The Avengers and The Wild Wild West . 1 +It 's everything you 'd expect -- but nothing more . 2 +Best indie of the year , so far . 4 +Hatfield and Hicks make the oddest of couples , and in this sense the movie becomes a study of the gambles of the publishing world , offering a case study that exists apart from all the movie 's political ramifications . 3 +It 's like going to a house party and watching the host defend himself against a frothing ex-girlfriend . 1 +That the Chuck Norris `` grenade gag '' occurs about 7 times during Windtalkers is a good indication of how serious-minded the film is . 2 +The plot is romantic comedy boilerplate from start to finish . 2 +It arrives with an impeccable pedigree , mongrel pep , and almost indecipherable plot complications . 2 +A film that clearly means to preach exclusively to the converted . 2 \ No newline at end of file diff --git a/tutorials/tutorial_for_developer.md b/tutorials/tutorial_for_developer.md new file mode 100644 index 00000000..b2ec0b98 --- /dev/null +++ b/tutorials/tutorial_for_developer.md @@ -0,0 +1,283 @@ +# fastNLP开发者指南 +#### 本教程涉及以下类： +- DataSet +- Sampler +- Batch +- Model +- Loss +- Metric +- Trainer +- Tester + +#### DataSet: 用于承载数据。 +1. DataSet里面每个元素只能是以下的三类`np.float64`, `np.int64`, `np.str`。如果传入的数据是`int`则被转换为`np.int64`, `float`被转为`np.float64`。 +2. DataSet可以将field设置为input或者target。其中被设置为input的field会被传递给Model.forward, 这个过程中我们是通过键匹配完成传递的。举例来说，假设DataSet中有'x1', 'x2', 'x3'被设置为了input，而 + - 函数是Model.forward(self, x1, x3), 那么DataSet中'x1', 'x3'会被传递给forward函数。多余的'x2'会被忽略 + - 函数是Model.forward(self, x1, x4), 这里多需要了一个'x4', 但是DataSet的input field中没有这个field，会报错。 + - 函数是Model.forward(self, x1, **kwargs), 会把'x1', 'x2', 'x3'都传入。但如果是Model.forward(self, x4, **kwargs)就会发生报错，因为没有'x4'。 +3. 对于设置为target的field的名称，我们建议取名为'target'（如果只有一个需要predict的值），但是不强制。后面会讲为什么target可以不强制。 +DataSet应该是不需要单独再开发的，如果有不能满足的场景，请在开发群提出或者github提交issue。 + +#### Sampler: 给定一个DataSet，返回一个序号的list，Batch按照这个list输出数据。 +Sampler需要继承fastNLP.core.sampler.BaseSampler +```python +class BaseSampler(object): + """The base class of all samplers. + + Sub-classes must implement the __call__ method. + __call__ takes a DataSet object and returns a list of int - the sampling indices. + """ +def __call__(self, *args, **kwargs): + raise NotImplementedError + +# 子类需要复写__call__方法。这个函数只能有一个必选参数, 且必须是DataSet类别， 否则Trainer没法调 +class SonSampler(BaseSampler): + def __init__(self, xxx): + # 可以实现init也不可以不实现。 + pass + def __call__(self, data_set): + pass +``` + +#### Batch: 将DataSet中设置为input和target的field取出来构成batch_x, batch_y +并且根据情况(主要根据数据类型能不能转为Tensor)将数据转换为pytorch的Tensor。batch中sample的取出顺序是由Sampler决定的。 +Sampler是传入一个DataSet，返回一个与DataSet等长的序号list，Batch一次会取出batch_size个sample(最后一个batch可能数量不足batch_size个)。 +举例： +1. SequentialSampler是顺序采样 + + 假设传入的DataSet长度是100, SequentialSampler返回的序号list就是[0, 1, ...,98, 99]. batch_size如果被设置为4，那么第一个batch所获取的instance就是[0, 1, 2, 3]这四个instance. 第二个batch所获取instace就是[4, 5, 6, 7], ...直到采完所有的sample。 +2. RandomSampler是随机采样 + + 假设传入的DataSet长度是100, RandomSampler返回的序号list可能是[0, 99, 20, 5, 3, 1, ...]. 依次按照batch_size的大小取出sample。 + +Batch应该不需要继承与开发，如果你有特殊需求请在开发群里提出。 + +#### Model：用户自定的Model +必须是nn.Module的子类 +1. 必须实现forward方法，并且forward方法不能出现*arg这种参数. 例如 + ```python + def forward(self, word_seq, *args): #这是不允许的. + # ... + pass + ``` + 返回值必须是dict的 + ```python + def forward(self, word_seq, seq_lens): + xxx = "xxx" + return {'pred': xxx} #return的值必须是dict的。里面的预测的key推荐使用pred，但是不做强制限制。输出元素数目不限。 + ``` +2. 如果实现了predict方法，在做evaluation的时候将调用predict方法而不是forward。如果没有predict方法，则在evaluation时调用forward方法。predict方法也不能使用*args这种参数形式，同时结果也必须返回一个dict，同样推荐key为'pred'。 + +#### Loss: 根据model.forward()返回的prediction(是一个dict)和batch_y计算相应的loss +1. 先介绍"键映射"。 如在DataSet, Model一节所看见的那样，fastNLP并不限制Model.forward()的返回值，也不限制DataSet中target field的key。计算的loss的时候，怎么才能知道从哪里取值呢？ +这里以CrossEntropyLoss为例，一般情况下, 计算CrossEntropy需要prediction和target两个值。而在CrossEntropyLoss初始化时可以传入两个参数(pred=None, target=None), 这两个参数接受的类型是str，假设(pred='output', target='label')，那么CrossEntropyLoss会使用'output'这个key在forward的output与batch_y中寻找值;'label'也是在forward的output与batch_y中寻找值。注意这里pred或target的来源并不一定非要来自于model.forward与batch_y，也可以只来自于forward的结果。 +2. 如何创建一个自己的loss + - 使用fastNLP.LossInForward, 在model.forward()的结果中包含一个为loss的key。 + - trainer中使用loss(假设loss=CrossEntropyLoss())的时候其实是 + los = loss(prediction, batch_y)，即直接调用的是`loss.__call__()`方法，但是CrossEntropyLoss里面并没有自己实现`__call__`方法，这是因为`__call__`在LossBase中实现了。所有的loss必须继承fastNLP.core.loss.LossBase, 下面先说一下LossBase的几个方法，见下一节。 +3. 尽量不要复写`__call__()`, `_init_param_map()`方法。 + +```python +class LossBase(): + def __init__(self): + self.param_map = {} # 一般情况下也不需要自己创建。调用_init_param_map()更好 + self._checked = False # 这个参数可以忽略 + + def _init_param_map(self, key_map=None, **kwargs): + # 这个函数是用于注册Loss的“键映射”，有两种传值方法， + # 第一种是通过key_map传入dict，取值是用value到forward和batch_y取 + # key_map = {'pred': 'output', 'target': 'label'} + # 第二种是自己写 + # _init_param_map(pred='output', target='label') + # 为什么会提供这么一个方法？通过调用这个方法会自动注册param_map，并会做一些检查，防止出现传入的key其实并不是get_loss + # 的一个参数。注意传入这个方法的参数必须都是需要做键映射的内容，其它loss参数不要传入。如果传入(pred=None, target=None) + # 则__call__()会到pred_dict与target_dict去寻找key为'pred'和'target'的值。 + # 但这个参数不是必须要调用的。 + + def __call__(self, pred_dict, target_dict, check=False): # check=False忽略这个参数，之后应该会被删除的 + # 这个函数主要会做一些check的工作，比如pred_dict与target_dict中是否包含了计算loss所必须的key等。检查通过，则调用get_loss + # 方法。 + fast_param = self._fast_param_map(predict_dict, target_dict): + if fast_param: + return self.get_loss(**fast_param) + # 如果没有fast_param则通过匹配参数然后调用get_loss完成 + xxxx + return loss # 返回为Tensor的loss + def _fast_param_map(self, pred_dict, target_dict): + # 这是一种快速计算loss的机制，因为在很多情况下其实都不需要通过"键映射"，比如计算loss时，pred_dict只有一个元素， + # target_dict也只有一个元素，那么无歧义地就可以把预测值与实际值用于计算loss, 基类判断了这种情况(可能还有其它无歧义的情况)。 + # 即_fast_param_map成功的话，就不需要使用键映射，这样即使在没有传递或者传递错误"键映射"的情况也可以直接计算loss。 + # 返回值是一个dict, 如果匹配成功，应该返回类似{'pred':value, 'target': value}的结果；如果dict为空则说明匹配失败， + # __call__方法会继续执行。 + + def get_loss(self, *args, **kwargs): + # 这个是一定需要实现的，计算loss的地方。 + # (1) get_loss中一定不能包含*arg这种参数形式。 + # (2) 如果包含**kwargs这种参数，这会将pred_dict与target_dict中所有参数传入。但是建议不要用这个参数 + raise NotImplementedError + +# 下面使用L1Loss举例 +class L1Loss(LossBase): # 继承LossBase + # 初始化需要映射的值，这里需要映射的值'pred', 'target'必须与get_loss需要参数名是对应的 + def __init__(self, pred=None, target=None): + super(L1Loss, self).__init__() + # 这里传入_init_param_map以使得pred和target被正确注册，但这一步不是必须的, 建议调用。传入_init_param_map的是用于 + # “键映射"的键值对。假设初始化__init__(pred=None, target=None, threshold=0.1)中threshold是用于控制loss计算的，则 + # 不要将threshold传入_init_param_map. + self._init_param_map(pred=pred, target=target) + + def get_loss(self, pred, target): + # 这里'pred', 'target'必须和初始化的映射是一致的。 + return F.l1_loss(input=pred, target=target) #直接返回一个loss即可 +``` + +### Metric: 根据Model.forward()或者Model.predict()的结果计算metric +metric的设计和loss的设计类似。都是传入pred_dict与target_dict进行计算。但是metric的pred_dict来源可能是Model.forward的返回值， 也可能是Model.predict(如果Model具有predict方法则会调用predict方法)的返回值，下面统一用pred_dict代替。 +1. 这里的"键映射"与loss的"键映射"是类似的。举例来说，若Metric(pred='output', target='label')，则使用'output'到pred_dict和target_dict中寻找pred, 用'label'寻找target。 +2. 如何创建一个自己的Metric方法 +Metric与loss的计算不同在于，Metric的计算有两个步骤。 + - **每个batch的输出**都会调用Metric的``__call__(pred_dict, target_dict)``方法，而``__call__``方法会调用evaluate()(需要实现)方法。 + - 在所有batch传入之后，调用Metric的get_metric()方法得到最终的metric值。 + - 所以Metric在调用evaluate方法时，根据拿到的数据: pred_dict与batch_y, 改变自己的状态(比如累加正确的次数，总的sample数等)。在调用get_metric()的时候给出一个最终计算结果。 + 所有的Metric必须继承自fastNLP.core.metrics.MetricBase. 例子见下一个cell +3. 尽量不要复写``__call__()``,``_init_param_map()``方法。 + +```python +class MetricBase: + def __init__(self): + self.param_map = {} # 一般情况下也不需要自己创建。调用_init_param_map()更好 + self._checked = False # 这个参数可以忽略 + + def _init_param_map(self, key_map=None, **kwargs): + # 这个函数是用于注册Metric的“键映射”，有两种传值方法， + # 第一种是通过key_map传入dict，取值是用value到forward和batch_y取 + # key_map = {'pred': 'output', 'target': 'label'} + # 第二种是自己写(建议使用改种方式) + # _init_param_map(pred='output', target='label') + # 为什么会提供这么一个方法？通过调用这个方法会自动注册param_map，并会做一些检查，防止出现传入的key其实并不是evaluate() + # 的一个参数。注意传入这个方法的参数必须都是需要做键映射的内容，其它evaluate参数不要传入。如果传入(pred=None, target=None) + # 则__call__()会到pred_dict与target_dict去寻找key为'pred'和'target'的值。 + # 但这个参数不是必须要调用的。 + pass + + def __call__(self, pred_dict, target_dict, check=False): # check=False忽略这个参数，之后应该会被删除的 + # 这个函数主要会做一些check的工作，比如pred_dict与target_dict中是否包含了计算evaluate所必须的key等。检查通过，则调用 + # evaluate方法。 + fast_param = self._fast_param_map(predict_dict, target_dict): + if fast_param: + return self.evaluate(**fast_param) + # 如果没有fast_param则通过匹配参数然后调用get_loss完成 + # xxxx + + def _fast_param_map(self, pred_dict, target_dict): + # 这是一种快速计算loss的机制，因为在很多情况下其实都不需要通过"键映射"，比如evaluate时，pred_dict只有一个元素， + # target_dict也只有一个元素，那么无歧义地就可以把预测值与实际值用于计算metric, 基类判断了这种情况(可能还有其它无歧义的 + # 情况)。即_fast_param_map成功的话，就不需要使用键映射，这样即使在没有传递或者传递错误"键映射"的情况也可以直接计算metric。 + # 返回值是一个dict, 如果匹配成功，应该返回类似{'pred':value, 'target': value}的结果；如果dict为空则说明匹配失败， + # __call__方法会继续尝试匹配。 + pass + + def evaluate(self, *args, **kwargs): + # 这个是一定需要实现的，累加metric状态 + # (1) evaluate()中一定不能包含*arg这种参数形式。 + # (2) 如果包含**kwargs这种参数，这会将pred_dict与target_dict中所有参数传入。但是建议不要用这个参数 + raise NotImplementedError + + def get_metric(self, reset=True): + # 这是一定需要实现的，获取最终的metric。返回值必须是一个dict。会在所有batch传入之后调用 + raise NotImplementedError + +# 下面使用AccuracyMetric举例 +class AccuracyMetric(MetricBase): # MetricBase + # 初始化需要映射的值，这里需要映射的值'pred', 'target'必须与evaluate()需要参数名是对应的 + def __init__(self, pred=None, target=None): + super(AccuracyMetric, self).__init__() + # 这里传入_init_param_map以使得pred和target被正确注册，但这一步不是必须的, 建议调用。传入_init_param_map的是用于 + # “键映射"的键值对。假设初始化__init__(pred=None, target=None, threshold=0.1)中threshold是用于控制loss计算的，则 + # 不要将threshold传入_init_param_map. + self._init_param_map(pred=pred, target=target) + + self.total = 0 # 用于累加一共有多少sample + self.corr = 0 # 用于累加一共有多少正确的sample + + def evaluate(self, pred, target): + # 对pred和target做一些基本的判断或者预处理等 + if pred.size()==target.size() and len(pred.size())=1: #如果pred已经做了argmax + pass + elif len(pred.size())==2 and len(target.size())==1: # pred还没有进行argmax + pred = pred.argmax(dim=1) + else: + raise ValueError("The shape of pred and target should be ((B, n_classes), (B, )) or (" + "(B,),(B,)).") + assert pred.size(0)==target.size(0), "Mismatch batch size." + # 进行相应的累加 + self.total += pred.size(0) + self.corr += torch.sum(torch.eq(pred, target).float()).item() + + def get_metric(self, reset=True): + # reset用于指示是否清空累加信息。默认为True + # 这个函数需要返回dict，可以包含多个metric。 + metric = {} + metric['acc'] = self.corr/self.total + if reset: + self.total = 0 + self.corr = 0 + return metric +``` + +#### Tester: 用于做evaluation，应该不需要更改 +重要的初始化参数有data, model, metric；比较重要的function是test()。 + +test中的运行过程 +``` +predict_func = 如果有model.predict则为model.predict, 否则是model.forward +for batch_x, batch_y in batch: +# (1) 同步数据与model +# (2) 根据predict_func的参数从batch_x中取出数据传入到predict_func中，得到结果pred_dict +# (3) 调用metric(pred_dict, batch_y +# (4) 当所有batch都运行完毕，会调用metric的get_metric方法，并且以返回的值作为evaluation的结果 +metric.get_metric() +``` + +#### Trainer: 对训练过程的封装。 +里面比较重要的function是train() +train()中的运行过程 +``` +(1) 创建batch + batch = Batch(dataset, batch_size, sampler=sampler) + for batch_x, batch_y in batch: + # ... + batch_x，batch_y都是dict。batch_x是DataSet中被设置为input的field；batch_y是DataSet中被设置为target的field。 + 两个dict中的key就是DataSet中的key，value会根据情况做好padding的tensor。 +(2)会将batch_x, batch_y中tensor移动到model所在的device +(3)根据model.forward的参数列表, 从batch_x中取出需要传递给forward的数据。 +(4)获取model.forward的输出结果pred_dict，并与batch_y一起传递给loss函数, 求得loss +(5)对loss进行反向梯度并更新参数 +(6) 如果有验证集，则需要做验证 + tester = Tester(model, dev_data，metric) + eval_results = tester.test() +(7) 如果eval_results是当前的最佳结果，则保存模型。 +``` + +#### 其他 +Trainer中还提供了"预跑"的功能。该功能通过check_code_level管理，如果check_code_level为-1，则不进行"预跑"。 + +check_code_level=0，1，2代表不同的提醒级别。 +目前不同提醒级别对应的是对DataSet中设置为input或target但又没有使用的field的提醒级别。 +0是忽略(默认)；1是会warning发生了未使用field的情况；2是出现了unused会直接报错并退出运行 + +"预跑"的主要目的有两个: +- 防止train完了之后进行evaluation的时候出现错误。之前的train就白费了 +- 由于存在"键映射"，直接运行导致的报错可能不太容易debug，通过"预跑"过程的报错会有一些debug提示 + +"预跑"会进行以下的操作： +- 使用很小的batch_size, 检查batch_x中是否包含Model.forward所需要的参数。只会运行两个循环。 +- 将Model.foward的输出pred_dict与batch_y输入到loss中， 并尝试backward. 不会更新参数，而且grad会被清零 + 如果传入了dev_data，还将进行metric的测试 +- 创建Tester，并传入少量数据，检测是否可以正常运行 + +"预跑"操作是在Trainer初始化的时候执行的。 + +正常情况下，应该不需要改动"预跑"的代码。但如果你遇到bug或者有什么好的建议，欢迎在开发群或者github提交issue。 + +