From c4ba75d160c508123ce536df98a5ccbea2ed5ad9 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Tue, 15 Jan 2019 14:30:37 +0800 Subject: [PATCH] code optimization * move used readers from reproduction to io/dataset_loader.py (API shall not call anything from reproduction/) --- fastNLP/api/api.py | 14 +- fastNLP/api/processor.py | 100 +++++ fastNLP/io/dataset_loader.py | 373 ++++++++++++++++++ reproduction/Biaffine_parser/main.py | 2 +- reproduction/Biaffine_parser/run.py | 7 +- reproduction/Biaffine_parser/util.py | 51 --- .../chinese_word_segment/cws_io/cws_reader.py | 194 --------- .../process/cws_processor.py | 103 ----- reproduction/pos_tag_model/pos_reader.py | 126 +----- reproduction/pos_tag_model/train_pos_tag.py | 5 +- 10 files changed, 489 insertions(+), 486 deletions(-) diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index 8368dcc9..b9bc7b70 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -9,9 +9,7 @@ from fastNLP.core.dataset import DataSet from fastNLP.api.utils import load_url from fastNLP.api.processor import ModelProcessor -from reproduction.chinese_word_segment.cws_io.cws_reader import ConllCWSReader -from reproduction.pos_tag_model.pos_reader import ZhConllPOSReader -from reproduction.Biaffine_parser.util import ConllxDataLoader, add_seg_tag +from fastNLP.io.dataset_loader import ConllCWSReader, ZhConllPOSReader, ConllxDataLoader, add_seg_tag from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.core.metrics import SpanFPreRecMetric @@ -31,6 +29,16 @@ class API: self._dict = None def predict(self, *args, **kwargs): + """Do prediction for the given input. + """ + raise NotImplementedError + + def test(self, file_path): + """Test performance over the given data set. + + :param str file_path: + :return: a dictionary of metric values + """ raise NotImplementedError def load(self, path, device): diff --git a/fastNLP/api/processor.py b/fastNLP/api/processor.py index 7354fe0f..6867dae8 100644 --- a/fastNLP/api/processor.py +++ b/fastNLP/api/processor.py @@ -322,3 +322,103 @@ class SetInputProcessor(Processor): def process(self, dataset): dataset.set_input(*self.fields, flag=self.flag) return dataset + + +class VocabIndexerProcessor(Processor): + """ + 根据DataSet创建Vocabulary，并将其用数字index。新生成的index的field会被放在new_added_filed_name, 如果没有提供 + new_added_field_name, 则覆盖原有的field_name. + + """ + + def __init__(self, field_name, new_added_filed_name=None, min_freq=1, max_size=None, + verbose=0, is_input=True): + """ + + :param field_name: 从哪个field_name创建词表，以及对哪个field_name进行index操作 + :param new_added_filed_name: index时，生成的index field的名称，如果不传入，则覆盖field_name. + :param min_freq: 创建的Vocabulary允许的单词最少出现次数. + :param max_size: 创建的Vocabulary允许的最大的单词数量 + :param verbose: 0, 不输出任何信息；1，输出信息 + :param bool is_input: + """ + super(VocabIndexerProcessor, self).__init__(field_name, new_added_filed_name) + self.min_freq = min_freq + self.max_size = max_size + + self.verbose = verbose + self.is_input = is_input + + def construct_vocab(self, *datasets): + """ + 使用传入的DataSet创建vocabulary + + :param datasets: DataSet类型的数据，用于构建vocabulary + :return: + """ + self.vocab = Vocabulary(min_freq=self.min_freq, max_size=self.max_size) + for dataset in datasets: + assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) + dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) + self.vocab.build_vocab() + if self.verbose: + print("Vocabulary Constructed, has {} items.".format(len(self.vocab))) + + def process(self, *datasets, only_index_dataset=None): + """ + 若还未建立Vocabulary，则使用dataset中的DataSet建立vocabulary；若已经有了vocabulary则使用已有的vocabulary。得到vocabulary + 后，则会index datasets与only_index_dataset。 + + :param datasets: DataSet类型的数据 + :param only_index_dataset: DataSet, or list of DataSet. 该参数中的内容只会被用于index，不会被用于生成vocabulary。 + :return: + """ + if len(datasets) == 0 and not hasattr(self, 'vocab'): + raise RuntimeError("You have to construct vocabulary first. Or you have to pass datasets to construct it.") + if not hasattr(self, 'vocab'): + self.construct_vocab(*datasets) + else: + if self.verbose: + print("Using constructed vocabulary with {} items.".format(len(self.vocab))) + to_index_datasets = [] + if len(datasets) != 0: + for dataset in datasets: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + to_index_datasets.append(dataset) + + if not (only_index_dataset is None): + if isinstance(only_index_dataset, list): + for dataset in only_index_dataset: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + to_index_datasets.append(dataset) + elif isinstance(only_index_dataset, DataSet): + to_index_datasets.append(only_index_dataset) + else: + raise TypeError('Only DataSet or list of DataSet is allowed, not {}.'.format(type(only_index_dataset))) + + for dataset in to_index_datasets: + assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) + dataset.apply(lambda ins: [self.vocab.to_index(token) for token in ins[self.field_name]], + new_field_name=self.new_added_field_name, is_input=self.is_input) + # 只返回一个，infer时为了跟其他processor保持一致 + if len(to_index_datasets) == 1: + return to_index_datasets[0] + + def set_vocab(self, vocab): + assert isinstance(vocab, Vocabulary), "Only fastNLP.core.Vocabulary is allowed, not {}.".format(type(vocab)) + self.vocab = vocab + + def delete_vocab(self): + del self.vocab + + def get_vocab_size(self): + return len(self.vocab) + + def set_verbose(self, verbose): + """ + 设置processor verbose状态。 + + :param verbose: int, 0，不输出任何信息；1，输出vocab 信息。 + :return: + """ + self.verbose = verbose diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index 27d8a360..2d157da3 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -90,6 +90,7 @@ class NativeDataSetLoader(DataSetLoader): """A simple example of DataSetLoader """ + def __init__(self): super(NativeDataSetLoader, self).__init__() @@ -107,6 +108,7 @@ class RawDataSetLoader(DataSetLoader): """A simple example of raw data reader """ + def __init__(self): super(RawDataSetLoader, self).__init__() @@ -142,6 +144,7 @@ class POSDataSetLoader(DataSetLoader): In this example, there are two sentences "Tom and Jerry ." and "Hello world !". Each word has its own label. """ + def __init__(self): super(POSDataSetLoader, self).__init__() @@ -540,3 +543,373 @@ class SNLIDataSetLoader(DataSetLoader): data_set.set_input("premise", "hypothesis", "premise_len", "hypothesis_len") data_set.set_target("truth") return data_set + + +class ConllCWSReader(object): + def __init__(self): + pass + + def load(self, path, cut_long_sent=False): + """ + 返回的DataSet只包含raw_sentence这个field，内容为str。 + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_char_lst(sample) + if res is None: + continue + line = ' '.join(res) + if cut_long_sent: + sents = cut_long_sentence(line) + else: + sents = [line] + for raw_sentence in sents: + ds.append(Instance(raw_sentence=raw_sentence)) + + return ds + + def get_char_lst(self, sample): + if len(sample) == 0: + return None + text = [] + for w in sample: + t1, t2, t3, t4 = w[1], w[3], w[6], w[7] + if t3 == '_': + return None + text.append(t1) + return text + + +class POSCWSReader(DataSetLoader): + """ + 支持读取以下的情况, 即每一行是一个词, 用空行作为两句话的界限. + 迈 N + 向 N + 充 N + ... + 泽 I-PER + 民 I-PER + + （ N + 一 N + 九 N + ... + + + :param filepath: + :return: + """ + + def __init__(self, in_word_splitter=None): + super().__init__() + self.in_word_splitter = in_word_splitter + + def load(self, filepath, in_word_splitter=None, cut_long_sent=False): + if in_word_splitter is None: + in_word_splitter = self.in_word_splitter + dataset = DataSet() + with open(filepath, 'r') as f: + words = [] + for line in f: + line = line.strip() + if len(line) == 0: # new line + if len(words) == 0: # 不能接受空行 + continue + line = ' '.join(words) + if cut_long_sent: + sents = cut_long_sentence(line) + else: + sents = [line] + for sent in sents: + instance = Instance(raw_sentence=sent) + dataset.append(instance) + words = [] + else: + line = line.split()[0] + if in_word_splitter is None: + words.append(line) + else: + words.append(line.split(in_word_splitter)[0]) + return dataset + + +class NaiveCWSReader(DataSetLoader): + """ + 这个reader假设了分词数据集为以下形式, 即已经用空格分割好内容了 + 这是 fastNLP , 一个 非常 good 的 包 . + 或者,即每个part后面还有一个pos tag + 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY + """ + + def __init__(self, in_word_splitter=None): + super().__init__() + + self.in_word_splitter = in_word_splitter + + def load(self, filepath, in_word_splitter=None, cut_long_sent=False): + """ + 允许使用的情况有(默认以\t或空格作为seg) + 这是 fastNLP , 一个 非常 good 的 包 . + 和 + 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY + 如果splitter不为None则认为是第二种情况, 且我们会按splitter分割"也/D", 然后取第一部分. 例如"也/D".split('/')[0] + :param filepath: + :param in_word_splitter: + :return: + """ + if in_word_splitter == None: + in_word_splitter = self.in_word_splitter + dataset = DataSet() + with open(filepath, 'r') as f: + for line in f: + line = line.strip() + if len(line.replace(' ', '')) == 0: # 不能接受空行 + continue + + if not in_word_splitter is None: + words = [] + for part in line.split(): + word = part.split(in_word_splitter)[0] + words.append(word) + line = ' '.join(words) + if cut_long_sent: + sents = cut_long_sentence(line) + else: + sents = [line] + for sent in sents: + instance = Instance(raw_sentence=sent) + dataset.append(instance) + + return dataset + + +def cut_long_sentence(sent, max_sample_length=200): + """ + 将长于max_sample_length的sentence截成多段，只会在有空格的地方发生截断。所以截取的句子可能长于或者短于max_sample_length + + :param sent: str. + :param max_sample_length: int. + :return: list of str. + """ + sent_no_space = sent.replace(' ', '') + cutted_sentence = [] + if len(sent_no_space) > max_sample_length: + parts = sent.strip().split() + new_line = '' + length = 0 + for part in parts: + length += len(part) + new_line += part + ' ' + if length > max_sample_length: + new_line = new_line[:-1] + cutted_sentence.append(new_line) + length = 0 + new_line = '' + if new_line != '': + cutted_sentence.append(new_line[:-1]) + else: + cutted_sentence.append(sent) + return cutted_sentence + + +class ZhConllPOSReader(object): + # 中文colln格式reader + def __init__(self): + pass + + def load(self, path): + """ + 返回的DataSet, 包含以下的field + words：list of str, + tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_one(sample) + if res is None: + continue + char_seq = [] + pos_seq = [] + for word, tag in zip(res[0], res[1]): + char_seq.extend(list(word)) + if len(word) == 1: + pos_seq.append('S-{}'.format(tag)) + elif len(word) > 1: + pos_seq.append('B-{}'.format(tag)) + for _ in range(len(word) - 2): + pos_seq.append('M-{}'.format(tag)) + pos_seq.append('E-{}'.format(tag)) + else: + raise ValueError("Zero length of word detected.") + + ds.append(Instance(words=char_seq, + tag=pos_seq)) + + return ds + + def get_one(self, sample): + if len(sample) == 0: + return None + text = [] + pos_tags = [] + for w in sample: + t1, t2, t3, t4 = w[1], w[3], w[6], w[7] + if t3 == '_': + return None + text.append(t1) + pos_tags.append(t2) + return text, pos_tags + + +class ConllPOSReader(object): + # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BIO的tag)。 + def __init__(self): + pass + + def load(self, path): + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_one(sample) + if res is None: + continue + char_seq = [] + pos_seq = [] + for word, tag in zip(res[0], res[1]): + if len(word) == 1: + char_seq.append(word) + pos_seq.append('S-{}'.format(tag)) + elif len(word) > 1: + pos_seq.append('B-{}'.format(tag)) + for _ in range(len(word) - 2): + pos_seq.append('M-{}'.format(tag)) + pos_seq.append('E-{}'.format(tag)) + char_seq.extend(list(word)) + else: + raise ValueError("Zero length of word detected.") + + ds.append(Instance(words=char_seq, + tag=pos_seq)) + + return ds + + +class ConllxDataLoader(object): + def load(self, path): + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.split('\t')) + if len(sample) > 0: + datalist.append(sample) + + data = [self.get_one(sample) for sample in datalist] + return list(filter(lambda x: x is not None, data)) + + def get_one(self, sample): + sample = list(map(list, zip(*sample))) + if len(sample) == 0: + return None + for w in sample[7]: + if w == '_': + print('Error Sample {}'.format(sample)) + return None + # return word_seq, pos_seq, head_seq, head_tag_seq + return sample[1], sample[3], list(map(int, sample[6])), sample[7] + + +def add_seg_tag(data): + """ + + :param data: list of ([word], [pos], [heads], [head_tags]) + :return: list of ([word], [pos]) + """ + + _processed = [] + for word_list, pos_list, _, _ in data: + new_sample = [] + for word, pos in zip(word_list, pos_list): + if len(word) == 1: + new_sample.append((word, 'S-' + pos)) + else: + new_sample.append((word[0], 'B-' + pos)) + for c in word[1:-1]: + new_sample.append((c, 'M-' + pos)) + new_sample.append((word[-1], 'E-' + pos)) + _processed.append(list(map(list, zip(*new_sample)))) + return _processed diff --git a/reproduction/Biaffine_parser/main.py b/reproduction/Biaffine_parser/main.py index 9028ff80..f4fd5836 100644 --- a/reproduction/Biaffine_parser/main.py +++ b/reproduction/Biaffine_parser/main.py @@ -5,7 +5,7 @@ sys.path.extend(['/home/yfshao/workdir/dev_fastnlp']) import torch import argparse -from reproduction.Biaffine_parser.util import ConllxDataLoader, add_seg_tag +from fastNLP.io.dataset_loader import ConllxDataLoader, add_seg_tag from fastNLP.core.dataset import DataSet from fastNLP.core.instance import Instance diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index e4928c63..ded7487d 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -4,20 +4,15 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) import fastNLP -import torch from fastNLP.core.trainer import Trainer from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.models.biaffine_parser import BiaffineParser, ParserMetric, ParserLoss -from fastNLP.core.vocabulary import Vocabulary -from fastNLP.core.dataset import DataSet from fastNLP.core.tester import Tester from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.io.model_io import ModelLoader -from fastNLP.io.embed_loader import EmbedLoader -from fastNLP.io.model_io import ModelSaver -from reproduction.Biaffine_parser.util import ConllxDataLoader, MyDataloader +from fastNLP.io.dataset_loader import ConllxDataLoader from fastNLP.api.processor import * BOS = '' diff --git a/reproduction/Biaffine_parser/util.py b/reproduction/Biaffine_parser/util.py index 793b1fb2..aa40e4e9 100644 --- a/reproduction/Biaffine_parser/util.py +++ b/reproduction/Biaffine_parser/util.py @@ -1,34 +1,3 @@ -class ConllxDataLoader(object): - def load(self, path): - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - data = [self.get_one(sample) for sample in datalist] - return list(filter(lambda x: x is not None, data)) - - def get_one(self, sample): - sample = list(map(list, zip(*sample))) - if len(sample) == 0: - return None - for w in sample[7]: - if w == '_': - print('Error Sample {}'.format(sample)) - return None - # return word_seq, pos_seq, head_seq, head_tag_seq - return sample[1], sample[3], list(map(int, sample[6])), sample[7] - - class MyDataloader: def load(self, data_path): with open(data_path, "r", encoding="utf-8") as f: @@ -56,23 +25,3 @@ class MyDataloader: return data -def add_seg_tag(data): - """ - - :param data: list of ([word], [pos], [heads], [head_tags]) - :return: list of ([word], [pos]) - """ - - _processed = [] - for word_list, pos_list, _, _ in data: - new_sample = [] - for word, pos in zip(word_list, pos_list): - if len(word) == 1: - new_sample.append((word, 'S-' + pos)) - else: - new_sample.append((word[0], 'B-' + pos)) - for c in word[1:-1]: - new_sample.append((c, 'M-' + pos)) - new_sample.append((word[-1], 'E-' + pos)) - _processed.append(list(map(list, zip(*new_sample)))) - return _processed \ No newline at end of file diff --git a/reproduction/chinese_word_segment/cws_io/cws_reader.py b/reproduction/chinese_word_segment/cws_io/cws_reader.py index 34bcf7dd..b28b04f6 100644 --- a/reproduction/chinese_word_segment/cws_io/cws_reader.py +++ b/reproduction/chinese_word_segment/cws_io/cws_reader.py @@ -1,197 +1,3 @@ -from fastNLP.core.dataset import DataSet -from fastNLP.core.instance import Instance -from fastNLP.io.dataset_loader import DataSetLoader - - -def cut_long_sentence(sent, max_sample_length=200): - """ - 将长于max_sample_length的sentence截成多段，只会在有空格的地方发生截断。所以截取的句子可能长于或者短于max_sample_length - - :param sent: str. - :param max_sample_length: int. - :return: list of str. - """ - sent_no_space = sent.replace(' ', '') - cutted_sentence = [] - if len(sent_no_space) > max_sample_length: - parts = sent.strip().split() - new_line = '' - length = 0 - for part in parts: - length += len(part) - new_line += part + ' ' - if length > max_sample_length: - new_line = new_line[:-1] - cutted_sentence.append(new_line) - length = 0 - new_line = '' - if new_line != '': - cutted_sentence.append(new_line[:-1]) - else: - cutted_sentence.append(sent) - return cutted_sentence - -class NaiveCWSReader(DataSetLoader): - """ - 这个reader假设了分词数据集为以下形式, 即已经用空格分割好内容了 - 这是 fastNLP , 一个 非常 good 的 包 . - 或者,即每个part后面还有一个pos tag - 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY - """ - def __init__(self, in_word_splitter=None): - super().__init__() - - self.in_word_splitter = in_word_splitter - - def load(self, filepath, in_word_splitter=None, cut_long_sent=False): - """ - 允许使用的情况有(默认以\t或空格作为seg) - 这是 fastNLP , 一个 非常 good 的 包 . - 和 - 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY - 如果splitter不为None则认为是第二种情况, 且我们会按splitter分割"也/D", 然后取第一部分. 例如"也/D".split('/')[0] - :param filepath: - :param in_word_splitter: - :return: - """ - if in_word_splitter == None: - in_word_splitter = self.in_word_splitter - dataset = DataSet() - with open(filepath, 'r') as f: - for line in f: - line = line.strip() - if len(line.replace(' ', ''))==0: # 不能接受空行 - continue - - if not in_word_splitter is None: - words = [] - for part in line.split(): - word = part.split(in_word_splitter)[0] - words.append(word) - line = ' '.join(words) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for sent in sents: - instance = Instance(raw_sentence=sent) - dataset.append(instance) - - return dataset - - -class POSCWSReader(DataSetLoader): - """ - 支持读取以下的情况, 即每一行是一个词, 用空行作为两句话的界限. - 迈 N - 向 N - 充 N - ... - 泽 I-PER - 民 I-PER - - （ N - 一 N - 九 N - ... - - - :param filepath: - :return: - """ - def __init__(self, in_word_splitter=None): - super().__init__() - self.in_word_splitter = in_word_splitter - - def load(self, filepath, in_word_splitter=None, cut_long_sent=False): - if in_word_splitter is None: - in_word_splitter = self.in_word_splitter - dataset = DataSet() - with open(filepath, 'r') as f: - words = [] - for line in f: - line = line.strip() - if len(line) == 0: # new line - if len(words)==0: # 不能接受空行 - continue - line = ' '.join(words) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for sent in sents: - instance = Instance(raw_sentence=sent) - dataset.append(instance) - words = [] - else: - line = line.split()[0] - if in_word_splitter is None: - words.append(line) - else: - words.append(line.split(in_word_splitter)[0]) - return dataset - - -class ConllCWSReader(object): - def __init__(self): - pass - - def load(self, path, cut_long_sent=False): - """ - 返回的DataSet只包含raw_sentence这个field，内容为str。 - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_char_lst(sample) - if res is None: - continue - line = ' '.join(res) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for raw_sentence in sents: - ds.append(Instance(raw_sentence=raw_sentence)) - - return ds - - def get_char_lst(self, sample): - if len(sample)==0: - return None - text = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - return text diff --git a/reproduction/chinese_word_segment/process/cws_processor.py b/reproduction/chinese_word_segment/process/cws_processor.py index 9e57d35a..be6ca6b1 100644 --- a/reproduction/chinese_word_segment/process/cws_processor.py +++ b/reproduction/chinese_word_segment/process/cws_processor.py @@ -226,109 +226,6 @@ class Pre2Post2BigramProcessor(BigramProcessor): return bigrams -# 这里需要建立vocabulary了，但是遇到了以下的问题 -# (1) 如果使用Processor的方式的话，但是在这种情况返回的不是dataset。所以建立vocabulary的工作用另外的方式实现，不借用 -# Processor了 -# TODO 如何将建立vocab和index这两步统一了？ - -class VocabIndexerProcessor(Processor): - """ - 根据DataSet创建Vocabulary，并将其用数字index。新生成的index的field会被放在new_added_filed_name, 如果没有提供 - new_added_field_name, 则覆盖原有的field_name. - - """ - def __init__(self, field_name, new_added_filed_name=None, min_freq=1, max_size=None, - verbose=0, is_input=True): - """ - - :param field_name: 从哪个field_name创建词表，以及对哪个field_name进行index操作 - :param new_added_filed_name: index时，生成的index field的名称，如果不传入，则覆盖field_name. - :param min_freq: 创建的Vocabulary允许的单词最少出现次数. - :param max_size: 创建的Vocabulary允许的最大的单词数量 - :param verbose: 0, 不输出任何信息；1，输出信息 - :param bool is_input: - """ - super(VocabIndexerProcessor, self).__init__(field_name, new_added_filed_name) - self.min_freq = min_freq - self.max_size = max_size - - self.verbose =verbose - self.is_input = is_input - - def construct_vocab(self, *datasets): - """ - 使用传入的DataSet创建vocabulary - - :param datasets: DataSet类型的数据，用于构建vocabulary - :return: - """ - self.vocab = Vocabulary(min_freq=self.min_freq, max_size=self.max_size) - for dataset in datasets: - assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: self.vocab.update(ins[self.field_name])) - self.vocab.build_vocab() - if self.verbose: - print("Vocabulary Constructed, has {} items.".format(len(self.vocab))) - - def process(self, *datasets, only_index_dataset=None): - """ - 若还未建立Vocabulary，则使用dataset中的DataSet建立vocabulary；若已经有了vocabulary则使用已有的vocabulary。得到vocabulary - 后，则会index datasets与only_index_dataset。 - - :param datasets: DataSet类型的数据 - :param only_index_dataset: DataSet, or list of DataSet. 该参数中的内容只会被用于index，不会被用于生成vocabulary。 - :return: - """ - if len(datasets)==0 and not hasattr(self,'vocab'): - raise RuntimeError("You have to construct vocabulary first. Or you have to pass datasets to construct it.") - if not hasattr(self, 'vocab'): - self.construct_vocab(*datasets) - else: - if self.verbose: - print("Using constructed vocabulary with {} items.".format(len(self.vocab))) - to_index_datasets = [] - if len(datasets)!=0: - for dataset in datasets: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - to_index_datasets.append(dataset) - - if not (only_index_dataset is None): - if isinstance(only_index_dataset, list): - for dataset in only_index_dataset: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - to_index_datasets.append(dataset) - elif isinstance(only_index_dataset, DataSet): - to_index_datasets.append(only_index_dataset) - else: - raise TypeError('Only DataSet or list of DataSet is allowed, not {}.'.format(type(only_index_dataset))) - - for dataset in to_index_datasets: - assert isinstance(dataset, DataSet), "Only DataSet class is allowed, not {}.".format(type(dataset)) - dataset.apply(lambda ins: [self.vocab.to_index(token) for token in ins[self.field_name]], - new_field_name=self.new_added_field_name, is_input=self.is_input) - # 只返回一个，infer时为了跟其他processor保持一致 - if len(to_index_datasets) == 1: - return to_index_datasets[0] - - def set_vocab(self, vocab): - assert isinstance(vocab, Vocabulary), "Only fastNLP.core.Vocabulary is allowed, not {}.".format(type(vocab)) - self.vocab = vocab - - def delete_vocab(self): - del self.vocab - - def get_vocab_size(self): - return len(self.vocab) - - def set_verbose(self, verbose): - """ - 设置processor verbose状态。 - - :param verbose: int, 0，不输出任何信息；1，输出vocab 信息。 - :return: - """ - self.verbose = verbose - class VocabProcessor(Processor): def __init__(self, field_name, min_freq=1, max_size=None): diff --git a/reproduction/pos_tag_model/pos_reader.py b/reproduction/pos_tag_model/pos_reader.py index c0a8c4cd..4ff58f4b 100644 --- a/reproduction/pos_tag_model/pos_reader.py +++ b/reproduction/pos_tag_model/pos_reader.py @@ -1,6 +1,5 @@ +from fastNLP.io.dataset_loader import ZhConllPOSReader -from fastNLP.core.dataset import DataSet -from fastNLP.core.instance import Instance def cut_long_sentence(sent, max_sample_length=200): sent_no_space = sent.replace(' ', '') @@ -24,129 +23,6 @@ def cut_long_sentence(sent, max_sample_length=200): return cutted_sentence -class ConllPOSReader(object): - # 返回的Dataset包含words(list of list, 里层的list是character), tag两个field(list of str, str是标有BIO的tag)。 - def __init__(self): - pass - - def load(self, path): - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_one(sample) - if res is None: - continue - char_seq = [] - pos_seq = [] - for word, tag in zip(res[0], res[1]): - if len(word)==1: - char_seq.append(word) - pos_seq.append('S-{}'.format(tag)) - elif len(word)>1: - pos_seq.append('B-{}'.format(tag)) - for _ in range(len(word)-2): - pos_seq.append('M-{}'.format(tag)) - pos_seq.append('E-{}'.format(tag)) - char_seq.extend(list(word)) - else: - raise ValueError("Zero length of word detected.") - - ds.append(Instance(words=char_seq, - tag=pos_seq)) - - return ds - - - -class ZhConllPOSReader(object): - # 中文colln格式reader - def __init__(self): - pass - - def load(self, path): - """ - 返回的DataSet, 包含以下的field - words：list of str, - tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_one(sample) - if res is None: - continue - char_seq = [] - pos_seq = [] - for word, tag in zip(res[0], res[1]): - char_seq.extend(list(word)) - if len(word)==1: - pos_seq.append('S-{}'.format(tag)) - elif len(word)>1: - pos_seq.append('B-{}'.format(tag)) - for _ in range(len(word)-2): - pos_seq.append('M-{}'.format(tag)) - pos_seq.append('E-{}'.format(tag)) - else: - raise ValueError("Zero length of word detected.") - - ds.append(Instance(words=char_seq, - tag=pos_seq)) - - return ds - - def get_one(self, sample): - if len(sample)==0: - return None - text = [] - pos_tags = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - pos_tags.append(t2) - return text, pos_tags - if __name__ == '__main__': reader = ZhConllPOSReader() d = reader.load('/home/hyan/train.conllx') diff --git a/reproduction/pos_tag_model/train_pos_tag.py b/reproduction/pos_tag_model/train_pos_tag.py index adc9359c..09a9ba02 100644 --- a/reproduction/pos_tag_model/train_pos_tag.py +++ b/reproduction/pos_tag_model/train_pos_tag.py @@ -10,13 +10,12 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) from fastNLP.api.pipeline import Pipeline -from fastNLP.api.processor import SeqLenProcessor +from fastNLP.api.processor import SeqLenProcessor, VocabIndexerProcessor from fastNLP.core.metrics import SpanFPreRecMetric from fastNLP.core.trainer import Trainer from fastNLP.io.config_io import ConfigLoader, ConfigSection from fastNLP.models.sequence_modeling import AdvSeqLabel -from reproduction.chinese_word_segment.process.cws_processor import VocabIndexerProcessor -from reproduction.pos_tag_model.pos_reader import ZhConllPOSReader +from fastNLP.io.dataset_loader import ZhConllPOSReader from fastNLP.api.processor import ModelProcessor, Index2WordProcessor cfgfile = './pos_tag.cfg'