diff --git a/docs/source/user/quickstart.rst b/docs/source/user/quickstart.rst index 43056a26..12e541b7 100644 --- a/docs/source/user/quickstart.rst +++ b/docs/source/user/quickstart.rst @@ -49,7 +49,7 @@ .. code-block:: python from fastNLP.models import CNNText - model = CNNText((len(vocab),50), num_classes=5, padding=2, dropout=0.1) + model = CNNText((len(vocab),50), num_classes=5, dropout=0.1) :class:`~fastNLP.models.CNNText` 的网络结构如下:: @@ -121,4 +121,4 @@ In Epoch:6/Step:12, got best dev performance:AccuracyMetric: acc=0.8 Reloaded the best model. -这份教程只是简单地介绍了使用 fastNLP 工作的流程，具体的细节分析见 :doc:`/user/tutorial_one` \ No newline at end of file +这份教程只是简单地介绍了使用 fastNLP 工作的流程，具体的细节分析见 :doc:`/user/tutorial_one` diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index ab020ce4..b011d15a 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -554,6 +554,7 @@ class DataSet(object): self.field_arrays[new_name].name = new_name else: raise KeyError("DataSet has no field named {}.".format(old_name)) + return self def set_target(self, *field_names, flag=True): """ @@ -593,7 +594,7 @@ class DataSet(object): try: self.field_arrays[name].is_input = flag except SetInputOrTargetException as e: - print(f"Cannot set field:{name} as input.") + print(f"Cannot set field:{name} as input, exception happens at the {e.index} value.") raise e else: raise KeyError("{} is not a valid field name.".format(name)) @@ -761,7 +762,20 @@ class DataSet(object): self._add_apply_field(results, new_field_name, kwargs) return results - + + def add_seq_len(self, field_name:str, new_field_name='seq_len'): + """ + 将使用len()直接对field_name中每个元素作用，将其结果作为seqence length, 并放入seq_len这个field。 + + :param field_name: str. + :return: + """ + if self.has_field(field_name=field_name): + self.apply_field(len, field_name, new_field_name=new_field_name) + else: + raise KeyError(f"Field:{field_name} not found.") + return self + def drop(self, func, inplace=True): """ func接受一个Instance，返回bool值。返回值为True时，该Instance会被移除或者加入到返回的DataSet中。 diff --git a/fastNLP/core/field.py b/fastNLP/core/field.py index c47771df..faa306f3 100644 --- a/fastNLP/core/field.py +++ b/fastNLP/core/field.py @@ -6,6 +6,7 @@ import numpy as np from typing import Any from abc import abstractmethod from copy import deepcopy +from collections import Counter class SetInputOrTargetException(Exception): def __init__(self, msg, index=None, field_name=None): @@ -61,6 +62,7 @@ class FieldArray: if value: self._cell_ndim = None self.dtype = None + self._ignore_type = value @property def is_input(self): @@ -223,6 +225,155 @@ class FieldArray: return self + def split(self, sep:str=None, inplace:bool=True): + """ + 依次对自身的元素使用.split()方法，应该只有当本field的元素为str时，该方法才有用。将返回值 + + :param sep: 分割符，如果为None则直接调用str.split()。 + :param inplace: 如果为True，则将新生成值替换本field。否则返回list。 + :return: List[List[str]] or self + """ + new_contents = [] + for index, cell in enumerate(self.content): + try: + new_contents.append(cell.split(sep)) + except Exception as e: + print(f"Exception happens when process value in index {index}.") + print(e) + return self._after_process(new_contents, inplace=inplace) + + def int(self, inplace:bool=True): + """ + 将本field中的值调用int(cell). 支持field中内容为以下两种情况(1)['1', '2', ...](即field中每个值为str的)， + (2) [['1', '2', ..], ['3', ..], ...](即field中每个值为一个list，list中的值会被依次转换。) + + :param inplace: 如果为True，则将新生成值替换本field。否则返回list。 + :return: List[int], List[List[int]], self + """ + new_contents = [] + for index, cell in enumerate(self.content): + try: + if isinstance(cell, list): + new_contents.append([int(value) for value in cell]) + else: + new_contents.append(int(cell)) + except Exception as e: + print(f"Exception happens when process value in index {index}.") + print(e) + return self._after_process(new_contents, inplace=inplace) + + def float(self, inplace=True): + """ + 将本field中的值调用float(cell). 支持field中内容为以下两种情况(1)['1', '2', ...](即field中每个值为str的)， + (2) [['1', '2', ..], ['3', ..], ...](即field中每个值为一个list，list中的值会被依次转换。) + + :param inplace: 如果为True，则将新生成值替换本field。否则返回list。 + :return: + """ + new_contents = [] + for index, cell in enumerate(self.content): + try: + if isinstance(cell, list): + new_contents.append([float(value) for value in cell]) + else: + new_contents.append(float(cell)) + except Exception as e: + print(f"Exception happens when process value in index {index}.") + print(e) + return self._after_process(new_contents, inplace=inplace) + + def bool(self, inplace=True): + """ + 将本field中的值调用bool(cell). 支持field中内容为以下两种情况(1)['1', '2', ...](即field中每个值为str的)， + (2) [['1', '2', ..], ['3', ..], ...](即field中每个值为一个list，list中的值会被依次转换。) + + :param inplace: 如果为True，则将新生成值替换本field。否则返回list。 + :return: + """ + new_contents = [] + for index, cell in enumerate(self.content): + try: + if isinstance(cell, list): + new_contents.append([bool(value) for value in cell]) + else: + new_contents.append(bool(cell)) + except Exception as e: + print(f"Exception happens when process value in index {index}.") + print(e) + + return self._after_process(new_contents, inplace=inplace) + + def lower(self, inplace=True): + """ + 将本field中的值调用cell.lower(). 支持field中内容为以下两种情况(1)['1', '2', ...](即field中每个值为str的)， + (2) [['1', '2', ..], ['3', ..], ...](即field中每个值为一个list，list中的值会被依次转换。) + + :param inplace: 如果为True，则将新生成值替换本field。否则返回list。 + :return: List[int], List[List[int]], self + """ + new_contents = [] + for index, cell in enumerate(self.content): + try: + if isinstance(cell, list): + new_contents.append([value.lower() for value in cell]) + else: + new_contents.append(cell.lower()) + except Exception as e: + print(f"Exception happens when process value in index {index}.") + print(e) + return self._after_process(new_contents, inplace=inplace) + + def upper(self, inplace=True): + """ + 将本field中的值调用cell.lower(). 支持field中内容为以下两种情况(1)['1', '2', ...](即field中每个值为str的)， + (2) [['1', '2', ..], ['3', ..], ...](即field中每个值为一个list，list中的值会被依次转换。) + + :param inplace: 如果为True，则将新生成值替换本field。否则返回list。 + :return: List[int], List[List[int]], self + """ + new_contents = [] + for index, cell in enumerate(self.content): + try: + if isinstance(cell, list): + new_contents.append([value.upper() for value in cell]) + else: + new_contents.append(cell.upper()) + except Exception as e: + print(f"Exception happens when process value in index {index}.") + print(e) + return self._after_process(new_contents, inplace=inplace) + + def value_count(self): + """ + 返回该field下不同value的数量。多用于统计label数量 + + :return: Counter, key是label，value是出现次数 + """ + count = Counter() + for cell in self.content: + count[cell] += 1 + return count + + def _after_process(self, new_contents, inplace): + """ + 当调用处理函数之后，决定是否要替换field。 + + :param new_contents: + :param inplace: + :return: self或者生成的content + """ + if inplace: + self.content = new_contents + try: + self.is_input = self.is_input + self.is_target = self.is_input + except SetInputOrTargetException as e: + print("The newly generated field cannot be set as input or target.") + raise e + return self + else: + return new_contents + def _get_ele_type_and_dim(cell:Any, dim=0): """ @@ -242,6 +393,8 @@ def _get_ele_type_and_dim(cell:Any, dim=0): dims = set([j for i,j in res]) if len(types)>1: raise SetInputOrTargetException("Mixed types detected: {}.".format(list(types))) + elif len(types)==0: + raise SetInputOrTargetException("Empty value encountered.") if len(dims)>1: raise SetInputOrTargetException("Mixed dimension detected: {}.".format(list(dims))) return types.pop(), dims.pop() @@ -257,6 +410,8 @@ def _get_ele_type_and_dim(cell:Any, dim=0): dims = set([j for i,j in res]) if len(types)>1: raise SetInputOrTargetException("Mixed types detected: {}.".format(list(types))) + elif len(types)==0: + raise SetInputOrTargetException("Empty value encountered.") if len(dims)>1: raise SetInputOrTargetException("Mixed dimension detected: {}.".format(list(dims))) return types.pop(), dims.pop() diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index 868d67b1..19c33c86 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -22,7 +22,7 @@ from .utils import _check_arg_dict_list from .utils import _get_func_signature from .utils import seq_len_to_mask from .vocabulary import Vocabulary - +from abc import abstractmethod class MetricBase(object): """ @@ -117,10 +117,12 @@ class MetricBase(object): def __init__(self): self.param_map = {} # key is param in function, value is input param. self._checked = False - + + @abstractmethod def evaluate(self, *args, **kwargs): raise NotImplementedError - + + @abstractmethod def get_metric(self, reset=True): raise NotImplemented diff --git a/fastNLP/core/utils.py b/fastNLP/core/utils.py index 9dab47b5..1eb2b70e 100644 --- a/fastNLP/core/utils.py +++ b/fastNLP/core/utils.py @@ -285,6 +285,7 @@ def _get_model_device(model): :param model: nn.Module :return: torch.device,None 如果返回值为None，说明这个模型没有任何参数。 """ + # TODO 这个函数存在一定的风险，因为同一个模型可能存在某些parameter不在显卡中，比如BertEmbedding assert isinstance(model, nn.Module) parameters = list(model.parameters()) @@ -295,6 +296,13 @@ def _get_model_device(model): def _build_args(func, **kwargs): + """ + 根据func的初始化参数，从kwargs中选择func需要的参数 + + :param func: callable + :param kwargs: 参数 + :return:dict. func中用到的参数 + """ spect = inspect.getfullargspec(func) if spect.varkw is not None: return kwargs diff --git a/fastNLP/core/vocabulary.py b/fastNLP/core/vocabulary.py index bca28e10..1d5d6f32 100644 --- a/fastNLP/core/vocabulary.py +++ b/fastNLP/core/vocabulary.py @@ -148,7 +148,7 @@ class Vocabulary(object): self.word2idx.update({w: i + start_idx for i, (w, _) in enumerate(words)}) self.build_reverse_vocab() self.rebuild = False - + def build_reverse_vocab(self): """ 基于 "word to index" dict, 构建 "index to word" dict. @@ -359,5 +359,7 @@ class Vocabulary(object): def __repr__(self): return "Vocabulary({}...)".format(list(self.word_count.keys())[:5]) + @_check_build_vocab def __iter__(self): - return iter(list(self.word_count.keys())) + for word, index in self.word2idx.items(): + yield word, index diff --git a/fastNLP/io/embed_loader.py b/fastNLP/io/embed_loader.py index 4119d93f..5237a8a7 100644 --- a/fastNLP/io/embed_loader.py +++ b/fastNLP/io/embed_loader.py @@ -26,6 +26,7 @@ class EmbeddingOption(Option): error=error ) + class EmbedLoader(BaseLoader): """ 别名：:class:`fastNLP.io.EmbedLoader` :class:`fastNLP.io.embed_loader.EmbedLoader` @@ -35,9 +36,9 @@ class EmbedLoader(BaseLoader): def __init__(self): super(EmbedLoader, self).__init__() - + @staticmethod - def load_with_vocab(embed_filepath, vocab, dtype=np.float32, normalize=True, error='ignore'): + def load_with_vocab(embed_filepath, vocab, dtype=np.float32, padding='', unknown='', normalize=True, error='ignore'): """ 从embed_filepath这个预训练的词向量中抽取出vocab这个词表的词的embedding。EmbedLoader将自动判断embed_filepath是 word2vec(第一行只有两个元素)还是glove格式的数据。 @@ -46,6 +47,8 @@ class EmbedLoader(BaseLoader): :param vocab: 词表 :class:`~fastNLP.Vocabulary` 类型，读取出现在vocab中的词的embedding。 没有出现在vocab中的词的embedding将通过找到的词的embedding的正态分布采样出来，以使得整个Embedding是同分布的。 :param dtype: 读出的embedding的类型 + :param str padding: 词表中padding的token + :param str unknown: 词表中unknown的token :param bool normalize: 是否将每个vector归一化到norm为1 :param str error: `ignore` , `strict` ; 如果 `ignore` ，错误将自动跳过; 如果 `strict` , 错误将抛出。 这里主要可能出错的地方在于词表有空行或者词表出现了维度不一致。 @@ -69,8 +72,14 @@ class EmbedLoader(BaseLoader): for idx, line in enumerate(f, start_idx): try: parts = line.strip().split() - if parts[0] in vocab: - index = vocab.to_index(parts[0]) + word = parts[0] + # 对齐unk与pad + if word==padding and vocab.padding is not None: + word = vocab.padding + elif word==unknown and vocab.unknown is not None: + word = vocab.unknown + if word in vocab: + index = vocab.to_index(word) matrix[index] = np.fromstring(' '.join(parts[1:]), sep=' ', dtype=dtype, count=dim) hit_flags[index] = True except Exception as e: @@ -102,8 +111,8 @@ class EmbedLoader(BaseLoader): :param str embed_filepath: 预训练的embedding的路径。 :param dtype: 读出的embedding的类型 - :param str padding: the padding tag for vocabulary. - :param str unknown: the unknown tag for vocabulary. + :param str padding: 词表中的padding的token. 并以此用做vocab的padding。 + :param str unknown: 词表中的unknown的token. 并以此用做vocab的unknown。 :param bool normalize: 是否将每个vector归一化到norm为1 :param str error: `ignore` , `strict` ; 如果 `ignore` ，错误将自动跳过; 如果 `strict` , 错误将抛出。这里主要可能出错的地 方在于词表有空行或者词表出现了维度不一致。 @@ -134,7 +143,7 @@ class EmbedLoader(BaseLoader): vocab.add_word(word) if unknown is not None and unknown == word: found_unknown = True - if found_pad is not None and padding == word: + if padding is not None and padding == word: found_pad = True except Exception as e: if error == 'ignore': diff --git a/fastNLP/io/file_utils.py b/fastNLP/io/file_utils.py new file mode 100644 index 00000000..11c7ab64 --- /dev/null +++ b/fastNLP/io/file_utils.py @@ -0,0 +1,255 @@ + +import os +from pathlib import Path +from urllib.parse import urlparse +import re +import requests +import tempfile +from tqdm import tqdm +import shutil +import hashlib + + +def cached_path(url_or_filename: str, cache_dir: Path=None) -> Path: + """ + 给定一个url或者文件名(可以是具体的文件名，也可以是文件)，先在cache_dir下寻找该文件是否存在，如果不存在则去下载, 并 + 将文件放入到 + """ + if cache_dir is None: + dataset_cache = Path(get_defalt_path()) + else: + dataset_cache = cache_dir + + parsed = urlparse(url_or_filename) + + if parsed.scheme in ("http", "https"): + # URL, so get it from the cache (downloading if necessary) + return get_from_cache(url_or_filename, dataset_cache) + elif parsed.scheme == "" and Path(os.path.join(dataset_cache, url_or_filename)).exists(): + # File, and it exists. + return Path(url_or_filename) + elif parsed.scheme == "": + # File, but it doesn't exist. + raise FileNotFoundError("file {} not found".format(url_or_filename)) + else: + # Something unknown + raise ValueError( + "unable to parse {} as a URL or as a local path".format(url_or_filename) + ) + +def get_filepath(filepath): + """ + 如果filepath中只有一个文件，则直接返回对应的全路径 + :param filepath: + :return: + """ + if os.path.isdir(filepath): + files = os.listdir(filepath) + if len(files)==1: + return os.path.join(filepath, files[0]) + else: + return filepath + return filepath + +def get_defalt_path(): + """ + 获取默认的fastNLP存放路径, 如果将FASTNLP_CACHE_PATH设置在了环境变量中，将使用环境变量的值，使得不用每个用户都去下载。 + + :return: + """ + if 'FASTNLP_CACHE_DIR' in os.environ: + fastnlp_cache_dir = os.environ.get('FASTNLP_CACHE_DIR') + if os.path.exists(fastnlp_cache_dir): + return fastnlp_cache_dir + raise RuntimeError("Some errors happens on cache directory.") + else: + raise RuntimeError("There function is not available right now.") + fastnlp_cache_dir = os.path.expanduser(os.path.join("~", ".fastNLP")) + return fastnlp_cache_dir + +def _get_base_url(name): + # 返回的URL结尾必须是/ + if 'FASTNLP_BASE_URL' in os.environ: + fastnlp_base_url = os.environ['FASTNLP_BASE_URL'] + return fastnlp_base_url + raise RuntimeError("There function is not available right now.") + +def split_filename_suffix(filepath): + """ + 给定filepath返回对应的name和suffix + :param filepath: + :return: filename, suffix + """ + filename = os.path.basename(filepath) + if filename.endswith('.tar.gz'): + return filename[:-7], '.tar.gz' + return os.path.splitext(filename) + +def get_from_cache(url: str, cache_dir: Path = None) -> Path: + """ + 尝试在cache_dir中寻找url定义的资源; 如果没有找到。则从url下载并将结果放在cache_dir下，缓存的名称由url的结果推断而来。 + 如果从url中下载的资源解压后有多个文件，则返回directory的路径; 如果只有一个资源，则返回具体的路径 + + """ + cache_dir.mkdir(parents=True, exist_ok=True) + + filename = re.sub(r".+/", "", url) + dir_name, suffix = split_filename_suffix(filename) + sep_index = dir_name[::-1].index('-') + if sep_index<0: + check_sum = None + else: + check_sum = dir_name[-sep_index+1:] + sep_index = len(dir_name) if sep_index==-1 else -sep_index-1 + dir_name = dir_name[:sep_index] + + # 寻找与它名字匹配的内容, 而不关心后缀 + match_dir_name = match_file(dir_name, cache_dir) + if match_dir_name: + dir_name = match_dir_name + cache_path = cache_dir / dir_name + + # get cache path to put the file + if cache_path.exists(): + return get_filepath(cache_path) + + # make HEAD request to check ETag TODO ETag可以用来判断资源是否已经更新了，之后需要加上 + response = requests.head(url, headers={"User-Agent": "fastNLP"}) + if response.status_code != 200: + raise IOError( + f"HEAD request failed for url {url} with status code {response.status_code}." + ) + + # add ETag to filename if it exists + # etag = response.headers.get("ETag") + + if not cache_path.exists(): + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + fd, temp_filename = tempfile.mkstemp() + print("%s not found in cache, downloading to %s"%(url, temp_filename)) + + # GET file object + req = requests.get(url, stream=True, headers={"User-Agent": "fastNLP"}) + content_length = req.headers.get("Content-Length") + total = int(content_length) if content_length is not None else None + progress = tqdm(unit="B", total=total) + sha256 = hashlib.sha256() + with open(temp_filename, "wb") as temp_file: + for chunk in req.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + progress.update(len(chunk)) + temp_file.write(chunk) + sha256.update(chunk) + # check sum + digit = sha256.hexdigest()[:8] + if not check_sum: + assert digit == check_sum, "File corrupted when download." + progress.close() + print(f"Finish download from {url}.") + + # 开始解压 + delete_temp_dir = None + if suffix in ('.zip', '.tar.gz'): + uncompress_temp_dir = tempfile.mkdtemp() + delete_temp_dir = uncompress_temp_dir + print(f"Start to uncompress file to {uncompress_temp_dir}.") + if suffix == '.zip': + unzip_file(Path(temp_filename), Path(uncompress_temp_dir)) + else: + untar_gz_file(Path(temp_filename), Path(uncompress_temp_dir)) + filenames = os.listdir(uncompress_temp_dir) + if len(filenames)==1: + if os.path.isdir(os.path.join(uncompress_temp_dir, filenames[0])): + uncompress_temp_dir = os.path.join(uncompress_temp_dir, filenames[0]) + + cache_path.mkdir(parents=True, exist_ok=True) + print("Finish un-compressing file.") + else: + uncompress_temp_dir = temp_filename + cache_path = str(cache_path) + suffix + success = False + try: + # 复制到指定的位置 + print(f"Copy file to {cache_path}.") + if os.path.isdir(uncompress_temp_dir): + for filename in os.listdir(uncompress_temp_dir): + shutil.copyfile(os.path.join(uncompress_temp_dir, filename), cache_path/filename) + else: + shutil.copyfile(uncompress_temp_dir, cache_path) + success = True + except Exception as e: + print(e) + raise e + finally: + if not success: + if cache_path.exists(): + if cache_path.is_file(): + os.remove(cache_path) + else: + shutil.rmtree(cache_path) + if delete_temp_dir: + shutil.rmtree(delete_temp_dir) + os.close(fd) + os.remove(temp_filename) + + return get_filepath(cache_path) + +def unzip_file(file: Path, to: Path): + # unpack and write out in CoNLL column-like format + from zipfile import ZipFile + + with ZipFile(file, "r") as zipObj: + # Extract all the contents of zip file in current directory + zipObj.extractall(to) + +def untar_gz_file(file:Path, to:Path): + import tarfile + + with tarfile.open(file, 'r:gz') as tar: + tar.extractall(to) + +def match_file(dir_name:str, cache_dir:str)->str: + """ + 匹配的原则是，在cache_dir下的文件: (1) 与dir_name完全一致; (2) 除了后缀以外和dir_name完全一致。 + 如果找到了两个匹配的结果将报错. 如果找到了则返回匹配的文件的名称; 没有找到返回空字符串 + + :param dir_name: 需要匹配的名称 + :param cache_dir: 在该目录下找匹配dir_name是否存在 + :return: str + """ + files = os.listdir(cache_dir) + matched_filenames = [] + for file_name in files: + if re.match(dir_name+'$', file_name) or re.match(dir_name+'\\..*', file_name): + matched_filenames.append(file_name) + if len(matched_filenames)==0: + return '' + elif len(matched_filenames)==1: + return matched_filenames[-1] + else: + raise RuntimeError(f"Duplicate matched files:{matched_filenames}, this should be caused by a bug.") + +if __name__ == '__main__': + cache_dir = Path('caches') + cache_dir = None + # 需要对cache_dir进行测试 + base_url = 'http://0.0.0.0:8888/file/download' + # if True: + # for filename in os.listdir(cache_dir): + # if os.path.isdir(os.path.join(cache_dir, filename)): + # shutil.rmtree(os.path.join(cache_dir, filename)) + # else: + # os.remove(os.path.join(cache_dir, filename)) + # 1. 测试.txt文件 + print(cached_path(base_url + '/{}'.format('txt_test-bcb4fe65.txt'), cache_dir)) + # 2. 测试.zip文件(只有一个文件) + print(cached_path(base_url + '/{}'.format('zip_test-40966d39.zip'), cache_dir)) + # 3. 测试.zip文件(有多个文件) + print(cached_path(base_url + '/{}'.format('zip_pack_test-70c0b20d.zip'), cache_dir)) + # 4. 测试.tar.gz文件 + print(cached_path(base_url + '/{}'.format('tar_gz_test-3e2679cf.tar.gz'), cache_dir)) + # 5. 测试.tar.gz多个文件 + print(cached_path(base_url + '/{}'.format('tar_gz_pack_test-08dfdccd.tar.gz'), cache_dir)) + + # 6. 测试.pkl文件 diff --git a/fastNLP/models/bert.py b/fastNLP/models/bert.py index 02227c0d..4846c7fa 100644 --- a/fastNLP/models/bert.py +++ b/fastNLP/models/bert.py @@ -30,7 +30,7 @@ class BertConfig: self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads - self.intermediate = intermediate_size + self.intermediate_size = intermediate_size self.hidden_act = hidden_act self.hidden_dropout_prob = hidden_dropout_prob self.attention_probs_dropout_prob = attention_probs_dropout_prob diff --git a/fastNLP/models/cnn_text_classification.py b/fastNLP/models/cnn_text_classification.py index 3a71a80a..081dd510 100644 --- a/fastNLP/models/cnn_text_classification.py +++ b/fastNLP/models/cnn_text_classification.py @@ -7,6 +7,7 @@ import torch.nn as nn from ..core.const import Const as C from ..modules import encoder +from fastNLP import seq_len_to_mask class CNNText(torch.nn.Module): @@ -21,15 +22,13 @@ class CNNText(torch.nn.Module): :param int num_classes: 一共有多少类 :param int,tuple(int) out_channels: 输出channel的数量。如果为list，则需要与kernel_sizes的数量保持一致 :param int,tuple(int) kernel_sizes: 输出channel的kernel大小。 - :param int padding: 对句子前后的pad的大小, 用0填充。 :param float dropout: Dropout的大小 """ def __init__(self, init_embed, num_classes, - kernel_nums=(3, 4, 5), - kernel_sizes=(3, 4, 5), - padding=0, + kernel_nums=(30, 40, 50), + kernel_sizes=(1, 3, 5), dropout=0.5): super(CNNText, self).__init__() @@ -38,8 +37,7 @@ class CNNText(torch.nn.Module): self.conv_pool = encoder.ConvMaxpool( in_channels=self.embed.embedding_dim, out_channels=kernel_nums, - kernel_sizes=kernel_sizes, - padding=padding) + kernel_sizes=kernel_sizes) self.dropout = nn.Dropout(dropout) self.fc = nn.Linear(sum(kernel_nums), num_classes) @@ -51,7 +49,11 @@ class CNNText(torch.nn.Module): :return output: dict of torch.LongTensor, [batch_size, num_classes] """ x = self.embed(words) # [N,L] -> [N,L,C] - x = self.conv_pool(x) # [N,L,C] -> [N,C] + if seq_len is not None: + mask = seq_len_to_mask(seq_len) + x = self.conv_pool(x, mask) + else: + x = self.conv_pool(x) # [N,L,C] -> [N,C] x = self.dropout(x) x = self.fc(x) # [N,C] -> [N, N_class] return {C.OUTPUT: x} diff --git a/fastNLP/modules/encoder/_bert.py b/fastNLP/modules/encoder/_bert.py new file mode 100644 index 00000000..fc62ea9c --- /dev/null +++ b/fastNLP/modules/encoder/_bert.py @@ -0,0 +1,625 @@ + + + +""" +这个页面的代码很大程度上参考了https://github.com/huggingface/pytorch-pretrained-BERT的代码 +""" + + +import torch +from torch import nn + +from ... import Vocabulary +import collections + +import os +import unicodedata +from ...io.file_utils import _get_base_url, cached_path +from .bert import BertModel +import numpy as np +from itertools import chain + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer`. + + Returns: + A list of wordpiece tokens. + """ + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, "r", encoding="utf-8") as reader: + while True: + token = reader.readline() + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, + do_lower_case=True, + never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + self.never_split = never_split + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = self._clean_text(text) + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case and token not in self.never_split: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + if text in self.never_split: + return [text] + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False + + +class BertTokenizer(object): + """Runs end-to-end tokenization: punctuation splitting + wordpiece""" + + def __init__(self, vocab_file, do_lower_case=True, max_len=None, do_basic_tokenize=True, + never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")): + """Constructs a BertTokenizer. + + Args: + vocab_file: Path to a one-wordpiece-per-line vocabulary file + do_lower_case: Whether to lower case the input + Only has an effect when do_wordpiece_only=False + do_basic_tokenize: Whether to do basic tokenization before wordpiece. + max_len: An artificial maximum length to truncate tokenized sequences to; + Effective maximum length is always the minimum of this + value (if specified) and the underlying BERT model's + sequence length. + never_split: List of tokens which will never be split during tokenization. + Only has an effect when do_wordpiece_only=False + """ + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict( + [(ids, tok) for tok, ids in self.vocab.items()]) + self.do_basic_tokenize = do_basic_tokenize + if do_basic_tokenize: + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, + never_split=never_split) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + self.max_len = max_len if max_len is not None else int(1e12) + + def tokenize(self, text): + split_tokens = [] + if self.do_basic_tokenize: + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def convert_tokens_to_ids(self, tokens): + """Converts a sequence of tokens into ids using the vocab.""" + ids = [] + for token in tokens: + ids.append(self.vocab[token]) + if len(ids) > self.max_len: + print( + "Token indices sequence length is longer than the specified maximum " + " sequence length for this BERT model ({} > {}). Running this" + " sequence through BERT will result in indexing errors".format(len(ids), self.max_len) + ) + return ids + + def convert_ids_to_tokens(self, ids): + """Converts a sequence of ids in wordpiece tokens using the vocab.""" + tokens = [] + for i in ids: + tokens.append(self.ids_to_tokens[i]) + return tokens + + def save_vocabulary(self, vocab_path): + """Save the tokenizer vocabulary to a directory or file.""" + index = 0 + if os.path.isdir(vocab_path): + vocab_file = os.path.join(vocab_path, VOCAB_NAME) + with open(vocab_file, "w", encoding="utf-8") as writer: + for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): + if index != token_index: + print("Saving vocabulary to {}: vocabulary indices are not consecutive." + " Please check that the vocabulary is not corrupted!".format(vocab_file)) + index = token_index + writer.write(token + u'\n') + index += 1 + return vocab_file + + @classmethod + def from_pretrained(cls, model_dir, *inputs, **kwargs): + """ + 给定path，直接读取vocab. + + """ + pretrained_model_name_or_path = os.path.join(model_dir, VOCAB_NAME) + print("loading vocabulary file {}".format(pretrained_model_name_or_path)) + max_len = 512 + kwargs['max_len'] = min(kwargs.get('max_len', int(1e12)), max_len) + # Instantiate tokenizer. + tokenizer = cls(pretrained_model_name_or_path, *inputs, **kwargs) + return tokenizer + +VOCAB_NAME = 'vocab.txt' + +class _WordBertModel(nn.Module): + def __init__(self, model_dir:str, vocab:Vocabulary, layers:str='-1', pool_method:str='first', include_cls_sep:bool=False): + super().__init__() + + self.tokenzier = BertTokenizer.from_pretrained(model_dir) + self.encoder = BertModel.from_pretrained(model_dir) + # 检查encoder_layer_number是否合理 + encoder_layer_number = len(self.encoder.encoder.layer) + self.layers = list(map(int, layers.split(','))) + for layer in self.layers: + if layer<0: + assert -layer<=encoder_layer_number, f"The layer index:{layer} is out of scope for " \ + f"a bert model with {encoder_layer_number} layers." + else: + assert layer