diff --git a/fastNLP/core/dist_trainer.py b/fastNLP/core/dist_trainer.py index 4a423933..00db6361 100644 --- a/fastNLP/core/dist_trainer.py +++ b/fastNLP/core/dist_trainer.py @@ -1,3 +1,6 @@ +""" +正在开发中的分布式训练代码 +""" import torch import torch.cuda import torch.optim @@ -41,7 +44,8 @@ def get_local_rank(): class DistTrainer(): - """Distributed Trainer that support distributed and mixed precision training + """ + Distributed Trainer that support distributed and mixed precision training """ def __init__(self, train_data, model, optimizer=None, loss=None, callbacks_all=None, callbacks_master=None, diff --git a/fastNLP/embeddings/contextual_embedding.py b/fastNLP/embeddings/contextual_embedding.py index 1831af4e..152b0ab9 100644 --- a/fastNLP/embeddings/contextual_embedding.py +++ b/fastNLP/embeddings/contextual_embedding.py @@ -1,4 +1,3 @@ - from abc import abstractmethod import torch @@ -9,6 +8,10 @@ from ..core.sampler import SequentialSampler from ..core.utils import _move_model_to_device, _get_model_device from .embedding import TokenEmbedding +__all__ = [ + "ContextualEmbedding" +] + class ContextualEmbedding(TokenEmbedding): def __init__(self, vocab: Vocabulary, word_dropout:float=0.0, dropout:float=0.0): diff --git a/fastNLP/io/config_io.py b/fastNLP/io/config_io.py index 4acdbb96..ac349080 100644 --- a/fastNLP/io/config_io.py +++ b/fastNLP/io/config_io.py @@ -1,7 +1,9 @@ """ 用于读入和处理和保存 config 文件 - .. todo:: + +.. todo:: 这个模块中的类可能被抛弃？ + """ __all__ = [ "ConfigLoader", diff --git a/fastNLP/io/loader/conll.py b/fastNLP/io/loader/conll.py index 43790c15..b2c89ecc 100644 --- a/fastNLP/io/loader/conll.py +++ b/fastNLP/io/loader/conll.py @@ -1,12 +1,12 @@ from typing import Dict, Union from .loader import Loader -from ... import DataSet +from ...core.dataset import DataSet from ..file_reader import _read_conll -from ... import Instance +from ...core.instance import Instance from .. import DataBundle from ..utils import check_loader_paths -from ... import Const +from ...core.const import Const class ConllLoader(Loader): diff --git a/fastNLP/io/loader/cws.py b/fastNLP/io/loader/cws.py index 46c07f28..3af28116 100644 --- a/fastNLP/io/loader/cws.py +++ b/fastNLP/io/loader/cws.py @@ -1,6 +1,6 @@ - from .loader import Loader -from ...core import DataSet, Instance +from ...core.dataset import DataSet +from ...core.instance import Instance class CWSLoader(Loader): diff --git a/fastNLP/io/loader/loader.py b/fastNLP/io/loader/loader.py index 4cf5bcf3..c59de29f 100644 --- a/fastNLP/io/loader/loader.py +++ b/fastNLP/io/loader/loader.py @@ -1,4 +1,4 @@ -from ... import DataSet +from ...core.dataset import DataSet from .. import DataBundle from ..utils import check_loader_paths from typing import Union, Dict diff --git a/fastNLP/io/loader/matching.py b/fastNLP/io/loader/matching.py index 05f113c1..58fa0d6f 100644 --- a/fastNLP/io/loader/matching.py +++ b/fastNLP/io/loader/matching.py @@ -1,12 +1,12 @@ import warnings from .loader import Loader from .json import JsonLoader -from ...core import Const +from ...core.const import Const from .. import DataBundle import os from typing import Union, Dict -from ...core import DataSet -from ...core import Instance +from ...core.dataset import DataSet +from ...core.instance import Instance class MNLILoader(Loader): diff --git a/fastNLP/io/pipe/classification.py b/fastNLP/io/pipe/classification.py index d370a28a..1b111e40 100644 --- a/fastNLP/io/pipe/classification.py +++ b/fastNLP/io/pipe/classification.py @@ -4,13 +4,14 @@ from ..base_loader import DataBundle from ...core.vocabulary import Vocabulary from ...core.const import Const from ..loader.classification import IMDBLoader, YelpFullLoader, SSTLoader, SST2Loader, YelpPolarityLoader -from ...core import DataSet, Instance +from ...core.dataset import DataSet +from ...core.instance import Instance from .utils import get_tokenizer, _indexize, _add_words_field, _drop_empty_instance from .pipe import Pipe import re nonalpnum = re.compile('[^0-9a-zA-Z?!\']+') -from ...core import cache_results +from ...core.utils import cache_results class _CLSPipe(Pipe): """ diff --git a/fastNLP/io/pipe/conll.py b/fastNLP/io/pipe/conll.py index e62d1a05..b9007344 100644 --- a/fastNLP/io/pipe/conll.py +++ b/fastNLP/io/pipe/conll.py @@ -1,7 +1,7 @@ from .pipe import Pipe from .. import DataBundle from .utils import iob2, iob2bioes -from ... import Const +from ...core.const import Const from ..loader.conll import Conll2003NERLoader, OntoNotesNERLoader from .utils import _indexize, _add_words_field diff --git a/fastNLP/io/pipe/matching.py b/fastNLP/io/pipe/matching.py index 1e551f1d..93e854b1 100644 --- a/fastNLP/io/pipe/matching.py +++ b/fastNLP/io/pipe/matching.py @@ -2,8 +2,8 @@ import math from .pipe import Pipe from .utils import get_tokenizer -from ...core import Const -from ...core import Vocabulary +from ...core.const import Const +from ...core.vocabulary import Vocabulary from ..loader.matching import SNLILoader, MNLILoader, QNLILoader, RTELoader, QuoraLoader diff --git a/fastNLP/io/pipe/utils.py b/fastNLP/io/pipe/utils.py index 59bee96e..5e9ff8dc 100644 --- a/fastNLP/io/pipe/utils.py +++ b/fastNLP/io/pipe/utils.py @@ -1,6 +1,6 @@ from typing import List -from ...core import Vocabulary -from ...core import Const +from ...core.vocabulary import Vocabulary +from ...core.const import Const def iob2(tags:List[str])->List[str]: """ diff --git a/reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py b/reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py index cec5ab76..0d292bdc 100644 --- a/reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py +++ b/reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py @@ -51,7 +51,7 @@ class ChineseNERLoader(DataSetLoader): :param paths: :param bool, bigrams: 是否包含生成bigram feature, [a, b, c, d] -> [ab, bc, cd, d] :param bool, trigrams: 是否包含trigram feature，[a, b, c, d] -> [abc, bcd, cd, d] - :return: DataBundle + :return: ~fastNLP.io.DataBundle 包含以下的fields raw_chars: List[str] chars: List[int]