lyhuang18
GitHub
6 years ago
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 1025 additions and 1040 deletions
Unified View
Diff Options
-
+22 -133README.md
-
+4 -3docs/requirements.txt
-
+2 -0docs/source/conf.py
-
+26 -34docs/source/fastNLP.core.rst
-
+12 -38docs/source/fastNLP.loader.rst
-
+10 -26docs/source/fastNLP.models.rst
-
+12 -30docs/source/fastNLP.modules.aggregation.rst
-
+9 -13docs/source/fastNLP.modules.decoder.rst
-
+18 -42docs/source/fastNLP.modules.encoder.rst
-
+2 -7docs/source/fastNLP.modules.interaction.rst
-
+6 -21docs/source/fastNLP.modules.rst
-
+4 -17docs/source/fastNLP.rst
-
+6 -18docs/source/fastNLP.saver.rst
-
BINdocs/source/figures/procedures_and_sequence_labeling.png
-
BINdocs/source/figures/text_classification.png
-
+47 -9docs/source/index.rst
-
+0 -7docs/source/modules.rst
-
+31 -0docs/source/user/installation.rst
-
+84 -0docs/source/user/quickstart.rst
-
+57 -31fastNLP/core/action.py
-
+15 -1fastNLP/core/loss.py
-
+26 -31fastNLP/core/metrics.py
-
+10 -2fastNLP/core/optimizer.py
-
+1 -1fastNLP/core/predictor.py
-
+55 -11fastNLP/core/preprocess.py
-
+40 -20fastNLP/core/tester.py
-
+36 -45fastNLP/core/trainer.py
-
+55 -2fastNLP/fastnlp.py
-
+3 -4fastNLP/loader/base_loader.py
-
+2 -2fastNLP/loader/config_loader.py
-
+62 -18fastNLP/loader/dataset_loader.py
-
+44 -2fastNLP/loader/embed_loader.py
-
+2 -2fastNLP/loader/model_loader.py
-
+6 -6fastNLP/models/cnn_text_classification.py
-
+2 -1fastNLP/modules/decoder/__init__.py
-
+3 -1fastNLP/modules/encoder/__init__.py
-
+5 -0fastNLP/modules/encoder/conv_maxpool.py
-
+0 -114reproduction/chinese_word_seg/cws_train.py
-
+12 -0reproduction/chinese_word_segment/cws.cfg
-
+6 -6reproduction/chinese_word_segment/run.py
-
+17 -11reproduction/pos_tag_model/pos_tag.cfg
-
+146 -0reproduction/pos_tag_model/train_pos_tag.py
-
+1 -0requirements.txt
-
+24 -0setup.py
-
+1 -2test/core/test_action.py
-
+7 -8test/loader/test_loader.py
-
+0 -138test/ner.py
-
+0 -129test/ner_decode.py
-
+23 -27test/readme_example.py
-
+8 -8test/seq_labeling.py
-
+8 -8test/test_cws.py
-
+44 -2test/test_fastNLP.py
-
+7 -7test/test_tester.py
-
+2 -2test/text_classify.py
+ 22
- 133
README.md
View File
| @@ -16,93 +16,19 @@ fastNLP is a modular Natural Language Processing system based on PyTorch, for fa | |||||
| - numpy>=1.14.2 | - numpy>=1.14.2 | ||||
| - torch==0.4.0 | - torch==0.4.0 | ||||
| - torchvision>=0.1.8 | - torchvision>=0.1.8 | ||||
| - tensorboardX | |||||
| ## Resources | ## Resources | ||||
| - [Documentation](https://github.com/fastnlp/fastNLP) | |||||
| - [Documentation](https://fastnlp.readthedocs.io/en/latest/) | |||||
| - [Source Code](https://github.com/fastnlp/fastNLP) | - [Source Code](https://github.com/fastnlp/fastNLP) | ||||
| ## Example | |||||
| ### Basic Usage | |||||
| A typical fastNLP routine is composed of four phases: loading dataset, pre-processing data, constructing model and training model. | |||||
| ```python | |||||
| from fastNLP.models.base_model import BaseModel | |||||
| from fastNLP.modules import encoder | |||||
| from fastNLP.modules import aggregation | |||||
| from fastNLP.modules import decoder | |||||
| from fastNLP.loader.dataset_loader import ClassDatasetLoader | |||||
| from fastNLP.loader.preprocess import ClassPreprocess | |||||
| from fastNLP.core.trainer import ClassificationTrainer | |||||
| from fastNLP.core.inference import ClassificationInfer | |||||
| class ClassificationModel(BaseModel): | |||||
| """ | |||||
| Simple text classification model based on CNN. | |||||
| """ | |||||
| def __init__(self, num_classes, vocab_size): | |||||
| super(ClassificationModel, self).__init__() | |||||
| self.emb = encoder.Embedding(nums=vocab_size, dims=300) | |||||
| self.enc = encoder.Conv( | |||||
| in_channels=300, out_channels=100, kernel_size=3) | |||||
| self.agg = aggregation.MaxPool() | |||||
| self.dec = decoder.MLP(100, num_classes=num_classes) | |||||
| def forward(self, x): | |||||
| x = self.emb(x) # [N,L] -> [N,L,C] | |||||
| x = self.enc(x) # [N,L,C_in] -> [N,L,C_out] | |||||
| x = self.agg(x) # [N,L,C] -> [N,C] | |||||
| x = self.dec(x) # [N,C] -> [N, N_class] | |||||
| return x | |||||
| data_dir = 'data' # directory to save data and model | |||||
| train_path = 'test/data_for_tests/text_classify.txt' # training set file | |||||
| # load dataset | |||||
| ds_loader = ClassDatasetLoader("train", train_path) | |||||
| data = ds_loader.load() | |||||
| # pre-process dataset | |||||
| pre = ClassPreprocess(data_dir) | |||||
| vocab_size, n_classes = pre.process(data, "data_train.pkl") | |||||
| # construct model | |||||
| model_args = { | |||||
| 'num_classes': n_classes, | |||||
| 'vocab_size': vocab_size | |||||
| } | |||||
| model = ClassificationModel(num_classes=n_classes, vocab_size=vocab_size) | |||||
| # train model | |||||
| train_args = { | |||||
| "epochs": 20, | |||||
| "batch_size": 50, | |||||
| "pickle_path": data_dir, | |||||
| "validate": False, | |||||
| "save_best_dev": False, | |||||
| "model_saved_path": None, | |||||
| "use_cuda": True, | |||||
| "learn_rate": 1e-3, | |||||
| "momentum": 0.9} | |||||
| trainer = ClassificationTrainer(train_args) | |||||
| trainer.train(model) | |||||
| # predict using model | |||||
| seqs = [x[0] for x in data] | |||||
| infer = ClassificationInfer(data_dir) | |||||
| labels_pred = infer.predict(model, seqs) | |||||
| ``` | |||||
| ## Installation | ## Installation | ||||
| Run the following commands to install fastNLP package. | |||||
| ```shell | |||||
| pip install fastNLP | |||||
| ``` | |||||
| ### Cloning From GitHub | ### Cloning From GitHub | ||||
| @@ -122,20 +48,26 @@ conda install pytorch torchvision -c pytorch | |||||
| pip3 install torch torchvision | pip3 install torch torchvision | ||||
| ``` | ``` | ||||
| ### TensorboardX Installation | |||||
| ```shell | |||||
| pip3 install tensorboardX | |||||
| ``` | |||||
| ## Project Structure | ## Project Structure | ||||
| ``` | ``` | ||||
| FastNLP | FastNLP | ||||
| ├── docs | ├── docs | ||||
| │ └── quick_tutorial.md | |||||
| ├── fastNLP | ├── fastNLP | ||||
| │ ├── action | |||||
| │ ├── core | |||||
| │ │ ├── action.py | │ │ ├── action.py | ||||
| │ │ ├── inference.py | |||||
| │ │ ├── __init__.py | │ │ ├── __init__.py | ||||
| │ │ ├── loss.py | |||||
| │ │ ├── metrics.py | │ │ ├── metrics.py | ||||
| │ │ ├── optimizer.py | │ │ ├── optimizer.py | ||||
| │ │ ├── predictor.py | |||||
| │ │ ├── preprocess.py | |||||
| │ │ ├── README.md | │ │ ├── README.md | ||||
| │ │ ├── tester.py | │ │ ├── tester.py | ||||
| │ │ └── trainer.py | │ │ └── trainer.py | ||||
| @@ -147,71 +79,28 @@ FastNLP | |||||
| │ │ ├── dataset_loader.py | │ │ ├── dataset_loader.py | ||||
| │ │ ├── embed_loader.py | │ │ ├── embed_loader.py | ||||
| │ │ ├── __init__.py | │ │ ├── __init__.py | ||||
| │ │ ├── model_loader.py | |||||
| │ │ └── preprocess.py | |||||
| │ │ └── model_loader.py | |||||
| │ ├── models | │ ├── models | ||||
| │ │ ├── base_model.py | |||||
| │ │ ├── char_language_model.py | |||||
| │ │ ├── cnn_text_classification.py | |||||
| │ │ ├── __init__.py | |||||
| │ │ └── sequence_modeling.py | |||||
| │ ├── modules | │ ├── modules | ||||
| │ │ ├── aggregation | │ │ ├── aggregation | ||||
| │ │ │ ├── attention.py | |||||
| │ │ │ ├── avg_pool.py | |||||
| │ │ │ ├── __init__.py | |||||
| │ │ │ ├── kmax_pool.py | |||||
| │ │ │ ├── max_pool.py | |||||
| │ │ │ └── self_attention.py | |||||
| │ │ ├── decoder | │ │ ├── decoder | ||||
| │ │ │ ├── CRF.py | |||||
| │ │ │ └── __init__.py | |||||
| │ │ ├── encoder | │ │ ├── encoder | ||||
| │ │ │ ├── char_embedding.py | |||||
| │ │ │ ├── conv_maxpool.py | |||||
| │ │ │ ├── conv.py | |||||
| │ │ │ ├── embedding.py | |||||
| │ │ │ ├── __init__.py | |||||
| │ │ │ ├── linear.py | |||||
| │ │ │ ├── lstm.py | |||||
| │ │ │ ├── masked_rnn.py | |||||
| │ │ │ └── variational_rnn.py | |||||
| │ │ ├── __init__.py | │ │ ├── __init__.py | ||||
| │ │ ├── interaction | │ │ ├── interaction | ||||
| │ │ │ └── __init__.py | |||||
| │ │ ├── other_modules.py | │ │ ├── other_modules.py | ||||
| │ │ └── utils.py | │ │ └── utils.py | ||||
| │ └── saver | │ └── saver | ||||
| │ ├── base_saver.py | |||||
| │ ├── __init__.py | |||||
| │ ├── logger.py | |||||
| │ └── model_saver.py | |||||
| ├── LICENSE | ├── LICENSE | ||||
| ├── README.md | ├── README.md | ||||
| ├── reproduction | ├── reproduction | ||||
| │ ├── Char-aware_NLM | |||||
| │ │ | |||||
| │ ├── CNN-sentence_classification | |||||
| │ │ | |||||
| │ ├── HAN-document_classification | |||||
| │ │ | |||||
| │ └── LSTM+self_attention_sentiment_analysis | |||||
| | | |||||
| ├── requirements.txt | ├── requirements.txt | ||||
| ├── setup.py | ├── setup.py | ||||
| └── test | └── test | ||||
| ├── core | |||||
| ├── data_for_tests | ├── data_for_tests | ||||
| │ ├── charlm.txt | |||||
| │ ├── config | |||||
| │ ├── cws_test | |||||
| │ ├── cws_train | |||||
| │ ├── people_infer.txt | |||||
| │ └── people.txt | |||||
| ├── test_charlm.py | |||||
| ├── test_cws.py | |||||
| ├── test_fastNLP.py | |||||
| ├── test_loader.py | |||||
| ├── test_seq_labeling.py | |||||
| ├── test_tester.py | |||||
| └── test_trainer.py | |||||
| ├── __init__.py | |||||
| ├── loader | |||||
| ├── modules | |||||
| └── readme_example.py | |||||
| ``` | ``` | ||||
+ 4
- 3
docs/requirements.txt
View File
| @@ -1,3 +1,4 @@ | |||||
| sphinx | |||||
| -e git://github.com/snide/sphinx_rtd_theme.git#egg=sphinx_rtd_theme | |||||
| sphinxcontrib.katex | |||||
| numpy>=1.14.2 | |||||
| http://download.pytorch.org/whl/cpu/torch-0.4.1-cp35-cp35m-linux_x86_64.whl | |||||
| torchvision>=0.1.8 | |||||
| sphinx-rtd-theme==0.4.1 | |||||
+ 2
- 0
docs/source/conf.py
View File
| @@ -42,6 +42,8 @@ release = '1.0' | |||||
| extensions = [ | extensions = [ | ||||
| 'sphinx.ext.autodoc', | 'sphinx.ext.autodoc', | ||||
| 'sphinx.ext.viewcode', | 'sphinx.ext.viewcode', | ||||
| 'sphinx.ext.autosummary', | |||||
| ] | ] | ||||
| # Add any paths that contain templates here, relative to this directory. | # Add any paths that contain templates here, relative to this directory. | ||||
+ 26
- 34
docs/source/fastNLP.core.rst
View File
| @@ -1,62 +1,54 @@ | |||||
| fastNLP.core package | |||||
| ==================== | |||||
| fastNLP.core | |||||
| ============= | |||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.core.action module | |||||
| -------------------------- | |||||
| fastNLP.core.action | |||||
| -------------------- | |||||
| .. automodule:: fastNLP.core.action | .. automodule:: fastNLP.core.action | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.core.metrics module | |||||
| --------------------------- | |||||
| fastNLP.core.loss | |||||
| ------------------ | |||||
| .. automodule:: fastNLP.core.loss | |||||
| :members: | |||||
| fastNLP.core.metrics | |||||
| --------------------- | |||||
| .. automodule:: fastNLP.core.metrics | .. automodule:: fastNLP.core.metrics | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.core.optimizer module | |||||
| ----------------------------- | |||||
| fastNLP.core.optimizer | |||||
| ----------------------- | |||||
| .. automodule:: fastNLP.core.optimizer | .. automodule:: fastNLP.core.optimizer | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.core.predictor module | |||||
| ----------------------------- | |||||
| fastNLP.core.predictor | |||||
| ----------------------- | |||||
| .. automodule:: fastNLP.core.predictor | .. automodule:: fastNLP.core.predictor | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.core.tester module | |||||
| -------------------------- | |||||
| fastNLP.core.preprocess | |||||
| ------------------------ | |||||
| .. automodule:: fastNLP.core.preprocess | |||||
| :members: | |||||
| fastNLP.core.tester | |||||
| -------------------- | |||||
| .. automodule:: fastNLP.core.tester | .. automodule:: fastNLP.core.tester | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.core.trainer module | |||||
| --------------------------- | |||||
| fastNLP.core.trainer | |||||
| --------------------- | |||||
| .. automodule:: fastNLP.core.trainer | .. automodule:: fastNLP.core.trainer | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.core | .. automodule:: fastNLP.core | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 12
- 38
docs/source/fastNLP.loader.rst
View File
| @@ -1,62 +1,36 @@ | |||||
| fastNLP.loader package | |||||
| ====================== | |||||
| fastNLP.loader | |||||
| =============== | |||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.loader.base\_loader module | |||||
| ---------------------------------- | |||||
| fastNLP.loader.base\_loader | |||||
| ---------------------------- | |||||
| .. automodule:: fastNLP.loader.base_loader | .. automodule:: fastNLP.loader.base_loader | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.loader.config\_loader module | |||||
| ------------------------------------ | |||||
| fastNLP.loader.config\_loader | |||||
| ------------------------------ | |||||
| .. automodule:: fastNLP.loader.config_loader | .. automodule:: fastNLP.loader.config_loader | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.loader.dataset\_loader module | |||||
| ------------------------------------- | |||||
| fastNLP.loader.dataset\_loader | |||||
| ------------------------------- | |||||
| .. automodule:: fastNLP.loader.dataset_loader | .. automodule:: fastNLP.loader.dataset_loader | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.loader.embed\_loader module | |||||
| ----------------------------------- | |||||
| fastNLP.loader.embed\_loader | |||||
| ----------------------------- | |||||
| .. automodule:: fastNLP.loader.embed_loader | .. automodule:: fastNLP.loader.embed_loader | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.loader.model\_loader module | |||||
| ----------------------------------- | |||||
| fastNLP.loader.model\_loader | |||||
| ----------------------------- | |||||
| .. automodule:: fastNLP.loader.model_loader | .. automodule:: fastNLP.loader.model_loader | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.loader.preprocess module | |||||
| -------------------------------- | |||||
| .. automodule:: fastNLP.loader.preprocess | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.loader | .. automodule:: fastNLP.loader | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 10
- 26
docs/source/fastNLP.models.rst
View File
| @@ -1,46 +1,30 @@ | |||||
| fastNLP.models package | |||||
| ====================== | |||||
| fastNLP.models | |||||
| =============== | |||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.models.base\_model module | |||||
| --------------------------------- | |||||
| fastNLP.models.base\_model | |||||
| --------------------------- | |||||
| .. automodule:: fastNLP.models.base_model | .. automodule:: fastNLP.models.base_model | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.models.char\_language\_model module | |||||
| ------------------------------------------- | |||||
| fastNLP.models.char\_language\_model | |||||
| ------------------------------------- | |||||
| .. automodule:: fastNLP.models.char_language_model | .. automodule:: fastNLP.models.char_language_model | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.models.cnn\_text\_classification module | |||||
| ----------------------------------------------- | |||||
| fastNLP.models.cnn\_text\_classification | |||||
| ----------------------------------------- | |||||
| .. automodule:: fastNLP.models.cnn_text_classification | .. automodule:: fastNLP.models.cnn_text_classification | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.models.sequence\_modeling module | |||||
| ---------------------------------------- | |||||
| fastNLP.models.sequence\_modeling | |||||
| ---------------------------------- | |||||
| .. automodule:: fastNLP.models.sequence_modeling | .. automodule:: fastNLP.models.sequence_modeling | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.models | .. automodule:: fastNLP.models | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 12
- 30
docs/source/fastNLP.modules.aggregation.rst
View File
| @@ -1,54 +1,36 @@ | |||||
| fastNLP.modules.aggregation package | |||||
| =================================== | |||||
| fastNLP.modules.aggregation | |||||
| ============================ | |||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.modules.aggregation.attention module | |||||
| -------------------------------------------- | |||||
| fastNLP.modules.aggregation.attention | |||||
| -------------------------------------- | |||||
| .. automodule:: fastNLP.modules.aggregation.attention | .. automodule:: fastNLP.modules.aggregation.attention | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.aggregation.avg\_pool module | |||||
| -------------------------------------------- | |||||
| fastNLP.modules.aggregation.avg\_pool | |||||
| -------------------------------------- | |||||
| .. automodule:: fastNLP.modules.aggregation.avg_pool | .. automodule:: fastNLP.modules.aggregation.avg_pool | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.aggregation.kmax\_pool module | |||||
| --------------------------------------------- | |||||
| fastNLP.modules.aggregation.kmax\_pool | |||||
| --------------------------------------- | |||||
| .. automodule:: fastNLP.modules.aggregation.kmax_pool | .. automodule:: fastNLP.modules.aggregation.kmax_pool | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.aggregation.max\_pool module | |||||
| -------------------------------------------- | |||||
| fastNLP.modules.aggregation.max\_pool | |||||
| -------------------------------------- | |||||
| .. automodule:: fastNLP.modules.aggregation.max_pool | .. automodule:: fastNLP.modules.aggregation.max_pool | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.aggregation.self\_attention module | |||||
| -------------------------------------------------- | |||||
| fastNLP.modules.aggregation.self\_attention | |||||
| -------------------------------------------- | |||||
| .. automodule:: fastNLP.modules.aggregation.self_attention | .. automodule:: fastNLP.modules.aggregation.self_attention | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.modules.aggregation | .. automodule:: fastNLP.modules.aggregation | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 9
- 13
docs/source/fastNLP.modules.decoder.rst
View File
| @@ -1,22 +1,18 @@ | |||||
| fastNLP.modules.decoder package | |||||
| =============================== | |||||
| fastNLP.modules.decoder | |||||
| ======================== | |||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.modules.decoder.CRF module | |||||
| ---------------------------------- | |||||
| fastNLP.modules.decoder.CRF | |||||
| ---------------------------- | |||||
| .. automodule:: fastNLP.modules.decoder.CRF | .. automodule:: fastNLP.modules.decoder.CRF | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.decoder.MLP | |||||
| ---------------------------- | |||||
| .. automodule:: fastNLP.modules.decoder.MLP | |||||
| :members: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.modules.decoder | .. automodule:: fastNLP.modules.decoder | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 18
- 42
docs/source/fastNLP.modules.encoder.rst
View File
| @@ -1,78 +1,54 @@ | |||||
| fastNLP.modules.encoder package | |||||
| =============================== | |||||
| fastNLP.modules.encoder | |||||
| ======================== | |||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.modules.encoder.char\_embedding module | |||||
| ---------------------------------------------- | |||||
| fastNLP.modules.encoder.char\_embedding | |||||
| ---------------------------------------- | |||||
| .. automodule:: fastNLP.modules.encoder.char_embedding | .. automodule:: fastNLP.modules.encoder.char_embedding | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.encoder.conv module | |||||
| ----------------------------------- | |||||
| fastNLP.modules.encoder.conv | |||||
| ----------------------------- | |||||
| .. automodule:: fastNLP.modules.encoder.conv | .. automodule:: fastNLP.modules.encoder.conv | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.encoder.conv\_maxpool module | |||||
| -------------------------------------------- | |||||
| fastNLP.modules.encoder.conv\_maxpool | |||||
| -------------------------------------- | |||||
| .. automodule:: fastNLP.modules.encoder.conv_maxpool | .. automodule:: fastNLP.modules.encoder.conv_maxpool | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.encoder.embedding module | |||||
| ---------------------------------------- | |||||
| fastNLP.modules.encoder.embedding | |||||
| ---------------------------------- | |||||
| .. automodule:: fastNLP.modules.encoder.embedding | .. automodule:: fastNLP.modules.encoder.embedding | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.encoder.linear module | |||||
| ------------------------------------- | |||||
| fastNLP.modules.encoder.linear | |||||
| ------------------------------- | |||||
| .. automodule:: fastNLP.modules.encoder.linear | .. automodule:: fastNLP.modules.encoder.linear | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.encoder.lstm module | |||||
| ----------------------------------- | |||||
| fastNLP.modules.encoder.lstm | |||||
| ----------------------------- | |||||
| .. automodule:: fastNLP.modules.encoder.lstm | .. automodule:: fastNLP.modules.encoder.lstm | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.encoder.masked\_rnn module | |||||
| ------------------------------------------ | |||||
| fastNLP.modules.encoder.masked\_rnn | |||||
| ------------------------------------ | |||||
| .. automodule:: fastNLP.modules.encoder.masked_rnn | .. automodule:: fastNLP.modules.encoder.masked_rnn | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.encoder.variational\_rnn module | |||||
| ----------------------------------------------- | |||||
| fastNLP.modules.encoder.variational\_rnn | |||||
| ----------------------------------------- | |||||
| .. automodule:: fastNLP.modules.encoder.variational_rnn | .. automodule:: fastNLP.modules.encoder.variational_rnn | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.modules.encoder | .. automodule:: fastNLP.modules.encoder | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 2
- 7
docs/source/fastNLP.modules.interaction.rst
View File
| @@ -1,10 +1,5 @@ | |||||
| fastNLP.modules.interaction package | |||||
| =================================== | |||||
| Module contents | |||||
| --------------- | |||||
| fastNLP.modules.interaction | |||||
| ============================ | |||||
| .. automodule:: fastNLP.modules.interaction | .. automodule:: fastNLP.modules.interaction | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 6
- 21
docs/source/fastNLP.modules.rst
View File
| @@ -1,8 +1,5 @@ | |||||
| fastNLP.modules package | |||||
| ======================= | |||||
| Subpackages | |||||
| ----------- | |||||
| fastNLP.modules | |||||
| ================ | |||||
| .. toctree:: | .. toctree:: | ||||
| @@ -11,30 +8,18 @@ Subpackages | |||||
| fastNLP.modules.encoder | fastNLP.modules.encoder | ||||
| fastNLP.modules.interaction | fastNLP.modules.interaction | ||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.modules.other\_modules module | |||||
| ------------------------------------- | |||||
| fastNLP.modules.other\_modules | |||||
| ------------------------------- | |||||
| .. automodule:: fastNLP.modules.other_modules | .. automodule:: fastNLP.modules.other_modules | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.modules.utils module | |||||
| ---------------------------- | |||||
| fastNLP.modules.utils | |||||
| ---------------------- | |||||
| .. automodule:: fastNLP.modules.utils | .. automodule:: fastNLP.modules.utils | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.modules | .. automodule:: fastNLP.modules | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 4
- 17
docs/source/fastNLP.rst
View File
| @@ -1,8 +1,5 @@ | |||||
| fastNLP package | |||||
| =============== | |||||
| Subpackages | |||||
| ----------- | |||||
| fastNLP | |||||
| ======== | |||||
| .. toctree:: | .. toctree:: | ||||
| @@ -12,22 +9,12 @@ Subpackages | |||||
| fastNLP.modules | fastNLP.modules | ||||
| fastNLP.saver | fastNLP.saver | ||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.fastnlp module | |||||
| ---------------------- | |||||
| fastNLP.fastnlp | |||||
| ---------------- | |||||
| .. automodule:: fastNLP.fastnlp | .. automodule:: fastNLP.fastnlp | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP | .. automodule:: fastNLP | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
+ 6
- 18
docs/source/fastNLP.saver.rst
View File
| @@ -1,30 +1,18 @@ | |||||
| fastNLP.saver package | |||||
| ===================== | |||||
| fastNLP.saver | |||||
| ============== | |||||
| Submodules | |||||
| ---------- | |||||
| fastNLP.saver.logger module | |||||
| --------------------------- | |||||
| fastNLP.saver.logger | |||||
| --------------------- | |||||
| .. automodule:: fastNLP.saver.logger | .. automodule:: fastNLP.saver.logger | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| fastNLP.saver.model\_saver module | |||||
| --------------------------------- | |||||
| fastNLP.saver.model\_saver | |||||
| --------------------------- | |||||
| .. automodule:: fastNLP.saver.model_saver | .. automodule:: fastNLP.saver.model_saver | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Module contents | |||||
| --------------- | |||||
| .. automodule:: fastNLP.saver | .. automodule:: fastNLP.saver | ||||
| :members: | :members: | ||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
BIN
docs/source/figures/procedures_and_sequence_labeling.png
View File
| Before | After |
|---|---|
|
|
| Width: 1079 | Height: 558 | Size: 51 kB |
BIN
docs/source/figures/text_classification.png
View File
| Before | After |
|---|---|
|
|
| Width: 1217 | Height: 543 | Size: 54 kB |
+ 47
- 9
docs/source/index.rst
View File
| @@ -1,16 +1,54 @@ | |||||
| .. fastNLP documentation master file, created by | |||||
| sphinx-quickstart on Mon Aug 20 17:06:44 2018. | |||||
| You can adapt this file completely to your liking, but it should at least | |||||
| contain the root `toctree` directive. | |||||
| fastNLP documentation | |||||
| ===================== | |||||
| fastNLP，目前仍在孵化中。 | |||||
| Welcome to fastNLP's documentation! | |||||
| =================================== | |||||
| Introduction | |||||
| ------------ | |||||
| fastNLP是一个基于PyTorch的模块化自然语言处理系统，用于快速开发NLP工具。 | |||||
| 它将基于深度学习的NLP模型划分为不同的模块。 | |||||
| 这些模块分为4类：encoder（编码），interaction（交互）, aggregration（聚合） and decoder（解码）， | |||||
| 而每个类别包含不同的实现模块。 | |||||
| 大多数当前的NLP模型可以构建在这些模块上，这极大地简化了开发NLP模型的过程。 | |||||
| fastNLP的架构如下左图所示： | |||||
| .. image:: figures/procedures_and_sequence_labeling.png | |||||
| 在constructing model部分，以序列标注（上右图）和文本分类（下图）为例进行说明： | |||||
| .. image:: figures/text_classification.png | |||||
| * encoder module：将输入编码为一些抽象表示，输入的是单词序列，输出向量序列。 | |||||
| * interaction module：使表示中的信息相互交互，输入的是向量序列，输出的也是向量序列。 | |||||
| * aggregation module：聚合和减少信息，输入向量序列，输出一个向量。 | |||||
| * decoder module：将表示解码为输出，输出一个label（文本分类）或者输出label序列（序列标注） | |||||
| 其中interaction module和aggregation module在模型中不一定存在，例如上面的序列标注模型。 | |||||
| User's Guide | |||||
| ------------ | |||||
| .. toctree:: | |||||
| :maxdepth: 2 | |||||
| user/installation | |||||
| user/quickstart | |||||
| API Reference | |||||
| ------------- | |||||
| If you are looking for information on a specific function, class or | |||||
| method, this part of the documentation is for you. | |||||
| .. toctree:: | .. toctree:: | ||||
| :maxdepth: 4 | |||||
| :caption: Contents: | |||||
| :maxdepth: 2 | |||||
| fastNLP | |||||
| fastNLP API <fastNLP> | |||||
+ 0
- 7
docs/source/modules.rst
View File
| @@ -1,7 +0,0 @@ | |||||
| fastNLP | |||||
| ======= | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| fastNLP | |||||
+ 31
- 0
docs/source/user/installation.rst
View File
| @@ -0,0 +1,31 @@ | |||||
| ============ | |||||
| Installation | |||||
| ============ | |||||
| .. contents:: | |||||
| :local: | |||||
| Cloning From GitHub | |||||
| ~~~~~~~~~~~~~~~~~~~ | |||||
| If you just want to use fastNLP, use: | |||||
| .. code:: shell | |||||
| git clone https://github.com/fastnlp/fastNLP | |||||
| cd fastNLP | |||||
| PyTorch Installation | |||||
| ~~~~~~~~~~~~~~~~~~~~ | |||||
| Visit the [PyTorch official website] for installation instructions based | |||||
| on your system. In general, you could use: | |||||
| .. code:: shell | |||||
| # using conda | |||||
| conda install pytorch torchvision -c pytorch | |||||
| # or using pip | |||||
| pip3 install torch torchvision | |||||
+ 84
- 0
docs/source/user/quickstart.rst
View File
| @@ -0,0 +1,84 @@ | |||||
| ========== | |||||
| Quickstart | |||||
| ========== | |||||
| Example | |||||
| ------- | |||||
| Basic Usage | |||||
| ~~~~~~~~~~~ | |||||
| A typical fastNLP routine is composed of four phases: loading dataset, | |||||
| pre-processing data, constructing model and training model. | |||||
| .. code:: python | |||||
| from fastNLP.models.base_model import BaseModel | |||||
| from fastNLP.modules import encoder | |||||
| from fastNLP.modules import aggregation | |||||
| from fastNLP.modules import decoder | |||||
| from fastNLP.loader.dataset_loader import ClassDatasetLoader | |||||
| from fastNLP.loader.preprocess import ClassPreprocess | |||||
| from fastNLP.core.trainer import ClassificationTrainer | |||||
| from fastNLP.core.inference import ClassificationInfer | |||||
| class ClassificationModel(BaseModel): | |||||
| """ | |||||
| Simple text classification model based on CNN. | |||||
| """ | |||||
| def __init__(self, num_classes, vocab_size): | |||||
| super(ClassificationModel, self).__init__() | |||||
| self.emb = encoder.Embedding(nums=vocab_size, dims=300) | |||||
| self.enc = encoder.Conv( | |||||
| in_channels=300, out_channels=100, kernel_size=3) | |||||
| self.agg = aggregation.MaxPool() | |||||
| self.dec = decoder.MLP(100, num_classes=num_classes) | |||||
| def forward(self, x): | |||||
| x = self.emb(x) # [N,L] -> [N,L,C] | |||||
| x = self.enc(x) # [N,L,C_in] -> [N,L,C_out] | |||||
| x = self.agg(x) # [N,L,C] -> [N,C] | |||||
| x = self.dec(x) # [N,C] -> [N, N_class] | |||||
| return x | |||||
| data_dir = 'data' # directory to save data and model | |||||
| train_path = 'test/data_for_tests/text_classify.txt' # training set file | |||||
| # load dataset | |||||
| ds_loader = ClassDatasetLoader("train", train_path) | |||||
| data = ds_loader.load() | |||||
| # pre-process dataset | |||||
| pre = ClassPreprocess(data_dir) | |||||
| vocab_size, n_classes = pre.process(data, "data_train.pkl") | |||||
| # construct model | |||||
| model_args = { | |||||
| 'num_classes': n_classes, | |||||
| 'vocab_size': vocab_size | |||||
| } | |||||
| model = ClassificationModel(num_classes=n_classes, vocab_size=vocab_size) | |||||
| # train model | |||||
| train_args = { | |||||
| "epochs": 20, | |||||
| "batch_size": 50, | |||||
| "pickle_path": data_dir, | |||||
| "validate": False, | |||||
| "save_best_dev": False, | |||||
| "model_saved_path": None, | |||||
| "use_cuda": True, | |||||
| "learn_rate": 1e-3, | |||||
| "momentum": 0.9} | |||||
| trainer = ClassificationTrainer(train_args) | |||||
| trainer.train(model) | |||||
| # predict using model | |||||
| seqs = [x[0] for x in data] | |||||
| infer = ClassificationInfer(data_dir) | |||||
| labels_pred = infer.predict(model, seqs) | |||||
+ 57
- 31
fastNLP/core/action.py
View File
| @@ -1,7 +1,3 @@ | |||||
| """ | |||||
| This file defines Action(s) and sample methods. | |||||
| """ | |||||
| from collections import Counter | from collections import Counter | ||||
| import numpy as np | import numpy as np | ||||
| @@ -9,13 +5,12 @@ import torch | |||||
| class Action(object): | class Action(object): | ||||
| """ | |||||
| Operations shared by Trainer, Tester, or Inference. | |||||
| """Operations shared by Trainer, Tester, or Inference. | |||||
| This is designed for reducing replicate codes. | This is designed for reducing replicate codes. | ||||
| - make_batch: produce a min-batch of data. @staticmethod | - make_batch: produce a min-batch of data. @staticmethod | ||||
| - pad: padding method used in sequence modeling. @staticmethod | - pad: padding method used in sequence modeling. @staticmethod | ||||
| - mode: change network mode for either train or test. (for PyTorch) @staticmethod | - mode: change network mode for either train or test. (for PyTorch) @staticmethod | ||||
| The base Action shall define operations shared by as much task-specific Actions as possible. | |||||
| """ | """ | ||||
| def __init__(self): | def __init__(self): | ||||
| @@ -24,18 +19,20 @@ class Action(object): | |||||
| @staticmethod | @staticmethod | ||||
| def make_batch(iterator, use_cuda, output_length=True, max_len=None): | def make_batch(iterator, use_cuda, output_length=True, max_len=None): | ||||
| """Batch and Pad data. | """Batch and Pad data. | ||||
| :param iterator: an iterator, (object that implements __next__ method) which returns the next sample. | :param iterator: an iterator, (object that implements __next__ method) which returns the next sample. | ||||
| :param use_cuda: bool, whether to use GPU | :param use_cuda: bool, whether to use GPU | ||||
| :param output_length: bool, whether to output the original length of the sequence before padding. (default: True) | :param output_length: bool, whether to output the original length of the sequence before padding. (default: True) | ||||
| :param max_len: int, maximum sequence length. Longer sequences will be clipped. (default: None) | :param max_len: int, maximum sequence length. Longer sequences will be clipped. (default: None) | ||||
| :return | |||||
| if output_length is True: | |||||
| :return : | |||||
| if output_length is True, | |||||
| (batch_x, seq_len): tuple of two elements | (batch_x, seq_len): tuple of two elements | ||||
| batch_x: list. Each entry is a list of features of a sample. [batch_size, max_len] | batch_x: list. Each entry is a list of features of a sample. [batch_size, max_len] | ||||
| seq_len: list. The length of the pre-padded sequence, if output_length is True. | seq_len: list. The length of the pre-padded sequence, if output_length is True. | ||||
| batch_y: list. Each entry is a list of labels of a sample. [batch_size, num_labels] | batch_y: list. Each entry is a list of labels of a sample. [batch_size, num_labels] | ||||
| if output_length is False: | |||||
| if output_length is False, | |||||
| batch_x: list. Each entry is a list of features of a sample. [batch_size, max_len] | batch_x: list. Each entry is a list of features of a sample. [batch_size, max_len] | ||||
| batch_y: list. Each entry is a list of labels of a sample. [batch_size, num_labels] | batch_y: list. Each entry is a list of labels of a sample. [batch_size, num_labels] | ||||
| """ | """ | ||||
| @@ -77,21 +74,21 @@ class Action(object): | |||||
| return batch | return batch | ||||
| @staticmethod | @staticmethod | ||||
| def mode(model, test=False): | |||||
| """ | |||||
| Train mode or Test mode. This is for PyTorch currently. | |||||
| :param model: | |||||
| :param test: | |||||
| def mode(model, is_test=False): | |||||
| """Train mode or Test mode. This is for PyTorch currently. | |||||
| :param model: a PyTorch model | |||||
| :param is_test: bool, whether in test mode or not. | |||||
| """ | """ | ||||
| if test: | |||||
| if is_test: | |||||
| model.eval() | model.eval() | ||||
| else: | else: | ||||
| model.train() | model.train() | ||||
| def convert_to_torch_tensor(data_list, use_cuda): | def convert_to_torch_tensor(data_list, use_cuda): | ||||
| """ | |||||
| convert lists into (cuda) Tensors. | |||||
| """Convert lists into (cuda) Tensors. | |||||
| :param data_list: 2-level lists | :param data_list: 2-level lists | ||||
| :param use_cuda: bool, whether to use GPU or not | :param use_cuda: bool, whether to use GPU or not | ||||
| :return data_list: PyTorch Tensor of shape [batch_size, max_seq_len] | :return data_list: PyTorch Tensor of shape [batch_size, max_seq_len] | ||||
| @@ -103,8 +100,8 @@ def convert_to_torch_tensor(data_list, use_cuda): | |||||
| def k_means_1d(x, k, max_iter=100): | def k_means_1d(x, k, max_iter=100): | ||||
| """ | |||||
| Perform k-means on 1-D data. | |||||
| """Perform k-means on 1-D data. | |||||
| :param x: list of int, representing points in 1-D. | :param x: list of int, representing points in 1-D. | ||||
| :param k: the number of clusters required. | :param k: the number of clusters required. | ||||
| :param max_iter: maximum iteration | :param max_iter: maximum iteration | ||||
| @@ -132,21 +129,28 @@ def k_means_1d(x, k, max_iter=100): | |||||
| def k_means_bucketing(all_inst, buckets): | def k_means_bucketing(all_inst, buckets): | ||||
| """ | |||||
| """Assign all instances into possible buckets using k-means, such that instances in the same bucket have similar lengths. | |||||
| :param all_inst: 3-level list | :param all_inst: 3-level list | ||||
| E.g. :: | |||||
| [ | [ | ||||
| [[word_11, word_12, word_13], [label_11. label_12]], # sample 1 | [[word_11, word_12, word_13], [label_11. label_12]], # sample 1 | ||||
| [[word_21, word_22, word_23], [label_21. label_22]], # sample 2 | [[word_21, word_22, word_23], [label_21. label_22]], # sample 2 | ||||
| ... | ... | ||||
| ] | ] | ||||
| :param buckets: list of int. The length of the list is the number of buckets. Each integer is the maximum length | :param buckets: list of int. The length of the list is the number of buckets. Each integer is the maximum length | ||||
| threshold for each bucket (This is usually None.). | threshold for each bucket (This is usually None.). | ||||
| :return data: 2-level list | :return data: 2-level list | ||||
| :: | |||||
| [ | [ | ||||
| [index_11, index_12, ...], # bucket 1 | [index_11, index_12, ...], # bucket 1 | ||||
| [index_21, index_22, ...], # bucket 2 | [index_21, index_22, ...], # bucket 2 | ||||
| ... | ... | ||||
| ] | ] | ||||
| """ | """ | ||||
| bucket_data = [[] for _ in buckets] | bucket_data = [[] for _ in buckets] | ||||
| num_buckets = len(buckets) | num_buckets = len(buckets) | ||||
| @@ -160,11 +164,16 @@ def k_means_bucketing(all_inst, buckets): | |||||
| class BaseSampler(object): | class BaseSampler(object): | ||||
| """ | |||||
| Base class for all samplers. | |||||
| """The base class of all samplers. | |||||
| """ | """ | ||||
| def __init__(self, data_set): | def __init__(self, data_set): | ||||
| """ | |||||
| :param data_set: multi-level list, of shape [num_example, *] | |||||
| """ | |||||
| self.data_set_length = len(data_set) | self.data_set_length = len(data_set) | ||||
| self.data = data_set | self.data = data_set | ||||
| @@ -176,11 +185,16 @@ class BaseSampler(object): | |||||
| class SequentialSampler(BaseSampler): | class SequentialSampler(BaseSampler): | ||||
| """ | |||||
| Sample data in the original order. | |||||
| """Sample data in the original order. | |||||
| """ | """ | ||||
| def __init__(self, data_set): | def __init__(self, data_set): | ||||
| """ | |||||
| :param data_set: multi-level list | |||||
| """ | |||||
| super(SequentialSampler, self).__init__(data_set) | super(SequentialSampler, self).__init__(data_set) | ||||
| def __iter__(self): | def __iter__(self): | ||||
| @@ -188,11 +202,16 @@ class SequentialSampler(BaseSampler): | |||||
| class RandomSampler(BaseSampler): | class RandomSampler(BaseSampler): | ||||
| """ | |||||
| Sample data in random permutation order. | |||||
| """Sample data in random permutation order. | |||||
| """ | """ | ||||
| def __init__(self, data_set): | def __init__(self, data_set): | ||||
| """ | |||||
| :param data_set: multi-level list | |||||
| """ | |||||
| super(RandomSampler, self).__init__(data_set) | super(RandomSampler, self).__init__(data_set) | ||||
| self.order = np.random.permutation(self.data_set_length) | self.order = np.random.permutation(self.data_set_length) | ||||
| @@ -201,11 +220,18 @@ class RandomSampler(BaseSampler): | |||||
| class Batchifier(object): | class Batchifier(object): | ||||
| """ | |||||
| Wrap random or sequential sampler to generate a mini-batch. | |||||
| """Wrap random or sequential sampler to generate a mini-batch. | |||||
| """ | """ | ||||
| def __init__(self, sampler, batch_size, drop_last=True): | def __init__(self, sampler, batch_size, drop_last=True): | ||||
| """ | |||||
| :param sampler: a Sampler object | |||||
| :param batch_size: int, the size of the mini-batch | |||||
| :param drop_last: bool, whether to drop the last examples that are not enough to make a mini-batch. | |||||
| """ | |||||
| super(Batchifier, self).__init__() | super(Batchifier, self).__init__() | ||||
| self.sampler = sampler | self.sampler = sampler | ||||
| self.batch_size = batch_size | self.batch_size = batch_size | ||||
| @@ -223,8 +249,7 @@ class Batchifier(object): | |||||
| class BucketBatchifier(Batchifier): | class BucketBatchifier(Batchifier): | ||||
| """ | |||||
| Partition all samples into multiple buckets, each of which contains sentences of approximately the same length. | |||||
| """Partition all samples into multiple buckets, each of which contains sentences of approximately the same length. | |||||
| In sampling, first random choose a bucket. Then sample data from it. | In sampling, first random choose a bucket. Then sample data from it. | ||||
| The number of buckets is decided dynamically by the variance of sentence lengths. | The number of buckets is decided dynamically by the variance of sentence lengths. | ||||
| """ | """ | ||||
| @@ -237,6 +262,7 @@ class BucketBatchifier(Batchifier): | |||||
| :param num_buckets: int, number of buckets for grouping these sequences. | :param num_buckets: int, number of buckets for grouping these sequences. | ||||
| :param drop_last: bool, useless currently. | :param drop_last: bool, useless currently. | ||||
| :param sampler: Sampler, useless currently. | :param sampler: Sampler, useless currently. | ||||
| """ | """ | ||||
| super(BucketBatchifier, self).__init__(sampler, batch_size, drop_last) | super(BucketBatchifier, self).__init__(sampler, batch_size, drop_last) | ||||
| buckets = ([None] * num_buckets) | buckets = ([None] * num_buckets) | ||||
+ 15
- 1
fastNLP/core/loss.py
View File
| @@ -8,8 +8,13 @@ class Loss(object): | |||||
| """ | """ | ||||
| def __init__(self, args): | def __init__(self, args): | ||||
| """ | |||||
| :param args: None or str, the name of a loss function. | |||||
| """ | |||||
| if args is None: | if args is None: | ||||
| # this is useful when | |||||
| # this is useful when Trainer.__init__ performs type check | |||||
| self._loss = None | self._loss = None | ||||
| elif isinstance(args, str): | elif isinstance(args, str): | ||||
| self._loss = self._borrow_from_pytorch(args) | self._loss = self._borrow_from_pytorch(args) | ||||
| @@ -17,10 +22,19 @@ class Loss(object): | |||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def get(self): | def get(self): | ||||
| """ | |||||
| :return self._loss: the loss function | |||||
| """ | |||||
| return self._loss | return self._loss | ||||
| @staticmethod | @staticmethod | ||||
| def _borrow_from_pytorch(loss_name): | def _borrow_from_pytorch(loss_name): | ||||
| """Given a name of a loss function, return it from PyTorch. | |||||
| :param loss_name: str, the name of a loss function | |||||
| :return loss: a PyTorch loss | |||||
| """ | |||||
| if loss_name == "cross_entropy": | if loss_name == "cross_entropy": | ||||
| return torch.nn.CrossEntropyLoss() | return torch.nn.CrossEntropyLoss() | ||||
| else: | else: | ||||
+ 26
- 31
fastNLP/core/metrics.py
View File
| @@ -1,11 +1,12 @@ | |||||
| import warnings | import warnings | ||||
| import numpy as np | import numpy as np | ||||
| import torch | import torch | ||||
| def _conver_numpy(x): | def _conver_numpy(x): | ||||
| """ | |||||
| convert input data to numpy array | |||||
| """convert input data to numpy array | |||||
| """ | """ | ||||
| if isinstance(x, np.ndarray): | if isinstance(x, np.ndarray): | ||||
| return x | return x | ||||
| @@ -17,21 +18,20 @@ def _conver_numpy(x): | |||||
| def _check_same_len(*arrays, axis=0): | def _check_same_len(*arrays, axis=0): | ||||
| """ | |||||
| check if input array list has same length for one dimension | |||||
| """check if input array list has same length for one dimension | |||||
| """ | """ | ||||
| lens = set([x.shape[axis] for x in arrays if x is not None]) | lens = set([x.shape[axis] for x in arrays if x is not None]) | ||||
| return len(lens) == 1 | return len(lens) == 1 | ||||
| def _label_types(y): | def _label_types(y): | ||||
| """ | |||||
| determine the type | |||||
| "binary" | |||||
| "multiclass" | |||||
| "multiclass-multioutput" | |||||
| "multilabel" | |||||
| "unknown" | |||||
| """Determine the type | |||||
| - "binary" | |||||
| - "multiclass" | |||||
| - "multiclass-multioutput" | |||||
| - "multilabel" | |||||
| - "unknown" | |||||
| """ | """ | ||||
| # never squeeze the first dimension | # never squeeze the first dimension | ||||
| y = y.squeeze() if y.shape[0] > 1 else y.resize(1, -1) | y = y.squeeze() if y.shape[0] > 1 else y.resize(1, -1) | ||||
| @@ -46,8 +46,8 @@ def _label_types(y): | |||||
| def _check_data(y_true, y_pred): | def _check_data(y_true, y_pred): | ||||
| """ | |||||
| check if y_true and y_pred is same type of data e.g both binary or multiclass | |||||
| """Check if y_true and y_pred is same type of data e.g both binary or multiclass | |||||
| """ | """ | ||||
| y_true, y_pred = _conver_numpy(y_true), _conver_numpy(y_pred) | y_true, y_pred = _conver_numpy(y_true), _conver_numpy(y_pred) | ||||
| if not _check_same_len(y_true, y_pred): | if not _check_same_len(y_true, y_pred): | ||||
| @@ -174,16 +174,13 @@ def classification_report(y_true, y_pred, labels=None, target_names=None, digits | |||||
| def accuracy_topk(y_true, y_prob, k=1): | def accuracy_topk(y_true, y_prob, k=1): | ||||
| """ | |||||
| Compute accuracy of y_true matching top-k probable | |||||
| """Compute accuracy of y_true matching top-k probable | |||||
| labels in y_prob. | labels in y_prob. | ||||
| Paras: | |||||
| y_ture - ndarray, true label, [n_samples] | |||||
| y_prob - ndarray, label probabilities, [n_samples, n_classes] | |||||
| k - int, k in top-k | |||||
| Returns: | |||||
| accuracy of top-k | |||||
| :param y_true: ndarray, true label, [n_samples] | |||||
| :param y_prob: ndarray, label probabilities, [n_samples, n_classes] | |||||
| :param k: int, k in top-k | |||||
| :return :accuracy of top-k | |||||
| """ | """ | ||||
| y_pred_topk = np.argsort(y_prob, axis=-1)[:, -1:-k - 1:-1] | y_pred_topk = np.argsort(y_prob, axis=-1)[:, -1:-k - 1:-1] | ||||
| @@ -195,16 +192,14 @@ def accuracy_topk(y_true, y_prob, k=1): | |||||
| def pred_topk(y_prob, k=1): | def pred_topk(y_prob, k=1): | ||||
| """ | |||||
| Return top-k predicted labels and corresponding probabilities. | |||||
| Args: | |||||
| y_prob - ndarray, size [n_samples, n_classes], probabilities on labels | |||||
| k - int, k of top-k | |||||
| Returns: | |||||
| y_pred_topk - ndarray, size [n_samples, k], predicted top-k labels | |||||
| y_prob_topk - ndarray, size [n_samples, k], probabilities for | |||||
| top-k labels | |||||
| """Return top-k predicted labels and corresponding probabilities. | |||||
| :param y_prob: ndarray, size [n_samples, n_classes], probabilities on labels | |||||
| :param k: int, k of top-k | |||||
| :returns | |||||
| y_pred_topk: ndarray, size [n_samples, k], predicted top-k labels | |||||
| y_prob_topk: ndarray, size [n_samples, k], probabilities for top-k labels | |||||
| """ | """ | ||||
| y_pred_topk = np.argsort(y_prob, axis=-1)[:, -1:-k - 1:-1] | y_pred_topk = np.argsort(y_prob, axis=-1)[:, -1:-k - 1:-1] | ||||
+ 10
- 2
fastNLP/core/optimizer.py
View File
| @@ -4,7 +4,6 @@ import torch | |||||
| class Optimizer(object): | class Optimizer(object): | ||||
| """Wrapper of optimizer from framework | """Wrapper of optimizer from framework | ||||
| names: arguments (type) | |||||
| 1. Adam: lr (float), weight_decay (float) | 1. Adam: lr (float), weight_decay (float) | ||||
| 2. AdaGrad | 2. AdaGrad | ||||
| 3. RMSProp | 3. RMSProp | ||||
| @@ -16,20 +15,29 @@ class Optimizer(object): | |||||
| """ | """ | ||||
| :param optimizer_name: str, the name of the optimizer | :param optimizer_name: str, the name of the optimizer | ||||
| :param kwargs: the arguments | :param kwargs: the arguments | ||||
| """ | """ | ||||
| self.optim_name = optimizer_name | self.optim_name = optimizer_name | ||||
| self.kwargs = kwargs | self.kwargs = kwargs | ||||
| @property | @property | ||||
| def name(self): | def name(self): | ||||
| """The name of the optimizer. | |||||
| :return: str | |||||
| """ | |||||
| return self.optim_name | return self.optim_name | ||||
| @property | @property | ||||
| def params(self): | def params(self): | ||||
| """The arguments used to create the optimizer. | |||||
| :return: dict of (str, *) | |||||
| """ | |||||
| return self.kwargs | return self.kwargs | ||||
| def construct_from_pytorch(self, model_params): | def construct_from_pytorch(self, model_params): | ||||
| """construct a optimizer from framework over given model parameters""" | |||||
| """Construct a optimizer from framework over given model parameters.""" | |||||
| if self.optim_name in ["SGD", "sgd"]: | if self.optim_name in ["SGD", "sgd"]: | ||||
| if "lr" in self.kwargs: | if "lr" in self.kwargs: | ||||
+ 1
- 1
fastNLP/core/predictor.py
View File
| @@ -70,7 +70,7 @@ class Predictor(object): | |||||
| def predict(self, network, data): | def predict(self, network, data): | ||||
| """Perform inference using the trained model. | """Perform inference using the trained model. | ||||
| :param network: a PyTorch model | |||||
| :param network: a PyTorch model (cpu) | |||||
| :param data: list of list of strings | :param data: list of list of strings | ||||
| :return: list of list of strings, [num_examples, tag_seq_length] | :return: list of list of strings, [num_examples, tag_seq_length] | ||||
| """ | """ | ||||
+ 55
- 11
fastNLP/core/preprocess.py
View File
| @@ -17,12 +17,24 @@ DEFAULT_WORD_TO_INDEX = {DEFAULT_PADDING_LABEL: 0, DEFAULT_UNKNOWN_LABEL: 1, | |||||
| # the first vocab in dict with the index = 5 | # the first vocab in dict with the index = 5 | ||||
| def save_pickle(obj, pickle_path, file_name): | def save_pickle(obj, pickle_path, file_name): | ||||
| """Save an object into a pickle file. | |||||
| :param obj: an object | |||||
| :param pickle_path: str, the directory where the pickle file is to be saved | |||||
| :param file_name: str, the name of the pickle file. In general, it should be ended by "pkl". | |||||
| """ | |||||
| with open(os.path.join(pickle_path, file_name), "wb") as f: | with open(os.path.join(pickle_path, file_name), "wb") as f: | ||||
| _pickle.dump(obj, f) | _pickle.dump(obj, f) | ||||
| print("{} saved in {}".format(file_name, pickle_path)) | print("{} saved in {}".format(file_name, pickle_path)) | ||||
| def load_pickle(pickle_path, file_name): | def load_pickle(pickle_path, file_name): | ||||
| """Load an object from a given pickle file. | |||||
| :param pickle_path: str, the directory where the pickle file is. | |||||
| :param file_name: str, the name of the pickle file. | |||||
| :return obj: an object stored in the pickle | |||||
| """ | |||||
| with open(os.path.join(pickle_path, file_name), "rb") as f: | with open(os.path.join(pickle_path, file_name), "rb") as f: | ||||
| obj = _pickle.load(f) | obj = _pickle.load(f) | ||||
| print("{} loaded from {}".format(file_name, pickle_path)) | print("{} loaded from {}".format(file_name, pickle_path)) | ||||
| @@ -30,7 +42,8 @@ def load_pickle(pickle_path, file_name): | |||||
| def pickle_exist(pickle_path, pickle_name): | def pickle_exist(pickle_path, pickle_name): | ||||
| """ | |||||
| """Check if a given pickle file exists in the directory. | |||||
| :param pickle_path: the directory of target pickle file | :param pickle_path: the directory of target pickle file | ||||
| :param pickle_name: the filename of target pickle file | :param pickle_name: the filename of target pickle file | ||||
| :return: True if file exists else False | :return: True if file exists else False | ||||
| @@ -45,6 +58,19 @@ def pickle_exist(pickle_path, pickle_name): | |||||
| class BasePreprocess(object): | class BasePreprocess(object): | ||||
| """Base class of all preprocessors. | |||||
| Preprocessors are responsible for converting data of strings into data of indices. | |||||
| During the pre-processing, the following pickle files will be built: | |||||
| - "word2id.pkl", a mapping from words(tokens) to indices | |||||
| - "id2word.pkl", a reversed dictionary | |||||
| - "label2id.pkl", a dictionary on labels | |||||
| - "id2label.pkl", a reversed dictionary on labels | |||||
| These four pickle files are expected to be saved in the given pickle directory once they are constructed. | |||||
| Preprocessors will check if those files are already in the directory and will reuse them in future calls. | |||||
| """ | |||||
| def __init__(self): | def __init__(self): | ||||
| self.word2index = None | self.word2index = None | ||||
| self.label2index = None | self.label2index = None | ||||
| @@ -59,6 +85,7 @@ class BasePreprocess(object): | |||||
| def run(self, train_dev_data, test_data=None, pickle_path="./", train_dev_split=0, cross_val=False, n_fold=10): | def run(self, train_dev_data, test_data=None, pickle_path="./", train_dev_split=0, cross_val=False, n_fold=10): | ||||
| """Main preprocessing pipeline. | """Main preprocessing pipeline. | ||||
| :param train_dev_data: three-level list, with either single label or multiple labels in a sample. | :param train_dev_data: three-level list, with either single label or multiple labels in a sample. | ||||
| :param test_data: three-level list, with either single label or multiple labels in a sample. (optional) | :param test_data: three-level list, with either single label or multiple labels in a sample. (optional) | ||||
| :param pickle_path: str, the path to save the pickle files. | :param pickle_path: str, the path to save the pickle files. | ||||
| @@ -67,6 +94,7 @@ class BasePreprocess(object): | |||||
| :param n_fold: int, the number of folds of cross validation. Only useful when cross_val is True. | :param n_fold: int, the number of folds of cross validation. Only useful when cross_val is True. | ||||
| :return results: a tuple of datasets after preprocessing. | :return results: a tuple of datasets after preprocessing. | ||||
| """ | """ | ||||
| if pickle_exist(pickle_path, "word2id.pkl") and pickle_exist(pickle_path, "class2id.pkl"): | if pickle_exist(pickle_path, "word2id.pkl") and pickle_exist(pickle_path, "class2id.pkl"): | ||||
| self.word2index = load_pickle(pickle_path, "word2id.pkl") | self.word2index = load_pickle(pickle_path, "word2id.pkl") | ||||
| self.label2index = load_pickle(pickle_path, "class2id.pkl") | self.label2index = load_pickle(pickle_path, "class2id.pkl") | ||||
| @@ -182,25 +210,31 @@ class SeqLabelPreprocess(BasePreprocess): | |||||
| """Preprocess pipeline, including building mapping from words to index, from index to words, | """Preprocess pipeline, including building mapping from words to index, from index to words, | ||||
| from labels/classes to index, from index to labels/classes. | from labels/classes to index, from index to labels/classes. | ||||
| data of three-level list which have multiple labels in each sample. | data of three-level list which have multiple labels in each sample. | ||||
| :: | |||||
| [ | [ | ||||
| [ [word_11, word_12, ...], [label_1, label_1, ...] ], | [ [word_11, word_12, ...], [label_1, label_1, ...] ], | ||||
| [ [word_21, word_22, ...], [label_2, label_1, ...] ], | [ [word_21, word_22, ...], [label_2, label_1, ...] ], | ||||
| ... | ... | ||||
| ] | ] | ||||
| """ | """ | ||||
| def __init__(self): | def __init__(self): | ||||
| super(SeqLabelPreprocess, self).__init__() | super(SeqLabelPreprocess, self).__init__() | ||||
| def build_dict(self, data): | def build_dict(self, data): | ||||
| """ | |||||
| Add new words with indices into self.word_dict, new labels with indices into self.label_dict. | |||||
| """Add new words with indices into self.word_dict, new labels with indices into self.label_dict. | |||||
| :param data: three-level list | :param data: three-level list | ||||
| :: | |||||
| [ | [ | ||||
| [ [word_11, word_12, ...], [label_1, label_1, ...] ], | [ [word_11, word_12, ...], [label_1, label_1, ...] ], | ||||
| [ [word_21, word_22, ...], [label_2, label_1, ...] ], | [ [word_21, word_22, ...], [label_2, label_1, ...] ], | ||||
| ... | ... | ||||
| ] | ] | ||||
| :return word2index: dict of {str, int} | :return word2index: dict of {str, int} | ||||
| label2index: dict of {str, int} | label2index: dict of {str, int} | ||||
| """ | """ | ||||
| @@ -216,14 +250,17 @@ class SeqLabelPreprocess(BasePreprocess): | |||||
| return word2index, label2index | return word2index, label2index | ||||
| def to_index(self, data): | def to_index(self, data): | ||||
| """ | |||||
| Convert word strings and label strings into indices. | |||||
| """Convert word strings and label strings into indices. | |||||
| :param data: three-level list | :param data: three-level list | ||||
| :: | |||||
| [ | [ | ||||
| [ [word_11, word_12, ...], [label_1, label_1, ...] ], | [ [word_11, word_12, ...], [label_1, label_1, ...] ], | ||||
| [ [word_21, word_22, ...], [label_2, label_1, ...] ], | [ [word_21, word_22, ...], [label_2, label_1, ...] ], | ||||
| ... | ... | ||||
| ] | ] | ||||
| :return data_index: the same shape as data, but each string is replaced by its corresponding index | :return data_index: the same shape as data, but each string is replaced by its corresponding index | ||||
| """ | """ | ||||
| data_index = [] | data_index = [] | ||||
| @@ -242,11 +279,14 @@ class ClassPreprocess(BasePreprocess): | |||||
| Preprocess pipeline, including building mapping from words to index, from index to words, | Preprocess pipeline, including building mapping from words to index, from index to words, | ||||
| from labels/classes to index, from index to labels/classes. | from labels/classes to index, from index to labels/classes. | ||||
| design for data of three-level list which has a single label in each sample. | design for data of three-level list which has a single label in each sample. | ||||
| :: | |||||
| [ | [ | ||||
| [ [word_11, word_12, ...], label_1 ], | [ [word_11, word_12, ...], label_1 ], | ||||
| [ [word_21, word_22, ...], label_2 ], | [ [word_21, word_22, ...], label_2 ], | ||||
| ... | ... | ||||
| ] | ] | ||||
| """ | """ | ||||
| def __init__(self): | def __init__(self): | ||||
| @@ -269,18 +309,21 @@ class ClassPreprocess(BasePreprocess): | |||||
| for word in sent: | for word in sent: | ||||
| if word not in word2index: | if word not in word2index: | ||||
| word2index[word[0]] = len(word2index) | |||||
| word2index[word] = len(word2index) | |||||
| return word2index, label2index | return word2index, label2index | ||||
| def to_index(self, data): | def to_index(self, data): | ||||
| """ | |||||
| Convert word strings and label strings into indices. | |||||
| """Convert word strings and label strings into indices. | |||||
| :param data: three-level list | :param data: three-level list | ||||
| :: | |||||
| [ | [ | ||||
| [ [word_11, word_12, ...], label_1 ], | [ [word_11, word_12, ...], label_1 ], | ||||
| [ [word_21, word_22, ...], label_2 ], | [ [word_21, word_22, ...], label_2 ], | ||||
| ... | ... | ||||
| ] | ] | ||||
| :return data_index: the same shape as data, but each string is replaced by its corresponding index | :return data_index: the same shape as data, but each string is replaced by its corresponding index | ||||
| """ | """ | ||||
| data_index = [] | data_index = [] | ||||
| @@ -295,14 +338,15 @@ class ClassPreprocess(BasePreprocess): | |||||
| def infer_preprocess(pickle_path, data): | def infer_preprocess(pickle_path, data): | ||||
| """ | |||||
| Preprocess over inference data. | |||||
| Transform three-level list of strings into that of index. | |||||
| """Preprocess over inference data. Transform three-level list of strings into that of index. | |||||
| :: | |||||
| [ | [ | ||||
| [word_11, word_12, ...], | [word_11, word_12, ...], | ||||
| [word_21, word_22, ...], | [word_21, word_22, ...], | ||||
| ... | ... | ||||
| ] | ] | ||||
| """ | """ | ||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | word2index = load_pickle(pickle_path, "word2id.pkl") | ||||
| data_index = [] | data_index = [] | ||||
+ 40
- 20
fastNLP/core/tester.py
View File
| @@ -38,7 +38,7 @@ class BaseTester(object): | |||||
| Obviously, "required_args" is the subset of "default_args". | Obviously, "required_args" is the subset of "default_args". | ||||
| The value in "default_args" to the keys in "required_args" is simply for type check. | The value in "default_args" to the keys in "required_args" is simply for type check. | ||||
| """ | """ | ||||
| # TODO: required arguments | |||||
| # add required arguments here | |||||
| required_args = {} | required_args = {} | ||||
| for req_key in required_args: | for req_key in required_args: | ||||
| @@ -56,7 +56,7 @@ class BaseTester(object): | |||||
| logger.error(msg) | logger.error(msg) | ||||
| raise ValueError(msg) | raise ValueError(msg) | ||||
| else: | else: | ||||
| # BeseTester doesn't care about extra arguments | |||||
| # BaseTester doesn't care about extra arguments | |||||
| pass | pass | ||||
| print(default_args) | print(default_args) | ||||
| @@ -69,8 +69,8 @@ class BaseTester(object): | |||||
| self.print_every_step = default_args["print_every_step"] | self.print_every_step = default_args["print_every_step"] | ||||
| self._model = None | self._model = None | ||||
| self.eval_history = [] | |||||
| self.batch_output = [] | |||||
| self.eval_history = [] # evaluation results of all batches | |||||
| self.batch_output = [] # outputs of all batches | |||||
| def test(self, network, dev_data): | def test(self, network, dev_data): | ||||
| if torch.cuda.is_available() and self.use_cuda: | if torch.cuda.is_available() and self.use_cuda: | ||||
| @@ -83,10 +83,10 @@ class BaseTester(object): | |||||
| self.eval_history.clear() | self.eval_history.clear() | ||||
| self.batch_output.clear() | self.batch_output.clear() | ||||
| iterator = iter(Batchifier(RandomSampler(dev_data), self.batch_size, drop_last=True)) | |||||
| iterator = iter(Batchifier(RandomSampler(dev_data), self.batch_size, drop_last=False)) | |||||
| step = 0 | step = 0 | ||||
| for batch_x, batch_y in self.make_batch(iterator, dev_data): | |||||
| for batch_x, batch_y in self.make_batch(iterator): | |||||
| with torch.no_grad(): | with torch.no_grad(): | ||||
| prediction = self.data_forward(network, batch_x) | prediction = self.data_forward(network, batch_x) | ||||
| eval_results = self.evaluate(prediction, batch_y) | eval_results = self.evaluate(prediction, batch_y) | ||||
| @@ -99,7 +99,7 @@ class BaseTester(object): | |||||
| print_output = "[test step {}] {}".format(step, eval_results) | print_output = "[test step {}] {}".format(step, eval_results) | ||||
| logger.info(print_output) | logger.info(print_output) | ||||
| if self.print_every_step > 0 and step % self.print_every_step == 0: | if self.print_every_step > 0 and step % self.print_every_step == 0: | ||||
| print(print_output) | |||||
| print(self.make_eval_output(prediction, eval_results)) | |||||
| step += 1 | step += 1 | ||||
| def mode(self, model, test): | def mode(self, model, test): | ||||
| @@ -115,28 +115,48 @@ class BaseTester(object): | |||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def evaluate(self, predict, truth): | def evaluate(self, predict, truth): | ||||
| """Compute evaluation metrics for the model. """ | |||||
| """Compute evaluation metrics. | |||||
| :param predict: Tensor | |||||
| :param truth: Tensor | |||||
| :return eval_results: can be anything. It will be stored in self.eval_history | |||||
| """ | |||||
| raise NotImplementedError | raise NotImplementedError | ||||
| @property | @property | ||||
| def metrics(self): | def metrics(self): | ||||
| """Return a list of metrics. """ | |||||
| """Compute and return metrics. | |||||
| Use self.eval_history to compute metrics over the whole dev set. | |||||
| Please refer to metrics.py for common metric functions. | |||||
| :return : variable number of outputs | |||||
| """ | |||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def show_matrices(self): | |||||
| """This is called by Trainer to print evaluation results on dev set during training. | |||||
| def show_metrics(self): | |||||
| """Customize evaluation outputs in Trainer. | |||||
| Called by Trainer to print evaluation results on dev set during training. | |||||
| Use self.metrics to fetch available metrics. | |||||
| :return print_str: str | :return print_str: str | ||||
| """ | """ | ||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def make_batch(self, iterator, data): | |||||
| def make_batch(self, iterator): | |||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def make_eval_output(self, predictions, eval_results): | |||||
| """Customize Tester outputs. | |||||
| :param predictions: Tensor | |||||
| :param eval_results: Tensor | |||||
| :return: str, to be printed. | |||||
| """ | |||||
| raise NotImplementedError | |||||
| class SeqLabelTester(BaseTester): | class SeqLabelTester(BaseTester): | ||||
| """ | |||||
| Tester for sequence labeling. | |||||
| """Tester for sequence labeling. | |||||
| """ | """ | ||||
| def __init__(self, **test_args): | def __init__(self, **test_args): | ||||
| @@ -194,15 +214,15 @@ class SeqLabelTester(BaseTester): | |||||
| batch_accuracy = np.mean([x[1] for x in self.eval_history]) | batch_accuracy = np.mean([x[1] for x in self.eval_history]) | ||||
| return batch_loss, batch_accuracy | return batch_loss, batch_accuracy | ||||
| def show_matrices(self): | |||||
| """ | |||||
| This is called by Trainer to print evaluation on dev set. | |||||
| def show_metrics(self): | |||||
| """This is called by Trainer to print evaluation on dev set. | |||||
| :return print_str: str | :return print_str: str | ||||
| """ | """ | ||||
| loss, accuracy = self.metrics() | loss, accuracy = self.metrics() | ||||
| return "dev loss={:.2f}, accuracy={:.2f}".format(loss, accuracy) | return "dev loss={:.2f}, accuracy={:.2f}".format(loss, accuracy) | ||||
| def make_batch(self, iterator, data): | |||||
| def make_batch(self, iterator): | |||||
| return Action.make_batch(iterator, use_cuda=self.use_cuda, output_length=True) | return Action.make_batch(iterator, use_cuda=self.use_cuda, output_length=True) | ||||
| @@ -211,12 +231,12 @@ class ClassificationTester(BaseTester): | |||||
| def __init__(self, **test_args): | def __init__(self, **test_args): | ||||
| """ | """ | ||||
| :param test_args: a dict-like object that has __getitem__ method, \ | |||||
| :param test_args: a dict-like object that has __getitem__ method. | |||||
| can be accessed by "test_args["key_str"]" | can be accessed by "test_args["key_str"]" | ||||
| """ | """ | ||||
| super(ClassificationTester, self).__init__(**test_args) | super(ClassificationTester, self).__init__(**test_args) | ||||
| def make_batch(self, iterator, data, max_len=None): | |||||
| def make_batch(self, iterator, max_len=None): | |||||
| return Action.make_batch(iterator, use_cuda=self.use_cuda, max_len=max_len) | return Action.make_batch(iterator, use_cuda=self.use_cuda, max_len=max_len) | ||||
| def data_forward(self, network, x): | def data_forward(self, network, x): | ||||
+ 36
- 45
fastNLP/core/trainer.py
View File
| @@ -1,10 +1,11 @@ | |||||
| import _pickle | |||||
| import copy | import copy | ||||
| import os | import os | ||||
| import time | import time | ||||
| from datetime import timedelta | from datetime import timedelta | ||||
| import torch | import torch | ||||
| import tensorboardX | |||||
| from tensorboardX import SummaryWriter | |||||
| from fastNLP.core.action import Action | from fastNLP.core.action import Action | ||||
| from fastNLP.core.action import RandomSampler, Batchifier | from fastNLP.core.action import RandomSampler, Batchifier | ||||
| @@ -15,16 +16,12 @@ from fastNLP.modules import utils | |||||
| from fastNLP.saver.logger import create_logger | from fastNLP.saver.logger import create_logger | ||||
| from fastNLP.saver.model_saver import ModelSaver | from fastNLP.saver.model_saver import ModelSaver | ||||
| DEFAULT_QUEUE_SIZE = 300 | |||||
| logger = create_logger(__name__, "./train_test.log") | logger = create_logger(__name__, "./train_test.log") | ||||
| class BaseTrainer(object): | class BaseTrainer(object): | ||||
| """Operations to train a model, including data loading, SGD, and validation. | |||||
| """Operations of training a model, including data loading, gradient descent, and validation. | |||||
| Subclasses must implement the following abstract methods: | |||||
| - grad_backward | |||||
| - get_loss | |||||
| """ | """ | ||||
| def __init__(self, **kwargs): | def __init__(self, **kwargs): | ||||
| @@ -32,10 +29,10 @@ class BaseTrainer(object): | |||||
| :param kwargs: dict of (key, value), or dict-like object. key is str. | :param kwargs: dict of (key, value), or dict-like object. key is str. | ||||
| The base trainer requires the following keys: | The base trainer requires the following keys: | ||||
| - epochs: int, the number of epochs in training | |||||
| - validate: bool, whether or not to validate on dev set | |||||
| - batch_size: int | |||||
| - pickle_path: str, the path to pickle files for pre-processing | |||||
| - epochs: int, the number of epochs in training | |||||
| - validate: bool, whether or not to validate on dev set | |||||
| - batch_size: int | |||||
| - pickle_path: str, the path to pickle files for pre-processing | |||||
| """ | """ | ||||
| super(BaseTrainer, self).__init__() | super(BaseTrainer, self).__init__() | ||||
| @@ -47,7 +44,7 @@ class BaseTrainer(object): | |||||
| """ | """ | ||||
| default_args = {"epochs": 3, "batch_size": 8, "validate": True, "use_cuda": True, "pickle_path": "./save/", | default_args = {"epochs": 3, "batch_size": 8, "validate": True, "use_cuda": True, "pickle_path": "./save/", | ||||
| "save_best_dev": True, "model_name": "default_model_name.pkl", "print_every_step": 1, | "save_best_dev": True, "model_name": "default_model_name.pkl", "print_every_step": 1, | ||||
| "loss": Loss(None), | |||||
| "loss": Loss(None), # used to pass type check | |||||
| "optimizer": Optimizer("Adam", lr=0.001, weight_decay=0) | "optimizer": Optimizer("Adam", lr=0.001, weight_decay=0) | ||||
| } | } | ||||
| """ | """ | ||||
| @@ -56,7 +53,7 @@ class BaseTrainer(object): | |||||
| Obviously, "required_args" is the subset of "default_args". | Obviously, "required_args" is the subset of "default_args". | ||||
| The value in "default_args" to the keys in "required_args" is simply for type check. | The value in "default_args" to the keys in "required_args" is simply for type check. | ||||
| """ | """ | ||||
| # TODO: required arguments | |||||
| # add required arguments here | |||||
| required_args = {} | required_args = {} | ||||
| for req_key in required_args: | for req_key in required_args: | ||||
| @@ -91,9 +88,12 @@ class BaseTrainer(object): | |||||
| self._loss_func = default_args["loss"].get() # return a pytorch loss function or None | self._loss_func = default_args["loss"].get() # return a pytorch loss function or None | ||||
| self._optimizer = None | self._optimizer = None | ||||
| self._optimizer_proto = default_args["optimizer"] | self._optimizer_proto = default_args["optimizer"] | ||||
| self._summary_writer = SummaryWriter(self.pickle_path + 'tensorboard_logs') | |||||
| self._graph_summaried = False | |||||
| def train(self, network, train_data, dev_data=None): | def train(self, network, train_data, dev_data=None): | ||||
| """General Training Procedure | """General Training Procedure | ||||
| :param network: a model | :param network: a model | ||||
| :param train_data: three-level list, the training set. | :param train_data: three-level list, the training set. | ||||
| :param dev_data: three-level list, the validation data (optional) | :param dev_data: three-level list, the validation data (optional) | ||||
| @@ -144,12 +144,13 @@ class BaseTrainer(object): | |||||
| print("Saved better model selected by validation.") | print("Saved better model selected by validation.") | ||||
| logger.info("Saved better model selected by validation.") | logger.info("Saved better model selected by validation.") | ||||
| valid_results = validator.show_matrices() | |||||
| valid_results = validator.show_metrics() | |||||
| print("[epoch {}] {}".format(epoch, valid_results)) | print("[epoch {}] {}".format(epoch, valid_results)) | ||||
| logger.info("[epoch {}] {}".format(epoch, valid_results)) | logger.info("[epoch {}] {}".format(epoch, valid_results)) | ||||
| def _train_step(self, data_iterator, network, **kwargs): | def _train_step(self, data_iterator, network, **kwargs): | ||||
| """Training process in one epoch. | """Training process in one epoch. | ||||
| kwargs should contain: | kwargs should contain: | ||||
| - n_print: int, print training information every n steps. | - n_print: int, print training information every n steps. | ||||
| - start: time.time(), the starting time of this step. | - start: time.time(), the starting time of this step. | ||||
| @@ -163,6 +164,11 @@ class BaseTrainer(object): | |||||
| loss = self.get_loss(prediction, batch_y) | loss = self.get_loss(prediction, batch_y) | ||||
| self.grad_backward(loss) | self.grad_backward(loss) | ||||
| self.update() | self.update() | ||||
| self._summary_writer.add_scalar("loss", loss.item(), global_step=step) | |||||
| if not self._graph_summaried: | |||||
| self._summary_writer.add_graph(network, batch_x) | |||||
| self._graph_summaried = True | |||||
| if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0: | if kwargs["n_print"] > 0 and step % kwargs["n_print"] == 0: | ||||
| end = time.time() | end = time.time() | ||||
| @@ -198,21 +204,6 @@ class BaseTrainer(object): | |||||
| network_copy = copy.deepcopy(network) | network_copy = copy.deepcopy(network) | ||||
| self.train(network_copy, train_data_cv[i], dev_data_cv[i]) | self.train(network_copy, train_data_cv[i], dev_data_cv[i]) | ||||
| def load_train_data(self, pickle_path): | |||||
| """ | |||||
| For task-specific processing. | |||||
| :param pickle_path: | |||||
| :return data_train | |||||
| """ | |||||
| file_path = os.path.join(pickle_path, "data_train.pkl") | |||||
| if os.path.exists(file_path): | |||||
| with open(file_path, 'rb') as f: | |||||
| data = _pickle.load(f) | |||||
| else: | |||||
| logger.error("cannot find training data {}. invalid input path for training data.".format(file_path)) | |||||
| raise RuntimeError("cannot find training data {}".format(file_path)) | |||||
| return data | |||||
| def make_batch(self, iterator): | def make_batch(self, iterator): | ||||
| raise NotImplementedError | raise NotImplementedError | ||||
| @@ -220,14 +211,13 @@ class BaseTrainer(object): | |||||
| Action.mode(network, test) | Action.mode(network, test) | ||||
| def define_optimizer(self): | def define_optimizer(self): | ||||
| """ | |||||
| Define framework-specific optimizer specified by the models. | |||||
| """Define framework-specific optimizer specified by the models. | |||||
| """ | """ | ||||
| self._optimizer = self._optimizer_proto.construct_from_pytorch(self._model.parameters()) | self._optimizer = self._optimizer_proto.construct_from_pytorch(self._model.parameters()) | ||||
| def update(self): | def update(self): | ||||
| """ | |||||
| Perform weight update on a model. | |||||
| """Perform weight update on a model. | |||||
| For PyTorch, just call optimizer to update. | For PyTorch, just call optimizer to update. | ||||
| """ | """ | ||||
| @@ -237,8 +227,8 @@ class BaseTrainer(object): | |||||
| raise NotImplementedError | raise NotImplementedError | ||||
| def grad_backward(self, loss): | def grad_backward(self, loss): | ||||
| """ | |||||
| Compute gradient with link rules. | |||||
| """Compute gradient with link rules. | |||||
| :param loss: a scalar where back-prop starts | :param loss: a scalar where back-prop starts | ||||
| For PyTorch, just do "loss.backward()" | For PyTorch, just do "loss.backward()" | ||||
| @@ -247,8 +237,8 @@ class BaseTrainer(object): | |||||
| loss.backward() | loss.backward() | ||||
| def get_loss(self, predict, truth): | def get_loss(self, predict, truth): | ||||
| """ | |||||
| Compute loss given prediction and ground truth. | |||||
| """Compute loss given prediction and ground truth. | |||||
| :param predict: prediction label vector | :param predict: prediction label vector | ||||
| :param truth: ground truth label vector | :param truth: ground truth label vector | ||||
| :return: a scalar | :return: a scalar | ||||
| @@ -256,8 +246,9 @@ class BaseTrainer(object): | |||||
| return self._loss_func(predict, truth) | return self._loss_func(predict, truth) | ||||
| def define_loss(self): | def define_loss(self): | ||||
| """ | |||||
| if the model defines a loss, use model's loss. | |||||
| """Define a loss for the trainer. | |||||
| If the model defines a loss, use model's loss. | |||||
| Otherwise, Trainer must has a loss argument, use it as loss. | Otherwise, Trainer must has a loss argument, use it as loss. | ||||
| These two losses cannot be defined at the same time. | These two losses cannot be defined at the same time. | ||||
| Trainer does not handle loss definition or choose default losses. | Trainer does not handle loss definition or choose default losses. | ||||
| @@ -274,7 +265,8 @@ class BaseTrainer(object): | |||||
| logger.info("The model didn't define loss, use Trainer's loss.") | logger.info("The model didn't define loss, use Trainer's loss.") | ||||
| def best_eval_result(self, validator): | def best_eval_result(self, validator): | ||||
| """ | |||||
| """Check if the current epoch yields better validation results. | |||||
| :param validator: a Tester instance | :param validator: a Tester instance | ||||
| :return: bool, True means current results on dev set is the best. | :return: bool, True means current results on dev set is the best. | ||||
| """ | """ | ||||
| @@ -289,15 +281,14 @@ class BaseTrainer(object): | |||||
| """ | """ | ||||
| if model_name[-4:] != ".pkl": | if model_name[-4:] != ".pkl": | ||||
| model_name += ".pkl" | model_name += ".pkl" | ||||
| ModelSaver(self.pickle_path + model_name).save_pytorch(network) | |||||
| ModelSaver(os.path.join(self.pickle_path, model_name)).save_pytorch(network) | |||||
| def _create_validator(self, valid_args): | def _create_validator(self, valid_args): | ||||
| raise NotImplementedError | raise NotImplementedError | ||||
| class SeqLabelTrainer(BaseTrainer): | class SeqLabelTrainer(BaseTrainer): | ||||
| """ | |||||
| Trainer for Sequence Labeling | |||||
| """Trainer for Sequence Labeling | |||||
| """ | """ | ||||
| @@ -327,11 +318,11 @@ class SeqLabelTrainer(BaseTrainer): | |||||
| return y | return y | ||||
| def get_loss(self, predict, truth): | def get_loss(self, predict, truth): | ||||
| """ | |||||
| Compute loss given prediction and ground truth. | |||||
| """Compute loss given prediction and ground truth. | |||||
| :param predict: prediction label vector, [batch_size, max_len, tag_size] | :param predict: prediction label vector, [batch_size, max_len, tag_size] | ||||
| :param truth: ground truth label vector, [batch_size, max_len] | :param truth: ground truth label vector, [batch_size, max_len] | ||||
| :return: a scalar | |||||
| :return loss: a scalar | |||||
| """ | """ | ||||
| batch_size, max_len = predict.size(0), predict.size(1) | batch_size, max_len = predict.size(0), predict.size(1) | ||||
| assert truth.shape == (batch_size, max_len) | assert truth.shape == (batch_size, max_len) | ||||
+ 55
- 2
fastNLP/fastnlp.py
View File
| @@ -1,3 +1,5 @@ | |||||
| import os | |||||
| from fastNLP.core.predictor import SeqLabelInfer, ClassificationInfer | from fastNLP.core.predictor import SeqLabelInfer, ClassificationInfer | ||||
| from fastNLP.core.preprocess import load_pickle | from fastNLP.core.preprocess import load_pickle | ||||
| from fastNLP.loader.config_loader import ConfigLoader, ConfigSection | from fastNLP.loader.config_loader import ConfigLoader, ConfigSection | ||||
| @@ -31,6 +33,22 @@ FastNLP_MODEL_COLLECTION = { | |||||
| "type": "seq_label", | "type": "seq_label", | ||||
| "config_file_name": "config", | "config_file_name": "config", | ||||
| "config_section_name": "text_class_model" | "config_section_name": "text_class_model" | ||||
| }, | |||||
| "pos_tag_model": { | |||||
| "url": "", | |||||
| "class": "sequence_modeling.AdvSeqLabel", | |||||
| "pickle": "pos_tag_model_v_0.pkl", | |||||
| "type": "seq_label", | |||||
| "config_file_name": "pos_tag.config", | |||||
| "config_section_name": "pos_tag_model" | |||||
| }, | |||||
| "text_classify_model": { | |||||
| "url": "", | |||||
| "class": "cnn_text_classification.CNNText", | |||||
| "pickle": "text_class_model_v0.pkl", | |||||
| "type": "text_class", | |||||
| "config_file_name": "text_classify.cfg", | |||||
| "config_section_name": "model" | |||||
| } | } | ||||
| } | } | ||||
| @@ -77,7 +95,7 @@ class FastNLP(object): | |||||
| print("Restore model class {}".format(str(model_class))) | print("Restore model class {}".format(str(model_class))) | ||||
| model_args = ConfigSection() | model_args = ConfigSection() | ||||
| ConfigLoader.load_config(self.model_dir + config_file, {section_name: model_args}) | |||||
| ConfigLoader.load_config(os.path.join(self.model_dir, config_file), {section_name: model_args}) | |||||
| print("Restore model hyper-parameters {}".format(str(model_args.data))) | print("Restore model hyper-parameters {}".format(str(model_args.data))) | ||||
| # fetch dictionary size and number of labels from pickle files | # fetch dictionary size and number of labels from pickle files | ||||
| @@ -91,7 +109,7 @@ class FastNLP(object): | |||||
| print("Model constructed.") | print("Model constructed.") | ||||
| # To do: framework independent | # To do: framework independent | ||||
| ModelLoader.load_pytorch(model, self.model_dir + FastNLP_MODEL_COLLECTION[model_name]["pickle"]) | |||||
| ModelLoader.load_pytorch(model, os.path.join(self.model_dir, FastNLP_MODEL_COLLECTION[model_name]["pickle"])) | |||||
| print("Model weights loaded.") | print("Model weights loaded.") | ||||
| self.model = model | self.model = model | ||||
| @@ -259,3 +277,38 @@ def interpret_word_seg_results(char_seq, label_seq): | |||||
| else: | else: | ||||
| raise ValueError("invalid label {}".format(label[0])) | raise ValueError("invalid label {}".format(label[0])) | ||||
| return words | return words | ||||
| def interpret_cws_pos_results(char_seq, label_seq): | |||||
| """Transform model output into user-friendly contents. | |||||
| :param char_seq: list of string | |||||
| :param label_seq: list of string, the same length as char_seq. | |||||
| :return outputs: list of tuple (words, pos_tag): | |||||
| """ | |||||
| def pos_tag_check(seq): | |||||
| """check whether all entries are the same """ | |||||
| return len(set(seq)) <= 1 | |||||
| word = [] | |||||
| word_pos = [] | |||||
| outputs = [] | |||||
| for char, label in zip(char_seq, label_seq): | |||||
| tmp = label.split("-") | |||||
| cws_label, pos_tag = tmp[0], tmp[1] | |||||
| if cws_label == "B" or cws_label == "M": | |||||
| word.append(char) | |||||
| word_pos.append(pos_tag) | |||||
| elif cws_label == "E": | |||||
| word.append(char) | |||||
| word_pos.append(pos_tag) | |||||
| if not pos_tag_check(word_pos): | |||||
| raise RuntimeError("character-wise pos tags inconsistent. ") | |||||
| outputs.append(("".join(word), word_pos[0])) | |||||
| word.clear() | |||||
| word_pos.clear() | |||||
| elif cws_label == "S": | |||||
| outputs.append((char, pos_tag)) | |||||
| return outputs | |||||
+ 3
- 4
fastNLP/loader/base_loader.py
View File
| @@ -1,9 +1,8 @@ | |||||
| class BaseLoader(object): | class BaseLoader(object): | ||||
| """docstring for BaseLoader""" | """docstring for BaseLoader""" | ||||
| def __init__(self, data_name, data_path): | |||||
| def __init__(self, data_path): | |||||
| super(BaseLoader, self).__init__() | super(BaseLoader, self).__init__() | ||||
| self.data_name = data_name | |||||
| self.data_path = data_path | self.data_path = data_path | ||||
| def load(self): | def load(self): | ||||
| @@ -25,8 +24,8 @@ class ToyLoader0(BaseLoader): | |||||
| For charLM | For charLM | ||||
| """ | """ | ||||
| def __init__(self, name, path): | |||||
| super(ToyLoader0, self).__init__(name, path) | |||||
| def __init__(self, data_path): | |||||
| super(ToyLoader0, self).__init__(data_path) | |||||
| def load(self): | def load(self): | ||||
| with open(self.data_path, 'r') as f: | with open(self.data_path, 'r') as f: | ||||
+ 2
- 2
fastNLP/loader/config_loader.py
View File
| @@ -9,7 +9,7 @@ class ConfigLoader(BaseLoader): | |||||
| """loader for configuration files""" | """loader for configuration files""" | ||||
| def __int__(self, data_name, data_path): | def __int__(self, data_name, data_path): | ||||
| super(ConfigLoader, self).__init__(data_name, data_path) | |||||
| super(ConfigLoader, self).__init__(data_path) | |||||
| self.config = self.parse(super(ConfigLoader, self).load()) | self.config = self.parse(super(ConfigLoader, self).load()) | ||||
| @staticmethod | @staticmethod | ||||
| @@ -100,7 +100,7 @@ class ConfigSection(object): | |||||
| if __name__ == "__main__": | if __name__ == "__main__": | ||||
| config = ConfigLoader('configLoader', 'there is no data') | |||||
| config = ConfigLoader('there is no data') | |||||
| section = {'General': ConfigSection(), 'My': ConfigSection(), 'A': ConfigSection()} | section = {'General': ConfigSection(), 'My': ConfigSection(), 'A': ConfigSection()} | ||||
| """ | """ | ||||
+ 62
- 18
fastNLP/loader/dataset_loader.py
View File
| @@ -6,8 +6,8 @@ from fastNLP.loader.base_loader import BaseLoader | |||||
| class DatasetLoader(BaseLoader): | class DatasetLoader(BaseLoader): | ||||
| """"loader for data sets""" | """"loader for data sets""" | ||||
| def __init__(self, data_name, data_path): | |||||
| super(DatasetLoader, self).__init__(data_name, data_path) | |||||
| def __init__(self, data_path): | |||||
| super(DatasetLoader, self).__init__(data_path) | |||||
| class POSDatasetLoader(DatasetLoader): | class POSDatasetLoader(DatasetLoader): | ||||
| @@ -31,8 +31,8 @@ class POSDatasetLoader(DatasetLoader): | |||||
| to label5. | to label5. | ||||
| """ | """ | ||||
| def __init__(self, data_name, data_path): | |||||
| super(POSDatasetLoader, self).__init__(data_name, data_path) | |||||
| def __init__(self, data_path): | |||||
| super(POSDatasetLoader, self).__init__(data_path) | |||||
| def load(self): | def load(self): | ||||
| assert os.path.exists(self.data_path) | assert os.path.exists(self.data_path) | ||||
| @@ -84,8 +84,8 @@ class TokenizeDatasetLoader(DatasetLoader): | |||||
| Data set loader for tokenization data sets | Data set loader for tokenization data sets | ||||
| """ | """ | ||||
| def __init__(self, data_name, data_path): | |||||
| super(TokenizeDatasetLoader, self).__init__(data_name, data_path) | |||||
| def __init__(self, data_path): | |||||
| super(TokenizeDatasetLoader, self).__init__(data_path) | |||||
| def load_pku(self, max_seq_len=32): | def load_pku(self, max_seq_len=32): | ||||
| """ | """ | ||||
| @@ -138,8 +138,8 @@ class TokenizeDatasetLoader(DatasetLoader): | |||||
| class ClassDatasetLoader(DatasetLoader): | class ClassDatasetLoader(DatasetLoader): | ||||
| """Loader for classification data sets""" | """Loader for classification data sets""" | ||||
| def __init__(self, data_name, data_path): | |||||
| super(ClassDatasetLoader, self).__init__(data_name, data_path) | |||||
| def __init__(self, data_path): | |||||
| super(ClassDatasetLoader, self).__init__(data_path) | |||||
| def load(self): | def load(self): | ||||
| assert os.path.exists(self.data_path) | assert os.path.exists(self.data_path) | ||||
| @@ -177,7 +177,7 @@ class ConllLoader(DatasetLoader): | |||||
| :param str data_name: the name of the conll data set | :param str data_name: the name of the conll data set | ||||
| :param str data_path: the path to the conll data set | :param str data_path: the path to the conll data set | ||||
| """ | """ | ||||
| super(ConllLoader, self).__init__(data_name, data_path) | |||||
| super(ConllLoader, self).__init__(data_path) | |||||
| self.data_set = self.parse(self.load()) | self.data_set = self.parse(self.load()) | ||||
| def load(self): | def load(self): | ||||
| @@ -209,8 +209,8 @@ class ConllLoader(DatasetLoader): | |||||
| class LMDatasetLoader(DatasetLoader): | class LMDatasetLoader(DatasetLoader): | ||||
| def __init__(self, data_name, data_path): | |||||
| super(LMDatasetLoader, self).__init__(data_name, data_path) | |||||
| def __init__(self, data_path): | |||||
| super(LMDatasetLoader, self).__init__(data_path) | |||||
| def load(self): | def load(self): | ||||
| if not os.path.exists(self.data_path): | if not os.path.exists(self.data_path): | ||||
| @@ -220,13 +220,57 @@ class LMDatasetLoader(DatasetLoader): | |||||
| return text.strip().split() | return text.strip().split() | ||||
| if __name__ == "__main__": | |||||
| class PeopleDailyCorpusLoader(DatasetLoader): | |||||
| """ | """ | ||||
| data = POSDatasetLoader("xxx", "../../test/data_for_tests/people.txt").load_lines() | |||||
| for example in data: | |||||
| for w, l in zip(example[0], example[1]): | |||||
| print(w, l) | |||||
| People Daily Corpus: Chinese word segmentation, POS tag, NER | |||||
| """ | """ | ||||
| ans = TokenizeDatasetLoader("xxx", "/home/zyfeng/Desktop/data/icwb2-data/training/test").load_pku() | |||||
| print(ans) | |||||
| def __init__(self, data_path): | |||||
| super(PeopleDailyCorpusLoader, self).__init__(data_path) | |||||
| def load(self): | |||||
| with open(self.data_path, "r", encoding="utf-8") as f: | |||||
| sents = f.readlines() | |||||
| pos_tag_examples = [] | |||||
| ner_examples = [] | |||||
| for sent in sents: | |||||
| inside_ne = False | |||||
| sent_pos_tag = [] | |||||
| sent_words = [] | |||||
| sent_ner = [] | |||||
| words = sent.strip().split()[1:] | |||||
| for word in words: | |||||
| if "[" in word and "]" in word: | |||||
| ner_tag = "U" | |||||
| print(word) | |||||
| elif "[" in word: | |||||
| inside_ne = True | |||||
| ner_tag = "B" | |||||
| word = word[1:] | |||||
| elif "]" in word: | |||||
| ner_tag = "L" | |||||
| word = word[:word.index("]")] | |||||
| if inside_ne is True: | |||||
| inside_ne = False | |||||
| else: | |||||
| raise RuntimeError("only ] appears!") | |||||
| else: | |||||
| if inside_ne is True: | |||||
| ner_tag = "I" | |||||
| else: | |||||
| ner_tag = "O" | |||||
| tmp = word.split("/") | |||||
| token, pos = tmp[0], tmp[1] | |||||
| sent_ner.append(ner_tag) | |||||
| sent_pos_tag.append(pos) | |||||
| sent_words.append(token) | |||||
| pos_tag_examples.append([sent_words, sent_pos_tag]) | |||||
| ner_examples.append([sent_words, sent_ner]) | |||||
| return pos_tag_examples, ner_examples | |||||
| if __name__ == "__main__": | |||||
| loader = PeopleDailyCorpusLoader("./") | |||||
| pos, ner = loader.load() | |||||
| print(pos[:10]) | |||||
| print(ner[:10]) | |||||
+ 44
- 2
fastNLP/loader/embed_loader.py
View File
| @@ -1,8 +1,50 @@ | |||||
| import _pickle | |||||
| import os | |||||
| import numpy as np | |||||
| from fastNLP.loader.base_loader import BaseLoader | from fastNLP.loader.base_loader import BaseLoader | ||||
| class EmbedLoader(BaseLoader): | class EmbedLoader(BaseLoader): | ||||
| """docstring for EmbedLoader""" | """docstring for EmbedLoader""" | ||||
| def __init__(self, data_name, data_path): | |||||
| super(EmbedLoader, self).__init__(data_name, data_path) | |||||
| def __init__(self, data_path): | |||||
| super(EmbedLoader, self).__init__(data_path) | |||||
| @staticmethod | |||||
| def load_embedding(emb_dim, emb_file, word_dict, emb_pkl): | |||||
| """Load the pre-trained embedding and combine with the given dictionary. | |||||
| :param emb_file: str, the pre-trained embedding. | |||||
| The embedding file should have the following format: | |||||
| Each line is a word embedding, where a word string is followed by multiple floats. | |||||
| Floats are separated by space. The word and the first float are separated by space. | |||||
| :param word_dict: dict, a mapping from word to index. | |||||
| :param emb_dim: int, the dimension of the embedding. Should be the same as pre-trained embedding. | |||||
| :param emb_pkl: str, the embedding pickle file. | |||||
| :return embedding_np: numpy array of shape (len(word_dict), emb_dim) | |||||
| TODO: fragile code | |||||
| """ | |||||
| # If the embedding pickle exists, load it and return. | |||||
| if os.path.exists(emb_pkl): | |||||
| with open(emb_pkl, "rb") as f: | |||||
| embedding_np = _pickle.load(f) | |||||
| return embedding_np | |||||
| # Otherwise, load the pre-trained embedding. | |||||
| with open(emb_file, "r", encoding="utf-8") as f: | |||||
| # begin with a random embedding | |||||
| embedding_np = np.random.uniform(-1, 1, size=(len(word_dict), emb_dim)) | |||||
| for line in f: | |||||
| line = line.strip().split() | |||||
| if len(line) != emb_dim + 1: | |||||
| # skip this line if two embedding dimension not match | |||||
| continue | |||||
| if line[0] in word_dict: | |||||
| # find the word and replace its embedding with a pre-trained one | |||||
| embedding_np[word_dict[line[0]]] = [float(i) for i in line[1:]] | |||||
| # save and return the result | |||||
| with open(emb_pkl, "wb") as f: | |||||
| _pickle.dump(embedding_np, f) | |||||
| return embedding_np | |||||
+ 2
- 2
fastNLP/loader/model_loader.py
View File
| @@ -8,8 +8,8 @@ class ModelLoader(BaseLoader): | |||||
| Loader for models. | Loader for models. | ||||
| """ | """ | ||||
| def __init__(self, data_name, data_path): | |||||
| super(ModelLoader, self).__init__(data_name, data_path) | |||||
| def __init__(self, data_path): | |||||
| super(ModelLoader, self).__init__(data_path) | |||||
| @staticmethod | @staticmethod | ||||
| def load_pytorch(empty_model, model_path): | def load_pytorch(empty_model, model_path): | ||||
+ 6
- 6
fastNLP/models/cnn_text_classification.py
View File
| @@ -5,7 +5,7 @@ import torch | |||||
| import torch.nn as nn | import torch.nn as nn | ||||
| # import torch.nn.functional as F | # import torch.nn.functional as F | ||||
| from fastNLP.modules.encoder.conv_maxpool import ConvMaxpool | |||||
| import fastNLP.modules.encoder as encoder | |||||
| class CNNText(torch.nn.Module): | class CNNText(torch.nn.Module): | ||||
| @@ -18,22 +18,22 @@ class CNNText(torch.nn.Module): | |||||
| def __init__(self, args): | def __init__(self, args): | ||||
| super(CNNText, self).__init__() | super(CNNText, self).__init__() | ||||
| class_num = args["num_classes"] | |||||
| num_classes = args["num_classes"] | |||||
| kernel_nums = [100, 100, 100] | kernel_nums = [100, 100, 100] | ||||
| kernel_sizes = [3, 4, 5] | kernel_sizes = [3, 4, 5] | ||||
| embed_num = args["vocab_size"] | |||||
| vocab_size = args["vocab_size"] | |||||
| embed_dim = 300 | embed_dim = 300 | ||||
| pretrained_embed = None | pretrained_embed = None | ||||
| drop_prob = 0.5 | drop_prob = 0.5 | ||||
| # no support for pre-trained embedding currently | # no support for pre-trained embedding currently | ||||
| self.embed = nn.Embedding(embed_num, embed_dim, padding_idx=0) | |||||
| self.conv_pool = ConvMaxpool( | |||||
| self.embed = encoder.embedding.Embedding(vocab_size, embed_dim) | |||||
| self.conv_pool = encoder.conv_maxpool.ConvMaxpool( | |||||
| in_channels=embed_dim, | in_channels=embed_dim, | ||||
| out_channels=kernel_nums, | out_channels=kernel_nums, | ||||
| kernel_sizes=kernel_sizes) | kernel_sizes=kernel_sizes) | ||||
| self.dropout = nn.Dropout(drop_prob) | self.dropout = nn.Dropout(drop_prob) | ||||
| self.fc = nn.Linear(sum(kernel_nums), class_num) | |||||
| self.fc = encoder.linear.Linear(sum(kernel_nums), num_classes) | |||||
| def forward(self, x): | def forward(self, x): | ||||
| x = self.embed(x) # [N,L] -> [N,L,C] | x = self.embed(x) # [N,L] -> [N,L,C] | ||||
+ 2
- 1
fastNLP/modules/decoder/__init__.py
View File
| @@ -1,3 +1,4 @@ | |||||
| from .CRF import ConditionalRandomField | from .CRF import ConditionalRandomField | ||||
| from .MLP import MLP | |||||
| __all__ = ["ConditionalRandomField"] | |||||
| __all__ = ["ConditionalRandomField", "MLP"] | |||||
+ 3
- 1
fastNLP/modules/encoder/__init__.py
View File
| @@ -2,8 +2,10 @@ from .embedding import Embedding | |||||
| from .linear import Linear | from .linear import Linear | ||||
| from .lstm import Lstm | from .lstm import Lstm | ||||
| from .conv import Conv | from .conv import Conv | ||||
| from .conv_maxpool import ConvMaxpool | |||||
| __all__ = ["Lstm", | __all__ = ["Lstm", | ||||
| "Embedding", | "Embedding", | ||||
| "Linear", | "Linear", | ||||
| "Conv"] | |||||
| "Conv", | |||||
| "ConvMaxpool"] | |||||
+ 5
- 0
fastNLP/modules/encoder/conv_maxpool.py
View File
| @@ -4,6 +4,7 @@ | |||||
| import torch | import torch | ||||
| import torch.nn as nn | import torch.nn as nn | ||||
| import torch.nn.functional as F | import torch.nn.functional as F | ||||
| from torch.nn.init import xavier_uniform_ | |||||
| class ConvMaxpool(nn.Module): | class ConvMaxpool(nn.Module): | ||||
| @@ -21,6 +22,7 @@ class ConvMaxpool(nn.Module): | |||||
| if isinstance(kernel_sizes, int): | if isinstance(kernel_sizes, int): | ||||
| out_channels = [out_channels] | out_channels = [out_channels] | ||||
| kernel_sizes = [kernel_sizes] | kernel_sizes = [kernel_sizes] | ||||
| self.convs = nn.ModuleList([nn.Conv1d( | self.convs = nn.ModuleList([nn.Conv1d( | ||||
| in_channels=in_channels, | in_channels=in_channels, | ||||
| out_channels=oc, | out_channels=oc, | ||||
| @@ -31,6 +33,9 @@ class ConvMaxpool(nn.Module): | |||||
| groups=groups, | groups=groups, | ||||
| bias=bias) | bias=bias) | ||||
| for oc, ks in zip(out_channels, kernel_sizes)]) | for oc, ks in zip(out_channels, kernel_sizes)]) | ||||
| for conv in self.convs: | |||||
| xavier_uniform_(conv.weight) # weight initialization | |||||
| else: | else: | ||||
| raise Exception( | raise Exception( | ||||
| 'Incorrect kernel sizes: should be list, tuple or int') | 'Incorrect kernel sizes: should be list, tuple or int') | ||||
+ 0
- 114
reproduction/chinese_word_seg/cws_train.py
View File
| @@ -1,114 +0,0 @@ | |||||
| import sys | |||||
| sys.path.append("..") | |||||
| from fastNLP.loader.config_loader import ConfigLoader, ConfigSection | |||||
| from fastNLP.core.trainer import SeqLabelTrainer | |||||
| from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, BaseLoader | |||||
| from fastNLP.core.preprocess import SeqLabelPreprocess, load_pickle | |||||
| from fastNLP.saver.model_saver import ModelSaver | |||||
| from fastNLP.loader.model_loader import ModelLoader | |||||
| from fastNLP.core.tester import SeqLabelTester | |||||
| from fastNLP.models.sequence_modeling import SeqLabeling | |||||
| from fastNLP.core.predictor import Predictor | |||||
| data_name = "pku_training.utf8" | |||||
| cws_data_path = "/home/zyfeng/data/pku_training.utf8" | |||||
| pickle_path = "./save/" | |||||
| data_infer_path = "/home/zyfeng/data/pku_test.utf8" | |||||
| def infer(): | |||||
| # Load infer configuration, the same as test | |||||
| test_args = ConfigSection() | |||||
| ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS_test": test_args}) | |||||
| # fetch dictionary size and number of labels from pickle files | |||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | |||||
| test_args["vocab_size"] = len(word2index) | |||||
| index2label = load_pickle(pickle_path, "id2class.pkl") | |||||
| test_args["num_classes"] = len(index2label) | |||||
| # Define the same model | |||||
| model = SeqLabeling(test_args) | |||||
| # Dump trained parameters into the model | |||||
| ModelLoader.load_pytorch(model, "./data_for_tests/saved_model.pkl") | |||||
| print("model loaded!") | |||||
| # Data Loader | |||||
| raw_data_loader = BaseLoader(data_name, data_infer_path) | |||||
| infer_data = raw_data_loader.load_lines() | |||||
| # Inference interface | |||||
| infer = Predictor(pickle_path) | |||||
| results = infer.predict(model, infer_data) | |||||
| print(results) | |||||
| print("Inference finished!") | |||||
| def train_test(): | |||||
| # Config Loader | |||||
| train_args = ConfigSection() | |||||
| test_args = ConfigSection() | |||||
| ConfigLoader("good_name", "good_path").load_config("./cws.cfg", {"train": train_args, "test": test_args}) | |||||
| # Data Loader | |||||
| loader = TokenizeDatasetLoader(data_name, cws_data_path) | |||||
| train_data = loader.load_pku() | |||||
| # Preprocessor | |||||
| preprocess = SeqLabelPreprocess() | |||||
| data_train, data_dev = preprocess.run(train_data, pickle_path=pickle_path, train_dev_split=0.3) | |||||
| train_args["vocab_size"] = preprocess.vocab_size | |||||
| train_args["num_classes"] = preprocess.num_classes | |||||
| # Trainer | |||||
| trainer = SeqLabelTrainer(train_args) | |||||
| # Model | |||||
| model = SeqLabeling(train_args) | |||||
| # Start training | |||||
| trainer.train(model, data_train, data_dev) | |||||
| print("Training finished!") | |||||
| # Saver | |||||
| saver = ModelSaver("./save/saved_model.pkl") | |||||
| saver.save_pytorch(model) | |||||
| print("Model saved!") | |||||
| # testing with validation set | |||||
| test(data_dev) | |||||
| def test(test_data): | |||||
| # Config Loader | |||||
| train_args = ConfigSection() | |||||
| ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS": train_args}) | |||||
| # Define the same model | |||||
| model = SeqLabeling(train_args) | |||||
| # Dump trained parameters into the model | |||||
| ModelLoader.load_pytorch(model, "./data_for_tests/saved_model.pkl") | |||||
| print("model loaded!") | |||||
| # Load test configuration | |||||
| test_args = ConfigSection() | |||||
| ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS_test": test_args}) | |||||
| # Tester | |||||
| tester = SeqLabelTester(test_args) | |||||
| # Start testing | |||||
| tester.test(model, test_data) | |||||
| # print test results | |||||
| print(tester.show_matrices()) | |||||
| print("model tested!") | |||||
| if __name__ == "__main__": | |||||
| train_test() | |||||
+ 12
- 0
reproduction/chinese_word_segment/cws.cfg
View File
| @@ -31,4 +31,16 @@ pickle_path = "./save/" | |||||
| use_crf = true | use_crf = true | ||||
| use_cuda = true | use_cuda = true | ||||
| rnn_hidden_units = 100 | rnn_hidden_units = 100 | ||||
| word_emb_dim = 100 | |||||
| [model] | |||||
| save_output = true | |||||
| validate_in_training = true | |||||
| save_dev_input = false | |||||
| save_loss = true | |||||
| batch_size = 640 | |||||
| pickle_path = "./save/" | |||||
| use_crf = true | |||||
| use_cuda = true | |||||
| rnn_hidden_units = 100 | |||||
| word_emb_dim = 100 | word_emb_dim = 100 | ||||
+ 6
- 6
reproduction/chinese_word_segment/run.py
View File
| @@ -27,7 +27,7 @@ data_infer_path = os.path.join(datadir, "infer.utf8") | |||||
| def infer(): | def infer(): | ||||
| # Config Loader | # Config Loader | ||||
| test_args = ConfigSection() | test_args = ConfigSection() | ||||
| ConfigLoader("config", "").load_config(cfgfile, {"POS_test": test_args}) | |||||
| ConfigLoader("config").load_config(cfgfile, {"POS_test": test_args}) | |||||
| # fetch dictionary size and number of labels from pickle files | # fetch dictionary size and number of labels from pickle files | ||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | word2index = load_pickle(pickle_path, "word2id.pkl") | ||||
| @@ -47,7 +47,7 @@ def infer(): | |||||
| raise | raise | ||||
| # Data Loader | # Data Loader | ||||
| raw_data_loader = BaseLoader(data_name, data_infer_path) | |||||
| raw_data_loader = BaseLoader(data_infer_path) | |||||
| infer_data = raw_data_loader.load_lines() | infer_data = raw_data_loader.load_lines() | ||||
| print('data loaded') | print('data loaded') | ||||
| @@ -63,10 +63,10 @@ def train(): | |||||
| # Config Loader | # Config Loader | ||||
| train_args = ConfigSection() | train_args = ConfigSection() | ||||
| test_args = ConfigSection() | test_args = ConfigSection() | ||||
| ConfigLoader("good_name", "good_path").load_config(cfgfile, {"train": train_args, "test": test_args}) | |||||
| ConfigLoader("good_path").load_config(cfgfile, {"train": train_args, "test": test_args}) | |||||
| # Data Loader | # Data Loader | ||||
| loader = TokenizeDatasetLoader(data_name, cws_data_path) | |||||
| loader = TokenizeDatasetLoader(cws_data_path) | |||||
| train_data = loader.load_pku() | train_data = loader.load_pku() | ||||
| # Preprocessor | # Preprocessor | ||||
| @@ -100,7 +100,7 @@ def train(): | |||||
| def test(): | def test(): | ||||
| # Config Loader | # Config Loader | ||||
| test_args = ConfigSection() | test_args = ConfigSection() | ||||
| ConfigLoader("config", "").load_config(cfgfile, {"POS_test": test_args}) | |||||
| ConfigLoader("config").load_config(cfgfile, {"POS_test": test_args}) | |||||
| # fetch dictionary size and number of labels from pickle files | # fetch dictionary size and number of labels from pickle files | ||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | word2index = load_pickle(pickle_path, "word2id.pkl") | ||||
| @@ -125,7 +125,7 @@ def test(): | |||||
| tester.test(model, dev_data) | tester.test(model, dev_data) | ||||
| # print test results | # print test results | ||||
| print(tester.show_matrices()) | |||||
| print(tester.show_metrics()) | |||||
| print("model tested!") | print("model tested!") | ||||
reproduction/chinese_word_seg/cws.cfg → reproduction/pos_tag_model/pos_tag.cfg
View File
| @@ -1,29 +1,35 @@ | |||||
| [train] | [train] | ||||
| epochs = 10 | |||||
| batch_size = 32 | |||||
| epochs = 30 | |||||
| batch_size = 64 | |||||
| pickle_path = "./save/" | pickle_path = "./save/" | ||||
| validate = true | validate = true | ||||
| save_best_dev = true | save_best_dev = true | ||||
| model_saved_path = "./save/" | model_saved_path = "./save/" | ||||
| rnn_hidden_units = 100 | rnn_hidden_units = 100 | ||||
| rnn_layers = 2 | |||||
| rnn_bi_direction = true | |||||
| word_emb_dim = 100 | word_emb_dim = 100 | ||||
| dropout = 0.5 | |||||
| use_crf = true | use_crf = true | ||||
| use_cuda = true | use_cuda = true | ||||
| print_every_step = 10 | |||||
| [test] | [test] | ||||
| save_output = true | save_output = true | ||||
| validate_in_training = true | validate_in_training = true | ||||
| save_dev_input = false | save_dev_input = false | ||||
| save_loss = true | save_loss = true | ||||
| batch_size = 64 | |||||
| batch_size = 640 | |||||
| pickle_path = "./save/" | |||||
| use_crf = true | |||||
| use_cuda = true | |||||
| [POS_test] | |||||
| save_output = true | |||||
| validate_in_training = true | |||||
| save_dev_input = false | |||||
| save_loss = true | |||||
| batch_size = 640 | |||||
| pickle_path = "./save/" | pickle_path = "./save/" | ||||
| rnn_hidden_units = 100 | |||||
| rnn_layers = 1 | |||||
| rnn_bi_direction = true | |||||
| word_emb_dim = 100 | |||||
| dropout = 0.5 | |||||
| use_crf = true | use_crf = true | ||||
| use_cuda = true | use_cuda = true | ||||
| rnn_hidden_units = 100 | |||||
| word_emb_dim = 100 | |||||
+ 146
- 0
reproduction/pos_tag_model/train_pos_tag.py
View File
| @@ -0,0 +1,146 @@ | |||||
| import os | |||||
| import sys | |||||
| sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) | |||||
| from fastNLP.loader.config_loader import ConfigLoader, ConfigSection | |||||
| from fastNLP.core.trainer import SeqLabelTrainer | |||||
| from fastNLP.loader.dataset_loader import PeopleDailyCorpusLoader, BaseLoader | |||||
| from fastNLP.core.preprocess import SeqLabelPreprocess, load_pickle | |||||
| from fastNLP.saver.model_saver import ModelSaver | |||||
| from fastNLP.loader.model_loader import ModelLoader | |||||
| from fastNLP.core.tester import SeqLabelTester | |||||
| from fastNLP.models.sequence_modeling import AdvSeqLabel | |||||
| from fastNLP.core.predictor import SeqLabelInfer | |||||
| # not in the file's dir | |||||
| if len(os.path.dirname(__file__)) != 0: | |||||
| os.chdir(os.path.dirname(__file__)) | |||||
| datadir = "/home/zyfeng/data/" | |||||
| cfgfile = './pos_tag.cfg' | |||||
| data_name = "CWS_POS_TAG_NER_people_daily.txt" | |||||
| pos_tag_data_path = os.path.join(datadir, data_name) | |||||
| pickle_path = "save" | |||||
| data_infer_path = os.path.join(datadir, "infer.utf8") | |||||
| def infer(): | |||||
| # Config Loader | |||||
| test_args = ConfigSection() | |||||
| ConfigLoader("config").load_config(cfgfile, {"POS_test": test_args}) | |||||
| # fetch dictionary size and number of labels from pickle files | |||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | |||||
| test_args["vocab_size"] = len(word2index) | |||||
| index2label = load_pickle(pickle_path, "id2class.pkl") | |||||
| test_args["num_classes"] = len(index2label) | |||||
| # Define the same model | |||||
| model = AdvSeqLabel(test_args) | |||||
| try: | |||||
| ModelLoader.load_pytorch(model, "./save/saved_model.pkl") | |||||
| print('model loaded!') | |||||
| except Exception as e: | |||||
| print('cannot load model!') | |||||
| raise | |||||
| # Data Loader | |||||
| raw_data_loader = BaseLoader(data_infer_path) | |||||
| infer_data = raw_data_loader.load_lines() | |||||
| print('data loaded') | |||||
| # Inference interface | |||||
| infer = SeqLabelInfer(pickle_path) | |||||
| results = infer.predict(model, infer_data) | |||||
| print(results) | |||||
| print("Inference finished!") | |||||
| def train(): | |||||
| # Config Loader | |||||
| train_args = ConfigSection() | |||||
| test_args = ConfigSection() | |||||
| ConfigLoader("good_name").load_config(cfgfile, {"train": train_args, "test": test_args}) | |||||
| # Data Loader | |||||
| loader = PeopleDailyCorpusLoader(pos_tag_data_path) | |||||
| train_data, _ = loader.load() | |||||
| # Preprocessor | |||||
| preprocessor = SeqLabelPreprocess() | |||||
| data_train, data_dev = preprocessor.run(train_data, pickle_path=pickle_path, train_dev_split=0.3) | |||||
| train_args["vocab_size"] = preprocessor.vocab_size | |||||
| train_args["num_classes"] = preprocessor.num_classes | |||||
| # Trainer | |||||
| trainer = SeqLabelTrainer(**train_args.data) | |||||
| # Model | |||||
| model = AdvSeqLabel(train_args) | |||||
| try: | |||||
| ModelLoader.load_pytorch(model, "./save/saved_model.pkl") | |||||
| print('model parameter loaded!') | |||||
| except Exception as e: | |||||
| print("No saved model. Continue.") | |||||
| pass | |||||
| # Start training | |||||
| trainer.train(model, data_train, data_dev) | |||||
| print("Training finished!") | |||||
| # Saver | |||||
| saver = ModelSaver("./save/saved_model.pkl") | |||||
| saver.save_pytorch(model) | |||||
| print("Model saved!") | |||||
| def test(): | |||||
| # Config Loader | |||||
| test_args = ConfigSection() | |||||
| ConfigLoader("config").load_config(cfgfile, {"POS_test": test_args}) | |||||
| # fetch dictionary size and number of labels from pickle files | |||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | |||||
| test_args["vocab_size"] = len(word2index) | |||||
| index2label = load_pickle(pickle_path, "id2class.pkl") | |||||
| test_args["num_classes"] = len(index2label) | |||||
| # load dev data | |||||
| dev_data = load_pickle(pickle_path, "data_dev.pkl") | |||||
| # Define the same model | |||||
| model = AdvSeqLabel(test_args) | |||||
| # Dump trained parameters into the model | |||||
| ModelLoader.load_pytorch(model, "./save/saved_model.pkl") | |||||
| print("model loaded!") | |||||
| # Tester | |||||
| tester = SeqLabelTester(**test_args.data) | |||||
| # Start testing | |||||
| tester.test(model, dev_data) | |||||
| # print test results | |||||
| print(tester.show_metrics()) | |||||
| print("model tested!") | |||||
| if __name__ == "__main__": | |||||
| import argparse | |||||
| parser = argparse.ArgumentParser(description='Run a chinese word segmentation model') | |||||
| parser.add_argument('--mode', help='set the model\'s model', choices=['train', 'test', 'infer']) | |||||
| args = parser.parse_args() | |||||
| if args.mode == 'train': | |||||
| train() | |||||
| elif args.mode == 'test': | |||||
| test() | |||||
| elif args.mode == 'infer': | |||||
| infer() | |||||
| else: | |||||
| print('no mode specified for model!') | |||||
| parser.print_help() | |||||
+ 1
- 0
requirements.txt
View File
| @@ -1,3 +1,4 @@ | |||||
| numpy>=1.14.2 | numpy>=1.14.2 | ||||
| torch==0.4.0 | torch==0.4.0 | ||||
| torchvision>=0.1.8 | torchvision>=0.1.8 | ||||
| tensorboardX | |||||
+ 24
- 0
setup.py
View File
| @@ -0,0 +1,24 @@ | |||||
| #!/usr/bin/env python | |||||
| # coding=utf-8 | |||||
| from setuptools import setup, find_packages | |||||
| with open('README.md') as f: | |||||
| readme = f.read() | |||||
| with open('LICENSE') as f: | |||||
| license = f.read() | |||||
| with open('requirements.txt') as f: | |||||
| reqs = f.read() | |||||
| setup( | |||||
| name='fastNLP', | |||||
| version='0.0.1', | |||||
| description='fastNLP: Deep Learning Toolkit for NLP, developed by Fudan FastNLP Team', | |||||
| long_description=readme, | |||||
| license=license, | |||||
| author='fudanNLP', | |||||
| python_requires='>=3.5', | |||||
| packages=find_packages(), | |||||
| install_requires=reqs.strip().split('\n'), | |||||
| ) | |||||
+ 1
- 2
test/core/test_action.py
View File
| @@ -1,9 +1,8 @@ | |||||
| import os | |||||
| import unittest | import unittest | ||||
| from fastNLP.core.action import Action, Batchifier, SequentialSampler | from fastNLP.core.action import Action, Batchifier, SequentialSampler | ||||
| class TestAction(unittest.TestCase): | class TestAction(unittest.TestCase): | ||||
| def test_case_1(self): | def test_case_1(self): | ||||
| x = [1, 2, 3, 4, 5, 6, 7, 8] | x = [1, 2, 3, 4, 5, 6, 7, 8] | ||||
+ 7
- 8
test/loader/test_loader.py
View File
| @@ -1,13 +1,12 @@ | |||||
| import os | |||||
| import configparser | import configparser | ||||
| import json | import json | ||||
| import os | |||||
| import unittest | import unittest | ||||
| from fastNLP.loader.config_loader import ConfigSection, ConfigLoader | from fastNLP.loader.config_loader import ConfigSection, ConfigLoader | ||||
| from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, POSDatasetLoader, LMDatasetLoader | from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, POSDatasetLoader, LMDatasetLoader | ||||
| class TestConfigLoader(unittest.TestCase): | class TestConfigLoader(unittest.TestCase): | ||||
| def test_case_ConfigLoader(self): | def test_case_ConfigLoader(self): | ||||
| @@ -33,8 +32,8 @@ class TestConfigLoader(unittest.TestCase): | |||||
| return dict | return dict | ||||
| test_arg = ConfigSection() | test_arg = ConfigSection() | ||||
| ConfigLoader("config", "").load_config(os.path.join("./test/loader", "config"), {"test": test_arg}) | |||||
| #ConfigLoader("config", "").load_config("/home/ygxu/github/fastNLP_testing/fastNLP/test/loader/config", | |||||
| ConfigLoader("config").load_config(os.path.join("./test/loader", "config"), {"test": test_arg}) | |||||
| # ConfigLoader("config").load_config("/home/ygxu/github/fastNLP_testing/fastNLP/test/loader/config", | |||||
| # {"test": test_arg}) | # {"test": test_arg}) | ||||
| #dict = read_section_from_config("/home/ygxu/github/fastNLP_testing/fastNLP/test/loader/config", "test") | #dict = read_section_from_config("/home/ygxu/github/fastNLP_testing/fastNLP/test/loader/config", "test") | ||||
| @@ -58,18 +57,18 @@ class TestConfigLoader(unittest.TestCase): | |||||
| class TestDatasetLoader(unittest.TestCase): | class TestDatasetLoader(unittest.TestCase): | ||||
| def test_case_TokenizeDatasetLoader(self): | def test_case_TokenizeDatasetLoader(self): | ||||
| loader = TokenizeDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8") | |||||
| loader = TokenizeDatasetLoader("./test/data_for_tests/cws_pku_utf_8") | |||||
| data = loader.load_pku(max_seq_len=32) | data = loader.load_pku(max_seq_len=32) | ||||
| print("pass TokenizeDatasetLoader test!") | print("pass TokenizeDatasetLoader test!") | ||||
| def test_case_POSDatasetLoader(self): | def test_case_POSDatasetLoader(self): | ||||
| loader = POSDatasetLoader("people", "./test/data_for_tests/people.txt") | |||||
| loader = POSDatasetLoader("./test/data_for_tests/people.txt") | |||||
| data = loader.load() | data = loader.load() | ||||
| datas = loader.load_lines() | datas = loader.load_lines() | ||||
| print("pass POSDatasetLoader test!") | print("pass POSDatasetLoader test!") | ||||
| def test_case_LMDatasetLoader(self): | def test_case_LMDatasetLoader(self): | ||||
| loader = LMDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8") | |||||
| loader = LMDatasetLoader("./test/data_for_tests/cws_pku_utf_8") | |||||
| data = loader.load() | data = loader.load() | ||||
| datas = loader.load_lines() | datas = loader.load_lines() | ||||
| print("pass TokenizeDatasetLoader test!") | print("pass TokenizeDatasetLoader test!") | ||||
+ 0
- 138
test/ner.py
View File
| @@ -1,138 +0,0 @@ | |||||
| import _pickle | |||||
| import os | |||||
| import numpy as np | |||||
| import torch | |||||
| from fastNLP.core.preprocess import SeqLabelPreprocess | |||||
| from fastNLP.core.tester import SeqLabelTester | |||||
| from fastNLP.core.trainer import SeqLabelTrainer | |||||
| from fastNLP.models.sequence_modeling import AdvSeqLabel | |||||
| class MyNERTrainer(SeqLabelTrainer): | |||||
| def __init__(self, train_args): | |||||
| super(MyNERTrainer, self).__init__(train_args) | |||||
| self.scheduler = None | |||||
| def define_optimizer(self): | |||||
| """ | |||||
| override | |||||
| :return: | |||||
| """ | |||||
| self.optimizer = torch.optim.Adam(self._model.parameters(), lr=0.001) | |||||
| self.scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, step_size=3000, gamma=0.5) | |||||
| def update(self): | |||||
| """ | |||||
| override | |||||
| :return: | |||||
| """ | |||||
| self.optimizer.step() | |||||
| self.scheduler.step() | |||||
| def _create_validator(self, valid_args): | |||||
| return MyNERTester(valid_args) | |||||
| def best_eval_result(self, validator): | |||||
| accuracy = validator.metrics() | |||||
| if accuracy > self.best_accuracy: | |||||
| self.best_accuracy = accuracy | |||||
| return True | |||||
| else: | |||||
| return False | |||||
| class MyNERTester(SeqLabelTester): | |||||
| def __init__(self, test_args): | |||||
| super(MyNERTester, self).__init__(test_args) | |||||
| def _evaluate(self, prediction, batch_y, seq_len): | |||||
| """ | |||||
| :param prediction: [batch_size, seq_len, num_classes] | |||||
| :param batch_y: [batch_size, seq_len] | |||||
| :param seq_len: [batch_size] | |||||
| :return: | |||||
| """ | |||||
| summ = 0 | |||||
| correct = 0 | |||||
| _, indices = torch.max(prediction, 2) | |||||
| for p, y, l in zip(indices, batch_y, seq_len): | |||||
| summ += l | |||||
| correct += np.sum(p[:l].cpu().numpy() == y[:l].cpu().numpy()) | |||||
| return float(correct / summ) | |||||
| def evaluate(self, predict, truth): | |||||
| return self._evaluate(predict, truth, self.seq_len) | |||||
| def metrics(self): | |||||
| return np.mean(self.eval_history) | |||||
| def show_matrices(self): | |||||
| return "dev accuracy={:.2f}".format(float(self.metrics())) | |||||
| def embedding_process(emb_file, word_dict, emb_dim, emb_pkl): | |||||
| if os.path.exists(emb_pkl): | |||||
| with open(emb_pkl, "rb") as f: | |||||
| embedding_np = _pickle.load(f) | |||||
| return embedding_np | |||||
| with open(emb_file, "r", encoding="utf-8") as f: | |||||
| embedding_np = np.random.uniform(-1, 1, size=(len(word_dict), emb_dim)) | |||||
| for line in f: | |||||
| line = line.strip().split() | |||||
| if len(line) != emb_dim + 1: | |||||
| continue | |||||
| if line[0] in word_dict: | |||||
| embedding_np[word_dict[line[0]]] = [float(i) for i in line[1:]] | |||||
| with open(emb_pkl, "wb") as f: | |||||
| _pickle.dump(embedding_np, f) | |||||
| return embedding_np | |||||
| def data_load(data_file): | |||||
| with open(data_file, "r", encoding="utf-8") as f: | |||||
| all_data = [] | |||||
| sent = [] | |||||
| label = [] | |||||
| for line in f: | |||||
| line = line.strip().split() | |||||
| if not len(line) <= 1: | |||||
| sent.append(line[0]) | |||||
| label.append(line[1]) | |||||
| else: | |||||
| all_data.append([sent, label]) | |||||
| sent = [] | |||||
| label = [] | |||||
| return all_data | |||||
| data_path = "data_for_tests/people.txt" | |||||
| pick_path = "data_for_tests/" | |||||
| emb_path = "data_for_tests/emb50.txt" | |||||
| save_path = "data_for_tests/" | |||||
| if __name__ == "__main__": | |||||
| data = data_load(data_path) | |||||
| preprocess = SeqLabelPreprocess() | |||||
| data_train, data_dev = preprocess.run(data, pickle_path=pick_path, train_dev_split=0.3) | |||||
| # emb = embedding_process(emb_path, p.word2index, 50, os.path.join(pick_path, "embedding.pkl")) | |||||
| emb = None | |||||
| args = {"epochs": 20, | |||||
| "batch_size": 1, | |||||
| "pickle_path": pick_path, | |||||
| "validate": True, | |||||
| "save_best_dev": True, | |||||
| "model_saved_path": save_path, | |||||
| "use_cuda": True, | |||||
| "vocab_size": preprocess.vocab_size, | |||||
| "num_classes": preprocess.num_classes, | |||||
| "word_emb_dim": 50, | |||||
| "rnn_hidden_units": 100 | |||||
| } | |||||
| # emb = torch.Tensor(emb).float().cuda() | |||||
| networks = AdvSeqLabel(args, emb) | |||||
| trainer = MyNERTrainer(args) | |||||
| trainer.train(networks, data_train, data_dev) | |||||
| print("Training finished!") | |||||
+ 0
- 129
test/ner_decode.py
View File
| @@ -1,129 +0,0 @@ | |||||
| import _pickle | |||||
| import os | |||||
| import torch | |||||
| from fastNLP.core.predictor import SeqLabelInfer | |||||
| from fastNLP.core.trainer import SeqLabelTrainer | |||||
| from fastNLP.loader.model_loader import ModelLoader | |||||
| from fastNLP.models.sequence_modeling import AdvSeqLabel | |||||
| class Decode(SeqLabelTrainer): | |||||
| def __init__(self, args): | |||||
| super(Decode, self).__init__(args) | |||||
| def decoder(self, network, sents, model_path): | |||||
| self.model = network | |||||
| self.model.load_state_dict(torch.load(model_path)) | |||||
| out_put = [] | |||||
| self.mode(network, test=True) | |||||
| for batch_x in sents: | |||||
| prediction = self.data_forward(self.model, batch_x) | |||||
| seq_tag = self.model.prediction(prediction, batch_x[1]) | |||||
| out_put.append(list(seq_tag)[0]) | |||||
| return out_put | |||||
| def process_sent(sents, word2id): | |||||
| sents_num = [] | |||||
| for s in sents: | |||||
| sent_num = [] | |||||
| for c in s: | |||||
| if c in word2id: | |||||
| sent_num.append(word2id[c]) | |||||
| else: | |||||
| sent_num.append(word2id["<unk>"]) | |||||
| sents_num.append(([sent_num], [len(sent_num)])) # batch_size is 1 | |||||
| return sents_num | |||||
| def process_tag(sents, tags, id2class): | |||||
| Tags = [] | |||||
| for ttt in tags: | |||||
| Tags.append([id2class[t] for t in ttt]) | |||||
| Segs = [] | |||||
| PosNers = [] | |||||
| for sent, tag in zip(sents, tags): | |||||
| word__ = [] | |||||
| lll__ = [] | |||||
| for c, t in zip(sent, tag): | |||||
| t = id2class[t] | |||||
| l = t.split("-") | |||||
| split_ = l[0] | |||||
| pn = l[1] | |||||
| if split_ == "S": | |||||
| word__.append(c) | |||||
| lll__.append(pn) | |||||
| word_1 = "" | |||||
| elif split_ == "E": | |||||
| word_1 += c | |||||
| word__.append(word_1) | |||||
| lll__.append(pn) | |||||
| word_1 = "" | |||||
| elif split_ == "B": | |||||
| word_1 = "" | |||||
| word_1 += c | |||||
| else: | |||||
| word_1 += c | |||||
| Segs.append(word__) | |||||
| PosNers.append(lll__) | |||||
| return Segs, PosNers | |||||
| pickle_path = "data_for_tests/" | |||||
| model_path = "data_for_tests/model_best_dev.pkl" | |||||
| if __name__ == "__main__": | |||||
| with open(os.path.join(pickle_path, "id2word.pkl"), "rb") as f: | |||||
| id2word = _pickle.load(f) | |||||
| with open(os.path.join(pickle_path, "word2id.pkl"), "rb") as f: | |||||
| word2id = _pickle.load(f) | |||||
| with open(os.path.join(pickle_path, "id2class.pkl"), "rb") as f: | |||||
| id2class = _pickle.load(f) | |||||
| sent = ["中共中央总书记、国家主席江泽民", | |||||
| "逆向处理输入序列并返回逆序后的序列"] # here is input | |||||
| args = {"epochs": 1, | |||||
| "batch_size": 1, | |||||
| "pickle_path": "data_for_tests/", | |||||
| "validate": True, | |||||
| "save_best_dev": True, | |||||
| "model_saved_path": "data_for_tests/", | |||||
| "use_cuda": False, | |||||
| "vocab_size": len(word2id), | |||||
| "num_classes": len(id2class), | |||||
| "word_emb_dim": 50, | |||||
| "rnn_hidden_units": 100, | |||||
| } | |||||
| """ | |||||
| network = AdvSeqLabel(args, None) | |||||
| decoder_ = Decode(args) | |||||
| tags_num = decoder_.decoder(network, process_sent(sent, word2id), model_path=model_path) | |||||
| output_seg, output_pn = process_tag(sent, tags_num, id2class) # here is output | |||||
| print(output_seg) | |||||
| print(output_pn) | |||||
| """ | |||||
| # Define the same model | |||||
| model = AdvSeqLabel(args, None) | |||||
| # Dump trained parameters into the model | |||||
| ModelLoader.load_pytorch(model, "./data_for_tests/model_best_dev.pkl") | |||||
| print("model loaded!") | |||||
| # Inference interface | |||||
| infer = SeqLabelInfer(pickle_path) | |||||
| sent = [[ch for ch in s] for s in sent] | |||||
| results = infer.predict(model, sent) | |||||
| for res in results: | |||||
| print(res) | |||||
| print("Inference finished!") | |||||
+ 23
- 27
test/readme_example.py
View File
| @@ -1,19 +1,13 @@ | |||||
| # python: 3.5 | |||||
| # pytorch: 0.4 | |||||
| ################ | |||||
| # Test cross validation. | |||||
| ################ | |||||
| from fastNLP.loader.preprocess import ClassPreprocess | |||||
| from fastNLP.core.loss import Loss | |||||
| from fastNLP.core.optimizer import Optimizer | |||||
| from fastNLP.core.predictor import ClassificationInfer | from fastNLP.core.predictor import ClassificationInfer | ||||
| from fastNLP.core.preprocess import ClassPreprocess | |||||
| from fastNLP.core.trainer import ClassificationTrainer | from fastNLP.core.trainer import ClassificationTrainer | ||||
| from fastNLP.loader.dataset_loader import ClassDatasetLoader | from fastNLP.loader.dataset_loader import ClassDatasetLoader | ||||
| from fastNLP.models.base_model import BaseModel | from fastNLP.models.base_model import BaseModel | ||||
| from fastNLP.modules import aggregation | from fastNLP.modules import aggregation | ||||
| from fastNLP.modules import encoder | |||||
| from fastNLP.modules import decoder | from fastNLP.modules import decoder | ||||
| from fastNLP.modules import encoder | |||||
| class ClassificationModel(BaseModel): | class ClassificationModel(BaseModel): | ||||
| @@ -28,7 +22,7 @@ class ClassificationModel(BaseModel): | |||||
| self.enc = encoder.Conv( | self.enc = encoder.Conv( | ||||
| in_channels=300, out_channels=100, kernel_size=3) | in_channels=300, out_channels=100, kernel_size=3) | ||||
| self.agg = aggregation.MaxPool() | self.agg = aggregation.MaxPool() | ||||
| self.dec = decoder.MLP(100, num_classes=num_classes) | |||||
| self.dec = decoder.MLP(size_layer=[100, num_classes]) | |||||
| def forward(self, x): | def forward(self, x): | ||||
| x = self.emb(x) # [N,L] -> [N,L,C] | x = self.emb(x) # [N,L] -> [N,L,C] | ||||
| @@ -38,18 +32,17 @@ class ClassificationModel(BaseModel): | |||||
| return x | return x | ||||
| data_dir = 'data' # directory to save data and model | |||||
| train_path = 'test/data_for_tests/text_classify.txt' # training set file | |||||
| data_dir = 'save/' # directory to save data and model | |||||
| train_path = './data_for_tests/text_classify.txt' # training set file | |||||
| # load dataset | # load dataset | ||||
| ds_loader = ClassDatasetLoader("train", train_path) | |||||
| ds_loader = ClassDatasetLoader(train_path) | |||||
| data = ds_loader.load() | data = ds_loader.load() | ||||
| # pre-process dataset | # pre-process dataset | ||||
| pre = ClassPreprocess(data, data_dir, cross_val=True, n_fold=5) | |||||
| # pre = ClassPreprocess(data, data_dir) | |||||
| n_classes = pre.num_classes | |||||
| vocab_size = pre.vocab_size | |||||
| pre = ClassPreprocess() | |||||
| train_set, dev_set = pre.run(data, train_dev_split=0.3, pickle_path=data_dir) | |||||
| n_classes, vocab_size = pre.num_classes, pre.vocab_size | |||||
| # construct model | # construct model | ||||
| model_args = { | model_args = { | ||||
| @@ -58,22 +51,25 @@ model_args = { | |||||
| } | } | ||||
| model = ClassificationModel(num_classes=n_classes, vocab_size=vocab_size) | model = ClassificationModel(num_classes=n_classes, vocab_size=vocab_size) | ||||
| # train model | |||||
| # construct trainer | |||||
| train_args = { | train_args = { | ||||
| "epochs": 10, | |||||
| "batch_size": 50, | |||||
| "epochs": 3, | |||||
| "batch_size": 16, | |||||
| "pickle_path": data_dir, | "pickle_path": data_dir, | ||||
| "validate": False, | "validate": False, | ||||
| "save_best_dev": False, | "save_best_dev": False, | ||||
| "model_saved_path": None, | "model_saved_path": None, | ||||
| "use_cuda": True, | "use_cuda": True, | ||||
| "learn_rate": 1e-3, | |||||
| "momentum": 0.9} | |||||
| trainer = ClassificationTrainer(train_args) | |||||
| # trainer.train(model, ['data_train.pkl', 'data_dev.pkl']) | |||||
| trainer.cross_validate(model) | |||||
| "loss": Loss("cross_entropy"), | |||||
| "optimizer": Optimizer("Adam", lr=0.001) | |||||
| } | |||||
| trainer = ClassificationTrainer(**train_args) | |||||
| # start training | |||||
| trainer.train(model, train_data=train_set, dev_data=dev_set) | |||||
| # predict using model | # predict using model | ||||
| data_infer = [x[0] for x in data] | data_infer = [x[0] for x in data] | ||||
| infer = ClassificationInfer(data_dir) | infer = ClassificationInfer(data_dir) | ||||
| labels_pred = infer.predict(model, data_infer) | |||||
| labels_pred = infer.predict(model.cpu(), data_infer) | |||||
| print(labels_pred) | |||||
+ 8
- 8
test/seq_labeling.py
View File
| @@ -33,7 +33,7 @@ data_infer_path = args.infer | |||||
| def infer(): | def infer(): | ||||
| # Load infer configuration, the same as test | # Load infer configuration, the same as test | ||||
| test_args = ConfigSection() | test_args = ConfigSection() | ||||
| ConfigLoader("config.cfg", "").load_config(config_dir, {"POS_infer": test_args}) | |||||
| ConfigLoader("config.cfg").load_config(config_dir, {"POS_infer": test_args}) | |||||
| # fetch dictionary size and number of labels from pickle files | # fetch dictionary size and number of labels from pickle files | ||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | word2index = load_pickle(pickle_path, "word2id.pkl") | ||||
| @@ -49,7 +49,7 @@ def infer(): | |||||
| print("model loaded!") | print("model loaded!") | ||||
| # Data Loader | # Data Loader | ||||
| raw_data_loader = BaseLoader("xxx", data_infer_path) | |||||
| raw_data_loader = BaseLoader(data_infer_path) | |||||
| infer_data = raw_data_loader.load_lines() | infer_data = raw_data_loader.load_lines() | ||||
| # Inference interface | # Inference interface | ||||
| @@ -65,11 +65,11 @@ def train_and_test(): | |||||
| # Config Loader | # Config Loader | ||||
| trainer_args = ConfigSection() | trainer_args = ConfigSection() | ||||
| model_args = ConfigSection() | model_args = ConfigSection() | ||||
| ConfigLoader("config.cfg", "").load_config(config_dir, { | |||||
| ConfigLoader("config.cfg").load_config(config_dir, { | |||||
| "test_seq_label_trainer": trainer_args, "test_seq_label_model": model_args}) | "test_seq_label_trainer": trainer_args, "test_seq_label_model": model_args}) | ||||
| # Data Loader | # Data Loader | ||||
| pos_loader = POSDatasetLoader("xxx", data_path) | |||||
| pos_loader = POSDatasetLoader(data_path) | |||||
| train_data = pos_loader.load_lines() | train_data = pos_loader.load_lines() | ||||
| # Preprocessor | # Preprocessor | ||||
| @@ -117,7 +117,7 @@ def train_and_test(): | |||||
| # Load test configuration | # Load test configuration | ||||
| tester_args = ConfigSection() | tester_args = ConfigSection() | ||||
| ConfigLoader("config.cfg", "").load_config(config_dir, {"test_seq_label_tester": tester_args}) | |||||
| ConfigLoader("config.cfg").load_config(config_dir, {"test_seq_label_tester": tester_args}) | |||||
| # Tester | # Tester | ||||
| tester = SeqLabelTester(save_output=False, | tester = SeqLabelTester(save_output=False, | ||||
| @@ -134,10 +134,10 @@ def train_and_test(): | |||||
| tester.test(model, data_dev) | tester.test(model, data_dev) | ||||
| # print test results | # print test results | ||||
| print(tester.show_matrices()) | |||||
| print(tester.show_metrics()) | |||||
| print("model tested!") | print("model tested!") | ||||
| if __name__ == "__main__": | if __name__ == "__main__": | ||||
| train_and_test() | |||||
| # infer() | |||||
| # train_and_test() | |||||
| infer() | |||||
+ 8
- 8
test/test_cws.py
View File
| @@ -22,7 +22,7 @@ data_infer_path = "data_for_tests/people_infer.txt" | |||||
| def infer(): | def infer(): | ||||
| # Load infer configuration, the same as test | # Load infer configuration, the same as test | ||||
| test_args = ConfigSection() | test_args = ConfigSection() | ||||
| ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS_test": test_args}) | |||||
| ConfigLoader("config.cfg").load_config("./data_for_tests/config", {"POS_test": test_args}) | |||||
| # fetch dictionary size and number of labels from pickle files | # fetch dictionary size and number of labels from pickle files | ||||
| word2index = load_pickle(pickle_path, "word2id.pkl") | word2index = load_pickle(pickle_path, "word2id.pkl") | ||||
| @@ -38,7 +38,7 @@ def infer(): | |||||
| print("model loaded!") | print("model loaded!") | ||||
| # Data Loader | # Data Loader | ||||
| raw_data_loader = BaseLoader(data_name, data_infer_path) | |||||
| raw_data_loader = BaseLoader(data_infer_path) | |||||
| infer_data = raw_data_loader.load_lines() | infer_data = raw_data_loader.load_lines() | ||||
| """ | """ | ||||
| Transform strings into list of list of strings. | Transform strings into list of list of strings. | ||||
| @@ -61,10 +61,10 @@ def infer(): | |||||
| def train_test(): | def train_test(): | ||||
| # Config Loader | # Config Loader | ||||
| train_args = ConfigSection() | train_args = ConfigSection() | ||||
| ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS": train_args}) | |||||
| ConfigLoader("config.cfg").load_config("./data_for_tests/config", {"POS": train_args}) | |||||
| # Data Loader | # Data Loader | ||||
| loader = TokenizeDatasetLoader(data_name, cws_data_path) | |||||
| loader = TokenizeDatasetLoader(cws_data_path) | |||||
| train_data = loader.load_pku() | train_data = loader.load_pku() | ||||
| # Preprocessor | # Preprocessor | ||||
| @@ -74,7 +74,7 @@ def train_test(): | |||||
| train_args["num_classes"] = p.num_classes | train_args["num_classes"] = p.num_classes | ||||
| # Trainer | # Trainer | ||||
| trainer = SeqLabelTrainer(train_args) | |||||
| trainer = SeqLabelTrainer(**train_args.data) | |||||
| # Model | # Model | ||||
| model = SeqLabeling(train_args) | model = SeqLabeling(train_args) | ||||
| @@ -99,16 +99,16 @@ def train_test(): | |||||
| # Load test configuration | # Load test configuration | ||||
| test_args = ConfigSection() | test_args = ConfigSection() | ||||
| ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS_test": test_args}) | |||||
| ConfigLoader("config.cfg").load_config("./data_for_tests/config", {"POS_test": test_args}) | |||||
| # Tester | # Tester | ||||
| tester = SeqLabelTester(test_args) | |||||
| tester = SeqLabelTester(**test_args.data) | |||||
| # Start testing | # Start testing | ||||
| tester.test(model, data_train) | tester.test(model, data_train) | ||||
| # print test results | # print test results | ||||
| print(tester.show_matrices()) | |||||
| print(tester.show_metrics()) | |||||
| print("model tested!") | print("model tested!") | ||||
+ 44
- 2
test/test_fastNLP.py
View File
| @@ -1,9 +1,12 @@ | |||||
| import sys | import sys | ||||
| sys.path.append("..") | sys.path.append("..") | ||||
| from fastNLP.fastnlp import FastNLP | from fastNLP.fastnlp import FastNLP | ||||
| from fastNLP.fastnlp import interpret_word_seg_results | |||||
| from fastNLP.fastnlp import interpret_word_seg_results, interpret_cws_pos_results | |||||
| PATH_TO_CWS_PICKLE_FILES = "/home/zyfeng/fastNLP/reproduction/chinese_word_segment/save/" | PATH_TO_CWS_PICKLE_FILES = "/home/zyfeng/fastNLP/reproduction/chinese_word_segment/save/" | ||||
| PATH_TO_POS_TAG_PICKLE_FILES = "/home/zyfeng/data/crf_seg/" | |||||
| PATH_TO_TEXT_CLASSIFICATION_PICKLE_FILES = "/home/zyfeng/data/text_classify/" | |||||
| def word_seg(): | def word_seg(): | ||||
| nlp = FastNLP(model_dir=PATH_TO_CWS_PICKLE_FILES) | nlp = FastNLP(model_dir=PATH_TO_CWS_PICKLE_FILES) | ||||
| @@ -39,5 +42,44 @@ def test_word_seg_interpret(): | |||||
| print(interpret_word_seg_results(chars, labels)) | print(interpret_word_seg_results(chars, labels)) | ||||
| def test_interpret_cws_pos_results(): | |||||
| foo = [ | |||||
| [('这', 'S-r'), ('是', 'S-v'), ('最', 'S-d'), ('好', 'S-a'), ('的', 'S-u'), ('基', 'B-p'), ('于', 'E-p'), ('深', 'B-d'), | |||||
| ('度', 'E-d'), ('学', 'B-v'), ('习', 'E-v'), ('的', 'S-u'), ('中', 'B-nz'), ('文', 'E-nz'), ('分', 'B-vn'), | |||||
| ('词', 'E-vn'), ('系', 'B-n'), ('统', 'E-n'), ('。', 'S-w')] | |||||
| ] | |||||
| chars = [x[0] for x in foo[0]] | |||||
| labels = [x[1] for x in foo[0]] | |||||
| print(interpret_cws_pos_results(chars, labels)) | |||||
| def pos_tag(): | |||||
| nlp = FastNLP(model_dir=PATH_TO_POS_TAG_PICKLE_FILES) | |||||
| nlp.load("pos_tag_model", config_file="pos_tag.config", section_name="pos_tag_model") | |||||
| text = ["这是最好的基于深度学习的中文分词系统。", | |||||
| "大王叫我来巡山。", | |||||
| "我党多年来致力于改善人民生活水平。"] | |||||
| results = nlp.run(text) | |||||
| for example in results: | |||||
| words, labels = [], [] | |||||
| for res in example: | |||||
| words.append(res[0]) | |||||
| labels.append(res[1]) | |||||
| print(interpret_cws_pos_results(words, labels)) | |||||
| def text_classify(): | |||||
| nlp = FastNLP(model_dir=PATH_TO_TEXT_CLASSIFICATION_PICKLE_FILES) | |||||
| nlp.load("text_classify_model", config_file="text_classify.cfg", section_name="model") | |||||
| text = [ | |||||
| "世界物联网大会明日在京召开龙头股启动在即", | |||||
| "乌鲁木齐市新增一处城市中心旅游目的地", | |||||
| "朱元璋的大明朝真的源于明教吗？——告诉你一个真实的“明教”"] | |||||
| results = nlp.run(text) | |||||
| print(results) | |||||
| """ | |||||
| ['finance', 'travel', 'history'] | |||||
| """ | |||||
| if __name__ == "__main__": | if __name__ == "__main__": | ||||
| word_seg() | |||||
| text_classify() | |||||
+ 7
- 7
test/test_tester.py
View File
| @@ -5,19 +5,19 @@ from fastNLP.loader.dataset_loader import TokenizeDatasetLoader | |||||
| from fastNLP.models.sequence_modeling import SeqLabeling | from fastNLP.models.sequence_modeling import SeqLabeling | ||||
| data_name = "pku_training.utf8" | data_name = "pku_training.utf8" | ||||
| cws_data_path = "/home/zyfeng/Desktop/data/pku_training.utf8" | |||||
| pickle_path = "data_for_tests" | pickle_path = "data_for_tests" | ||||
| def foo(): | def foo(): | ||||
| loader = TokenizeDatasetLoader(data_name, "./data_for_tests/cws_pku_utf_8") | |||||
| loader = TokenizeDatasetLoader("./data_for_tests/cws_pku_utf_8") | |||||
| train_data = loader.load_pku() | train_data = loader.load_pku() | ||||
| train_args = ConfigSection() | train_args = ConfigSection() | ||||
| ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS": train_args}) | |||||
| ConfigLoader("config.cfg").load_config("./data_for_tests/config", {"POS": train_args}) | |||||
| # Preprocessor | # Preprocessor | ||||
| p = SeqLabelPreprocess(train_data, pickle_path) | |||||
| p = SeqLabelPreprocess() | |||||
| train_data = p.run(train_data) | |||||
| train_args["vocab_size"] = p.vocab_size | train_args["vocab_size"] = p.vocab_size | ||||
| train_args["num_classes"] = p.num_classes | train_args["num_classes"] = p.num_classes | ||||
| @@ -26,11 +26,11 @@ def foo(): | |||||
| valid_args = {"save_output": True, "validate_in_training": True, "save_dev_input": True, | valid_args = {"save_output": True, "validate_in_training": True, "save_dev_input": True, | ||||
| "save_loss": True, "batch_size": 8, "pickle_path": "./data_for_tests/", | "save_loss": True, "batch_size": 8, "pickle_path": "./data_for_tests/", | ||||
| "use_cuda": True} | "use_cuda": True} | ||||
| validator = SeqLabelTester(valid_args) | |||||
| validator = SeqLabelTester(**valid_args) | |||||
| print("start validation.") | print("start validation.") | ||||
| validator.test(model) | |||||
| print(validator.show_matrices()) | |||||
| validator.test(model, train_data) | |||||
| print(validator.show_metrics()) | |||||
| if __name__ == "__main__": | if __name__ == "__main__": | ||||
+ 2
- 2
test/text_classify.py
View File
| @@ -34,7 +34,7 @@ config_dir = args.config | |||||
| def infer(): | def infer(): | ||||
| # load dataset | # load dataset | ||||
| print("Loading data...") | print("Loading data...") | ||||
| ds_loader = ClassDatasetLoader("train", train_data_dir) | |||||
| ds_loader = ClassDatasetLoader(train_data_dir) | |||||
| data = ds_loader.load() | data = ds_loader.load() | ||||
| unlabeled_data = [x[0] for x in data] | unlabeled_data = [x[0] for x in data] | ||||
| @@ -69,7 +69,7 @@ def train(): | |||||
| # load dataset | # load dataset | ||||
| print("Loading data...") | print("Loading data...") | ||||
| ds_loader = ClassDatasetLoader("train", train_data_dir) | |||||
| ds_loader = ClassDatasetLoader(train_data_dir) | |||||
| data = ds_loader.load() | data = ds_loader.load() | ||||
| print(data[0]) | print(data[0]) | ||||