From 1c9a0b5875bd48e7b6ad3fec268a8f34772e7245 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 14 May 2019 20:59:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8A=8A=E6=9A=82=E4=B8=8D=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E7=9A=84=E5=8A=9F=E8=83=BD=E7=A7=BB=E5=88=B0=20legacy=20?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/callback.py | 6 ++++-- fastNLP/models/enas_trainer.py | 4 ++-- {fastNLP => legacy}/api/README.md | 0 {fastNLP => legacy}/api/__init__.py | 0 {fastNLP => legacy}/api/api.py | 8 ++++---- {fastNLP => legacy}/api/converter.py | 0 {fastNLP => legacy}/api/examples.py | 0 {fastNLP => legacy}/api/pipeline.py | 0 {fastNLP => legacy}/api/processor.py | 15 ++++++--------- {fastNLP => legacy}/api/utils.py | 2 +- {fastNLP => legacy}/automl/__init__.py | 0 {fastNLP => legacy}/automl/enas_controller.py | 0 {fastNLP => legacy}/automl/enas_model.py | 0 {fastNLP => legacy}/automl/enas_trainer.py | 12 ++++++------ {fastNLP => legacy}/automl/enas_utils.py | 0 {fastNLP => legacy}/component/__init__.py | 0 {fastNLP => legacy}/component/bert_tokenizer.py | 0 {test => legacy/test}/api/test_pipeline.py | 0 {test => legacy/test}/api/test_processor.py | 0 {test => legacy/test}/automl/test_enas.py | 0 20 files changed, 23 insertions(+), 24 deletions(-) rename {fastNLP => legacy}/api/README.md (100%) rename {fastNLP => legacy}/api/__init__.py (100%) rename {fastNLP => legacy}/api/api.py (98%) rename {fastNLP => legacy}/api/converter.py (100%) rename {fastNLP => legacy}/api/examples.py (100%) rename {fastNLP => legacy}/api/pipeline.py (100%) rename {fastNLP => legacy}/api/processor.py (98%) rename {fastNLP => legacy}/api/utils.py (98%) rename {fastNLP => legacy}/automl/__init__.py (100%) rename {fastNLP => legacy}/automl/enas_controller.py (100%) rename {fastNLP => legacy}/automl/enas_model.py (100%) rename {fastNLP => legacy}/automl/enas_trainer.py (98%) rename {fastNLP => legacy}/automl/enas_utils.py (100%) rename {fastNLP => legacy}/component/__init__.py (100%) rename {fastNLP => legacy}/component/bert_tokenizer.py (100%) rename {test => legacy/test}/api/test_pipeline.py (100%) rename {test => legacy/test}/api/test_processor.py (100%) rename {test => legacy/test}/automl/test_enas.py (100%) diff --git a/fastNLP/core/callback.py b/fastNLP/core/callback.py index 76a9e2e2..c944ec96 100644 --- a/fastNLP/core/callback.py +++ b/fastNLP/core/callback.py @@ -66,8 +66,9 @@ from ..io.model_io import ModelSaver, ModelLoader try: from tensorboardX import SummaryWriter + tensorboardX_flag = True except: - pass + tensorboardX_flag = False class Callback(object): @@ -581,7 +582,8 @@ class TensorboardCallback(Callback): path = os.path.join("./", 'tensorboard_logs_{}'.format(self.trainer.start_time)) else: path = os.path.join(save_dir, 'tensorboard_logs_{}'.format(self.trainer.start_time)) - self._summary_writer = SummaryWriter(path) + if tensorboardX_flag: + self._summary_writer = SummaryWriter(path) def on_batch_begin(self, batch_x, batch_y, indices): if "model" in self.options and self.graph_added is False: diff --git a/fastNLP/models/enas_trainer.py b/fastNLP/models/enas_trainer.py index cce93556..9cd7d8d0 100644 --- a/fastNLP/models/enas_trainer.py +++ b/fastNLP/models/enas_trainer.py @@ -78,7 +78,7 @@ class ENASTrainer(Trainer): results['seconds'] = 0. return results try: - if torch.cuda.is_available() and self.use_cuda: + if torch.cuda.is_available() and "cuda" in self.device: self.model = self.model.cuda() self._model_device = self.model.parameters().__next__().device self._mode(self.model, is_test=False) @@ -337,7 +337,7 @@ class ENASTrainer(Trainer): # policy loss loss = -log_probs*utils.get_variable(adv, - self.use_cuda, + 'cuda' in self.device, requires_grad=False) loss = loss.sum() # or loss.mean() diff --git a/fastNLP/api/README.md b/legacy/api/README.md similarity index 100% rename from fastNLP/api/README.md rename to legacy/api/README.md diff --git a/fastNLP/api/__init__.py b/legacy/api/__init__.py similarity index 100% rename from fastNLP/api/__init__.py rename to legacy/api/__init__.py diff --git a/fastNLP/api/api.py b/legacy/api/api.py similarity index 98% rename from fastNLP/api/api.py rename to legacy/api/api.py index 2e7cbfcf..0b5d3cd3 100644 --- a/fastNLP/api/api.py +++ b/legacy/api/api.py @@ -5,13 +5,13 @@ import torch warnings.filterwarnings('ignore') import os -from ..core.dataset import DataSet +from fastNLP.core.dataset import DataSet from .utils import load_url from .processor import ModelProcessor -from ..io.dataset_loader import _cut_long_sentence, ConllLoader -from ..core.instance import Instance +from fastNLP.io.dataset_loader import _cut_long_sentence, ConllLoader +from fastNLP.core.instance import Instance from ..api.pipeline import Pipeline -from ..core.metrics import SpanFPreRecMetric +from fastNLP.core.metrics import SpanFPreRecMetric from .processor import IndexerProcessor # TODO add pretrain urls diff --git a/fastNLP/api/converter.py b/legacy/api/converter.py similarity index 100% rename from fastNLP/api/converter.py rename to legacy/api/converter.py diff --git a/fastNLP/api/examples.py b/legacy/api/examples.py similarity index 100% rename from fastNLP/api/examples.py rename to legacy/api/examples.py diff --git a/fastNLP/api/pipeline.py b/legacy/api/pipeline.py similarity index 100% rename from fastNLP/api/pipeline.py rename to legacy/api/pipeline.py diff --git a/fastNLP/api/processor.py b/legacy/api/processor.py similarity index 98% rename from fastNLP/api/processor.py rename to legacy/api/processor.py index 3c60e621..4c442ed2 100644 --- a/fastNLP/api/processor.py +++ b/legacy/api/processor.py @@ -3,10 +3,10 @@ from collections import defaultdict import torch -from ..core.batch import Batch -from ..core.dataset import DataSet -from ..core.sampler import SequentialSampler -from ..core.vocabulary import Vocabulary +from fastNLP.core.batch import Batch +from fastNLP.core.dataset import DataSet +from fastNLP.core.sampler import SequentialSampler +from fastNLP.core.vocabulary import Vocabulary class Processor(object): @@ -232,7 +232,7 @@ class SeqLenProcessor(Processor): return dataset -from ..core.utils import _build_args +from fastNLP.core.utils import _build_args class ModelProcessor(Processor): @@ -257,10 +257,7 @@ class ModelProcessor(Processor): data_iterator = Batch(dataset, batch_size=self.batch_size, sampler=SequentialSampler()) batch_output = defaultdict(list) - if hasattr(self.model, "predict"): - predict_func = self.model.predict - else: - predict_func = self.model.forward + predict_func = self.model.forward with torch.no_grad(): for batch_x, _ in data_iterator: refined_batch_x = _build_args(predict_func, **batch_x) diff --git a/fastNLP/api/utils.py b/legacy/api/utils.py similarity index 98% rename from fastNLP/api/utils.py rename to legacy/api/utils.py index e8e7c42a..184e5fe6 100644 --- a/fastNLP/api/utils.py +++ b/legacy/api/utils.py @@ -22,7 +22,7 @@ except ImportError: try: from tqdm.auto import tqdm except: - from ..core.utils import _pseudo_tqdm as tqdm + from fastNLP.core.utils import _pseudo_tqdm as tqdm # matches bfd8deac from resnet18-bfd8deac.pth HASH_REGEX = re.compile(r'-([a-f0-9]*)\.') diff --git a/fastNLP/automl/__init__.py b/legacy/automl/__init__.py similarity index 100% rename from fastNLP/automl/__init__.py rename to legacy/automl/__init__.py diff --git a/fastNLP/automl/enas_controller.py b/legacy/automl/enas_controller.py similarity index 100% rename from fastNLP/automl/enas_controller.py rename to legacy/automl/enas_controller.py diff --git a/fastNLP/automl/enas_model.py b/legacy/automl/enas_model.py similarity index 100% rename from fastNLP/automl/enas_model.py rename to legacy/automl/enas_model.py diff --git a/fastNLP/automl/enas_trainer.py b/legacy/automl/enas_trainer.py similarity index 98% rename from fastNLP/automl/enas_trainer.py rename to legacy/automl/enas_trainer.py index 8f51c2cd..e3524aa9 100644 --- a/fastNLP/automl/enas_trainer.py +++ b/legacy/automl/enas_trainer.py @@ -11,15 +11,15 @@ import torch try: from tqdm.auto import tqdm except: - from ..core.utils import _pseudo_tqdm as tqdm + from fastNLP.core.utils import _pseudo_tqdm as tqdm -from ..core.batch import Batch -from ..core.callback import CallbackException -from ..core.dataset import DataSet -from ..core.utils import _move_dict_value_to_device +from fastNLP.core.batch import Batch +from fastNLP.core.callback import CallbackException +from fastNLP.core.dataset import DataSet +from fastNLP.core.utils import _move_dict_value_to_device import fastNLP from . import enas_utils as utils -from ..core.utils import _build_args +from fastNLP.core.utils import _build_args from torch.optim import Adam diff --git a/fastNLP/automl/enas_utils.py b/legacy/automl/enas_utils.py similarity index 100% rename from fastNLP/automl/enas_utils.py rename to legacy/automl/enas_utils.py diff --git a/fastNLP/component/__init__.py b/legacy/component/__init__.py similarity index 100% rename from fastNLP/component/__init__.py rename to legacy/component/__init__.py diff --git a/fastNLP/component/bert_tokenizer.py b/legacy/component/bert_tokenizer.py similarity index 100% rename from fastNLP/component/bert_tokenizer.py rename to legacy/component/bert_tokenizer.py diff --git a/test/api/test_pipeline.py b/legacy/test/api/test_pipeline.py similarity index 100% rename from test/api/test_pipeline.py rename to legacy/test/api/test_pipeline.py diff --git a/test/api/test_processor.py b/legacy/test/api/test_processor.py similarity index 100% rename from test/api/test_processor.py rename to legacy/test/api/test_processor.py diff --git a/test/automl/test_enas.py b/legacy/test/automl/test_enas.py similarity index 100% rename from test/automl/test_enas.py rename to legacy/test/automl/test_enas.py