diff --git a/maas_lib/preprocessors/space/dialog_intent_preprocessor.py b/maas_lib/preprocessors/space/dialog_intent_preprocessor.py index 43ced78c..b8c5d34e 100644 --- a/maas_lib/preprocessors/space/dialog_intent_preprocessor.py +++ b/maas_lib/preprocessors/space/dialog_intent_preprocessor.py @@ -44,6 +44,4 @@ class DialogIntentPreprocessor(Preprocessor): Dict[str, Any]: the preprocessed data """ - # idx = self.text_field.get_ids(data) - - return {'user_idx': idx} + return self.text_field.preprocessor(data) diff --git a/tests/preprocessors/nlp/test_dialog_intent.py b/tests/preprocessors/nlp/test_dialog_intent.py new file mode 100644 index 00000000..3f0500b7 --- /dev/null +++ b/tests/preprocessors/nlp/test_dialog_intent.py @@ -0,0 +1,26 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import unittest + +from tests.case.nlp.dialog_intent_case import test_case + +from maas_lib.preprocessors import DialogIntentPreprocessor +from maas_lib.utils.constant import Fields, InputFields +from maas_lib.utils.logger import get_logger + +logger = get_logger() + + +class DialogGenerationPreprocessorTest(unittest.TestCase): + + def test_tokenize(self): + modeldir = '/Users/yangliu/Desktop/space-dialog-intent' + processor = DialogIntentPreprocessor(model_dir=modeldir) + + for item in test_case: + print(item) + print(processor(item)) + + +if __name__ == '__main__': + unittest.main()