intent preprocessor ready

maas_lib/preprocessors/space/dialog_intent_preprocessor.py

@@ -44,6 +44,4 @@ class DialogIntentPreprocessor(Preprocessor):
             Dict[str, Any]: the preprocessed data
         """
 
-        # idx = self.text_field.get_ids(data)
-
-        return {'user_idx': idx}
+        return self.text_field.preprocessor(data)

tests/preprocessors/nlp/test_dialog_intent.py

@@ -0,0 +1,26 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import unittest
+
+from tests.case.nlp.dialog_intent_case import test_case
+
+from maas_lib.preprocessors import DialogIntentPreprocessor
+from maas_lib.utils.constant import Fields, InputFields
+from maas_lib.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class DialogGenerationPreprocessorTest(unittest.TestCase):
+
+    def test_tokenize(self):
+        modeldir = '/Users/yangliu/Desktop/space-dialog-intent'
+        processor = DialogIntentPreprocessor(model_dir=modeldir)
+
+        for item in test_case:
+            print(item)
+            print(processor(item))
+
+
+if __name__ == '__main__':
+    unittest.main()