delete get_tokenizer in fastNLP/io/utils.py

fastNLP/io/utils.py

@@ -76,15 +76,3 @@ def check_loader_paths(paths: Union[str, Dict[str, str]]) -> Dict[str, str]:
             raise ValueError("Empty paths is not allowed.")
     else:
         raise TypeError(f"paths only supports str and dict. not {type(paths)}.")
-
-
-def get_tokenizer():
-    try:
-        import spacy
-        spacy.prefer_gpu()
-        en = spacy.load('en')
-        logger.info('use spacy tokenizer')
-        return lambda x: [w.text for w in en.tokenizer(x)]
-    except Exception as e:
-        logger.error('use raw tokenizer')
-        return lambda x: x.split()