Browse Source

[update] multi criteria cws

tags/v0.5.5
yunfan 4 years ago
parent
commit
75d593702a
1 changed files with 1 additions and 1 deletions
  1. +1
    -1
      reproduction/multi-criteria-cws/data-prepare.py

+ 1
- 1
reproduction/multi-criteria-cws/data-prepare.py View File

@@ -51,7 +51,7 @@ def preprocess(text):
def to_sentence_list(text, split_long_sentence=False): def to_sentence_list(text, split_long_sentence=False):
text = preprocess(text) text = preprocess(text)
delimiter = set() delimiter = set()
delimiter.update("。!?:;…、,(),;!?、,\"'")
delimiter.update("。!?:;…、,(),;!?、.\"'")
delimiter.add("……") delimiter.add("……")
sent_list = [] sent_list = []
sent = [] sent = []


Loading…
Cancel
Save