Browse Source

fix dataset

tags/v0.2.0
yunfan 5 years ago
parent
commit
1806bbdbec
2 changed files with 9 additions and 2 deletions
  1. +7
    -2
      fastNLP/api/parser.py
  2. +2
    -0
      fastNLP/core/dataset.py

+ 7
- 2
fastNLP/api/parser.py View File

@@ -3,6 +3,7 @@ from fastNLP.core.dataset import DataSet
from fastNLP.core.predictor import Predictor
from fastNLP.api.pipeline import Pipeline
from fastNLP.api.processor import *
from fastNLP.models.biaffine_parser import BiaffineParser


class DependencyParser(API):
@@ -23,9 +24,13 @@ class DependencyParser(API):
def build(self):
pipe = Pipeline()

# build pipeline
word_seq = 'word_seq'
pos_seq = 'pos_seq'
pipe.add_processor(Num2TagProcessor('<NUM>', word_seq))
pipe.add_processor(Num2TagProcessor('<NUM>', 'raw_sentence', word_seq))
pipe.add_processor(IndexerProcessor(word_vocab, word_seq, word_seq+'_idx'))
pipe.add_processor(IndexerProcessor(pos_vocab, pos_seq, pos_seq+'_idx'))
pipe.add_processor()

# load model parameters
self.model = BiaffineParser()
self.pipeline = pipe

+ 2
- 0
fastNLP/core/dataset.py View File

@@ -86,6 +86,8 @@ class DataSet(object):
return self.field_arrays[name]

def __len__(self):
if len(self.field_arrays) == 0:
return 0
field = iter(self.field_arrays.values()).__next__()
return len(field)



Loading…
Cancel
Save