From ab55f25e2088137e4cc2400a534984e927459112 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Fri, 31 Aug 2018 10:46:56 +0800 Subject: [PATCH] Updates to Trainer/Tester/fastnlp 1. Tester has a parameter "print_every_step" to control printing. print_every_step == 0 means NO print. 2. Tester's evaluate return (list of) floats, rather than torch.cuda.tensor 3. Trainer also has a parameter "print_every_step". The same usage. 4. In training, validation steps are not shown. 5. Updates to code comments. 6. fastnlp.py is ready for CWS. test_fastNLP.py works. --- fastNLP/core/preprocess.py | 4 +- fastNLP/core/tester.py | 4 +- fastNLP/core/trainer.py | 79 +++++++---------------- fastNLP/fastnlp.py | 17 +++-- fastnlp-architecture.jpg | Bin 35615 -> 0 bytes reproduction/chinese_word_segment/run.py | 27 ++++---- test/seq_labeling.py | 4 +- test/test_fastNLP.py | 2 +- 8 files changed, 61 insertions(+), 76 deletions(-) delete mode 100644 fastnlp-architecture.jpg diff --git a/fastNLP/core/preprocess.py b/fastNLP/core/preprocess.py index 1805f4eb..99bf45ba 100644 --- a/fastNLP/core/preprocess.py +++ b/fastNLP/core/preprocess.py @@ -19,13 +19,13 @@ DEFAULT_WORD_TO_INDEX = {DEFAULT_PADDING_LABEL: 0, DEFAULT_UNKNOWN_LABEL: 1, def save_pickle(obj, pickle_path, file_name): with open(os.path.join(pickle_path, file_name), "wb") as f: _pickle.dump(obj, f) - print("{} saved in {}.".format(file_name, pickle_path)) + print("{} saved in {}".format(file_name, pickle_path)) def load_pickle(pickle_path, file_name): with open(os.path.join(pickle_path, file_name), "rb") as f: obj = _pickle.load(f) - print("{} loaded from {}.".format(file_name, pickle_path)) + print("{} loaded from {}".format(file_name, pickle_path)) return obj diff --git a/fastNLP/core/tester.py b/fastNLP/core/tester.py index a3844991..c168822e 100644 --- a/fastNLP/core/tester.py +++ b/fastNLP/core/tester.py @@ -98,7 +98,7 @@ class BaseTester(object): print_output = "[test step {}] {}".format(step, eval_results) logger.info(print_output) - if step % self.print_every_step == 0: + if self.print_every_step > 0 and step % self.print_every_step == 0: print(print_output) step += 1 @@ -187,7 +187,7 @@ class SeqLabelTester(BaseTester): # make sure "results" is in the same device as "truth" results = results.to(truth) accuracy = torch.sum(results == truth.view((-1,))).to(torch.float) / results.shape[0] - return [loss.data, accuracy.data] + return [float(loss), float(accuracy)] def metrics(self): batch_loss = np.mean([x[0] for x in self.eval_history]) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index 52872d1d..a609caa3 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -4,7 +4,6 @@ import os import time from datetime import timedelta -import numpy as np import torch from fastNLP.core.action import Action @@ -47,7 +46,7 @@ class BaseTrainer(object): Otherwise, error will raise. """ default_args = {"epochs": 3, "batch_size": 8, "validate": True, "use_cuda": True, "pickle_path": "./save/", - "save_best_dev": True, "model_name": "default_model_name.pkl", + "save_best_dev": True, "model_name": "default_model_name.pkl", "print_every_step": 1, "loss": Loss(None), "optimizer": Optimizer("Adam", lr=0.001, weight_decay=0) } @@ -86,6 +85,7 @@ class BaseTrainer(object): self.save_best_dev = default_args["save_best_dev"] self.use_cuda = default_args["use_cuda"] self.model_name = default_args["model_name"] + self.print_every_step = default_args["print_every_step"] self._model = None self._loss_func = default_args["loss"].get() # return a pytorch loss function or None @@ -93,48 +93,35 @@ class BaseTrainer(object): self._optimizer_proto = default_args["optimizer"] def train(self, network, train_data, dev_data=None): - """General Training Steps + """General Training Procedure :param network: a model :param train_data: three-level list, the training set. :param dev_data: three-level list, the validation data (optional) - - The method is framework independent. - Work by calling the following methods: - - prepare_input - - mode - - define_optimizer - - data_forward - - get_loss - - grad_backward - - update - Subclasses must implement these methods with a specific framework. """ - # prepare model and data, transfer model to gpu if available + # transfer model to gpu if available if torch.cuda.is_available() and self.use_cuda: self._model = network.cuda() + # self._model is used to access model-specific loss else: self._model = network - # define tester over dev data + # define Tester over dev data if self.validate: default_valid_args = {"save_output": True, "validate_in_training": True, "save_dev_input": True, "save_loss": True, "batch_size": self.batch_size, "pickle_path": self.pickle_path, - "use_cuda": self.use_cuda} + "use_cuda": self.use_cuda, "print_every_step": 0} validator = self._create_validator(default_valid_args) logger.info("validator defined as {}".format(str(validator))) + # optimizer and loss self.define_optimizer() logger.info("optimizer defined as {}".format(str(self._optimizer))) self.define_loss() logger.info("loss function defined as {}".format(str(self._loss_func))) - # main training epochs - n_samples = len(train_data) - n_batches = n_samples // self.batch_size - n_print = 1 + # main training procedure start = time.time() logger.info("training epochs started") - for epoch in range(1, self.n_epochs + 1): logger.info("training epoch {}".format(epoch)) @@ -144,23 +131,30 @@ class BaseTrainer(object): data_iterator = iter(Batchifier(RandomSampler(train_data), self.batch_size, drop_last=False)) logger.info("prepared data iterator") - self._train_step(data_iterator, network, start=start, n_print=n_print, epoch=epoch) + # one forward and backward pass + self._train_step(data_iterator, network, start=start, n_print=self.print_every_step, epoch=epoch) + # validation if self.validate: logger.info("validation started") validator.test(network, dev_data) if self.save_best_dev and self.best_eval_result(validator): self.save_model(network, self.model_name) - print("saved better model selected by dev") - logger.info("saved better model selected by dev") + print("Saved better model selected by validation.") + logger.info("Saved better model selected by validation.") valid_results = validator.show_matrices() print("[epoch {}] {}".format(epoch, valid_results)) logger.info("[epoch {}] {}".format(epoch, valid_results)) def _train_step(self, data_iterator, network, **kwargs): - """Training process in one epoch.""" + """Training process in one epoch. + kwargs should contain: + - n_print: int, print training information every n steps. + - start: time.time(), the starting time of this step. + - epoch: int, + """ step = 0 for batch_x, batch_y in self.make_batch(data_iterator): @@ -287,10 +281,11 @@ class BaseTrainer(object): raise NotImplementedError def save_model(self, network, model_name): - """ + """Save this model with such a name. + This method may be called multiple times by Trainer to overwritten a better model. + :param network: the PyTorch model :param model_name: str - model_best_dev.pkl may be overwritten by a better model in future epochs. """ if model_name[-4:] != ".pkl": model_name += ".pkl" @@ -300,33 +295,9 @@ class BaseTrainer(object): raise NotImplementedError -class ToyTrainer(BaseTrainer): - """ - An example to show the definition of Trainer. - """ - - def __init__(self, training_args): - super(ToyTrainer, self).__init__(training_args) - - def load_train_data(self, data_path): - data_train = _pickle.load(open(data_path + "/data_train.pkl", "rb")) - data_dev = _pickle.load(open(data_path + "/data_train.pkl", "rb")) - return data_train, data_dev, 0, 1 - - def data_forward(self, network, x): - return network(x) - - def grad_backward(self, loss): - self._model.zero_grad() - loss.backward() - - def get_loss(self, pred, truth): - return np.mean(np.square(pred - truth)) - - class SeqLabelTrainer(BaseTrainer): """ - Trainer for Sequence Modeling + Trainer for Sequence Labeling """ @@ -384,7 +355,7 @@ class SeqLabelTrainer(BaseTrainer): class ClassificationTrainer(BaseTrainer): - """Trainer for classification.""" + """Trainer for text classification.""" def __init__(self, **train_args): super(ClassificationTrainer, self).__init__(**train_args) diff --git a/fastNLP/fastnlp.py b/fastNLP/fastnlp.py index 2590eaf5..fce796af 100644 --- a/fastNLP/fastnlp.py +++ b/fastNLP/fastnlp.py @@ -1,4 +1,4 @@ -# from fastNLP.core.predictor import SeqLabelInfer, ClassificationInfer +from fastNLP.core.predictor import SeqLabelInfer, ClassificationInfer from fastNLP.core.preprocess import load_pickle from fastNLP.loader.config_loader import ConfigLoader, ConfigSection from fastNLP.loader.model_loader import ModelLoader @@ -74,9 +74,11 @@ class FastNLP(object): self._download(model_name, FastNLP_MODEL_COLLECTION[model_name]["url"]) model_class = self._get_model_class(FastNLP_MODEL_COLLECTION[model_name]["class"]) + print("Restore model class {}".format(str(model_class))) model_args = ConfigSection() ConfigLoader.load_config(self.model_dir + config_file, {section_name: model_args}) + print("Restore model hyper-parameters {}".format(str(model_args.data))) # fetch dictionary size and number of labels from pickle files word2index = load_pickle(self.model_dir, "word2id.pkl") @@ -86,14 +88,16 @@ class FastNLP(object): # Construct the model model = model_class(model_args) + print("Model constructed.") # To do: framework independent ModelLoader.load_pytorch(model, self.model_dir + FastNLP_MODEL_COLLECTION[model_name]["pickle"]) + print("Model weights loaded.") self.model = model self.infer_type = FastNLP_MODEL_COLLECTION[model_name]["type"] - print("Model loaded. ") + print("Inference ready.") def run(self, raw_input): """ @@ -168,10 +172,15 @@ class FastNLP(object): :param language: str, one of ('zh', 'en'), Chinese or English. :return data: list of list of string, each string is a token. """ + assert language in ("zh", "en") data = [] - delimiter = " " if language is "en" else "" for sent in text: - tokens = sent.strip().split(delimiter) + if language == "en": + tokens = sent.strip().split() + elif language == "zh": + tokens = [char for char in sent] + else: + raise RuntimeError("Unknown language {}".format(language)) data.append(tokens) return data diff --git a/fastnlp-architecture.jpg b/fastnlp-architecture.jpg deleted file mode 100644 index 5e5c504e4db523c20f6cfffd4d4ec80003a26f3c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 35615 zcmeFZ1yohtx<9-S1w{}+Is^rzLAu!j(%lUr(gGq4!bT8KIs~Mj5s#!$(APBt!%RL~K+v0)9778lrt=p(~?%ci&KJ5YC z2X5os!F|Feg7!$+2>q!Y9{b1e)VoxoCC&IMLkH9xuity$!yq6eA||1EMoUM}z{$nU z!^_7n_EKCzQc7AzRZU$(Q%hUN_>GCFnYo3fy@R8Zvx}>n&nI6$|Ic3nA|k&=MaO)L zjY~_<$jr*l$;~S*gOyiQR#n%uw6?W(bar+343CVCjZaKYO)oAjudJ@EZ)|QI9vz>Y zo}FJ@Uj32_1wj3aSbvx7ALPOT<+^qIHtKEkUvi<`at04noZEMvu%Y3KD5D$MJ$lOi z@h+Zdcxp-WJt_{B1N_(ThcF1JITvXTe~I>+WdAwAy#Gs*{avu%<(dXCQBlCjL&X6g zzzGIjn%Dmx|AWSW{vO6XTHw8O!J?hBXXnlv5t|2s<9g9o*fKforL0j$v4S- z+c~plGq*x_Hj%&vmED#4`_9WocD{9ZasVe?0TTGzb>*8x$jxmVhzy@GeBqn@z$FCF zTv;0lOpPFcL5by)^lN_T)!kXoufNJaUr#>rq#H~tLjq;lNFXJ+b5run0STPoRNc_6 zh8}ww4M0}l%BkXJNWe@H35bw}4*$Ie%Doh8@YW<~_eHvBzyP8@8R5-CkxB9QhN2;T zm;QoCAo5Vc7nIqr?aqjd{(a<>HDT_b#4xmf9^*{W;AZg{9Y?FCs z8g4vL!sW!NHKdG&F9Dq3OKM(MB%dB7kHZ)x4_IS^pkHV=uYB>8wd3#LDZlvrP{%UX zs2FDXC zAxZ+h-i7Cj!jgtOB{)yM#w2fmLt4=H&*MrTDNBaKS;pTrzQd|^QgshGGnJEvv1J;qiBIPJ_L$3h|G(?B~@6G=ngAn?K zW%)Au0Fm*T|Eto9CorBAlRsrS7|WD}4O<)xVy6(rmI3Pjx^Vusfd1Wo(L9UKts&d1 zkV12fKb%HfP#v@RAJQDbe_NRWI(^F)SK@QibY_0fOwhCJI;{FqM^9bfE{{?+jAb=g zaI6+kH)-;AqLbGP)o+j56fV_PM9(|U;pHj&=Wz98d+#lO)-GyvsCnnnaz~C^h{x>8 zc-pthfnHxR&k{jj4hy(iR}69IjMf_aJk?B%GI`b~nYncp5|;F&B5qH%qeQRe7q^Q+aBjP?05`bYBG+{nq z7cf4;Lm5BsE#iDDeXVV6xU2#*tLcWt z=$TT&q^1?+9L|f1dYdc4He)I#RJ-lD_Bj+>-ui?|8o;l|KQ!E_tttuWUp9!AGC~3> zeiTtqtl4Wf6?e?c;%!0>t3wMbdv{waOc~uB*;w)bpUHKy06%73t-#i}I#pRW0<2(b zM}|^q@1L)dvl|N?rnCJ>wh-iEilfve^)GQ?0r5Uw0zpV1LCv^DZklU1;VNNdb{?Ny zPpTRfE8)(-Y4)tF!(*J%F-$f~x9Dt|&S^^0S9~oG?h4r@xI1s_b7_nedbVck>ew!< zye!%4;CW@to6k9abq`dQQl=hA0RD*+37pSdYWSKhAb~D1(6)IrMwx3wTo~UtvSl9* zO#E`B%DDf@lUlP)M*^RG+JduXq(@FpwP|0tL`$rSoT0{T<|iNQ^UpUZ8p3jrK&LsX zD~|H5r@rb$046{``bRze$Maa?4%f{8z;wy^%t!y(k6VMwo4%Er*da8TE5dAcdDICP zX?$S6dr}mUW7R~Cs*H~CFab@xJfQ2XmUMS5Xq8&2$3uh|pT_jUA#!n!Gx_o~6tv*@ znSZbtf9%P><6H$7piV$}L98>I_ol1@Q>?CMiu}dq4OC{V$n%8_b-+#IYUVJtXBP*& z&w;C4y_cw1HSSYFHlK6lENInVJNV*h5-&+R62nc5vZl5k=|~DvaJM#HPw}2MZNDH- z#9w#wh%Rw1wB+YGJDqN}I=-y^@*4Wdj1M(Q!meEm`-`L02Ry@Frt@dVJG~t5dup8> z_uNf`n@;j%7Fl19ePDFY8n`L8bH{J3g$4WesjSP|5;*$d@_F|iZPxGjfo>fo=|c=YvUdvi&rCP`%z=3$1H zsLlv6rW~r@%#~MokDsfwnS_6636Ve$78l;n?ma}GSid00&nmHY&eOLzyeGJ==lZFL zFj;1>7!s)*DKqgw)%78o)Utf+#yv{Zt8qR4=%VPeiL;ov`o3l1&8HF1p5yHHvt2mv zilVt?;&tiuV!p?x(-y-Ql8}_~0KlOG{=BUJwBjvT{U8&nBv7&&=!>)fMvhl$0d&A0 zk0NMm;}OXO64>5^o{(%Xg6kUYSB?a(t_*L`y!bI>bm&m~1PvE2PRt9}z0Y>1mplr{ zJJ4L0Z`Ghz_%xH~kHMMk+}GIYr87{;lKdZ^BtK-@vt`=R7@xQ3nKk)ju8T^pz%UVF zon3n{TB0jPJxnNy%f+N9>>Zq*efs?zIuWY@A&D-HJA?(oysRJMV()+Hx9b3iwj))h zF7keE6I?lEXLOs;^sI5L`JZk*XB$pERO8R^BrFY-mLAP&kD|QIt5ns2&f8}+HOY6V z5aVc(VX35>kIk+dWojy%U4$;<@m5hbj>M+YHp61N{e*j@cXj@)WAwZC(o~eYKi%z5 z^X*)w#avBe_yy0-DnH%La$Z?S5iPXngpf zeIcz4)W7qpJWdraaP4+bML%H?zly%Fn1`ov+Gxu6i8^tIumS3dA^&&BSivJ+-0;TO zoSDv2nmV4)G=&Nk?pC(Q#&^8y$K+UI_Sy3l=}XbN(mewJ4i0gfe+c2kLFQNiEA@r5 zu@!m7?z>E$mn9L)#mYg$^)a);jjkmHF4;ReK_ni&feUjzsb`G47}!(^k7tKBgy|4d zV=PQMEwb8_@_qG3BDZg`g{o@BSb1b#q>W#7+})>k7Y~s&p9pRj41O znh+Fmr%7kt5C4cM=Bhy>8p6NaIGhZ}fi`gx$|@4N_^8%ITclN#Kb?KVO9jG_8 zXdn;RZ|C>>dOqPzYBF0;#Pc}O<-BO3W!hr+&%4-?Ba zH7hj=Q?*TVhPz~JscPvPYAbl*SV}p(Vw*`?D5MFA;u|U# z2(-oi#9^)Gy(MWDCH5xb-tEWB*__3WPQn#j1$El4oE2&7FXQNJI*g+EUV0@>H7yr= zMn$D=>l9q=<&*bi_O^9m6DL0%v>y@lG>DRv|6WJZ=d&^Gl$#Cjv1yyq`hK_n-WFR@ zdA4PZU*b}szOGz#1nH6Jj{upRb#zMOdRoM5!xfgOrTZR)&4UK5R)Qt<-9Z^;en!oF zb1<`PIrVKrqMobr!o;;mp$uoUb9R9@6>e{guMXqZO@~iW4-nInTC=h3o5q9@GhVIs zL1y^%OTCQw!OpEmBG#NG?c8tPh&H3GY=SuYu}!$}US<^=5n@Hmxg%K^hV% zh&ol!_%^pSVU3R@p^Mvz+CQIDla|Jxn*ooj$nEv|<5>#R6AcCwVLnQJ(!`%O`FDeU z9xZrn*bR=!KO@Z_xHrA&bSR^7c5zwyGy<2ck~(P?2{6d2pJQ!woyxiv!bxqR^d{A# zta5uI&CME}iwlQEt(`30i=@#K4E7Y1%NM>J9W_i>As_KbKDZ}MjYU7{P`mXU0vZ19 zsIRIwaq@LIMx;7G8{;y9{kv*q5<5T^vgV&xip^~)zGODCbK=20-d@Z~3kz;y|J=cS zl>{fzm(=l4qj|ii-Jgk9hf9S7#y)K7Hni42onCWZX+o-ks$4wFa9%@^VsN-j>J1# z0uvL)MqZ+y35U%^=dF~*ekX&_^*KlsdDCbf$v)i6k>ct!2DJKIo;g;HM<}Dv?Qz4K z+rubHWJn;~ULFY)YyFD$b=2b*2QCRY7q0%V`v^*78O9hoZBX}jGw9#BX#)^tdzLFhw^*|_M&iI|KAM zMgj)K`DjW#@9%a!{bF_dMwCP?;)LM0D7It|#tL(Z8_~X2SSaf-KB_Qta<)%6Jg*nf zXvaedH%h$;)Enhuh1p)eLhYRj{&qe9i87tJV3>0tKB*1NiusW|Agv*kNeVc04=9e; z#L?YP$gK2Pv)a}T*42o4`HolKTBd(nV2~0?QNmws> z;DU%P$_fBh3+o8+`0uQJoe0{tqYQt8^D~vGlXyM5Tmy6~AFmmfk`~%%H*~f3XaugR zlY7lgeC%^lQ$hm{>R59uD+wRPz8+Pgdim%M+br3U%SqnD3;taVM(y(Ln31Ko1sJso z0$5W>KzM9IS)z#&0MJEb#qj`TaXge{C=z&r1-UsxsHbXANcN5GEE~ufW!M@8VGI@0 z!k|T2oCx|x$Pnlt|0ApbI)?TLawHHo-iQPYWkHx|LPQS<_(NB?ib3&)K|G2RWe{xS zS3`e1P{Q*s>=nnKN$+XAOhg-trk$QV$#w8}&?z1wUKZ>lV^1A26Wfpl%K`o0SNPaw>9nib)!m#feW*v6>k?D3Z*2d@ckN66yYyUgj!S1el))hb zXn{+`y2`=L%(F;4tc1qekPTtit}V9CY=b)WYg=3_4#}Y%>q44_#-u6Y*J44KjPj`Uu39~3lgB#82Hp}i)pbEA^jNyUCP&e9O7$p zZ7}*C$=N?`f1+-`@U(*IZQ;qa)UNeYoK~WioczCFF~5b<^5Fzh(|maB&!2n^xhs@A zs)Wvk1n#l~Hty-U>Q=f6+f5~}Urk7SF6qc6@f8OcK6;^N@&^D6eziz|l3Wr||C`%y z_^0=s6Bu{^?W)cK=cfRK`wAKQ4%Xp;ake*Zo#6Zh4F3la$G-W7}zWvA`C>sjT9a&z){g9w(hF5t3VNS8c=L5^G25jP}*X(8rLaIxs@V^4|$ zIV7-a4qf*hxVhqlT8SY6K^Ex#P$350n!jxu-+%XNq#TK}NUfRHn4kgb(`hjC^f*ZG zMPNkwyrDOP)-5K#lX!n+d7#q?;qEF!H_Wrd#N<(#GRt1tqTyu5N`IujDBQ~5qjvP?* z#F}Jm@z3p*=PFvB$-3GMijk#bDxlrl6W)Mt*QSMT?-O*!-w)Pvk&#EeesJ6`m;3d} zOr_IKsOn5ey+vgW`KWyQLSMGN-b`VdIOp?K6A2*1CkY8;@fORbg;Ziw4ZJ;x@!-*} zt{h@v=9z)L`=URs{rd6UwyYQfnU1YKMZdUMe_-mB#QE%)ju6*&|Vys_r)*lai zF?~==J^%p%gHli_J}Vw~{!B*ee@bBd@yr5p^616AKO0$X1cgcx#YmLj-HK2u#4SeA zKL`B(E+VH8YJa9H`StzHL${Jm4J&TNlRj;#ZTy`1`8_vJ^>@;>_2E(4T%^qNzKOy! zI~8AuSFfs1jmHay#AU5(OPBHSB0y9mT;#us3hM2ZVfDx4vs9biS0ts4a>ejiN>Bed z8@j5GkB1_uICgR)^ModGRs|LQ4fT>xX%r5sdeHzeTV>0oQPkKO;haq<4a`3(=7})_-U*{m(`JJN30#;IJVk&HPLnEBLc+x2JXUTpbKOLWFna@N&Gt zcX9uL<&~bMnEXB3*=Pv2TzW|Qz7o3bZzIR^YiP6#4}34&5g%I-2+%*kmxMp@d<@!{ zcSxWSbPw8<(Dr_f-NtWYV+3s>T-?7x$G*M-F*tRKKb3qzTKjA0(tn#pM_1pZYG|U^tOtH+S!D+OZo446N?Ds55 z2TtlmN!fnbAH`$L6O=+op!m{+!l}T z+n-a3`5A4=OBcY$hs5C=a-SDKYkHb@0boA3W1h@pQ)!CjE?re0v;5xwj5*ZhTe8Y7 zf)o0B5<2h*1YRpI#(nYJwa<;u@EW5$I-qCF{_sP{`3y9eVzPqbxD7Pg)(Eat@yFvH zlkOOXJq^o{Yl2pUOY-$@V@K$YF81XTbiV=#fc8ApN_a=emvz<9F8L-9M5YB5Z_XB$ zC4aBcTv@U_+39gh%u#XG0HXan5=f#&T-zR||6U`EE`55S7TZYnQgascEq?Fi&>yPQ z!BgHfoAdvk3D4i-KmsLiA?sg2YHJ2={)b9I=i#huuXR!dp?*$`1b!EW(_co)Z<{ak zW2YP&gJ;Tk@GtSPjTAFvoOo;*1qm^5msso2B_5|0X?RQ?om$ZodM^&ucd$cr1~WNf zRymtI1%@xu0vNDF{|lVw1LIC^jqSxI;bJiSied~yH3s`JJ1z5+0H?Nd?#`!6gHaVE)=rKX(2l!VG`iQQQmq&apIn*QD0;}x=NpP; z^}ELcI4XK2HM34pq(lZ=%GWb~^B(?2m+Dt9!2B35dh@Tt}7Eo}t7QkkBYljMZA-9)It zX}X?YPA7suBchu3T7b+fBtpZ=-XLiE3uH1hlzn2*J_VL2hx3IzR zvk>Ib_;H8%wt;Q&^taP(@>}zyu{%^jUChQY7GV)Tq-o)&?KA}>zJAF?)6%!2&(Nll zcuivU)8B?FDk<8^@athqr&I9`a5b4v%V|6&vUb{5stGYGH*bA{wVKL{BB_UazZ z7J2J*lD}z8@da*_+nlFJC1!Tiicv9$4yv|f9nY7EbHpHbQ_LQ+WDe(8V&F4kzFN;BXya-w>5IG&*?Dp_u=RZ^EzY>5H~W2%Rv}b7=u}-5Uu~yi0@y_MFJ* z*xenQa_&c#4~zrzTI!(5>pC5Y6Pz%KM}=nHcbqTXBIFBV-yFd&Z* zeRUjvot6h(O@OXb^p_j?F}k#f1_TJ8sHmO=KyFp8DSo#f&u@$vf;Q8Mq)#^Se&RcY z#)DLirZO(uC#Cd3Ox2u%ReD(`Ym#^VEd`)cE^VaTjj4($OTvo;EJSkY(#{@9@0kvy zPRaJ9znss3W@=t@=Nt9awXCthu_xdNu5xk9d|9)+7zeZCHxnqsqI}&P zMx){Y+)5^;w$%HOyN`&r>_PfrK`ASzYM6XqayFPQ1BnvW8dw}U1BcpmO9k;Pzvf2s zlJu(@cO{#C3yp6jJ-o9dH~(3jlbMXA%mz4kV*kV|{)f(fMhePjbtVOX_*vP>cQe=G z1bem?v(g7&E$v+g+pST(CunU}`C87oJUO+N)kJ)iy@c`={e*3g*eu)7&$bQBaMvobeRVF58HnAX#lF+NZ0k6b(oT{$xLD3~6&8kHQZhG|@6A zsCc*_jv0v$vH(2Ksc|3JbvD+k+M;h3r)EVA>k+A_GBh04@vMd*q6UXvknJF_MzYE( ze6wnBXUux5aj|)BM|#_ZWQXwPGQnxGb+qPn8T)r#GEEa_iSJ%5Q)Jq#81oUOd2#rC zF?V>9A$(qev(rlvKS~yISzvnbAd&hWX61Cy7mBOzBex#9p;Us)y-;6?@`UpZhG0|M zSRJ!*p1!2Ru%7PR)7|3~kAhaB^#I3Y-;)`kryks#_V`UV+hrei+{v!hk|@R}^!(Od zdbLn~Yxkb&7Z@)m2i8cX`a@9xKUNM$`@SvR=5d+w%oRqV^;D6n5~S;<{VZS7p? zQQN8DB9zzeS5a9rcNRxUXAS)DEvprt%hv>TAt%Zg3SWmw7$59g-KgJd%m_5nN$&^t zK~5*xTaQUfM;!w7xh-h5tetk=ydqkB$I-#>uF;1Fc)dc|1W~*{BY1z(8W{~lAZ9IR z0k=QBx%D4r?c`e4WEMr5+t|$d_K&F(o7prySSy;~vnCq*VB{>=>cFp4@Q!}7-cP$B zJriJopb^2!O7TE6C z-Ofgca7>?^nQjVEbQxeSEv2-IFq-?E;rX-RH>g`2s)PTO+oR?vB=R;5BmKjE3+-t> zq2@g+n`7xANk!F7?P>Y4S()kJ!5BN$QyIA9@$v zFI|y9Y_<5)upVLs5#rk_F@@5QZB>GT`MH; zAr6c@ELIwRp2=P&&ztRxLD$QRZ;s5T@{quQmErZuulPoh<{Z6Lf8qb{+y3zKa^ZHJ zPYn@#1f&-)#V@q+R!L{3VGNAlaM#fNM52sBv+FS|;8Ns#%*M1(Aq98ofHQ3l0vzMV zT2X#ZK`UMYahsM&k&M!>^J!Th5lvn#6^&`ce?5I^WRat);asIG^#fm)4R|0el+Tr& ztYS1xPP^0wsphwD?i~cIby>Dh=$0k zabH|8MeHsV*O2uWuK8n>(eF?w`z$o%WScflYAJdFd$r;o1s~O62V-@?)QPNB@cRW# z=dXIYsp3i!Eqq&S$xA{TTf#w0oQ+tzb1s4E!v;$~wPrqSlXM4_bllsq>r?Spt*<;F z4iJb@n&`l4I?65MMgQLCMw1n2ZI(B0$yS$T^&K@I2WC7HkZZdr#EtdTgRO3=a4oX$ znFMLncQg^eIP<>kb`@EFZl>6x?C#Tb>6)lFaUx`&2g2!=X&)yb(t;V~0^6?kO6+oH z<)|wn-#nqP_-yiBa)hG41`E<0=NP=hBhNK9(gb`~Sd3yD0Z@jckw9~e<6b(=M)ca; z8WPa?E`|SQK<{v&{uXci+fZODna%cO?6Tg?UU#1Lt4Cz3FGaK?6?#s~v2K`)q`Z$* z^$1F0;U0zSZ0q~*pjRJfP>U=R3~r4)Z0WOi+rN({VS&;_w%frLKzIonRStr!Gi!*} zp?+V<3`569E$9O&VmODk(&qVg_OvB8x8}Qff@7@Z*`g_q9Xo`5!bEE4;x$O7)Iix) zJ!_u@cg=*p#YB|H6Mp2ZS`{6cm%VLUZ@m&m)O_{&6XqtRnG2N#9%IxUFL1P_rFe<( zVE@cs{foA*8VP_@h%pHJ6_@;-yMnB+{o<}BK|I_XHRT_b(7*EkpJD%w+-}O&_I3_ZAEsbFx2E|^D6i;fa+!2Sx2<7180=FDDAu$*7l&J=~zaZ^xt5rz5tE{Hx;P#1=E$w_Mj^qCWvJ)4Pg*uF+Cs}CW=cj#T(x7R2j;!2=+8On%EEkwK`h^ zI~aEp&gEcjs#TH8zG`33)^n|)C}xt+>9DKQsdg;f`o8S*XKdJG5X;%hFJE_)-O|zY zNv$fYD)}@g-B$WZj3nqV@;t=DzQeLVa7o~J$pyAHMm1)=Di%wC^LQLX0S#S9=xR(? z_tEgWvr?b4_3e^&t~TXv`Zn`D6#E@)o#StUss(y=#m`FaMD_Igmc`97;4H96l3yiU zj}1U*+MJ>ZyQga;M&KUHF>$E zZ7Kq1-aZi$zi+rb`q%rej%01ZjC^G2m zN|Sc6_>zufVW{!siOL5>aOBC0Z$5Skx+II&BpUfE`)1htqBBNr_E}9t zZ9mf5);Oz(d*;i@!Vy0V=OfN2D!gr;6r(7?D1L!i)Ji`%1+lU6k zEA{;NZ-uYfS>*;pMf)qt1FAJr9Nz73v|T@PvF!ABC;wRsuWD+%|o$x`x=gy%l^m+%*D* zLmAnk7FpgDC9Q!QG0y5=IF4|fB${6vq=JM(D@Z7ehnm70P9WA45O8INgL@@zij_b` z1EP>HN7f751e87a`S@*Vm|)r<%%w&K(eR=P)Jg2WrK`wP9NKl|qx6LJ=xulip1kU&pu3yGU*uQ4|hlBG9no9y-9q>ZSns+742YH{%Z}gI>BGzj>geR#Z4yKR za%z2rTf$UBjw4t#g$d>w1maH?62e^9SGUj$dC~@T_iVERVZ%xX|=mSM=$%r<(QsWB-94iY3H1jG`?+ zWS8Z+nMI#ahg%FqYxA}G?(ZusOmt}uuj?8YTe$faOmJ6z9(uvS_?+>(TbNzDChXAE z-`$~N&@eZuACH~gPuMFwVC8ARI4Q`%G&6LQrxyVc{(hPp-4z z_PKDLNgjtb?8OR2+p)BjQ9X;J)v_$~LVCV^^^=zGbm^sSRd*Qo4gVS3N)dsZNZ5ZQ zeiZv*f!4I6e`h(1-~FV>(<+dW^$UCXR0!|^dv;-)P?O-WkI7lw zm?gF1y@G^067Z$@F4UhlqdQVmK-=KGA_OBorZC2;A{I0j!lJT@duIw6mx$>^4mfodGwiH8Xi>9=2n_|NioAH0lKlF3Jl^9CwX{9_)vOK9+V*SiBIF%BdQTmnvtNOLi)l&S(A^dh@*T$Pe zCm-^*FyWMs%l$iyIGw2-*o!*R&hHGTGy4jX-mt$fKp|cy-|pQxC4aooT5#@_ZG&-1AV3S{nW^k(vKj%+O_766vVX`P^>`u+o9MPy2gOVC4(>w z+KfB8;b=%?p{#2lZUH5(3*k9JsHjzu0d_u@qcsZkBo_t~s$+X99Fzjzf^7H%0se#s zV;!3lHB6qjYb&a7?jPx{nq2lm?@=fo=CBSSf$wk*#?s91EW~p>X^EFB&OaUYedM8G z&}N|r1BN5zvhIE(MbX+%McZYfs6whJ?bPAqx%!qhxcUaXu5v*czNjBhiVWog=<5`z z54VQ&6)N`L5RA_}EE?vc3R#{gw}A_QYTp2IwS@%wL$?pnZ#+QM4NnF5mpnsh;7tOG z_B`32((+U-xgj<_y~jF&_MKv<q~Y@ro_a#g`QihMEn_BGO%ZGP2VGHKXGC3vw1mP8j5oLBK%$aQiI6 zqAFS#Xi1`fHFBGKL}a~d-uCcu`lck4wL1cJJ@>~1OH$@lW`XqTrt@uQ{l?hN)BN&y z(AVx_Pz0l4etO0}H(zH<6;HWjnb6)5=27Pt3(1i2!chXPYy1X&S1Wj)TWoE2l)TP8 zS4xoKlKq90;#L`AyLig(l4_MURz_pi2r^$qynuKm| zkbp__7-mn`jM8Gr@FWrC@^H-@kd!3z~uuyOrpl*`lsPlBIrn&3#Eq zDmX&j&3)V5L5vFCk2XqH0%yqnYHh?u1lINu<;K0(Yw!n2ZppoW*<{&6 zaeNKxRA}SXDD)3S_>ABSUiPC1!}dD^aKE>Wvrv#s2xAtnC^5f1B3o^EW3|^wo^KEk zJv`U=LrvF4Q)A*pM9nduj|=PZUjH-o2=g9#8?WoJEk+Le!1@D^lPk0+%Sc_)c!JmJ zN5&r#t~>PPKjx->Ze#y3$hc8w^X)04?50`B zxMTS4%$pR{qa>FbEZvj`PnTPTpZFWrkHYe|WuY*(>$XT(!cXr{PT7_Qq=y8B>4Ey9 zh+&KIpHhVpd4v|X-}xlOZKaG_v#^?#`Mo-SX#C>^cCW_9<}3H0FETF!-159EPX3Y; zMXaL_!@bE|@#C?3_meGa z{~XBw|FKnh9S+8iPlBO{hE9Z!KA43Gs)w)+kdiet26K9U?W_Dt(jG>Po4)V1e{E@H zVpEd^6(*EJwG~t2kV#1*ydxdsa|+7It&B7;`e%_@nFG@*En{Z=`_EsdmTf0E<|2A+ z&x*@kaXiv(7IH1e~&vSQ)P<*EM<>w(o2{m#+l+}xZL z4EYR;VgZFSfhll#R`qg}azv5iA_HNpQQJY--S@V2no-I6d)AEZ|;j zIRE<5{+qbq4B+(UFxpW^CGRs)x2v($}jN!lHU~GBM3tU+FnXPoDutEo_K{O zZzr(qRwP)6onpO#3nbuPsFmi$J0-|m_$?bo(x#%@7dG5Suvw8b1$vkh%2`vA*%k-d zZL-vKc#lS}$MWcI5$!UkB}p$=et8zN-<5+389mRE%$^drIXj%z&A@$sSYV0YkWR#9 z8UPfXafezm8;lcFlHCn5eg)@OwlKDMT~OgPIFnBnj(cZd0{?XYkZ~muPrWO!dGUy$KbR_=>DUPlP z8JkaHc#85RM)Y5F(EnNP=WP?ryRYsFiKJj+V(9$tedoE=dD_IhYY418TE9~7v7&Qi zXr&CoQ77yfd1kj382Y{_oHjm7w=!)PUWJHyUvl#7d3j%8avUwS*)NEqzk(qzUp{tm zwNKt-!Rezoo`z|&I+yOk&My1faqqF$gGji?M=l46X*Z3)9s*RT}X~&s&SUF=0#y3m}ox6=D!$UYu zmrBbM`HvL+uk;HvRAHM>W4^qmiSg@~;FvGx>XrTSK{KTfzpfZhx3VUcAz5`gKc_}d zHknvv!dcluM{eR!XT2}V(46Sx=X$-Y-Lfvx30*neYgYqJwgO_5%8-=g-edgGZ23GF z4QrVpSYUj6oESB3lkeBa5ni~^R-lWSdg;MU@bnBZOG*t7mQ6Y}{mi7}wa>?GY3p8|lAn>vAe zp?ERmtHX8s#2#mzbb(lbg$~zhIzOYqoj0^mR5@I86G?g-mCELBI<1-A_sb9Bq>SgD zh(F>b>>$WbB2 zu)E~Qsi_)*-VO1YG2)OQzd&*+P>4}BiMPFt?`G*1O0P*Hz*ty7+K0Qg+Bgab>Xv)A+*QZ6LQ&!v4&;fvIM+*ui0x89t*|tH&vJpxnVXkSS}jw7qih_R6;< zqtb3*h0z#DK#f2jJt^$t5U(yfe4L%rbx35>bT?07G1P#Oy|Z})8&iK%v#gCe8p_Z2 zl*T&&;0P1KUYAtdZoEYEl`T|ki@VM37Lw_VH3!=v(0XZ!VU~0kHNkAUfIGLy%JWHm z*)YpeS!L;^v`Z_YO~T6KpC`NWP|WR`lrq?OabaA?r1wic)@Quyl=dMU@U8Yv|MEa` zr*)-4o5FCG&V8x`GSuxRzIEEeEEa#B>@A)vo0z5hRV!M>cc#%9QoU&6olyE55Y*$v zJIp7V72lpzJL0aQ%sy|Z=TC}~SZH=$mKw8WVfpD=v?5Iv_GTrQc5OLmy~5(zLF4D* z7)8Q~@wXLnsSj;dCHQD@Hf2gv=aB1ad*6}^Cv)m^)Kb`tYm(^;DnSa>u9KqVR>BzP zX_;PkF4&PDOtJV+&ehSOJF;02wQ~I^in(}H`;tTGH=2)tq*k50Tu1?hJ9Yp&) z`l=rg%ZVcPpDU;`|3sD*|y^4l@9`eAv|XRs!=8ALbLSQeaPjh58@`^ z_X3i8aB!>8&3?=h>)&doyLQZm--v+j;2qjYkPcZN$G$$JyS|eQziI|K%XW&|V>s$L zn9S2?0|O^32eSV&eRn+h_!RvHj+gujOgl=}g6vPG?+`$bx6yB|@jUkj4zTAhJ)hpx zj37KyKzGzTv{DB;FiLTLpyJ^@fbe()UV9z-9Ski73MkHZl&>2hO_2ZVQsh#_(L={% zLJ`}j%D;CjJzfSD`8BlL?%aU&Yk<%Iq)9WchPAC$T)Ctw%G6yT4YP}f5llDq@GN)B zQeJi9{4_cSZd<{p`R4@q>3c_Y;>Eg^4@{+BoN5@94}vy6TR(G0rz0Y7BCM$U3GFSH ztKET|-4~=V4gAVLhgf0R+_;`a7Z)3~bb5rkF#=cr5yCsV@%%sutq?y73EX1=(*s3h z4X-c2l$TF2FF}AXvlyHPlzed$Xp3wbbWmjiTmz6rnjy#rJ&1Qf?*NuV+J*-a8_6el zAlnE|#f7wk;uVGt7#TFbAc1LU!l`R$Z|*Qd!D;>9_Wm>DId1mlnUo-3PHh5qSxsmz z5k)ROUIWA6l4xIU(~BK{JGieIj{Q>YH|Lpd>vYj2Q5qumXmZ}2#(_(~dc%&Bg#44j z^G#cl?2=QUdJKyIRKlV+hx4Jaoi0Wc{R&zM03ll&w&?{e9<>J2bhgJavXs zqtA}@YR0E7a&rc5r4SLF_5OY2+nRT3OKD&*k<0WO!BTh0Wkm>a*q6HwZ#P$WpHX(U zG43BnfBzgoK#rvzs(Qpmkdj4=PI~wi`p!z1uU*%{=tpkf@!P97HFd>Z0oKc_(Bt%L zgszeXTcN)_7l^n@kI?s-ZoQ0|4|Moz;a?a3 zUOHzb=GEDEgvoi@9KP!BR}mv%V3M5Exc{qmp#n&($3~^N+V5BY-cE?L6ZU_q@hG(| zrx#|sYzRl0GlXoug+{k-4~FcIR#c)5p{;Zo`Q6uM1~{4u2;X4RwTftw^Ea_mp0R}( zeZ#DU*_BeSZ)Vb#dufwf$6+~naq%@6Sp1g#tM}t+rRB%I+=Cp_Uzd7#+-&IzjMm(F zXB^AxG6RAkr+H&)btSsqG{Hfd_s*#)x~>B7zD8x9`}p{*`S(F%lGk+8a;6PAWY4V1 zar8pFJ9F~K*aHL0ekrb@aoTEkuVk43ojhepb@Sn4l$ z*3?Bg0qoi#0EJGmRqmvLlrgNnBC)AkYgL2y3ROFvU?Z<|OXIm-plh-9a>7Y)rc&oY zAH%{_eVJb2EY>ZL&6i#|S4m${OarnB^0VcX#G#gJV&%7fWZRvP9gnIY0ZY(q9p6}= z!y8ZYL2kBoqPQH;P0Q2Uaw3SlTV!jU8g_&$t**EtFE8|iU@*4mSqPoKrV~bEY|1_J zys#~&M=&d_Q&=#I)14^kw~n2HCs}>@n-#}IW~0onzrIsmQ<7Ya18T*MtQwwG4izyi zKBd=LY^mJ$zO;mXK5&88sFiM&B|7y_PwI4=2%5wh@;OKPUFRo+H?5$~`{t11sY{>A z3s2kKX+MRjB>!|(*FAcc8ND}+7HnD-z#X}V{_ehqkXrx{7m#n2GdW*Nlet>AQ};dK zMVqI(r5DG(r@l&ar;5H}RlB){#~;5OVgdQfcNFk$pfACUL({i^+za%#b8#*hA#m?! zh!Vf;i-rCaM9)a$ddma?UVNe6Lw76rOd}C=2nZU_SD`JTJB0#>51=~l0YP&;FkS~y zA&Z6MMbH>{YQpLM`xoVfP)J)_TfTkEl%ienB|Cz5lxTStSaIm4g2zoC3{P3=*swTU zo77j>K3kMy3|nqgxUu-~?z13cjueh_)C*m*@Oqa%o3_vC@6)^|nZTUfc`wjj|IZo& zM)B+^(Zw`ar35)f2k1dDl>K>6?BDmts>_RZ2yAdeF!>}-xNHI}s%q0r@VR=KgWQ=# zWi7;`(~qhasfCC1r5hyvUHhI(~V&s@BFeG`v5P zfkaX3e0LjRx{7Zwel~w-fBR#oH~(vy3^`?$;5 zycgJ}=`U9x?FbJ>)4NoazOTPHrG?yKQkJ~^nm6nz>kImmY=P_im;7qghMb8jNFX1C zbN}$MkqpY`t$<`|W}4TNYpzE4K`7z^&m(RD;UfOaQH~4vFin?i!Z6}n=~m;cr}Ur zE!kiBQa#@_dg*qQ@ODtZuyr>*u}VNT?VYX0;-@lnX{9`rwXlFa=7<^Pj@5w9_{1<&XRv@V zhlTkK>{DZ`_1P!7zVok0z^T$Qk5Pi#qbrk^zPsGIkt+J)qn*`*KwSNHrWR?x(GdAq z%E#I0p_c2$l@Svi(=Xh086{q^;io_PI501qReK{VS2kwdFyH~iG5G-zcZE|z#Ffhr zvc8xJa?DJ?0Z$Acw?u-Wr^NI3RRO=t%rP#UU(;jI3__b6R;9(q>Gye#6J4*uwwPPmF((jEcBka zlUqoBp21vd`0iflSDuIXK97?4&+^^O{A@GEclKG{GQl>O*=HyP;v~)?%lhhmv1ZxC z!3o0JiSTaW*Umh}&+-v>L)k`lvq#g%GOGg>T>~k&b*lrUan{2)CXoP>+IgO)%)xF$ zy2G>YkE)!%e9=$!+U^voa(@stXXv6SKFSiew$`nqu^Ou%<6I4dcx*L(;k2qvVpmCN|Jw<`gyt;k{u_j*p!`r-EZUXvGLWo%f})`MU9|MIK$8T zK)iTNRXsAnnh?r%z_G#6xhM|9Kjpnqn4h>NZ8?7=`B*=S#%a$h7byv>7Z1FAV?E2fC55j(u?%2LQoVD=|zw(ML@drjsj9O^w4{cbOIqjxPyCdH+nqx zlzaBM_dd`0g9nCu^OZ?5vu3T|dKWFxD{FfTTgt*-OM{hBYd-(3Y5#JJm??V4=}=L_ z=oK-OBwZJTYH8Qt@fj!X6>$ca2Il)Ok*MU`J*YmImtGlXTSZF{NvxB#vIl}s8#Dc6 zO`-oyNo?JX47)n*P(BPZy=vN;w&+OjD=zh&!&3vlu%BG+p?6|1)tz8kx~ZfAH`680 z(97|bqR1g{NE%-N6973!*;^GeL6x~6a~P5JS?#p*QzQ2M!V3>n`$UOviSI)j<}L0* zCeEY)i*y)xspKE>;Y%Yru+wPV&yfef+3*+p!XN%e%?RA{K{7zu6^Enko^z|ish##m zOek03*zjHjrE#|Ip^&+s3j6;Xpa0(RH!I9c<4Kw`sgA%~Jfd7>&xE8*;wedArS~ke z>M!sOx^H0dVR_Q#Z_6|_voFSsjz4(6QpK~<)AT~8z@X45n214&8xf}MkLy-m(YoeV zMA8}jo}z}lHUPikd3&Pm9;?WTQ{4I;A4I7eYK$tIF%5rcNs8q0;q~MU&r!72q|qyP zU+9MrJQ?=E05Tvoxf!-u1)0v=jtkf()QC|siny>L{I-2g&1PpS0`FsoxSm?cm`BXH zD%AB1tLmNqg1Uriu6`#0zS&W5uB$50Uj2bIPfSz0MQ+}`{Kp!@J$eYgi;Z0%DA?V= zbr$-e0)ZTdAEA{gGmuuENu-z)OEOrJUF_8PXoqL5RDfa6(Q&bNBT3RKVT)C4dr2Ru z33;K>G5tbMSPq${gLg?q>341K-}5^x4txj_29f$Mx|kqD>HUD}Za0>V1;4^2J!B3i+0^Hgl4GusGe>b@IYf<+gh5Ua& z3MJzs*SiPTVfG}j{6bXVFV%}kEB5=N+pop`8I& zVU?cgrKwys`CHZ2w3of3RQj;9MS8+e{|m#o&BGWD?VBGkO`1CWpGOQc?9xv{2^GW> z4J<|9+qzm)WcsD995RTFrX6)eD{|bdY@KZFC7(>za^)#5HPnmChCPf_Jb}8==~&@f z&_Q?Oo#4UjBJ+vIvK@r6odoSzL@lCE&^cVx6W&*U=)E_eWx79KFigk77+z+2%^2>T zNFFU)Qa$8WVbPo^8So4jAr-b|uDA8DAbYH=*&@pN1nb3m7AqO{6jstu`&r4^6=Kws zL#c6rz_d4pOJLkjKcCQoWI+UzNeygxcwC;e+r8C^04^hyUIv#>HqI-^6Deb;LQYmP z<|YgzkaRY(3IEx+^dKT(25U%2$~dWV6aL z2Ni7{FW-ASD8A;mL!1_WT$Ce58(OPg(^A&eUN6R6@I-JpopEjP3&SnEmjZHYJs$v5 zw00ruC(?EyQs<_3wltwRjxZbz4|K7wmH4W(J;)XnP1zv{kK33x#lP=@&KQ;hACQFv z!y!uKE<|rGZjE0DFqDx6wg- zd%)k|vEK%#ZQla|6dgz0zQK{*H=IBz_D^2S|9Qqop{rx^FZ>_hp$zT#A`n&fV68)- z!N6w>hB-~fG^jX_lmV$;XeA=d4-3NIS}^{p*9AgF!|n#k9LA(5OVHrp)He#b?OKJJ zU=Cs&l6eJ8YZl%-;R#IKx*ISoZ7aGxKeloM(em+SxaaJq!!Cp;Fn@1rx^IqB?S>msK?M8W1o4d*KCWDNez4Os8 z@8}M zEUz<5_^{(o*7^D_6c$^y%V>DBEnjh9V9(GYZN1z7uG)xR*nnk%Y%w;rt4tcA@ zYh`Mdt3Jj-utkBK{=QzcD)kkX*H@j2_?auHl0iP{jTZ$9jqcXDPYRzfjW#qhWmzNu za^fH2#h*iwzHI>i)>5S=^n!~DU4v3=hdW%X9e?}IK4_B++xdD=e_}B3yTX@RtNI zJEC?W9RYZ#>^2~p&DQTim_X+J^op*{1D-`_6`mk&7jg=LvjtSvnO(^9svY57$T;S; zG(9*AsnO0fa9ULZ-S?RFF61T@yTi0Bv@v0;Tv(_0|fz7^KYkLL(~rAqY=wEz!*a8W&Yq-!O}8915g&foju_9y9Ou$Dk^o*1RmP4?GgS7KGWIdcT+SXSXLI;Vokes&r^^zB^jDTJ7hA@-O20Dz;_rDszo*yeJrPa*ZS4H1bn8(%vi{&UrV%j_^u+g?kf?? zN*mpAd7)U)~tf55`Dr2CcU$ zdY^+>ZP_X=d5BJmp{`}7ED7##JmbjK@zud(7!7k9efwTW~5BCjd84@M3b5Vi;41A+| z`ZyU=e-?>;N6VII{7S?7%Xwwlqu4I{=MzUbIPW?I9${WZ`9Ovf(#-pk zvjX%{&i-lw!UQr3jMb!?;|02>k4r9I4}e)TOQMcVvA)P_k(41N4(j$;y*$SeQ?=~( zG-hDE(c0Oedvi5%h`E_jbRwAJa&0Vc|7-Zb%sl^<`Pw(ks=PM0r-d(N5t~TcFOD_S zxv?K>j#vnqvV4Bz1^HzX6GAbOG>5oTy>sD6{hWb*?XwFjSuA~W+sI5_!5N_F{-5bt z4*VvA`cr?$mlwg)HO^R%(KAW)bgtb}OmR3L8usdzVzI>wX?{ee@0UPCKKq~v)0&tT zW01BllL^zfrogT18}N&wiv3{=`S{AK(v*^lS0}<==RW&vBy7TVI0Zcvsy~@~E7EHc z#{s3y#dGk@dnHGALuSrmEn(oT!i_qsw$IB0lHti%d$n1TI`^Jb_3Lm z*eNaA__7N*|JwyqCswb{BTDg$*z=%VLy;POCvKwC^4IS4v8H0#8@hu4qy~ybtj`S( z=WR#&{oX(16LA3^y;U_5NulQc>kX>VU^r|`EnMDQMK!Y7Y?Eq=#Ti#Ft#=RcEL<{8 zwahwO`b6QF)vZm>8FO>5NGId8Hz2F_LK47o{GmtM`}?~E_2=0D9AVf1u;L+iiePAM zRf@>sj@)`PM&h=lyu7lo^Fgt!PBI~rtR4@J8u^kzYq#dx(pEA(*_a&Wuhx$!I)$q0 z!k=3je8M~t)%D|Eu+GZnv|PBwrv>MBwh?Aqnq8Y$FTa?Wj1PBTcNPsx7-&j@v4G71 z8w)i$)>U92@bG)cj%iY1&c&z8fWW8Q^5c=o z$q%y*3#4u`Fg{II zJTHFG`gwSz`7_hZ&@`6UeV)*?x68wIqjA)8A?AUl(oRBB{8=NiTBNIeWqpza)XYqo z3?8crih`@kOS=$n9*%u5Hb)Eh;am?kwaAv{+G@u<%-H21uq_omo;Z4ZrWl^DCjtZ%b86qp~}JS z@c6ArX?2OyjCvn}02%vp#U+pX9`5-naN=8^9HJVB;ah?+j4WA!x z_`=UCl7TFcHZvy*adoqHFk*M;z4b|JdWLxFqE0D<0Yaf?B20scKSBKp{;|PCB7}j5 zJKl!QtYLdDE5b*dYA7a=vjxU*a}+tN^IV$rHVRYm-gFm|)qSORyttrf>2pqDur|EE zrchdXKt040x4gjqIqsn&%zH_jE4sT$Pni5o_uYKITKBl8j6Ba=%w($}$W^#k)~sh@y0Kk%-;2oJ-_11HlJhd!9tlGJWOPxAXUAF! zDiTZNxiuk+G<5d9(~RYgb+V%+cf;!|4TaQ&3HmlDY&!7=u32}EI;CTj}(CPrZU{Fm+E$nuCW*_o;ULm4_ zT$CR+T}G>cL%3U9_IKzB_~LqoPZ{LU)Un^Pg~9t?x>8q<94s3*L#k%aS2=kZ?1_>ynQ-9E>0vfF!QP%f%) zxs`+oc5^&gvu)eRCJ>N@6d1Ppt{M5bp!5O_Nyr`z_zzW>x8TXEe9gk0PW5x`BpoAp zae<3&#DFEhPSjo14q1cRpbB3`t9wt)3vxQ4pLpN*Cwkzny{h=7wv?tY5Bu@KF*i+` z(PE7#=bm-;?PssSK15eQe?zs~5k@W^l&r|$QRFRu$^UgfE1~}`WQcHe>UBVkJF8&e ztettF?A=on@?>{^(Wyz9jr*LXi*;5h>fVJEIB=w@Xt0nqGkliQgU}#FW}xhlbfYmp z$tI-wGgZ0C@(z!_IbPPr>ITLM#sS&}kQ^W+PA0U0%4s&UQ}sQ|d-Onc+CkD3Mod)- zR_2xgaCMmva-t%=Yf4Gp+6T(sSoQ3G!y#^c4ExfJ-i%+jmrgFAvDr$G6cSTGj8!o> zQ@+gA!f%FC&0N!TuO zIg>8?nE6Q^O>a@-iPD1=Z)Q&8k4330TAs;fGe%X}A}f=9L8Ns7X!` z5h1Rppm1{+);NLK_db>jpLPjq7)dsH&4UO@ZBTC;yMt4gSG|;O@~%CT%ZOcvT1Akf zH4rkE*}Bmx2Lv-x7Z>+O7WT&+^22MG&}n{9fA^~RpMKfZ)QS>gu1VQi@)cGUUXG6@ z8Sj!?IBpIAIdkaJM?%LgE8 zRAK-gLkK9*u!i`qiD?CzNg!I-*GqPPq99tW;L~zKXRWz zE}lGj0?vgckYKQ+tnMAczC){2v~?&Ityq*C5dPM0>M*>JY_ zbmG)eYLam=JC4NV^96{z)(I8)?~ga9hF>@_`V=$Uxt5409%EWk#0tvqVo3iQdu7rHX=k_1vR9fl{pn9{`?akRLwCHLLUpsC*(sVQ z8DpgRX{bz9Zf|A1_HsKe{*@N-KbUL&j)j*1iCx4?A6wiq*@Yxhl^&hmb_QFkryDy! zE8GTZn|y~m2JhVeYTINE2pl@_mA3%-#RLuIJSpe?#Xga5jV2wFw5Nf)yQhIW&-K;T z(;G4k7h>ee?P?pNT3@CxVqIiA-fF;9d$}6}QrDioF)b2wXX(VyFtQx)A)u5W;6v9L z^kdJDmJ12S*xQm1xFropvl@mGn^KDq4wZrWH~zXOEFxntA?TvQd23^-gPtNm&+Mzw z{#D7pEKPwTAgH{V@WQtKp_tEkgrTe0PxkR7$e&L6|U^TAT9Kj()%lG`A=-}KK%XzXg_q>Nc?0a!6U$h8pcO* zS%1X<|A{xiAkECY#@zmwV649g1@0sI{|jC_X#XMnCQS392yn^S*;6V@sVsVSJm_o z)7Hp0iN7zt?W^V%v6w5n?iaqjBL%edK8SC}tt(h>0!}>kTNVKeH2)qD`!a2PQQk>3 z+DmN!%60v?Z^yX>_v770!5tj>mZ*T7rvvKYXZWo@9wTMvkHh}ByKl)5|JLWYZ8z7Q zwo*IgN;)X_Q9cddXbHY&z|S&#Aj$s%2u-Ndrfelr=rBZRrq5N8C;OGZF)1sBC^A zirbI=2)^lQBh&LK53(y~jD&~N6>ZaQx*ES!Qzh=bO3l~_;uPPL&NFcr2noMukN<*LMyVh{<=Z6=SSDUiy;|zQO>&$fV6cAnsqglKg4s`$ygc zTSnSlNbsY~9lBkJB-o5a!L^BRY{9~&nig36E_&R-;UOq~ob~hk#Zz;%{^2Qg>{=Hg zhC#fy1MU5)*V8rq!?Uq3B!juG4-IuM&^!rb&*>XKc=9H^I~c2k?_i8&>0ayazl6%* zY9Cs9s_qf}S}cIA6M>oS%c{z{YVTq>e{bH>@=?uDF3~-Na}bK&ol?gF><+v(V2UcA zIXEoT$>o}v4OmDej(}=DU2IDWKopl&9(j${Q~RQ;(8U|Y0V308xPYc8@=J!x8N*Bd zecE-Veh9~3o}O*w28H2vzB5nL;3HYl?`qDq*Y{K4}3MW4JehGWVVOE1F=EUhc2hMQMU-S;b&BxN00 z1;Q(K9UUV-g<8?1$}?^(QU#A?p9(dzZq`e&Z zA9z({I@!Sd{5Whmf7MLAFwQ*k9Qk|45{VJ#5k`+C`|e~cVp={OZ4cwWF^O`>jZPPY zvoc+mgKhg+lVcyFVHNou4$Uspm8BC$#OHkRjL)jfKNJ;rGL2E@nZImgXi;ks{9ub443(;t7?|4bZWP7<(DmGy2`dk-h4P2g_G7J`a zJx6O~B072*ExCx^gBCJZEbCO6S#5;$-GNK89_4ytFip0GT-(ERSUb`UT1%Ar9Hunt zKNoL&CHQ0@P-M-js(-*$w@v116Orbh9M zQ?;n5FmmR#H$%WF2;A<`XQC5C(%j?Rrj@MS)>fGhd+u>(P7@+}=@&hrLkE|AGHu7j zB*);54#gRVlCu!wV>jf_Urrr>A}3WXw}sLM&2bNb!fa{QU5Y67&eTEq$D~I|8F2S8 zNsKg|`DqKIp4aj_$AXHau4I(JB(Ab-Z@l&nsJmbBxgf!;$X1G6vpkWxP{Ww+5!1pr zh#I8`pf%2Ps!)_n+kKE%Mp7dSIl{UhiljD=!Y{*3Eh86bYTIQGlVwYF#4m89wojHv zeh>j6$)XaGVcHx$k?}y-y<0Q2TE8>-{~tty{}mBuATc0CG}!9`Pacr!w_3!5VCV1( zJqk$6Ccm-H_7d`b_JiznP!zgtv(asPV~&^FS5S;!B%Ml4u=F9LVobS>Z1nt9-?&?I z=5^o47JhUA{16GaEik+ZnS)}`Jov|^|A%S>{-|jAN89#+a+k8H2c`M?PozF)<-G6e z!>c}bA0>OcK#?D%mM4fZy*_RCE}CCRpVUC&v}u+j*F%`zm9htA99Y>wp{p+?#1lO+ zmlmObcs;#GyheRhC2sFR03JFv9OUZi1pMc<|D|yOaBJ6znMD&)q%#5^Kw^+?H4+z3 zoGGJQ_ij6Ziyf%LXiW6Q9|JhX$$~PWjSU{RRNngNaRTB5p1TsXYJy{*KMwWYyvCQI hha1sP`77VHpM3-W^S__ZO#c~||BTDu`x)5n{VxVO!q@-+ diff --git a/reproduction/chinese_word_segment/run.py b/reproduction/chinese_word_segment/run.py index 188d478d..66d01038 100644 --- a/reproduction/chinese_word_segment/run.py +++ b/reproduction/chinese_word_segment/run.py @@ -6,21 +6,21 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) from fastNLP.loader.config_loader import ConfigLoader, ConfigSection from fastNLP.core.trainer import SeqLabelTrainer from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, BaseLoader -from fastNLP.loader.preprocess import POSPreprocess, load_pickle +from fastNLP.core.preprocess import SeqLabelPreprocess, load_pickle from fastNLP.saver.model_saver import ModelSaver from fastNLP.loader.model_loader import ModelLoader from fastNLP.core.tester import SeqLabelTester from fastNLP.models.sequence_modeling import AdvSeqLabel -from fastNLP.core.inference import SeqLabelInfer +from fastNLP.core.predictor import SeqLabelInfer # not in the file's dir if len(os.path.dirname(__file__)) != 0: os.chdir(os.path.dirname(__file__)) -datadir = 'icwb2-data' -cfgfile = 'cws.cfg' +datadir = "/home/zyfeng/data/" +cfgfile = './cws.cfg' data_name = "pku_training.utf8" -cws_data_path = os.path.join(datadir, "training/pku_training.utf8") +cws_data_path = os.path.join(datadir, "pku_training.utf8") pickle_path = "save" data_infer_path = os.path.join(datadir, "infer.utf8") @@ -70,9 +70,10 @@ def train(): train_data = loader.load_pku() # Preprocessor - p = POSPreprocess(train_data, pickle_path, train_dev_split=0.3) - train_args["vocab_size"] = p.vocab_size - train_args["num_classes"] = p.num_classes + preprocessor = SeqLabelPreprocess() + data_train, data_dev = preprocessor.run(train_data, pickle_path=pickle_path, train_dev_split=0.3) + train_args["vocab_size"] = preprocessor.vocab_size + train_args["num_classes"] = preprocessor.num_classes # Trainer trainer = SeqLabelTrainer(**train_args.data) @@ -83,10 +84,11 @@ def train(): ModelLoader.load_pytorch(model, "./save/saved_model.pkl") print('model parameter loaded!') except Exception as e: + print("No saved model. Continue.") pass # Start training - trainer.train(model) + trainer.train(model, data_train, data_dev) print("Training finished!") # Saver @@ -106,6 +108,9 @@ def test(): index2label = load_pickle(pickle_path, "id2class.pkl") test_args["num_classes"] = len(index2label) + # load dev data + dev_data = load_pickle(pickle_path, "data_dev.pkl") + # Define the same model model = AdvSeqLabel(test_args) @@ -114,10 +119,10 @@ def test(): print("model loaded!") # Tester - tester = SeqLabelTester(test_args) + tester = SeqLabelTester(**test_args.data) # Start testing - tester.test(model) + tester.test(model, dev_data) # print test results print(tester.show_matrices()) diff --git a/test/seq_labeling.py b/test/seq_labeling.py index 4cf5e86f..a9488834 100644 --- a/test/seq_labeling.py +++ b/test/seq_labeling.py @@ -123,7 +123,7 @@ def train_and_test(): tester = SeqLabelTester(save_output=False, save_loss=False, save_best_dev=False, - batch_size=8, + batch_size=4, use_cuda=False, pickle_path=pickle_path, model_name="seq_label_in_test.pkl", @@ -140,4 +140,4 @@ def train_and_test(): if __name__ == "__main__": train_and_test() - infer() + # infer() diff --git a/test/test_fastNLP.py b/test/test_fastNLP.py index 9d0f2dee..c29db1fb 100644 --- a/test/test_fastNLP.py +++ b/test/test_fastNLP.py @@ -3,7 +3,7 @@ import sys sys.path.append("..") from fastNLP.fastnlp import FastNLP -PATH_TO_CWS_PICKLE_FILES = "/home/zyfeng/data/save/" +PATH_TO_CWS_PICKLE_FILES = "/home/zyfeng/fastNLP/reproduction/chinese_word_segment/save/" def word_seg(): nlp = FastNLP(model_dir=PATH_TO_CWS_PICKLE_FILES)