You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

train.py 2.4 kB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071
  1. import os
  2. from sedna.core.lifelong_learning import LifelongLearning
  3. from sedna.common.config import Context, BaseConfig
  4. from sedna.datasources import TxtDataParse
  5. from basemodel import Model
  6. def _load_txt_dataset(dataset_url):
  7. # use original dataset url
  8. original_dataset_url = Context.get_parameters('original_dataset_url', "")
  9. dataset_urls = dataset_url.split()
  10. dataset_urls = [
  11. os.path.join(
  12. os.path.dirname(original_dataset_url),
  13. dataset_url) for dataset_url in dataset_urls]
  14. return dataset_urls[:-1], dataset_urls[-1]
  15. def train(estimator, train_data):
  16. task_definition = {
  17. "method": "TaskDefinitionSimple"
  18. }
  19. task_allocation = {
  20. "method": "TaskAllocationSimple"
  21. }
  22. ll_job = LifelongLearning(estimator,
  23. task_definition=task_definition,
  24. task_relationship_discovery=None,
  25. task_allocation=task_allocation,
  26. task_remodeling=None,
  27. inference_integrate=None,
  28. task_update_decision=None,
  29. unseen_task_allocation=None,
  30. unseen_sample_recognition=None,
  31. unseen_sample_re_recognition=None
  32. )
  33. ll_job.train(train_data)
  34. def update(estimator, train_data):
  35. ll_job = LifelongLearning(estimator,
  36. task_definition=None,
  37. task_relationship_discovery=None,
  38. task_allocation=None,
  39. task_remodeling=None,
  40. inference_integrate=None,
  41. task_update_decision=None,
  42. unseen_task_allocation=None,
  43. unseen_sample_recognition=None,
  44. unseen_sample_re_recognition=None
  45. )
  46. ll_job.update(train_data)
  47. def run():
  48. estimator = Model(num_class=31, epochs=1)
  49. train_dataset_url = BaseConfig.train_dataset_url
  50. train_data = TxtDataParse(data_type="train", func=_load_txt_dataset)
  51. train_data.parse(train_dataset_url, use_raw=False)
  52. train(estimator, train_data)
  53. if __name__ == '__main__':
  54. run()