diff --git a/mindinsight/explainer/encapsulator/explain_job_encap.py b/mindinsight/explainer/encapsulator/explain_job_encap.py index 5e45facc..3eda8680 100644 --- a/mindinsight/explainer/encapsulator/explain_job_encap.py +++ b/mindinsight/explainer/encapsulator/explain_job_encap.py @@ -91,4 +91,5 @@ class ExplainJobEncap(ExplainDataEncap): saliency_info["metrics"] = list(job.metrics) info["saliency"] = saliency_info info["uncertainty"] = {"enabled": job.uncertainty_enabled} + info["status"] = job.status return info diff --git a/mindinsight/explainer/manager/explain_loader.py b/mindinsight/explainer/manager/explain_loader.py index d6b9f275..b7f3b950 100644 --- a/mindinsight/explainer/manager/explain_loader.py +++ b/mindinsight/explainer/manager/explain_loader.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -50,6 +50,8 @@ _SAMPLE_FIELD_NAMES = [ class _LoaderStatus(Enum): STOP = 'STOP' LOADING = 'LOADING' + PENDING = 'PENDING' + LOADED = 'LOADED' def _round(score): @@ -75,13 +77,13 @@ class ExplainLoader: 'create_time': os.stat(summary_dir).st_ctime, 'update_time': os.stat(summary_dir).st_mtime, 'query_time': os.stat(summary_dir).st_ctime, - 'uncertainty_enabled': False + 'uncertainty_enabled': False, } self._samples = defaultdict(dict) self._metadata = {'explainers': [], 'metrics': [], 'labels': [], 'min_confidence': 0.5} self._benchmark = {'explainer_score': defaultdict(dict), 'label_score': defaultdict(dict)} - self._status = _LoaderStatus.STOP.value + self._status = _LoaderStatus.PENDING.value self._status_mutex = threading.Lock() @property @@ -270,7 +272,9 @@ class ExplainLoader: def load(self): """Start loading data from the latest summary file to the loader.""" - self.status = _LoaderStatus.LOADING.value + if self.status != _LoaderStatus.LOADED.value: + self.status = _LoaderStatus.LOADING.value + filenames = [] for filename in FileHandler.list_dir(self._loader_info['summary_dir']): if FileHandler.is_file(FileHandler.join(self._loader_info['summary_dir'], filename)): @@ -286,15 +290,20 @@ class ExplainLoader: try: file_changed, is_end, event_dict = self._parser.list_events(filenames) except UnknownError: + is_end = True break if file_changed: logger.info('Summary file in %s update, reload the data in the summary.', self._loader_info['summary_dir']) self._clear_job() + if self.status != _LoaderStatus.STOP.value: + self.status = _LoaderStatus.LOADING.value if event_dict: self._import_data_from_event(event_dict) self._reform_sample_info() + if is_end: + self.status = _LoaderStatus.LOADED.value @property def status(self): diff --git a/mindinsight/explainer/manager/explain_manager.py b/mindinsight/explainer/manager/explain_manager.py index f04a5c81..7d650698 100644 --- a/mindinsight/explainer/manager/explain_manager.py +++ b/mindinsight/explainer/manager/explain_manager.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ from mindinsight.datavisual.common.enums import BaseEnum from mindinsight.datavisual.data_access.file_handler import FileHandler from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher from mindinsight.explainer.common.log import logger -from mindinsight.explainer.manager.explain_loader import ExplainLoader +from mindinsight.explainer.manager.explain_loader import ExplainLoader, _LoaderStatus from mindinsight.utils.exceptions import ParamValueError, UnknownError _MAX_LOADERS_NUM = 3 @@ -91,6 +91,9 @@ class ExplainManager: if loader_id in self._loader_pool: self._loader_pool[loader_id].query_time = datetime.now().timestamp() self._loader_pool.move_to_end(loader_id, last=True) + loader = self._loader_pool[loader_id] + if loader.status == _LoaderStatus.STOP.value: + self._reload_data_again() return self._loader_pool[loader_id] try: diff --git a/tests/ut/explainer/encapsulator/mock_explain_manager.py b/tests/ut/explainer/encapsulator/mock_explain_manager.py index a54db862..32ee9c0e 100644 --- a/tests/ut/explainer/encapsulator/mock_explain_manager.py +++ b/tests/ut/explainer/encapsulator/mock_explain_manager.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ from datetime import datetime from mindinsight.explainer.encapsulator.explain_job_encap import ExplainJobEncap +from mindinsight.explainer.manager.explain_loader import _LoaderStatus class MockExplainJob: @@ -60,6 +61,7 @@ class MockExplainJob: ] } ] + self.status = _LoaderStatus.LOADED.value def retrieve_image(self, image_id): """Get original image binary.""" diff --git a/tests/ut/explainer/manager/test_explain_loader.py b/tests/ut/explainer/manager/test_explain_loader.py index f719b15a..310286c2 100644 --- a/tests/ut/explainer/manager/test_explain_loader.py +++ b/tests/ut/explainer/manager/test_explain_loader.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,6 +24,12 @@ from mindinsight.explainer.manager.explain_loader import _LoaderStatus from mindinsight.explainer.manager.explain_parser import ExplainParser +class _MockStat: + def __init__(self, _): + self.st_ctime = 1 + self.st_mtime = 1 + self.st_size = 1 + class TestExplainLoader: """Test explain loader class.""" @@ -37,11 +43,6 @@ class TestExplainLoader: mock_list_dir.return_value = ['events.summary.123.host_explain'] mock_list_events.return_value = (True, False, None) - class _MockStat: - def __init__(self, _): - self.st_ctime = 1 - self.st_mtime = 1 - self.st_size = 1 mock_stat.side_effect = _MockStat @@ -50,8 +51,8 @@ class TestExplainLoader: summary_dir='./summary_dir') def _stop_loader(explain_loader): - time.sleep(0.01) - assert explain_loader.status == _LoaderStatus.LOADING.value + while explain_loader.status != _LoaderStatus.LOADING.value: + time.sleep(0.01) explain_loader.stop() thread = threading.Thread(target=_stop_loader, args=[loader], daemon=True) @@ -59,3 +60,22 @@ class TestExplainLoader: loader.load() assert loader.status == _LoaderStatus.STOP.value + + @patch.object(ExplainParser, 'list_events') + @patch.object(FileHandler, 'list_dir') + @patch.object(FileHandler, 'is_file') + @patch.object(os, 'stat') + def test_loaded_with_is_end(self, mock_stat, mock_is_file, mock_list_dir, mock_list_events): + """Test loading function.""" + mock_is_file.return_value = True + mock_list_dir.return_value = ['events.summary.123.host_explain'] + mock_list_events.return_value = (True, True, None) + + mock_stat.side_effect = _MockStat + + loader = ExplainLoader( + loader_id='./summary_dir', + summary_dir='./summary_dir') + + loader.load() + assert loader.status == _LoaderStatus.LOADED.value