remove lineage public APIs

mindinsight/backend/lineagemgr/lineage_api.py

@@ -21,7 +21,7 @@ from flask import Blueprint, jsonify, request
 from mindinsight.conf import settings
 from mindinsight.datavisual.utils.tools import get_train_id
 from mindinsight.datavisual.data_transform.data_manager import DATA_MANAGER
-from mindinsight.lineagemgr.api.model import general_filter_summary_lineage, general_get_summary_lineage
+from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage
 from mindinsight.utils.exceptions import MindInsightException, ParamValueError
 from mindinsight.lineagemgr.cache_item_updater import update_lineage_object
 
@@ -69,7 +69,7 @@ def _get_lineage_info(search_condition):
     """
     summary_base_dir = str(settings.SUMMARY_BASE_DIR)
     try:
-        lineage_info = general_filter_summary_lineage(
+        lineage_info = filter_summary_lineage(
             data_manager=DATA_MANAGER,
             search_condition=search_condition,
             added=True)
@@ -137,7 +137,7 @@ def get_dataset_graph():
     summary_base_dir = str(settings.SUMMARY_BASE_DIR)
     summary_dir = get_train_id(request)
     try:
-        dataset_graph = general_get_summary_lineage(
+        dataset_graph = get_summary_lineage(
             DATA_MANAGER,
             summary_dir=summary_dir,
             keys=['dataset_graph']

mindinsight/lineagemgr/__init__.py

@@ -12,15 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
-"""
-Lineagemgr Module Introduction.
-
-This module provides Python APIs to query the lineage of models.
-The APIs can be used to get the lineage information of the models. For example,
-what hyperparameter is used in the model training, which model has the highest
-accuracy among all the versions, etc.
-"""
-from mindinsight.lineagemgr.api.model import get_summary_lineage, filter_summary_lineage
-
-
-__all__ = ["get_summary_lineage", "filter_summary_lineage"]

mindinsight/lineagemgr/api/__init__.py

@@ -1,14 +0,0 @@
-# Copyright 2019 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================

mindinsight/lineagemgr/api/model.py → mindinsight/lineagemgr/model.py

@@ -29,39 +29,7 @@ from mindinsight.lineagemgr.querier.querier import Querier
 from mindinsight.utils.exceptions import MindInsightException
 
 
-def get_summary_lineage(summary_dir, keys=None):
-    """
-    Get the lineage information according to summary directory and keys.
-
-    The function queries lineage information of single train process
-    corresponding to the given summary directory. Users can query the
-    information according to `keys`.
-
-    Args:
-        summary_dir (str): The summary directory. It contains summary logs for
-            one training.
-        keys (list[str]): The filter keys of lineage information. The acceptable
-            keys are `metric`, `user_defined`, `hyper_parameters`, `algorithm`,
-            `train_dataset`, `model`, `valid_dataset` and `dataset_graph`.
-            If it is `None`, all information will be returned. Default: None.
-
-    Returns:
-        dict, the lineage information for one training.
-
-    Raises:
-        LineageParamSummaryPathError: If summary path is invalid.
-        LineageQuerySummaryDataError: If querying summary data fails.
-        LineageFileNotFoundError: If the summary log file is not found.
-
-    Examples:
-        >>> summary_dir = "/path/to/summary"
-        >>> summary_lineage_info = get_summary_lineage(summary_dir)
-        >>> hyper_parameters = get_summary_lineage(summary_dir, keys=["hyper_parameters"])
-    """
-    return general_get_summary_lineage(summary_dir=summary_dir, keys=keys)
-
-
-def general_get_summary_lineage(data_manager=None, summary_dir=None, keys=None):
+def get_summary_lineage(data_manager=None, summary_dir=None, keys=None):
     """
     Get summary lineage from data_manager or parsing from summaries.
 
@@ -116,131 +84,7 @@ def general_get_summary_lineage(data_manager=None, summary_dir=None, keys=None):
     return result[0]
 
 
-def filter_summary_lineage(summary_base_dir, search_condition=None):
-    """
-    Filter the lineage information under summary base directory according to search condition.
-
-    Users can filter and sort all lineage information according to the search
-    condition. The supported filter fields include `summary_dir`, `network`,
-    etc. The filter conditions include `eq`, `lt`, `gt`, `le`, `ge` and `in`.
-    If the value type of filter condition is `str`, such as summary_dir and
-    lineage_type, then its key can only be `in` and `eq`. At the same time,
-    the combined use of these fields and conditions is supported. If you want
-    to sort based on filter fields, the field of `sorted_name` and `sorted_type`
-    should be specified.
-
-    Users can use `lineage_type` to decide what kind of lineage information to
-    query. If the `lineage_type` is not defined, the query result is all lineage
-    information.
-
-    Users can paginate query result based on `offset` and `limit`. The `offset`
-    refers to page number. The `limit` refers to the number in one page.
-
-    Args:
-        summary_base_dir (str): The summary base directory. It contains summary
-            directories generated by training.
-        search_condition (dict): The search condition. When filtering and
-            sorting, in addition to the following supported fields, fields
-            prefixed with `metric/` and `user_defined/` are also supported.
-            For example, the field should be `metric/accuracy` if the key
-            of `metrics` parameter is `accuracy`. The fields prefixed with
-            `metric/` and `user_defined/` are related to the `metrics`
-            parameter in the training script and user defined information in
-            TrainLineage/EvalLineage callback, respectively. Default: None.
-
-            - summary_dir (dict): The filter condition of summary directory.
-
-            - loss_function (dict): The filter condition of loss function.
-
-            - train_dataset_path (dict): The filter condition of train dataset path.
-
-            - train_dataset_count (dict): The filter condition of train dataset count.
-
-            - test_dataset_path (dict): The filter condition of test dataset path.
-
-            - test_dataset_count (dict): The filter condition of test dataset count.
-
-            - network (dict): The filter condition of network.
-
-            - optimizer (dict): The filter condition of optimizer.
-
-            - learning_rate (dict): The filter condition of learning rate.
-
-            - epoch (dict): The filter condition of epoch.
-
-            - batch_size (dict): The filter condition of batch size.
-
-            - device_num (dict): The filter condition of device num.
-
-            - loss (dict): The filter condition of loss.
-
-            - model_size (dict): The filter condition of model size.
-
-            - dataset_mark (dict): The filter condition of dataset mark.
-
-            - lineage_type (dict): The filter condition of lineage type. It decides
-              what kind of lineage information to query. Its value can be `dataset`
-              or `model`, e.g., {'in': ['dataset', 'model']}, {'eq': 'model'}, etc.
-              If its values contain `dataset`, the query result will contain the
-              lineage information related to data augmentation. If its values contain
-              `model`, the query result will contain model lineage information.
-              If it is not defined or it is a dict like {'in': ['dataset', 'model']},
-              the query result is all lineage information.
-
-            - offset (int): Page number, the value range is [0, 100000].
-
-            - limit (int): The number in one page, the value range is [1, 100].
-
-            - sorted_name (str): Specify which field to sort by.
-
-            - sorted_type (str): Specify sort order. It can be `ascending` or
-              `descending`.
-
-    Returns:
-        dict, lineage information under summary base directory according to
-        search condition.
-
-    Raises:
-        LineageSearchConditionParamError: If search_condition param is invalid.
-        LineageParamSummaryPathError: If summary path is invalid.
-        LineageFileNotFoundError: If the summary log file is not found.
-        LineageQuerySummaryDataError: If querying summary log file data fails.
-
-    Examples:
-        >>> summary_base_dir = "/path/to/summary_base"
-        >>> search_condition = {
-        >>>     'summary_dir': {
-        >>>         'in': [
-        >>>             os.path.join(summary_base_dir, 'summary_1'),
-        >>>             os.path.join(summary_base_dir, 'summary_2'),
-        >>>             os.path.join(summary_base_dir, 'summary_3')
-        >>>         ]
-        >>>     },
-        >>>     'loss': {
-        >>>         'gt': 2.0
-        >>>     },
-        >>>     'batch_size': {
-        >>>         'ge': 128,
-        >>>         'le': 256
-        >>>     },
-        >>>     'metric/accuracy': {
-        >>>         'lt': 0.1
-        >>>     },
-        >>>     'sorted_name': 'summary_dir',
-        >>>     'sorted_type': 'descending',
-        >>>     'limit': 3,
-        >>>     'offset': 0,
-        >>>     'lineage_type': {
-        >>>         'eq': 'model'
-        >>>     }
-        >>> }
-        >>> summary_lineage = filter_summary_lineage(summary_base_dir)
-        >>> summary_lineage_filter = filter_summary_lineage(summary_base_dir, search_condition)
-    """
-    return general_filter_summary_lineage(summary_base_dir=summary_base_dir, search_condition=search_condition)
-
-
-def general_filter_summary_lineage(data_manager=None, summary_base_dir=None, search_condition=None, added=False):
+def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_condition=None, added=False):
     """
     Filter summary lineage from data_manager or parsing from summaries.
 

tests/st/func/lineagemgr/api/__init__.py

@@ -1,14 +0,0 @@
-# Copyright 2019 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================

tests/st/func/lineagemgr/cache/test_lineage_cache.py

@@ -26,10 +26,9 @@ import pytest
 
 from mindinsight.datavisual.data_transform.data_manager import DataManager
 from mindinsight.lineagemgr.cache_item_updater import LineageCacheItemUpdater
-from mindinsight.lineagemgr.api.model import general_filter_summary_lineage, \
-    general_get_summary_lineage
+from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage
 
-from ..api.test_model_api import LINEAGE_INFO_RUN1, LINEAGE_FILTRATION_EXCEPT_RUN, \
+from ..test_model import LINEAGE_INFO_RUN1, LINEAGE_FILTRATION_EXCEPT_RUN, \
     LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2
 from ..conftest import BASE_SUMMARY_DIR
 from .....ut.lineagemgr.querier import event_data
@@ -56,7 +55,7 @@ class TestModelApi(TestCase):
     @pytest.mark.env_single
     def test_get_summary_lineage(self):
         """Test the interface of get_summary_lineage."""
-        total_res = general_get_summary_lineage(data_manager=self._data_manger, summary_dir="./run1")
+        total_res = get_summary_lineage(data_manager=self._data_manger, summary_dir="./run1")
         expect_total_res = LINEAGE_INFO_RUN1
         assert_equal_lineages(expect_total_res, total_res, self.assertDictEqual)
 
@@ -81,7 +80,7 @@ class TestModelApi(TestCase):
         search_condition = {
             'sorted_name': 'summary_dir'
         }
-        res = general_filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition)
+        res = filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -98,5 +97,5 @@ class TestModelApi(TestCase):
                 "in": ['./dir_with_empty_lineage']
             }
         }
-        res = general_filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition)
+        res = filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition)
         assert_equal_lineages(expect_result, res, self.assertDictEqual)

tests/st/func/lineagemgr/collection/model/test_model_lineage.py

@@ -27,7 +27,7 @@ from unittest import TestCase, mock
 
 import numpy as np
 import pytest
-from mindinsight.lineagemgr import get_summary_lineage
+from mindinsight.lineagemgr.model import get_summary_lineage
 from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors
 from mindinsight.utils.exceptions import MindInsightException
 
@@ -90,12 +90,12 @@ class TestModelLineage(TestCase):
         train_callback = TrainLineage(SUMMARY_DIR, True, self.user_defined_info)
         train_callback.initial_learning_rate = 0.12
         train_callback.end(RunContext(self.run_context))
-        res = get_summary_lineage(SUMMARY_DIR)
+        res = get_summary_lineage(summary_dir=SUMMARY_DIR)
         assert res.get('hyper_parameters', {}).get('epoch') == 10
         run_context = self.run_context
         run_context['epoch_num'] = 14
         train_callback.end(RunContext(run_context))
-        res = get_summary_lineage(SUMMARY_DIR)
+        res = get_summary_lineage(summary_dir=SUMMARY_DIR)
         assert res.get('hyper_parameters', {}).get('epoch') == 14
 
     @pytest.mark.scene_eval(3)
@@ -186,7 +186,7 @@ class TestModelLineage(TestCase):
         run_context_customized['train_network'] = net
         train_callback.begin(RunContext(run_context_customized))
         train_callback.end(RunContext(run_context_customized))
-        res = get_summary_lineage(SUMMARY_DIR)
+        res = get_summary_lineage(summary_dir=SUMMARY_DIR)
         assert res.get('hyper_parameters', {}).get('loss_function') \
                == 'SoftmaxCrossEntropyWithLogits'
         assert res.get('algorithm', {}).get('network') == 'ResNet'

tests/st/func/lineagemgr/api/test_model_api.py → tests/st/func/lineagemgr/test_model.py

@@ -25,13 +25,13 @@ from unittest import TestCase
 
 import pytest
 
-from mindinsight.lineagemgr import filter_summary_lineage, get_summary_lineage
+from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage
 from mindinsight.lineagemgr.common.exceptions.exceptions import (LineageFileNotFoundError, LineageParamSummaryPathError,
                                                                  LineageParamTypeError, LineageParamValueError,
                                                                  LineageSearchConditionParamError)
-from ..conftest import BASE_SUMMARY_DIR, DATASET_GRAPH, SUMMARY_DIR, SUMMARY_DIR_2
-from .....ut.lineagemgr.querier import event_data
-from .....utils.tools import assert_equal_lineages
+from .conftest import BASE_SUMMARY_DIR, DATASET_GRAPH, SUMMARY_DIR, SUMMARY_DIR_2
+from ....ut.lineagemgr.querier import event_data
+from ....utils.tools import assert_equal_lineages
 
 LINEAGE_INFO_RUN1 = {
     'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
@@ -170,9 +170,9 @@ class TestModelApi(TestCase):
     @pytest.mark.env_single
     def test_get_summary_lineage(self):
         """Test the interface of get_summary_lineage."""
-        total_res = get_summary_lineage(SUMMARY_DIR)
-        partial_res1 = get_summary_lineage(SUMMARY_DIR, ['hyper_parameters'])
-        partial_res2 = get_summary_lineage(SUMMARY_DIR, ['metric', 'algorithm'])
+        total_res = get_summary_lineage(None, SUMMARY_DIR)
+        partial_res1 = get_summary_lineage(None, SUMMARY_DIR, ['hyper_parameters'])
+        partial_res2 = get_summary_lineage(None, SUMMARY_DIR, ['metric', 'algorithm'])
         expect_total_res = LINEAGE_INFO_RUN1
         expect_partial_res1 = {
             'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
@@ -200,14 +200,14 @@ class TestModelApi(TestCase):
         assert_equal_lineages(expect_partial_res2, partial_res2, self.assertDictEqual)
 
         # the lineage summary file is empty
-        result = get_summary_lineage(self.dir_with_empty_lineage)
+        result = get_summary_lineage(None, self.dir_with_empty_lineage)
         assert {} == result
 
         # keys is empty list
         expect_result = {
             'summary_dir': SUMMARY_DIR
         }
-        result = get_summary_lineage(SUMMARY_DIR, [])
+        result = get_summary_lineage(None, SUMMARY_DIR, [])
         assert expect_result == result
 
     @pytest.mark.level0
@@ -223,6 +223,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path does not exist or is not a dir.',
             get_summary_lineage,
+            None,
             '/tmp/fake/dir'
         )
 
@@ -231,6 +232,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             get_summary_lineage,
+            None,
             'tmp'
         )
 
@@ -239,6 +241,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             get_summary_lineage,
+            None,
             ['/root/linage1', '/root/lineage2']
         )
 
@@ -247,6 +250,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             get_summary_lineage,
+            None,
             '',
             keys=None
         )
@@ -256,6 +260,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             get_summary_lineage,
+            None,
             '\\',
             keys=None
         )
@@ -273,6 +278,7 @@ class TestModelApi(TestCase):
             LineageParamValueError,
             'Keys must be in',
             get_summary_lineage,
+            None,
             SUMMARY_DIR,
             ['metric', 'fake_name']
         )
@@ -281,6 +287,7 @@ class TestModelApi(TestCase):
             LineageParamTypeError,
             'Keys must be list.',
             get_summary_lineage,
+            None,
             SUMMARY_DIR,
             0
         )
@@ -289,6 +296,7 @@ class TestModelApi(TestCase):
             LineageParamTypeError,
             'Keys must be list.',
             get_summary_lineage,
+            None,
             SUMMARY_DIR,
             0.1
         )
@@ -297,6 +305,7 @@ class TestModelApi(TestCase):
             LineageParamTypeError,
             'Keys must be list.',
             get_summary_lineage,
+            None,
             SUMMARY_DIR,
             True
         )
@@ -305,6 +314,7 @@ class TestModelApi(TestCase):
             LineageParamTypeError,
             'Element of keys must be str.',
             get_summary_lineage,
+            None,
             SUMMARY_DIR,
             [1, 2, 3]
         )
@@ -313,6 +323,7 @@ class TestModelApi(TestCase):
             LineageParamTypeError,
             'Keys must be list.',
             get_summary_lineage,
+            None,
             SUMMARY_DIR,
             (3, 4)
         )
@@ -321,6 +332,7 @@ class TestModelApi(TestCase):
             LineageParamTypeError,
             'Keys must be list.',
             get_summary_lineage,
+            None,
             SUMMARY_DIR,
             {'a': 'b'}
         )
@@ -346,7 +358,7 @@ class TestModelApi(TestCase):
         search_condition = {
             'sorted_name': 'summary_dir'
         }
-        res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
+        res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -357,7 +369,7 @@ class TestModelApi(TestCase):
             'object': [],
             'count': 0
         }
-        res = filter_summary_lineage(self.dir_with_empty_lineage)
+        res = filter_summary_lineage(None, self.dir_with_empty_lineage)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -395,7 +407,7 @@ class TestModelApi(TestCase):
             ],
             'count': 2
         }
-        partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
+        partial_res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(partial_res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -433,7 +445,7 @@ class TestModelApi(TestCase):
             ],
             'count': 2
         }
-        partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
+        partial_res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(partial_res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -462,7 +474,7 @@ class TestModelApi(TestCase):
             ],
             'count': 3
         }
-        partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1)
+        partial_res1 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition1)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(partial_res1.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -481,7 +493,7 @@ class TestModelApi(TestCase):
             'object': [],
             'count': 0
         }
-        partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2)
+        partial_res2 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition2)
         assert expect_result == partial_res2
 
     @pytest.mark.level0
@@ -511,7 +523,7 @@ class TestModelApi(TestCase):
             ],
             'count': 1
         }
-        res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
+        res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
         assert expect_result == res
 
     @pytest.mark.level0
@@ -527,6 +539,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             filter_summary_lineage,
+            None,
             'relative_path'
         )
 
@@ -535,6 +548,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path does not exist or is not a dir.',
             filter_summary_lineage,
+            None,
             '/path/does/not/exist'
         )
 
@@ -543,6 +557,7 @@ class TestModelApi(TestCase):
             LineageFileNotFoundError,
             'There is no summary log file under summary_base_dir.',
             filter_summary_lineage,
+            None,
             self.empty_dir
         )
 
@@ -562,6 +577,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The search_condition element summary_dir should be dict.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -574,6 +590,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The sorted_name have to exist when sorted_type exists.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -584,6 +601,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'Invalid search_condition type, it should be dict.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -596,6 +614,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The limit must be int.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -616,6 +635,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The offset must be int.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -630,6 +650,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The search attribute not supported.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -651,6 +672,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The sorted_type must be ascending or descending',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -665,6 +687,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The compare condition should be in',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -679,6 +702,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The parameter metric/accuracy is invalid.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -703,6 +727,7 @@ class TestModelApi(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             filter_summary_lineage,
+            None,
             BASE_SUMMARY_DIR,
             search_condition
         )
@@ -727,7 +752,7 @@ class TestModelApi(TestCase):
             'object': [],
             'count': 0
         }
-        partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1)
+        partial_res1 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition1)
         assert expect_result == partial_res1
 
         # the (offset + 1) * limit > count
@@ -743,7 +768,7 @@ class TestModelApi(TestCase):
             'object': [],
             'count': 2
         }
-        partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2)
+        partial_res2 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition2)
         assert expect_result == partial_res2
 
     @pytest.mark.level0
@@ -766,6 +791,7 @@ class TestModelApi(TestCase):
                 LineageSearchConditionParamError,
                 f'The parameter {condition_key} is invalid. Its operation should be `in` or `eq`.',
                 filter_summary_lineage,
+                None,
                 BASE_SUMMARY_DIR,
                 search_condition
             )
@@ -781,6 +807,7 @@ class TestModelApi(TestCase):
                 LineageSearchConditionParamError,
                 f'The parameter {condition_key} is invalid. More than one operation.',
                 filter_summary_lineage,
+                None,
                 BASE_SUMMARY_DIR,
                 search_condition
             )
@@ -804,6 +831,7 @@ class TestModelApi(TestCase):
                 LineageSearchConditionParamError,
                 "The parameter lineage_type is invalid. It should be 'dataset' or 'model'.",
                 filter_summary_lineage,
+                None,
                 BASE_SUMMARY_DIR,
                 search_condition
             )
@@ -825,6 +853,7 @@ class TestModelApi(TestCase):
                 LineageSearchConditionParamError,
                 'The sorted_name must be in',
                 filter_summary_lineage,
+                None,
                 BASE_SUMMARY_DIR,
                 search_condition
             )

tests/ut/backend/lineagemgr/test_lineage_api.py

@@ -70,7 +70,7 @@ class TestSearchModel(TestCase):
         self.url = '/v1/mindinsight/lineagemgr/lineages'
 
     @mock.patch('mindinsight.backend.lineagemgr.lineage_api.settings')
-    @mock.patch('mindinsight.backend.lineagemgr.lineage_api.general_filter_summary_lineage')
+    @mock.patch('mindinsight.backend.lineagemgr.lineage_api.filter_summary_lineage')
     def test_search_model_success(self, *args):
         """Test the success of model_success."""
         base_dir = '/path/to/test_lineage_summary_dir_base'
@@ -113,7 +113,7 @@ class TestSearchModel(TestCase):
         self.assertDictEqual(expect_result, response.get_json())
 
     @mock.patch('mindinsight.backend.lineagemgr.lineage_api.settings')
-    @mock.patch('mindinsight.backend.lineagemgr.lineage_api.general_filter_summary_lineage')
+    @mock.patch('mindinsight.backend.lineagemgr.lineage_api.filter_summary_lineage')
     def test_search_model_fail(self, *args):
         """Test the function of model_lineage with exception."""
         response = self.app_client.post(self.url, data='xxx')

tests/ut/lineagemgr/api/__init__.py

@@ -1,14 +0,0 @@
-# Copyright 2019 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================

tests/ut/lineagemgr/api/test_model.py → tests/ut/lineagemgr/test_model.py

@@ -16,8 +16,7 @@
 from unittest import TestCase, mock
 from unittest.mock import MagicMock
 
-from mindinsight.lineagemgr import get_summary_lineage, filter_summary_lineage
-from mindinsight.lineagemgr.api.model import _convert_relative_path_to_abspath
+from mindinsight.lineagemgr.model import get_summary_lineage, filter_summary_lineage, _convert_relative_path_to_abspath
 from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamSummaryPathError, \
     LineageFileNotFoundError, LineageSummaryParseException, LineageQuerierParamException, \
     LineageQuerySummaryDataError, LineageSearchConditionParamError, LineageParamTypeError, \
@@ -28,8 +27,8 @@ from mindinsight.lineagemgr.common.path_parser import SummaryPathParser
 class TestModel(TestCase):
     """Test the function of get_summary_lineage and filter_summary_lineage."""
 
-    @mock.patch('mindinsight.lineagemgr.api.model.Querier')
-    @mock.patch('mindinsight.lineagemgr.api.model.LineageParser')
+    @mock.patch('mindinsight.lineagemgr.model.Querier')
+    @mock.patch('mindinsight.lineagemgr.model.LineageParser')
     @mock.patch('os.path.isdir')
     def test_get_summary_lineage_success(self, isdir_mock, parser_mock, qurier_mock):
         """Test the function of get_summary_lineage."""
@@ -40,7 +39,7 @@ class TestModel(TestCase):
         qurier_mock.return_value = mock_querier
         mock_querier.get_summary_lineage.return_value = [{'algorithm': {'network': 'ResNet'}}]
         summary_dir = '/path/to/summary_dir'
-        result = get_summary_lineage(summary_dir, keys=['algorithm'])
+        result = get_summary_lineage(None, summary_dir, keys=['algorithm'])
         self.assertEqual(result, {'algorithm': {'network': 'ResNet'}})
 
     def test_get_summary_lineage_failed(self):
@@ -50,6 +49,7 @@ class TestModel(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             get_summary_lineage,
+            None,
             invalid_path
         )
 
@@ -63,6 +63,7 @@ class TestModel(TestCase):
             LineageFileNotFoundError,
             'no summary log file under summary_dir',
             get_summary_lineage,
+            None,
             '/path/to/summary_dir'
         )
 
@@ -81,10 +82,10 @@ class TestModel(TestCase):
         mock_parser.return_value = None
         mock_file_handler = MagicMock()
         mock_file_handler.size = 1
-        result = get_summary_lineage('/path/to/summary_dir')
+        result = get_summary_lineage(None, '/path/to/summary_dir')
         assert {} == result
 
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_path')
+    @mock.patch('mindinsight.lineagemgr.model.validate_path')
     def test_convert_relative_path_to_abspath(self, validate_path_mock):
         """Test the function of converting realtive path to abspath."""
         validate_path_mock.return_value = '/path/to/summary_base_dir/summary_dir'
@@ -129,11 +130,11 @@ class TestModel(TestCase):
 
 class TestFilterAPI(TestCase):
     """Test the function of filter_summary_lineage."""
-    @mock.patch('mindinsight.lineagemgr.api.model.LineageOrganizer')
-    @mock.patch('mindinsight.lineagemgr.api.model.Querier')
+    @mock.patch('mindinsight.lineagemgr.model.LineageOrganizer')
+    @mock.patch('mindinsight.lineagemgr.model.Querier')
     @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_lineage_summaries')
-    @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath')
-    @mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir')
+    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
+    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
     def test_filter_summary_lineage(self, validate_path_mock, convert_path_mock,
                                     latest_summary_mock, qurier_mock, organizer_mock):
         """Test the function of filter_summary_lineage."""
@@ -154,7 +155,7 @@ class TestFilterAPI(TestCase):
         mock_querier.filter_summary_lineage.return_value = [{'loss': 3.0}]
 
         summary_base_dir = '/path/to/summary_base_dir'
-        result = filter_summary_lineage(summary_base_dir)
+        result = filter_summary_lineage(None, summary_base_dir)
         self.assertEqual(result, [{'loss': 3.0}])
 
     def test_invalid_path(self):
@@ -164,11 +165,12 @@ class TestFilterAPI(TestCase):
             LineageParamSummaryPathError,
             'The summary path is invalid.',
             filter_summary_lineage,
+            None,
             invalid_path
         )
 
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_condition')
-    @mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir')
+    @mock.patch('mindinsight.lineagemgr.model.validate_condition')
+    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
     def test_invalid_search_condition(self, mock_path, mock_valid):
         """Test filter_summary_lineage with invalid invalid param."""
         mock_path.return_value = None
@@ -178,14 +180,15 @@ class TestFilterAPI(TestCase):
             LineageSearchConditionParamError,
             'Invalid search_condition type.',
             filter_summary_lineage,
+            None,
             '/path/to/summary/dir',
             'invalid_condition'
         )
 
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition')
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_condition')
+    @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition')
+    @mock.patch('mindinsight.lineagemgr.model.validate_condition')
     @mock.patch('mindinsight.lineagemgr.common.utils.validate_path')
-    @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath')
+    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
     def test_failed_to_convert_path(self, mock_convert, *args):
         """Test filter_summary_lineage with invalid invalid param."""
         mock_convert.side_effect = LineageParamValueError('invalid path')
@@ -194,14 +197,15 @@ class TestFilterAPI(TestCase):
             LineageParamSummaryPathError,
             'invalid path',
             filter_summary_lineage,
+            None,
             '/path/to/summary/dir',
             {}
         )
 
-    @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath')
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition')
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_condition')
-    @mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir')
+    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
+    @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition')
+    @mock.patch('mindinsight.lineagemgr.model.validate_condition')
+    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
     @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
     def test_failed_to_get_summary_filesh(self, mock_parse, *args):
         """Test filter_summary_lineage with invalid invalid param."""
@@ -212,21 +216,22 @@ class TestFilterAPI(TestCase):
             LineageFileNotFoundError,
             'There is no summary log file under summary_base_dir.',
             filter_summary_lineage,
+            None,
             path
         )
 
-    @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath')
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition')
-    @mock.patch('mindinsight.lineagemgr.api.model.validate_condition')
-    @mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir')
+    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
+    @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition')
+    @mock.patch('mindinsight.lineagemgr.model.validate_condition')
+    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
     @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
-    @mock.patch('mindinsight.lineagemgr.api.model.Querier')
+    @mock.patch('mindinsight.lineagemgr.model.Querier')
     def test_failed_to_querier(self, mock_query, mock_parse, *args):
         """Test filter_summary_lineage with invalid invalid param."""
         mock_query.side_effect = LineageSummaryParseException()
         mock_parse.return_value = ['/path/to/summary/file']
         args[0].return_value = None
-        res = filter_summary_lineage('/path/to/summary')
+        res = filter_summary_lineage(None, '/path/to/summary')
         assert res == {'object': [], 'count': 0}
 
         mock_query.side_effect = LineageQuerierParamException(['keys'], 'key')
@@ -234,5 +239,6 @@ class TestFilterAPI(TestCase):
             LineageQuerySummaryDataError,
             'Filter summary lineage failed.',
             filter_summary_lineage,
+            None,
             '/path/to/summary/dir'
         )