!687 clean lineage code after removing lineage public APIs, delete get_summary_lineage(), modify related st and ut

Merge pull request !687 from luopengting/lineage

mindinsight/backend/lineagemgr/lineage_api.py

@@ -14,16 +14,15 @@
 # ============================================================================
 """Lineage restful api."""
 import json
-import os
 
 from flask import Blueprint, jsonify, request
 
 from mindinsight.conf import settings
 from mindinsight.datavisual.utils.tools import get_train_id
 from mindinsight.datavisual.data_transform.data_manager import DATA_MANAGER
-from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage
-from mindinsight.utils.exceptions import MindInsightException, ParamValueError
 from mindinsight.lineagemgr.cache_item_updater import update_lineage_object
+from mindinsight.lineagemgr.model import filter_summary_lineage
+from mindinsight.utils.exceptions import MindInsightException, ParamValueError
 
 BLUEPRINT = Blueprint("lineage", __name__, url_prefix=settings.URL_PATH_PREFIX+settings.API_PREFIX)
 
@@ -67,26 +66,8 @@ def _get_lineage_info(search_condition):
     Raises:
         MindInsightException: If method fails to be called.
     """
-    summary_base_dir = str(settings.SUMMARY_BASE_DIR)
     try:
-        lineage_info = filter_summary_lineage(
-            data_manager=DATA_MANAGER,
-            search_condition=search_condition,
-            added=True)
-
-        lineages = lineage_info['object']
-
-        summary_base_dir = os.path.realpath(summary_base_dir)
-        length = len(summary_base_dir)
-
-        for lineage in lineages:
-            summary_dir = lineage['summary_dir']
-            summary_dir = os.path.realpath(summary_dir)
-            if summary_base_dir == summary_dir:
-                relative_dir = './'
-            else:
-                relative_dir = os.path.join(os.curdir, summary_dir[length+1:])
-            lineage['summary_dir'] = relative_dir
+        lineage_info = filter_summary_lineage(data_manager=DATA_MANAGER, search_condition=search_condition)
 
     except MindInsightException as exception:
         raise MindInsightException(exception.error, exception.message, http_code=400)
@@ -134,29 +115,27 @@ def get_dataset_graph():
         >>> GET http://xxxx/v1/mindinsight/datasets/dataset_graph?train_id=xxx
     """
 
-    summary_base_dir = str(settings.SUMMARY_BASE_DIR)
-    summary_dir = get_train_id(request)
+    train_id = get_train_id(request)
+    search_condition = {
+        'summary_dir': {
+            'in': [train_id]
+        }
+    }
+    result = {}
     try:
-        dataset_graph = get_summary_lineage(
-            DATA_MANAGER,
-            summary_dir=summary_dir,
-            keys=['dataset_graph']
-        )
+        objects = filter_summary_lineage(DATA_MANAGER, search_condition).get('object')
     except MindInsightException as exception:
         raise MindInsightException(exception.error, exception.message, http_code=400)
 
-    if dataset_graph:
-        summary_dir_result = dataset_graph.get('summary_dir')
-        base_dir_len = len(summary_base_dir)
-        if summary_base_dir == summary_dir_result:
-            relative_dir = './'
-        else:
-            relative_dir = os.path.join(
-                os.curdir, summary_dir[base_dir_len + 1:]
-            )
-        dataset_graph['summary_dir'] = relative_dir
-
-    return jsonify(dataset_graph)
+    if objects:
+        lineage_obj = objects[0]
+        dataset_graph = lineage_obj.get('dataset_graph')
+
+        if dataset_graph:
+            result.update({'dataset_graph': dataset_graph})
+            result.update({'summary_dir': lineage_obj.get('summary_dir')})
+
+    return jsonify(result)
 
 
 def init_module(app):

mindinsight/datavisual/data_transform/data_manager.py

@@ -210,6 +210,11 @@ class CachedTrainJob:
         """Threading lock with given key."""
         return self._key_locks.setdefault(key, threading.Lock())
 
+    @property
+    def train_id(self):
+        """Get train id."""
+        return self._basic_info.train_id
+
 
 class TrainJob:
     """

mindinsight/lineagemgr/cache_item_updater.py

@@ -72,12 +72,13 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater):
 
     def _lineage_parsing(self, cache_item):
         """Parse summaries and return lineage parser."""
+        train_id = cache_item.train_id
         summary_dir = cache_item.abs_summary_dir
         update_time = cache_item.basic_info.update_time
 
         cached_lineage_item = cache_item.get(key=LINEAGE, raise_exception=False)
         if cached_lineage_item is None:
-            lineage_parser = LineageParser(summary_dir, update_time)
+            lineage_parser = LineageParser(train_id, summary_dir, update_time)
         else:
             lineage_parser = cached_lineage_item
             with cache_item.lock_key(LINEAGE):

mindinsight/lineagemgr/common/exceptions/error_code.py

@@ -57,7 +57,6 @@ class LineageErrors(LineageErrorCodes):
     PARAM_STEP_NUM_ERROR = 18 | _MINDSPORE_COLLECTOR_ERROR
 
     # Model lineage error codes.
-    LINEAGE_PARAM_OPERATION_ERROR = 0 | _MODEL_LINEAGE_API_ERROR_MASK
     LINEAGE_PARAM_METRIC_ERROR = 1 | _MODEL_LINEAGE_API_ERROR_MASK
     LINEAGE_PARAM_LOSS_FUNCTION_ERROR = 4 | _MODEL_LINEAGE_API_ERROR_MASK
     LINEAGE_PARAM_TRAIN_DATASET_PATH_ERROR = 5 | _MODEL_LINEAGE_API_ERROR_MASK
@@ -79,7 +78,6 @@ class LineageErrors(LineageErrorCodes):
     LINEAGE_DIR_NOT_EXIST_ERROR = 20 | _MODEL_LINEAGE_API_ERROR_MASK
     LINEAGE_SUMMARY_DATA_ERROR = 21 | _MODEL_LINEAGE_API_ERROR_MASK
     LINEAGE_FILE_NOT_FOUND_ERROR = 22 | _MODEL_LINEAGE_API_ERROR_MASK
-    LINEAGE_PARAM_SUMMARY_PATH_ERROR = 23 | _MODEL_LINEAGE_API_ERROR_MASK
     LINEAGE_SEARCH_CONDITION_PARAM_ERROR = 24 | _MODEL_LINEAGE_API_ERROR_MASK
 
     LINEAGE_PARAM_LINEAGE_TYPE_ERROR = 25 | _MODEL_LINEAGE_API_ERROR_MASK
@@ -110,7 +108,6 @@ class LineageErrorMsg(Enum):
                                  "mindspore.train.summary.SummaryRecord"
     PARAM_RAISE_EXCEPTION_ERROR = "Invalid value for raise_exception. It should be True or False."
     # Lineage error messages.
-    LINEAGE_PARAM_SUMMARY_PATH_ERROR = "The parameter summary path error: {}"
     LINEAGE_SUMMARY_DATA_ERROR = "Query summary data error: {}"
     LINEAGE_FILE_NOT_FOUND_ERROR = "File not found error: {}"
     LINEAGE_DIR_NOT_EXIST_ERROR = "Dir not exist error: {}"

mindinsight/lineagemgr/common/exceptions/exceptions.py

@@ -106,16 +106,6 @@ class LineageEventFieldNotExistException(MindInsightException):
         )
 
 
-class LineageParamSummaryPathError(MindInsightException):
-    """The lineage parameter summary path error."""
-    def __init__(self, msg):
-        super(LineageParamSummaryPathError, self).__init__(
-            error=LineageErrors.LINEAGE_PARAM_SUMMARY_PATH_ERROR,
-            message=LineageErrorMsg.LINEAGE_PARAM_SUMMARY_PATH_ERROR.value.format(msg),
-            http_code=400
-        )
-
-
 class LineageQuerySummaryDataError(MindInsightException):
     """Query summary data error in lineage module."""
     def __init__(self, msg):
@@ -136,16 +126,6 @@ class LineageFileNotFoundError(MindInsightException):
         )
 
 
-class LineageDirNotExistError(MindInsightException):
-    """Directory not exist in lineage module."""
-    def __init__(self, msg):
-        super(LineageDirNotExistError, self).__init__(
-            error=LineageErrors.LINEAGE_DIR_NOT_EXIST_ERROR,
-            message=LineageErrorMsg.LINEAGE_DIR_NOT_EXIST_ERROR.value.format(msg),
-            http_code=400
-        )
-
-
 class LineageSearchConditionParamError(MindInsightException):
     """Search condition param is invalid in lineage module."""
     def __init__(self, msg):

mindinsight/lineagemgr/common/utils.py

@@ -15,30 +15,16 @@
 """Lineage utils."""
 import os
 import re
-from pathlib import Path
 
 from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
-from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError, LineageParamTypeError, \
-    LineageDirNotExistError, LineageParamSummaryPathError
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError
 from mindinsight.lineagemgr.common.log import logger as log
-from mindinsight.lineagemgr.common.validator.validate import validate_path
 
 
 def enum_to_list(enum):
     return [enum_ele.value for enum_ele in enum]
 
 
-def normalize_summary_dir(summary_dir):
-    """Normalize summary dir."""
-    try:
-        summary_dir = validate_path(summary_dir)
-    except (LineageParamValueError, LineageDirNotExistError) as error:
-        log.error(str(error))
-        log.exception(error)
-        raise LineageParamSummaryPathError(str(error.message))
-    return summary_dir
-
-
 def get_timestamp(filename):
     """Get timestamp from filename."""
     timestamp = int(re.search(SummaryWatcher().SUMMARY_FILENAME_REGEX, filename)[1])
@@ -68,25 +54,3 @@ def make_directory(path):
             log.error("No write permission on the directory(%r), error = %r", path, err)
             raise LineageParamTypeError("No write permission on the directory.")
     return real_path
-
-
-def get_relative_path(path, base_path):
-    """
-    Get relative path based on base_path.
-
-    Args:
-        path (str): absolute path.
-        base_path: absolute base path.
-
-    Returns:
-        str, relative path based on base_path.
-
-    """
-    try:
-        r_path = str(Path(path).relative_to(Path(base_path)))
-    except ValueError:
-        raise LineageParamValueError("The path %r does not start with %r." % (path, base_path))
-
-    if r_path == ".":
-        r_path = ""
-    return os.path.join("./", r_path)

mindinsight/lineagemgr/common/validator/validate.py

@@ -13,13 +13,11 @@
 # limitations under the License.
 # ============================================================================
 """Validate the parameters."""
-import os
 import re
 from marshmallow import ValidationError
 
 from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg
-from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, \
-    LineageParamValueError, LineageDirNotExistError
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, LineageParamValueError
 from mindinsight.lineagemgr.common.log import logger as log
 from mindinsight.lineagemgr.common.validator.validate_path import safe_normalize_path
 from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING
@@ -224,38 +222,6 @@ def validate_raise_exception(raise_exception):
         )
 
 
-def validate_filter_key(keys):
-    """
-    Verify the keys of filtering is valid or not.
-
-    Args:
-        keys (list): The keys to get the relative lineage info.
-
-    Raises:
-        LineageParamTypeError: If keys is not list.
-        LineageParamValueError: If the value of keys is invalid.
-    """
-    filter_keys = [
-        'metric', 'hyper_parameters', 'algorithm',
-        'train_dataset', 'model', 'valid_dataset',
-        'dataset_graph'
-    ]
-
-    if not isinstance(keys, list):
-        log.error("Keys must be list.")
-        raise LineageParamTypeError("Keys must be list.")
-
-    for element in keys:
-        if not isinstance(element, str):
-            log.error("Element of keys must be str.")
-            raise LineageParamTypeError("Element of keys must be str.")
-
-    if not set(keys).issubset(filter_keys):
-        err_msg = "Keys must be in {}.".format(filter_keys)
-        log.error(err_msg)
-        raise LineageParamValueError(err_msg)
-
-
 def validate_condition(search_condition):
     """
     Verify the param in search_condition is valid or not.
@@ -310,31 +276,6 @@ def validate_condition(search_condition):
             raise LineageParamValueError(err_msg)
 
 
-def validate_path(summary_path):
-    """
-    Verify the summary path is valid or not.
-
-    Args:
-        summary_path (str): The summary path which is a dir.
-
-    Raises:
-        LineageParamValueError: If the input param value is invalid.
-        LineageDirNotExistError: If the summary path is invalid.
-    """
-    try:
-        summary_path = safe_normalize_path(
-            summary_path, "summary_path", None, check_absolute_path=True
-        )
-    except ValidationError:
-        log.error("The summary path is invalid.")
-        raise LineageParamValueError("The summary path is invalid.")
-    if not os.path.isdir(summary_path):
-        log.error("The summary path does not exist or is not a dir.")
-        raise LineageDirNotExistError("The summary path does not exist or is not a dir.")
-
-    return summary_path
-
-
 def validate_user_defined_info(user_defined_info):
     """
     Validate user defined info， delete the item if its key is in lineage.

mindinsight/lineagemgr/lineage_parser.py

@@ -15,7 +15,6 @@
 """This file is used to parse lineage info."""
 import os
 
-from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
 from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSummaryAnalyzeException, \
     LineageEventNotExistException, LineageEventFieldNotExistException, LineageFileNotFoundError, \
     MindInsightException
@@ -64,8 +63,9 @@ class SuperLineageObj:
 
 class LineageParser:
     """Lineage parser."""
-    def __init__(self, summary_dir, update_time=None, added_info=None):
+    def __init__(self, train_id, summary_dir, update_time=None, added_info=None):
         self._summary_dir = summary_dir
+        self._train_id = train_id
         self._update_time = update_time
         self._added_info = added_info
 
@@ -154,7 +154,7 @@ class LineageParser:
         """Update lineage object."""
         if self._super_lineage_obj is None:
             lineage_obj = LineageObj(
-                self._summary_dir,
+                self._train_id,
                 train_lineage=lineage_info.train_lineage,
                 evaluation_lineage=lineage_info.eval_lineage,
                 dataset_graph=lineage_info.dataset_graph,
@@ -177,52 +177,13 @@ class LineageParser:
 
 class LineageOrganizer:
     """Lineage organizer."""
-    def __init__(self, data_manager=None, summary_base_dir=None):
+    def __init__(self, data_manager):
         self._data_manager = data_manager
-        self._summary_base_dir = summary_base_dir
-        self._check_params()
         self._super_lineage_objs = {}
         self._organize_from_cache()
-        self._organize_from_disk()
-
-    def _check_params(self):
-        """Check params."""
-        if self._data_manager is not None and self._summary_base_dir is not None:
-            self._summary_base_dir = None
-
-    def _organize_from_disk(self):
-        """Organize lineage objs from disk."""
-        if self._summary_base_dir is None:
-            return
-        summary_watcher = SummaryWatcher()
-        relative_dirs = summary_watcher.list_summary_directories(
-            summary_base_dir=self._summary_base_dir
-        )
-
-        no_lineage_count = 0
-        for item in relative_dirs:
-            relative_dir = item.get('relative_path')
-            update_time = item.get('update_time')
-            abs_summary_dir = os.path.realpath(os.path.join(self._summary_base_dir, relative_dir))
-
-            try:
-                lineage_parser = LineageParser(abs_summary_dir, update_time)
-                super_lineage_obj = lineage_parser.super_lineage_obj
-                if super_lineage_obj is not None:
-                    self._super_lineage_objs.update({abs_summary_dir: super_lineage_obj})
-            except LineageFileNotFoundError:
-                no_lineage_count += 1
-
-        if no_lineage_count == len(relative_dirs):
-            logger.info('There is no summary log file under summary_base_dir.')
-            raise LineageFileNotFoundError(
-                'There is no summary log file under summary_base_dir.'
-            )
 
     def _organize_from_cache(self):
         """Organize lineage objs from cache."""
-        if self._data_manager is None:
-            return
         brief_cache = self._data_manager.get_brief_cache()
         cache_items = brief_cache.cache_items
         for relative_dir, cache_train_job in cache_items.items():

mindinsight/lineagemgr/model.py

@@ -13,20 +13,15 @@
 # limitations under the License.
 # ============================================================================
 """This file is used to define the model lineage python api."""
-import os
 import numpy as np
 import pandas as pd
 
-from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError, \
-    LineageQuerySummaryDataError, LineageParamSummaryPathError, \
-    LineageQuerierParamException, LineageDirNotExistError, LineageSearchConditionParamError, \
-    LineageParamTypeError, LineageSummaryParseException
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageQuerySummaryDataError, \
+    LineageQuerierParamException, LineageSearchConditionParamError, LineageParamTypeError, LineageSummaryParseException
 from mindinsight.lineagemgr.common.log import logger as log
-from mindinsight.lineagemgr.common.utils import normalize_summary_dir, get_relative_path
 from mindinsight.lineagemgr.common.validator.model_parameter import SearchModelConditionParameter
-from mindinsight.lineagemgr.common.validator.validate import validate_filter_key, validate_search_model_condition, \
-    validate_condition, validate_path, validate_train_id
-from mindinsight.lineagemgr.lineage_parser import LineageParser, LineageOrganizer
+from mindinsight.lineagemgr.common.validator.validate import validate_search_model_condition, validate_condition
+from mindinsight.lineagemgr.lineage_parser import LineageOrganizer
 from mindinsight.lineagemgr.querier.querier import Querier
 from mindinsight.optimizer.common.enums import ReasonCode
 from mindinsight.optimizer.utils.utils import is_simple_numpy_number
@@ -38,62 +33,7 @@ _USER_DEFINED_PREFIX = "[U]"
 USER_DEFINED_INFO_LIMIT = 100
 
 
-def get_summary_lineage(data_manager=None, summary_dir=None, keys=None):
-    """
-    Get summary lineage from data_manager or parsing from summaries.
-
-    One of data_manager or summary_dir needs to be specified. Support getting
-    super_lineage_obj from data_manager or parsing summaries by summary_dir.
-
-    Args:
-        data_manager (DataManager): Data manager defined as
-            mindinsight.datavisual.data_transform.data_manager.DataManager
-        summary_dir (str): The summary directory. It contains summary logs for
-            one training.
-        keys (list[str]): The filter keys of lineage information. The acceptable
-            keys are `metric`, `user_defined`, `hyper_parameters`, `algorithm`,
-            `train_dataset`, `model`, `valid_dataset` and `dataset_graph`.
-            If it is `None`, all information will be returned. Default: None.
-
-    Returns:
-        dict, the lineage information for one training.
-
-    Raises:
-        LineageParamSummaryPathError: If summary path is invalid.
-        LineageQuerySummaryDataError: If querying summary data fails.
-        LineageFileNotFoundError: If the summary log file is not found.
-
-    """
-    default_result = {}
-    if data_manager is None and summary_dir is None:
-        raise LineageParamTypeError("One of data_manager or summary_dir needs to be specified.")
-    if data_manager is not None and summary_dir is None:
-        raise LineageParamTypeError("If data_manager is specified, the summary_dir needs to be "
-                                    "specified as relative path.")
-
-    if keys is not None:
-        validate_filter_key(keys)
-
-    if data_manager is None:
-        normalize_summary_dir(summary_dir)
-        super_lineage_obj = LineageParser(summary_dir).super_lineage_obj
-    else:
-        validate_train_id(summary_dir)
-        super_lineage_obj = LineageOrganizer(data_manager=data_manager).get_super_lineage_obj(summary_dir)
-
-    if super_lineage_obj is None:
-        return default_result
-
-    try:
-        result = Querier({summary_dir: super_lineage_obj}).get_summary_lineage(summary_dir, keys)
-    except (LineageQuerierParamException, LineageParamTypeError) as error:
-        log.error(str(error))
-        log.exception(error)
-        raise LineageQuerySummaryDataError("Get summary lineage failed.")
-    return result[0]
-
-
-def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_condition=None, added=False):
+def filter_summary_lineage(data_manager, search_condition=None):
     """
     Filter summary lineage from data_manager or parsing from summaries.
 
@@ -103,18 +43,8 @@ def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_cond
     Args:
         data_manager (DataManager): Data manager defined as
             mindinsight.datavisual.data_transform.data_manager.DataManager
-        summary_base_dir (str): The summary base directory. It contains summary
-            directories generated by training.
         search_condition (dict): The search condition.
     """
-    if data_manager is None and summary_base_dir is None:
-        raise LineageParamTypeError("One of data_manager or summary_base_dir needs to be specified.")
-
-    if data_manager is None:
-        summary_base_dir = normalize_summary_dir(summary_base_dir)
-    else:
-        summary_base_dir = data_manager.summary_base_dir
-
     search_condition = {} if search_condition is None else search_condition
 
     try:
@@ -126,18 +56,8 @@ def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_cond
         raise LineageSearchConditionParamError(str(error.message))
 
     try:
-        search_condition = _convert_relative_path_to_abspath(summary_base_dir, search_condition)
-    except (LineageParamValueError, LineageDirNotExistError) as error:
-        log.error(str(error))
-        log.exception(error)
-        raise LineageParamSummaryPathError(str(error.message))
-
-    try:
-        lineage_objects = LineageOrganizer(data_manager, summary_base_dir).super_lineage_objs
-        result = Querier(lineage_objects).filter_summary_lineage(
-            condition=search_condition,
-            added=added
-        )
+        lineage_objects = LineageOrganizer(data_manager).super_lineage_objs
+        result = Querier(lineage_objects).filter_summary_lineage(condition=search_condition)
     except LineageSummaryParseException:
         result = {'object': [], 'count': 0}
     except (LineageQuerierParamException, LineageParamTypeError) as error:
@@ -148,53 +68,6 @@ def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_cond
     return result
 
 
-def _convert_relative_path_to_abspath(summary_base_dir, search_condition):
-    """
-    Convert relative path to absolute path.
-
-    Args:
-        summary_base_dir (str): The summary base directory.
-        search_condition (dict): The search condition.
-
-    Returns:
-        dict, the updated search_condition.
-
-    Raises:
-        LineageParamValueError: If the value of input_name is invalid.
-    """
-    if ("summary_dir" not in search_condition) or (not search_condition.get("summary_dir")):
-        return search_condition
-
-    summary_dir_condition = search_condition.get("summary_dir")
-
-    for key in ['in', 'not_in']:
-        if key in summary_dir_condition:
-            summary_paths = []
-            for summary_dir in summary_dir_condition.get(key):
-                if summary_dir.startswith('./'):
-                    abs_dir = os.path.join(
-                        summary_base_dir, summary_dir[2:]
-                    )
-                    abs_dir = validate_path(abs_dir)
-                else:
-                    abs_dir = validate_path(summary_dir)
-                summary_paths.append(abs_dir)
-            search_condition.get('summary_dir')[key] = summary_paths
-
-    if 'eq' in summary_dir_condition:
-        summary_dir = summary_dir_condition.get('eq')
-        if summary_dir.startswith('./'):
-            abs_dir = os.path.join(
-                summary_base_dir, summary_dir[2:]
-            )
-            abs_dir = validate_path(abs_dir)
-        else:
-            abs_dir = validate_path(summary_dir)
-        search_condition.get('summary_dir')['eq'] = abs_dir
-
-    return search_condition
-
-
 def get_flattened_lineage(data_manager, search_condition=None):
     """
     Get lineage data in a table from data manager.
@@ -207,10 +80,10 @@ def get_flattened_lineage(data_manager, search_condition=None):
         Dict[str, list]: A dict contains keys and values from lineages.
 
     """
-    summary_base_dir, flatten_dict, user_count = data_manager.summary_base_dir, {'train_id': []}, 0
+    flatten_dict, user_count = {'train_id': []}, 0
     lineages = filter_summary_lineage(data_manager=data_manager, search_condition=search_condition).get("object", [])
     for index, lineage in enumerate(lineages):
-        flatten_dict['train_id'].append(get_relative_path(lineage.get("summary_dir"), summary_base_dir))
+        flatten_dict['train_id'].append(lineage.get("summary_dir"))
         for key, val in _flatten_lineage(lineage.get('model_lineage', {})):
             if key.startswith(_USER_DEFINED_PREFIX) and key not in flatten_dict:
                 if user_count > USER_DEFINED_INFO_LIMIT:

mindinsight/lineagemgr/querier/querier.py

@@ -189,53 +189,7 @@ class Querier:
             raise LineageParamTypeError("Init param should be a dict.")
         return super_lineage_objs
 
-    def get_summary_lineage(self, summary_dir=None, filter_keys=None):
-        """
-        Get summary lineage information.
-
-        If a summary dir is specified, the special summary lineage information
-        will be found. If the summary dir is `None`, all summary lineage
-        information will be found.
-
-        Returns the content corresponding to the specified field in the filter
-        key. The contents of the filter key include `metric`, `hyper_parameters`,
-        `algorithm`, `train_dataset`, `valid_dataset` and `model`. You can
-        specify multiple filter keys in the `filter_keys`. If the parameter is
-        `None`, complete information will be returned.
-
-        Args:
-            summary_dir (Union[str, None]): Summary log dir. Default: None.
-            filter_keys (Union[list[str], None]): Filter keys. Default: None.
-
-        Returns:
-            list[dict], summary lineage information.
-        """
-
-        if filter_keys is None:
-            filter_keys = LineageFilterKey.get_key_list()
-        else:
-            for key in filter_keys:
-                if not LineageFilterKey.is_valid_filter_key(key):
-                    raise LineageQuerierParamException(
-                        filter_keys, 'The filter key {} is invalid.'.format(key)
-                    )
-
-        if summary_dir is None:
-            result = [
-                item.lineage_obj.get_summary_info(filter_keys) for item in self._super_lineage_objs.values()
-            ]
-        elif summary_dir in self._super_lineage_objs:
-            lineage_obj = self._super_lineage_objs[summary_dir].lineage_obj
-            result = [lineage_obj.get_summary_info(filter_keys)]
-        else:
-            raise LineageQuerierParamException(
-                'summary_dir',
-                'Summary dir {} does not exist.'.format(summary_dir)
-            )
-
-        return result
-
-    def filter_summary_lineage(self, condition=None, added=False):
+    def filter_summary_lineage(self, condition=None):
         """
         Filter and sort lineage information based on the specified condition.
 
@@ -298,8 +252,7 @@ class Querier:
                 lineage_object.update(item.lineage_obj.to_model_lineage_dict())
             if LineageType.DATASET.value in lineage_types:
                 lineage_object.update(item.lineage_obj.to_dataset_lineage_dict())
-            if added:
-                lineage_object.update({"added_info": item.added_info})
+            lineage_object.update({"added_info": item.added_info})
             object_items.append(lineage_object)
 
         lineage_info = {

tests/st/func/lineagemgr/cache/__init__.py

@@ -1,14 +0,0 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================

tests/st/func/lineagemgr/cache/test_lineage_cache.py

@@ -1,100 +0,0 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-
-"""
-Function:
-    Test the query module about lineage information.
-Usage:
-    The query module test should be run after lineagemgr/collection/model/test_model_lineage.py
-    pytest lineagemgr
-"""
-from unittest import TestCase
-
-import pytest
-
-from mindinsight.datavisual.data_transform.data_manager import DataManager
-from mindinsight.lineagemgr.cache_item_updater import LineageCacheItemUpdater
-from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage
-
-from ..test_model import LINEAGE_INFO_RUN1, LINEAGE_FILTRATION_EXCEPT_RUN, \
-    LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2
-from ..conftest import BASE_SUMMARY_DIR
-from .....ut.lineagemgr.querier import event_data
-from .....utils.tools import assert_equal_lineages
-
-
-@pytest.mark.usefixtures("create_summary_dir")
-class TestModelApi(TestCase):
-    """Test get lineage from data_manager."""
-    @classmethod
-    def setup_class(cls):
-        data_manager = DataManager(BASE_SUMMARY_DIR)
-        data_manager.register_brief_cache_item_updater(LineageCacheItemUpdater())
-        data_manager.start_load_data().join()
-
-        cls._data_manger = data_manager
-
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_get_summary_lineage(self):
-        """Test the interface of get_summary_lineage."""
-        total_res = get_summary_lineage(data_manager=self._data_manger, summary_dir="./run1")
-        expect_total_res = LINEAGE_INFO_RUN1
-        assert_equal_lineages(expect_total_res, total_res, self.assertDictEqual)
-
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_filter_summary_lineage(self):
-        """Test the interface of filter_summary_lineage."""
-        expect_result = {
-            'customized': event_data.CUSTOMIZED__1,
-            'object': [
-                LINEAGE_FILTRATION_EXCEPT_RUN,
-                LINEAGE_FILTRATION_RUN1,
-                LINEAGE_FILTRATION_RUN2
-            ],
-            'count': 3
-        }
-
-        search_condition = {
-            'sorted_name': 'summary_dir'
-        }
-        res = filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition)
-        expect_objects = expect_result.get('object')
-        for idx, res_object in enumerate(res.get('object')):
-            expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
-        assert_equal_lineages(expect_result, res, self.assertDictEqual)
-
-        expect_result = {
-            'customized': {},
-            'object': [],
-            'count': 0
-        }
-
-        search_condition = {
-            'summary_dir': {
-                "in": ['./dir_with_empty_lineage']
-            }
-        }
-        res = filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition)
-        assert_equal_lineages(expect_result, res, self.assertDictEqual)

tests/st/func/lineagemgr/collection/model/test_model_lineage.py

@@ -27,7 +27,7 @@ from unittest import TestCase, mock
 
 import numpy as np
 import pytest
-from mindinsight.lineagemgr.model import get_summary_lineage
+from mindinsight.lineagemgr.model import filter_summary_lineage
 from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors
 from mindinsight.utils.exceptions import MindInsightException
 
@@ -37,10 +37,11 @@ from mindspore.dataset.engine import MindDataset
 from mindspore.nn import Momentum, SoftmaxCrossEntropyWithLogits, WithLossCell
 from mindspore.train.callback import ModelCheckpoint, RunContext, SummaryStep, _ListCallback
 from mindspore.train.summary import SummaryRecord
-from tests.utils.lineage_writer.model_lineage import AnalyzeObject, EvalLineage, TrainLineage
 
-from ...conftest import SUMMARY_DIR, SUMMARY_DIR_2, SUMMARY_DIR_3
 from .train_one_step import TrainOneStep
+from ...conftest import SUMMARY_DIR, SUMMARY_DIR_2, SUMMARY_DIR_3, BASE_SUMMARY_DIR, LINEAGE_DATA_MANAGER
+from ......utils.lineage_writer.model_lineage import AnalyzeObject, EvalLineage, TrainLineage
+from ......utils.tools import get_relative_path
 
 
 @pytest.mark.usefixtures("create_summary_dir")
@@ -76,6 +77,13 @@ class TestModelLineage(TestCase):
             "version": "v1"
         }
 
+        train_id = get_relative_path(SUMMARY_DIR, BASE_SUMMARY_DIR)
+        cls._search_condition = {
+            'summary_dir': {
+                'eq': train_id
+            }
+        }
+
     @pytest.mark.scene_train(2)
     @pytest.mark.level0
     @pytest.mark.platform_arm_ascend_training
@@ -90,13 +98,17 @@ class TestModelLineage(TestCase):
         train_callback = TrainLineage(SUMMARY_DIR, True, self.user_defined_info)
         train_callback.initial_learning_rate = 0.12
         train_callback.end(RunContext(self.run_context))
-        res = get_summary_lineage(summary_dir=SUMMARY_DIR)
-        assert res.get('hyper_parameters', {}).get('epoch') == 10
+
+        LINEAGE_DATA_MANAGER.start_load_data().join()
+        res = filter_summary_lineage(LINEAGE_DATA_MANAGER, self._search_condition)
+        assert res.get('object')[0].get('model_lineage', {}).get('epoch') == 10
         run_context = self.run_context
         run_context['epoch_num'] = 14
         train_callback.end(RunContext(run_context))
-        res = get_summary_lineage(summary_dir=SUMMARY_DIR)
-        assert res.get('hyper_parameters', {}).get('epoch') == 14
+
+        LINEAGE_DATA_MANAGER.start_load_data().join()
+        res = filter_summary_lineage(LINEAGE_DATA_MANAGER, self._search_condition)
+        assert res.get('object')[0].get('model_lineage', {}).get('epoch') == 14
 
     @pytest.mark.scene_eval(3)
     @pytest.mark.level0
@@ -186,12 +198,13 @@ class TestModelLineage(TestCase):
         run_context_customized['train_network'] = net
         train_callback.begin(RunContext(run_context_customized))
         train_callback.end(RunContext(run_context_customized))
-        res = get_summary_lineage(summary_dir=SUMMARY_DIR)
-        assert res.get('hyper_parameters', {}).get('loss_function') \
-               == 'SoftmaxCrossEntropyWithLogits'
-        assert res.get('algorithm', {}).get('network') == 'ResNet'
-        assert res.get('hyper_parameters', {}).get('optimizer') == 'Momentum'
 
+        LINEAGE_DATA_MANAGER.start_load_data().join()
+        res = filter_summary_lineage(LINEAGE_DATA_MANAGER, self._search_condition)
+        assert res.get('object')[0].get('model_lineage', {}).get('loss_function') \
+               == 'SoftmaxCrossEntropyWithLogits'
+        assert res.get('object')[0].get('model_lineage', {}).get('network') == 'ResNet'
+        assert res.get('object')[0].get('model_lineage', {}).get('optimizer') == 'Momentum'
 
     @pytest.mark.scene_exception(1)
     @pytest.mark.level0

tests/st/func/lineagemgr/conftest.py

@@ -21,6 +21,9 @@ import tempfile
 
 import pytest
 
+from mindinsight.datavisual.data_transform.data_manager import DataManager
+from mindinsight.lineagemgr.cache_item_updater import LineageCacheItemUpdater
+
 from ....utils import mindspore
 from ....utils.mindspore.dataset.engine.serializer_deserializer import SERIALIZED_PIPELINE
 
@@ -31,6 +34,9 @@ SUMMARY_DIR = os.path.join(BASE_SUMMARY_DIR, 'run1')
 SUMMARY_DIR_2 = os.path.join(BASE_SUMMARY_DIR, 'run2')
 SUMMARY_DIR_3 = os.path.join(BASE_SUMMARY_DIR, 'except_run')
 
+LINEAGE_DATA_MANAGER = DataManager(BASE_SUMMARY_DIR)
+LINEAGE_DATA_MANAGER.register_brief_cache_item_updater(LineageCacheItemUpdater())
+
 COLLECTION_MODULE = 'TestModelLineage'
 API_MODULE = 'TestModelApi'
 DATASET_GRAPH = SERIALIZED_PIPELINE

tests/st/func/lineagemgr/test_model.py

@@ -24,20 +24,18 @@ import os
 from unittest import TestCase
 import pytest
 
-from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage
-from mindinsight.lineagemgr.common.exceptions.exceptions import (LineageFileNotFoundError, LineageParamSummaryPathError,
-                                                                 LineageParamTypeError, LineageParamValueError,
-                                                                 LineageSearchConditionParamError)
+from mindinsight.lineagemgr.model import filter_summary_lineage
 from mindinsight.datavisual.data_transform import data_manager
 from mindinsight.lineagemgr.cache_item_updater import LineageCacheItemUpdater
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSearchConditionParamError
 from mindinsight.lineagemgr.model import get_flattened_lineage
 
-from .conftest import BASE_SUMMARY_DIR, DATASET_GRAPH, SUMMARY_DIR, SUMMARY_DIR_2
+from .conftest import BASE_SUMMARY_DIR, DATASET_GRAPH, SUMMARY_DIR, SUMMARY_DIR_2, LINEAGE_DATA_MANAGER
 from ....ut.lineagemgr.querier import event_data
-from ....utils.tools import assert_equal_lineages
+from ....utils.tools import assert_equal_lineages, get_relative_path
 
 LINEAGE_INFO_RUN1 = {
-    'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
+    'summary_dir': './run1',
     'metric': {
         'accuracy': 0.78
     },
@@ -67,7 +65,7 @@ LINEAGE_INFO_RUN1 = {
     'dataset_graph': DATASET_GRAPH
 }
 LINEAGE_FILTRATION_EXCEPT_RUN = {
-    'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'),
+    'summary_dir': './except_run',
     'model_lineage': {
         'loss_function': 'SoftmaxCrossEntropyWithLogits',
         'train_dataset_path': None,
@@ -86,10 +84,11 @@ LINEAGE_FILTRATION_EXCEPT_RUN = {
         'metric': {},
         'dataset_mark': 2
     },
-    'dataset_graph': DATASET_GRAPH
+    'dataset_graph': DATASET_GRAPH,
+    'added_info': {}
 }
 LINEAGE_FILTRATION_RUN1 = {
-    'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
+    'summary_dir': './run1',
     'model_lineage': {
         'loss_function': 'SoftmaxCrossEntropyWithLogits',
         'train_dataset_path': None,
@@ -114,10 +113,11 @@ LINEAGE_FILTRATION_RUN1 = {
         },
         'dataset_mark': 2
     },
-    'dataset_graph': DATASET_GRAPH
+    'dataset_graph': DATASET_GRAPH,
+    'added_info': {}
 }
 LINEAGE_FILTRATION_RUN2 = {
-    'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
+    'summary_dir': './run2',
     'model_lineage': {
         'loss_function': "SoftmaxCrossEntropyWithLogits",
         'train_dataset_path': None,
@@ -138,7 +138,8 @@ LINEAGE_FILTRATION_RUN2 = {
         },
         'dataset_mark': 3
     },
-    'dataset_graph': DATASET_GRAPH
+    'dataset_graph': DATASET_GRAPH,
+    'added_info': {}
 }
 
 
@@ -149,12 +150,12 @@ class TestModelApi(TestCase):
     @classmethod
     def setup_class(cls):
         """The initial process."""
-        cls.dir_with_empty_lineage = os.path.join(
+        cls._dir_with_empty_lineage = os.path.join(
             BASE_SUMMARY_DIR, 'dir_with_empty_lineage'
         )
-        os.makedirs(cls.dir_with_empty_lineage)
+        os.makedirs(cls._dir_with_empty_lineage)
         ms_file = os.path.join(
-            cls.dir_with_empty_lineage, 'empty.summary.1581499502.bms_MS'
+            cls._dir_with_empty_lineage, 'empty.summary.1581499502.bms_MS'
         )
         lineage_file = ms_file + '_lineage'
         with open(ms_file, mode='w'):
@@ -162,156 +163,11 @@ class TestModelApi(TestCase):
         with open(lineage_file, mode='w'):
             pass
 
-        cls.empty_dir = os.path.join(BASE_SUMMARY_DIR, 'empty_dir')
-        os.makedirs(cls.empty_dir)
-
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_get_summary_lineage(self):
-        """Test the interface of get_summary_lineage."""
-        total_res = get_summary_lineage(None, SUMMARY_DIR)
-        partial_res1 = get_summary_lineage(None, SUMMARY_DIR, ['hyper_parameters'])
-        partial_res2 = get_summary_lineage(None, SUMMARY_DIR, ['metric', 'algorithm'])
-        expect_total_res = LINEAGE_INFO_RUN1
-        expect_partial_res1 = {
-            'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
-            'hyper_parameters': {
-                'optimizer': 'Momentum',
-                'learning_rate': 0.12,
-                'loss_function': 'SoftmaxCrossEntropyWithLogits',
-                'epoch': 14,
-                'parallel_mode': 'stand_alone',
-                'device_num': 2,
-                'batch_size': 32
-            }
-        }
-        expect_partial_res2 = {
-            'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
-            'metric': {
-                'accuracy': 0.78
-            },
-            'algorithm': {
-                'network': 'ResNet'
-            }
-        }
-        assert_equal_lineages(expect_total_res, total_res, self.assertDictEqual)
-        assert_equal_lineages(expect_partial_res1, partial_res1, self.assertDictEqual)
-        assert_equal_lineages(expect_partial_res2, partial_res2, self.assertDictEqual)
-
-        # the lineage summary file is empty
-        result = get_summary_lineage(None, self.dir_with_empty_lineage)
-        assert {} == result
-
-        # keys is empty list
-        expect_result = {
-            'summary_dir': SUMMARY_DIR
-        }
-        result = get_summary_lineage(None, SUMMARY_DIR, [])
-        assert expect_result == result
-
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_get_summary_lineage_exception_1(self):
-        """Test the interface of get_summary_lineage with exception."""
-        # summary path does not exist
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path does not exist or is not a dir.',
-            get_summary_lineage,
-            None,
-            '/tmp/fake/dir'
-        )
-
-        # summary path is relative path
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            get_summary_lineage,
-            None,
-            'tmp'
-        )
-
-        # the type of input param is invalid
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            get_summary_lineage,
-            None,
-            ['/root/linage1', '/root/lineage2']
-        )
-
-        # summary path is empty str
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            get_summary_lineage,
-            None,
-            '',
-            keys=None
-        )
+        empty_dir = os.path.join(BASE_SUMMARY_DIR, 'empty_dir')
+        os.makedirs(empty_dir)
+        cls._empty_train_id = get_relative_path(empty_dir, LINEAGE_DATA_MANAGER.summary_base_dir)
 
-        # summary path invalid
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            get_summary_lineage,
-            None,
-            '\\',
-            keys=None
-        )
-
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_get_summary_lineage_exception_2(self):
-        """Test the interface of get_summary_lineage with exception."""
-        # keys is invalid
-        self.assertRaisesRegex(
-            LineageParamValueError,
-            'Keys must be in',
-            get_summary_lineage,
-            None,
-            SUMMARY_DIR,
-            ['metric', 'fake_name']
-        )
-
-        self.assertRaisesRegex(
-            LineageParamTypeError,
-            'Element of keys must be str.',
-            get_summary_lineage,
-            None,
-            SUMMARY_DIR,
-            [1, 2, 3]
-        )
-
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_get_summary_lineage_exception_3(self):
-        """Test the interface of get_summary_lineage with exception."""
-        for keys in [0, 0.1, True, (3, 4), {'a': 'b'}]:
-            self.assertRaisesRegex(
-                LineageParamTypeError,
-                'Keys must be list.',
-                get_summary_lineage,
-                None,
-                SUMMARY_DIR,
-                keys
-            )
+        LINEAGE_DATA_MANAGER.start_load_data().join()
 
     @pytest.mark.level0
     @pytest.mark.platform_arm_ascend_training
@@ -334,18 +190,7 @@ class TestModelApi(TestCase):
         search_condition = {
             'sorted_name': 'summary_dir'
         }
-        res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
-        expect_objects = expect_result.get('object')
-        for idx, res_object in enumerate(res.get('object')):
-            expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
-        assert_equal_lineages(expect_result, res, self.assertDictEqual)
-
-        expect_result = {
-            'customized': {},
-            'object': [],
-            'count': 0
-        }
-        res = filter_summary_lineage(None, self.dir_with_empty_lineage)
+        res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -362,8 +207,8 @@ class TestModelApi(TestCase):
         search_condition = {
             'summary_dir': {
                 'in': [
-                    SUMMARY_DIR,
-                    SUMMARY_DIR_2
+                    get_relative_path(SUMMARY_DIR, LINEAGE_DATA_MANAGER.summary_base_dir),
+                    get_relative_path(SUMMARY_DIR_2, LINEAGE_DATA_MANAGER.summary_base_dir)
                 ]
             },
             'metric/accuracy': {
@@ -383,7 +228,7 @@ class TestModelApi(TestCase):
             ],
             'count': 2
         }
-        partial_res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
+        partial_res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(partial_res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -421,7 +266,7 @@ class TestModelApi(TestCase):
             ],
             'count': 2
         }
-        partial_res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
+        partial_res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(partial_res.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -450,7 +295,7 @@ class TestModelApi(TestCase):
             ],
             'count': 3
         }
-        partial_res1 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition1)
+        partial_res1 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition1)
         expect_objects = expect_result.get('object')
         for idx, res_object in enumerate(partial_res1.get('object')):
             expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
@@ -469,9 +314,30 @@ class TestModelApi(TestCase):
             'object': [],
             'count': 0
         }
-        partial_res2 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition2)
+        partial_res2 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition2)
         assert expect_result == partial_res2
 
+    @pytest.mark.level0
+    @pytest.mark.platform_arm_ascend_training
+    @pytest.mark.platform_x86_gpu_training
+    @pytest.mark.platform_x86_ascend_training
+    @pytest.mark.platform_x86_cpu
+    @pytest.mark.env_single
+    def test_filter_summary_lineage_condition_4(self):
+        """Test the abnormal execution of the filter_summary_lineage interface."""
+        expect_result = {
+            'customized': {},
+            'object': [],
+            'count': 0
+        }
+        search_condition = {
+            'summary_dir': {
+                'eq': self._empty_train_id
+            }
+        }
+        res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition)
+        assert expect_result == res
+
     @pytest.mark.level0
     @pytest.mark.platform_arm_ascend_training
     @pytest.mark.platform_x86_gpu_training
@@ -480,10 +346,10 @@ class TestModelApi(TestCase):
     @pytest.mark.env_single
     def test_filter_summary_lineage_with_lineage_type(self):
         """Test the interface of filter_summary_lineage with lineage_type."""
-        summary_dir = os.path.join(BASE_SUMMARY_DIR, 'except_run')
+        train_id = './except_run'
         search_condition = {
             'summary_dir': {
-                'in': [summary_dir]
+                'in': [train_id]
             },
             'lineage_type': {
                 'eq': 'dataset'
@@ -493,50 +359,16 @@ class TestModelApi(TestCase):
             'customized': {},
             'object': [
                 {
-                    'summary_dir': summary_dir,
-                    'dataset_graph': DATASET_GRAPH
+                    'summary_dir': train_id,
+                    'dataset_graph': DATASET_GRAPH,
+                    'added_info': {}
                 }
             ],
             'count': 1
         }
-        res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition)
+        res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition)
         assert expect_result == res
 
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_filter_summary_lineage_exception_1(self):
-        """Test the abnormal execution of the filter_summary_lineage interface."""
-        # summary base dir is relative path
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            filter_summary_lineage,
-            None,
-            'relative_path'
-        )
-
-        # summary base dir does not exist
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path does not exist or is not a dir.',
-            filter_summary_lineage,
-            None,
-            '/path/does/not/exist'
-        )
-
-        # no summary log file under summary_base_dir
-        self.assertRaisesRegex(
-            LineageFileNotFoundError,
-            'There is no summary log file under summary_base_dir.',
-            filter_summary_lineage,
-            None,
-            self.empty_dir
-        )
-
     @pytest.mark.level0
     @pytest.mark.platform_arm_ascend_training
     @pytest.mark.platform_x86_gpu_training
@@ -553,8 +385,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The search_condition element summary_dir should be dict.',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -566,8 +397,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The sorted_name have to exist when sorted_type exists.',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -577,8 +407,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'Invalid search_condition type, it should be dict.',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -590,8 +419,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The limit must be int.',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -611,8 +439,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The offset must be int.',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -626,8 +453,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The search attribute not supported.',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -648,8 +474,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The sorted_type must be ascending or descending',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -663,8 +488,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The compare condition should be in',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -678,8 +502,7 @@ class TestModelApi(TestCase):
             LineageSearchConditionParamError,
             'The parameter metric/accuracy is invalid.',
             filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
+            LINEAGE_DATA_MANAGER,
             search_condition
         )
 
@@ -690,31 +513,6 @@ class TestModelApi(TestCase):
     @pytest.mark.platform_x86_cpu
     @pytest.mark.env_single
     def test_filter_summary_lineage_exception_5(self):
-        """Test the abnormal execution of the filter_summary_lineage interface."""
-        # the summary dir is invalid in search condition
-        search_condition = {
-            'summary_dir': {
-                'in': [
-                    'xxx'
-                ]
-            }
-        }
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            filter_summary_lineage,
-            None,
-            BASE_SUMMARY_DIR,
-            search_condition
-        )
-
-    @pytest.mark.level0
-    @pytest.mark.platform_arm_ascend_training
-    @pytest.mark.platform_x86_gpu_training
-    @pytest.mark.platform_x86_ascend_training
-    @pytest.mark.platform_x86_cpu
-    @pytest.mark.env_single
-    def test_filter_summary_lineage_exception_6(self):
         """Test the abnormal execution of the filter_summary_lineage interface."""
         # gt > lt
         search_condition1 = {
@@ -728,7 +526,7 @@ class TestModelApi(TestCase):
             'object': [],
             'count': 0
         }
-        partial_res1 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition1)
+        partial_res1 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition1)
         assert expect_result == partial_res1
 
         # the (offset + 1) * limit > count
@@ -744,7 +542,7 @@ class TestModelApi(TestCase):
             'object': [],
             'count': 2
         }
-        partial_res2 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition2)
+        partial_res2 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition2)
         assert expect_result == partial_res2
 
     @pytest.mark.level0
@@ -753,7 +551,7 @@ class TestModelApi(TestCase):
     @pytest.mark.platform_x86_ascend_training
     @pytest.mark.platform_x86_cpu
     @pytest.mark.env_single
-    def test_filter_summary_lineage_exception_7(self):
+    def test_filter_summary_lineage_exception_6(self):
         """Test the abnormal execution of the filter_summary_lineage interface."""
         condition_keys = ["summary_dir", "lineage_type", "loss_function", "optimizer", "network", "dataset_mark"]
         for condition_key in condition_keys:
@@ -767,8 +565,7 @@ class TestModelApi(TestCase):
                 LineageSearchConditionParamError,
                 f'The parameter {condition_key} is invalid. Its operation should be `eq`, `in` or `not_in`.',
                 filter_summary_lineage,
-                None,
-                BASE_SUMMARY_DIR,
+                LINEAGE_DATA_MANAGER,
                 search_condition
             )
 
@@ -778,7 +575,7 @@ class TestModelApi(TestCase):
     @pytest.mark.platform_x86_ascend_training
     @pytest.mark.platform_x86_cpu
     @pytest.mark.env_single
-    def test_filter_summary_lineage_exception_8(self):
+    def test_filter_summary_lineage_exception_7(self):
         """Test the abnormal execution of the filter_summary_lineage interface."""
         invalid_lineage_types = ['xxx', None]
         for lineage_type in invalid_lineage_types:
@@ -791,8 +588,7 @@ class TestModelApi(TestCase):
                 LineageSearchConditionParamError,
                 "The parameter lineage_type is invalid. It should be 'dataset' or 'model'.",
                 filter_summary_lineage,
-                None,
-                BASE_SUMMARY_DIR,
+                LINEAGE_DATA_MANAGER,
                 search_condition
             )
 
@@ -802,7 +598,7 @@ class TestModelApi(TestCase):
     @pytest.mark.platform_x86_ascend_training
     @pytest.mark.platform_x86_cpu
     @pytest.mark.env_single
-    def test_filter_summary_lineage_exception_9(self):
+    def test_filter_summary_lineage_exception_8(self):
         """Test the abnormal execution of the filter_summary_lineage interface."""
         invalid_sorted_names = ['xxx', 'metric_', 1]
         for sorted_name in invalid_sorted_names:
@@ -813,8 +609,7 @@ class TestModelApi(TestCase):
                 LineageSearchConditionParamError,
                 'The sorted_name must be in',
                 filter_summary_lineage,
-                None,
-                BASE_SUMMARY_DIR,
+                LINEAGE_DATA_MANAGER,
                 search_condition
             )
 

tests/ut/backend/lineagemgr/test_lineage_api.py

@@ -14,7 +14,6 @@
 # ============================================================================
 """Test the module of lineage_api."""
 import json
-import os
 from unittest import TestCase, mock
 
 from flask import Response
@@ -73,21 +72,21 @@ class TestSearchModel(TestCase):
     @mock.patch('mindinsight.backend.lineagemgr.lineage_api.filter_summary_lineage')
     def test_search_model_success(self, *args):
         """Test the success of model_success."""
-        base_dir = '/path/to/test_lineage_summary_dir_base'
         args[0].return_value = {
             'object': [
                 {
-                    'summary_dir': base_dir,
-                    'model_lineage': LINEAGE_FILTRATION_BASE
+                    'summary_dir': './',
+                    'model_lineage': LINEAGE_FILTRATION_BASE,
+                    'added_info': {}
                 },
                 {
-                    'summary_dir': os.path.join(base_dir, 'run1'),
-                    'model_lineage': LINEAGE_FILTRATION_RUN1
+                    'summary_dir': './run1',
+                    'model_lineage': LINEAGE_FILTRATION_RUN1,
+                    'added_info': {}
                 }
             ],
             'count': 2
         }
-        args[1].SUMMARY_BASE_DIR = base_dir
 
         body_data = {
             'limit': 10,
@@ -101,11 +100,13 @@ class TestSearchModel(TestCase):
             'object': [
                 {
                     'summary_dir': './',
-                    'model_lineage': LINEAGE_FILTRATION_BASE
+                    'model_lineage': LINEAGE_FILTRATION_BASE,
+                    'added_info': {}
                 },
                 {
                     'summary_dir': './run1',
-                    'model_lineage': LINEAGE_FILTRATION_RUN1
+                    'model_lineage': LINEAGE_FILTRATION_RUN1,
+                    'added_info': {}
                 }
             ],
             'count': 2

tests/ut/lineagemgr/querier/test_querier.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 # ============================================================================
 """Test the querier module."""
+import os
 import time
 
 from unittest import TestCase, mock
@@ -22,7 +23,7 @@ from google.protobuf.json_format import ParseDict
 
 import mindinsight.datavisual.proto_files.mindinsight_lineage_pb2 as summary_pb2
 from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, LineageQuerierParamException
-from mindinsight.lineagemgr.lineage_parser import LineageOrganizer
+from mindinsight.lineagemgr.lineage_parser import LineageParser
 from mindinsight.lineagemgr.querier.querier import Querier
 from mindinsight.lineagemgr.summary.lineage_summary_analyzer import LineageInfo
 
@@ -104,6 +105,7 @@ def create_filtration_result(summary_dir, train_event_dict,
             "user_defined": {}
         },
         "dataset_graph": dataset_dict,
+        'added_info': {}
     }
     return filtration_result
 
@@ -162,42 +164,42 @@ LINEAGE_INFO_1 = {
     'dataset_graph': event_data.DATASET_DICT_0
 }
 LINEAGE_FILTRATION_0 = create_filtration_result(
-    '/path/to/summary0',
+    './summary0',
     event_data.EVENT_TRAIN_DICT_0,
     event_data.EVENT_EVAL_DICT_0,
     event_data.METRIC_0,
     event_data.DATASET_DICT_0
 )
 LINEAGE_FILTRATION_1 = create_filtration_result(
-    '/path/to/summary1',
+    './summary1',
     event_data.EVENT_TRAIN_DICT_1,
     event_data.EVENT_EVAL_DICT_1,
     event_data.METRIC_1,
     event_data.DATASET_DICT_0
 )
 LINEAGE_FILTRATION_2 = create_filtration_result(
-    '/path/to/summary2',
+    './summary2',
     event_data.EVENT_TRAIN_DICT_2,
     event_data.EVENT_EVAL_DICT_2,
     event_data.METRIC_2,
     event_data.DATASET_DICT_0
 )
 LINEAGE_FILTRATION_3 = create_filtration_result(
-    '/path/to/summary3',
+    './summary3',
     event_data.EVENT_TRAIN_DICT_3,
     event_data.EVENT_EVAL_DICT_3,
     event_data.METRIC_3,
     event_data.DATASET_DICT_0
 )
 LINEAGE_FILTRATION_4 = create_filtration_result(
-    '/path/to/summary4',
+    './summary4',
     event_data.EVENT_TRAIN_DICT_4,
     event_data.EVENT_EVAL_DICT_4,
     event_data.METRIC_4,
     event_data.DATASET_DICT_0
 )
 LINEAGE_FILTRATION_5 = {
-    "summary_dir": '/path/to/summary5',
+    "summary_dir": './summary5',
     "model_lineage": {
         "loss_function":
             event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'],
@@ -219,10 +221,11 @@ LINEAGE_FILTRATION_5 = {
         "dataset_mark": '2',
         "user_defined": {}
     },
-    "dataset_graph": event_data.DATASET_DICT_0
+    "dataset_graph": event_data.DATASET_DICT_0,
+    "added_info": {}
 }
 LINEAGE_FILTRATION_6 = {
-    "summary_dir": '/path/to/summary6',
+    "summary_dir": './summary6',
     "model_lineage": {
         "loss_function": None,
         "train_dataset_path": None,
@@ -243,15 +246,16 @@ LINEAGE_FILTRATION_6 = {
         "dataset_mark": '2',
         "user_defined": {}
     },
-    "dataset_graph": event_data.DATASET_DICT_0
+    "dataset_graph": event_data.DATASET_DICT_0,
+    "added_info": {}
 }
 
 
 class TestQuerier(TestCase):
     """Test the class of `Querier`."""
+    _MOCK_DATA_MANAGER = MagicMock()
 
     @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_lineage_summaries')
-    @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryWatcher.list_summary_directories')
     @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_user_defined_info')
     @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_summary_infos')
     @mock.patch('mindinsight.lineagemgr.lineage_parser.FileHandler')
@@ -263,139 +267,23 @@ class TestQuerier(TestCase):
             event_data.EVENT_DATASET_DICT_0
         )
         args[1].return_value = []
-        args[3].return_value = ['path']
+        args[2].return_value = ['path']
         mock_file_handler = MagicMock()
         mock_file_handler.size = 1
 
-        args[2].return_value = [{'relative_path': './', 'update_time': 1}]
-        single_summary_path = '/path/to/summary0'
-        lineage_objects = LineageOrganizer(summary_base_dir=single_summary_path).super_lineage_objs
-        self.single_querier = Querier(lineage_objects)
+        summary_dir = '/path/test/'
 
         lineage_infos = get_lineage_infos()
         args[0].side_effect = lineage_infos
-        summary_base_dir = '/path/to'
-        relative_dirs = []
+        lineage_objects = {}
         for i in range(7):
-            relative_dirs.append(dict(relative_path=f'./summary{i}', update_time=time.time() - i))
-        args[2].return_value = relative_dirs
-        lineage_objects = LineageOrganizer(summary_base_dir=summary_base_dir).super_lineage_objs
-        self.multi_querier = Querier(lineage_objects)
-
-    def test_get_summary_lineage_success_1(self):
-        """Test the success of get_summary_lineage."""
-        expected_result = [LINEAGE_INFO_0]
-        result = self.single_querier.get_summary_lineage()
-        assert_equal_lineages(expected_result, result, self.assertListEqual)
-
-    def test_get_summary_lineage_success_2(self):
-        """Test the success of get_summary_lineage."""
-        expected_result = [LINEAGE_INFO_0]
-        result = self.single_querier.get_summary_lineage()
-        assert_equal_lineages(expected_result, result, self.assertListEqual)
-
-    def test_get_summary_lineage_success_3(self):
-        """Test the success of get_summary_lineage."""
-        expected_result = [
-            {
-                'summary_dir': '/path/to/summary0',
-                'model': event_data.EVENT_TRAIN_DICT_0['train_lineage']['model'],
-                'algorithm': event_data.EVENT_TRAIN_DICT_0['train_lineage']['algorithm']
-            }
-        ]
-        result = self.single_querier.get_summary_lineage(
-            filter_keys=['model', 'algorithm']
-        )
-        assert_equal_lineages(expected_result, result, self.assertListEqual)
-
-    def test_get_summary_lineage_success_4(self):
-        """Test the success of get_summary_lineage."""
-        expected_result = [
-            LINEAGE_INFO_0,
-            LINEAGE_INFO_1,
-            {
-                'summary_dir': '/path/to/summary2',
-                **event_data.EVENT_TRAIN_DICT_2['train_lineage'],
-                'metric': event_data.METRIC_2,
-                'valid_dataset': event_data.EVENT_EVAL_DICT_2['evaluation_lineage']['valid_dataset'],
-                'dataset_graph': event_data.DATASET_DICT_0
-            },
-            {
-                'summary_dir': '/path/to/summary3',
-                **event_data.EVENT_TRAIN_DICT_3['train_lineage'],
-                'metric': event_data.METRIC_3,
-                'valid_dataset': event_data.EVENT_EVAL_DICT_3['evaluation_lineage']['valid_dataset'],
-                'dataset_graph': event_data.DATASET_DICT_0
-            },
-            {
-                'summary_dir': '/path/to/summary4',
-                **event_data.EVENT_TRAIN_DICT_4['train_lineage'],
-                'metric': event_data.METRIC_4,
-                'valid_dataset': event_data.EVENT_EVAL_DICT_4['evaluation_lineage']['valid_dataset'],
-                'dataset_graph': event_data.DATASET_DICT_0
-            },
-            {
-                'summary_dir': '/path/to/summary5',
-                **event_data.EVENT_TRAIN_DICT_5['train_lineage'],
-                'metric': {},
-                'valid_dataset': {},
-                'dataset_graph': event_data.DATASET_DICT_0
-            },
-            {
-                'summary_dir': '/path/to/summary6',
-                'hyper_parameters': {},
-                'algorithm': {},
-                'model': {},
-                'train_dataset': {},
-                'metric': event_data.METRIC_5,
-                'valid_dataset': event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset'],
-                'dataset_graph': event_data.DATASET_DICT_0
-            }
-        ]
-        result = self.multi_querier.get_summary_lineage()
-        assert_equal_lineages(expected_result, result, self.assertListEqual)
-
-    def test_get_summary_lineage_success_5(self):
-        """Test the success of get_summary_lineage."""
-        expected_result = [LINEAGE_INFO_1]
-        result = self.multi_querier.get_summary_lineage(
-            summary_dir='/path/to/summary1'
-        )
-        assert_equal_lineages(expected_result, result, self.assertListEqual)
-
-    def test_get_summary_lineage_success_6(self):
-        """Test the success of get_summary_lineage."""
-        expected_result = [
-            {
-                'summary_dir': '/path/to/summary0',
-                'hyper_parameters': event_data.EVENT_TRAIN_DICT_0['train_lineage']['hyper_parameters'],
-                'train_dataset': event_data.EVENT_TRAIN_DICT_0['train_lineage']['train_dataset'],
-                'metric': event_data.METRIC_0,
-                'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset']
-            }
-        ]
-        filter_keys = [
-            'metric', 'hyper_parameters', 'train_dataset', 'valid_dataset'
-        ]
-        result = self.multi_querier.get_summary_lineage(
-            summary_dir='/path/to/summary0', filter_keys=filter_keys
-        )
-        assert_equal_lineages(expected_result, result, self.assertListEqual)
-
-    def test_get_summary_lineage_fail(self):
-        """Test the function of get_summary_lineage with exception."""
-        filter_keys = ['xxx']
-        self.assertRaises(
-            LineageQuerierParamException,
-            self.multi_querier.get_summary_lineage,
-            filter_keys=filter_keys
-        )
+            train_id = f'./summary{i}'
+            summary_dir = os.path.join(summary_dir, train_id)
+            update_time = time.time() - i
+            lineage_parser = LineageParser(train_id, summary_dir, update_time)
+            lineage_objects.update({train_id: lineage_parser.super_lineage_obj})
 
-        self.assertRaises(
-            LineageQuerierParamException,
-            self.multi_querier.get_summary_lineage,
-            summary_dir='xxx'
-        )
+        self.multi_querier = Querier(lineage_objects)
 
     def test_filter_summary_lineage_success_1(self):
         """Test the success of filter_summary_lineage."""

tests/ut/lineagemgr/querier/test_query_model.py

@@ -133,6 +133,7 @@ class TestLineageObj(TestCase):
         )
         expected_result['model_lineage']['dataset_mark'] = None
         expected_result.pop('dataset_graph')
+        expected_result.pop('added_info')
         result = self.lineage_obj.to_model_lineage_dict()
         assert_equal_lineages(expected_result, result, self.assertDictEqual)
 

tests/ut/lineagemgr/test_model.py

@@ -16,224 +16,60 @@
 from unittest import TestCase, mock
 from unittest.mock import MagicMock
 
-from mindinsight.lineagemgr.model import get_summary_lineage, filter_summary_lineage, \
-    _convert_relative_path_to_abspath, get_flattened_lineage
-from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamSummaryPathError, \
-    LineageFileNotFoundError, LineageSummaryParseException, LineageQuerierParamException, \
-    LineageQuerySummaryDataError, LineageSearchConditionParamError, LineageParamTypeError, \
-    LineageParamValueError
+from mindinsight.lineagemgr.model import filter_summary_lineage, get_flattened_lineage
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSummaryParseException, \
+    LineageQuerierParamException, LineageQuerySummaryDataError, LineageSearchConditionParamError, LineageParamTypeError
 from mindinsight.lineagemgr.common.path_parser import SummaryPathParser
 from ...st.func.lineagemgr.test_model import LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2
 
 
-class TestModel(TestCase):
-    """Test the function of get_summary_lineage and filter_summary_lineage."""
-
-    @mock.patch('mindinsight.lineagemgr.model.Querier')
-    @mock.patch('mindinsight.lineagemgr.model.LineageParser')
-    @mock.patch('os.path.isdir')
-    def test_get_summary_lineage_success(self, isdir_mock, parser_mock, qurier_mock):
-        """Test the function of get_summary_lineage."""
-        isdir_mock.return_value = True
-        parser_mock.return_value = MagicMock()
-
-        mock_querier = MagicMock()
-        qurier_mock.return_value = mock_querier
-        mock_querier.get_summary_lineage.return_value = [{'algorithm': {'network': 'ResNet'}}]
-        summary_dir = '/path/to/summary_dir'
-        result = get_summary_lineage(None, summary_dir, keys=['algorithm'])
-        self.assertEqual(result, {'algorithm': {'network': 'ResNet'}})
-
-    def test_get_summary_lineage_failed(self):
-        """Test get_summary_lineage failed."""
-        invalid_path = '../fake_dir'
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            get_summary_lineage,
-            None,
-            invalid_path
-        )
-
-    @mock.patch('mindinsight.lineagemgr.common.utils.validate_path')
-    @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
-    def test_get_summary_lineage_failed2(self, mock_summary, mock_valid):
-        """Test get_summary_lineage failed."""
-        mock_summary.return_value = []
-        mock_valid.return_value = '/path/to/summary/dir'
-        self.assertRaisesRegex(
-            LineageFileNotFoundError,
-            'no summary log file under summary_dir',
-            get_summary_lineage,
-            None,
-            '/path/to/summary_dir'
-        )
-
-    @mock.patch('mindinsight.lineagemgr.lineage_parser.FileHandler')
-    @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageParser._parse_summary_log')
-    @mock.patch('mindinsight.lineagemgr.common.utils.validate_path')
-    @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
-    def test_get_summary_lineage_failed3(self,
-                                         mock_summary,
-                                         mock_valid,
-                                         mock_parser,
-                                         mock_file_handler):
-        """Test get_summary_lineage failed."""
-        mock_summary.return_value = ['/path/to/summary/file']
-        mock_valid.return_value = '/path/to/summary_dir'
-        mock_parser.return_value = None
-        mock_file_handler = MagicMock()
-        mock_file_handler.size = 1
-        result = get_summary_lineage(None, '/path/to/summary_dir')
-        assert {} == result
-
-    @mock.patch('mindinsight.lineagemgr.model.validate_path')
-    def test_convert_relative_path_to_abspath(self, validate_path_mock):
-        """Test the function of converting realtive path to abspath."""
-        validate_path_mock.return_value = '/path/to/summary_base_dir/summary_dir'
-        summary_base_dir = '/path/to/summary_base_dir'
-        search_condition = {
-            'summary_dir': {
-                'in': ['/path/to/summary_base_dir']
-            }
-        }
-        result = _convert_relative_path_to_abspath(summary_base_dir,
-                                                   search_condition)
-        self.assertDictEqual(
-            result, {'summary_dir': {'in': ['/path/to/summary_base_dir/summary_dir']}})
-
-        search_condition = {
-            'summary_dir': {
-                'in': ['./summary_dir']
-            }
-        }
-        result = _convert_relative_path_to_abspath(summary_base_dir, search_condition)
-        self.assertDictEqual(
-            result, {'summary_dir': {'in': ['/path/to/summary_base_dir/summary_dir']}}
-        )
-
-        search_condition = {
-            'summary_dir': {
-                'eq': '/summary_dir'
-            }
-        }
-        result = _convert_relative_path_to_abspath(summary_base_dir, search_condition)
-        self.assertDictEqual(
-            result, {'summary_dir': {'eq': '/path/to/summary_base_dir/summary_dir'}})
-
-        search_condition = {
-            'summary_dir': None
-        }
-        result = _convert_relative_path_to_abspath(summary_base_dir, search_condition)
-        self.assertDictEqual(
-            result, search_condition
-        )
-
-
 class TestFilterAPI(TestCase):
     """Test the function of filter_summary_lineage."""
-    @mock.patch('mindinsight.lineagemgr.model.LineageOrganizer')
+    _MOCK_DATA_MANAGER = MagicMock()
+
     @mock.patch('mindinsight.lineagemgr.model.Querier')
     @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_lineage_summaries')
-    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
-    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
-    def test_filter_summary_lineage(self, validate_path_mock, convert_path_mock,
-                                    latest_summary_mock, qurier_mock, organizer_mock):
+    def test_filter_summary_lineage(self, latest_summary_mock, qurier_mock):
         """Test the function of filter_summary_lineage."""
-        convert_path_mock.return_value = {
-            'summary_dir': {
-                'in': ['/path/to/summary_base_dir']
-            },
-            'loss': {
-                'gt': 2.0
-            }
-        }
-        organizer_mock = MagicMock()
-        organizer_mock.super_lineage_objs = None
-        validate_path_mock.return_value = True
         latest_summary_mock.return_value = ['/path/to/summary_base_dir/summary_dir']
         mock_querier = MagicMock()
         qurier_mock.return_value = mock_querier
         mock_querier.filter_summary_lineage.return_value = [{'loss': 3.0}]
 
-        summary_base_dir = '/path/to/summary_base_dir'
-        result = filter_summary_lineage(None, summary_base_dir)
+        result = filter_summary_lineage(self._MOCK_DATA_MANAGER)
         self.assertEqual(result, [{'loss': 3.0}])
 
-    def test_invalid_path(self):
-        """Test filter_summary_lineage with invalid path."""
-        invalid_path = '../fake_dir'
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'The summary path is invalid.',
-            filter_summary_lineage,
-            None,
-            invalid_path
-        )
-
     @mock.patch('mindinsight.lineagemgr.model.validate_condition')
-    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
-    def test_invalid_search_condition(self, mock_path, mock_valid):
+    def test_invalid_search_condition(self, mock_valid):
         """Test filter_summary_lineage with invalid invalid param."""
-        mock_path.return_value = None
         mock_valid.side_effect = LineageParamTypeError(
             'Invalid search_condition type.')
         self.assertRaisesRegex(
             LineageSearchConditionParamError,
             'Invalid search_condition type.',
             filter_summary_lineage,
-            None,
-            '/path/to/summary/dir',
+            self._MOCK_DATA_MANAGER,
             'invalid_condition'
         )
 
-    @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition')
-    @mock.patch('mindinsight.lineagemgr.model.validate_condition')
-    @mock.patch('mindinsight.lineagemgr.common.utils.validate_path')
-    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
-    def test_failed_to_convert_path(self, mock_convert, *args):
+    def test_failed_to_get_summary_files(self):
         """Test filter_summary_lineage with invalid invalid param."""
-        mock_convert.side_effect = LineageParamValueError('invalid path')
-        args[0].return_value = None
-        self.assertRaisesRegex(
-            LineageParamSummaryPathError,
-            'invalid path',
-            filter_summary_lineage,
-            None,
-            '/path/to/summary/dir',
-            {}
-        )
-
-    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
-    @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition')
-    @mock.patch('mindinsight.lineagemgr.model.validate_condition')
-    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
-    @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
-    def test_failed_to_get_summary_filesh(self, mock_parse, *args):
-        """Test filter_summary_lineage with invalid invalid param."""
-        path = '/path/to/summary/dir'
-        mock_parse.return_value = []
-        args[0].return_value = path
-        self.assertRaisesRegex(
-            LineageFileNotFoundError,
-            'There is no summary log file under summary_base_dir.',
-            filter_summary_lineage,
-            None,
-            path
-        )
+        default_result = {
+            'customized': {},
+            'object': [],
+            'count': 0
+        }
+        assert default_result == filter_summary_lineage(self._MOCK_DATA_MANAGER)
 
-    @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath')
     @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition')
     @mock.patch('mindinsight.lineagemgr.model.validate_condition')
-    @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir')
     @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
     @mock.patch('mindinsight.lineagemgr.model.Querier')
-    def test_failed_to_querier(self, mock_query, mock_parse, *args):
+    def test_failed_to_querier(self, mock_query, *args):
         """Test filter_summary_lineage with invalid invalid param."""
         mock_query.side_effect = LineageSummaryParseException()
-        mock_parse.return_value = ['/path/to/summary/file']
         args[0].return_value = None
-        res = filter_summary_lineage(None, '/path/to/summary')
+        res = filter_summary_lineage(self._MOCK_DATA_MANAGER)
         assert res == {'object': [], 'count': 0}
 
         mock_query.side_effect = LineageQuerierParamException(['keys'], 'key')
@@ -241,8 +77,7 @@ class TestFilterAPI(TestCase):
             LineageQuerySummaryDataError,
             'Filter summary lineage failed.',
             filter_summary_lineage,
-            None,
-            '/path/to/summary/dir'
+            self._MOCK_DATA_MANAGER
         )
 
     @mock.patch('mindinsight.lineagemgr.model.filter_summary_lineage')
@@ -251,8 +86,7 @@ class TestFilterAPI(TestCase):
         mock_data = {
             'object': [LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2]
         }
-        mock_datamanager = MagicMock()
-        mock_datamanager.summary_base_dir = '/tmp/'
+        mock_data_manager = MagicMock()
         mock_filter_summary_lineage.return_value = mock_data
-        result = get_flattened_lineage(mock_datamanager, None)
+        result = get_flattened_lineage(mock_data_manager)
         assert result.get('[U]info') == ['info1', None]

tests/utils/tools.py

@@ -19,12 +19,14 @@ import io
 import os
 import shutil
 import json
+from pathlib import Path
 
 from urllib.parse import urlencode
-
 import numpy as np
 from PIL import Image
 
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError
+
 
 def get_url(url, params):
     """
@@ -138,3 +140,25 @@ def assert_equal_lineages(lineages1, lineages2, assert_func, decimal_num=2):
     else:
         deal_float_for_dict(lineages1, lineages2, decimal_num)
     assert_func(lineages1, lineages2)
+
+
+def get_relative_path(path, base_path):
+    """
+    Get relative path based on base_path.
+
+    Args:
+        path (str): absolute path.
+        base_path: absolute base path.
+
+    Returns:
+        str, relative path based on base_path.
+
+    """
+    try:
+        r_path = str(Path(path).relative_to(Path(base_path)))
+    except ValueError:
+        raise LineageParamValueError("The path %r does not start with %r." % (path, base_path))
+
+    if r_path == ".":
+        r_path = ""
+    return os.path.join("./", r_path)