diff --git a/mindinsight/backend/lineagemgr/lineage_api.py b/mindinsight/backend/lineagemgr/lineage_api.py index edec1210..359f97e9 100644 --- a/mindinsight/backend/lineagemgr/lineage_api.py +++ b/mindinsight/backend/lineagemgr/lineage_api.py @@ -14,16 +14,15 @@ # ============================================================================ """Lineage restful api.""" import json -import os from flask import Blueprint, jsonify, request from mindinsight.conf import settings from mindinsight.datavisual.utils.tools import get_train_id from mindinsight.datavisual.data_transform.data_manager import DATA_MANAGER -from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage -from mindinsight.utils.exceptions import MindInsightException, ParamValueError from mindinsight.lineagemgr.cache_item_updater import update_lineage_object +from mindinsight.lineagemgr.model import filter_summary_lineage +from mindinsight.utils.exceptions import MindInsightException, ParamValueError BLUEPRINT = Blueprint("lineage", __name__, url_prefix=settings.URL_PATH_PREFIX+settings.API_PREFIX) @@ -67,26 +66,8 @@ def _get_lineage_info(search_condition): Raises: MindInsightException: If method fails to be called. """ - summary_base_dir = str(settings.SUMMARY_BASE_DIR) try: - lineage_info = filter_summary_lineage( - data_manager=DATA_MANAGER, - search_condition=search_condition, - added=True) - - lineages = lineage_info['object'] - - summary_base_dir = os.path.realpath(summary_base_dir) - length = len(summary_base_dir) - - for lineage in lineages: - summary_dir = lineage['summary_dir'] - summary_dir = os.path.realpath(summary_dir) - if summary_base_dir == summary_dir: - relative_dir = './' - else: - relative_dir = os.path.join(os.curdir, summary_dir[length+1:]) - lineage['summary_dir'] = relative_dir + lineage_info = filter_summary_lineage(data_manager=DATA_MANAGER, search_condition=search_condition) except MindInsightException as exception: raise MindInsightException(exception.error, exception.message, http_code=400) @@ -134,29 +115,27 @@ def get_dataset_graph(): >>> GET http://xxxx/v1/mindinsight/datasets/dataset_graph?train_id=xxx """ - summary_base_dir = str(settings.SUMMARY_BASE_DIR) - summary_dir = get_train_id(request) + train_id = get_train_id(request) + search_condition = { + 'summary_dir': { + 'in': [train_id] + } + } + result = {} try: - dataset_graph = get_summary_lineage( - DATA_MANAGER, - summary_dir=summary_dir, - keys=['dataset_graph'] - ) + objects = filter_summary_lineage(DATA_MANAGER, search_condition).get('object') except MindInsightException as exception: raise MindInsightException(exception.error, exception.message, http_code=400) - if dataset_graph: - summary_dir_result = dataset_graph.get('summary_dir') - base_dir_len = len(summary_base_dir) - if summary_base_dir == summary_dir_result: - relative_dir = './' - else: - relative_dir = os.path.join( - os.curdir, summary_dir[base_dir_len + 1:] - ) - dataset_graph['summary_dir'] = relative_dir - - return jsonify(dataset_graph) + if objects: + lineage_obj = objects[0] + dataset_graph = lineage_obj.get('dataset_graph') + + if dataset_graph: + result.update({'dataset_graph': dataset_graph}) + result.update({'summary_dir': lineage_obj.get('summary_dir')}) + + return jsonify(result) def init_module(app): diff --git a/mindinsight/datavisual/data_transform/data_manager.py b/mindinsight/datavisual/data_transform/data_manager.py index 8e2a01c6..b16431d5 100644 --- a/mindinsight/datavisual/data_transform/data_manager.py +++ b/mindinsight/datavisual/data_transform/data_manager.py @@ -210,6 +210,11 @@ class CachedTrainJob: """Threading lock with given key.""" return self._key_locks.setdefault(key, threading.Lock()) + @property + def train_id(self): + """Get train id.""" + return self._basic_info.train_id + class TrainJob: """ diff --git a/mindinsight/lineagemgr/cache_item_updater.py b/mindinsight/lineagemgr/cache_item_updater.py index bb7b5b59..5039a8ed 100644 --- a/mindinsight/lineagemgr/cache_item_updater.py +++ b/mindinsight/lineagemgr/cache_item_updater.py @@ -72,12 +72,13 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater): def _lineage_parsing(self, cache_item): """Parse summaries and return lineage parser.""" + train_id = cache_item.train_id summary_dir = cache_item.abs_summary_dir update_time = cache_item.basic_info.update_time cached_lineage_item = cache_item.get(key=LINEAGE, raise_exception=False) if cached_lineage_item is None: - lineage_parser = LineageParser(summary_dir, update_time) + lineage_parser = LineageParser(train_id, summary_dir, update_time) else: lineage_parser = cached_lineage_item with cache_item.lock_key(LINEAGE): diff --git a/mindinsight/lineagemgr/common/exceptions/error_code.py b/mindinsight/lineagemgr/common/exceptions/error_code.py index c896e718..3e380b2b 100644 --- a/mindinsight/lineagemgr/common/exceptions/error_code.py +++ b/mindinsight/lineagemgr/common/exceptions/error_code.py @@ -57,7 +57,6 @@ class LineageErrors(LineageErrorCodes): PARAM_STEP_NUM_ERROR = 18 | _MINDSPORE_COLLECTOR_ERROR # Model lineage error codes. - LINEAGE_PARAM_OPERATION_ERROR = 0 | _MODEL_LINEAGE_API_ERROR_MASK LINEAGE_PARAM_METRIC_ERROR = 1 | _MODEL_LINEAGE_API_ERROR_MASK LINEAGE_PARAM_LOSS_FUNCTION_ERROR = 4 | _MODEL_LINEAGE_API_ERROR_MASK LINEAGE_PARAM_TRAIN_DATASET_PATH_ERROR = 5 | _MODEL_LINEAGE_API_ERROR_MASK @@ -79,7 +78,6 @@ class LineageErrors(LineageErrorCodes): LINEAGE_DIR_NOT_EXIST_ERROR = 20 | _MODEL_LINEAGE_API_ERROR_MASK LINEAGE_SUMMARY_DATA_ERROR = 21 | _MODEL_LINEAGE_API_ERROR_MASK LINEAGE_FILE_NOT_FOUND_ERROR = 22 | _MODEL_LINEAGE_API_ERROR_MASK - LINEAGE_PARAM_SUMMARY_PATH_ERROR = 23 | _MODEL_LINEAGE_API_ERROR_MASK LINEAGE_SEARCH_CONDITION_PARAM_ERROR = 24 | _MODEL_LINEAGE_API_ERROR_MASK LINEAGE_PARAM_LINEAGE_TYPE_ERROR = 25 | _MODEL_LINEAGE_API_ERROR_MASK @@ -110,7 +108,6 @@ class LineageErrorMsg(Enum): "mindspore.train.summary.SummaryRecord" PARAM_RAISE_EXCEPTION_ERROR = "Invalid value for raise_exception. It should be True or False." # Lineage error messages. - LINEAGE_PARAM_SUMMARY_PATH_ERROR = "The parameter summary path error: {}" LINEAGE_SUMMARY_DATA_ERROR = "Query summary data error: {}" LINEAGE_FILE_NOT_FOUND_ERROR = "File not found error: {}" LINEAGE_DIR_NOT_EXIST_ERROR = "Dir not exist error: {}" diff --git a/mindinsight/lineagemgr/common/exceptions/exceptions.py b/mindinsight/lineagemgr/common/exceptions/exceptions.py index 3856ab21..291e615d 100644 --- a/mindinsight/lineagemgr/common/exceptions/exceptions.py +++ b/mindinsight/lineagemgr/common/exceptions/exceptions.py @@ -106,16 +106,6 @@ class LineageEventFieldNotExistException(MindInsightException): ) -class LineageParamSummaryPathError(MindInsightException): - """The lineage parameter summary path error.""" - def __init__(self, msg): - super(LineageParamSummaryPathError, self).__init__( - error=LineageErrors.LINEAGE_PARAM_SUMMARY_PATH_ERROR, - message=LineageErrorMsg.LINEAGE_PARAM_SUMMARY_PATH_ERROR.value.format(msg), - http_code=400 - ) - - class LineageQuerySummaryDataError(MindInsightException): """Query summary data error in lineage module.""" def __init__(self, msg): @@ -136,16 +126,6 @@ class LineageFileNotFoundError(MindInsightException): ) -class LineageDirNotExistError(MindInsightException): - """Directory not exist in lineage module.""" - def __init__(self, msg): - super(LineageDirNotExistError, self).__init__( - error=LineageErrors.LINEAGE_DIR_NOT_EXIST_ERROR, - message=LineageErrorMsg.LINEAGE_DIR_NOT_EXIST_ERROR.value.format(msg), - http_code=400 - ) - - class LineageSearchConditionParamError(MindInsightException): """Search condition param is invalid in lineage module.""" def __init__(self, msg): diff --git a/mindinsight/lineagemgr/common/utils.py b/mindinsight/lineagemgr/common/utils.py index fe295574..21cae5ea 100644 --- a/mindinsight/lineagemgr/common/utils.py +++ b/mindinsight/lineagemgr/common/utils.py @@ -15,30 +15,16 @@ """Lineage utils.""" import os import re -from pathlib import Path from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher -from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError, LineageParamTypeError, \ - LineageDirNotExistError, LineageParamSummaryPathError +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError from mindinsight.lineagemgr.common.log import logger as log -from mindinsight.lineagemgr.common.validator.validate import validate_path def enum_to_list(enum): return [enum_ele.value for enum_ele in enum] -def normalize_summary_dir(summary_dir): - """Normalize summary dir.""" - try: - summary_dir = validate_path(summary_dir) - except (LineageParamValueError, LineageDirNotExistError) as error: - log.error(str(error)) - log.exception(error) - raise LineageParamSummaryPathError(str(error.message)) - return summary_dir - - def get_timestamp(filename): """Get timestamp from filename.""" timestamp = int(re.search(SummaryWatcher().SUMMARY_FILENAME_REGEX, filename)[1]) @@ -68,25 +54,3 @@ def make_directory(path): log.error("No write permission on the directory(%r), error = %r", path, err) raise LineageParamTypeError("No write permission on the directory.") return real_path - - -def get_relative_path(path, base_path): - """ - Get relative path based on base_path. - - Args: - path (str): absolute path. - base_path: absolute base path. - - Returns: - str, relative path based on base_path. - - """ - try: - r_path = str(Path(path).relative_to(Path(base_path))) - except ValueError: - raise LineageParamValueError("The path %r does not start with %r." % (path, base_path)) - - if r_path == ".": - r_path = "" - return os.path.join("./", r_path) diff --git a/mindinsight/lineagemgr/common/validator/validate.py b/mindinsight/lineagemgr/common/validator/validate.py index 2f0e64c1..9d4a4242 100644 --- a/mindinsight/lineagemgr/common/validator/validate.py +++ b/mindinsight/lineagemgr/common/validator/validate.py @@ -13,13 +13,11 @@ # limitations under the License. # ============================================================================ """Validate the parameters.""" -import os import re from marshmallow import ValidationError from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg -from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, \ - LineageParamValueError, LineageDirNotExistError +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, LineageParamValueError from mindinsight.lineagemgr.common.log import logger as log from mindinsight.lineagemgr.common.validator.validate_path import safe_normalize_path from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING @@ -224,38 +222,6 @@ def validate_raise_exception(raise_exception): ) -def validate_filter_key(keys): - """ - Verify the keys of filtering is valid or not. - - Args: - keys (list): The keys to get the relative lineage info. - - Raises: - LineageParamTypeError: If keys is not list. - LineageParamValueError: If the value of keys is invalid. - """ - filter_keys = [ - 'metric', 'hyper_parameters', 'algorithm', - 'train_dataset', 'model', 'valid_dataset', - 'dataset_graph' - ] - - if not isinstance(keys, list): - log.error("Keys must be list.") - raise LineageParamTypeError("Keys must be list.") - - for element in keys: - if not isinstance(element, str): - log.error("Element of keys must be str.") - raise LineageParamTypeError("Element of keys must be str.") - - if not set(keys).issubset(filter_keys): - err_msg = "Keys must be in {}.".format(filter_keys) - log.error(err_msg) - raise LineageParamValueError(err_msg) - - def validate_condition(search_condition): """ Verify the param in search_condition is valid or not. @@ -310,31 +276,6 @@ def validate_condition(search_condition): raise LineageParamValueError(err_msg) -def validate_path(summary_path): - """ - Verify the summary path is valid or not. - - Args: - summary_path (str): The summary path which is a dir. - - Raises: - LineageParamValueError: If the input param value is invalid. - LineageDirNotExistError: If the summary path is invalid. - """ - try: - summary_path = safe_normalize_path( - summary_path, "summary_path", None, check_absolute_path=True - ) - except ValidationError: - log.error("The summary path is invalid.") - raise LineageParamValueError("The summary path is invalid.") - if not os.path.isdir(summary_path): - log.error("The summary path does not exist or is not a dir.") - raise LineageDirNotExistError("The summary path does not exist or is not a dir.") - - return summary_path - - def validate_user_defined_info(user_defined_info): """ Validate user defined info, delete the item if its key is in lineage. diff --git a/mindinsight/lineagemgr/lineage_parser.py b/mindinsight/lineagemgr/lineage_parser.py index 9df28899..ed895ce5 100644 --- a/mindinsight/lineagemgr/lineage_parser.py +++ b/mindinsight/lineagemgr/lineage_parser.py @@ -15,7 +15,6 @@ """This file is used to parse lineage info.""" import os -from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSummaryAnalyzeException, \ LineageEventNotExistException, LineageEventFieldNotExistException, LineageFileNotFoundError, \ MindInsightException @@ -64,8 +63,9 @@ class SuperLineageObj: class LineageParser: """Lineage parser.""" - def __init__(self, summary_dir, update_time=None, added_info=None): + def __init__(self, train_id, summary_dir, update_time=None, added_info=None): self._summary_dir = summary_dir + self._train_id = train_id self._update_time = update_time self._added_info = added_info @@ -154,7 +154,7 @@ class LineageParser: """Update lineage object.""" if self._super_lineage_obj is None: lineage_obj = LineageObj( - self._summary_dir, + self._train_id, train_lineage=lineage_info.train_lineage, evaluation_lineage=lineage_info.eval_lineage, dataset_graph=lineage_info.dataset_graph, @@ -177,52 +177,13 @@ class LineageParser: class LineageOrganizer: """Lineage organizer.""" - def __init__(self, data_manager=None, summary_base_dir=None): + def __init__(self, data_manager): self._data_manager = data_manager - self._summary_base_dir = summary_base_dir - self._check_params() self._super_lineage_objs = {} self._organize_from_cache() - self._organize_from_disk() - - def _check_params(self): - """Check params.""" - if self._data_manager is not None and self._summary_base_dir is not None: - self._summary_base_dir = None - - def _organize_from_disk(self): - """Organize lineage objs from disk.""" - if self._summary_base_dir is None: - return - summary_watcher = SummaryWatcher() - relative_dirs = summary_watcher.list_summary_directories( - summary_base_dir=self._summary_base_dir - ) - - no_lineage_count = 0 - for item in relative_dirs: - relative_dir = item.get('relative_path') - update_time = item.get('update_time') - abs_summary_dir = os.path.realpath(os.path.join(self._summary_base_dir, relative_dir)) - - try: - lineage_parser = LineageParser(abs_summary_dir, update_time) - super_lineage_obj = lineage_parser.super_lineage_obj - if super_lineage_obj is not None: - self._super_lineage_objs.update({abs_summary_dir: super_lineage_obj}) - except LineageFileNotFoundError: - no_lineage_count += 1 - - if no_lineage_count == len(relative_dirs): - logger.info('There is no summary log file under summary_base_dir.') - raise LineageFileNotFoundError( - 'There is no summary log file under summary_base_dir.' - ) def _organize_from_cache(self): """Organize lineage objs from cache.""" - if self._data_manager is None: - return brief_cache = self._data_manager.get_brief_cache() cache_items = brief_cache.cache_items for relative_dir, cache_train_job in cache_items.items(): diff --git a/mindinsight/lineagemgr/model.py b/mindinsight/lineagemgr/model.py index 7c48f20d..e8eec69b 100644 --- a/mindinsight/lineagemgr/model.py +++ b/mindinsight/lineagemgr/model.py @@ -13,20 +13,15 @@ # limitations under the License. # ============================================================================ """This file is used to define the model lineage python api.""" -import os import numpy as np import pandas as pd -from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError, \ - LineageQuerySummaryDataError, LineageParamSummaryPathError, \ - LineageQuerierParamException, LineageDirNotExistError, LineageSearchConditionParamError, \ - LineageParamTypeError, LineageSummaryParseException +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageQuerySummaryDataError, \ + LineageQuerierParamException, LineageSearchConditionParamError, LineageParamTypeError, LineageSummaryParseException from mindinsight.lineagemgr.common.log import logger as log -from mindinsight.lineagemgr.common.utils import normalize_summary_dir, get_relative_path from mindinsight.lineagemgr.common.validator.model_parameter import SearchModelConditionParameter -from mindinsight.lineagemgr.common.validator.validate import validate_filter_key, validate_search_model_condition, \ - validate_condition, validate_path, validate_train_id -from mindinsight.lineagemgr.lineage_parser import LineageParser, LineageOrganizer +from mindinsight.lineagemgr.common.validator.validate import validate_search_model_condition, validate_condition +from mindinsight.lineagemgr.lineage_parser import LineageOrganizer from mindinsight.lineagemgr.querier.querier import Querier from mindinsight.optimizer.common.enums import ReasonCode from mindinsight.optimizer.utils.utils import is_simple_numpy_number @@ -38,62 +33,7 @@ _USER_DEFINED_PREFIX = "[U]" USER_DEFINED_INFO_LIMIT = 100 -def get_summary_lineage(data_manager=None, summary_dir=None, keys=None): - """ - Get summary lineage from data_manager or parsing from summaries. - - One of data_manager or summary_dir needs to be specified. Support getting - super_lineage_obj from data_manager or parsing summaries by summary_dir. - - Args: - data_manager (DataManager): Data manager defined as - mindinsight.datavisual.data_transform.data_manager.DataManager - summary_dir (str): The summary directory. It contains summary logs for - one training. - keys (list[str]): The filter keys of lineage information. The acceptable - keys are `metric`, `user_defined`, `hyper_parameters`, `algorithm`, - `train_dataset`, `model`, `valid_dataset` and `dataset_graph`. - If it is `None`, all information will be returned. Default: None. - - Returns: - dict, the lineage information for one training. - - Raises: - LineageParamSummaryPathError: If summary path is invalid. - LineageQuerySummaryDataError: If querying summary data fails. - LineageFileNotFoundError: If the summary log file is not found. - - """ - default_result = {} - if data_manager is None and summary_dir is None: - raise LineageParamTypeError("One of data_manager or summary_dir needs to be specified.") - if data_manager is not None and summary_dir is None: - raise LineageParamTypeError("If data_manager is specified, the summary_dir needs to be " - "specified as relative path.") - - if keys is not None: - validate_filter_key(keys) - - if data_manager is None: - normalize_summary_dir(summary_dir) - super_lineage_obj = LineageParser(summary_dir).super_lineage_obj - else: - validate_train_id(summary_dir) - super_lineage_obj = LineageOrganizer(data_manager=data_manager).get_super_lineage_obj(summary_dir) - - if super_lineage_obj is None: - return default_result - - try: - result = Querier({summary_dir: super_lineage_obj}).get_summary_lineage(summary_dir, keys) - except (LineageQuerierParamException, LineageParamTypeError) as error: - log.error(str(error)) - log.exception(error) - raise LineageQuerySummaryDataError("Get summary lineage failed.") - return result[0] - - -def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_condition=None, added=False): +def filter_summary_lineage(data_manager, search_condition=None): """ Filter summary lineage from data_manager or parsing from summaries. @@ -103,18 +43,8 @@ def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_cond Args: data_manager (DataManager): Data manager defined as mindinsight.datavisual.data_transform.data_manager.DataManager - summary_base_dir (str): The summary base directory. It contains summary - directories generated by training. search_condition (dict): The search condition. """ - if data_manager is None and summary_base_dir is None: - raise LineageParamTypeError("One of data_manager or summary_base_dir needs to be specified.") - - if data_manager is None: - summary_base_dir = normalize_summary_dir(summary_base_dir) - else: - summary_base_dir = data_manager.summary_base_dir - search_condition = {} if search_condition is None else search_condition try: @@ -126,18 +56,8 @@ def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_cond raise LineageSearchConditionParamError(str(error.message)) try: - search_condition = _convert_relative_path_to_abspath(summary_base_dir, search_condition) - except (LineageParamValueError, LineageDirNotExistError) as error: - log.error(str(error)) - log.exception(error) - raise LineageParamSummaryPathError(str(error.message)) - - try: - lineage_objects = LineageOrganizer(data_manager, summary_base_dir).super_lineage_objs - result = Querier(lineage_objects).filter_summary_lineage( - condition=search_condition, - added=added - ) + lineage_objects = LineageOrganizer(data_manager).super_lineage_objs + result = Querier(lineage_objects).filter_summary_lineage(condition=search_condition) except LineageSummaryParseException: result = {'object': [], 'count': 0} except (LineageQuerierParamException, LineageParamTypeError) as error: @@ -148,53 +68,6 @@ def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_cond return result -def _convert_relative_path_to_abspath(summary_base_dir, search_condition): - """ - Convert relative path to absolute path. - - Args: - summary_base_dir (str): The summary base directory. - search_condition (dict): The search condition. - - Returns: - dict, the updated search_condition. - - Raises: - LineageParamValueError: If the value of input_name is invalid. - """ - if ("summary_dir" not in search_condition) or (not search_condition.get("summary_dir")): - return search_condition - - summary_dir_condition = search_condition.get("summary_dir") - - for key in ['in', 'not_in']: - if key in summary_dir_condition: - summary_paths = [] - for summary_dir in summary_dir_condition.get(key): - if summary_dir.startswith('./'): - abs_dir = os.path.join( - summary_base_dir, summary_dir[2:] - ) - abs_dir = validate_path(abs_dir) - else: - abs_dir = validate_path(summary_dir) - summary_paths.append(abs_dir) - search_condition.get('summary_dir')[key] = summary_paths - - if 'eq' in summary_dir_condition: - summary_dir = summary_dir_condition.get('eq') - if summary_dir.startswith('./'): - abs_dir = os.path.join( - summary_base_dir, summary_dir[2:] - ) - abs_dir = validate_path(abs_dir) - else: - abs_dir = validate_path(summary_dir) - search_condition.get('summary_dir')['eq'] = abs_dir - - return search_condition - - def get_flattened_lineage(data_manager, search_condition=None): """ Get lineage data in a table from data manager. @@ -207,10 +80,10 @@ def get_flattened_lineage(data_manager, search_condition=None): Dict[str, list]: A dict contains keys and values from lineages. """ - summary_base_dir, flatten_dict, user_count = data_manager.summary_base_dir, {'train_id': []}, 0 + flatten_dict, user_count = {'train_id': []}, 0 lineages = filter_summary_lineage(data_manager=data_manager, search_condition=search_condition).get("object", []) for index, lineage in enumerate(lineages): - flatten_dict['train_id'].append(get_relative_path(lineage.get("summary_dir"), summary_base_dir)) + flatten_dict['train_id'].append(lineage.get("summary_dir")) for key, val in _flatten_lineage(lineage.get('model_lineage', {})): if key.startswith(_USER_DEFINED_PREFIX) and key not in flatten_dict: if user_count > USER_DEFINED_INFO_LIMIT: diff --git a/mindinsight/lineagemgr/querier/querier.py b/mindinsight/lineagemgr/querier/querier.py index e8fbc57d..92ce508d 100644 --- a/mindinsight/lineagemgr/querier/querier.py +++ b/mindinsight/lineagemgr/querier/querier.py @@ -189,53 +189,7 @@ class Querier: raise LineageParamTypeError("Init param should be a dict.") return super_lineage_objs - def get_summary_lineage(self, summary_dir=None, filter_keys=None): - """ - Get summary lineage information. - - If a summary dir is specified, the special summary lineage information - will be found. If the summary dir is `None`, all summary lineage - information will be found. - - Returns the content corresponding to the specified field in the filter - key. The contents of the filter key include `metric`, `hyper_parameters`, - `algorithm`, `train_dataset`, `valid_dataset` and `model`. You can - specify multiple filter keys in the `filter_keys`. If the parameter is - `None`, complete information will be returned. - - Args: - summary_dir (Union[str, None]): Summary log dir. Default: None. - filter_keys (Union[list[str], None]): Filter keys. Default: None. - - Returns: - list[dict], summary lineage information. - """ - - if filter_keys is None: - filter_keys = LineageFilterKey.get_key_list() - else: - for key in filter_keys: - if not LineageFilterKey.is_valid_filter_key(key): - raise LineageQuerierParamException( - filter_keys, 'The filter key {} is invalid.'.format(key) - ) - - if summary_dir is None: - result = [ - item.lineage_obj.get_summary_info(filter_keys) for item in self._super_lineage_objs.values() - ] - elif summary_dir in self._super_lineage_objs: - lineage_obj = self._super_lineage_objs[summary_dir].lineage_obj - result = [lineage_obj.get_summary_info(filter_keys)] - else: - raise LineageQuerierParamException( - 'summary_dir', - 'Summary dir {} does not exist.'.format(summary_dir) - ) - - return result - - def filter_summary_lineage(self, condition=None, added=False): + def filter_summary_lineage(self, condition=None): """ Filter and sort lineage information based on the specified condition. @@ -298,8 +252,7 @@ class Querier: lineage_object.update(item.lineage_obj.to_model_lineage_dict()) if LineageType.DATASET.value in lineage_types: lineage_object.update(item.lineage_obj.to_dataset_lineage_dict()) - if added: - lineage_object.update({"added_info": item.added_info}) + lineage_object.update({"added_info": item.added_info}) object_items.append(lineage_object) lineage_info = { diff --git a/tests/st/func/lineagemgr/cache/__init__.py b/tests/st/func/lineagemgr/cache/__init__.py deleted file mode 100644 index e3077430..00000000 --- a/tests/st/func/lineagemgr/cache/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ diff --git a/tests/st/func/lineagemgr/cache/test_lineage_cache.py b/tests/st/func/lineagemgr/cache/test_lineage_cache.py deleted file mode 100644 index 8e53f2d3..00000000 --- a/tests/st/func/lineagemgr/cache/test_lineage_cache.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -""" -Function: - Test the query module about lineage information. -Usage: - The query module test should be run after lineagemgr/collection/model/test_model_lineage.py - pytest lineagemgr -""" -from unittest import TestCase - -import pytest - -from mindinsight.datavisual.data_transform.data_manager import DataManager -from mindinsight.lineagemgr.cache_item_updater import LineageCacheItemUpdater -from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage - -from ..test_model import LINEAGE_INFO_RUN1, LINEAGE_FILTRATION_EXCEPT_RUN, \ - LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2 -from ..conftest import BASE_SUMMARY_DIR -from .....ut.lineagemgr.querier import event_data -from .....utils.tools import assert_equal_lineages - - -@pytest.mark.usefixtures("create_summary_dir") -class TestModelApi(TestCase): - """Test get lineage from data_manager.""" - @classmethod - def setup_class(cls): - data_manager = DataManager(BASE_SUMMARY_DIR) - data_manager.register_brief_cache_item_updater(LineageCacheItemUpdater()) - data_manager.start_load_data().join() - - cls._data_manger = data_manager - - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_get_summary_lineage(self): - """Test the interface of get_summary_lineage.""" - total_res = get_summary_lineage(data_manager=self._data_manger, summary_dir="./run1") - expect_total_res = LINEAGE_INFO_RUN1 - assert_equal_lineages(expect_total_res, total_res, self.assertDictEqual) - - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_filter_summary_lineage(self): - """Test the interface of filter_summary_lineage.""" - expect_result = { - 'customized': event_data.CUSTOMIZED__1, - 'object': [ - LINEAGE_FILTRATION_EXCEPT_RUN, - LINEAGE_FILTRATION_RUN1, - LINEAGE_FILTRATION_RUN2 - ], - 'count': 3 - } - - search_condition = { - 'sorted_name': 'summary_dir' - } - res = filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition) - expect_objects = expect_result.get('object') - for idx, res_object in enumerate(res.get('object')): - expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') - assert_equal_lineages(expect_result, res, self.assertDictEqual) - - expect_result = { - 'customized': {}, - 'object': [], - 'count': 0 - } - - search_condition = { - 'summary_dir': { - "in": ['./dir_with_empty_lineage'] - } - } - res = filter_summary_lineage(data_manager=self._data_manger, search_condition=search_condition) - assert_equal_lineages(expect_result, res, self.assertDictEqual) diff --git a/tests/st/func/lineagemgr/collection/model/test_model_lineage.py b/tests/st/func/lineagemgr/collection/model/test_model_lineage.py index b5928d4d..857add95 100644 --- a/tests/st/func/lineagemgr/collection/model/test_model_lineage.py +++ b/tests/st/func/lineagemgr/collection/model/test_model_lineage.py @@ -27,7 +27,7 @@ from unittest import TestCase, mock import numpy as np import pytest -from mindinsight.lineagemgr.model import get_summary_lineage +from mindinsight.lineagemgr.model import filter_summary_lineage from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors from mindinsight.utils.exceptions import MindInsightException @@ -37,10 +37,11 @@ from mindspore.dataset.engine import MindDataset from mindspore.nn import Momentum, SoftmaxCrossEntropyWithLogits, WithLossCell from mindspore.train.callback import ModelCheckpoint, RunContext, SummaryStep, _ListCallback from mindspore.train.summary import SummaryRecord -from tests.utils.lineage_writer.model_lineage import AnalyzeObject, EvalLineage, TrainLineage -from ...conftest import SUMMARY_DIR, SUMMARY_DIR_2, SUMMARY_DIR_3 from .train_one_step import TrainOneStep +from ...conftest import SUMMARY_DIR, SUMMARY_DIR_2, SUMMARY_DIR_3, BASE_SUMMARY_DIR, LINEAGE_DATA_MANAGER +from ......utils.lineage_writer.model_lineage import AnalyzeObject, EvalLineage, TrainLineage +from ......utils.tools import get_relative_path @pytest.mark.usefixtures("create_summary_dir") @@ -76,6 +77,13 @@ class TestModelLineage(TestCase): "version": "v1" } + train_id = get_relative_path(SUMMARY_DIR, BASE_SUMMARY_DIR) + cls._search_condition = { + 'summary_dir': { + 'eq': train_id + } + } + @pytest.mark.scene_train(2) @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @@ -90,13 +98,17 @@ class TestModelLineage(TestCase): train_callback = TrainLineage(SUMMARY_DIR, True, self.user_defined_info) train_callback.initial_learning_rate = 0.12 train_callback.end(RunContext(self.run_context)) - res = get_summary_lineage(summary_dir=SUMMARY_DIR) - assert res.get('hyper_parameters', {}).get('epoch') == 10 + + LINEAGE_DATA_MANAGER.start_load_data().join() + res = filter_summary_lineage(LINEAGE_DATA_MANAGER, self._search_condition) + assert res.get('object')[0].get('model_lineage', {}).get('epoch') == 10 run_context = self.run_context run_context['epoch_num'] = 14 train_callback.end(RunContext(run_context)) - res = get_summary_lineage(summary_dir=SUMMARY_DIR) - assert res.get('hyper_parameters', {}).get('epoch') == 14 + + LINEAGE_DATA_MANAGER.start_load_data().join() + res = filter_summary_lineage(LINEAGE_DATA_MANAGER, self._search_condition) + assert res.get('object')[0].get('model_lineage', {}).get('epoch') == 14 @pytest.mark.scene_eval(3) @pytest.mark.level0 @@ -186,12 +198,13 @@ class TestModelLineage(TestCase): run_context_customized['train_network'] = net train_callback.begin(RunContext(run_context_customized)) train_callback.end(RunContext(run_context_customized)) - res = get_summary_lineage(summary_dir=SUMMARY_DIR) - assert res.get('hyper_parameters', {}).get('loss_function') \ - == 'SoftmaxCrossEntropyWithLogits' - assert res.get('algorithm', {}).get('network') == 'ResNet' - assert res.get('hyper_parameters', {}).get('optimizer') == 'Momentum' + LINEAGE_DATA_MANAGER.start_load_data().join() + res = filter_summary_lineage(LINEAGE_DATA_MANAGER, self._search_condition) + assert res.get('object')[0].get('model_lineage', {}).get('loss_function') \ + == 'SoftmaxCrossEntropyWithLogits' + assert res.get('object')[0].get('model_lineage', {}).get('network') == 'ResNet' + assert res.get('object')[0].get('model_lineage', {}).get('optimizer') == 'Momentum' @pytest.mark.scene_exception(1) @pytest.mark.level0 diff --git a/tests/st/func/lineagemgr/conftest.py b/tests/st/func/lineagemgr/conftest.py index db2c14f7..9ebe6f4f 100644 --- a/tests/st/func/lineagemgr/conftest.py +++ b/tests/st/func/lineagemgr/conftest.py @@ -21,6 +21,9 @@ import tempfile import pytest +from mindinsight.datavisual.data_transform.data_manager import DataManager +from mindinsight.lineagemgr.cache_item_updater import LineageCacheItemUpdater + from ....utils import mindspore from ....utils.mindspore.dataset.engine.serializer_deserializer import SERIALIZED_PIPELINE @@ -31,6 +34,9 @@ SUMMARY_DIR = os.path.join(BASE_SUMMARY_DIR, 'run1') SUMMARY_DIR_2 = os.path.join(BASE_SUMMARY_DIR, 'run2') SUMMARY_DIR_3 = os.path.join(BASE_SUMMARY_DIR, 'except_run') +LINEAGE_DATA_MANAGER = DataManager(BASE_SUMMARY_DIR) +LINEAGE_DATA_MANAGER.register_brief_cache_item_updater(LineageCacheItemUpdater()) + COLLECTION_MODULE = 'TestModelLineage' API_MODULE = 'TestModelApi' DATASET_GRAPH = SERIALIZED_PIPELINE diff --git a/tests/st/func/lineagemgr/test_model.py b/tests/st/func/lineagemgr/test_model.py index 388d6b9a..e9da3a11 100644 --- a/tests/st/func/lineagemgr/test_model.py +++ b/tests/st/func/lineagemgr/test_model.py @@ -24,20 +24,18 @@ import os from unittest import TestCase import pytest -from mindinsight.lineagemgr.model import filter_summary_lineage, get_summary_lineage -from mindinsight.lineagemgr.common.exceptions.exceptions import (LineageFileNotFoundError, LineageParamSummaryPathError, - LineageParamTypeError, LineageParamValueError, - LineageSearchConditionParamError) +from mindinsight.lineagemgr.model import filter_summary_lineage from mindinsight.datavisual.data_transform import data_manager from mindinsight.lineagemgr.cache_item_updater import LineageCacheItemUpdater +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSearchConditionParamError from mindinsight.lineagemgr.model import get_flattened_lineage -from .conftest import BASE_SUMMARY_DIR, DATASET_GRAPH, SUMMARY_DIR, SUMMARY_DIR_2 +from .conftest import BASE_SUMMARY_DIR, DATASET_GRAPH, SUMMARY_DIR, SUMMARY_DIR_2, LINEAGE_DATA_MANAGER from ....ut.lineagemgr.querier import event_data -from ....utils.tools import assert_equal_lineages +from ....utils.tools import assert_equal_lineages, get_relative_path LINEAGE_INFO_RUN1 = { - 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'summary_dir': './run1', 'metric': { 'accuracy': 0.78 }, @@ -67,7 +65,7 @@ LINEAGE_INFO_RUN1 = { 'dataset_graph': DATASET_GRAPH } LINEAGE_FILTRATION_EXCEPT_RUN = { - 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'), + 'summary_dir': './except_run', 'model_lineage': { 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, @@ -86,10 +84,11 @@ LINEAGE_FILTRATION_EXCEPT_RUN = { 'metric': {}, 'dataset_mark': 2 }, - 'dataset_graph': DATASET_GRAPH + 'dataset_graph': DATASET_GRAPH, + 'added_info': {} } LINEAGE_FILTRATION_RUN1 = { - 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'summary_dir': './run1', 'model_lineage': { 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, @@ -114,10 +113,11 @@ LINEAGE_FILTRATION_RUN1 = { }, 'dataset_mark': 2 }, - 'dataset_graph': DATASET_GRAPH + 'dataset_graph': DATASET_GRAPH, + 'added_info': {} } LINEAGE_FILTRATION_RUN2 = { - 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'), + 'summary_dir': './run2', 'model_lineage': { 'loss_function': "SoftmaxCrossEntropyWithLogits", 'train_dataset_path': None, @@ -138,7 +138,8 @@ LINEAGE_FILTRATION_RUN2 = { }, 'dataset_mark': 3 }, - 'dataset_graph': DATASET_GRAPH + 'dataset_graph': DATASET_GRAPH, + 'added_info': {} } @@ -149,12 +150,12 @@ class TestModelApi(TestCase): @classmethod def setup_class(cls): """The initial process.""" - cls.dir_with_empty_lineage = os.path.join( + cls._dir_with_empty_lineage = os.path.join( BASE_SUMMARY_DIR, 'dir_with_empty_lineage' ) - os.makedirs(cls.dir_with_empty_lineage) + os.makedirs(cls._dir_with_empty_lineage) ms_file = os.path.join( - cls.dir_with_empty_lineage, 'empty.summary.1581499502.bms_MS' + cls._dir_with_empty_lineage, 'empty.summary.1581499502.bms_MS' ) lineage_file = ms_file + '_lineage' with open(ms_file, mode='w'): @@ -162,156 +163,11 @@ class TestModelApi(TestCase): with open(lineage_file, mode='w'): pass - cls.empty_dir = os.path.join(BASE_SUMMARY_DIR, 'empty_dir') - os.makedirs(cls.empty_dir) - - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_get_summary_lineage(self): - """Test the interface of get_summary_lineage.""" - total_res = get_summary_lineage(None, SUMMARY_DIR) - partial_res1 = get_summary_lineage(None, SUMMARY_DIR, ['hyper_parameters']) - partial_res2 = get_summary_lineage(None, SUMMARY_DIR, ['metric', 'algorithm']) - expect_total_res = LINEAGE_INFO_RUN1 - expect_partial_res1 = { - 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), - 'hyper_parameters': { - 'optimizer': 'Momentum', - 'learning_rate': 0.12, - 'loss_function': 'SoftmaxCrossEntropyWithLogits', - 'epoch': 14, - 'parallel_mode': 'stand_alone', - 'device_num': 2, - 'batch_size': 32 - } - } - expect_partial_res2 = { - 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), - 'metric': { - 'accuracy': 0.78 - }, - 'algorithm': { - 'network': 'ResNet' - } - } - assert_equal_lineages(expect_total_res, total_res, self.assertDictEqual) - assert_equal_lineages(expect_partial_res1, partial_res1, self.assertDictEqual) - assert_equal_lineages(expect_partial_res2, partial_res2, self.assertDictEqual) - - # the lineage summary file is empty - result = get_summary_lineage(None, self.dir_with_empty_lineage) - assert {} == result - - # keys is empty list - expect_result = { - 'summary_dir': SUMMARY_DIR - } - result = get_summary_lineage(None, SUMMARY_DIR, []) - assert expect_result == result - - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_get_summary_lineage_exception_1(self): - """Test the interface of get_summary_lineage with exception.""" - # summary path does not exist - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path does not exist or is not a dir.', - get_summary_lineage, - None, - '/tmp/fake/dir' - ) - - # summary path is relative path - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - get_summary_lineage, - None, - 'tmp' - ) - - # the type of input param is invalid - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - get_summary_lineage, - None, - ['/root/linage1', '/root/lineage2'] - ) - - # summary path is empty str - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - get_summary_lineage, - None, - '', - keys=None - ) + empty_dir = os.path.join(BASE_SUMMARY_DIR, 'empty_dir') + os.makedirs(empty_dir) + cls._empty_train_id = get_relative_path(empty_dir, LINEAGE_DATA_MANAGER.summary_base_dir) - # summary path invalid - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - get_summary_lineage, - None, - '\\', - keys=None - ) - - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_get_summary_lineage_exception_2(self): - """Test the interface of get_summary_lineage with exception.""" - # keys is invalid - self.assertRaisesRegex( - LineageParamValueError, - 'Keys must be in', - get_summary_lineage, - None, - SUMMARY_DIR, - ['metric', 'fake_name'] - ) - - self.assertRaisesRegex( - LineageParamTypeError, - 'Element of keys must be str.', - get_summary_lineage, - None, - SUMMARY_DIR, - [1, 2, 3] - ) - - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_get_summary_lineage_exception_3(self): - """Test the interface of get_summary_lineage with exception.""" - for keys in [0, 0.1, True, (3, 4), {'a': 'b'}]: - self.assertRaisesRegex( - LineageParamTypeError, - 'Keys must be list.', - get_summary_lineage, - None, - SUMMARY_DIR, - keys - ) + LINEAGE_DATA_MANAGER.start_load_data().join() @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @@ -334,18 +190,7 @@ class TestModelApi(TestCase): search_condition = { 'sorted_name': 'summary_dir' } - res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition) - expect_objects = expect_result.get('object') - for idx, res_object in enumerate(res.get('object')): - expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') - assert_equal_lineages(expect_result, res, self.assertDictEqual) - - expect_result = { - 'customized': {}, - 'object': [], - 'count': 0 - } - res = filter_summary_lineage(None, self.dir_with_empty_lineage) + res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition) expect_objects = expect_result.get('object') for idx, res_object in enumerate(res.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') @@ -362,8 +207,8 @@ class TestModelApi(TestCase): search_condition = { 'summary_dir': { 'in': [ - SUMMARY_DIR, - SUMMARY_DIR_2 + get_relative_path(SUMMARY_DIR, LINEAGE_DATA_MANAGER.summary_base_dir), + get_relative_path(SUMMARY_DIR_2, LINEAGE_DATA_MANAGER.summary_base_dir) ] }, 'metric/accuracy': { @@ -383,7 +228,7 @@ class TestModelApi(TestCase): ], 'count': 2 } - partial_res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition) + partial_res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') @@ -421,7 +266,7 @@ class TestModelApi(TestCase): ], 'count': 2 } - partial_res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition) + partial_res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') @@ -450,7 +295,7 @@ class TestModelApi(TestCase): ], 'count': 3 } - partial_res1 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition1) + partial_res1 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition1) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res1.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') @@ -469,9 +314,30 @@ class TestModelApi(TestCase): 'object': [], 'count': 0 } - partial_res2 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition2) + partial_res2 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition2) assert expect_result == partial_res2 + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_condition_4(self): + """Test the abnormal execution of the filter_summary_lineage interface.""" + expect_result = { + 'customized': {}, + 'object': [], + 'count': 0 + } + search_condition = { + 'summary_dir': { + 'eq': self._empty_train_id + } + } + res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition) + assert expect_result == res + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_gpu_training @@ -480,10 +346,10 @@ class TestModelApi(TestCase): @pytest.mark.env_single def test_filter_summary_lineage_with_lineage_type(self): """Test the interface of filter_summary_lineage with lineage_type.""" - summary_dir = os.path.join(BASE_SUMMARY_DIR, 'except_run') + train_id = './except_run' search_condition = { 'summary_dir': { - 'in': [summary_dir] + 'in': [train_id] }, 'lineage_type': { 'eq': 'dataset' @@ -493,50 +359,16 @@ class TestModelApi(TestCase): 'customized': {}, 'object': [ { - 'summary_dir': summary_dir, - 'dataset_graph': DATASET_GRAPH + 'summary_dir': train_id, + 'dataset_graph': DATASET_GRAPH, + 'added_info': {} } ], 'count': 1 } - res = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition) + res = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition) assert expect_result == res - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_filter_summary_lineage_exception_1(self): - """Test the abnormal execution of the filter_summary_lineage interface.""" - # summary base dir is relative path - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - filter_summary_lineage, - None, - 'relative_path' - ) - - # summary base dir does not exist - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path does not exist or is not a dir.', - filter_summary_lineage, - None, - '/path/does/not/exist' - ) - - # no summary log file under summary_base_dir - self.assertRaisesRegex( - LineageFileNotFoundError, - 'There is no summary log file under summary_base_dir.', - filter_summary_lineage, - None, - self.empty_dir - ) - @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_gpu_training @@ -553,8 +385,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The search_condition element summary_dir should be dict.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -566,8 +397,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The sorted_name have to exist when sorted_type exists.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -577,8 +407,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'Invalid search_condition type, it should be dict.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -590,8 +419,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The limit must be int.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -611,8 +439,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The offset must be int.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -626,8 +453,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The search attribute not supported.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -648,8 +474,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The sorted_type must be ascending or descending', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -663,8 +488,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The compare condition should be in', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -678,8 +502,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The parameter metric/accuracy is invalid.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -690,31 +513,6 @@ class TestModelApi(TestCase): @pytest.mark.platform_x86_cpu @pytest.mark.env_single def test_filter_summary_lineage_exception_5(self): - """Test the abnormal execution of the filter_summary_lineage interface.""" - # the summary dir is invalid in search condition - search_condition = { - 'summary_dir': { - 'in': [ - 'xxx' - ] - } - } - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - filter_summary_lineage, - None, - BASE_SUMMARY_DIR, - search_condition - ) - - @pytest.mark.level0 - @pytest.mark.platform_arm_ascend_training - @pytest.mark.platform_x86_gpu_training - @pytest.mark.platform_x86_ascend_training - @pytest.mark.platform_x86_cpu - @pytest.mark.env_single - def test_filter_summary_lineage_exception_6(self): """Test the abnormal execution of the filter_summary_lineage interface.""" # gt > lt search_condition1 = { @@ -728,7 +526,7 @@ class TestModelApi(TestCase): 'object': [], 'count': 0 } - partial_res1 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition1) + partial_res1 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition1) assert expect_result == partial_res1 # the (offset + 1) * limit > count @@ -744,7 +542,7 @@ class TestModelApi(TestCase): 'object': [], 'count': 2 } - partial_res2 = filter_summary_lineage(None, BASE_SUMMARY_DIR, search_condition2) + partial_res2 = filter_summary_lineage(LINEAGE_DATA_MANAGER, search_condition2) assert expect_result == partial_res2 @pytest.mark.level0 @@ -753,7 +551,7 @@ class TestModelApi(TestCase): @pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_cpu @pytest.mark.env_single - def test_filter_summary_lineage_exception_7(self): + def test_filter_summary_lineage_exception_6(self): """Test the abnormal execution of the filter_summary_lineage interface.""" condition_keys = ["summary_dir", "lineage_type", "loss_function", "optimizer", "network", "dataset_mark"] for condition_key in condition_keys: @@ -767,8 +565,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, f'The parameter {condition_key} is invalid. Its operation should be `eq`, `in` or `not_in`.', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -778,7 +575,7 @@ class TestModelApi(TestCase): @pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_cpu @pytest.mark.env_single - def test_filter_summary_lineage_exception_8(self): + def test_filter_summary_lineage_exception_7(self): """Test the abnormal execution of the filter_summary_lineage interface.""" invalid_lineage_types = ['xxx', None] for lineage_type in invalid_lineage_types: @@ -791,8 +588,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, "The parameter lineage_type is invalid. It should be 'dataset' or 'model'.", filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) @@ -802,7 +598,7 @@ class TestModelApi(TestCase): @pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_cpu @pytest.mark.env_single - def test_filter_summary_lineage_exception_9(self): + def test_filter_summary_lineage_exception_8(self): """Test the abnormal execution of the filter_summary_lineage interface.""" invalid_sorted_names = ['xxx', 'metric_', 1] for sorted_name in invalid_sorted_names: @@ -813,8 +609,7 @@ class TestModelApi(TestCase): LineageSearchConditionParamError, 'The sorted_name must be in', filter_summary_lineage, - None, - BASE_SUMMARY_DIR, + LINEAGE_DATA_MANAGER, search_condition ) diff --git a/tests/ut/backend/lineagemgr/test_lineage_api.py b/tests/ut/backend/lineagemgr/test_lineage_api.py index 8ff5b222..e36960e9 100644 --- a/tests/ut/backend/lineagemgr/test_lineage_api.py +++ b/tests/ut/backend/lineagemgr/test_lineage_api.py @@ -14,7 +14,6 @@ # ============================================================================ """Test the module of lineage_api.""" import json -import os from unittest import TestCase, mock from flask import Response @@ -73,21 +72,21 @@ class TestSearchModel(TestCase): @mock.patch('mindinsight.backend.lineagemgr.lineage_api.filter_summary_lineage') def test_search_model_success(self, *args): """Test the success of model_success.""" - base_dir = '/path/to/test_lineage_summary_dir_base' args[0].return_value = { 'object': [ { - 'summary_dir': base_dir, - 'model_lineage': LINEAGE_FILTRATION_BASE + 'summary_dir': './', + 'model_lineage': LINEAGE_FILTRATION_BASE, + 'added_info': {} }, { - 'summary_dir': os.path.join(base_dir, 'run1'), - 'model_lineage': LINEAGE_FILTRATION_RUN1 + 'summary_dir': './run1', + 'model_lineage': LINEAGE_FILTRATION_RUN1, + 'added_info': {} } ], 'count': 2 } - args[1].SUMMARY_BASE_DIR = base_dir body_data = { 'limit': 10, @@ -101,11 +100,13 @@ class TestSearchModel(TestCase): 'object': [ { 'summary_dir': './', - 'model_lineage': LINEAGE_FILTRATION_BASE + 'model_lineage': LINEAGE_FILTRATION_BASE, + 'added_info': {} }, { 'summary_dir': './run1', - 'model_lineage': LINEAGE_FILTRATION_RUN1 + 'model_lineage': LINEAGE_FILTRATION_RUN1, + 'added_info': {} } ], 'count': 2 diff --git a/tests/ut/lineagemgr/querier/test_querier.py b/tests/ut/lineagemgr/querier/test_querier.py index c8a4a6f2..78b3bf7b 100644 --- a/tests/ut/lineagemgr/querier/test_querier.py +++ b/tests/ut/lineagemgr/querier/test_querier.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================ """Test the querier module.""" +import os import time from unittest import TestCase, mock @@ -22,7 +23,7 @@ from google.protobuf.json_format import ParseDict import mindinsight.datavisual.proto_files.mindinsight_lineage_pb2 as summary_pb2 from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, LineageQuerierParamException -from mindinsight.lineagemgr.lineage_parser import LineageOrganizer +from mindinsight.lineagemgr.lineage_parser import LineageParser from mindinsight.lineagemgr.querier.querier import Querier from mindinsight.lineagemgr.summary.lineage_summary_analyzer import LineageInfo @@ -104,6 +105,7 @@ def create_filtration_result(summary_dir, train_event_dict, "user_defined": {} }, "dataset_graph": dataset_dict, + 'added_info': {} } return filtration_result @@ -162,42 +164,42 @@ LINEAGE_INFO_1 = { 'dataset_graph': event_data.DATASET_DICT_0 } LINEAGE_FILTRATION_0 = create_filtration_result( - '/path/to/summary0', + './summary0', event_data.EVENT_TRAIN_DICT_0, event_data.EVENT_EVAL_DICT_0, event_data.METRIC_0, event_data.DATASET_DICT_0 ) LINEAGE_FILTRATION_1 = create_filtration_result( - '/path/to/summary1', + './summary1', event_data.EVENT_TRAIN_DICT_1, event_data.EVENT_EVAL_DICT_1, event_data.METRIC_1, event_data.DATASET_DICT_0 ) LINEAGE_FILTRATION_2 = create_filtration_result( - '/path/to/summary2', + './summary2', event_data.EVENT_TRAIN_DICT_2, event_data.EVENT_EVAL_DICT_2, event_data.METRIC_2, event_data.DATASET_DICT_0 ) LINEAGE_FILTRATION_3 = create_filtration_result( - '/path/to/summary3', + './summary3', event_data.EVENT_TRAIN_DICT_3, event_data.EVENT_EVAL_DICT_3, event_data.METRIC_3, event_data.DATASET_DICT_0 ) LINEAGE_FILTRATION_4 = create_filtration_result( - '/path/to/summary4', + './summary4', event_data.EVENT_TRAIN_DICT_4, event_data.EVENT_EVAL_DICT_4, event_data.METRIC_4, event_data.DATASET_DICT_0 ) LINEAGE_FILTRATION_5 = { - "summary_dir": '/path/to/summary5', + "summary_dir": './summary5', "model_lineage": { "loss_function": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'], @@ -219,10 +221,11 @@ LINEAGE_FILTRATION_5 = { "dataset_mark": '2', "user_defined": {} }, - "dataset_graph": event_data.DATASET_DICT_0 + "dataset_graph": event_data.DATASET_DICT_0, + "added_info": {} } LINEAGE_FILTRATION_6 = { - "summary_dir": '/path/to/summary6', + "summary_dir": './summary6', "model_lineage": { "loss_function": None, "train_dataset_path": None, @@ -243,15 +246,16 @@ LINEAGE_FILTRATION_6 = { "dataset_mark": '2', "user_defined": {} }, - "dataset_graph": event_data.DATASET_DICT_0 + "dataset_graph": event_data.DATASET_DICT_0, + "added_info": {} } class TestQuerier(TestCase): """Test the class of `Querier`.""" + _MOCK_DATA_MANAGER = MagicMock() @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_lineage_summaries') - @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryWatcher.list_summary_directories') @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_user_defined_info') @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_summary_infos') @mock.patch('mindinsight.lineagemgr.lineage_parser.FileHandler') @@ -263,139 +267,23 @@ class TestQuerier(TestCase): event_data.EVENT_DATASET_DICT_0 ) args[1].return_value = [] - args[3].return_value = ['path'] + args[2].return_value = ['path'] mock_file_handler = MagicMock() mock_file_handler.size = 1 - args[2].return_value = [{'relative_path': './', 'update_time': 1}] - single_summary_path = '/path/to/summary0' - lineage_objects = LineageOrganizer(summary_base_dir=single_summary_path).super_lineage_objs - self.single_querier = Querier(lineage_objects) + summary_dir = '/path/test/' lineage_infos = get_lineage_infos() args[0].side_effect = lineage_infos - summary_base_dir = '/path/to' - relative_dirs = [] + lineage_objects = {} for i in range(7): - relative_dirs.append(dict(relative_path=f'./summary{i}', update_time=time.time() - i)) - args[2].return_value = relative_dirs - lineage_objects = LineageOrganizer(summary_base_dir=summary_base_dir).super_lineage_objs - self.multi_querier = Querier(lineage_objects) - - def test_get_summary_lineage_success_1(self): - """Test the success of get_summary_lineage.""" - expected_result = [LINEAGE_INFO_0] - result = self.single_querier.get_summary_lineage() - assert_equal_lineages(expected_result, result, self.assertListEqual) - - def test_get_summary_lineage_success_2(self): - """Test the success of get_summary_lineage.""" - expected_result = [LINEAGE_INFO_0] - result = self.single_querier.get_summary_lineage() - assert_equal_lineages(expected_result, result, self.assertListEqual) - - def test_get_summary_lineage_success_3(self): - """Test the success of get_summary_lineage.""" - expected_result = [ - { - 'summary_dir': '/path/to/summary0', - 'model': event_data.EVENT_TRAIN_DICT_0['train_lineage']['model'], - 'algorithm': event_data.EVENT_TRAIN_DICT_0['train_lineage']['algorithm'] - } - ] - result = self.single_querier.get_summary_lineage( - filter_keys=['model', 'algorithm'] - ) - assert_equal_lineages(expected_result, result, self.assertListEqual) - - def test_get_summary_lineage_success_4(self): - """Test the success of get_summary_lineage.""" - expected_result = [ - LINEAGE_INFO_0, - LINEAGE_INFO_1, - { - 'summary_dir': '/path/to/summary2', - **event_data.EVENT_TRAIN_DICT_2['train_lineage'], - 'metric': event_data.METRIC_2, - 'valid_dataset': event_data.EVENT_EVAL_DICT_2['evaluation_lineage']['valid_dataset'], - 'dataset_graph': event_data.DATASET_DICT_0 - }, - { - 'summary_dir': '/path/to/summary3', - **event_data.EVENT_TRAIN_DICT_3['train_lineage'], - 'metric': event_data.METRIC_3, - 'valid_dataset': event_data.EVENT_EVAL_DICT_3['evaluation_lineage']['valid_dataset'], - 'dataset_graph': event_data.DATASET_DICT_0 - }, - { - 'summary_dir': '/path/to/summary4', - **event_data.EVENT_TRAIN_DICT_4['train_lineage'], - 'metric': event_data.METRIC_4, - 'valid_dataset': event_data.EVENT_EVAL_DICT_4['evaluation_lineage']['valid_dataset'], - 'dataset_graph': event_data.DATASET_DICT_0 - }, - { - 'summary_dir': '/path/to/summary5', - **event_data.EVENT_TRAIN_DICT_5['train_lineage'], - 'metric': {}, - 'valid_dataset': {}, - 'dataset_graph': event_data.DATASET_DICT_0 - }, - { - 'summary_dir': '/path/to/summary6', - 'hyper_parameters': {}, - 'algorithm': {}, - 'model': {}, - 'train_dataset': {}, - 'metric': event_data.METRIC_5, - 'valid_dataset': event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset'], - 'dataset_graph': event_data.DATASET_DICT_0 - } - ] - result = self.multi_querier.get_summary_lineage() - assert_equal_lineages(expected_result, result, self.assertListEqual) - - def test_get_summary_lineage_success_5(self): - """Test the success of get_summary_lineage.""" - expected_result = [LINEAGE_INFO_1] - result = self.multi_querier.get_summary_lineage( - summary_dir='/path/to/summary1' - ) - assert_equal_lineages(expected_result, result, self.assertListEqual) - - def test_get_summary_lineage_success_6(self): - """Test the success of get_summary_lineage.""" - expected_result = [ - { - 'summary_dir': '/path/to/summary0', - 'hyper_parameters': event_data.EVENT_TRAIN_DICT_0['train_lineage']['hyper_parameters'], - 'train_dataset': event_data.EVENT_TRAIN_DICT_0['train_lineage']['train_dataset'], - 'metric': event_data.METRIC_0, - 'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'] - } - ] - filter_keys = [ - 'metric', 'hyper_parameters', 'train_dataset', 'valid_dataset' - ] - result = self.multi_querier.get_summary_lineage( - summary_dir='/path/to/summary0', filter_keys=filter_keys - ) - assert_equal_lineages(expected_result, result, self.assertListEqual) - - def test_get_summary_lineage_fail(self): - """Test the function of get_summary_lineage with exception.""" - filter_keys = ['xxx'] - self.assertRaises( - LineageQuerierParamException, - self.multi_querier.get_summary_lineage, - filter_keys=filter_keys - ) + train_id = f'./summary{i}' + summary_dir = os.path.join(summary_dir, train_id) + update_time = time.time() - i + lineage_parser = LineageParser(train_id, summary_dir, update_time) + lineage_objects.update({train_id: lineage_parser.super_lineage_obj}) - self.assertRaises( - LineageQuerierParamException, - self.multi_querier.get_summary_lineage, - summary_dir='xxx' - ) + self.multi_querier = Querier(lineage_objects) def test_filter_summary_lineage_success_1(self): """Test the success of filter_summary_lineage.""" diff --git a/tests/ut/lineagemgr/querier/test_query_model.py b/tests/ut/lineagemgr/querier/test_query_model.py index 88d7691b..589c9583 100644 --- a/tests/ut/lineagemgr/querier/test_query_model.py +++ b/tests/ut/lineagemgr/querier/test_query_model.py @@ -133,6 +133,7 @@ class TestLineageObj(TestCase): ) expected_result['model_lineage']['dataset_mark'] = None expected_result.pop('dataset_graph') + expected_result.pop('added_info') result = self.lineage_obj.to_model_lineage_dict() assert_equal_lineages(expected_result, result, self.assertDictEqual) diff --git a/tests/ut/lineagemgr/test_model.py b/tests/ut/lineagemgr/test_model.py index 7d9c89b3..fecf37b7 100644 --- a/tests/ut/lineagemgr/test_model.py +++ b/tests/ut/lineagemgr/test_model.py @@ -16,224 +16,60 @@ from unittest import TestCase, mock from unittest.mock import MagicMock -from mindinsight.lineagemgr.model import get_summary_lineage, filter_summary_lineage, \ - _convert_relative_path_to_abspath, get_flattened_lineage -from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamSummaryPathError, \ - LineageFileNotFoundError, LineageSummaryParseException, LineageQuerierParamException, \ - LineageQuerySummaryDataError, LineageSearchConditionParamError, LineageParamTypeError, \ - LineageParamValueError +from mindinsight.lineagemgr.model import filter_summary_lineage, get_flattened_lineage +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSummaryParseException, \ + LineageQuerierParamException, LineageQuerySummaryDataError, LineageSearchConditionParamError, LineageParamTypeError from mindinsight.lineagemgr.common.path_parser import SummaryPathParser from ...st.func.lineagemgr.test_model import LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2 -class TestModel(TestCase): - """Test the function of get_summary_lineage and filter_summary_lineage.""" - - @mock.patch('mindinsight.lineagemgr.model.Querier') - @mock.patch('mindinsight.lineagemgr.model.LineageParser') - @mock.patch('os.path.isdir') - def test_get_summary_lineage_success(self, isdir_mock, parser_mock, qurier_mock): - """Test the function of get_summary_lineage.""" - isdir_mock.return_value = True - parser_mock.return_value = MagicMock() - - mock_querier = MagicMock() - qurier_mock.return_value = mock_querier - mock_querier.get_summary_lineage.return_value = [{'algorithm': {'network': 'ResNet'}}] - summary_dir = '/path/to/summary_dir' - result = get_summary_lineage(None, summary_dir, keys=['algorithm']) - self.assertEqual(result, {'algorithm': {'network': 'ResNet'}}) - - def test_get_summary_lineage_failed(self): - """Test get_summary_lineage failed.""" - invalid_path = '../fake_dir' - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - get_summary_lineage, - None, - invalid_path - ) - - @mock.patch('mindinsight.lineagemgr.common.utils.validate_path') - @mock.patch.object(SummaryPathParser, 'get_lineage_summaries') - def test_get_summary_lineage_failed2(self, mock_summary, mock_valid): - """Test get_summary_lineage failed.""" - mock_summary.return_value = [] - mock_valid.return_value = '/path/to/summary/dir' - self.assertRaisesRegex( - LineageFileNotFoundError, - 'no summary log file under summary_dir', - get_summary_lineage, - None, - '/path/to/summary_dir' - ) - - @mock.patch('mindinsight.lineagemgr.lineage_parser.FileHandler') - @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageParser._parse_summary_log') - @mock.patch('mindinsight.lineagemgr.common.utils.validate_path') - @mock.patch.object(SummaryPathParser, 'get_lineage_summaries') - def test_get_summary_lineage_failed3(self, - mock_summary, - mock_valid, - mock_parser, - mock_file_handler): - """Test get_summary_lineage failed.""" - mock_summary.return_value = ['/path/to/summary/file'] - mock_valid.return_value = '/path/to/summary_dir' - mock_parser.return_value = None - mock_file_handler = MagicMock() - mock_file_handler.size = 1 - result = get_summary_lineage(None, '/path/to/summary_dir') - assert {} == result - - @mock.patch('mindinsight.lineagemgr.model.validate_path') - def test_convert_relative_path_to_abspath(self, validate_path_mock): - """Test the function of converting realtive path to abspath.""" - validate_path_mock.return_value = '/path/to/summary_base_dir/summary_dir' - summary_base_dir = '/path/to/summary_base_dir' - search_condition = { - 'summary_dir': { - 'in': ['/path/to/summary_base_dir'] - } - } - result = _convert_relative_path_to_abspath(summary_base_dir, - search_condition) - self.assertDictEqual( - result, {'summary_dir': {'in': ['/path/to/summary_base_dir/summary_dir']}}) - - search_condition = { - 'summary_dir': { - 'in': ['./summary_dir'] - } - } - result = _convert_relative_path_to_abspath(summary_base_dir, search_condition) - self.assertDictEqual( - result, {'summary_dir': {'in': ['/path/to/summary_base_dir/summary_dir']}} - ) - - search_condition = { - 'summary_dir': { - 'eq': '/summary_dir' - } - } - result = _convert_relative_path_to_abspath(summary_base_dir, search_condition) - self.assertDictEqual( - result, {'summary_dir': {'eq': '/path/to/summary_base_dir/summary_dir'}}) - - search_condition = { - 'summary_dir': None - } - result = _convert_relative_path_to_abspath(summary_base_dir, search_condition) - self.assertDictEqual( - result, search_condition - ) - - class TestFilterAPI(TestCase): """Test the function of filter_summary_lineage.""" - @mock.patch('mindinsight.lineagemgr.model.LineageOrganizer') + _MOCK_DATA_MANAGER = MagicMock() + @mock.patch('mindinsight.lineagemgr.model.Querier') @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_lineage_summaries') - @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath') - @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir') - def test_filter_summary_lineage(self, validate_path_mock, convert_path_mock, - latest_summary_mock, qurier_mock, organizer_mock): + def test_filter_summary_lineage(self, latest_summary_mock, qurier_mock): """Test the function of filter_summary_lineage.""" - convert_path_mock.return_value = { - 'summary_dir': { - 'in': ['/path/to/summary_base_dir'] - }, - 'loss': { - 'gt': 2.0 - } - } - organizer_mock = MagicMock() - organizer_mock.super_lineage_objs = None - validate_path_mock.return_value = True latest_summary_mock.return_value = ['/path/to/summary_base_dir/summary_dir'] mock_querier = MagicMock() qurier_mock.return_value = mock_querier mock_querier.filter_summary_lineage.return_value = [{'loss': 3.0}] - summary_base_dir = '/path/to/summary_base_dir' - result = filter_summary_lineage(None, summary_base_dir) + result = filter_summary_lineage(self._MOCK_DATA_MANAGER) self.assertEqual(result, [{'loss': 3.0}]) - def test_invalid_path(self): - """Test filter_summary_lineage with invalid path.""" - invalid_path = '../fake_dir' - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'The summary path is invalid.', - filter_summary_lineage, - None, - invalid_path - ) - @mock.patch('mindinsight.lineagemgr.model.validate_condition') - @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir') - def test_invalid_search_condition(self, mock_path, mock_valid): + def test_invalid_search_condition(self, mock_valid): """Test filter_summary_lineage with invalid invalid param.""" - mock_path.return_value = None mock_valid.side_effect = LineageParamTypeError( 'Invalid search_condition type.') self.assertRaisesRegex( LineageSearchConditionParamError, 'Invalid search_condition type.', filter_summary_lineage, - None, - '/path/to/summary/dir', + self._MOCK_DATA_MANAGER, 'invalid_condition' ) - @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition') - @mock.patch('mindinsight.lineagemgr.model.validate_condition') - @mock.patch('mindinsight.lineagemgr.common.utils.validate_path') - @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath') - def test_failed_to_convert_path(self, mock_convert, *args): + def test_failed_to_get_summary_files(self): """Test filter_summary_lineage with invalid invalid param.""" - mock_convert.side_effect = LineageParamValueError('invalid path') - args[0].return_value = None - self.assertRaisesRegex( - LineageParamSummaryPathError, - 'invalid path', - filter_summary_lineage, - None, - '/path/to/summary/dir', - {} - ) - - @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath') - @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition') - @mock.patch('mindinsight.lineagemgr.model.validate_condition') - @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir') - @mock.patch.object(SummaryPathParser, 'get_lineage_summaries') - def test_failed_to_get_summary_filesh(self, mock_parse, *args): - """Test filter_summary_lineage with invalid invalid param.""" - path = '/path/to/summary/dir' - mock_parse.return_value = [] - args[0].return_value = path - self.assertRaisesRegex( - LineageFileNotFoundError, - 'There is no summary log file under summary_base_dir.', - filter_summary_lineage, - None, - path - ) + default_result = { + 'customized': {}, + 'object': [], + 'count': 0 + } + assert default_result == filter_summary_lineage(self._MOCK_DATA_MANAGER) - @mock.patch('mindinsight.lineagemgr.model._convert_relative_path_to_abspath') @mock.patch('mindinsight.lineagemgr.model.validate_search_model_condition') @mock.patch('mindinsight.lineagemgr.model.validate_condition') - @mock.patch('mindinsight.lineagemgr.model.normalize_summary_dir') @mock.patch.object(SummaryPathParser, 'get_lineage_summaries') @mock.patch('mindinsight.lineagemgr.model.Querier') - def test_failed_to_querier(self, mock_query, mock_parse, *args): + def test_failed_to_querier(self, mock_query, *args): """Test filter_summary_lineage with invalid invalid param.""" mock_query.side_effect = LineageSummaryParseException() - mock_parse.return_value = ['/path/to/summary/file'] args[0].return_value = None - res = filter_summary_lineage(None, '/path/to/summary') + res = filter_summary_lineage(self._MOCK_DATA_MANAGER) assert res == {'object': [], 'count': 0} mock_query.side_effect = LineageQuerierParamException(['keys'], 'key') @@ -241,8 +77,7 @@ class TestFilterAPI(TestCase): LineageQuerySummaryDataError, 'Filter summary lineage failed.', filter_summary_lineage, - None, - '/path/to/summary/dir' + self._MOCK_DATA_MANAGER ) @mock.patch('mindinsight.lineagemgr.model.filter_summary_lineage') @@ -251,8 +86,7 @@ class TestFilterAPI(TestCase): mock_data = { 'object': [LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2] } - mock_datamanager = MagicMock() - mock_datamanager.summary_base_dir = '/tmp/' + mock_data_manager = MagicMock() mock_filter_summary_lineage.return_value = mock_data - result = get_flattened_lineage(mock_datamanager, None) + result = get_flattened_lineage(mock_data_manager) assert result.get('[U]info') == ['info1', None] diff --git a/tests/utils/tools.py b/tests/utils/tools.py index 6c15cb39..a2f59ec4 100644 --- a/tests/utils/tools.py +++ b/tests/utils/tools.py @@ -19,12 +19,14 @@ import io import os import shutil import json +from pathlib import Path from urllib.parse import urlencode - import numpy as np from PIL import Image +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError + def get_url(url, params): """ @@ -138,3 +140,25 @@ def assert_equal_lineages(lineages1, lineages2, assert_func, decimal_num=2): else: deal_float_for_dict(lineages1, lineages2, decimal_num) assert_func(lineages1, lineages2) + + +def get_relative_path(path, base_path): + """ + Get relative path based on base_path. + + Args: + path (str): absolute path. + base_path: absolute base path. + + Returns: + str, relative path based on base_path. + + """ + try: + r_path = str(Path(path).relative_to(Path(base_path))) + except ValueError: + raise LineageParamValueError("The path %r does not start with %r." % (path, base_path)) + + if r_path == ".": + r_path = "" + return os.path.join("./", r_path)