remove redundant code in lineagemgr

mindinsight/lineagemgr/common/utils.py

@@ -13,15 +13,13 @@
 # limitations under the License.
 # ============================================================================
 """Lineage utils."""
-import os
 import re
 
 from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
-from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError
-from mindinsight.lineagemgr.common.log import logger as log
 
 
 def enum_to_list(enum):
+    """Enum to list."""
     return [enum_ele.value for enum_ele in enum]
 
 
@@ -29,28 +27,3 @@ def get_timestamp(filename):
     """Get timestamp from filename."""
     timestamp = int(re.search(SummaryWatcher().SUMMARY_FILENAME_REGEX, filename)[1])
     return timestamp
-
-
-def make_directory(path):
-    """Make directory."""
-    if path is None or not isinstance(path, str) or not path.strip():
-        log.error("Invalid input path: %r.", path)
-        raise LineageParamTypeError("Invalid path type")
-
-    # convert relative path to abs path
-    path = os.path.realpath(path)
-    log.debug("The abs path is %r", path)
-
-    # check path exist and its write permissions]
-    if os.path.exists(path):
-        real_path = path
-    else:
-        # All exceptions need to be caught because create directory maybe have some limit(permissions)
-        log.debug("The directory(%s) doesn't exist, will create it", path)
-        try:
-            os.makedirs(path, exist_ok=True)
-            real_path = path
-        except PermissionError as err:
-            log.error("No write permission on the directory(%r), error = %r", path, err)
-            raise LineageParamTypeError("No write permission on the directory.")
-    return real_path

mindinsight/lineagemgr/common/validator/model_parameter.py

@@ -15,37 +15,13 @@
 """Define schema of model lineage input parameters."""
 from marshmallow import Schema, fields, ValidationError, pre_load, validates
 
-from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrorMsg, \
-    LineageErrors
-from mindinsight.lineagemgr.common.exceptions.exceptions import \
-    LineageParamTypeError, LineageParamValueError
-from mindinsight.lineagemgr.common.log import logger
+from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrorMsg, LineageErrors
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, LineageParamValueError
 from mindinsight.lineagemgr.common.utils import enum_to_list
 from mindinsight.lineagemgr.querier.querier import LineageType
 from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING
 from mindinsight.utils.exceptions import MindInsightException
 
-try:
-    from mindspore.dataset.engine import Dataset
-except (ImportError, ModuleNotFoundError):
-    logger.error('MindSpore Not Found!')
-
-
-
-class EvalParameter(Schema):
-    """Define the parameter schema for Evaluation job."""
-    valid_dataset = fields.Function(allow_none=True)
-    metrics = fields.Dict(allow_none=True)
-
-    @pre_load
-    def check_valid_dataset(self, data, **kwargs):
-        valid_dataset = data.get("valid_dataset")
-        if valid_dataset and not isinstance(valid_dataset, Dataset):
-            raise ValidationError({'valid_dataset': [
-                "Parameter valid_dataset must be an instance of "
-                "mindspore.dataengine.datasets.Dataset"]})
-        return data
-
 
 class SearchModelConditionParameter(Schema):
     """Define the search model condition parameter schema."""

mindinsight/lineagemgr/common/validator/validate.py

@@ -13,34 +13,12 @@
 # limitations under the License.
 # ============================================================================
 """Validate the parameters."""
-import re
-from marshmallow import ValidationError
-
 from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg
 from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, LineageParamValueError
 from mindinsight.lineagemgr.common.log import logger as log
-from mindinsight.lineagemgr.common.validator.validate_path import safe_normalize_path
 from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING
 from mindinsight.utils.exceptions import MindInsightException, ParamValueError
 
-
-# Named string regular expression
-_name_re = r"^\w+[0-9a-zA-Z\_\.]*$"
-
-TRAIN_RUN_CONTEXT_ERROR_MAPPING = {
-    'optimizer': LineageErrors.PARAM_OPTIMIZER_ERROR,
-    'loss_fn': LineageErrors.PARAM_LOSS_FN_ERROR,
-    'net_outputs': LineageErrors.PARAM_NET_OUTPUTS_ERROR,
-    'train_network': LineageErrors.PARAM_TRAIN_NETWORK_ERROR,
-    'train_dataset': LineageErrors.PARAM_DATASET_ERROR,
-    'epoch_num': LineageErrors.PARAM_EPOCH_NUM_ERROR,
-    'batch_num': LineageErrors.PARAM_BATCH_NUM_ERROR,
-    'parallel_mode': LineageErrors.PARAM_TRAIN_PARALLEL_ERROR,
-    'device_number': LineageErrors.PARAM_DEVICE_NUMBER_ERROR,
-    'list_callback': LineageErrors.PARAM_CALLBACK_LIST_ERROR,
-    'train_dataset_size': LineageErrors.PARAM_DATASET_SIZE_ERROR,
-}
-
 SEARCH_MODEL_ERROR_MAPPING = {
     'summary_dir': LineageErrors.LINEAGE_PARAM_SUMMARY_DIR_ERROR,
     'loss_function': LineageErrors.LINEAGE_PARAM_LOSS_FUNCTION_ERROR,
@@ -64,19 +42,6 @@ SEARCH_MODEL_ERROR_MAPPING = {
     'lineage_type': LineageErrors.LINEAGE_PARAM_LINEAGE_TYPE_ERROR
 }
 
-
-TRAIN_RUN_CONTEXT_ERROR_MSG_MAPPING = {
-    'optimizer': LineageErrorMsg.PARAM_OPTIMIZER_ERROR.value,
-    'loss_fn': LineageErrorMsg.PARAM_LOSS_FN_ERROR.value,
-    'net_outputs': LineageErrorMsg.PARAM_NET_OUTPUTS_ERROR.value,
-    'train_network': LineageErrorMsg.PARAM_TRAIN_NETWORK_ERROR.value,
-    'epoch_num': LineageErrorMsg.PARAM_EPOCH_NUM_ERROR.value,
-    'batch_num': LineageErrorMsg.PARAM_BATCH_NUM_ERROR.value,
-    'parallel_mode': LineageErrorMsg.PARAM_TRAIN_PARALLEL_ERROR.value,
-    'device_number': LineageErrorMsg.PARAM_DEVICE_NUMBER_ERROR.value,
-    'list_callback': LineageErrorMsg.PARAM_CALLBACK_LIST_ERROR.value
-}
-
 SEARCH_MODEL_ERROR_MSG_MAPPING = {
     'summary_dir': LineageErrorMsg.LINEAGE_PARAM_SUMMARY_DIR_ERROR.value,
     'loss_function': LineageErrorMsg.LINEAGE_LOSS_FUNCTION_ERROR.value,
@@ -101,84 +66,6 @@ SEARCH_MODEL_ERROR_MSG_MAPPING = {
 }
 
 
-EVAL_RUN_CONTEXT_ERROR_MAPPING = {
-    'valid_dataset': LineageErrors.PARAM_DATASET_ERROR,
-    'metrics': LineageErrors.PARAM_EVAL_METRICS_ERROR
-}
-
-EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING = {
-    'metrics': LineageErrorMsg.PARAM_EVAL_METRICS_ERROR.value,
-}
-
-
-def validate_int_params(int_param, param_name):
-    """
-    Verify the parameter which type is integer valid or not.
-
-    Args:
-        int_param (int): parameter that is integer,
-            including epoch, dataset_batch_size, step_num
-        param_name (str): the name of parameter,
-            including epoch, dataset_batch_size, step_num
-
-    Raises:
-        MindInsightException: If the parameters are invalid.
-    """
-    if not isinstance(int_param, int) or int_param <= 0 or int_param > pow(2, 63) - 1:
-        if param_name == 'step_num':
-            log.error('Invalid step_num. The step number should be a positive integer.')
-            raise MindInsightException(error=LineageErrors.PARAM_STEP_NUM_ERROR,
-                                       message=LineageErrorMsg.PARAM_STEP_NUM_ERROR.value)
-
-        if param_name == 'dataset_batch_size':
-            log.error('Invalid dataset_batch_size. '
-                      'The batch size should be a positive integer.')
-            raise MindInsightException(error=LineageErrors.PARAM_BATCH_SIZE_ERROR,
-                                       message=LineageErrorMsg.PARAM_BATCH_SIZE_ERROR.value)
-
-
-def validate_file_path(file_path, allow_empty=False):
-    """
-    Verify that the file_path is valid.
-
-    Args:
-        file_path (str): Input file path.
-        allow_empty (bool): Whether file_path can be empty.
-
-    Raises:
-        MindInsightException: If the parameters are invalid.
-    """
-    try:
-        if allow_empty and not file_path:
-            return file_path
-        return safe_normalize_path(file_path, raise_key='dataset_path', safe_prefixes=None)
-    except ValidationError as error:
-        log.error(str(error))
-        raise MindInsightException(error=LineageErrors.PARAM_FILE_PATH_ERROR,
-                                   message=str(error))
-
-
-def validate_eval_run_context(schema, data):
-    """
-    Validate mindspore evaluation job run_context data according to schema.
-
-    Args:
-        schema (Schema): data schema.
-        data (dict): data to check schema.
-
-    Raises:
-        MindInsightException: If the parameters are invalid.
-    """
-    errors = schema().validate(data)
-    for error_key, error_msg in errors.items():
-        if error_key in EVAL_RUN_CONTEXT_ERROR_MAPPING.keys():
-            error_code = EVAL_RUN_CONTEXT_ERROR_MAPPING.get(error_key)
-            if EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key):
-                error_msg = EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key)
-            log.error(error_msg)
-            raise MindInsightException(error=error_code, message=error_msg)
-
-
 def validate_search_model_condition(schema, data):
     """
     Validate search model condition.
@@ -203,25 +90,6 @@ def validate_search_model_condition(schema, data):
             raise MindInsightException(error=error_code, message=error_msg)
 
 
-def validate_raise_exception(raise_exception):
-    """
-    Validate raise_exception.
-
-    Args:
-        raise_exception (bool): decide raise exception or not,
-            if True, raise exception; else, catch exception and continue.
-
-    Raises:
-        MindInsightException: If the parameters are invalid.
-    """
-    if not isinstance(raise_exception, bool):
-        log.error("Invalid raise_exception. It should be True or False.")
-        raise MindInsightException(
-            error=LineageErrors.PARAM_RAISE_EXCEPTION_ERROR,
-            message=LineageErrorMsg.PARAM_RAISE_EXCEPTION_ERROR.value
-        )
-
-
 def validate_condition(search_condition):
     """
     Verify the param in search_condition is valid or not.
@@ -276,44 +144,6 @@ def validate_condition(search_condition):
             raise LineageParamValueError(err_msg)
 
 
-def validate_user_defined_info(user_defined_info):
-    """
-    Validate user defined info， delete the item if its key is in lineage.
-
-    Args:
-        user_defined_info (dict): The user defined info.
-
-    Raises:
-        LineageParamTypeError: If the type of parameters is invalid.
-        LineageParamValueError: If user defined keys have been defined in lineage.
-
-    """
-    if not isinstance(user_defined_info, dict):
-        log.error("Invalid user defined info. It should be a dict.")
-        raise LineageParamTypeError("Invalid user defined info. It should be dict.")
-    for key, value in user_defined_info.items():
-        if not isinstance(key, str):
-            error_msg = "Dict key type {} is not supported in user defined info." \
-                        "Only str is permitted now.".format(type(key))
-            log.error(error_msg)
-            raise LineageParamTypeError(error_msg)
-        if not isinstance(value, (int, str, float)):
-            error_msg = "Dict value type {} is not supported in user defined info." \
-                        "Only str, int and float are permitted now.".format(type(value))
-            log.error(error_msg)
-            raise LineageParamTypeError(error_msg)
-
-    field_map = set(FIELD_MAPPING.keys())
-    user_defined_keys = set(user_defined_info.keys())
-    insertion = list(field_map & user_defined_keys)
-
-    if insertion:
-        for key in insertion:
-            user_defined_info.pop(key)
-        raise LineageParamValueError("There are some keys have defined in lineage. "
-                                     "Duplicated key(s): %s. " % insertion)
-
-
 def validate_train_id(relative_path):
     """
     Check if train_id is valid.
@@ -385,27 +215,3 @@ def validate_added_info(added_info: dict):
                 raise LineageParamValueError("'remark' must be str.")
             # length of remark should be in [0, 128].
             validate_range("length of remark", len(value), min_value=0, max_value=128)
-
-
-def validate_str_by_regular(target, reg=None, flag=re.ASCII):
-    """
-    Validate string by given regular.
-
-    Args:
-        target: target string.
-        reg: pattern.
-        flag: pattern mode.
-
-    Raises:
-        LineageParamValueError, if string not match given pattern.
-
-    Returns:
-        bool, if target matches pattern, return True.
-
-    """
-    if reg is None:
-        reg = _name_re
-    if re.match(reg, target, flag) is None:
-        raise LineageParamValueError("'{}' is illegal, it should be match "
-                                     "regular'{}' by flags'{}'".format(target, reg, flag))
-    return True

mindinsight/lineagemgr/summary/_summary_adapter.py

@@ -14,6 +14,7 @@
 # ============================================================================
 """The converter between proto format event of lineage and dict."""
 
+
 def organize_graph(graph_message):
     """
     Convert a dataset graph to its dict format.

tests/st/func/lineagemgr/collection/model/__init__.py

@@ -12,3 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
+"""Lineage collection."""

tests/st/func/lineagemgr/collection/model/test_model_lineage.py

@@ -28,8 +28,6 @@ from unittest import TestCase, mock
 import numpy as np
 import pytest
 from mindinsight.lineagemgr.model import filter_summary_lineage
-from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors
-from mindinsight.utils.exceptions import MindInsightException
 
 from mindspore.application.model_zoo.resnet import ResNet
 from mindspore.common.tensor import Tensor
@@ -213,14 +211,12 @@ class TestModelLineage(TestCase):
     @pytest.mark.platform_x86_ascend_training
     @pytest.mark.platform_x86_cpu
     @pytest.mark.env_single
-    @mock.patch('tests.utils.lineage_writer.model_lineage.validate_eval_run_context')
-    @mock.patch.object(AnalyzeObject, 'get_file_size', return_value=64)
-    def test_raise_exception_record_trainlineage(self, *args):
+    @mock.patch.object(AnalyzeObject, 'get_file_size')
+    def test_raise_exception_record_trainlineage(self, mock_analyze):
         """Test exception when error happened after recording training infos."""
+        mock_analyze.return_value = 64
         if os.path.exists(SUMMARY_DIR_3):
             shutil.rmtree(SUMMARY_DIR_3)
-        args[1].side_effect = MindInsightException(error=LineageErrors.PARAM_RUN_CONTEXT_ERROR,
-                                                   message="RunContext error.")
         train_callback = TrainLineage(SUMMARY_DIR_3, True)
         train_callback.begin(RunContext(self.run_context))
         full_file_name = train_callback.lineage_summary.lineage_log_path

tests/utils/lineage_writer/model_lineage.py

@@ -19,17 +19,13 @@ import os
 import numpy as np
 from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrorMsg, LineageErrors
 from mindinsight.lineagemgr.common.log import logger as log
-from mindinsight.lineagemgr.common.utils import make_directory
-from mindinsight.lineagemgr.common.validator.model_parameter import EvalParameter
-from mindinsight.lineagemgr.common.validator.validate import (validate_eval_run_context, validate_file_path,
-                                                              validate_int_params,
-                                                              validate_raise_exception,
-                                                              validate_user_defined_info)
 from mindinsight.utils.exceptions import MindInsightException
 
 from ._summary_record import LineageSummary
 from .base import Metadata
-from .utils import try_except, LineageParamRunContextError, LineageGetModelFileError, LineageLogError
+from .utils import try_except, LineageParamRunContextError, LineageGetModelFileError, LineageLogError, \
+    validate_int_params, validate_file_path, validate_raise_exception, \
+    validate_user_defined_info, make_directory
 
 try:
     from mindspore.common.tensor import Tensor
@@ -308,7 +304,6 @@ class EvalLineage(Callback):
             raise LineageParamRunContextError(error_msg)
 
         run_context_args = run_context.original_args()
-        validate_eval_run_context(EvalParameter, run_context_args)
 
         valid_dataset = run_context_args.get('valid_dataset')
 

tests/utils/lineage_writer/utils.py

@@ -13,9 +13,16 @@
 # limitations under the License.
 # ============================================================================
 """Lineage writer utils."""
+import os
 from functools import wraps
 
+from marshmallow import ValidationError
+
 from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg
+from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError, LineageParamValueError
+from mindinsight.lineagemgr.common.log import logger as log
+from mindinsight.lineagemgr.common.validator.validate_path import safe_normalize_path
+from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING
 from mindinsight.utils.exceptions import MindInsightException
 
 
@@ -81,3 +88,141 @@ def try_except(logger):
 
         return wrapper
     return try_except_decorate
+
+
+def validate_int_params(int_param, param_name):
+    """
+    Verify the parameter which type is integer valid or not.
+
+    Args:
+        int_param (int): parameter that is integer,
+            including epoch, dataset_batch_size, step_num
+        param_name (str): the name of parameter,
+            including epoch, dataset_batch_size, step_num
+
+    Raises:
+        MindInsightException: If the parameters are invalid.
+    """
+    if not isinstance(int_param, int) or int_param <= 0 or int_param > pow(2, 63) - 1:
+        if param_name == 'step_num':
+            log.error('Invalid step_num. The step number should be a positive integer.')
+            raise MindInsightException(error=LineageErrors.PARAM_STEP_NUM_ERROR,
+                                       message=LineageErrorMsg.PARAM_STEP_NUM_ERROR.value)
+
+        if param_name == 'dataset_batch_size':
+            log.error('Invalid dataset_batch_size. '
+                      'The batch size should be a positive integer.')
+            raise MindInsightException(error=LineageErrors.PARAM_BATCH_SIZE_ERROR,
+                                       message=LineageErrorMsg.PARAM_BATCH_SIZE_ERROR.value)
+
+
+def validate_file_path(file_path, allow_empty=False):
+    """
+    Verify that the file_path is valid.
+
+    Args:
+        file_path (str): Input file path.
+        allow_empty (bool): Whether file_path can be empty.
+
+    Raises:
+        MindInsightException: If the parameters are invalid.
+    """
+    try:
+        if allow_empty and not file_path:
+            return file_path
+        return safe_normalize_path(file_path, raise_key='dataset_path', safe_prefixes=None)
+    except ValidationError as error:
+        log.error(str(error))
+        raise MindInsightException(error=LineageErrors.PARAM_FILE_PATH_ERROR,
+                                   message=str(error))
+
+
+EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING = {
+    'metrics': LineageErrorMsg.PARAM_EVAL_METRICS_ERROR.value,
+}
+EVAL_RUN_CONTEXT_ERROR_MAPPING = {
+    'valid_dataset': LineageErrors.PARAM_DATASET_ERROR,
+    'metrics': LineageErrors.PARAM_EVAL_METRICS_ERROR
+}
+
+
+def validate_raise_exception(raise_exception):
+    """
+    Validate raise_exception.
+
+    Args:
+        raise_exception (bool): decide raise exception or not,
+            if True, raise exception; else, catch exception and continue.
+
+    Raises:
+        MindInsightException: If the parameters are invalid.
+    """
+    if not isinstance(raise_exception, bool):
+        log.error("Invalid raise_exception. It should be True or False.")
+        raise MindInsightException(
+            error=LineageErrors.PARAM_RAISE_EXCEPTION_ERROR,
+            message=LineageErrorMsg.PARAM_RAISE_EXCEPTION_ERROR.value
+        )
+
+
+def validate_user_defined_info(user_defined_info):
+    """
+    Validate user defined info， delete the item if its key is in lineage.
+
+    Args:
+        user_defined_info (dict): The user defined info.
+
+    Raises:
+        LineageParamTypeError: If the type of parameters is invalid.
+        LineageParamValueError: If user defined keys have been defined in lineage.
+
+    """
+    if not isinstance(user_defined_info, dict):
+        log.error("Invalid user defined info. It should be a dict.")
+        raise LineageParamTypeError("Invalid user defined info. It should be dict.")
+    for key, value in user_defined_info.items():
+        if not isinstance(key, str):
+            error_msg = "Dict key type {} is not supported in user defined info." \
+                        "Only str is permitted now.".format(type(key))
+            log.error(error_msg)
+            raise LineageParamTypeError(error_msg)
+        if not isinstance(value, (int, str, float)):
+            error_msg = "Dict value type {} is not supported in user defined info." \
+                        "Only str, int and float are permitted now.".format(type(value))
+            log.error(error_msg)
+            raise LineageParamTypeError(error_msg)
+
+    field_map = set(FIELD_MAPPING.keys())
+    user_defined_keys = set(user_defined_info.keys())
+    insertion = list(field_map & user_defined_keys)
+
+    if insertion:
+        for key in insertion:
+            user_defined_info.pop(key)
+        raise LineageParamValueError("There are some keys have defined in lineage. "
+                                     "Duplicated key(s): %s. " % insertion)
+
+
+def make_directory(path):
+    """Make directory."""
+    if path is None or not isinstance(path, str) or not path.strip():
+        log.error("Invalid input path: %r.", path)
+        raise LineageParamTypeError("Invalid path type")
+
+    # convert relative path to abs path
+    path = os.path.realpath(path)
+    log.debug("The abs path is %r", path)
+
+    # check path exist and its write permissions]
+    if os.path.exists(path):
+        real_path = path
+    else:
+        # All exceptions need to be caught because create directory maybe have some limit(permissions)
+        log.debug("The directory(%s) doesn't exist, will create it", path)
+        try:
+            os.makedirs(path, exist_ok=True)
+            real_path = path
+        except PermissionError as err:
+            log.error("No write permission on the directory(%r), error = %r", path, err)
+            raise LineageParamTypeError("No write permission on the directory.")
+    return real_path