From f54d57cd06d7a5296adb644c8afc9d8ff4a59657 Mon Sep 17 00:00:00 2001 From: luopengting Date: Fri, 18 Sep 2020 15:24:51 +0800 Subject: [PATCH] modify http_code to 400 in lineagemgr, move deprecated exceptions to tests --- .../common/exceptions/exceptions.py | 68 ++++++--------- mindinsight/lineagemgr/common/utils.py | 38 +-------- tests/utils/lineage_writer/model_lineage.py | 5 +- tests/utils/lineage_writer/utils.py | 83 +++++++++++++++++++ 4 files changed, 112 insertions(+), 82 deletions(-) create mode 100644 tests/utils/lineage_writer/utils.py diff --git a/mindinsight/lineagemgr/common/exceptions/exceptions.py b/mindinsight/lineagemgr/common/exceptions/exceptions.py index c5de07c6..3856ab21 100644 --- a/mindinsight/lineagemgr/common/exceptions/exceptions.py +++ b/mindinsight/lineagemgr/common/exceptions/exceptions.py @@ -23,7 +23,8 @@ class LineageParamTypeError(MindInsightException): def __init__(self, msg): super(LineageParamTypeError, self).__init__( error=LineageErrors.PARAM_TYPE_ERROR, - message=LineageErrorMsg.PARAM_TYPE_ERROR.value.format(msg) + message=LineageErrorMsg.PARAM_TYPE_ERROR.value.format(msg), + http_code=400 ) @@ -33,27 +34,8 @@ class LineageParamValueError(MindInsightException): def __init__(self, msg): super(LineageParamValueError, self).__init__( error=LineageErrors.PARAM_VALUE_ERROR, - message=LineageErrorMsg.PARAM_VALUE_ERROR.value.format(msg) - ) - - -class LineageParamRunContextError(MindInsightException): - """The input parameter run_context error in lineage module.""" - - def __init__(self, msg): - super(LineageParamRunContextError, self).__init__( - error=LineageErrors.PARAM_RUN_CONTEXT_ERROR, - message=LineageErrorMsg.PARAM_RUN_CONTEXT_ERROR.value.format(msg) - ) - - -class LineageGetModelFileError(MindInsightException): - """The get model file error in lineage module.""" - - def __init__(self, msg): - super(LineageGetModelFileError, self).__init__( - error=LineageErrors.LINEAGE_GET_MODEL_FILE_ERROR, - message=LineageErrorMsg.LINEAGE_GET_MODEL_FILE_ERROR.value.format(msg) + message=LineageErrorMsg.PARAM_VALUE_ERROR.value.format(msg), + http_code=400 ) @@ -65,7 +47,8 @@ class LineageSummaryAnalyzeException(MindInsightException): msg = '' super(LineageSummaryAnalyzeException, self).__init__( error=LineageErrors.SUMMARY_ANALYZE_ERROR, - message=LineageErrorMsg.SUMMARY_ANALYZE_ERROR.value.format(msg) + message=LineageErrorMsg.SUMMARY_ANALYZE_ERROR.value.format(msg), + http_code=400 ) @@ -74,16 +57,8 @@ class LineageVerificationException(MindInsightException): def __init__(self, msg): super(LineageVerificationException, self).__init__( error=LineageErrors.SUMMARY_VERIFICATION_ERROR, - message=LineageErrorMsg.SUMMARY_VERIFICATION_ERROR.value.format(msg) - ) - - -class LineageLogError(MindInsightException): - """The lineage collector error.""" - def __init__(self, msg): - super(LineageLogError, self).__init__( - error=LineageErrors.LOG_LINEAGE_INFO_ERROR, - message=LineageErrorMsg.LOG_LINEAGE_INFO_ERROR.value.format(msg) + message=LineageErrorMsg.SUMMARY_VERIFICATION_ERROR.value.format(msg), + http_code=400 ) @@ -93,7 +68,8 @@ class LineageEventNotExistException(MindInsightException): def __init__(self): super(LineageEventNotExistException, self).__init__( error=LineageErrors.EVENT_NOT_EXIST_ERROR, - message=LineageErrorMsg.EVENT_NOT_EXIST_ERROR.value + message=LineageErrorMsg.EVENT_NOT_EXIST_ERROR.value, + http_code=400 ) @@ -103,7 +79,8 @@ class LineageQuerierParamException(MindInsightException): def __init__(self, *msg): super(LineageQuerierParamException, self).__init__( error=LineageErrors.QUERIER_PARAM_ERROR, - message=LineageErrorMsg.QUERIER_PARAM_ERROR.value.format(*msg) + message=LineageErrorMsg.QUERIER_PARAM_ERROR.value.format(*msg), + http_code=400 ) @@ -113,7 +90,8 @@ class LineageSummaryParseException(MindInsightException): def __init__(self): super(LineageSummaryParseException, self).__init__( error=LineageErrors.SUMMARY_PARSE_FAIL_ERROR, - message=LineageErrorMsg.SUMMARY_PARSE_FAIL_ERROR.value + message=LineageErrorMsg.SUMMARY_PARSE_FAIL_ERROR.value, + http_code=400 ) @@ -123,7 +101,8 @@ class LineageEventFieldNotExistException(MindInsightException): def __init__(self, msg): super(LineageEventFieldNotExistException, self).__init__( error=LineageErrors.EVENT_FIELD_NOT_EXIST_ERROR, - message=LineageErrorMsg.EVENT_FIELD_NOT_EXIST_ERROR.value.format(msg) + message=LineageErrorMsg.EVENT_FIELD_NOT_EXIST_ERROR.value.format(msg), + http_code=400 ) @@ -132,7 +111,8 @@ class LineageParamSummaryPathError(MindInsightException): def __init__(self, msg): super(LineageParamSummaryPathError, self).__init__( error=LineageErrors.LINEAGE_PARAM_SUMMARY_PATH_ERROR, - message=LineageErrorMsg.LINEAGE_PARAM_SUMMARY_PATH_ERROR.value.format(msg) + message=LineageErrorMsg.LINEAGE_PARAM_SUMMARY_PATH_ERROR.value.format(msg), + http_code=400 ) @@ -141,7 +121,8 @@ class LineageQuerySummaryDataError(MindInsightException): def __init__(self, msg): super(LineageQuerySummaryDataError, self).__init__( error=LineageErrors.LINEAGE_SUMMARY_DATA_ERROR, - message=LineageErrorMsg.LINEAGE_SUMMARY_DATA_ERROR.value.format(msg) + message=LineageErrorMsg.LINEAGE_SUMMARY_DATA_ERROR.value.format(msg), + http_code=400 ) @@ -150,7 +131,8 @@ class LineageFileNotFoundError(MindInsightException): def __init__(self, msg): super(LineageFileNotFoundError, self).__init__( error=LineageErrors.LINEAGE_FILE_NOT_FOUND_ERROR, - message=LineageErrorMsg.LINEAGE_FILE_NOT_FOUND_ERROR.value.format(msg) + message=LineageErrorMsg.LINEAGE_FILE_NOT_FOUND_ERROR.value.format(msg), + http_code=400 ) @@ -159,7 +141,8 @@ class LineageDirNotExistError(MindInsightException): def __init__(self, msg): super(LineageDirNotExistError, self).__init__( error=LineageErrors.LINEAGE_DIR_NOT_EXIST_ERROR, - message=LineageErrorMsg.LINEAGE_DIR_NOT_EXIST_ERROR.value.format(msg) + message=LineageErrorMsg.LINEAGE_DIR_NOT_EXIST_ERROR.value.format(msg), + http_code=400 ) @@ -168,5 +151,6 @@ class LineageSearchConditionParamError(MindInsightException): def __init__(self, msg): super(LineageSearchConditionParamError, self).__init__( error=LineageErrors.LINEAGE_SEARCH_CONDITION_PARAM_ERROR, - message=LineageErrorMsg.LINEAGE_SEARCH_CONDITION_PARAM_ERROR.value.format(msg) + message=LineageErrorMsg.LINEAGE_SEARCH_CONDITION_PARAM_ERROR.value.format(msg), + http_code=400 ) diff --git a/mindinsight/lineagemgr/common/utils.py b/mindinsight/lineagemgr/common/utils.py index eca1e485..fe295574 100644 --- a/mindinsight/lineagemgr/common/utils.py +++ b/mindinsight/lineagemgr/common/utils.py @@ -15,54 +15,19 @@ """Lineage utils.""" import os import re -from functools import wraps from pathlib import Path from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher -from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \ - LineageGetModelFileError, LineageLogError, LineageParamValueError, LineageParamTypeError, \ +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError, LineageParamTypeError, \ LineageDirNotExistError, LineageParamSummaryPathError from mindinsight.lineagemgr.common.log import logger as log from mindinsight.lineagemgr.common.validator.validate import validate_path -from mindinsight.utils.exceptions import MindInsightException def enum_to_list(enum): return [enum_ele.value for enum_ele in enum] -def try_except(logger): - """ - Catch or raise exceptions while collecting lineage. - - Args: - logger (logger): The logger instance which logs the warning info. - - Returns: - function, the decorator which we use to retry the decorated function. - """ - def try_except_decorate(func): - @wraps(func) - def wrapper(self, *args, **kwargs): - try: - func(self, *args, **kwargs) - except (AttributeError, MindInsightException, - LineageParamRunContextError, LineageLogError, - LineageGetModelFileError, IOError) as err: - logger.error(err) - - try: - raise_except = self.raise_exception - except AttributeError: - raise_except = False - - if raise_except is True: - raise - - return wrapper - return try_except_decorate - - def normalize_summary_dir(summary_dir): """Normalize summary dir.""" try: @@ -82,7 +47,6 @@ def get_timestamp(filename): def make_directory(path): """Make directory.""" - real_path = None if path is None or not isinstance(path, str) or not path.strip(): log.error("Invalid input path: %r.", path) raise LineageParamTypeError("Invalid path type") diff --git a/tests/utils/lineage_writer/model_lineage.py b/tests/utils/lineage_writer/model_lineage.py index 931e683d..82f9c61d 100644 --- a/tests/utils/lineage_writer/model_lineage.py +++ b/tests/utils/lineage_writer/model_lineage.py @@ -18,10 +18,8 @@ import os import numpy as np from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrorMsg, LineageErrors -from mindinsight.lineagemgr.common.exceptions.exceptions import (LineageGetModelFileError, LineageLogError, - LineageParamRunContextError) from mindinsight.lineagemgr.common.log import logger as log -from mindinsight.lineagemgr.common.utils import make_directory, try_except +from mindinsight.lineagemgr.common.utils import make_directory from mindinsight.lineagemgr.common.validator.model_parameter import EvalParameter from mindinsight.lineagemgr.common.validator.validate import (validate_eval_run_context, validate_file_path, validate_int_params, @@ -31,6 +29,7 @@ from mindinsight.utils.exceptions import MindInsightException from ._summary_record import LineageSummary from .base import Metadata +from .utils import try_except, LineageParamRunContextError, LineageGetModelFileError, LineageLogError try: from mindspore.common.tensor import Tensor diff --git a/tests/utils/lineage_writer/utils.py b/tests/utils/lineage_writer/utils.py new file mode 100644 index 00000000..7acd8a9e --- /dev/null +++ b/tests/utils/lineage_writer/utils.py @@ -0,0 +1,83 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Lineage writer utils.""" +from functools import wraps + +from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg +from mindinsight.utils.exceptions import MindInsightException + + +class LineageParamRunContextError(MindInsightException): + """The input parameter run_context error in lineage module.""" + + def __init__(self, msg): + super(LineageParamRunContextError, self).__init__( + error=LineageErrors.PARAM_RUN_CONTEXT_ERROR, + message=LineageErrorMsg.PARAM_RUN_CONTEXT_ERROR.value.format(msg), + http_code=400 + ) + + +class LineageGetModelFileError(MindInsightException): + """The get model file error in lineage module.""" + + def __init__(self, msg): + super(LineageGetModelFileError, self).__init__( + error=LineageErrors.LINEAGE_GET_MODEL_FILE_ERROR, + message=LineageErrorMsg.LINEAGE_GET_MODEL_FILE_ERROR.value.format(msg), + http_code=400 + ) + + +class LineageLogError(MindInsightException): + """The lineage collector error.""" + def __init__(self, msg): + super(LineageLogError, self).__init__( + error=LineageErrors.LOG_LINEAGE_INFO_ERROR, + message=LineageErrorMsg.LOG_LINEAGE_INFO_ERROR.value.format(msg), + http_code=400 + ) + + +def try_except(logger): + """ + Catch or raise exceptions while collecting lineage. + + Args: + logger (logger): The logger instance which logs the warning info. + + Returns: + function, the decorator which we use to retry the decorated function. + """ + def try_except_decorate(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + try: + func(self, *args, **kwargs) + except (AttributeError, MindInsightException, + LineageParamRunContextError, LineageLogError, + LineageGetModelFileError, IOError) as err: + logger.error(err) + + try: + raise_except = self.raise_exception + except AttributeError: + raise_except = False + + if raise_except is True: + raise + + return wrapper + return try_except_decorate