auto choose nodes when create watchpoints add actual_value when hit watchpoints add error_code when to show the Nan and Inf errortags/v1.1.0
| @@ -21,7 +21,7 @@ This module also provide the available conditions to condition_collections api. | |||
| import math | |||
| from enum import Enum | |||
| from mindinsight.conditionmgr.log import logger | |||
| from mindinsight.debugger.conditionmgr.log import logger | |||
| class ConditionIdEnum(Enum): | |||
| @@ -53,9 +53,6 @@ class ConditionIdEnum(Enum): | |||
| WEIGHT_NOT_CHANGED = "weight_not_changed" | |||
| WEIGHT_CHANGE_TOO_LARGE = "weight_change_too_large" | |||
| WEIGHT_CHANGE_TOO_SMALL = "weight_change_too_small" | |||
| TENSOR_CHANGE_TOO_LARGE = "tensor_change_too_large" | |||
| TENSOR_CHANGE_TOO_SMALL = "tensor_change_too_small" | |||
| TENSOR_NOT_CHANGED = "tensor_not_changed" | |||
| ACTIVATION_RANGE = "activation_range" | |||
| TENSOR_RANGE = "tensor_range" | |||
| @@ -17,19 +17,19 @@ Condition list. | |||
| This module provide the detail conditions list. | |||
| """ | |||
| from mindinsight.conditionmgr.condition import Condition | |||
| from mindinsight.conditionmgr.condition import OptimizePhaseEnum | |||
| from mindinsight.conditionmgr.condition import ConditionParameter | |||
| from mindinsight.conditionmgr.condition import ValueTypeEnum | |||
| from mindinsight.conditionmgr.condition import TargetTypeEnum | |||
| from mindinsight.conditionmgr.condition import PlatformEnum | |||
| from mindinsight.conditionmgr.condition import ParamTypeEnum | |||
| from mindinsight.conditionmgr.condition import ConditionIdEnum | |||
| from mindinsight.conditionmgr.condition import check_initialization_available | |||
| from mindinsight.conditionmgr.condition import check_normal_param_range | |||
| from mindinsight.conditionmgr.condition import check_percentage_param_range | |||
| from mindinsight.conditionmgr.condition import check_abs_param_range | |||
| from mindinsight.conditionmgr.condition import check_not_nan | |||
| from mindinsight.debugger.conditionmgr.condition import Condition | |||
| from mindinsight.debugger.conditionmgr.condition import OptimizePhaseEnum | |||
| from mindinsight.debugger.conditionmgr.condition import ConditionParameter | |||
| from mindinsight.debugger.conditionmgr.condition import ValueTypeEnum | |||
| from mindinsight.debugger.conditionmgr.condition import TargetTypeEnum | |||
| from mindinsight.debugger.conditionmgr.condition import PlatformEnum | |||
| from mindinsight.debugger.conditionmgr.condition import ParamTypeEnum | |||
| from mindinsight.debugger.conditionmgr.condition import ConditionIdEnum | |||
| from mindinsight.debugger.conditionmgr.condition import check_initialization_available | |||
| from mindinsight.debugger.conditionmgr.condition import check_normal_param_range | |||
| from mindinsight.debugger.conditionmgr.condition import check_percentage_param_range | |||
| from mindinsight.debugger.conditionmgr.condition import check_abs_param_range | |||
| from mindinsight.debugger.conditionmgr.condition import check_not_nan | |||
| CONDITION_LIST = [ | |||
| @@ -569,89 +569,10 @@ CONDITION_LIST = [ | |||
| supported_platforms=(PlatformEnum.ASCEND, PlatformEnum.GPU), | |||
| minimum_debugger_capability=(1, 1) | |||
| ), | |||
| Condition( | |||
| condition_id=ConditionIdEnum.TENSOR_CHANGE_TOO_LARGE, | |||
| abbr="TCL", | |||
| # Send this condition to MindSpore will use WatchCondition.Condition.tensor_change_too_large | |||
| optimize_phase=OptimizePhaseEnum.TENSOR_CHECK, | |||
| parameters=[ | |||
| ConditionParameter( | |||
| name="abs_update_ratio_mean_gt", | |||
| value_type=ValueTypeEnum.FLOAT64, | |||
| valid_test_func=check_abs_param_range, | |||
| default_value=1e-1 | |||
| ), | |||
| ConditionParameter( | |||
| name="epsilon", | |||
| value_type=ValueTypeEnum.FLOAT64, | |||
| support_disable=False, | |||
| default_value=1e-9, | |||
| visible_on_ui=False | |||
| ) | |||
| ], | |||
| supported_target_type=TargetTypeEnum.TENSOR, | |||
| supported_platforms=(PlatformEnum.ASCEND, PlatformEnum.GPU), | |||
| minimum_debugger_capability=(1, 1) | |||
| ), | |||
| Condition( | |||
| condition_id=ConditionIdEnum.TENSOR_CHANGE_TOO_SMALL, | |||
| abbr="TCS", | |||
| # Send this condition to MindSpore will use WatchCondition.Condition.tensor_change_too_small | |||
| optimize_phase=OptimizePhaseEnum.TENSOR_CHECK, | |||
| parameters=[ | |||
| ConditionParameter( | |||
| name="abs_update_ratio_mean_lt", | |||
| value_type=ValueTypeEnum.FLOAT64, | |||
| valid_test_func=check_abs_param_range, | |||
| default_value=1e-4 | |||
| ), | |||
| ConditionParameter( | |||
| name="epsilon", | |||
| value_type=ValueTypeEnum.FLOAT64, | |||
| support_disable=False, | |||
| default_value=1e-9, | |||
| visible_on_ui=False | |||
| ) | |||
| ], | |||
| supported_target_type=TargetTypeEnum.TENSOR, | |||
| supported_platforms=(PlatformEnum.ASCEND, PlatformEnum.GPU), | |||
| minimum_debugger_capability=(1, 1) | |||
| ), | |||
| Condition( | |||
| condition_id=ConditionIdEnum.TENSOR_NOT_CHANGED, | |||
| abbr="TNC", | |||
| # Send this condition to MindSpore will use WatchCondition.Condition.tensor_not_changed | |||
| optimize_phase=OptimizePhaseEnum.TENSOR_CHECK, | |||
| parameters=[ | |||
| ConditionParameter( | |||
| name="rtol", | |||
| value_type=ValueTypeEnum.FLOAT64, | |||
| valid_test_func=check_abs_param_range, | |||
| default_value=1e-5 | |||
| ), | |||
| ConditionParameter( | |||
| name="atol", | |||
| value_type=ValueTypeEnum.FLOAT64, | |||
| support_disable=False, | |||
| default_value=1e-8, | |||
| visible_on_ui=False | |||
| ), | |||
| ConditionParameter( | |||
| name="equal_nan", | |||
| value_type=ValueTypeEnum.BOOL, | |||
| support_disable=False, | |||
| default_value=False, | |||
| visible_on_ui=False | |||
| ) | |||
| ], | |||
| supported_target_type=TargetTypeEnum.TENSOR, | |||
| supported_platforms=(PlatformEnum.ASCEND, PlatformEnum.GPU), | |||
| minimum_debugger_capability=(1, 1) | |||
| ), | |||
| Condition( | |||
| condition_id=ConditionIdEnum.ACTIVATION_RANGE, | |||
| abbr="AR", | |||
| # Send this condition to MindSpore will use WatchCondition.Condition.tensor_not_changed | |||
| # Send this condition to MindSpore will use WatchCondition.Condition.activation_range | |||
| optimize_phase=OptimizePhaseEnum.TENSOR_CHECK, | |||
| parameters=[ | |||
| ConditionParameter( | |||
| @@ -696,7 +617,7 @@ CONDITION_LIST = [ | |||
| Condition( | |||
| condition_id=ConditionIdEnum.TENSOR_RANGE, | |||
| abbr="TR", | |||
| # Send this condition to MindSpore will use WatchCondition.Condition.tensor_not_changed | |||
| # Send this condition to MindSpore will use WatchCondition.Condition.tensor_range | |||
| optimize_phase=OptimizePhaseEnum.TENSOR_CHECK, | |||
| parameters=[ | |||
| ConditionParameter( | |||
| @@ -17,10 +17,10 @@ Condition manager.. | |||
| This module provide condition manager function. | |||
| """ | |||
| from mindinsight.conditionmgr.condition import Condition | |||
| from mindinsight.conditionmgr.condition import TargetTypeEnum | |||
| from mindinsight.conditionmgr.condition_list import CONDITION_LIST | |||
| from mindinsight.conditionmgr.log import logger | |||
| from mindinsight.debugger.conditionmgr.condition import Condition | |||
| from mindinsight.debugger.conditionmgr.condition import TargetTypeEnum | |||
| from mindinsight.debugger.conditionmgr.condition_list import CONDITION_LIST | |||
| from mindinsight.debugger.conditionmgr.log import logger | |||
| class ConditionMgr: | |||
| @@ -19,11 +19,11 @@ This module predefine recommend watchpoints. | |||
| """ | |||
| import queue as Queue | |||
| from mindinsight.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.conditionmgr.condition import TargetTypeEnum | |||
| from mindinsight.conditionmgr.condition import ConditionIdEnum | |||
| from mindinsight.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.conditionmgr.log import logger | |||
| from mindinsight.debugger.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.debugger.conditionmgr.condition import TargetTypeEnum | |||
| from mindinsight.debugger.conditionmgr.condition import ConditionIdEnum | |||
| from mindinsight.debugger.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.debugger.conditionmgr.log import logger | |||
| from mindinsight.conf import settings | |||
| @@ -43,7 +43,6 @@ class _WatchPointData: | |||
| "id": self.watch_condition.get("condition"), | |||
| "params": [{ | |||
| "name": param.get_parameter_name(), | |||
| "disable": False, | |||
| "value": param.value | |||
| } for param in self.watch_condition.get("params")] | |||
| } | |||
| @@ -81,7 +80,7 @@ def recommend_watchpoints(condition_mgr: ConditionMgr, graph_stream, condition_c | |||
| return watch_points | |||
| # add weight watch points | |||
| merged_info = _get_basic_node_info(TargetTypeEnum.WEIGHT.value, graph_stream) | |||
| merged_info = get_basic_node_info(TargetTypeEnum.WEIGHT.value, graph_stream) | |||
| _recommend_weight_initialization(merged_info, condition_mgr, watch_points, condition_context) | |||
| _recommend_weight_change_too_large(merged_info, condition_mgr, watch_points, condition_context) | |||
| @@ -92,11 +91,11 @@ def recommend_watchpoints(condition_mgr: ConditionMgr, graph_stream, condition_c | |||
| _recommend_weight_change_too_small(condition_mgr, trainable_weight_nodes, watch_points, condition_context) | |||
| # add gradient watch points | |||
| merged_info = _get_basic_node_info(TargetTypeEnum.GRADIENT.value, graph_stream) | |||
| merged_info = get_basic_node_info(TargetTypeEnum.GRADIENT.value, graph_stream) | |||
| _recommend_gradient_vanishing(merged_info, condition_mgr, watch_points, condition_context) | |||
| # add tensor watch points | |||
| merged_info = _get_basic_node_info(TargetTypeEnum.TENSOR.value, graph_stream) | |||
| merged_info = get_basic_node_info(TargetTypeEnum.TENSOR.value, graph_stream) | |||
| _recommend_overflow_ascend_chip(merged_info, condition_mgr, watch_points, condition_context) | |||
| _recommend_tensor_overflow(merged_info, condition_mgr, watch_points, condition_context) | |||
| _recommend_tensor_all_zero(merged_info, condition_mgr, watch_points, condition_context) | |||
| @@ -272,7 +271,7 @@ def _recommend_weight_initialization(basic_info_nodes, condition_mgr, watch_poin | |||
| watch_points.append(weight_initialization_watchpoint) | |||
| def _get_basic_node_info(node_category, graph_stream): | |||
| def get_basic_node_info(node_category, graph_stream): | |||
| """Get node merged info.""" | |||
| basic_info_nodes = _get_basic_node_info_by_node_category(node_category, graph_stream) | |||
| merged_info = _merge_nodes(basic_info_nodes, graph_stream.whole_graph) | |||
| @@ -13,6 +13,8 @@ | |||
| # limitations under the License. | |||
| # ============================================================================ | |||
| """Implement the debugger grpc server.""" | |||
| import copy | |||
| from functools import wraps | |||
| from mindinsight.debugger.common.log import LOGGER as log | |||
| @@ -443,10 +445,22 @@ class DebuggerGrpcServer(grpc_server_base.EventListenerServicer): | |||
| continue | |||
| watchpoint_hit = { | |||
| 'tensor_proto': watchpoint_hit_proto.tensor, | |||
| 'watchpoint': watchpoint_stream.get_watchpoint_by_id(watchpoint_hit_proto.id), | |||
| 'watchpoint': copy.deepcopy(watchpoint_stream.get_watchpoint_by_id(watchpoint_hit_proto.id)), | |||
| 'node_name': ui_node_name, | |||
| 'graph_name': graph_name | |||
| } | |||
| hit_params = {} | |||
| for param in watchpoint_hit_proto.watch_condition.params: | |||
| if param.actual_value: | |||
| hit_params[param.name] = param.actual_value | |||
| for i, param in enumerate(watchpoint_hit['watchpoint'].condition['params']): | |||
| name = param['name'] | |||
| if name in hit_params.keys(): | |||
| watchpoint_hit['watchpoint'].condition['params'][i]['actual_value'] = hit_params[name] | |||
| else: | |||
| watchpoint_hit['watchpoint'].condition['params'][i]['actual_value'] = None | |||
| if watchpoint_hit_proto.error_code: | |||
| watchpoint_hit['error_code'] = watchpoint_hit_proto.error_code | |||
| watchpoint_hits.append(watchpoint_hit) | |||
| self._received_hit = watchpoint_hits | |||
| reply = get_ack_reply() | |||
| @@ -20,9 +20,9 @@ from threading import Thread | |||
| import grpc | |||
| from mindinsight.conditionmgr.condition import ConditionContext | |||
| from mindinsight.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.conditionmgr.recommender import recommend_watchpoints | |||
| from mindinsight.debugger.conditionmgr.condition import ConditionContext | |||
| from mindinsight.debugger.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.debugger.conditionmgr.recommender import recommend_watchpoints | |||
| from mindinsight.conf import settings | |||
| from mindinsight.datavisual.data_transform.graph import NodeTypeEnum | |||
| from mindinsight.datavisual.utils.tools import to_float | |||
| @@ -517,7 +517,6 @@ class DebuggerServer: | |||
| "params": [ | |||
| { | |||
| "name": "abs_mean_gt", | |||
| "disable": false, | |||
| "value": 1.1 | |||
| } | |||
| ] | |||
| @@ -13,12 +13,12 @@ | |||
| # limitations under the License. | |||
| # ============================================================================ | |||
| """Define the watchpoint stream.""" | |||
| from mindinsight.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.debugger.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.debugger.common.exceptions.exceptions import DebuggerParamValueError | |||
| from mindinsight.debugger.common.log import LOGGER as log | |||
| from mindinsight.debugger.common.utils import is_scope_type | |||
| from mindinsight.debugger.proto.debug_grpc_pb2 import SetCMD, WatchCondition | |||
| from mindinsight.conditionmgr.condition import ConditionIdEnum | |||
| from mindinsight.debugger.conditionmgr.condition import ConditionIdEnum | |||
| WATCHPOINT_CONDITION_MAPPING = { | |||
| @@ -46,11 +46,8 @@ WATCHPOINT_CONDITION_MAPPING = { | |||
| ConditionIdEnum.WEIGHT_TOO_SMALL.value: WatchCondition.Condition.tensor_too_small, | |||
| ConditionIdEnum.GRADIENT_VANISHING.value: WatchCondition.Condition.tensor_too_small, | |||
| ConditionIdEnum.TENSOR_ALL_ZERO.value: WatchCondition.Condition.tensor_all_zero, | |||
| ConditionIdEnum.TENSOR_CHANGE_TOO_LARGE.value: WatchCondition.Condition.tensor_change_too_large, | |||
| ConditionIdEnum.WEIGHT_CHANGE_TOO_LARGE.value: WatchCondition.Condition.tensor_change_too_large, | |||
| ConditionIdEnum.TENSOR_CHANGE_TOO_SMALL.value: WatchCondition.Condition.tensor_change_too_small, | |||
| ConditionIdEnum.WEIGHT_CHANGE_TOO_SMALL.value: WatchCondition.Condition.tensor_change_too_small, | |||
| ConditionIdEnum.TENSOR_NOT_CHANGED.value: WatchCondition.Condition.tensor_not_changed, | |||
| ConditionIdEnum.WEIGHT_NOT_CHANGED.value: WatchCondition.Condition.tensor_not_changed | |||
| } | |||
| @@ -288,7 +285,7 @@ class Watchpoint: | |||
| param_proto = set_cmd.watch_condition.params.add() | |||
| param_proto.name = param.get('name') | |||
| param_proto.value = param.get('value') | |||
| param_proto.disabled = param.get('disable') | |||
| param_proto.disabled = False | |||
| # Only one parameter of condition in current version. | |||
| set_cmd.watch_condition.value = param.get('value') | |||
| @@ -317,6 +314,7 @@ class WatchpointHit: | |||
| self.node_name = node_name | |||
| self.slot = tensor_proto.slot | |||
| self.graph_name = graph_name | |||
| self.error_code = 0 | |||
| @property | |||
| def tensor_full_name(self): | |||
| @@ -13,8 +13,8 @@ | |||
| # limitations under the License. | |||
| # ============================================================================ | |||
| """Define the graph stream handler.""" | |||
| from mindinsight.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.conditionmgr.condition import TargetTypeEnum as CategoryTypeEnum | |||
| from mindinsight.debugger.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.debugger.conditionmgr.condition import TargetTypeEnum as CategoryTypeEnum | |||
| from mindinsight.debugger.common.exceptions.exceptions import DebuggerParamValueError, \ | |||
| DebuggerNodeNotInGraphError, DebuggerGraphNotExistError | |||
| from mindinsight.debugger.common.log import LOGGER as log | |||
| @@ -13,7 +13,7 @@ | |||
| # limitations under the License. | |||
| # ============================================================================ | |||
| """Define the watchpoint stream handler.""" | |||
| from mindinsight.conditionmgr.condition import ValueTypeEnum | |||
| from mindinsight.debugger.conditionmgr.condition import ValueTypeEnum | |||
| from mindinsight.debugger.common.exceptions.exceptions import DebuggerParamValueError, \ | |||
| DebuggerParamTypeError | |||
| from mindinsight.debugger.common.log import LOGGER as log | |||
| @@ -256,7 +256,6 @@ class WatchpointHandler(StreamHandlerBase): | |||
| "params": [ | |||
| { | |||
| "name": "abs_mean_gt", | |||
| "disable": false, | |||
| "value": 1.1 | |||
| } | |||
| ] | |||
| @@ -403,6 +402,8 @@ class WatchpointHitHandler(StreamHandlerBase): | |||
| node_name=value.get('node_name'), | |||
| graph_name=value.get('graph_name') | |||
| ) | |||
| if 'error_code' in value.keys(): | |||
| watchpoint_hit.error_code = value.get('error_code') | |||
| # get all hit watchpoints according to node name ans tensor slot | |||
| watchpoint_hits = self._get_watchpoints_by_tensor_name(watchpoint_hit.node_name, | |||
| watchpoint_hit.slot) | |||
| @@ -482,10 +483,24 @@ class WatchpointHitHandler(StreamHandlerBase): | |||
| dict, tensor hit info. | |||
| """ | |||
| res = {} | |||
| watch_points = [tensor_hit.watchpoint for tensor_hit in tensor_hits] | |||
| watch_points = [] | |||
| error_codes = set() | |||
| for tensor_hit in tensor_hits: | |||
| error_code = tensor_hit.error_code | |||
| watchpoint = tensor_hit.watchpoint | |||
| watchpoint['error_code'] = error_code | |||
| watch_points.append(watchpoint) | |||
| error_codes.add(error_code) | |||
| summarized_error_code = error_codes.pop() | |||
| while error_codes: | |||
| temp = error_codes.pop() | |||
| summarized_error_code = summarized_error_code | temp | |||
| if watch_points: | |||
| res = { | |||
| 'slot': slot, | |||
| 'summarized_error_code': summarized_error_code, | |||
| 'watch_points': watch_points | |||
| } | |||
| return res | |||
| @@ -608,7 +623,6 @@ def set_default_param(condition_mgr, watch_condition): | |||
| "params": [ | |||
| { | |||
| "name": "abs_mean_gt", | |||
| "disable": false, | |||
| "value": 1.1 | |||
| } | |||
| ] | |||
| @@ -625,7 +639,6 @@ def set_default_param(condition_mgr, watch_condition): | |||
| if not param.visible_on_ui and not param.support_disable: | |||
| watch_condition["params"].append({ | |||
| "name": param.name, | |||
| "disable": False, | |||
| "value": param.default_value | |||
| }) | |||
| watch_condition["abbr"] = condition.abbr | |||
| @@ -14,8 +14,10 @@ | |||
| # ============================================================================ | |||
| """This module is aimed to deal with watchpoint commands.""" | |||
| from mindinsight.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.conditionmgr.condition import ConditionIdEnum | |||
| from mindinsight.debugger.conditionmgr.common.utils import NodeBasicInfo | |||
| from mindinsight.debugger.conditionmgr.condition import ConditionIdEnum, TargetTypeEnum | |||
| from mindinsight.debugger.conditionmgr.recommender import get_basic_node_info | |||
| from mindinsight.debugger.stream_handler.watchpoint_handler import validate_watch_condition | |||
| from mindinsight.debugger.common.exceptions.exceptions import DebuggerParamValueError, \ | |||
| DebuggerCreateWatchPointError, DebuggerUpdateWatchPointError, \ | |||
| DebuggerDeleteWatchPointError | |||
| @@ -44,7 +46,6 @@ class WatchpointOperator: | |||
| "params": [ | |||
| { | |||
| "name": "abs_mean_gt", | |||
| "disable": false, | |||
| "value": 1.1 | |||
| } | |||
| ] | |||
| @@ -73,6 +74,14 @@ class WatchpointOperator: | |||
| node_names=params.get('watch_nodes'), | |||
| search_pattern=params.get('search_pattern'), | |||
| graph_name=params.get('graph_name')) | |||
| validate_watch_condition(self._condition_mgr, watch_condition) | |||
| condition_id = watch_condition.get('id') | |||
| condition = self._condition_mgr.get_condition(condition_id) | |||
| if condition.supported_target_type in [TargetTypeEnum.ACTIVATION, TargetTypeEnum.GRADIENT, | |||
| TargetTypeEnum.WEIGHT]: | |||
| watch_nodes = get_basic_node_info(condition.supported_target_type.value, self._graph_stream).copy() | |||
| watchpoint_stream = self._watchpoint_stream | |||
| watch_point_id = watchpoint_stream.create_watchpoint( | |||
| self._condition_mgr, watch_condition, watch_nodes, params.get('watch_point_id')) | |||
| @@ -1,184 +1 @@ | |||
| { | |||
| "graph": { | |||
| "nodes": [ | |||
| { | |||
| "name": "Default/args0", | |||
| "full_name": "Default/args0", | |||
| "type": "Parameter", | |||
| "input": {}, | |||
| "output": { | |||
| "Default/TransData-op99": { | |||
| "shape": [ | |||
| [ | |||
| 32, | |||
| 1, | |||
| 32, | |||
| 32 | |||
| ] | |||
| ], | |||
| "edge_type": "data", | |||
| "independent_layout": false, | |||
| "data_type": "DT_TENSOR[DT_FLOAT32]", | |||
| "slot_mapping": [ | |||
| [ | |||
| "0", | |||
| "" | |||
| ] | |||
| ] | |||
| } | |||
| }, | |||
| "slots": [ | |||
| { | |||
| "slot": "0", | |||
| "statistics": { | |||
| "overall_max": 6.0, | |||
| "overall_min": 1.0, | |||
| "overall_avg": 3.5, | |||
| "overall_count": 6, | |||
| "overall_nan_count": 0, | |||
| "overall_neg_inf_count": 0, | |||
| "overall_pos_inf_count": 0, | |||
| "overall_zero_count": 0.0, | |||
| "overall_neg_zero_count": 0.0, | |||
| "overall_pos_zero_count": 6.0 | |||
| }, | |||
| "shape": [ | |||
| 2, | |||
| 3 | |||
| ], | |||
| "has_prev_step": true | |||
| } | |||
| ], | |||
| "graph_name": "graph_0" | |||
| }, | |||
| { | |||
| "name": "Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op190", | |||
| "full_name": "Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op190", | |||
| "type": "Cast", | |||
| "input": { | |||
| "Default/TransData-op99": { | |||
| "shape": [ | |||
| [ | |||
| 32, | |||
| 1, | |||
| 32, | |||
| 32 | |||
| ] | |||
| ], | |||
| "edge_type": "data", | |||
| "independent_layout": false, | |||
| "data_type": "DT_TENSOR[DT_FLOAT32]", | |||
| "slot_mapping": [ | |||
| [ | |||
| "0", | |||
| "" | |||
| ] | |||
| ] | |||
| } | |||
| }, | |||
| "output": {}, | |||
| "slots": [ | |||
| { | |||
| "slot": "0", | |||
| "statistics": { | |||
| "overall_max": 6.0, | |||
| "overall_min": 1.0, | |||
| "overall_avg": 3.5, | |||
| "overall_count": 6, | |||
| "overall_nan_count": 0, | |||
| "overall_neg_inf_count": 0, | |||
| "overall_pos_inf_count": 0, | |||
| "overall_zero_count": 0.0, | |||
| "overall_neg_zero_count": 0.0, | |||
| "overall_pos_zero_count": 6.0 | |||
| }, | |||
| "shape": [ | |||
| 2, | |||
| 3 | |||
| ] | |||
| } | |||
| ], | |||
| "graph_name": "graph_0" | |||
| }, | |||
| { | |||
| "name": "Default/TransData-op99", | |||
| "full_name": "Default/TransData-op99", | |||
| "type": "TransData", | |||
| "input": { | |||
| "Default/args0": { | |||
| "shape": [ | |||
| [ | |||
| 32, | |||
| 1, | |||
| 32, | |||
| 32 | |||
| ] | |||
| ], | |||
| "edge_type": "data", | |||
| "independent_layout": false, | |||
| "data_type": "DT_TENSOR[DT_FLOAT32]", | |||
| "slot_mapping": [ | |||
| [ | |||
| "0", | |||
| "" | |||
| ] | |||
| ] | |||
| } | |||
| }, | |||
| "output": { | |||
| "Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op190": { | |||
| "shape": [ | |||
| [ | |||
| 32, | |||
| 1, | |||
| 32, | |||
| 32 | |||
| ] | |||
| ], | |||
| "edge_type": "data", | |||
| "independent_layout": false, | |||
| "data_type": "DT_TENSOR[DT_FLOAT32]", | |||
| "slot_mapping": [ | |||
| [ | |||
| "0", | |||
| "" | |||
| ] | |||
| ] | |||
| } | |||
| }, | |||
| "slots": [ | |||
| { | |||
| "slot": "0", | |||
| "watch_points": [ | |||
| { | |||
| "id": 1, | |||
| "watch_condition": { | |||
| "id": "inf", | |||
| "params": [], | |||
| "abbr": "INF" | |||
| } | |||
| } | |||
| ], | |||
| "statistics": { | |||
| "overall_max": 6.0, | |||
| "overall_min": 1.0, | |||
| "overall_avg": 3.5, | |||
| "overall_count": 6, | |||
| "overall_nan_count": 0, | |||
| "overall_neg_inf_count": 0, | |||
| "overall_pos_inf_count": 0, | |||
| "overall_zero_count": 0.0, | |||
| "overall_neg_zero_count": 0.0, | |||
| "overall_pos_zero_count": 6.0 | |||
| }, | |||
| "shape": [ | |||
| 2, | |||
| 3 | |||
| ] | |||
| } | |||
| ], | |||
| "graph_name": "graph_0" | |||
| } | |||
| ] | |||
| } | |||
| } | |||
| {"graph": {"nodes": [{"name": "Default/args0", "full_name": "Default/args0", "type": "Parameter", "input": {}, "output": {"Default/TransData-op99": {"shape": [[32, 1, 32, 32]], "edge_type": "data", "independent_layout": false, "data_type": "DT_TENSOR[DT_FLOAT32]", "slot_mapping": [["0", ""]]}}, "slots": [{"slot": "0", "statistics": {"overall_max": 6.0, "overall_min": 1.0, "overall_avg": 3.5, "overall_count": 6, "overall_nan_count": 0, "overall_neg_inf_count": 0, "overall_pos_inf_count": 0, "overall_zero_count": 0.0, "overall_neg_zero_count": 0.0, "overall_pos_zero_count": 6.0}, "shape": [2, 3], "has_prev_step": true}], "graph_name": "graph_0"}, {"name": "Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op190", "full_name": "Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op190", "type": "Cast", "input": {"Default/TransData-op99": {"shape": [[32, 1, 32, 32]], "edge_type": "data", "independent_layout": false, "data_type": "DT_TENSOR[DT_FLOAT32]", "slot_mapping": [["0", ""]]}}, "output": {}, "slots": [{"slot": "0", "statistics": {"overall_max": 6.0, "overall_min": 1.0, "overall_avg": 3.5, "overall_count": 6, "overall_nan_count": 0, "overall_neg_inf_count": 0, "overall_pos_inf_count": 0, "overall_zero_count": 0.0, "overall_neg_zero_count": 0.0, "overall_pos_zero_count": 6.0}, "shape": [2, 3]}], "graph_name": "graph_0"}, {"name": "Default/TransData-op99", "full_name": "Default/TransData-op99", "type": "TransData", "input": {"Default/args0": {"shape": [[32, 1, 32, 32]], "edge_type": "data", "independent_layout": false, "data_type": "DT_TENSOR[DT_FLOAT32]", "slot_mapping": [["0", ""]]}}, "output": {"Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op190": {"shape": [[32, 1, 32, 32]], "edge_type": "data", "independent_layout": false, "data_type": "DT_TENSOR[DT_FLOAT32]", "slot_mapping": [["0", ""]]}}, "slots": [{"slot": "0", "summarized_error_code": 0, "watch_points": [{"id": 1, "watch_condition": {"id": "inf", "params": [], "abbr": "INF"}, "error_code": 0}], "statistics": {"overall_max": 6.0, "overall_min": 1.0, "overall_avg": 3.5, "overall_count": 6, "overall_nan_count": 0, "overall_neg_inf_count": 0, "overall_pos_inf_count": 0, "overall_zero_count": 0.0, "overall_neg_zero_count": 0.0, "overall_pos_zero_count": 6.0}, "shape": [2, 3]}], "graph_name": "graph_0"}]}} | |||
| @@ -1 +1 @@ | |||
| {"watch_point_hits": [{"node_name": "Default/TransData-op99", "tensors": [{"slot": "0", "watch_points": [{"id": 1, "watch_condition": {"id": "inf", "params": [], "abbr": "INF"}}]}], "graph_name": "graph_0"}, {"node_name": "Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25", "tensors": [{"slot": "0", "watch_points": [{"id": 1, "watch_condition": {"id": "inf", "params": [], "abbr": "INF"}}]}], "graph_name": "graph_0"}]} | |||
| {"watch_point_hits": [{"node_name": "Default/TransData-op99", "tensors": [{"slot": "0", "summarized_error_code": 0, "watch_points": [{"id": 1, "watch_condition": {"id": "inf", "params": [], "abbr": "INF"}, "error_code": 0}]}], "graph_name": "graph_0"}, {"node_name": "Default/optimizer-Momentum/ApplyMomentum[8]_1/ApplyMomentum-op25", "tensors": [{"slot": "0", "summarized_error_code": 0, "watch_points": [{"id": 1, "watch_condition": {"id": "inf", "params": [], "abbr": "INF"}, "error_code": 0}]}], "graph_name": "graph_0"}]} | |||
| @@ -5,9 +5,11 @@ | |||
| "tensors": [ | |||
| { | |||
| "slot": "0", | |||
| "summarized_error_code": 0, | |||
| "watch_points": [ | |||
| { | |||
| "id": 1, | |||
| "error_code": 0, | |||
| "watch_condition": { | |||
| "condition": "MAX_GT", | |||
| "param": 1 | |||
| @@ -25,7 +25,7 @@ from unittest import mock, TestCase | |||
| import pytest | |||
| from google.protobuf import json_format | |||
| from mindinsight.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.debugger.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.debugger.common.exceptions.exceptions import DebuggerParamValueError, \ | |||
| DebuggerParamTypeError | |||
| from mindinsight.debugger.common.log import LOGGER as log | |||
| @@ -23,7 +23,7 @@ from unittest.mock import MagicMock | |||
| import numpy as np | |||
| from mindinsight.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.debugger.conditionmgr.conditionmgr import ConditionMgr | |||
| from mindinsight.debugger.common.utils import get_ack_reply, ServerStatus | |||
| from mindinsight.debugger.debugger_cache import DebuggerCache | |||
| from mindinsight.debugger.debugger_grpc_server import DebuggerGrpcServer | |||
| @@ -190,7 +190,7 @@ class TestDebuggerServer: | |||
| def test_create_watchpoint_with_wrong_state(self): | |||
| """Test create watchpoint with wrong state.""" | |||
| with pytest.raises(DebuggerCreateWatchPointError, match='Failed to create watchpoint'): | |||
| self._server.create_watchpoint({'watch_condition': {'condition': 'INF'}}) | |||
| self._server.create_watchpoint({'watch_condition': {'id': 'inf'}}) | |||
| @mock.patch.object(MetadataHandler, 'state', 'waiting') | |||
| @mock.patch.object(GraphHandler, 'get_node_basic_info', return_value=[MagicMock()]) | |||
| @@ -199,7 +199,7 @@ class TestDebuggerServer: | |||
| def test_create_watchpoint(self, *args): | |||
| """Test create watchpoint.""" | |||
| args[0].return_value = 1 | |||
| res = self._server.create_watchpoint({'watch_condition': {'condition': 'INF'}, | |||
| res = self._server.create_watchpoint({'watch_condition': {'id': 'inf'}, | |||
| 'watch_nodes': ['watch_node_name']}) | |||
| assert res == {'id': 1, 'metadata': {'enable_recheck': False, 'state': 'waiting'}} | |||