bugfix for comparing scope_level using string

mindinsight/profiler/analyser/analyser.py

@@ -90,7 +90,8 @@ class AicoreTypeAnalyser(BaseAnalyser):
         Returns:
             list[Union[str, float]], the converted data.
         """
-        return [row[0], float(row[1]), int(row[2]), float(row[3])]
+        return [row[0], self._format_float_data(float(row[1])),
+                int(row[2]), self._format_float_data(float(row[3]))]
 
 
 class AicoreDetailAnalyser(BaseAnalyser):
@@ -270,7 +271,7 @@ class AicoreDetailAnalyser(BaseAnalyser):
             list[Union[str, float]], the operator detail information in one row.
         """
         framework_info = framework_infos.get(row[0])
-        return [framework_info[1], framework_info[2], float(row[1]),
+        return [framework_info[1], framework_info[2], self._format_float_data(float(row[1])),
                 framework_info[3], framework_info[0], framework_info[4]]
 
 
@@ -402,5 +403,5 @@ class AicpuDetailAnalyser(BaseAnalyser):
         Returns:
             list[Union[str, float]], the converted data.
         """
-        return [int(row[0]), row[1], float(row[2]), float(row[3]), int(row[4]),
-                int(row[5])]
+        return [int(row[0]), row[1], self._format_float_data(float(row[2])),
+                self._format_float_data(float(row[3])), int(row[4]), int(row[5])]

mindinsight/profiler/analyser/base_analyser.py

@@ -253,3 +253,12 @@ class BaseAnalyser(ABC):
             'object': self._result,
             'size': self._size
         }
+
+    @staticmethod
+    def _format_float_data(float_data):
+        """Format the float data."""
+        if float_data < 1:
+            float_data = round(float_data, 6)
+        else:
+            float_data = round(float_data, 3)
+        return float_data

mindinsight/profiler/analyser/cpu_analyser.py

@@ -24,8 +24,7 @@ class CpuOpTypeAnalyser(GpuAnalyser):
     _col_names = validate.CPU_TYPE_COL
     _csv_file_to_analyse = 'cpu_op_type_info_{}.csv'
 
-    @staticmethod
-    def _convert_field_type(row):
+    def _convert_field_type(self, row):
         """
         Convert the field type to the specific type.
 
@@ -36,7 +35,10 @@ class CpuOpTypeAnalyser(GpuAnalyser):
             list, the converted data.
         """
         try:
-            return [row[0], int(row[1]), int(row[2]), float(row[3]), float(row[4]), float(row[5])*100]
+            return [row[0], int(row[1]), int(row[2]),
+                    self._format_float_data(float(row[3])),
+                    self._format_float_data(float(row[4])),
+                    self._format_float_data(float(row[5])*100)]
         except IndexError as err:
             log.exception(err)
             raise ProfilerRawFileException('failed to get HOST CPU operator type data.')
@@ -47,8 +49,7 @@ class CpuOpInfoAnalyser(GpuAnalyser):
     _col_names = validate.CPU_DETAIL_COL
     _csv_file_to_analyse = 'cpu_op_detail_info_{}.csv'
 
-    @staticmethod
-    def _convert_field_type(row):
+    def _convert_field_type(self, row):
         """
         Convert the field type to the specific type.
 
@@ -59,7 +60,9 @@ class CpuOpInfoAnalyser(GpuAnalyser):
             list, the converted data.
         """
         try:
-            return [row[0], row[1], row[2], row[3], int(row[4]), float(row[5]), float(row[6]), float(row[7]), row[8]]
+            return [row[0], row[1], row[2], row[3], int(row[4]),
+                    self._format_float_data(float(row[5])), self._format_float_data(float(row[6])),
+                    self._format_float_data(float(row[7])), row[8]]
         except IndexError as err:
             log.exception(err)
             raise ProfilerRawFileException('failed to get HOST CPU operator detail data.')

mindinsight/profiler/analyser/gpu_analyser.py

@@ -43,8 +43,7 @@ class GpuAnalyser(BaseAnalyser):
             for info in csv_reader:
                 self._data.append(self._convert_field_type(info))
 
-    @staticmethod
-    def _convert_field_type(row):
+    def _convert_field_type(self, row):
         """
         Convert the field type to the specific type.
 
@@ -74,8 +73,7 @@ class GpuOpTypeAnalyser(GpuAnalyser):
     _col_names = ["op_type", "type_occurrences", "total_time", "proportion", "avg_time"]
     _csv_file_to_analyse = 'gpu_op_type_info_{}.csv'
 
-    @staticmethod
-    def _convert_field_type(row):
+    def _convert_field_type(self, row):
         """
         Convert the field type to the specific type.
 
@@ -85,7 +83,8 @@ class GpuOpTypeAnalyser(GpuAnalyser):
         Returns:
             list, the converted data.
         """
-        return [row[0], int(row[1]), float(row[2]), float(row[3])*100, float(row[4])]
+        return [row[0], int(row[1]), self._format_float_data(float(row[2])),
+                self._format_float_data(float(row[3])*100), self._format_float_data(float(row[4]))]
 
 
 class GpuOpInfoAnalyser(GpuAnalyser):
@@ -95,8 +94,7 @@ class GpuOpInfoAnalyser(GpuAnalyser):
                   "proportion", "cuda_activity_cost_time", "cuda_activity_call_count"]
     _csv_file_to_analyse = 'gpu_op_detail_info_{}.csv'
 
-    @staticmethod
-    def _convert_field_type(row):
+    def _convert_field_type(self, row):
         """
         Convert the field type to the specific type.
 
@@ -106,8 +104,9 @@ class GpuOpInfoAnalyser(GpuAnalyser):
         Returns:
             list, the converted data.
         """
-        return [row[0], row[1], row[2], row[3], int(row[4]), float(row[5]),
-                float(row[6]), float(row[7]), float(row[8]), int(row[9])]
+        return [row[0], row[1], row[2], row[3], int(row[4]),
+                self._format_float_data(float(row[5])), self._format_float_data(float(row[6])),
+                self._format_float_data(float(row[7])), self._format_float_data(float(row[8])), int(row[9])]
 
 
 class GpuCudaActivityAnalyser(GpuAnalyser):
@@ -117,8 +116,7 @@ class GpuCudaActivityAnalyser(GpuAnalyser):
                   "avg_duration", "max_duration", "min_duration"]
     _csv_file_to_analyse = 'gpu_activity_data_{}.csv'
 
-    @staticmethod
-    def _convert_field_type(row):
+    def _convert_field_type(self, row):
         """
         Convert the field type to the specific type.
 
@@ -129,4 +127,5 @@ class GpuCudaActivityAnalyser(GpuAnalyser):
             list, the converted data.
         """
         return [row[0], row[1], row[2], row[3], row[4], row[5], int(row[6]),
-                float(row[7]), float(row[8]), float(row[9]), float(row[10])]
+                self._format_float_data(float(row[7])), self._format_float_data(float(row[8])),
+                self._format_float_data(float(row[9])), self._format_float_data(float(row[10]))]

mindinsight/profiler/analyser/timeline_analyser.py

@@ -68,8 +68,9 @@ class TimelineAnalyser(BaseAnalyser):
                 with open(file_path, 'r') as f_obj:
                     timeline = json.load(f_obj)
                     for idx, time_item in enumerate(timeline):
-                        if time_item["tid"] == "Name Scope" and \
-                                                time_item["scope_level"] >= scope_name_num:
+                        if time_item["tid"] == 100001 and \
+                                time_item["ph"] != "M" and \
+                                int(time_item["scope_level"]) >= int(scope_name_num):
                             timeline[idx] = None
                     timeline = list(filter(lambda x: x, timeline))
             except (IOError, OSError, json.JSONDecodeError) as err:
@@ -110,4 +111,7 @@ class TimelineAnalyser(BaseAnalyser):
         else:
             logger.info('No timeline summary file. Please check the output path.')
 
+        if not timeline_summary.get("max_scope_name_num"):
+            timeline_summary["max_scope_name_num"] = 0
+
         return timeline_summary

mindinsight/profiler/common/util.py

@@ -42,7 +42,11 @@ def analyse_device_list_from_profiler_dir(profiler_dir):
 
     device_id_list = set()
     gpu_device_id_list = set()
+
+    depth = 0
     for _, _, filenames in os.walk(profiler_dir):
+        if depth == 1:
+            break
         for filename in filenames:
             if filename.startswith("step_trace_raw"):
                 items = filename.split("_")
@@ -57,6 +61,7 @@ def analyse_device_list_from_profiler_dir(profiler_dir):
                 device_id_list.add(device_num)
             elif device_num.isdigit() and '_'.join(items[:-1]) in gpu_profiler_file_prefix:
                 gpu_device_id_list.add(device_num)
+        depth += 1
 
     if device_id_list:
         result_list = sorted(list(device_id_list))

tests/st/func/profiler/test_op_analyser.py

@@ -31,7 +31,7 @@ OP_GATHER_V2_INFO = {
     ],
     'object': [
         [
-            'GatherV2-op55', 'GatherV2', 42.220212142857136, 'Default',
+            'GatherV2-op55', 'GatherV2', 42.220, 'Default',
             'Default/network-TrainStepWrap/network-VirtualDatasetCellTriple/'
             '_backbone-NetWithLossClass/network-WideDeepModel/GatherV2-op55',
             {
@@ -53,7 +53,7 @@ OP_GATHER_V2_INFO = {
             }
         ],
         [
-            'GatherV2-op33', 'GatherV2', 0.9352293333333332, 'Default',
+            'GatherV2-op33', 'GatherV2', 0.935229, 'Default',
             'Default/network-TrainStepWrap/network-VirtualDatasetCellTriple/'
             '_backbone-NetWithLossClass/network-WideDeepModel/GatherV2-op33',
             {
@@ -107,15 +107,15 @@ class TestOpAnalyser:
         expect_result = {
             'col_name': ['op_type', 'execution_time', 'execution_frequency', 'percent'],
             'object': [
-                ['UnsortedSegmentSum', 44.607826, 2, 35.28],
-                ['GatherV2', 43.155441, 2, 34.13],
-                ['Slice', 20.376315, 16, 16.12],
-                ['Concat', 5.808454, 4, 4.59],
-                ['Split', 2.714277, 2, 2.15],
-                ['MatMul', 1.936681, 15, 1.53],
-                ['Mul', 1.902949, 32, 1.51],
-                ['StridedSliceGrad', 1.506834, 2, 1.19],
-                ['TransData', 1.115158, 30, 0.88],
+                ['UnsortedSegmentSum', 44.608, 2, 35.28],
+                ['GatherV2', 43.155, 2, 34.13],
+                ['Slice', 20.376, 16, 16.12],
+                ['Concat', 5.808, 4, 4.59],
+                ['Split', 2.714, 2, 2.15],
+                ['MatMul', 1.937, 15, 1.53],
+                ['Mul', 1.903, 32, 1.51],
+                ['StridedSliceGrad', 1.507, 2, 1.19],
+                ['TransData', 1.115, 30, 0.88],
                 ['ReluGrad', 0.854069, 5, 0.68],
                 ['Cast', 0.484685, 15, 0.38],
                 ['ReLU', 0.483282, 5, 0.38],
@@ -146,6 +146,9 @@ class TestOpAnalyser:
             }
         }
         result = self._analyser_aicore_type.query(condition)
+        print("gzh test")
+        print(expect_result)
+        print(result)
         assert expect_result == result
 
     @pytest.mark.level0
@@ -159,8 +162,8 @@ class TestOpAnalyser:
         expect_result = {
             'col_name': ['op_type', 'execution_time', 'execution_frequency', 'percent'],
             'object': [
-                ['MatMul', 1.936681, 15, 1.53],
-                ['Mul', 1.902949, 32, 1.51]
+                ['MatMul', 1.937, 15, 1.53],
+                ['Mul', 1.903, 32, 1.51]
             ],
             'size': 2
         }

tests/utils/resource/profiler/aicore_intermediate_1_type.csv

@@ -3,4 +3,4 @@ AtomicAddrClean,0.007283,6,0.49
 Cast,0.053395,13,3.63
 TransData,0.121800,5,8.23
 Conv2D,0.063656,2,4.33
-MatMul,1.085982,9,73.80
+MatMul,1.085,9,73.80

tests/utils/resource/profiler/ascend_timeline_summary_0.json

@@ -1 +1 @@
-{"total_time": 23147.013999999992, "num_of_streams": 2, "num_of_ops": 213, "op_exe_times": 112}
+{"total_time": 23147.013999999992, "num_of_streams": 2, "num_of_ops": 213, "op_exe_times": 112, "max_scope_name_num": 0}

tests/utils/resource/profiler/gpu_timeline_summary_0.json

@@ -1 +1 @@
-{"total_time": 3875.638703999999, "num_of_streams": 1, "num_of_ops": 66, "op_exe_times": 66}
+{"total_time": 3875.638703999999, "num_of_streams": 1, "num_of_ops": 66, "op_exe_times": 66, "max_scope_name_num": 0}