From 7133172f4c5a0c64be83a56ce1ff1451cc8b6454 Mon Sep 17 00:00:00 2001 From: gzhcv Date: Tue, 26 Jan 2021 10:16:06 +0800 Subject: [PATCH] Add cpu analyser of profiler module --- mindinsight/profiler/analyser/__init__.py | 4 +- mindinsight/profiler/analyser/cpu_analyser.py | 55 +++++++++++++++++++ .../profiler/common/validator/validate.py | 14 ++++- tests/st/func/profiler/test_validator.py | 2 +- .../common/validator/test_validator.py | 4 +- 5 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 mindinsight/profiler/analyser/cpu_analyser.py diff --git a/mindinsight/profiler/analyser/__init__.py b/mindinsight/profiler/analyser/__init__.py index 494d4273..f5ce40a1 100644 --- a/mindinsight/profiler/analyser/__init__.py +++ b/mindinsight/profiler/analyser/__init__.py @@ -13,5 +13,5 @@ # limitations under the License. # ============================================================================ """The analyser module.""" -from . import analyser, minddata_pipeline_analyser, step_trace_analyser, \ - minddata_analyser, timeline_analyser, gpu_analyser, memory_usage_analyser, minddata_cpu_utilization_analyser +from . import analyser, minddata_pipeline_analyser, step_trace_analyser, minddata_analyser, \ + timeline_analyser, cpu_analyser, gpu_analyser, memory_usage_analyser, minddata_cpu_utilization_analyser diff --git a/mindinsight/profiler/analyser/cpu_analyser.py b/mindinsight/profiler/analyser/cpu_analyser.py new file mode 100644 index 00000000..01284b0e --- /dev/null +++ b/mindinsight/profiler/analyser/cpu_analyser.py @@ -0,0 +1,55 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The cpu base analyser.""" +from mindinsight.profiler.analyser.gpu_analyser import GpuAnalyser +from mindinsight.profiler.common.validator import validate + +class CpuOpTypeAnalyser(GpuAnalyser): + """Cpu operation type analyser.""" + _col_names = validate.CPU_TYPE_COL + _csv_file_to_analyse = 'cpu_op_type_info_{}.csv' + + @staticmethod + def _convert_field_type(row): + """ + Convert the field type to the specific type. + + Args: + row (list): One row data from parsed data. + + Returns: + list, the converted data. + """ + return [row[0], int(row[1]), int(row[2]), float(row[3]), float(row[4]), float(row[5])*100] + + +class CpuOpInfoAnalyser(GpuAnalyser): + """Cpu operation detail info analyser.""" + _col_names = validate.CPU_DETAIL_COL + _csv_file_to_analyse = 'cpu_op_detail_info_{}.csv' + + @staticmethod + def _convert_field_type(row): + """ + Convert the field type to the specific type. + + Args: + row (list): One row data from parsed data. + + Returns: + list, the converted data. + """ + return [row[0], row[1], row[2], row[3], int(row[4]), float(row[5]), + float(row[6]), float(row[7]), row[8]] diff --git a/mindinsight/profiler/common/validator/validate.py b/mindinsight/profiler/common/validator/validate.py index 0b1ffcf8..e42e3725 100644 --- a/mindinsight/profiler/common/validator/validate.py +++ b/mindinsight/profiler/common/validator/validate.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,6 +34,10 @@ GPU_ACTIVITY_COL = ["name", "type", "op_full_name", "stream_id", GPU_DETAIL_COL = ["op_side", "op_type", "op_name", "op_full_name", "op_occurrences", "op_total_time", "op_avg_time", "proportion", "cuda_activity_cost_time", "cuda_activity_call_count"] +CPU_TYPE_COL = ["op_type", "type_occurrences", "execution_frequency", "total_compute_time", + "avg_time", "percent"] +CPU_DETAIL_COL = ["op_side", "op_type", "op_name", "full_op_name", "op_occurrences", + "op_total_time", "op_avg_time", "total_time_proportion", "subgraph"] MINDDATA_PIPELINE_COL = [ 'op_id', 'op_type', 'num_workers', 'output_queue_average_size', 'output_queue_length', 'output_queue_usage_rate', 'sample_interval', @@ -82,14 +86,18 @@ def validate_condition(search_condition): search_scope = GPU_DETAIL_COL elif op_type == "gpu_cuda_activity": search_scope = GPU_ACTIVITY_COL + elif op_type == "cpu_op_type": + search_scope = CPU_TYPE_COL + elif op_type == "cpu_op_info": + search_scope = CPU_DETAIL_COL else: raise ProfilerOpTypeException( "The op_type must in ['aicpu_type','aicpu_detail', 'aicore_type', 'aicore_detail', " - "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity']") + "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity', 'cpu_op_type', 'cpu_op_info']") else: raise ProfilerOpTypeException( "The op_type must in ['aicpu_type','aicpu_detail', 'aicore_type', 'aicore_detail', " - "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity']") + "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity', 'cpu_op_type', 'cpu_op_info']") if "group_condition" in search_condition: validate_group_condition(search_condition) diff --git a/tests/st/func/profiler/test_validator.py b/tests/st/func/profiler/test_validator.py index 9cc5c8ef..0ac2c907 100644 --- a/tests/st/func/profiler/test_validator.py +++ b/tests/st/func/profiler/test_validator.py @@ -73,7 +73,7 @@ class TestValidate: condition_list = [{'op_type': "xxx"}, {}] exception_message = "The op_type in search_condition error, The op_type must in " \ "['aicpu_type','aicpu_detail', 'aicore_type', 'aicore_detail', "\ - "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity']" + "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity', 'cpu_op_type', 'cpu_op_info']" for condition in condition_list: with pytest.raises(ProfilerOpTypeException) as exc_info: validate_condition(condition) diff --git a/tests/ut/profiler/common/validator/test_validator.py b/tests/ut/profiler/common/validator/test_validator.py index ccc468bd..43846606 100644 --- a/tests/ut/profiler/common/validator/test_validator.py +++ b/tests/ut/profiler/common/validator/test_validator.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -252,7 +252,7 @@ class TestValidateCondition: condition_list = [{'op_type': "xxx"}, {}] exception_message = "The op_type in search_condition error, The op_type must in " \ "['aicpu_type','aicpu_detail', 'aicore_type', 'aicore_detail', "\ - "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity']" + "'gpu_op_type', 'gpu_op_info', 'gpu_cuda_activity', 'cpu_op_type', 'cpu_op_info']" for condition in condition_list: with pytest.raises(ProfilerOpTypeException) as exc_info: validate_condition(condition)