Huawei_Technology
/
mindspore-mindinsight
Not watched
1
0
Fork 0
Code
Releases 11 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 8fd5ebb453
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '8fd5ebb453'
${ noResults }
mindspore-mindinsight/mindinsight/domain/graph/utils.py

484 lines
21 kB
Raw Blame History 

  1. # Copyright 2021 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ============================================================================

  15. """Parse utils module."""

  16. 

  17. import os

  18. 

  19. import xlsxwriter

  20. 

  21. from mindinsight.domain.graph.base import InputType, OutputType

  22. 

  23. 

  24. class Toolkit:

  25.     """Toolkit."""

  26. 

  27.     PLACEHOLDER = '-'

  28. 

  29.     def __init__(self, dump_dir, constants, parameters, operators):

  30.         self.dump_dir = dump_dir

  31.         self.constants = constants

  32.         self.parameters = parameters

  33.         self.operators = operators

  34. 

  35.     def export_xlsx(self, file_path):

  36.         """

  37.         Export graph data to Excel file.

  38. 

  39.         Args:

  40.             file_path (str) : Excel file path.

  41.         """

  42.         target_dir = os.path.dirname(file_path)

  43.         if not os.path.isdir(target_dir):

  44.             print(f'Directory {target_dir} not exists')

  45.             return

  46. 

  47.         workbook = xlsxwriter.Workbook(file_path)

  48. 

  49.         # text_v_align: 1-top, 2-middle, 3-bottom

  50.         # text_h_align: 1-left, 2-center, 3-right

  51.         styles = dict(

  52.             header_left_fmt=workbook.add_format(dict(

  53.                 text_v_align=2, text_h_align=1,

  54.                 font_color='#000000', bg_color='#d9d9d9',

  55.                 bold=True,

  56.             )),

  57.             header_center_fmt=workbook.add_format(dict(

  58.                 text_v_align=2, text_h_align=2,

  59.                 font_color='#000000', bg_color='#d9d9d9',

  60.                 bold=True,

  61.             )),

  62.             content_left_fmt=workbook.add_format(dict(

  63.                 text_v_align=2, text_h_align=1,

  64.                 text_wrap=False,

  65.             )),

  66.             content_center_fmt=workbook.add_format(dict(

  67.                 text_v_align=2, text_h_align=2,

  68.                 text_wrap=False,

  69.             )),

  70.             content_wrapped_fmt=workbook.add_format(dict(

  71.                 text_v_align=2, text_h_align=1,

  72.                 text_wrap=True,

  73.             )),

  74.         )

  75. 

  76.         self._add_info_worksheet(workbook, styles)

  77.         self._add_constant_worksheet(workbook, styles)

  78.         self._add_parameter_worksheet(workbook, styles)

  79.         self._add_operator_worksheet(workbook, styles)

  80.         self._add_statistics_worksheet(workbook, styles)

  81.         self._add_source_worksheet(workbook, styles)

  82. 

  83.         for worksheet in workbook.sheetnames.values():

  84.             worksheet.freeze_panes(1, 0)

  85.             worksheet.freeze_panes(1, 1)

  86. 

  87.         workbook.close()

  88. 

  89.     def _convert_column_indices(self, metas):

  90.         """

  91.         Convert column metas into indices mapping.

  92. 

  93.         Args:

  94.             metas (list): Column metas.

  95. 

  96.         Returns:

  97.             dict, holds the indicess of columns.

  98.         """

  99.         mapping = {}

  100.         for index, (name, _, _) in enumerate(metas):

  101.             mapping[name] = index

  102.         return mapping

  103. 

  104.     def _add_info_worksheet(self, workbook, styles):

  105.         """

  106.         Add info worksheet.

  107. 

  108.         Args:

  109.             workbook (WorkBook): Excel workbook.

  110.             styles (dict): Workbook styles.

  111.         """

  112.         worksheet = workbook.add_worksheet('info')

  113. 

  114.         # column metas contain column names, styles and widths

  115.         column_metas = [

  116.             ('argument', styles['header_center_fmt'], 20),

  117.             ('value', styles['header_left_fmt'], 150),

  118.         ]

  119.         for index, (column, fmt, width) in enumerate(column_metas):

  120.             worksheet.set_column(index, index, width)

  121.             worksheet.write(0, index, column, fmt)

  122.         worksheet.autofilter(0, 0, 0, len(column_metas) - 1)

  123. 

  124.         indices = self._convert_column_indices(column_metas)

  125.         worksheet.write(1, indices.get('argument'), 'dump-dir', styles['content_center_fmt'])

  126.         worksheet.write(1, indices.get('value'), self.dump_dir or '', styles['content_left_fmt'])

  127. 

  128.     def _get_operator_input_info(self, operator, input_types):

  129.         """

  130.         Add operator worksheet.

  131. 

  132.         Args:

  133.             operator (Operator): Operator.

  134.             input_types (dict): Input types.

  135. 

  136.         Returns:

  137.             dict, input info content.

  138.         """

  139.         input_content = ''

  140.         input_dtype_content = ''

  141.         input_shape_content = ''

  142. 

  143.         for op_input in operator.inputs:

  144.             if op_input.type == InputType.OPERATOR:

  145.                 op = input_types[InputType.OPERATOR][op_input.op_id]

  146.                 if op.type == 'Load':

  147.                     input_content += f'{op.type}_{op.name}' + '\n'

  148.                 else:

  149.                     input_content += f'{op.type}_{op.op_id}' + '\n'

  150.                     if op_input.info:

  151.                         input_dtype_content += str(op_input.info['dtype']) + '\n'

  152.                         input_shape_content += str(op_input.info.get('shape') or Toolkit.PLACEHOLDER) + '\n'

  153.                     else:

  154.                         input_dtype_content += Toolkit.PLACEHOLDER + '\n'

  155.                         input_shape_content += Toolkit.PLACEHOLDER + '\n'

  156.             elif op_input.type == InputType.PARAMETER:

  157.                 input_content += op_input.name + '\n'

  158.                 param = input_types[InputType.PARAMETER][op_input.name]

  159.                 if param.output:

  160.                     input_dtype_content += param.output.info['dtype'] + '\n'

  161.                     input_shape_content += str(param.output.info.get('shape') or Toolkit.PLACEHOLDER) + '\n'

  162.                 else:

  163.                     input_dtype_content += Toolkit.PLACEHOLDER + '\n'

  164.                     input_shape_content += Toolkit.PLACEHOLDER + '\n'

  165.             elif op_input.type == InputType.CONSTANT:

  166.                 input_content += op_input.name + '\n'

  167.                 cst = input_types[InputType.CONSTANT][op_input.name]

  168.                 if cst.output.type == OutputType.TENSOR:

  169.                     input_dtype_content += cst.output.info.get('dtype') or Toolkit.PLACEHOLDER + '\n'

  170.                     input_shape_content += str(cst.output.info.get('shape') or Toolkit.PLACEHOLDER) + '\n'

  171.                 else:

  172.                     input_dtype_content += Toolkit.PLACEHOLDER + '\n'

  173.                     input_shape_content += Toolkit.PLACEHOLDER + '\n'

  174.             else:

  175.                 input_content += op_input.name + '\n'

  176.                 input_dtype_content += Toolkit.PLACEHOLDER + '\n'

  177.                 input_shape_content += Toolkit.PLACEHOLDER + '\n'

  178. 

  179.         return {

  180.             'input': input_content.strip(),

  181.             'input_dtype': input_dtype_content.strip(),

  182.             'input_shape': input_shape_content.strip(),

  183.         }

  184. 

  185.     def _add_operator_worksheet(self, workbook, styles):

  186.         """

  187.         Add operator worksheet.

  188. 

  189.         Args:

  190.             workbook (WorkBook): Excel workbook.

  191.             styles (dict): Workbook styles.

  192.         """

  193.         constant_mapping = dict((constant.name, constant) for constant in self.constants)

  194.         parameter_mapping = dict((parameter.name, parameter) for parameter in self.parameters)

  195.         operator_mapping = dict((operator.op_id, operator) for operator in self.operators)

  196.         input_types = {

  197.             InputType.CONSTANT: constant_mapping,

  198.             InputType.PARAMETER: parameter_mapping,

  199.             InputType.OPERATOR: operator_mapping,

  200.         }

  201. 

  202.         worksheet = workbook.add_worksheet('operator')

  203. 

  204.         # column metas contain column names, styles and widths

  205.         column_metas = [

  206.             ('operator', styles['header_left_fmt'], 30),

  207.             ('input', styles['header_left_fmt'], 30),

  208.             ('input_dtype', styles['header_left_fmt'], 20),

  209.             ('input_shape', styles['header_left_fmt'], 25),

  210.             ('output_dtype', styles['header_left_fmt'], 20),

  211.             ('output_shape', styles['header_left_fmt'], 25),

  212.             ('downstream', styles['header_left_fmt'], 30),

  213.             ('name', styles['header_center_fmt'], 10),

  214.             ('attrs', styles['header_left_fmt'], 30),

  215.             ('full_name', styles['header_left_fmt'], 20),

  216.             ('device_id', styles['header_left_fmt'], 20),

  217.             ('graph_name', styles['header_left_fmt'], 30),

  218.             ('stack', styles['header_left_fmt'], 150),

  219.         ]

  220.         for index, (column, fmt, width) in enumerate(column_metas):

  221.             worksheet.set_column(index, index, width)

  222.             worksheet.write(0, index, column, fmt)

  223.         worksheet.autofilter(0, 0, 0, len(column_metas) - 1)

  224. 

  225.         indices = self._convert_column_indices(column_metas)

  226.         for index, operator in enumerate(self.operators):

  227.             if operator.type == 'Load':

  228.                 operator_content = f'{operator.type}_{operator.name}'

  229.             else:

  230.                 operator_content = f'{operator.type}_{operator.op_id}'

  231. 

  232.             worksheet.write(index + 1, indices.get('operator'), operator_content, styles['content_left_fmt'])

  233. 

  234.             if operator.type == 'make_tuple':

  235.                 worksheet.write(index + 1, indices.get('device_id'), operator.device_id, styles['content_left_fmt'])

  236.                 worksheet.write(index + 1, indices.get('graph_name'), operator.graph_name, styles['content_left_fmt'])

  237.                 continue

  238. 

  239.             input_info = self._get_operator_input_info(operator, input_types)

  240.             worksheet.write(index + 1, indices.get('input'), input_info['input'], styles['content_wrapped_fmt'])

  241.             worksheet.write(

  242.                 index + 1, indices.get('input_dtype'),

  243.                 input_info['input_dtype'], styles['content_wrapped_fmt'])

  244.             worksheet.write(

  245.                 index + 1, indices.get('input_shape'),

  246.                 input_info['input_shape'], styles['content_wrapped_fmt'])

  247. 

  248.             output_dtype_content = ''

  249.             output_shape_content = ''

  250.             if operator.output and operator.output.type == OutputType.TENSOR:

  251.                 output_dtype_content = operator.output.info['dtype']

  252.                 output_shape_content = str(operator.output.info['shape'])

  253.             elif operator.output and operator.output.type == OutputType.TUPLE:

  254.                 output_dtype_content = '\n'.join([

  255.                     Toolkit.PLACEHOLDER if dtype is None else dtype

  256.                     for dtype in operator.output.info['dtypes']

  257.                 ])

  258.                 output_shape_content = '\n'.join([

  259.                     Toolkit.PLACEHOLDER if shape is None else str(shape)

  260.                     for shape in operator.output.info['shapes']

  261.                 ])

  262.             worksheet.write(

  263.                 index + 1, indices.get('output_dtype'),

  264.                 output_dtype_content, styles['content_wrapped_fmt'])

  265.             worksheet.write(

  266.                 index + 1, indices.get('output_shape'),

  267.                 output_shape_content, styles['content_wrapped_fmt'])

  268. 

  269.             downstream_content = ''

  270.             for op_id in operator.downstream:

  271.                 op = operator_mapping[op_id]

  272.                 downstream_content += f'{op.type}_{op.op_id}' + '\n'

  273.             worksheet.write(

  274.                 index + 1, indices.get('downstream'),

  275.                 downstream_content.strip(), styles['content_wrapped_fmt'])

  276. 

  277.             worksheet.write(index + 1, indices.get('name'), operator.name, styles['content_center_fmt'])

  278.             worksheet.write(index + 1, indices.get('attrs'), str(operator.attrs), styles['content_left_fmt'])

  279.             worksheet.write(index + 1, indices.get('full_name'), operator.full_name, styles['content_left_fmt'])

  280.             worksheet.write(index + 1, indices.get('device_id'), operator.device_id, styles['content_left_fmt'])

  281.             worksheet.write(index + 1, indices.get('graph_name'), operator.graph_name, styles['content_left_fmt'])

  282. 

  283.             stack_content = ''

  284.             for source in operator.stack:

  285.                 stack_content += f'{source.file_path}:{source.line_no}\n{source.code_line}\n'

  286.             worksheet.write(index + 1, indices.get('stack'), stack_content.strip(), styles['content_wrapped_fmt'])

  287. 

  288.     def _add_parameter_worksheet(self, workbook, styles):

  289.         """

  290.         Add parameter worksheet.

  291. 

  292.         Args:

  293.             workbook (WorkBook): Excel workbook.

  294.             styles (dict): Workbook styles.

  295.         """

  296.         worksheet = workbook.add_worksheet('parameter')

  297. 

  298.         # column metas contain column names, styles and widths

  299.         column_metas = [

  300.             ('name', styles['header_left_fmt'], 50),

  301.             ('output_dtype', styles['header_left_fmt'], 20),

  302.             ('output_shape', styles['header_left_fmt'], 25),

  303.             ('downstream', styles['header_left_fmt'], 30),

  304.             ('device_id', styles['header_left_fmt'], 20),

  305.             ('graph_name', styles['header_left_fmt'], 30),

  306.         ]

  307.         for index, (column, fmt, width) in enumerate(column_metas):

  308.             worksheet.set_column(index, index, width)

  309.             worksheet.write(0, index, column, fmt)

  310.         worksheet.autofilter(0, 0, 0, len(column_metas) - 1)

  311. 

  312.         indices = self._convert_column_indices(column_metas)

  313.         operator_mapping = dict((operator.op_id, operator) for operator in self.operators)

  314.         for index, parameter in enumerate(self.parameters):

  315.             worksheet.write(index + 1, indices.get('name'), parameter.name, styles['content_left_fmt'])

  316.             worksheet.write(

  317.                 index + 1, indices.get('output_dtype'),

  318.                 parameter.output.info['dtype'], styles['content_left_fmt'])

  319.             worksheet.write(

  320.                 index + 1, indices.get('output_shape'),

  321.                 str(parameter.output.info['shape']), styles['content_left_fmt'])

  322. 

  323.             downstream_nodes = [operator_mapping[op_id] for op_id in parameter.downstream]

  324.             downstream_content = ''

  325.             for op in downstream_nodes:

  326.                 if op.type == 'Load':

  327.                     downstream_content += f'{op.type}_{op.name}' + '\n'

  328.                 else:

  329.                     downstream_content += f'{op.type}_{op.op_id}' + '\n'

  330.             worksheet.write(

  331.                 index + 1, indices.get('downstream'),

  332.                 downstream_content.strip(), styles['content_wrapped_fmt'])

  333. 

  334.             worksheet.write(index + 1, indices.get('device_id'), parameter.device_id, styles['content_left_fmt'])

  335.             worksheet.write(index + 1, indices.get('graph_name'), parameter.graph_name, styles['content_left_fmt'])

  336. 

  337.     def _add_constant_worksheet(self, workbook, styles):

  338.         """

  339.         Add constant worksheet.

  340. 

  341.         Args:

  342.             workbook (WorkBook): Excel workbook.

  343.             styles (dict): Workbook styles.

  344.         """

  345.         worksheet = workbook.add_worksheet('constant')

  346. 

  347.         # column metas contain column names, styles and widths

  348.         column_metas = [

  349.             ('name', styles['header_left_fmt'], 10),

  350.             ('value', styles['header_left_fmt'], 30),

  351.             ('downstream', styles['header_left_fmt'], 30),

  352.             ('device_id', styles['header_left_fmt'], 20),

  353.             ('graph_name', styles['header_left_fmt'], 30),

  354.         ]

  355.         for index, (column, fmt, width) in enumerate(column_metas):

  356.             worksheet.set_column(index, index, width)

  357.             worksheet.write(0, index, column, fmt)

  358.         worksheet.autofilter(0, 0, 0, len(column_metas) - 1)

  359. 

  360.         indices = self._convert_column_indices(column_metas)

  361.         operator_mapping = dict((operator.op_id, operator) for operator in self.operators)

  362.         for index, constant in enumerate(self.constants):

  363.             worksheet.write(index + 1, indices.get('name'), constant.name, styles['content_left_fmt'])

  364. 

  365.             if constant.output.type == OutputType.NONE:

  366.                 value_content = 'NONE'

  367.             elif constant.output.type == OutputType.TENSOR:

  368.                 value_content = 'TENSOR'

  369.             else:

  370.                 value_content = constant.output.info['value']

  371.             worksheet.write(index + 1, indices.get('value'), value_content, styles['content_left_fmt'])

  372. 

  373.             downstream_nodes = [operator_mapping[op_id] for op_id in constant.downstream]

  374.             downstream_content = ''

  375.             for op in downstream_nodes:

  376.                 if op.type == 'Load':

  377.                     downstream_content += f'{op.type}_{op.name}' + '\n'

  378.                 else:

  379.                     downstream_content += f'{op.type}_{op.op_id}' + '\n'

  380.             worksheet.write(

  381.                 index + 1, indices.get('downstream'),

  382.                 downstream_content.strip(), styles['content_wrapped_fmt'])

  383. 

  384.             worksheet.write(index + 1, indices.get('device_id'), constant.device_id, styles['content_left_fmt'])

  385.             worksheet.write(index + 1, indices.get('graph_name'), constant.graph_name, styles['content_left_fmt'])

  386. 

  387.     def _add_statistics_worksheet(self, workbook, styles):

  388.         """

  389.         Add statistics worksheet.

  390. 

  391.         Args:

  392.             workbook (WorkBook): Excel workbook.

  393.             styles (dict): Workbook styles.

  394.         """

  395.         worksheet = workbook.add_worksheet('statistics')

  396. 

  397.         # column metas contain column names, styles and widths

  398.         column_metas = [

  399.             ('operator', styles['header_left_fmt'], 30),

  400.             ('count', styles['header_center_fmt'], 20),

  401.         ]

  402.         for index, (column, fmt, width) in enumerate(column_metas):

  403.             worksheet.set_column(index, index, width)

  404.             worksheet.write(0, index, column, fmt)

  405.         worksheet.autofilter(0, 0, 0, len(column_metas) - 1)

  406. 

  407.         operator_type_set = set()

  408.         for operator in self.operators:

  409.             operator_type_set.add(operator.type)

  410. 

  411.         operator_types = sorted(list(operator_type_set))

  412.         stats = dict(zip(operator_types, [0]*len(operator_types)))

  413.         for operator in self.operators:

  414.             stats[operator.type] += 1

  415. 

  416.         indices = self._convert_column_indices(column_metas)

  417.         for index, operator_type in enumerate(operator_types):

  418.             worksheet.write(index + 1, indices.get('operator'), operator_type, styles['content_left_fmt'])

  419.             worksheet.write(index + 1, indices.get('count'), stats[operator_type], styles['content_center_fmt'])

  420. 

  421.     def _add_source_worksheet(self, workbook, styles):

  422.         """

  423.         Add source worksheet.

  424. 

  425.         Args:

  426.             workbook (WorkBook): Excel workbook.

  427.             styles (dict): Workbook styles.

  428.         """

  429.         worksheet = workbook.add_worksheet('source')

  430. 

  431.         # column metas contain column names, styles and widths

  432.         column_metas = [

  433.             ('stack', styles['header_left_fmt'], 150),

  434.             ('operator', styles['header_left_fmt'], 30),

  435.             ('full_name', styles['header_left_fmt'], 20),

  436.             ('device_id', styles['header_left_fmt'], 20),

  437.             ('graph_name', styles['header_left_fmt'], 30),

  438.         ]

  439.         for index, (column, fmt, width) in enumerate(column_metas):

  440.             worksheet.set_column(index, index, width)

  441.             worksheet.write(0, index, column, fmt)

  442.         worksheet.autofilter(0, 0, 0, len(column_metas) - 1)

  443. 

  444.         source_mapping = {}

  445.         for operator in self.operators:

  446.             if not operator.stack:

  447.                 continue

  448.             stack = [f'{source.file_path}:{source.line_no}\n{source.code_line}' for source in operator.stack]

  449.             key = '\n'.join(stack)

  450.             if key in source_mapping:

  451.                 source_mapping[key].append(operator)

  452.             else:

  453.                 source_mapping[key] = [operator]

  454. 

  455.         row = 0

  456.         indices = self._convert_column_indices(column_metas)

  457.         for key in source_mapping:

  458.             operators = source_mapping[key]

  459.             operators.sort(key=lambda x: int(x.op_id))

  460. 

  461.             if len(operators) == 1:

  462.                 worksheet.write(row + 1, indices.get('stack'), key, styles['content_wrapped_fmt'])

  463.             else:

  464.                 worksheet.merge_range(

  465.                     row + 1, indices.get('stack'),

  466.                     row+len(operators), 0, key, styles['content_wrapped_fmt'])

  467. 

  468.             for index, operator in enumerate(operators):

  469.                 operator_content = f'{operator.type}_{operator.op_id}'

  470.                 worksheet.write(

  471.                     row + index + 1, indices.get('operator'),

  472.                     operator_content, styles['content_left_fmt'])

  473.                 worksheet.write(

  474.                     row + index + 1, indices.get('full_name'),

  475.                     operator.full_name, styles['content_left_fmt'])

  476.                 worksheet.write(

  477.                     row + index + 1, indices.get('device_id'),

  478.                     operator.device_id, styles['content_left_fmt'])

  479.                 worksheet.write(

  480.                     row + index + 1, indices.get('graph_name'),

  481.                     operator.graph_name, styles['content_left_fmt'])

  482. 

  483.             row += len(operators)