add parse_summary function

we can use the function to transfer summary file to csv file and image so that it is easy to read and process

mindinsight/datavisual/common/log.py

@@ -17,3 +17,5 @@ from mindinsight.utils.log import setup_logger
 
 logger = setup_logger("datavisual", "datavisual")
 restful_logger = setup_logger("restful_api", "restful_api")
+parse_summary_logger = setup_logger("parse_summary", "parse_summary", console=True,
+                                    formatter='[%(levelname)s]%(message)s')

mindinsight/datavisual/data_transform/summary_parser/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Summary parser module used to parse summary data and save it to csv file and image."""

mindinsight/datavisual/data_transform/summary_parser/event_parser.py

@@ -0,0 +1,245 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Scalar Writer.
+
+This module write scalar into a  csv file.
+"""
+import os
+import struct
+
+from google.protobuf.message import DecodeError
+
+from mindinsight.datavisual.utils import crc32
+from mindinsight.datavisual.common import exceptions
+from mindinsight.datavisual.common.log import parse_summary_logger
+from mindinsight.datavisual.proto_files import lazy_read_pb2
+from mindinsight.datavisual.data_access.file_handler import FileHandler
+from mindinsight.datavisual.data_transform.summary_parser.image_writer import ImageWriter
+from mindinsight.datavisual.data_transform.summary_parser.scalar_writer import ScalarWriter
+from mindinsight.datavisual.data_transform.ms_data_loader import _SummaryParser
+from mindinsight.utils.exceptions import UnknownError
+
+HEADER_SIZE = 8
+CRC_STR_SIZE = 4
+MAX_EVENT_STRING = 500000000
+SCALAR = 'scalar_value'
+IMAGE = 'image'
+
+
+class EventParser():
+    """Parse summary file and save it to csv file and image."""
+    def __init__(self, summary_dir, output):
+        self._summary_dir = summary_dir
+        self._output = output
+        self._scalar_writer = ScalarWriter(self._output)
+        self._image_writer = ImageWriter(FileHandler.join(self._output, IMAGE))
+        self._current = 0
+        self._file_size = 0
+        self._process_info = 0
+        self._image_check = False
+        self._scalar_check = False
+
+    def parse(self):
+        """Load summary file and parse file content."""
+        try:
+            if not (self._check_filepath() and self._check_create_filepath(
+                    self._output) and self._check_create_filepath(FileHandler.join(self._output, IMAGE))):
+                return
+
+            summary_parser = _SummaryParser(self._summary_dir)
+            summary_files = summary_parser.filter_files(os.listdir(self._summary_dir))
+
+            if not summary_files:
+                parse_summary_logger.error('Path %s has no summary file.', self._summary_dir)
+                return
+
+            summary_files = summary_parser.sort_files(summary_files)
+
+            filename = summary_files[-1]
+            file_path = FileHandler.join(self._summary_dir, filename)
+
+            if not os.access(file_path, os.R_OK):
+                parse_summary_logger.error('Path %s is not accessible, please check the file-authority.', file_path)
+                return
+
+            self._summary_file_handler = FileHandler(file_path, 'rb')
+
+            self._file_size = os.path.getsize(file_path)
+            # when current parsed size bigger than self._process_info, print process
+            self._process_info = self._file_size // 10
+
+            parse_summary_logger.info("loading %s", file_path)
+            result = self._load(self._summary_file_handler)
+
+            self._scalar_writer.write()
+
+            warning = ''
+
+            if not self._scalar_check:
+                warning = warning + " the summary file contains no scalar value"
+            if not self._image_check:
+                warning = warning + " the summary file contains no image"
+            if result:
+                if warning:
+                    parse_summary_logger.warning(warning)
+                parse_summary_logger.info("parsing summary file finished")
+
+        except Exception as ex:
+            parse_summary_logger.error("Parse summary file failed, detail: %r", str(ex))
+            raise UnknownError(str(ex))
+
+    def _load(self, file_handler):
+        """
+        Load a log file data.
+
+        Args:
+            file_handler (FileHandler): A file handler.
+
+        Returns:
+            bool, True if the summary file is finished loading.
+        """
+        while True:
+            try:
+                event_str = self._event_load(file_handler)
+                if event_str is None:
+                    return True
+                if len(event_str) > MAX_EVENT_STRING:
+                    parse_summary_logger.warning("file_path: %s, event string: %d exceeds %d and drop it.",
+                                                 file_handler.file_path, len(event_str), MAX_EVENT_STRING)
+                    continue
+                self._event_parse(event_str)
+            except exceptions.CRCFailedError:
+                parse_summary_logger.error("Check crc faild, file_path=%s, offset=%s.", file_handler.file_path,
+                                           file_handler.offset)
+                return False
+            except (OSError, DecodeError, exceptions.MindInsightException) as ex:
+                parse_summary_logger.error("Parse file fail, detail: %r, file path: %s.", str(ex),
+                                           file_handler.file_path)
+                return False
+
+    def _event_load(self, file_handler):
+        """
+        Load binary string to event string.
+
+        Args:
+            file_handler (FileHandler): A file handler.
+
+        Returns:
+            bytes, MindSpore event in bytes.
+        """
+        # read the header
+        header_str = file_handler.read(HEADER_SIZE)
+
+        if not header_str:
+            return None
+
+        header_crc_str = file_handler.read(CRC_STR_SIZE)
+        if not header_crc_str:
+            header_crc_str = ''
+
+        if len(header_str) != HEADER_SIZE or len(header_crc_str) != CRC_STR_SIZE:
+            parse_summary_logger.error("Check header size and crc, record truncated at offset %s, file_path=%s.",
+                                       file_handler.offset, file_handler.file_path)
+            return None
+        if not crc32.CheckValueAgainstData(header_crc_str, header_str, HEADER_SIZE):
+            raise exceptions.CRCFailedError()
+
+        # read the event body if integrity of header is verified
+        header = struct.unpack('Q', header_str)
+        event_len = int(header[0])
+
+        event_str = file_handler.read(event_len)
+        if not event_str:
+            event_str = ''
+        event_crc_str = file_handler.read(CRC_STR_SIZE)
+        if not event_crc_str:
+            event_crc_str = ''
+
+        if len(event_str) != event_len or len(event_crc_str) != CRC_STR_SIZE:
+            parse_summary_logger.error("Check event crc, record truncated at offset %d, file_path: %s.",
+                                       file_handler.offset, file_handler.file_path)
+            return None
+        if not crc32.CheckValueAgainstData(event_crc_str, event_str, event_len):
+            raise exceptions.CRCFailedError()
+        self._current += HEADER_SIZE + 2 * CRC_STR_SIZE + event_len
+        if self._current > self._process_info:
+            parse_summary_logger.info("current process: %d/%d, %d%%", self._current, self._file_size,
+                                      100 * self._current // self._file_size)
+            self._process_info += self._file_size // 10
+        return event_str
+
+    def _event_parse(self, event_str):
+        """
+        Transform `Event` data to event and extract the scalar and image data.
+
+        Args:
+            event_str (str): Message event string in summary proto, data read from file handler.
+        """
+
+        plugins = [SCALAR, IMAGE]
+
+        event = lazy_read_pb2.Event.FromString(event_str)
+
+        if event.HasField('summary'):
+            for value in event.summary.value:
+                for plugin in plugins:
+                    if not value.HasField(plugin):
+                        continue
+                    self._parse_summary_value(value.tag, event.step, event.wall_time, value, plugin)
+
+    def _parse_summary_value(self, tag, step, wall_time, value, plugin):
+        """
+        Parse summary value and write corresponding file according to plugin.
+
+        Args:
+            tag (str): value tag
+            step (int): train step
+            wall_time (float): Timestamp
+            value (Summary.Value): Value message in summary file.
+            plugin (str): Plugin value.
+        """
+        if plugin == SCALAR:
+            self._scalar_writer.add((tag, step, wall_time, value.scalar_value))
+            self._scalar_check = True
+
+        elif plugin == IMAGE:
+            self._image_writer.add((tag, step, value.image.encoded_image))
+            self._image_writer.write()
+            self._image_check = True
+
+    def _check_filepath(self):
+        """Check file path existence, accessible and available"""
+        if os.path.exists(self._summary_dir):
+            if not os.path.isdir(self._summary_dir):
+                parse_summary_logger.error('Path of summary directory is not a valid directory.')
+                return False
+            if not os.access(self._summary_dir, os.R_OK | os.X_OK):
+                parse_summary_logger.error('Path %s is not accessible, please check the file-authority.',
+                                           self._summary_dir)
+            return True
+        parse_summary_logger.error('Path of summary directory not exists.')
+        return False
+
+    def _check_create_filepath(self, filepath):
+        """Check file path existence, accessible and available, if not exist create the file"""
+        permissions = os.R_OK | os.W_OK | os.X_OK
+        os.umask(permissions << 3 | permissions)
+        if os.path.exists(filepath):
+            parse_summary_logger.error('Path %s has already existed, please choose a new output path.', filepath)
+            return False
+        mode = permissions << 6
+        os.makedirs(filepath, mode=mode)
+        return True

mindinsight/datavisual/data_transform/summary_parser/image_writer.py

@@ -0,0 +1,56 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Image Writer.
+
+This module write scalar into a  csv file.
+"""
+import os
+import re
+
+from mindinsight.datavisual.data_transform.summary_parser.writer import Writer
+
+
+class ImageWriter(Writer):
+    """ImageWriter write image into a png file."""
+    def __init__(self, file_path):
+        """
+        Init ImageWriter.
+
+        Args:
+            file_path (str): A directory path, e.g. '/output/image/'.
+        """
+        self._file_path = file_path
+        self._image_data = []
+
+    def add(self, value):
+        """
+        Add value.
+
+        Args:
+            value (object): tag and step and image value.
+        """
+        self._image_data.append(value)
+
+    def write(self):
+        """Write file."""
+        for i in range(len(self._image_data)):
+            tag = self._image_data[i][0]
+            tag = tag.replace('/', '_')
+            tag = re.sub(r'[^a-zA-Z0-9_]+', '', tag)
+            with os.fdopen(os.open("{}/{}_{}.png".format(self._file_path, tag, self._image_data[i][1]),
+                                   os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as fp:
+                fp.write(self._image_data[i][2])
+        self._image_data = []

mindinsight/datavisual/data_transform/summary_parser/scalar_writer.py

@@ -0,0 +1,51 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Scalar Writer.
+
+This module write scalar into a csv file.
+"""
+import csv
+import os
+
+from mindinsight.datavisual.data_transform.summary_parser.writer import Writer
+
+
+class ScalarWriter(Writer):
+    """ScalarWriter write scalar into a  csv file."""
+    def __init__(self, file_path):
+        """
+        Init ScalarWriter.
+
+        Args:
+            file_path (str): A directory path, e.g. '/output/'.
+        """
+        self._file_path = file_path
+        self._scalar_data = [("tag", "step", "wall_time", "value")]
+
+    def add(self, value):
+        """
+        Add value.
+
+        Args:
+            value (object): wall_time, tag and step and scalar value.
+        """
+        self._scalar_data.append(value)
+
+    def write(self):
+        """Write file."""
+        with os.fdopen(os.open('{}/scalar.csv'.format(self._file_path), os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fp:
+            writer = csv.writer(fp, dialect='excel')
+            writer.writerows(self._scalar_data)

mindinsight/datavisual/data_transform/summary_parser/writer.py

@@ -0,0 +1,32 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Base writer."""
+from abc import abstractmethod
+
+
+class Writer:
+    """Base writer for writers."""
+    @abstractmethod
+    def add(self, value):
+        """
+        Abstract method for adding value.
+
+        Args:
+            value (object): scalar, tensor or image value with wall_time, tag and step.
+        """
+
+    @abstractmethod
+    def write(self):
+        """Abstract method for writing file."""

mindinsight/datavisual/proto_files/lazy_read.proto

@@ -0,0 +1,45 @@
+syntax = "proto2";
+
+package mindinsight.summary;
+option cc_enable_arenas = true;
+
+// Event Protocol buffer, Top define
+message Event {
+  // Timestamp
+  required double wall_time = 1;
+
+  // The step of train.
+  optional int64 step = 2;
+
+  oneof what {
+    // An event file was started, with the specified version.
+    // Now version is "Mindspore.Event:1"
+    string version = 3;
+
+    // Summary data
+    Summary summary = 5;
+
+  }
+}
+
+
+// A Summary is a set of named values that be produced regularly during training
+message Summary {
+    message Image {
+        required bytes encoded_image = 4;
+    }
+
+    message Value {
+        // Tag name for the data.
+        required string tag = 1;
+
+        // Value associated with the tag.
+        oneof value {
+            float scalar_value = 3;
+            Image image = 4;
+        }
+    }
+
+    // Set of values for the summary.
+    repeated Value value = 1;
+}

mindinsight/datavisual/proto_files/lazy_read_pb2.py

@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: lazy_read.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='lazy_read.proto',
+  package='mindinsight.summary',
+  syntax='proto2',
+  serialized_options=b'\370\001\001',
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0flazy_read.proto\x12\x13mindinsight.summary\"t\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12/\n\x07summary\x18\x05 \x01(\x0b\x32\x1c.mindinsight.summary.SummaryH\x00\x42\x06\n\x04what\"\xc8\x01\n\x07Summary\x12\x31\n\x05value\x18\x01 \x03(\x0b\x32\".mindinsight.summary.Summary.Value\x1a\x1e\n\x05Image\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1aj\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12\x33\n\x05image\x18\x04 \x01(\x0b\x32\".mindinsight.summary.Summary.ImageH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01'
+)
+
+
+
+
+_EVENT = _descriptor.Descriptor(
+  name='Event',
+  full_name='mindinsight.summary.Event',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='wall_time', full_name='mindinsight.summary.Event.wall_time', index=0,
+      number=1, type=1, cpp_type=5, label=2,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='step', full_name='mindinsight.summary.Event.step', index=1,
+      number=2, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='version', full_name='mindinsight.summary.Event.version', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='summary', full_name='mindinsight.summary.Event.summary', index=3,
+      number=5, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+    _descriptor.OneofDescriptor(
+      name='what', full_name='mindinsight.summary.Event.what',
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+  ],
+  serialized_start=40,
+  serialized_end=156,
+)
+
+
+_SUMMARY_IMAGE = _descriptor.Descriptor(
+  name='Image',
+  full_name='mindinsight.summary.Summary.Image',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='encoded_image', full_name='mindinsight.summary.Summary.Image.encoded_image', index=0,
+      number=4, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=b"",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=221,
+  serialized_end=251,
+)
+
+_SUMMARY_VALUE = _descriptor.Descriptor(
+  name='Value',
+  full_name='mindinsight.summary.Summary.Value',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='tag', full_name='mindinsight.summary.Summary.Value.tag', index=0,
+      number=1, type=9, cpp_type=9, label=2,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='scalar_value', full_name='mindinsight.summary.Summary.Value.scalar_value', index=1,
+      number=3, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='image', full_name='mindinsight.summary.Summary.Value.image', index=2,
+      number=4, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+    _descriptor.OneofDescriptor(
+      name='value', full_name='mindinsight.summary.Summary.Value.value',
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+  ],
+  serialized_start=253,
+  serialized_end=359,
+)
+
+_SUMMARY = _descriptor.Descriptor(
+  name='Summary',
+  full_name='mindinsight.summary.Summary',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='mindinsight.summary.Summary.value', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[_SUMMARY_IMAGE, _SUMMARY_VALUE, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=159,
+  serialized_end=359,
+)
+
+_EVENT.fields_by_name['summary'].message_type = _SUMMARY
+_EVENT.oneofs_by_name['what'].fields.append(
+  _EVENT.fields_by_name['version'])
+_EVENT.fields_by_name['version'].containing_oneof = _EVENT.oneofs_by_name['what']
+_EVENT.oneofs_by_name['what'].fields.append(
+  _EVENT.fields_by_name['summary'])
+_EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what']
+_SUMMARY_IMAGE.containing_type = _SUMMARY
+_SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE
+_SUMMARY_VALUE.containing_type = _SUMMARY
+_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
+  _SUMMARY_VALUE.fields_by_name['scalar_value'])
+_SUMMARY_VALUE.fields_by_name['scalar_value'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']
+_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
+  _SUMMARY_VALUE.fields_by_name['image'])
+_SUMMARY_VALUE.fields_by_name['image'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']
+_SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE
+DESCRIPTOR.message_types_by_name['Event'] = _EVENT
+DESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), {
+  'DESCRIPTOR' : _EVENT,
+  '__module__' : 'lazy_read_pb2'
+  # @@protoc_insertion_point(class_scope:mindinsight.summary.Event)
+  })
+_sym_db.RegisterMessage(Event)
+
+Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), {
+
+  'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), {
+    'DESCRIPTOR' : _SUMMARY_IMAGE,
+    '__module__' : 'lazy_read_pb2'
+    # @@protoc_insertion_point(class_scope:mindinsight.summary.Summary.Image)
+    })
+  ,
+
+  'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), {
+    'DESCRIPTOR' : _SUMMARY_VALUE,
+    '__module__' : 'lazy_read_pb2'
+    # @@protoc_insertion_point(class_scope:mindinsight.summary.Summary.Value)
+    })
+  ,
+  'DESCRIPTOR' : _SUMMARY,
+  '__module__' : 'lazy_read_pb2'
+  # @@protoc_insertion_point(class_scope:mindinsight.summary.Summary)
+  })
+_sym_db.RegisterMessage(Summary)
+_sym_db.RegisterMessage(Summary.Image)
+_sym_db.RegisterMessage(Summary.Value)
+
+
+DESCRIPTOR._options = None
+# @@protoc_insertion_point(module_scope)

mindinsight/scripts/parse_summary.py

@@ -0,0 +1,116 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Parse summary file."""
+
+import argparse
+import os
+import datetime
+
+from mindinsight.utils.command import BaseCommand
+from mindinsight.datavisual.data_transform.summary_parser.event_parser import EventParser
+
+
+class FilepathAction(argparse.Action):
+    """File directory action class definition."""
+
+    def __call__(self, parser_in, namespace, values, option_string=None):
+        """
+        Inherited __call__ method from argparse.Action.
+
+        Args:
+            parser_in (ArgumentParser): Passed-in argument parser.
+            namespace (Namespace): Namespace object to hold arguments.
+            values (object): Argument values with type depending on argument definition.
+            option_string (str): Optional string for specific argument name. Default: None.
+        """
+        summary_dir = values
+        if summary_dir.startswith('~'):
+            summary_dir = os.path.realpath(os.path.expanduser(summary_dir))
+
+        if not summary_dir.startswith('/'):
+            summary_dir = os.path.realpath(os.path.join(os.getcwd(), summary_dir))
+
+        summary_dir = os.path.realpath(summary_dir)
+        setattr(namespace, self.dest, summary_dir)
+
+
+class OutputDirAction(argparse.Action):
+    """File directory action class definition."""
+
+    def __call__(self, parser_in, namespace, values, option_string=None):
+        """
+        Inherited __call__ method from argparse.Action.
+
+        Args:
+            parser_in (ArgumentParser): Passed-in argument parser.
+            namespace (Namespace): Namespace object to hold arguments.
+            values (object): Argument values with type depending on argument definition.
+            option_string (str): Optional string for specific argument name. Default: None.
+        """
+        output = values
+        if output.startswith('~'):
+            output = os.path.realpath(os.path.expanduser(output))
+
+        if not output.startswith('/'):
+            output = os.path.realpath(os.path.join(os.getcwd(), output))
+
+        output = os.path.realpath(output)
+        setattr(namespace, self.dest, output)
+
+
+class Command(BaseCommand):
+    """Start mindinsight service."""
+
+    name = 'parse_summary'
+    description = 'Parse summary file'
+
+    def add_arguments(self, parser):
+        """
+        Add arguments to parser.
+
+        Args:
+            parser (ArgumentParser): Specify parser to which arguments are added.
+        """
+        parser.add_argument(
+            '--summary-dir',
+            type=str,
+            action=FilepathAction,
+            default=os.path.realpath(os.getcwd()),
+            help="""
+                    Optional, specify path for summary file directory. 
+                    Default directory is the current working directory.
+                """)
+
+        parser.add_argument(
+            '--output',
+            type=str,
+            action=OutputDirAction,
+            default=os.path.realpath(os.getcwd()),
+            help="""
+                    Optional, specify path for converted file directory. Default output 
+                    directory is `output` folder in the current working directory.
+                """)
+
+    def run(self, args):
+        """
+        Execute for start command.
+
+        Args:
+            args (Namespace): Parsed arguments to hold customized parameters.
+        """
+        date_time = datetime.datetime.now().strftime('output_%Y%m%d_%H%M%S_%f')
+        date_time = os.path.join(args.output, date_time)
+        eventparser = EventParser(args.summary_dir, date_time)
+        eventparser.parse()