You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

log_operations.py 3.2 kB

5 years ago
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
  1. # Copyright 2019 Huawei Technologies Co., Ltd
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # ============================================================================
  15. """log operations module."""
  16. import json
  17. import os
  18. import time
  19. from tests.ut.datavisual.utils.log_generators.graph_log_generator import GraphLogGenerator
  20. from tests.ut.datavisual.utils.log_generators.images_log_generator import ImagesLogGenerator
  21. from tests.ut.datavisual.utils.log_generators.scalars_log_generator import ScalarsLogGenerator
  22. from mindinsight.datavisual.common.enums import PluginNameEnum
  23. log_generators = {
  24. PluginNameEnum.GRAPH.value: GraphLogGenerator(),
  25. PluginNameEnum.IMAGE.value: ImagesLogGenerator(),
  26. PluginNameEnum.SCALAR.value: ScalarsLogGenerator()
  27. }
  28. class LogOperations:
  29. """Log Operations class."""
  30. @staticmethod
  31. def generate_log(plugin_name, log_dir, log_settings, valid=True):
  32. """
  33. Generate log.
  34. Args:
  35. plugin_name (str): Plugin name, contains 'graph', 'image', and 'scalar'.
  36. log_dir (str): Log path to write log.
  37. log_settings (dict): Info about the log, e.g.:
  38. {
  39. current_time (int): Timestamp in summary file name, not necessary.
  40. graph_base_path (str): Path of graph_bas.json, necessary for `graph`.
  41. steps (list[int]): Steps for `image` and `scalar`, default is [1].
  42. tag (str): Tag name, default is 'default_tag'.
  43. }
  44. valid (bool): If true, summary name will be valid.
  45. Returns:
  46. str, Summary log path.
  47. """
  48. current_time = log_settings.get('time', int(time.time()))
  49. current_time = int(current_time)
  50. log_generator = log_generators.get(plugin_name)
  51. if valid:
  52. temp_path = os.path.join(log_dir, '%s.%s' % ('test.summary', str(current_time)))
  53. else:
  54. temp_path = os.path.join(log_dir, '%s.%s' % ('test.invalid', str(current_time)))
  55. if plugin_name == PluginNameEnum.GRAPH.value:
  56. graph_base_path = log_settings.get('graph_base_path')
  57. with open(graph_base_path, 'r') as load_f:
  58. graph_dict = json.load(load_f)
  59. graph_dict = log_generator.generate_log(temp_path, graph_dict)
  60. return temp_path, graph_dict
  61. steps_list = log_settings.get('steps', [1])
  62. tag_name = log_settings.get('tag', 'default_tag')
  63. metadata, values = log_generator.generate_log(temp_path, steps_list, tag_name)
  64. return temp_path, metadata, values
  65. @staticmethod
  66. def get_log_generator(plugin_name):
  67. """Get log generator."""
  68. return log_generators.get(plugin_name)

MindInsight为MindSpore提供了简单易用的调优调试能力。在训练过程中,可以将标量、张量、图像、计算图、模型超参、训练耗时等数据记录到文件中,通过MindInsight可视化页面进行查看及分析。