You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

tools.py 4.8 kB

5 years ago
5 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169
  1. # Copyright 2019 Huawei Technologies Co., Ltd
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # ============================================================================
  15. """Common Tools."""
  16. import imghdr
  17. import math
  18. import os
  19. from numbers import Number
  20. from urllib.parse import unquote
  21. from mindinsight.datavisual.common.exceptions import MaxCountExceededError
  22. from mindinsight.utils import exceptions
  23. _IMG_EXT_TO_MIMETYPE = {
  24. 'bmp': 'image/bmp',
  25. 'gif': 'image/gif',
  26. 'jpeg': 'image/jpeg',
  27. 'png': 'image/png',
  28. }
  29. _DEFAULT_IMAGE_MIMETYPE = 'application/octet-stream'
  30. def find_app_package():
  31. """Find package in current directory."""
  32. backend_dir = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, os.pardir, "backend"))
  33. packages = []
  34. for file in os.listdir(backend_dir):
  35. file_path = os.path.join(backend_dir, file)
  36. if os.path.isfile(file_path):
  37. continue
  38. if not os.path.isfile(os.path.join(file_path, '__init__.py')):
  39. continue
  40. rel_path = os.path.relpath(file_path, backend_dir)
  41. package = rel_path.replace(os.path.sep, '.')
  42. package = f"mindinsight.backend.{package}"
  43. packages.append(package)
  44. return packages
  45. def to_str(bytes_or_text, encode="utf-8"):
  46. """Bytes transform string."""
  47. if isinstance(bytes_or_text, bytes):
  48. return bytes_or_text.decode(encode)
  49. if isinstance(bytes_or_text, str):
  50. return bytes_or_text
  51. raise TypeError("Param isn't str or bytes type, param={}".format(bytes_or_text))
  52. def to_int(param, param_name):
  53. """
  54. Transfer param to int type.
  55. Args:
  56. param (Any): A param transformed.
  57. param_name (str): Param name.
  58. Returns:
  59. int, value after transformed.
  60. """
  61. try:
  62. param = int(param)
  63. except ValueError:
  64. raise exceptions.ParamTypeError(param_name, 'Integer')
  65. return param
  66. def str_to_bool(param, param_name):
  67. """
  68. Check param and transform it to bool.
  69. Args:
  70. param (str): 'true' or 'false' is valid.
  71. param_name (str): Param name.
  72. Returns:
  73. bool, if param is 'true', case insensitive.
  74. Raises:
  75. ParamValueError: If the value of param is not 'false' and 'true'.
  76. """
  77. if not isinstance(param, str):
  78. raise exceptions.ParamTypeError(param_name, 'str')
  79. if param.lower() not in ['false', 'true']:
  80. raise exceptions.ParamValueError("The value of %s must be 'false' or 'true'." % param_name)
  81. param = (param.lower() == 'true')
  82. return param
  83. def get_img_mimetype(img_data):
  84. """
  85. Recognize image headers and generate image MIMETYPE.
  86. Args:
  87. img_data (bin): Binary character stream of image.
  88. Returns:
  89. str, a MIMETYPE of the give image.
  90. """
  91. image_type = imghdr.what(None, img_data)
  92. mimetype = _IMG_EXT_TO_MIMETYPE.get(image_type, _DEFAULT_IMAGE_MIMETYPE)
  93. return mimetype
  94. def get_train_id(request):
  95. """
  96. Get train ID from requst query string and unquote content.
  97. Args:
  98. request (FlaskRequest): Http request instance.
  99. Returns:
  100. str, unquoted train ID.
  101. """
  102. train_id = request.args.get('train_id')
  103. if train_id is not None:
  104. try:
  105. train_id = unquote(train_id, errors='strict')
  106. except UnicodeDecodeError:
  107. raise exceptions.UrlDecodeError('Unquote train id error with strict mode')
  108. return train_id
  109. def if_nan_inf_to_none(name, value):
  110. """
  111. Transform value to None if it is NaN or Inf.
  112. Args:
  113. name (str): Name of value.
  114. value (float): A number transformed.
  115. Returns:
  116. float, if value is NaN or Inf, return None.
  117. """
  118. if not isinstance(value, Number):
  119. raise exceptions.ParamTypeError(name, 'number')
  120. if math.isnan(value) or math.isinf(value):
  121. value = None
  122. return value
  123. class Counter:
  124. """Count accumulator with limit checking."""
  125. def __init__(self, max_count=None, init_count=0):
  126. self._count = init_count
  127. self._max_count = max_count
  128. def add(self, value=1):
  129. """Add value."""
  130. if self._max_count is not None and self._count + value > self._max_count:
  131. raise MaxCountExceededError()
  132. self._count += value

MindInsight为MindSpore提供了简单易用的调优调试能力。在训练过程中,可以将标量、张量、图像、计算图、模型超参、训练耗时等数据记录到文件中,通过MindInsight可视化页面进行查看及分析。