You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

conftest.py 4.4 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115
  1. # Copyright 2020 Huawei Technologies Co., Ltd
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # ============================================================================
  15. """
  16. Description: This file is used for some common util.
  17. """
  18. import os
  19. import shutil
  20. from unittest.mock import Mock
  21. import pytest
  22. from flask import Response
  23. from mindinsight.conf import settings
  24. from mindinsight.datavisual.data_transform import data_manager
  25. from mindinsight.datavisual.data_transform.data_manager import DataManager
  26. from mindinsight.datavisual.data_transform.loader_generators.loader_generator import MAX_DATA_LOADER_SIZE
  27. from mindinsight.datavisual.utils import tools
  28. from ....utils.log_operations import LogOperations
  29. from ....utils.tools import check_loading_done
  30. from . import constants
  31. from . import globals as gbl
  32. summaries_metadata = None
  33. mock_data_manager = None
  34. summary_base_dir = constants.SUMMARY_BASE_DIR
  35. @pytest.fixture(autouse=True)
  36. def set_summary_base_dir(monkeypatch):
  37. """Mock settings.SUMMARY_BASE_DIR."""
  38. monkeypatch.setattr(settings, 'SUMMARY_BASE_DIR', summary_base_dir)
  39. @pytest.fixture(scope="session")
  40. def init_summary_logs():
  41. """Init summary logs."""
  42. try:
  43. if os.path.exists(summary_base_dir):
  44. shutil.rmtree(summary_base_dir)
  45. permissions = os.R_OK | os.W_OK | os.X_OK
  46. mode = permissions << 6
  47. if not os.path.exists(summary_base_dir):
  48. os.mkdir(summary_base_dir, mode=mode)
  49. global summaries_metadata, mock_data_manager
  50. log_operations = LogOperations()
  51. summaries_metadata = log_operations.create_summary_logs(summary_base_dir, constants.SUMMARY_DIR_NUM_FIRST,
  52. constants.SUMMARY_DIR_PREFIX)
  53. mock_data_manager = DataManager(summary_base_dir)
  54. mock_data_manager.start_load_data(reload_interval=0)
  55. check_loading_done(mock_data_manager)
  56. summaries_metadata.update(
  57. log_operations.create_summary_logs(summary_base_dir, constants.SUMMARY_DIR_NUM_SECOND,
  58. constants.SUMMARY_DIR_PREFIX,
  59. constants.SUMMARY_DIR_NUM_FIRST))
  60. summaries_metadata.update(
  61. log_operations.create_multiple_logs(summary_base_dir, constants.MULTIPLE_DIR_NAME,
  62. constants.MULTIPLE_LOG_NUM))
  63. summaries_metadata.update(
  64. log_operations.create_reservoir_log(summary_base_dir, constants.RESERVOIR_DIR_NAME,
  65. constants.RESERVOIR_STEP_NUM))
  66. mock_data_manager.start_load_data(reload_interval=0)
  67. # Sleep 1 sec to make sure the status of mock_data_manager changed to LOADING.
  68. check_loading_done(mock_data_manager, first_sleep_time=1)
  69. # Maximum number of loads is `MAX_DATA_LOADER_SIZE`.
  70. for i in range(len(summaries_metadata) - MAX_DATA_LOADER_SIZE):
  71. summaries_metadata.pop("./%s%d" % (constants.SUMMARY_DIR_PREFIX, i))
  72. yield
  73. finally:
  74. if os.path.exists(summary_base_dir):
  75. shutil.rmtree(summary_base_dir)
  76. @pytest.fixture(autouse=True)
  77. def populate_globals():
  78. """Populate globals."""
  79. gbl.summaries_metadata = summaries_metadata
  80. gbl.mock_data_manager = mock_data_manager
  81. @pytest.fixture
  82. def client():
  83. """This fixture is flask client."""
  84. gbl.mock_data_manager.start_load_data(reload_interval=0)
  85. check_loading_done(gbl.mock_data_manager)
  86. data_manager.DATA_MANAGER = gbl.mock_data_manager
  87. packages = ["mindinsight.backend.datavisual"]
  88. mock_obj = Mock(return_value=packages)
  89. tools.find_app_package = mock_obj
  90. from mindinsight.backend.application import APP
  91. APP.response_class = Response
  92. app_client = APP.test_client()
  93. yield app_client

MindInsight为MindSpore提供了简单易用的调优调试能力。在训练过程中,可以将标量、张量、图像、计算图、模型超参、训练耗时等数据记录到文件中,通过MindInsight可视化页面进行查看及分析。