You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

memory_dumper.cc 6.8 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. /**
  2. * Copyright 2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include "common/debug/memory_dumper.h"
  17. #include <string>
  18. #include "framework/common/debug/log.h"
  19. #include "framework/common/debug/ge_log.h"
  20. #include "framework/common/util.h"
  21. #include "framework/common/ge_inner_error_codes.h"
  22. using std::string;
  23. namespace {
  24. const int kInvalidFd = (-1);
  25. } // namespace
  26. namespace ge {
  27. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY MemoryDumper::MemoryDumper() : fd_(kInvalidFd) {}
  28. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY MemoryDumper::~MemoryDumper() { Close(); }
  29. // Dump the data to the file
  30. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::DumpToFile(const char *filename, void *data,
  31. int64_t len) {
  32. #ifdef FMK_SUPPORT_DUMP
  33. GE_CHECK_NOTNULL(filename);
  34. GE_CHECK_NOTNULL(data);
  35. if (len == 0) {
  36. GELOGE(FAILED, "[Check][Param]Failed, data length is 0.");
  37. REPORT_INNER_ERROR("E19999", "Check param failed, data length is 0.");
  38. return PARAM_INVALID;
  39. }
  40. // Open the file
  41. int fd = OpenFile(filename);
  42. if (fd == kInvalidFd) {
  43. GELOGE(FAILED, "[Open][File]Failed, filename:%s.", filename);
  44. REPORT_INNER_ERROR("E19999", "Opne file failed, filename:%s.", filename);
  45. return FAILED;
  46. }
  47. // Write the data to the file
  48. Status ret = SUCCESS;
  49. int32_t mmpa_ret = mmWrite(fd, data, len);
  50. // mmWrite return -1:Failed to write data to file;return -2:Invalid parameter
  51. if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
  52. GELOGE(FAILED, "[Write][Data]Failed, errno:%d, errmsg:%s", mmpa_ret, strerror(errno));
  53. REPORT_INNER_ERROR("E19999", "Write data failed, errno:%d, errmsg:%s.",
  54. mmpa_ret, strerror(errno));
  55. ret = FAILED;
  56. }
  57. // Close the file
  58. if (mmClose(fd) != EN_OK) { // mmClose return 0: success
  59. GELOGE(FAILED, "[Close][File]Failed, error_code:%u, filename:%s errmsg:%s.", ret, filename, strerror(errno));
  60. REPORT_INNER_ERROR("E19999", "Close file failed, error_code:%u, filename:%s errmsg:%s.",
  61. ret, filename, strerror(errno));
  62. ret = FAILED;
  63. }
  64. return ret;
  65. #else
  66. GELOGW("need to define FMK_SUPPORT_DUMP for dump op input and output.");
  67. return SUCCESS;
  68. #endif
  69. }
  70. // Open file
  71. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Open(const char *filename) {
  72. GE_CHK_BOOL_RET_STATUS(filename != nullptr, FAILED, "Incorrect parameter. filename is nullptr");
  73. // Try to remove file first for reduce the close time by overwriting way
  74. // (The process of file closing will be about 100~200ms slower per file when written by overwriting way)
  75. // If remove file failed, then try to open it with overwriting way
  76. int ret = remove(filename);
  77. // If remove file failed, print the warning log
  78. if (ret != 0) {
  79. GELOGW("Remove file failed.");
  80. }
  81. fd_ = OpenFile(filename);
  82. if (fd_ == kInvalidFd) {
  83. GELOGE(FAILED, "[Open][File]Failed, filename:%s.", filename);
  84. REPORT_INNER_ERROR("E19999", "Open file:%s failed.", filename);
  85. return FAILED;
  86. }
  87. return SUCCESS;
  88. }
  89. // Dump the data to file
  90. FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Dump(void *data, uint32_t len) const {
  91. GE_CHK_BOOL_RET_STATUS(data != nullptr, FAILED, "Incorrect parameter. data is nullptr");
  92. #ifdef FMK_SUPPORT_DUMP
  93. int32_t mmpa_ret = mmWrite(fd_, data, len);
  94. // mmWrite return -1:failed to write data to file;return -2:invalid parameter
  95. if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
  96. GELOGE(FAILED, "[Write][Data]Failed, errno:%d, errmsg:%s", mmpa_ret, strerror(errno));
  97. REPORT_INNER_ERROR("E19999", "Write data to file failed, errno:%d, errmsg:%s.",
  98. mmpa_ret, strerror(errno));
  99. return FAILED;
  100. }
  101. return SUCCESS;
  102. #else
  103. GELOGW("need to define FMK_SUPPORT_DUMP for dump op input and output.");
  104. return SUCCESS;
  105. #endif
  106. }
  107. // Close file
  108. void MemoryDumper::Close() noexcept {
  109. // Close file
  110. if (fd_ != kInvalidFd && mmClose(fd_) != EN_OK) {
  111. GELOGW("Close file failed, errmsg:%s.", strerror(errno));
  112. }
  113. fd_ = kInvalidFd;
  114. }
  115. // Open file
  116. int MemoryDumper::OpenFile(const char *filename) {
  117. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(filename == nullptr, return kInvalidFd, "Incorrect parameter. filename is nullptr");
  118. // Find the last separator
  119. int path_split_pos = static_cast<int>(strlen(filename) - 1);
  120. for (; path_split_pos >= 0; path_split_pos--) {
  121. GE_IF_BOOL_EXEC(filename[path_split_pos] == '\\' || filename[path_split_pos] == '/', break;)
  122. }
  123. // Get the absolute path
  124. string real_path;
  125. char tmp_path[MMPA_MAX_PATH] = {0};
  126. GE_IF_BOOL_EXEC(
  127. -1 != path_split_pos, string prefix_path = std::string(filename).substr(0, path_split_pos);
  128. string last_path = std::string(filename).substr(path_split_pos, strlen(filename) - 1);
  129. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(prefix_path.length() >= MMPA_MAX_PATH,
  130. return kInvalidFd, "Prefix path is too long!");
  131. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(mmRealPath(prefix_path.c_str(), tmp_path, MMPA_MAX_PATH) != EN_OK, return kInvalidFd,
  132. "Dir %s does not exit, errmsg:%s.", prefix_path.c_str(), strerror(errno));
  133. real_path = std::string(tmp_path) + last_path;)
  134. GE_IF_BOOL_EXEC(
  135. path_split_pos == -1 || path_split_pos == 0,
  136. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(strlen(filename) >= MMPA_MAX_PATH, return kInvalidFd, "Prefix path is too long!");
  137. GE_IF_BOOL_EXEC(mmRealPath(filename, tmp_path, MMPA_MAX_PATH) != EN_OK,
  138. GELOGI("File %s does not exit, it will be created.", filename));
  139. real_path = std::string(tmp_path);)
  140. // Open file, only the current user can read and write, to avoid malicious application access
  141. // Using the O_EXCL, if the file already exists,return failed to avoid privilege escalation vulnerability.
  142. mmMode_t mode = M_IRUSR | M_IWUSR;
  143. int32_t fd = mmOpen2(real_path.c_str(), M_RDWR | M_CREAT | M_APPEND, mode);
  144. if (fd == EN_ERROR || fd == EN_INVALID_PARAM) {
  145. GELOGE(kInvalidFd, "[Open][File]Failed. errno:%d, errmsg:%s, filename:%s.",
  146. fd, strerror(errno), filename);
  147. return kInvalidFd;
  148. }
  149. return fd;
  150. }
  151. } // namespace ge

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示