From: @hugo1 Reviewed-by: @wqtshg,@sheng-nan Signed-off-by: @wqtshgtags/v1.3.0
| @@ -341,7 +341,6 @@ set(TRAIN_SRC_LIST | |||
| "init/gelib.cc" | |||
| "model/ge_model.cc" | |||
| "model/ge_root_model.cc" | |||
| "omm/csa_interact.cc" | |||
| "opskernel_manager/ops_kernel_manager.cc" | |||
| "opskernel_manager/ops_kernel_builder_manager.cc" | |||
| "session/inner_session.cc" | |||
| @@ -416,7 +415,6 @@ set(TRAIN_SRC_LIST | |||
| set(INFER_SRC_LIST | |||
| "graph/manager/trans_var_data_utils.cc" | |||
| "omm/csa_interact.cc" | |||
| "common/fp16_t.cc" | |||
| "common/formats/utils/formats_trans_utils.cc" | |||
| "common/formats/format_transfers/datatype_transfer.cc" | |||
| @@ -57,7 +57,6 @@ set(SRC_LIST | |||
| "../common/profiling/ge_profiling.cc" | |||
| "../graph/load/graph_loader.cc" | |||
| "../graph/execute/graph_execute.cc" | |||
| "../omm/csa_interact.cc" | |||
| "../graph/manager/graph_manager_utils.cc" | |||
| "../graph/manager/graph_var_manager.cc" | |||
| "../graph/manager/graph_mem_allocator.cc" | |||
| @@ -11,7 +11,6 @@ local_ge_executor_src_files := \ | |||
| ../common/profiling/ge_profiling.cc \ | |||
| ../graph/load/graph_loader.cc \ | |||
| ../graph/execute/graph_execute.cc \ | |||
| ../omm/csa_interact.cc \ | |||
| ../graph/manager/graph_manager_utils.cc \ | |||
| ../graph/manager/graph_var_manager.cc \ | |||
| ../graph/manager/rdma_pool_allocator.cc \ | |||
| @@ -4,7 +4,6 @@ COMMON_LOCAL_SRC_FILES := \ | |||
| proto/fusion_model.proto \ | |||
| proto/optimizer_priority.proto \ | |||
| graph/manager/trans_var_data_utils.cc \ | |||
| omm/csa_interact.cc \ | |||
| common/fp16_t.cc \ | |||
| common/formats/utils/formats_trans_utils.cc \ | |||
| common/formats/format_transfers/datatype_transfer.cc \ | |||
| @@ -256,7 +256,6 @@ LIBGE_LOCAL_SRC_FILES := \ | |||
| init/gelib.cc \ | |||
| model/ge_model.cc \ | |||
| model/ge_root_model.cc \ | |||
| omm/csa_interact.cc \ | |||
| opskernel_manager/ops_kernel_manager.cc \ | |||
| opskernel_manager/ops_kernel_builder_manager.cc \ | |||
| session/inner_session.cc \ | |||
| @@ -21,7 +21,6 @@ | |||
| #include "graph/load/model_manager/model_manager.h" | |||
| #include "graph/load/model_manager/davinci_model.h" | |||
| #include "omm/csa_interact.h" | |||
| namespace ge { | |||
| using Uint32Pair = pair<uint32_t, uint32_t>; | |||
| @@ -490,12 +489,10 @@ Status GraphExecutor::AsyncExecuteModel(const GeRootModelPtr &ge_root_model, con | |||
| } catch (std::bad_alloc &) { | |||
| REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); | |||
| GELOGE(MEMALLOC_FAILED, "RunAsync failed, bad memory allocation occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return MEMALLOC_FAILED; | |||
| } catch (...) { | |||
| REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); | |||
| GELOGE(FAILED, "RunAsync failed, some exceptions occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return FAILED; | |||
| } | |||
| @@ -509,18 +506,15 @@ Status GraphExecutor::DataInput(const InputData &input_data, OutputData &output_ | |||
| Status ret = model_manager->DataInput(input_data, output_data); | |||
| if (ret != SUCCESS) { | |||
| GELOGE(ret, "DataInput: DataInput failed."); | |||
| CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return ret; | |||
| } | |||
| } catch (std::bad_alloc &) { | |||
| REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); | |||
| GELOGE(MEMALLOC_FAILED, "DataInput failed, bad memory allocation occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return MEMALLOC_FAILED; | |||
| } catch (...) { | |||
| REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); | |||
| GELOGE(FAILED, "DataInput failed, some exceptions occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return FAILED; | |||
| } | |||
| @@ -535,18 +529,15 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vector<Inp | |||
| Status ret = model_manager->GetInputOutputDescInfo(model_id, input_desc, output_desc); | |||
| if (ret != SUCCESS) { | |||
| GELOGE(ret, "GetInputOutputDescInfo failed."); | |||
| CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return ret; | |||
| } | |||
| } catch (std::bad_alloc &) { | |||
| REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); | |||
| GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return MEMALLOC_FAILED; | |||
| } catch (...) { | |||
| REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); | |||
| GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return FAILED; | |||
| } | |||
| @@ -564,18 +555,15 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vector<Inp | |||
| new_model_desc); | |||
| if (ret != SUCCESS) { | |||
| GELOGE(ret, "GetInputOutputDescInfo failed."); | |||
| CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return ret; | |||
| } | |||
| } catch (std::bad_alloc &) { | |||
| REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); | |||
| GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return MEMALLOC_FAILED; | |||
| } catch (...) { | |||
| REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); | |||
| GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !"); | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return FAILED; | |||
| } | |||
| @@ -24,7 +24,6 @@ | |||
| #include "graph/ge_context.h" | |||
| #include "graph/load/model_manager/model_manager.h" | |||
| #include "graph/manager/graph_var_manager.h" | |||
| #include "omm/csa_interact.h" | |||
| namespace ge { | |||
| Status GraphLoader::UnloadModel(uint32_t model_id) { | |||
| @@ -40,7 +39,6 @@ Status GraphLoader::UnloadModel(uint32_t model_id) { | |||
| ret = model_manager->Unload(model_id); | |||
| if (ret != SUCCESS) { | |||
| GELOGE(ret, "UnloadModel: Unload failed. model id:%u", model_id); | |||
| CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_UNLOAD); | |||
| return ret; | |||
| } | |||
| GELOGI("UnLoad model success, model id:%u.", model_id); | |||
| @@ -55,7 +53,6 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge | |||
| REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X", | |||
| GetContext().DeviceId(), rt_ret); | |||
| GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); | |||
| CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_LOAD); | |||
| return RT_FAILED; | |||
| } | |||
| if (ge_root_model_ptr == nullptr) { | |||
| @@ -69,8 +66,6 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge | |||
| Status ret = model_manager->LoadModelOnline(model_id, ge_root_model_ptr, listener); | |||
| if (ret != SUCCESS) { | |||
| GELOGE(ret, "LoadModel: Load failed. ret = %u", ret); | |||
| CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_LOAD); | |||
| rt_ret = rtDeviceReset(GetContext().DeviceId()); | |||
| if (rt_ret != RT_ERROR_NONE) { | |||
| REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", | |||
| @@ -94,7 +89,6 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge | |||
| } | |||
| GELOGE(ret, "LoadModel: Start failed."); | |||
| CsaInteract::GetInstance().WriteErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return ret; | |||
| } | |||
| rt_ret = rtDeviceReset(GetContext().DeviceId()); | |||
| @@ -247,7 +241,6 @@ Status GraphLoader::GetMemoryInfo(int64_t &free) { | |||
| REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X", | |||
| GetContext().DeviceId(), rt_ret); | |||
| GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); | |||
| CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_LOAD); | |||
| return RT_FAILED; | |||
| } | |||
| size_t total_mem = 0; | |||
| @@ -50,7 +50,6 @@ | |||
| #include "graph/utils/type_utils.h" | |||
| #include "init/gelib.h" | |||
| #include "mmpa/mmpa_api.h" | |||
| #include "omm/csa_interact.h" | |||
| #include "runtime/base.h" | |||
| #include "runtime/dev.h" | |||
| #include "runtime/event.h" | |||
| @@ -2718,7 +2717,6 @@ Status DavinciModel::ReturnNoOutput(uint32_t data_id) { | |||
| void *DavinciModel::Run(DavinciModel *model) { | |||
| GE_CHK_BOOL_EXEC(model != nullptr, | |||
| CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| return nullptr, "model_pointer is null!") | |||
| bool seq_end_flag = false; | |||
| uint32_t model_id = model->Id(); | |||
| @@ -2742,7 +2740,6 @@ void *DavinciModel::Run(DavinciModel *model) { | |||
| bool rslt_flg = true; | |||
| if (model->GetDataInputer() == nullptr) { | |||
| GELOGW("Data inputer is nullptr."); | |||
| CsaInteract::GetInstance().StoreInternalErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| break; | |||
| } | |||
| @@ -2763,7 +2760,6 @@ void *DavinciModel::Run(DavinciModel *model) { | |||
| ret = model->SyncVarData(); | |||
| GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( | |||
| ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||
| CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| continue, "Copy input data to model failed."); // [No need to check value] | |||
| GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(Model_SyncVarData, "Model Run SyncVarData")); | |||
| @@ -2773,7 +2769,6 @@ void *DavinciModel::Run(DavinciModel *model) { | |||
| ret = model->CopyInputData(current_data, false); | |||
| GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( | |||
| ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||
| CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| continue, "Copy input data to model failed."); // [No need to check value] | |||
| if (model->is_online_infer_dynamic_ && !model->is_getnext_sink_dynamic_) { | |||
| model->cur_dynamic_dims_.clear(); | |||
| @@ -2794,7 +2789,6 @@ void *DavinciModel::Run(DavinciModel *model) { | |||
| rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); | |||
| GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||
| (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||
| CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||
| continue); | |||
| GELOGI("rtModelExecute end"); | |||
| GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); | |||
| @@ -2812,7 +2806,6 @@ void *DavinciModel::Run(DavinciModel *model) { | |||
| rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); | |||
| (void)model->ReturnResult(current_data.index, false, seq_end_flag, | |||
| data_wrapper->GetOutput()); // [No need to check value] | |||
| CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||
| continue); | |||
| } | |||
| @@ -2841,7 +2834,6 @@ void *DavinciModel::Run(DavinciModel *model) { | |||
| GELOGI("run iterator count is %lu, model_id:%u", model->iterator_count_, model->model_id_); | |||
| } | |||
| CsaInteract::GetInstance().WriteInternalErrorCode(); | |||
| GELOGI("Model run end, model id:%u", model->model_id_); | |||
| return nullptr; | |||
| } | |||
| @@ -19,7 +19,6 @@ | |||
| #include "graph/utils/tensor_utils.h" | |||
| #include "graph/utils/type_utils.h" | |||
| #include "graph/ge_context.h" | |||
| #include "omm/csa_interact.h" | |||
| namespace ge { | |||
| namespace hybrid { | |||
| @@ -163,7 +162,6 @@ Status HybridModelAsyncExecutor::RunInternal() { | |||
| ret = PreRun(current_data, args); | |||
| GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( | |||
| ret != SUCCESS, (void) HandleResult(ret, current_data.index, args, data_wrapper->GetOutput()); | |||
| CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC); | |||
| continue, "[Invoke][PreRun] failed, model_id:%u.", model_id_); // [No need to check value] | |||
| if (pipe_executor_ != nullptr) { | |||
| @@ -181,7 +179,6 @@ Status HybridModelAsyncExecutor::RunInternal() { | |||
| } | |||
| ret = HandleResult(ret, current_data.index, args, data_wrapper->GetOutput()); | |||
| if (ret != SUCCESS) { | |||
| CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||
| continue; | |||
| } | |||
| @@ -191,7 +188,6 @@ Status HybridModelAsyncExecutor::RunInternal() { | |||
| GELOGI("run iterator count is %lu, model_id:%u", iterator_count_, model_id_); | |||
| } | |||
| CsaInteract::GetInstance().WriteInternalErrorCode(); | |||
| GELOGI("Model run end, model id:%u", model_id_); | |||
| return SUCCESS; | |||
| } | |||
| @@ -42,7 +42,6 @@ | |||
| #include "graph/manager/graph_mem_allocator.h" | |||
| #include "graph/manager/host_mem_manager.h" | |||
| #include "graph/manager/graph_var_manager.h" | |||
| #include "omm/csa_interact.h" | |||
| #include "runtime/kernel.h" | |||
| #include "opskernel_manager/ops_kernel_builder_manager.h" | |||
| #include "external/runtime/rt_error_codes.h" | |||
| @@ -376,10 +375,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status GELib::InitSystemWithOpt | |||
| } | |||
| GE_CHK_STATUS_RET(HostMemManager::Instance().Initialize()); | |||
| // Update CSA file | |||
| CsaInteract::GetInstance().Init(options.device_id, GetContext().TraceId()); | |||
| Status ret = CsaInteract::GetInstance().WriteJobState(JOBSTATE_RUNNING, JOBSUBSTATE_ENV_INIT); | |||
| GE_LOGE_IF(ret != SUCCESS, "[Write][JobState] failed, ret:%u ", ret); | |||
| // set device id | |||
| GELOGI("set logical device id:%u", options.device_id); | |||
| @@ -408,10 +403,6 @@ Status GELib::SystemShutdownWithOptions(const Options &options) { | |||
| GE_CHK_RT(rtDeviceReset(options.device_id)); | |||
| // Update CSA file | |||
| Status ret = CsaInteract::GetInstance().WriteJobState(JOBSTATE_SUCCEED); | |||
| GE_LOGE_IF(ret != SUCCESS, "[Write][JobState] failed, ret:%u ", ret); | |||
| is_system_inited = false; | |||
| is_shutdown = true; | |||
| GELOGI("%s finalize GELib success.", mode.c_str()); | |||
| @@ -1,265 +0,0 @@ | |||
| /** | |||
| * Copyright 2019-2020 Huawei Technologies Co., Ltd | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| */ | |||
| #include "omm/csa_interact.h" | |||
| #include "framework/common/debug/ge_log.h" | |||
| #include "framework/common/debug/log.h" | |||
| #include "framework/common/util.h" | |||
| #include "graph/ge_context.h" | |||
| #include "graph/manager/graph_var_manager.h" | |||
| #include "graph/utils/tensor_utils.h" | |||
| #include "mmpa/mmpa_api.h" | |||
| #include "nlohmann/json.hpp" | |||
| namespace ge { | |||
| namespace { | |||
| const char FMK_STATUS_FILE_DIR_ENV[] = "FMK_STATUS_FILE_DIR"; | |||
| const char JOBSTATE_FILE_NAME[] = "jobstateupdate_framework"; | |||
| const char HCOM_DETECT_FILE_NAME[] = "hcom_detection_result"; | |||
| const char FILE_SEPARATE[] = "/"; | |||
| } // namespace | |||
| /// | |||
| /// @brief Obtain CsaInteract instance | |||
| /// @return CsaInteract instance | |||
| /// | |||
| CsaInteract &CsaInteract::GetInstance() { | |||
| static CsaInteract instance; | |||
| return instance; | |||
| } | |||
| /// | |||
| /// @brief CsaInteract instance initialization | |||
| /// @param [in] dev_index device index | |||
| /// @param [in] job_id job id | |||
| /// @return void | |||
| /// | |||
| void CsaInteract::Init(int32_t dev_index, int64_t job_id) { | |||
| if (!is_init_) { | |||
| dev_index_ = dev_index; | |||
| job_id_ = job_id; | |||
| char file_dir_env[MMPA_MAX_PATH] = { 0x00 }; | |||
| INT32 res = mmGetEnv(FMK_STATUS_FILE_DIR_ENV, file_dir_env, MMPA_MAX_PATH); | |||
| string csa_path_prefix; | |||
| if (res == EN_OK) { | |||
| csa_path_prefix = file_dir_env; | |||
| } | |||
| if (!csa_path_prefix.empty()) { | |||
| job_state_file_ = csa_path_prefix + std::to_string(dev_index_) + FILE_SEPARATE + JOBSTATE_FILE_NAME; | |||
| hcom_detect_file_ = csa_path_prefix + std::to_string(dev_index_) + FILE_SEPARATE + HCOM_DETECT_FILE_NAME; | |||
| } | |||
| is_init_ = true; | |||
| } | |||
| } | |||
| /// | |||
| /// @brief Update job state file | |||
| /// @param [in] job_state job state | |||
| /// @param [in] job_sub_state detailed job state | |||
| /// @param [in] module_ret_errcode sub module training failure error code | |||
| /// @param [in] error_module error module identified by FMK | |||
| /// @return Status | |||
| /// | |||
| Status CsaInteract::WriteJobState(JobState job_state, JobSubState job_sub_state, uint32_t module_ret_errcode, | |||
| ErrorModule error_module) { | |||
| if (!is_init_) { | |||
| GELOGE(INTERNAL_ERROR, "[Init][CsaInteract] obj has not init, can't WriteJobState"); | |||
| REPORT_INNER_ERROR("E19999", "WriteJobState failed before init. "); | |||
| return INTERNAL_ERROR; | |||
| } | |||
| if ((curr_state_ == JOBSTATE_FAILED) || (curr_state_ == JOBSTATE_KILLED)) { | |||
| return SUCCESS; | |||
| } | |||
| if (job_state_file_.empty()) { | |||
| return SUCCESS; | |||
| } | |||
| std::string content; | |||
| try { | |||
| nlohmann::json content_json; | |||
| content_json["job_id"] = job_id_; | |||
| content_json["jobstate"] = job_state; | |||
| // Only the running or running failure state has a job sub state | |||
| if ((job_state == JOBSTATE_RUNNING) || (job_state == JOBSTATE_FAILED)) { | |||
| content_json["job_sub_state"] = job_sub_state; | |||
| } | |||
| content_json["time"] = CurrentTimeInStr(); | |||
| // Write error code only if run failed | |||
| if (job_state == JOBSTATE_FAILED) { | |||
| content_json["errorcode"] = module_ret_errcode; | |||
| content_json["errmodule"] = error_module; | |||
| } | |||
| content = content_json.dump(); | |||
| } catch (const nlohmann::json::exception &e) { | |||
| GELOGE(INTERNAL_ERROR, "[Create][JsonObject] exception:%s job_state:%u job_sub_state:%u.", | |||
| e.what(), job_state, job_sub_state); | |||
| REPORT_INNER_ERROR("E19999", "Create json object failed. exception:%s job_state:%u job_sub_state:%u.", | |||
| e.what(), job_state, job_sub_state); | |||
| return INTERNAL_ERROR; | |||
| } | |||
| if (WriteFile(job_state_file_, content) != SUCCESS) { | |||
| // The error log subfunction has been printed and will not print again | |||
| return INTERNAL_ERROR; | |||
| } | |||
| curr_state_ = job_state; | |||
| return SUCCESS; | |||
| } | |||
| /// | |||
| /// @brief Update error code in the job state file | |||
| /// @param [in] module_ret_errcode sub module training failure error code | |||
| /// @param [in] error_module error module identified by FMK | |||
| /// @param [in] job_sub_state detailed job state | |||
| /// @return void | |||
| /// | |||
| void CsaInteract::WriteErrorCode(uint32_t module_ret_errcode, ErrorModule error_module, JobSubState job_sub_state) { | |||
| // The error log subfunction has been printed and will not print again | |||
| Status ret = WriteJobState(JOBSTATE_FAILED, job_sub_state, module_ret_errcode, error_module); | |||
| if (ret != SUCCESS) { | |||
| GELOGW("write error code fail. ret_code: %u, status: %u", module_ret_errcode, job_sub_state); | |||
| } | |||
| } | |||
| /// | |||
| /// @brief Record errors that occurred durning the training | |||
| /// @param [in] module_ret_errcode sub module training failure error code | |||
| /// @param [in] error_module error module identified by FMK | |||
| /// @param [in] job_sub_state detailed job state | |||
| /// @return void | |||
| /// | |||
| void CsaInteract::StoreInternalErrorCode(uint32_t module_ret_errcode, ErrorModule error_module, | |||
| JobSubState job_sub_state) { | |||
| is_have_internal_error_ = true; | |||
| csa_error_code_.module_ret_errcode = module_ret_errcode; | |||
| csa_error_code_.error_module = error_module; | |||
| csa_error_code_.job_sub_state = job_sub_state; | |||
| } | |||
| /// | |||
| /// @brief Update training error code in the job state file | |||
| /// @return void | |||
| /// | |||
| void CsaInteract::WriteInternalErrorCode() { | |||
| if (is_have_internal_error_) { | |||
| WriteErrorCode(csa_error_code_.module_ret_errcode, csa_error_code_.error_module, csa_error_code_.job_sub_state); | |||
| } | |||
| } | |||
| /// | |||
| /// @brief Update network connectivity detect file | |||
| /// @param [in] content network connectivity content | |||
| /// @return Status | |||
| /// | |||
| Status CsaInteract::WriteHcomDetection(const std::string &content) { | |||
| if (!is_init_) { | |||
| GELOGE(INTERNAL_ERROR, "[Init][CsaInteract] obj has not init, can't WriteJobState"); | |||
| REPORT_INNER_ERROR("E19999", "WriteHcomDetection failed before init."); | |||
| return INTERNAL_ERROR; | |||
| } | |||
| if (hcom_detect_file_.empty()) { | |||
| return SUCCESS; | |||
| } | |||
| return WriteFile(hcom_detect_file_, content); | |||
| } | |||
| /// | |||
| /// @ingroup WriteFile | |||
| /// @brief Write the content into the file. If the file does not exist, create the file | |||
| /// @param [in] file_name: File name to be written | |||
| /// @param [in] content: Contents to be written | |||
| /// @return Status | |||
| /// | |||
| Status CsaInteract::WriteFile(const std::string &file_name, const std::string &content) { | |||
| // if file path is not exist, then make path | |||
| INT32 flags = M_WRONLY | O_TRUNC | M_CREAT; | |||
| int32_t fd = mmOpen2(file_name.c_str(), flags, M_IRUSR | M_IWUSR | M_UMASK_GRPREAD); | |||
| if (fd == EN_ERROR) { | |||
| if (MakePath(file_name) != SUCCESS) { | |||
| GELOGE(INTERNAL_ERROR, "[Create][File Path] errno is %d", errno); | |||
| REPORT_CALL_ERROR("E19999", "MakePath failed. errno is %d", errno); | |||
| return INTERNAL_ERROR; | |||
| } | |||
| fd = mmOpen2(file_name.c_str(), flags, M_IRUSR | M_IWUSR | M_UMASK_GRPREAD); | |||
| if (fd == EN_ERROR) { | |||
| GELOGE(INTERNAL_ERROR, "[Open][File] errno is %d file_name: %s", errno, file_name.c_str()); | |||
| REPORT_CALL_ERROR("E19999", "mmOpen2 failed. errno is %d file_name: %s", errno, file_name.c_str()); | |||
| return INTERNAL_ERROR; | |||
| } | |||
| } | |||
| mmSsize_t ret = mmWrite(fd, reinterpret_cast<void *>(const_cast<char *>(content.c_str())), content.length()); | |||
| if (ret == EN_ERROR) { | |||
| GELOGE(INTERNAL_ERROR, "[Write][File] errno is %d", errno); | |||
| REPORT_CALL_ERROR("E19999", "mmWrite failed. errno is %d", errno); | |||
| ret = mmClose(fd); | |||
| if (ret == EN_ERROR) { | |||
| GELOGE(INTERNAL_ERROR, "[Close][File] error is %d", errno); | |||
| REPORT_CALL_ERROR("E19999", "mmClose failed. error is %d", errno); | |||
| } | |||
| return INTERNAL_ERROR; | |||
| } | |||
| ret = mmClose(fd); | |||
| if (ret == EN_ERROR) { | |||
| GELOGE(INTERNAL_ERROR, "[Close][File] error is %d", errno); | |||
| REPORT_CALL_ERROR("E19999", "mmClose failed. error is %d", errno); | |||
| return INTERNAL_ERROR; | |||
| } | |||
| return SUCCESS; | |||
| } | |||
| /// | |||
| /// @ingroup MakePath | |||
| /// @brief Verify whether the file path exists, if not, recursively create the folder | |||
| /// @param [in] file_name: File name to be verified | |||
| /// @return Status | |||
| /// | |||
| Status CsaInteract::MakePath(const std::string &file_name) { | |||
| std::size_t found = file_name.find_last_of("/"); | |||
| if (found == std::string::npos) { | |||
| return PARAM_INVALID; | |||
| } | |||
| std::string file_path = file_name.substr(0, found + 1); | |||
| if (mmAccess(file_path.c_str()) == EN_OK) { | |||
| return SUCCESS; | |||
| } | |||
| found = file_path.find_first_of("/"); | |||
| while (found != std::string::npos) { | |||
| std::string pre_path = file_path.substr(0, found + 1); | |||
| if (mmAccess(pre_path.c_str()) != EN_OK) { | |||
| if (mmMkdir(pre_path.c_str(), M_IRWXU) != EN_OK) { | |||
| GELOGE(INTERNAL_ERROR, "[Create][FileDir] fail, errno is %d, pre_path:%s", errno, pre_path.c_str()); | |||
| REPORT_CALL_ERROR("E19999", "mmMkdir failed. errno is %d pre_path:%s", errno, pre_path.c_str()); | |||
| return INTERNAL_ERROR; | |||
| } | |||
| } | |||
| found = file_path.find_first_of("/", found + 1); | |||
| } | |||
| return SUCCESS; | |||
| } | |||
| } // namespace ge | |||
| @@ -1,183 +0,0 @@ | |||
| /** | |||
| * Copyright 2019-2020 Huawei Technologies Co., Ltd | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| */ | |||
| #ifndef GE_OMM_CSA_INTERACT_H_ | |||
| #define GE_OMM_CSA_INTERACT_H_ | |||
| #include <string> | |||
| #include "framework/common/ge_inner_error_codes.h" | |||
| namespace ge { | |||
| enum JobState { | |||
| JOBSTATE_WAITING = 1, | |||
| JOBSTATE_RUNNING, | |||
| JOBSTATE_KILLING, | |||
| JOBSTATE_SUCCEED, | |||
| JOBSTATE_FAILED, | |||
| JOBSTATE_KILLED, | |||
| JOBSTATE_UNKOWN | |||
| }; | |||
| enum JobSubState { | |||
| JOBSUBSTATE_ENV_INIT = 201, | |||
| JOBSUBSTATE_ENV_FIN, | |||
| JOBSUBSTATE_RESOUCE_ALLOC, | |||
| JOBSUBSTATE_MODEL_COMPILE, | |||
| JOBSUBSTATE_GRAPH_PREPARE, | |||
| JOBSUBSTATE_GRAPH_SPLIT, | |||
| JOBSUBSTATE_GRAPH_OPTIMIZE, | |||
| JOBSUBSTATE_GRAPH_BUILD, | |||
| JOBSUBSTATE_GRAPH_LOAD, | |||
| JOBSUBSTATE_GRAPH_EXEC, | |||
| JOBSUBSTATE_GRAPH_UNLOAD, | |||
| JOBSUBSTATE_OTHER | |||
| }; | |||
| enum ErrorModule { | |||
| ERROR_MODULE_DRIVER = 0x01, | |||
| ERROR_MODULE_RUNTIME = 0x04, | |||
| ERROR_MODULE_CCE = 0x06, | |||
| ERROR_MODULE_FMK = 0x08, | |||
| ERROR_MODULE_HCCL = 0x12 | |||
| }; | |||
| struct CsaErrorCode { | |||
| CsaErrorCode() | |||
| : module_ret_errcode(0), | |||
| error_module(ERROR_MODULE_FMK), | |||
| job_sub_state(JOBSUBSTATE_OTHER) {} | |||
| ~CsaErrorCode() {} | |||
| uint32_t module_ret_errcode; | |||
| ErrorModule error_module; | |||
| JobSubState job_sub_state; | |||
| }; | |||
| class CsaInteract { | |||
| public: | |||
| /// | |||
| /// @brief Obtain CsaInteract instance | |||
| /// @return CsaInteract instance | |||
| /// | |||
| static CsaInteract& GetInstance(); | |||
| /// | |||
| /// @brief CsaInteract instance initialization | |||
| /// @param [in] dev_index device index | |||
| /// @param [in] job_id job id | |||
| /// @return void | |||
| /// | |||
| void Init(int32_t dev_index, int64_t job_id); | |||
| /// | |||
| /// @brief Update job state file | |||
| /// @param [in] job_state job state | |||
| /// @param [in] job_sub_state detailed job state | |||
| /// @param [in] module_ret_errcode sub module training failure error code | |||
| /// @param [in] error_module error module identified by FMK | |||
| /// @return Status | |||
| /// | |||
| Status WriteJobState(JobState job_state, | |||
| JobSubState job_sub_state = JOBSUBSTATE_OTHER, | |||
| uint32_t module_ret_errcode = SUCCESS, | |||
| ErrorModule error_module = ERROR_MODULE_FMK); | |||
| /// | |||
| /// @brief Update error code in the job state file | |||
| /// @param [in] module_ret_errcode sub module training failure error code | |||
| /// @param [in] error_module error module identified by FMK | |||
| /// @param [in] job_sub_state detailed job state | |||
| /// @return void | |||
| /// | |||
| void WriteErrorCode(uint32_t module_ret_errcode, ErrorModule error_module, | |||
| JobSubState job_sub_state); | |||
| /// | |||
| /// @brief Record errors that occurred durning the training | |||
| /// @param [in] module_ret_errcode sub module training failure error code | |||
| /// @param [in] error_module error module identified by FMK | |||
| /// @param [in] job_sub_state detailed job state | |||
| /// @return void | |||
| /// | |||
| void StoreInternalErrorCode(uint32_t module_ret_errcode, | |||
| ErrorModule error_module, | |||
| JobSubState job_sub_state); | |||
| /// | |||
| /// @brief Update training error code in the job state file | |||
| /// @return void | |||
| /// | |||
| void WriteInternalErrorCode(); | |||
| /// | |||
| /// @brief Update network connectivity detect file | |||
| /// @param [in] content network connectivity content | |||
| /// @return Status | |||
| /// | |||
| Status WriteHcomDetection(const std::string& content); | |||
| private: | |||
| CsaInteract() | |||
| : dev_index_(0), | |||
| job_id_(0), | |||
| is_init_(false), | |||
| curr_state_(JOBSTATE_UNKOWN), | |||
| is_have_internal_error_(false) {} | |||
| ~CsaInteract() {} | |||
| CsaInteract(const CsaInteract&) = delete; | |||
| CsaInteract(CsaInteract&&) = delete; | |||
| CsaInteract& operator=(const CsaInteract&) = delete; | |||
| CsaInteract& operator=(CsaInteract&&) = delete; | |||
| /// | |||
| /// @ingroup WriteFile | |||
| /// @brief Write the content into the file. If the file does not exist, create the file | |||
| /// @param [in] file_name: File name to be written | |||
| /// @param [in] content: Contents to be written | |||
| /// @return Status | |||
| /// | |||
| Status WriteFile(const std::string& file_name, const std::string& content); | |||
| /// | |||
| /// @ingroup MakePath | |||
| /// @brief Verify whether the file path exists, if not, recursively create the folder | |||
| /// @param [in] file_name: File name to be verified | |||
| /// @return Status | |||
| /// | |||
| Status MakePath(const std::string& file_name); | |||
| // device index | |||
| int32_t dev_index_; | |||
| // job id | |||
| int64_t job_id_; | |||
| // is initialization complete | |||
| bool is_init_; | |||
| // current job state | |||
| JobState curr_state_; | |||
| // job state file | |||
| std::string job_state_file_; | |||
| // network connectivity detect file | |||
| std::string hcom_detect_file_; | |||
| // identification of internal errors that occurred during the training | |||
| bool is_have_internal_error_; | |||
| // error code information | |||
| CsaErrorCode csa_error_code_; | |||
| }; | |||
| } // namespace ge | |||
| #endif // GE_OMM_CSA_INTERACT_H_ | |||
| @@ -287,7 +287,6 @@ set(COMMON_SRC_FILES | |||
| "${GE_CODE_DIR}/ge/graph/load/model_manager/zero_copy_task.cc" | |||
| "${GE_CODE_DIR}/ge/graph/load/model_manager/cpu_queue_schedule.cc" | |||
| "${GE_CODE_DIR}/ge/graph/load/model_manager/aipp_utils.cc" | |||
| "${GE_CODE_DIR}/ge/omm/csa_interact.cc" | |||
| "${GE_CODE_DIR}/ge/graph/load/model_manager/tbe_handle_store.cc" | |||
| "${GE_CODE_DIR}/ge/common/kernel_store.cc" | |||
| "${GE_CODE_DIR}/ge/common/tbe_kernel_store.cc" | |||
| @@ -391,7 +390,6 @@ set(GRAPH_PARTITION_COMMON_SRC_FILES | |||
| set(GRAPH_LOAD_COMMON_SRC_FILES | |||
| "${GE_CODE_DIR}/ge/graph/load/graph_loader.cc" | |||
| "${GE_CODE_DIR}/ge/graph/manager/graph_manager_utils.cc" | |||
| "${GE_CODE_DIR}/ge/omm/csa_interact.cc" | |||
| "${GE_CODE_DIR}/ge/graph/manager/graph_mem_allocator.cc" | |||
| "${GE_CODE_DIR}/ge/graph/manager/graph_var_manager.cc" | |||
| "${GE_CODE_DIR}/ge/graph/manager/trans_var_data_utils.cc" | |||
| @@ -22,7 +22,6 @@ | |||
| #include "graph/execute/graph_execute.h" | |||
| #include "graph/load/model_manager/model_manager.h" | |||
| #include "graph/load/model_manager/davinci_model.h" | |||
| #include "omm/csa_interact.h" | |||
| #undef private | |||
| #undef public | |||