pcm-coordinator/internal/cron/cron.go

/*

 Copyright (c) [2023] [pcm]
 [pcm-coordinator] is licensed under Mulan PSL v2.
 You can use this software according to the terms and conditions of the Mulan PSL v2.
 You may obtain a copy of Mulan PSL v2 at:
          http://license.coscl.org.cn/MulanPSL2
 THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 EITHER EXPaRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 See the Mulan PSL v2 for more details.

*/

package cron

import (
	"github.com/zeromicro/go-zero/core/logx"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/logic/schedule"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/service/utils/status"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/svc"
)

func AddCronGroup(svc *svc.ServiceContext) {

	svc.Cron.AddFunc("*/5  *  *  *  *  ?", func() {
		UpdateAiAdapterMaps(svc)
	})

	//svc.Cron.AddFunc("30  *  *  *  *  ?", func() {
	//	adapterList, err := svc.Scheduler.AiStorages.GetAdaptersByType("1")
	//	if err != nil {
	//		logx.Errorf(err.Error())
	//		return
	//	}
	//	stat.UpdateClusterResources(svc, adapterList)
	//})

	svc.Cron.AddFunc("@hourly", func() {
		status.UpdateAutoStoppedInstance(svc)
	})

	svc.Cron.AddFunc("1 * * * * *", func() {
		queryResource := schedule.NewQueryResourcesLogic(svc.HttpClient.R().Context(), svc)
		trainResrc, err := queryResource.QueryResourcesByClusterId(nil, "Train")
		if err != nil {
			logx.Error(err)
		}
		svc.Scheduler.AiService.LocalCache[schedule.QUERY_TRAIN_RESOURCES] = trainResrc
		inferResrc, err := queryResource.QueryResourcesByClusterId(nil, "Inference")
		if err != nil {
			logx.Error(err)
		}
		svc.Scheduler.AiService.LocalCache[schedule.QUERY_INFERENCE_RESOURCES] = inferResrc
	})

	//更新hpc任务状态
	svc.Cron.AddFunc("*/5  *  *  *  *  ?", func() {
		status.UpdateHpcTaskStatus(svc)
	})

	//更新推理任务状态
	svc.Cron.AddFunc("*/5  *  *  *  *  ?", func() {
		tasks, err := svc.Scheduler.AiStorages.GetInferDeployInstanceListLastMonth()
		if err != nil {
			logx.Error(err)
		}
		svc.Scheduler.AiService.Si.UpdateDeployInstanceStatusBatch(tasks, true)
	})

	//更新训练任务状态
	svc.Cron.AddFunc("*/10  *  *  *  *  ?", func() {
		tasks, err := svc.Scheduler.AiStorages.AllTaskLastMonth()
		if err != nil {
			logx.Error(err)
		}
		go svc.Scheduler.AiService.St.UpdateTaskStatus(tasks)
		go svc.Scheduler.AiService.St.UpdateAiTaskStatus(tasks)
	})
	svc.Cron.AddFunc("*/30  *  *  *  *  ?", func() {
		status.UpdateCloudTaskStatus(svc)
	})
}