From e553d022338d12868d27694036e5628c742cdbb0 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 31 May 2022 11:55:10 +0800 Subject: [PATCH 01/90] #1249 add models --- models/point_account.go | 32 ++++++++++++++++++ models/point_account_log.go | 16 +++++++++ models/point_limit_config.go | 23 +++++++++++++ models/point_operate_record.go | 36 +++++++++++++++++++++ models/point_periodic_task.go | 28 ++++++++++++++++ models/point_task_accomplish_log.go | 12 +++++++ models/point_task_config.go | 25 ++++++++++++++ services/reward/account/account.go | 9 ++++++ services/reward/operate/callback.go | 4 +++ services/reward/operate/operator.go | 45 ++++++++++++++++++++++++++ services/reward/point/point_operate.go | 16 +++++++++ services/reward/reward.go | 6 ++++ services/task/point_task.go | 10 ++++++ 13 files changed, 262 insertions(+) create mode 100644 models/point_account.go create mode 100644 models/point_account_log.go create mode 100644 models/point_limit_config.go create mode 100644 models/point_operate_record.go create mode 100644 models/point_periodic_task.go create mode 100644 models/point_task_accomplish_log.go create mode 100644 models/point_task_config.go create mode 100644 services/reward/account/account.go create mode 100644 services/reward/operate/callback.go create mode 100644 services/reward/operate/operator.go create mode 100644 services/reward/point/point_operate.go create mode 100644 services/reward/reward.go create mode 100644 services/task/point_task.go diff --git a/models/point_account.go b/models/point_account.go new file mode 100644 index 000000000..f889d5d4f --- /dev/null +++ b/models/point_account.go @@ -0,0 +1,32 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type PointAccountStatus int + +// Possible PointAccountStatus types. +const ( + PointAccountNormal PointAccountStatus = iota + 1 // 1 + PointAccountFreeze // 2 + PointAccountDeleted // 3 +) + +type PointAccount struct { + ID int64 `xorm:"pk autoincr"` + Balance int64 `xorm:"NOT NULL DEFAULT 0"` + TotalEarned int64 `xorm:"NOT NULL DEFAULT 0"` + TotalConsumed int64 `xorm:"NOT NULL DEFAULT 0"` + UserId int64 `xorm:"INDEX NOT NULL"` + Status string `xorm:"NOT NULL"` + Version int64 `xorm:"NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +func (account *PointAccount) Increase(amount int64) error { + return nil +} + +func (account *PointAccount) Decrease(amount int64) error { + return nil +} diff --git a/models/point_account_log.go b/models/point_account_log.go new file mode 100644 index 000000000..ae718fe0f --- /dev/null +++ b/models/point_account_log.go @@ -0,0 +1,16 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type PointAccountLog struct { + ID int64 `xorm:"pk autoincr"` + AccountId int64 `xorm:"INDEX NOT NULL"` + UserId int64 `xorm:"INDEX NOT NULL"` + Type string `xorm:"NOT NULL"` + RelatedId string `xorm:"INDEX NOT NULL"` + PointsAmount int64 `xorm:"NOT NULL"` + AmountBefore int64 `xorm:"NOT NULL"` + AmountAfter int64 `xorm:"NOT NULL"` + AccountVersion int64 `xorm:"NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} diff --git a/models/point_limit_config.go b/models/point_limit_config.go new file mode 100644 index 000000000..60fb5e735 --- /dev/null +++ b/models/point_limit_config.go @@ -0,0 +1,23 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +const ( + LimitConfigRefreshRateOnce = "ONCE" + LimitConfigRefreshRateDaily = "DAILY" +) +const ( + LimitTargetRangeAllUser = "ALL_USER" + LimitTargetRangeSingleUser = "SINGLE_USER" +) + +type PointLimitConfig struct { + ID int64 `xorm:"pk autoincr"` + Tittle string + RefreshRate string `xorm:"NOT NULL"` + TargetRange string `xorm:"NOT NULL"` + LimitNum int64 `xorm:"NOT NULL"` + Creator int64 `xorm:"NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + DeletedAt timeutil.TimeStamp `xorm:"deleted"` +} diff --git a/models/point_operate_record.go b/models/point_operate_record.go new file mode 100644 index 000000000..d2dda7863 --- /dev/null +++ b/models/point_operate_record.go @@ -0,0 +1,36 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type RewardSourceType string + +const ( + SourceTypeAccomplishPointTask RewardSourceType = "ACCOMPLISH_POINT_TASK" + SourceTypeAdminOperate RewardSourceType = "ADMIN_OPERATE" + SourceTypeRunCloudbrainTask RewardSourceType = "RUN_CLOUBRAIN_TASK" +) + +const ( + OperateTypeIncrease = "INCREASE_POINT" + OperateTypeDecrease = "DECREASE_POINT" +) + +const ( + OperateStatusOperating = "OPERATING" + OperateStatusSucceeded = "SUCCEEDED" + OperateStatusFailed = "FAILED" +) + +type PointOperateRecord struct { + ID int64 `xorm:"pk autoincr"` + UserId int64 `xorm:"INDEX NOT NULL"` + PointsAmount int64 `xorm:"NOT NULL"` + RelatedType string `xorm:"NOT NULL"` + RelatedId string `xorm:"INDEX NOT NULL"` + OperateType string `xorm:"NOT NULL"` + OperateRate string `xorm:"NOT NULL default once"` + Status string `xorm:"NOT NULL"` + Remark string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} diff --git a/models/point_periodic_task.go b/models/point_periodic_task.go new file mode 100644 index 000000000..0d4297f2f --- /dev/null +++ b/models/point_periodic_task.go @@ -0,0 +1,28 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type PeriodicTaskStatus int + +// Possible PeriodicTaskStatus types. +const ( + PeriodicTaskStatusRunning PointAccountStatus = iota + 1 // 1 + PeriodicTaskStatusSuccess // 2 + PeriodicTaskStatusFailed // 3 +) + +type PeriodicTask struct { + ID int64 `xorm:"pk autoincr"` + Type string `xorm:"NOT NULL"` + OperateRecordId int64 `xorm:"INDEX NOT NULL"` + IntervalSecond int64 `xorm:"NOT NULL"` + PointsAmount int64 `xorm:"NOT NULL"` + NextExecuteTime timeutil.TimeStamp + SuccessCount int `xorm:"NOT NULL default 0"` + FailedCount int `xorm:"NOT NULL default 0"` + Status string `xorm:"NOT NULL"` + ExitCode string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + FinishedUnix timeutil.TimeStamp `xorm:"INDEX"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} diff --git a/models/point_task_accomplish_log.go b/models/point_task_accomplish_log.go new file mode 100644 index 000000000..82c45e163 --- /dev/null +++ b/models/point_task_accomplish_log.go @@ -0,0 +1,12 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type PointTaskAccomplishLog struct { + ID int64 `xorm:"pk autoincr"` + ConfigId int64 `xorm:"NOT NULL"` + TaskCode int64 `xorm:"NOT NULL"` + UserId int64 `xorm:"INDEX NOT NULL"` + RelatedId int64 `xorm:"INDEX NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} diff --git a/models/point_task_config.go b/models/point_task_config.go new file mode 100644 index 000000000..070e3d29e --- /dev/null +++ b/models/point_task_config.go @@ -0,0 +1,25 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" +) + +const ( + TaskConfigRefreshRateOnce = "ONCE" + TaskConfigRefreshRateDaily = "DAILY" +) + +//PointTaskConfig Only add and delete are allowed, edit is not allowed +//so if you want to edit config for some task code,please delete first and add new one +type PointTaskConfig struct { + ID int64 `xorm:"pk autoincr"` + TaskCode string `xorm:"NOT NULL"` + Tittle string `xorm:"NOT NULL"` + RefreshRate string `xorm:"NOT NULL"` + Times int `xorm:"NOT NULL"` + AwardPoints int `xorm:"NOT NULL"` + Status int `xorm:"NOT NULL"` + Creator int64 `xorm:"NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + DeletedAt timeutil.TimeStamp `xorm:"deleted"` +} diff --git a/services/reward/account/account.go b/services/reward/account/account.go new file mode 100644 index 000000000..4967b368e --- /dev/null +++ b/services/reward/account/account.go @@ -0,0 +1,9 @@ +package account + +func IncreaseAmount(userId int64, amount int64) error { + return nil +} + +func DecreaseAmount(userId int64, amount int64) error { + return nil +} diff --git a/services/reward/operate/callback.go b/services/reward/operate/callback.go new file mode 100644 index 000000000..27c42f443 --- /dev/null +++ b/services/reward/operate/callback.go @@ -0,0 +1,4 @@ +package operate + +type CallbackHandler struct { +} diff --git a/services/reward/operate/operator.go b/services/reward/operate/operator.go new file mode 100644 index 000000000..63d12b970 --- /dev/null +++ b/services/reward/operate/operator.go @@ -0,0 +1,45 @@ +package operate + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/services/reward" +) + +type RewardOperateContext struct { + SourceType models.RewardSourceType + RelatedId string + Remark string + Reward reward.Reward + TargetUserId int64 + RequestId string +} + +type RewardOperateResponse int + +const ( + RewardOperateSuccess RewardOperateResponse = iota + 1 + RewardOperateBalanceNotEnough +) + +func (t RewardOperateResponse) IsSuccess() bool { + return t == RewardOperateSuccess +} + +type RewardOperator interface { + IsOperated(ctx RewardOperateContext) bool + IsLimited(ctx RewardOperateContext) bool + Operate(ctx RewardOperateContext) error +} + +func Operate(operator RewardOperator, ctx RewardOperateContext) error { + if operator.IsOperated(ctx) { + return nil + } + if operator.IsLimited(ctx) { + return nil + } + if err := operator.Operate(ctx); err != nil { + return err + } + return nil +} diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go new file mode 100644 index 000000000..f91ca11b3 --- /dev/null +++ b/services/reward/point/point_operate.go @@ -0,0 +1,16 @@ +package point + +import "code.gitea.io/gitea/services/reward/operate" + +type PointOperator struct { +} + +func (operator *PointOperator) IsOperated(ctx operate.RewardOperateContext) bool { + return true +} +func (operator *PointOperator) IsLimited(ctx operate.RewardOperateContext) bool { + return true +} +func (operator *PointOperator) Operate(ctx operate.RewardOperateContext) error { + return nil +} diff --git a/services/reward/reward.go b/services/reward/reward.go new file mode 100644 index 000000000..8ec0e0471 --- /dev/null +++ b/services/reward/reward.go @@ -0,0 +1,6 @@ +package reward + +type Reward struct { + Amount int + Type string +} diff --git a/services/task/point_task.go b/services/task/point_task.go new file mode 100644 index 000000000..b72fbffdc --- /dev/null +++ b/services/task/point_task.go @@ -0,0 +1,10 @@ +package task + +func Accomplish() error { + //1、幂等性判断 + //2、获取任务配置 + //3、判断任务是否可以完成 + //4、生成任务记录 + //5、触发奖励发放 + return nil +} From 57a590de791a2742130591b3c7fc4693f4d9f1b0 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 31 May 2022 17:43:46 +0800 Subject: [PATCH 02/90] update task and reward --- models/error.go | 12 +++ models/point_account_log.go | 2 +- models/point_operate_record.go | 14 ++- models/point_task_accomplish_log.go | 12 --- models/point_task_config.go | 25 ------ models/task_accomplish_log.go | 51 +++++++++++ models/task_config.go | 53 +++++++++++ modules/auth/wechat/access_token.go | 8 +- modules/redis/redis_key/key_base.go | 2 + modules/redis/redis_key/task_redis_key.go | 16 ++++ services/reward/{operate => }/callback.go | 2 +- services/reward/{operate => }/operator.go | 25 ++++-- services/reward/point/point_operate.go | 11 ++- services/reward/reward.go | 2 +- services/task/limiter.go | 39 ++++++++ services/task/point_task.go | 10 --- services/task/task.go | 104 ++++++++++++++++++++++ services/task/task_config.go | 28 ++++++ 18 files changed, 347 insertions(+), 69 deletions(-) delete mode 100644 models/point_task_accomplish_log.go delete mode 100644 models/point_task_config.go create mode 100644 models/task_accomplish_log.go create mode 100644 models/task_config.go create mode 100644 modules/redis/redis_key/task_redis_key.go rename services/reward/{operate => }/callback.go (67%) rename services/reward/{operate => }/operator.go (58%) create mode 100644 services/task/limiter.go delete mode 100644 services/task/point_task.go create mode 100644 services/task/task.go create mode 100644 services/task/task_config.go diff --git a/models/error.go b/models/error.go index 46917e15e..19afa9d8b 100755 --- a/models/error.go +++ b/models/error.go @@ -2012,3 +2012,15 @@ func IsErrTagNotExist(err error) bool { _, ok := err.(ErrTagNotExist) return ok } + +type ErrRecordNotExist struct { +} + +func IsErrRecordNotExist(err error) bool { + _, ok := err.(ErrRecordNotExist) + return ok +} + +func (err ErrRecordNotExist) Error() string { + return fmt.Sprintf("record not exist in database") +} diff --git a/models/point_account_log.go b/models/point_account_log.go index ae718fe0f..f699495e7 100644 --- a/models/point_account_log.go +++ b/models/point_account_log.go @@ -7,7 +7,7 @@ type PointAccountLog struct { AccountId int64 `xorm:"INDEX NOT NULL"` UserId int64 `xorm:"INDEX NOT NULL"` Type string `xorm:"NOT NULL"` - RelatedId string `xorm:"INDEX NOT NULL"` + SourceId string `xorm:"INDEX NOT NULL"` PointsAmount int64 `xorm:"NOT NULL"` AmountBefore int64 `xorm:"NOT NULL"` AmountAfter int64 `xorm:"NOT NULL"` diff --git a/models/point_operate_record.go b/models/point_operate_record.go index d2dda7863..b0ffb094c 100644 --- a/models/point_operate_record.go +++ b/models/point_operate_record.go @@ -5,9 +5,15 @@ import "code.gitea.io/gitea/modules/timeutil" type RewardSourceType string const ( - SourceTypeAccomplishPointTask RewardSourceType = "ACCOMPLISH_POINT_TASK" - SourceTypeAdminOperate RewardSourceType = "ADMIN_OPERATE" - SourceTypeRunCloudbrainTask RewardSourceType = "RUN_CLOUBRAIN_TASK" + SourceTypeAccomplishTask RewardSourceType = "ACCOMPLISH_TASK" + SourceTypeAdminOperate RewardSourceType = "ADMIN_OPERATE" + SourceTypeRunCloudbrainTask RewardSourceType = "RUN_CLOUBRAIN_TASK" +) + +type RewardType string + +const ( + RewardTypePoint RewardType = "POINT" ) const ( @@ -26,7 +32,7 @@ type PointOperateRecord struct { UserId int64 `xorm:"INDEX NOT NULL"` PointsAmount int64 `xorm:"NOT NULL"` RelatedType string `xorm:"NOT NULL"` - RelatedId string `xorm:"INDEX NOT NULL"` + SourceId string `xorm:"INDEX NOT NULL"` OperateType string `xorm:"NOT NULL"` OperateRate string `xorm:"NOT NULL default once"` Status string `xorm:"NOT NULL"` diff --git a/models/point_task_accomplish_log.go b/models/point_task_accomplish_log.go deleted file mode 100644 index 82c45e163..000000000 --- a/models/point_task_accomplish_log.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import "code.gitea.io/gitea/modules/timeutil" - -type PointTaskAccomplishLog struct { - ID int64 `xorm:"pk autoincr"` - ConfigId int64 `xorm:"NOT NULL"` - TaskCode int64 `xorm:"NOT NULL"` - UserId int64 `xorm:"INDEX NOT NULL"` - RelatedId int64 `xorm:"INDEX NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` -} diff --git a/models/point_task_config.go b/models/point_task_config.go deleted file mode 100644 index 070e3d29e..000000000 --- a/models/point_task_config.go +++ /dev/null @@ -1,25 +0,0 @@ -package models - -import ( - "code.gitea.io/gitea/modules/timeutil" -) - -const ( - TaskConfigRefreshRateOnce = "ONCE" - TaskConfigRefreshRateDaily = "DAILY" -) - -//PointTaskConfig Only add and delete are allowed, edit is not allowed -//so if you want to edit config for some task code,please delete first and add new one -type PointTaskConfig struct { - ID int64 `xorm:"pk autoincr"` - TaskCode string `xorm:"NOT NULL"` - Tittle string `xorm:"NOT NULL"` - RefreshRate string `xorm:"NOT NULL"` - Times int `xorm:"NOT NULL"` - AwardPoints int `xorm:"NOT NULL"` - Status int `xorm:"NOT NULL"` - Creator int64 `xorm:"NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - DeletedAt timeutil.TimeStamp `xorm:"deleted"` -} diff --git a/models/task_accomplish_log.go b/models/task_accomplish_log.go new file mode 100644 index 000000000..51976c401 --- /dev/null +++ b/models/task_accomplish_log.go @@ -0,0 +1,51 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "time" +) + +type TaskAccomplishLog struct { + ID int64 `xorm:"pk autoincr"` + ConfigId int64 `xorm:"NOT NULL"` + TaskCode string `xorm:"NOT NULL"` + UserId int64 `xorm:"INDEX NOT NULL"` + SourceId string `xorm:"INDEX NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + +type LimiterPeriod struct { + StartTime time.Time + EndTime time.Time +} + +func getTaskAccomplishLog(tl *TaskAccomplishLog) (*TaskAccomplishLog, error) { + has, err := x.Get(tl) + if err != nil { + return nil, err + } else if !has { + return nil, ErrRecordNotExist{} + } + return tl, nil +} + +func GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskCode string) (*TaskAccomplishLog, error) { + t := &TaskAccomplishLog{ + SourceId: sourceId, + TaskCode: taskCode, + } + return getTaskAccomplishLog(t) +} + +func CountOnceTask(configId int64, userId int64, period *LimiterPeriod) (int64, error) { + if period == nil { + return x.Where("config_id = ? and user_id = ?", configId, userId).Count(&TaskAccomplishLog{}) + } else { + return x.Where("config_id = ? and user_id = ? and created_unix >= ? and created_unix < ? ", configId, userId, period.StartTime.Unix(), period.EndTime.Unix()).Count(&TaskAccomplishLog{}) + } + +} + +func InsertTaskAccomplishLog(tl *TaskAccomplishLog) (int64, error) { + return x.Insert(tl) +} diff --git a/models/task_config.go b/models/task_config.go new file mode 100644 index 000000000..f74237b59 --- /dev/null +++ b/models/task_config.go @@ -0,0 +1,53 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "fmt" +) + +type TaskType string + +const ( + TaskTypeComment TaskType = "COMMENT" +) + +func (t *TaskType) String() string { + return fmt.Sprint(t) +} + +const ( + TaskConfigRefreshRateNotCycle = "NOT_CYCLE" + TaskConfigRefreshRateDaily = "DAILY" +) + +//PointTaskConfig Only add and delete are allowed, edit is not allowed +//so if you want to edit config for some task code,please delete first and add new one +type TaskConfig struct { + ID int64 `xorm:"pk autoincr"` + TaskCode string `xorm:"NOT NULL"` + Tittle string `xorm:"NOT NULL"` + RefreshRate string `xorm:"NOT NULL"` + Times int64 `xorm:"NOT NULL"` + AwardType string `xorm:"NOT NULL"` + AwardAmount int64 `xorm:"NOT NULL"` + Creator int64 `xorm:"NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + DeletedAt timeutil.TimeStamp `xorm:"deleted"` +} + +func getTaskConfig(t *TaskConfig) (*TaskConfig, error) { + has, err := x.Get(t) + if err != nil { + return nil, err + } else if !has { + return nil, ErrRecordNotExist{} + } + return t, nil +} + +func GetTaskConfigByTaskCode(taskCode string) (*TaskConfig, error) { + t := &TaskConfig{ + TaskCode: taskCode, + } + return getTaskConfig(t) +} diff --git a/modules/auth/wechat/access_token.go b/modules/auth/wechat/access_token.go index f9516e3e1..af62c3e7b 100644 --- a/modules/auth/wechat/access_token.go +++ b/modules/auth/wechat/access_token.go @@ -7,14 +7,12 @@ import ( "time" ) -const EMPTY_REDIS_VAL = "Nil" - var accessTokenLock = redis_lock.NewDistributeLock(redis_key.AccessTokenLockKey()) func GetWechatAccessToken() string { token, _ := redis_client.Get(redis_key.WechatAccessTokenKey()) if token != "" { - if token == EMPTY_REDIS_VAL { + if token == redis_key.EMPTY_REDIS_VAL { return "" } live, _ := redis_client.TTL(redis_key.WechatAccessTokenKey()) @@ -39,7 +37,7 @@ func refreshAndGetAccessToken() string { defer accessTokenLock.UnLock() token, _ := redis_client.Get(redis_key.WechatAccessTokenKey()) if token != "" { - if token == EMPTY_REDIS_VAL { + if token == redis_key.EMPTY_REDIS_VAL { return "" } return token @@ -59,7 +57,7 @@ func callAccessTokenAndUpdateCache() string { } if token == "" { - redis_client.Setex(redis_key.WechatAccessTokenKey(), EMPTY_REDIS_VAL, 10*time.Second) + redis_client.Setex(redis_key.WechatAccessTokenKey(), redis_key.EMPTY_REDIS_VAL, 10*time.Second) return "" } redis_client.Setex(redis_key.WechatAccessTokenKey(), token, time.Duration(r.Expires_in)*time.Second) diff --git a/modules/redis/redis_key/key_base.go b/modules/redis/redis_key/key_base.go index 0efc6ed38..797720c62 100644 --- a/modules/redis/redis_key/key_base.go +++ b/modules/redis/redis_key/key_base.go @@ -4,6 +4,8 @@ import "strings" const KEY_SEPARATE = ":" +const EMPTY_REDIS_VAL = "Nil" + func KeyJoin(keys ...string) string { var build strings.Builder for _, v := range keys { diff --git a/modules/redis/redis_key/task_redis_key.go b/modules/redis/redis_key/task_redis_key.go new file mode 100644 index 000000000..b33e575fb --- /dev/null +++ b/modules/redis/redis_key/task_redis_key.go @@ -0,0 +1,16 @@ +package redis_key + +import ( + "code.gitea.io/gitea/models" + "fmt" +) + +const TASK_REDIS_PREFIX = "task" + +func TaskAccomplishLock(userId int64, sourceId string, taskType models.TaskType) string { + return KeyJoin(TASK_REDIS_PREFIX, fmt.Sprint(userId), sourceId, taskType.String(), "accomplish") +} + +func TaskConfig(taskType models.TaskType) string { + return KeyJoin(TASK_REDIS_PREFIX, "config", taskType.String()) +} diff --git a/services/reward/operate/callback.go b/services/reward/callback.go similarity index 67% rename from services/reward/operate/callback.go rename to services/reward/callback.go index 27c42f443..b67ffa673 100644 --- a/services/reward/operate/callback.go +++ b/services/reward/callback.go @@ -1,4 +1,4 @@ -package operate +package reward type CallbackHandler struct { } diff --git a/services/reward/operate/operator.go b/services/reward/operator.go similarity index 58% rename from services/reward/operate/operator.go rename to services/reward/operator.go index 63d12b970..848ba703d 100644 --- a/services/reward/operate/operator.go +++ b/services/reward/operator.go @@ -1,17 +1,22 @@ -package operate +package reward import ( "code.gitea.io/gitea/models" - "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/services/reward/point" + "errors" + "fmt" ) +var RewardOperatorMap = map[string]RewardOperator{ + fmt.Sprint(models.RewardTypePoint): new(point.PointOperator), +} + type RewardOperateContext struct { SourceType models.RewardSourceType - RelatedId string + SourceId string Remark string - Reward reward.Reward + Reward Reward TargetUserId int64 - RequestId string } type RewardOperateResponse int @@ -31,7 +36,11 @@ type RewardOperator interface { Operate(ctx RewardOperateContext) error } -func Operate(operator RewardOperator, ctx RewardOperateContext) error { +func Send(ctx RewardOperateContext) error { + operator := GetOperator(ctx.Reward.Type) + if operator == nil { + return errors.New("operator of reward type is not exist") + } if operator.IsOperated(ctx) { return nil } @@ -43,3 +52,7 @@ func Operate(operator RewardOperator, ctx RewardOperateContext) error { } return nil } + +func GetOperator(rewardType string) RewardOperator { + return RewardOperatorMap[rewardType] +} diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index f91ca11b3..5a6c18bff 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -1,16 +1,19 @@ package point -import "code.gitea.io/gitea/services/reward/operate" +import ( + "code.gitea.io/gitea/services/reward" +) type PointOperator struct { } -func (operator *PointOperator) IsOperated(ctx operate.RewardOperateContext) bool { +func (operator *PointOperator) IsOperated(ctx reward.RewardOperateContext) bool { + //todo return true } -func (operator *PointOperator) IsLimited(ctx operate.RewardOperateContext) bool { +func (operator *PointOperator) IsLimited(ctx reward.RewardOperateContext) bool { return true } -func (operator *PointOperator) Operate(ctx operate.RewardOperateContext) error { +func (operator *PointOperator) Operate(ctx reward.RewardOperateContext) error { return nil } diff --git a/services/reward/reward.go b/services/reward/reward.go index 8ec0e0471..ca1c1f3cd 100644 --- a/services/reward/reward.go +++ b/services/reward/reward.go @@ -1,6 +1,6 @@ package reward type Reward struct { - Amount int + Amount int64 Type string } diff --git a/services/task/limiter.go b/services/task/limiter.go new file mode 100644 index 000000000..6c2cd4f44 --- /dev/null +++ b/services/task/limiter.go @@ -0,0 +1,39 @@ +package task + +import ( + "code.gitea.io/gitea/models" + "time" +) + +var LimiterMap = map[string]Limiter{ + models.TaskConfigRefreshRateNotCycle: new(NoCycleLimiter), + models.TaskConfigRefreshRateDaily: new(DailyLimiter), +} + +type Limiter interface { + GetCurrentPeriod() *models.LimiterPeriod +} + +type NoCycleLimiter struct { +} + +func (l *NoCycleLimiter) GetCurrentPeriod() *models.LimiterPeriod { + return nil +} + +type DailyLimiter struct { +} + +func (l *DailyLimiter) GetCurrentPeriod() *models.LimiterPeriod { + t := time.Now() + startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) + endTime := startTime.Add(24 * time.Hour) + return &models.LimiterPeriod{ + StartTime: startTime, + EndTime: endTime, + } +} + +func GetLimiter(refreshRateype string) Limiter { + return LimiterMap[refreshRateype] +} diff --git a/services/task/point_task.go b/services/task/point_task.go deleted file mode 100644 index b72fbffdc..000000000 --- a/services/task/point_task.go +++ /dev/null @@ -1,10 +0,0 @@ -package task - -func Accomplish() error { - //1、幂等性判断 - //2、获取任务配置 - //3、判断任务是否可以完成 - //4、生成任务记录 - //5、触发奖励发放 - return nil -} diff --git a/services/task/task.go b/services/task/task.go new file mode 100644 index 000000000..3b702f179 --- /dev/null +++ b/services/task/task.go @@ -0,0 +1,104 @@ +package task + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/services/reward" + "errors" + "time" +) + +func Accomplish(userId int64, taskType models.TaskType, sourceId string) { + go accomplish(userId, taskType, sourceId) +} + +func accomplish(userId int64, taskType models.TaskType, sourceId string) error { + //lock + var taskLock = redis_lock.NewDistributeLock(redis_key.TaskAccomplishLock(userId, sourceId, taskType)) + if !taskLock.Lock(3 * time.Second) { + log.Info("duplicated task request,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) + return nil + } + defer taskLock.UnLock() + + //is handled before? + isHandled, err := isHandled(taskType, sourceId) + if err != nil { + log.Error("Get isHandled error,%v", err) + return err + } + if isHandled { + log.Info("task has been handled,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) + return nil + } + + //get task config + config, err := GetTaskConfig(taskType) + if err != nil { + log.Error("GetTaskConfig error,%v", err) + return err + } + if config == nil { + log.Info("task config not exist,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) + return nil + } + + //is limited? + isLimited, err := IsLimited(userId, config) + if err != nil { + log.Error("get limited error,%v", err) + return err + } + if isLimited { + log.Info("task accomplish maximum times are reached,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) + return nil + } + + //add log + models.InsertTaskAccomplishLog(&models.TaskAccomplishLog{ + ConfigId: config.ID, + TaskCode: config.TaskCode, + UserId: userId, + SourceId: sourceId, + }) + + //reward + reward.Send(reward.RewardOperateContext{ + SourceType: models.SourceTypeAccomplishTask, + SourceId: sourceId, + Reward: reward.Reward{ + Amount: config.AwardAmount, + Type: config.AwardType, + }, + TargetUserId: userId, + }) + + return nil +} + +func isHandled(taskType models.TaskType, sourceId string) (bool, error) { + _, err := models.GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskType.String()) + if err != nil { + if models.IsErrRecordNotExist(err) { + return false, nil + } + return false, err + } + return true, nil + +} + +func IsLimited(userId int64, config *models.TaskConfig) (bool, error) { + limiter := GetLimiter(config.RefreshRate) + if limiter == nil { + return false, errors.New("task config incorrect") + } + n, err := models.CountOnceTask(config.ID, userId, limiter.GetCurrentPeriod()) + if err != nil { + return false, err + } + return n >= config.Times, nil + +} diff --git a/services/task/task_config.go b/services/task/task_config.go new file mode 100644 index 000000000..ccdf4c08a --- /dev/null +++ b/services/task/task_config.go @@ -0,0 +1,28 @@ +package task + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" + "encoding/json" +) + +func GetTaskConfig(taskType models.TaskType) (*models.TaskConfig, error) { + configStr, _ := redis_client.Get(redis_key.TaskConfig(taskType)) + if configStr != "" { + if configStr == redis_key.EMPTY_REDIS_VAL { + return nil, nil + } + config := new(models.TaskConfig) + json.Unmarshal([]byte(configStr), config) + return config, nil + } + config, err := models.GetTaskConfigByTaskCode(taskType.String()) + if err != nil { + if models.IsErrRecordNotExist(err) { + return nil, nil + } + return nil, err + } + return config, nil +} From 1a2327fc2d86220414a2cecdc6c1fe2a8392e6f9 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 31 May 2022 17:56:31 +0800 Subject: [PATCH 03/90] update --- models/task_accomplish_log.go | 2 +- services/task/task.go | 2 +- services/task/task_config.go | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/models/task_accomplish_log.go b/models/task_accomplish_log.go index 51976c401..ed2927678 100644 --- a/models/task_accomplish_log.go +++ b/models/task_accomplish_log.go @@ -37,7 +37,7 @@ func GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskCode string) (*Task return getTaskAccomplishLog(t) } -func CountOnceTask(configId int64, userId int64, period *LimiterPeriod) (int64, error) { +func CountInTaskPeriod(configId int64, userId int64, period *LimiterPeriod) (int64, error) { if period == nil { return x.Where("config_id = ? and user_id = ?", configId, userId).Count(&TaskAccomplishLog{}) } else { diff --git a/services/task/task.go b/services/task/task.go index 3b702f179..c5f65240b 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -95,7 +95,7 @@ func IsLimited(userId int64, config *models.TaskConfig) (bool, error) { if limiter == nil { return false, errors.New("task config incorrect") } - n, err := models.CountOnceTask(config.ID, userId, limiter.GetCurrentPeriod()) + n, err := models.CountInTaskPeriod(config.ID, userId, limiter.GetCurrentPeriod()) if err != nil { return false, err } diff --git a/services/task/task_config.go b/services/task/task_config.go index ccdf4c08a..6812f5d67 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -5,10 +5,14 @@ import ( "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "encoding/json" + "time" ) +//GetTaskConfig get task config from redis cache first +// if not exist in redis, find in db and refresh the redis key func GetTaskConfig(taskType models.TaskType) (*models.TaskConfig, error) { - configStr, _ := redis_client.Get(redis_key.TaskConfig(taskType)) + redisKey := redis_key.TaskConfig(taskType) + configStr, _ := redis_client.Get(redisKey) if configStr != "" { if configStr == redis_key.EMPTY_REDIS_VAL { return nil, nil @@ -22,7 +26,10 @@ func GetTaskConfig(taskType models.TaskType) (*models.TaskConfig, error) { if models.IsErrRecordNotExist(err) { return nil, nil } + redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) return nil, err } + jsonStr, _ := json.Marshal(config) + redis_client.Setex(redisKey, string(jsonStr), 30*24*time.Hour) return config, nil } From 7f64382856e26cd7db4323297f0c836ce94f6b4a Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 1 Jun 2022 18:09:27 +0800 Subject: [PATCH 04/90] add reward --- models/limit_config.go | 33 ++++++ models/point_account.go | 77 ++++++++++++- models/point_account_log.go | 11 +- models/point_limit_config.go | 23 ---- models/point_operate_record.go | 42 ------- models/reward_operate_record.go | 84 ++++++++++++++ models/task_accomplish_log.go | 6 +- models/task_config.go | 4 +- modules/redis/redis_client/client.go | 45 ++++++++ modules/redis/redis_key/account_redis_key.go | 17 +++ modules/redis/redis_key/limit_redis_key.go | 26 +++++ modules/redis/redis_key/reward_redis_key.go | 7 ++ modules/redis/redis_key/task_redis_key.go | 5 +- modules/util/uuid_util.go | 10 ++ services/reward/account/account.go | 9 -- services/reward/callback.go | 4 - services/reward/limiter/limiter.go | 93 +++++++++++++++ services/reward/operator.go | 106 ++++++++++++++---- .../reward/point/account/point_account.go | 58 ++++++++++ services/reward/point/point_operate.go | 33 +++++- services/task/limiter.go | 39 ------- services/task/period/handler.go | 50 +++++++++ services/task/task.go | 21 ++-- services/task/task_config.go | 2 +- 24 files changed, 640 insertions(+), 165 deletions(-) create mode 100644 models/limit_config.go delete mode 100644 models/point_limit_config.go delete mode 100644 models/point_operate_record.go create mode 100644 models/reward_operate_record.go create mode 100644 modules/redis/redis_key/account_redis_key.go create mode 100644 modules/redis/redis_key/limit_redis_key.go create mode 100644 modules/redis/redis_key/reward_redis_key.go create mode 100644 modules/util/uuid_util.go delete mode 100644 services/reward/account/account.go delete mode 100644 services/reward/callback.go create mode 100644 services/reward/limiter/limiter.go create mode 100644 services/reward/point/account/point_account.go delete mode 100644 services/task/limiter.go create mode 100644 services/task/period/handler.go diff --git a/models/limit_config.go b/models/limit_config.go new file mode 100644 index 000000000..273af0de1 --- /dev/null +++ b/models/limit_config.go @@ -0,0 +1,33 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type LimitConfig struct { + ID int64 `xorm:"pk autoincr"` + Tittle string + RefreshRate string `xorm:"NOT NULL"` + Scope string `xorm:"NOT NULL"` + LimitNum int64 `xorm:"NOT NULL"` + LimitCode string `xorm:"NOT NULL"` + Creator int64 `xorm:"NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + DeletedAt timeutil.TimeStamp `xorm:"deleted"` +} + +func findLimitConfig(tl *LimitConfig) ([]LimitConfig, error) { + r := make([]LimitConfig, 0) + err := x.Find(r, tl) + if err != nil { + return nil, err + } else if len(r) == 0 { + return nil, ErrRecordNotExist{} + } + return r, nil +} + +func GetLimitConfigByLimitCode(limitCode string) ([]LimitConfig, error) { + t := &LimitConfig{ + LimitCode: limitCode, + } + return findLimitConfig(t) +} diff --git a/models/point_account.go b/models/point_account.go index f889d5d4f..7fa38cb7a 100644 --- a/models/point_account.go +++ b/models/point_account.go @@ -6,27 +6,92 @@ type PointAccountStatus int // Possible PointAccountStatus types. const ( - PointAccountNormal PointAccountStatus = iota + 1 // 1 - PointAccountFreeze // 2 - PointAccountDeleted // 3 + PointAccountNormal int = iota + 1 // 1 + PointAccountFreeze // 2 + PointAccountDeleted // 3 ) type PointAccount struct { ID int64 `xorm:"pk autoincr"` + AccountCode string `xorm:"INDEX NOT NULL"` Balance int64 `xorm:"NOT NULL DEFAULT 0"` TotalEarned int64 `xorm:"NOT NULL DEFAULT 0"` TotalConsumed int64 `xorm:"NOT NULL DEFAULT 0"` UserId int64 `xorm:"INDEX NOT NULL"` - Status string `xorm:"NOT NULL"` + Status int `xorm:"NOT NULL"` Version int64 `xorm:"NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } -func (account *PointAccount) Increase(amount int64) error { +func (account *PointAccount) Increase(amount int64, sourceId string) error { + sess := x.NewSession() + defer sess.Close() + sql := "update point_account set balance = balance + ?,total_earned = total_earned + ? ,version = version + 1 where account_code = ? " + _, err := sess.Exec(sql, amount, amount, account.AccountCode) + if err != nil { + sess.Rollback() + return err + } + accountLog := &PointAccountLog{ + AccountCode: account.AccountCode, + UserId: account.UserId, + Type: IncreaseAccountBalance, + SourceId: sourceId, + PointsAmount: amount, + BalanceBefore: account.Balance, + BalanceAfter: account.Balance + amount, + AccountVersion: account.Version, + } + _, err = sess.Insert(accountLog) + if err != nil { + sess.Rollback() + return err + } + sess.Commit() return nil } -func (account *PointAccount) Decrease(amount int64) error { +func (account *PointAccount) Decrease(amount int64, sourceId string) error { + sess := x.NewSession() + defer sess.Close() + sql := "update point_account set balance = balance - ?,total_consumed = total_consumed + ? ,version = version + 1 where account_code = ? " + _, err := sess.Exec(sql, amount, amount, account.AccountCode) + if err != nil { + sess.Rollback() + return err + } + accountLog := &PointAccountLog{ + AccountCode: account.AccountCode, + UserId: account.UserId, + Type: DecreaseAccountBalance, + SourceId: sourceId, + PointsAmount: amount, + BalanceBefore: account.Balance, + BalanceAfter: account.Balance - amount, + AccountVersion: account.Version, + } + _, err = sess.Insert(accountLog) + if err != nil { + sess.Rollback() + return err + } + sess.Commit() return nil } + +func GetAccountByUserId(userId int64) (*PointAccount, error) { + p := &PointAccount{} + has, err := x.Where("user_id = ?", userId).Get(p) + if err != nil { + return nil, err + } + if !has { + return nil, nil + } + return p, nil +} + +func InsertAccount(tl *PointAccount) (int64, error) { + return x.Insert(tl) +} diff --git a/models/point_account_log.go b/models/point_account_log.go index f699495e7..3ed39ed77 100644 --- a/models/point_account_log.go +++ b/models/point_account_log.go @@ -2,15 +2,20 @@ package models import "code.gitea.io/gitea/modules/timeutil" +const ( + IncreaseAccountBalance = "increase" + DecreaseAccountBalance = "decrease" +) + type PointAccountLog struct { ID int64 `xorm:"pk autoincr"` - AccountId int64 `xorm:"INDEX NOT NULL"` + AccountCode string `xorm:"INDEX NOT NULL"` UserId int64 `xorm:"INDEX NOT NULL"` Type string `xorm:"NOT NULL"` SourceId string `xorm:"INDEX NOT NULL"` PointsAmount int64 `xorm:"NOT NULL"` - AmountBefore int64 `xorm:"NOT NULL"` - AmountAfter int64 `xorm:"NOT NULL"` + BalanceBefore int64 `xorm:"NOT NULL"` + BalanceAfter int64 `xorm:"NOT NULL"` AccountVersion int64 `xorm:"NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } diff --git a/models/point_limit_config.go b/models/point_limit_config.go deleted file mode 100644 index 60fb5e735..000000000 --- a/models/point_limit_config.go +++ /dev/null @@ -1,23 +0,0 @@ -package models - -import "code.gitea.io/gitea/modules/timeutil" - -const ( - LimitConfigRefreshRateOnce = "ONCE" - LimitConfigRefreshRateDaily = "DAILY" -) -const ( - LimitTargetRangeAllUser = "ALL_USER" - LimitTargetRangeSingleUser = "SINGLE_USER" -) - -type PointLimitConfig struct { - ID int64 `xorm:"pk autoincr"` - Tittle string - RefreshRate string `xorm:"NOT NULL"` - TargetRange string `xorm:"NOT NULL"` - LimitNum int64 `xorm:"NOT NULL"` - Creator int64 `xorm:"NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - DeletedAt timeutil.TimeStamp `xorm:"deleted"` -} diff --git a/models/point_operate_record.go b/models/point_operate_record.go deleted file mode 100644 index b0ffb094c..000000000 --- a/models/point_operate_record.go +++ /dev/null @@ -1,42 +0,0 @@ -package models - -import "code.gitea.io/gitea/modules/timeutil" - -type RewardSourceType string - -const ( - SourceTypeAccomplishTask RewardSourceType = "ACCOMPLISH_TASK" - SourceTypeAdminOperate RewardSourceType = "ADMIN_OPERATE" - SourceTypeRunCloudbrainTask RewardSourceType = "RUN_CLOUBRAIN_TASK" -) - -type RewardType string - -const ( - RewardTypePoint RewardType = "POINT" -) - -const ( - OperateTypeIncrease = "INCREASE_POINT" - OperateTypeDecrease = "DECREASE_POINT" -) - -const ( - OperateStatusOperating = "OPERATING" - OperateStatusSucceeded = "SUCCEEDED" - OperateStatusFailed = "FAILED" -) - -type PointOperateRecord struct { - ID int64 `xorm:"pk autoincr"` - UserId int64 `xorm:"INDEX NOT NULL"` - PointsAmount int64 `xorm:"NOT NULL"` - RelatedType string `xorm:"NOT NULL"` - SourceId string `xorm:"INDEX NOT NULL"` - OperateType string `xorm:"NOT NULL"` - OperateRate string `xorm:"NOT NULL default once"` - Status string `xorm:"NOT NULL"` - Remark string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` -} diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go new file mode 100644 index 000000000..ca1f52168 --- /dev/null +++ b/models/reward_operate_record.go @@ -0,0 +1,84 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "fmt" +) + +type RewardSourceType string + +const ( + SourceTypeAccomplishTask RewardSourceType = "ACCOMPLISH_TASK" + SourceTypeAdminOperate RewardSourceType = "ADMIN_OPERATE" + SourceTypeRunCloudbrainTask RewardSourceType = "RUN_CLOUBRAIN_TASK" +) + +func (r *RewardSourceType) String() string { + return fmt.Sprint(r) +} + +type RewardType string + +const ( + RewardTypePoint RewardType = "POINT" +) + +func (r *RewardType) String() string { + return fmt.Sprint(r) +} + +const ( + OperateTypeIncrease = "INCREASE_POINT" + OperateTypeDecrease = "DECREASE_POINT" +) + +const ( + OperateStatusOperating = "OPERATING" + OperateStatusSucceeded = "SUCCEEDED" + OperateStatusFailed = "FAILED" +) + +type RewardOperateRecord struct { + ID int64 `xorm:"pk autoincr"` + UserId int64 `xorm:"INDEX NOT NULL"` + Amount int64 `xorm:"NOT NULL"` + RewardType string `xorm:"NOT NULL"` + SourceType string `xorm:"NOT NULL"` + SourceId string `xorm:"INDEX NOT NULL"` + RequestId string `xorm:"INDEX NOT NULL"` + OperateType string `xorm:"NOT NULL"` + CycleIntervalSeconds int64 `xorm:"NOT NULL default 0"` + Status string `xorm:"NOT NULL"` + Remark string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) { + has, err := x.Get(tl) + if err != nil { + return nil, err + } else if !has { + return nil, ErrRecordNotExist{} + } + return tl, nil +} + +func GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId string) (*RewardOperateRecord, error) { + t := &RewardOperateRecord{ + SourceType: sourceType, + RequestId: requestId, + } + return getPointOperateRecord(t) +} + +func InsertAwardOperateRecord(tl *RewardOperateRecord) (int64, error) { + return x.Insert(tl) +} + +func UpdateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus string) (int64, error) { + r := &RewardOperateRecord{ + Status: newStatus, + } + return x.Cols("status").Where("source_type=? and requestId=? and status=?", sourceType, requestId, oldStatus).Update(r) +} diff --git a/models/task_accomplish_log.go b/models/task_accomplish_log.go index ed2927678..707c214f5 100644 --- a/models/task_accomplish_log.go +++ b/models/task_accomplish_log.go @@ -7,6 +7,7 @@ import ( type TaskAccomplishLog struct { ID int64 `xorm:"pk autoincr"` + LogId string `xorm:"INDEX NOT NULL"` ConfigId int64 `xorm:"NOT NULL"` TaskCode string `xorm:"NOT NULL"` UserId int64 `xorm:"INDEX NOT NULL"` @@ -14,9 +15,10 @@ type TaskAccomplishLog struct { CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } -type LimiterPeriod struct { +type PeriodResult struct { StartTime time.Time EndTime time.Time + LeftTime time.Duration } func getTaskAccomplishLog(tl *TaskAccomplishLog) (*TaskAccomplishLog, error) { @@ -37,7 +39,7 @@ func GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskCode string) (*Task return getTaskAccomplishLog(t) } -func CountInTaskPeriod(configId int64, userId int64, period *LimiterPeriod) (int64, error) { +func CountInTaskPeriod(configId int64, userId int64, period *PeriodResult) (int64, error) { if period == nil { return x.Where("config_id = ? and user_id = ?", configId, userId).Count(&TaskAccomplishLog{}) } else { diff --git a/models/task_config.go b/models/task_config.go index f74237b59..036f4e315 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -16,8 +16,8 @@ func (t *TaskType) String() string { } const ( - TaskConfigRefreshRateNotCycle = "NOT_CYCLE" - TaskConfigRefreshRateDaily = "DAILY" + PeriodNotCycle = "NOT_CYCLE" + PeriodDaily = "DAILY" ) //PointTaskConfig Only add and delete are allowed, edit is not allowed diff --git a/modules/redis/redis_client/client.go b/modules/redis/redis_client/client.go index 437aecdae..2c487a72c 100644 --- a/modules/redis/redis_client/client.go +++ b/modules/redis/redis_client/client.go @@ -85,3 +85,48 @@ func TTL(key string) (int, error) { return n, nil } + +func IncrBy(key string, n int64) (int64, error) { + redisClient := labelmsg.Get() + defer redisClient.Close() + + reply, err := redisClient.Do("INCRBY", key, n) + if err != nil { + return 0, err + } + i, err := strconv.ParseInt(fmt.Sprint(reply), 10, 64) + return i, nil + +} + +func Expire(key string, expireSeconds int64) error { + redisClient := labelmsg.Get() + defer redisClient.Close() + + _, err := redisClient.Do("EXPIRE ", key, expireSeconds) + if err != nil { + return err + } + return nil + +} + +//GetInt64 get redis value by Get(key) +//and then parse the value to int64 +//return {isExist(bool)} {value(int64)} {error(error)} +func GetInt64(key string) (bool, int64, error) { + str, err := Get(key) + if err != nil { + return false, 0, err + } + if str == "" { + return false, 0, nil + } + + i, err := strconv.ParseInt(str, 10, 64) + if err != nil { + return false, 0, err + } + return true, i, nil + +} diff --git a/modules/redis/redis_key/account_redis_key.go b/modules/redis/redis_key/account_redis_key.go new file mode 100644 index 000000000..f36a8ea5c --- /dev/null +++ b/modules/redis/redis_key/account_redis_key.go @@ -0,0 +1,17 @@ +package redis_key + +import "fmt" + +const ACCOUNT_REDIS_PREFIX = "account" + +func PointAccountOperateLock(accountCode string) string { + return KeyJoin(ACCOUNT_REDIS_PREFIX, accountCode, "operate", "lock") +} + +func PointAccountDetail(userId int64) string { + return KeyJoin(ACCOUNT_REDIS_PREFIX, fmt.Sprint(userId), "detail") +} + +func PointAccountInitLock(userId int64) string { + return KeyJoin(ACCOUNT_REDIS_PREFIX, fmt.Sprint(userId), "init", "lock") +} diff --git a/modules/redis/redis_key/limit_redis_key.go b/modules/redis/redis_key/limit_redis_key.go new file mode 100644 index 000000000..e9d8352a2 --- /dev/null +++ b/modules/redis/redis_key/limit_redis_key.go @@ -0,0 +1,26 @@ +package redis_key + +import ( + "code.gitea.io/gitea/models" + "fmt" +) + +const LIMIT_REDIS_PREFIX = "limit" + +func LimitCount(userId int64, limitCode string, period *models.PeriodResult) string { + if userId == 0 { + if period == nil { + return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, "count") + } + return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, fmt.Sprint(period.StartTime.Unix()), fmt.Sprint(period.EndTime.Unix()), "count") + } + if period == nil { + return KeyJoin(LIMIT_REDIS_PREFIX, "uid", fmt.Sprint(userId), limitCode, "count") + } + return KeyJoin(LIMIT_REDIS_PREFIX, "uid", fmt.Sprint(userId), limitCode, fmt.Sprint(period.StartTime.Unix()), fmt.Sprint(period.EndTime.Unix()), "count") + +} + +func LimitConfig(limitCode string) string { + return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, "config") +} diff --git a/modules/redis/redis_key/reward_redis_key.go b/modules/redis/redis_key/reward_redis_key.go new file mode 100644 index 000000000..df8c0ca16 --- /dev/null +++ b/modules/redis/redis_key/reward_redis_key.go @@ -0,0 +1,7 @@ +package redis_key + +const REWARD_REDIS_PREFIX = "reward" + +func RewardSendLock(requestId string, sourceType string) string { + return KeyJoin(REWARD_REDIS_PREFIX, requestId, sourceType, "send") +} diff --git a/modules/redis/redis_key/task_redis_key.go b/modules/redis/redis_key/task_redis_key.go index b33e575fb..2eb8c21d1 100644 --- a/modules/redis/redis_key/task_redis_key.go +++ b/modules/redis/redis_key/task_redis_key.go @@ -2,13 +2,12 @@ package redis_key import ( "code.gitea.io/gitea/models" - "fmt" ) const TASK_REDIS_PREFIX = "task" -func TaskAccomplishLock(userId int64, sourceId string, taskType models.TaskType) string { - return KeyJoin(TASK_REDIS_PREFIX, fmt.Sprint(userId), sourceId, taskType.String(), "accomplish") +func TaskAccomplishLock(sourceId string, taskType models.TaskType) string { + return KeyJoin(TASK_REDIS_PREFIX, sourceId, taskType.String(), "accomplish") } func TaskConfig(taskType models.TaskType) string { diff --git a/modules/util/uuid_util.go b/modules/util/uuid_util.go new file mode 100644 index 000000000..301c6ff38 --- /dev/null +++ b/modules/util/uuid_util.go @@ -0,0 +1,10 @@ +package util + +import ( + gouuid "github.com/satori/go.uuid" + "strings" +) + +func UUID() string { + return strings.ReplaceAll(gouuid.NewV4().String(), "-", "") +} diff --git a/services/reward/account/account.go b/services/reward/account/account.go deleted file mode 100644 index 4967b368e..000000000 --- a/services/reward/account/account.go +++ /dev/null @@ -1,9 +0,0 @@ -package account - -func IncreaseAmount(userId int64, amount int64) error { - return nil -} - -func DecreaseAmount(userId int64, amount int64) error { - return nil -} diff --git a/services/reward/callback.go b/services/reward/callback.go deleted file mode 100644 index b67ffa673..000000000 --- a/services/reward/callback.go +++ /dev/null @@ -1,4 +0,0 @@ -package reward - -type CallbackHandler struct { -} diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go new file mode 100644 index 000000000..aca8af22e --- /dev/null +++ b/services/reward/limiter/limiter.go @@ -0,0 +1,93 @@ +package limiter + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/services/task/period" + "encoding/json" + "errors" + "fmt" + "time" +) + +type limiterRunner struct { + limiters []models.LimitConfig + index int + userId int64 + amount int64 + limitCode string +} + +func newLimiterRunner(limitCode string, userId, amount int64) *limiterRunner { + return &limiterRunner{ + userId: userId, + amount: amount, + limitCode: limitCode, + index: 0, + } +} + +func (l *limiterRunner) Run() error { + if err := l.LoadLimiters(l.limitCode); err != nil { + return err + } + //todo 验证未配置的情况 + for l.index <= len(l.limiters) { + err := l.limit(l.limiters[l.index]) + if err != nil { + return err + } + l.index += 1 + } + return nil +} + +func (l *limiterRunner) limit(r models.LimitConfig) error { + p, err := period.GetPeriod(r.RefreshRate) + if err != nil { + return err + } + redisKey := redis_key.LimitCount(l.userId, r.LimitCode, p) + usedNum, err := redis_client.IncrBy(redisKey, l.amount) + //if it is the first time,set expire time + if usedNum == l.amount && p != nil { + //todo 验证浮点精确度 + redis_client.Expire(redisKey, int64(p.LeftTime.Seconds())) + } + if usedNum > r.LimitNum { + redis_client.IncrBy(redisKey, -1*l.amount) + return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) + } + return nil +} + +func (l *limiterRunner) LoadLimiters(limitCode string) error { + redisKey := redis_key.LimitConfig(limitCode) + val, _ := redis_client.Get(redisKey) + if val != "" { + if val == redis_key.EMPTY_REDIS_VAL { + return nil + } + limiters := make([]models.LimitConfig, 0) + json.Unmarshal([]byte(val), limiters) + return nil + } + limiters, err := models.GetLimitConfigByLimitCode(limitCode) + if err != nil { + if models.IsErrRecordNotExist(err) { + redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) + return nil + } + return err + } + jsonStr, _ := json.Marshal(limiters) + redis_client.Setex(redisKey, string(jsonStr), 30*24*time.Hour) + + return nil +} + +func CheckLimit(limitCode string, userId, amount int64) error { + r := newLimiterRunner(limitCode, userId, amount) + return r.Run() +} diff --git a/services/reward/operator.go b/services/reward/operator.go index 848ba703d..321562474 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -2,9 +2,13 @@ package reward import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/services/reward/point" "errors" "fmt" + "time" ) var RewardOperatorMap = map[string]RewardOperator{ @@ -12,47 +16,109 @@ var RewardOperatorMap = map[string]RewardOperator{ } type RewardOperateContext struct { - SourceType models.RewardSourceType - SourceId string - Remark string - Reward Reward - TargetUserId int64 -} - -type RewardOperateResponse int - -const ( - RewardOperateSuccess RewardOperateResponse = iota + 1 - RewardOperateBalanceNotEnough -) - -func (t RewardOperateResponse) IsSuccess() bool { - return t == RewardOperateSuccess + SourceType models.RewardSourceType + SourceId string + Remark string + Reward Reward + TargetUserId int64 + RequestId string + OperateType string + CycleIntervalSeconds int64 } type RewardOperator interface { - IsOperated(ctx RewardOperateContext) bool IsLimited(ctx RewardOperateContext) bool Operate(ctx RewardOperateContext) error } func Send(ctx RewardOperateContext) error { + //add lock + var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardSendLock(ctx.RequestId, ctx.SourceType.String())) + if !rewardLock.Lock(3 * time.Second) { + log.Info("duplicated reward request,targetUserId=%d requestId=%s", ctx.TargetUserId, ctx.RequestId) + return nil + } + defer rewardLock.UnLock() + + //is handled before? + isHandled, err := isHandled(ctx.SourceType, ctx.RequestId) + if err != nil { + log.Error("reward is handled error,%v", err) + return err + } + if isHandled { + log.Info("reward has been handled,ctx=%+v", ctx) + return nil + } + + //get operator operator := GetOperator(ctx.Reward.Type) if operator == nil { return errors.New("operator of reward type is not exist") } - if operator.IsOperated(ctx) { - return nil - } + + //is limited? if operator.IsLimited(ctx) { return nil } + + //new reward operate record + if err := initAwardOperateRecord(ctx); err != nil { + return err + } + + //operate if err := operator.Operate(ctx); err != nil { + updateAwardOperateRecordStatus(ctx.SourceType.String(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) return err } + + //if not a cycle operate,update status to success + if ctx.CycleIntervalSeconds > 0 { + updateAwardOperateRecordStatus(ctx.SourceType.String(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusSucceeded) + } return nil } func GetOperator(rewardType string) RewardOperator { return RewardOperatorMap[rewardType] } + +func isHandled(sourceType models.RewardSourceType, requestId string) (bool, error) { + _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType.String(), requestId) + if err != nil { + if models.IsErrRecordNotExist(err) { + return false, nil + } + return false, err + } + return true, nil + +} + +func initAwardOperateRecord(ctx RewardOperateContext) error { + _, err := models.InsertAwardOperateRecord(&models.RewardOperateRecord{ + UserId: ctx.TargetUserId, + Amount: ctx.Reward.Amount, + RewardType: ctx.Reward.Type, + SourceType: ctx.SourceType.String(), + SourceId: ctx.SourceId, + RequestId: ctx.RequestId, + OperateType: ctx.OperateType, + CycleIntervalSeconds: ctx.CycleIntervalSeconds, + Status: models.OperateStatusOperating, + Remark: ctx.Remark, + }) + if err != nil { + return err + } + return nil +} + +func updateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus string) error { + _, err := models.UpdateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus) + if err != nil { + return err + } + return nil +} diff --git a/services/reward/point/account/point_account.go b/services/reward/point/account/point_account.go new file mode 100644 index 000000000..9ff5001fc --- /dev/null +++ b/services/reward/point/account/point_account.go @@ -0,0 +1,58 @@ +package account + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/modules/util" + "encoding/json" + "time" +) + +func GetAccount(userId int64) (*models.PointAccount, error) { + redisKey := redis_key.PointAccountDetail(userId) + val, _ := redis_client.Get(redisKey) + if val != "" { + account := &models.PointAccount{} + json.Unmarshal([]byte(val), account) + return account, nil + } + account, err := models.GetAccountByUserId(userId) + if err != nil { + if models.IsErrRecordNotExist(err) { + a, err := InitAccount(userId) + if err != nil { + return nil, err + } + return a, nil + } + return nil, err + } + jsonStr, _ := json.Marshal(account) + redis_client.Setex(redisKey, string(jsonStr), 24*time.Hour) + return account, nil +} + +func InitAccount(userId int64) (*models.PointAccount, error) { + lock := redis_lock.NewDistributeLock(redis_key.PointAccountInitLock(userId)) + if lock.LockWithWait(3*time.Second, 3*time.Second) { + defer lock.UnLock() + account, _ := models.GetAccountByUserId(userId) + if account == nil { + models.InsertAccount(&models.PointAccount{ + Balance: 0, + TotalEarned: 0, + TotalConsumed: 0, + UserId: userId, + Status: models.PointAccountNormal, + Version: 0, + AccountCode: util.UUID(), + }) + return models.GetAccountByUserId(userId) + } + return account, nil + } + return nil, nil + +} diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 5a6c18bff..ddcac515b 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -1,19 +1,44 @@ package point import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/services/reward/limiter" + "code.gitea.io/gitea/services/reward/point/account" + "errors" + "time" ) type PointOperator struct { } -func (operator *PointOperator) IsOperated(ctx reward.RewardOperateContext) bool { - //todo - return true -} func (operator *PointOperator) IsLimited(ctx reward.RewardOperateContext) bool { + if err := limiter.CheckLimit(ctx.Reward.Type, ctx.TargetUserId, ctx.Reward.Amount); err != nil { + return false + } return true } + func (operator *PointOperator) Operate(ctx reward.RewardOperateContext) error { + a, err := account.GetAccount(ctx.TargetUserId) + if err != nil || a == nil { + return errors.New("get account error") + } + + lock := redis_lock.NewDistributeLock(redis_key.PointAccountOperateLock(a.AccountCode)) + if lock.LockWithWait(3*time.Second, 3*time.Second) { + defer lock.UnLock() + na, _ := account.GetAccount(ctx.TargetUserId) + if ctx.OperateType == models.OperateTypeIncrease { + na.Increase(ctx.Reward.Amount, ctx.SourceId) + } else if ctx.OperateType == models.OperateTypeDecrease { + na.Decrease(ctx.Reward.Amount, ctx.SourceId) + } + + } else { + return errors.New("Get account operate lock failed") + } return nil } diff --git a/services/task/limiter.go b/services/task/limiter.go deleted file mode 100644 index 6c2cd4f44..000000000 --- a/services/task/limiter.go +++ /dev/null @@ -1,39 +0,0 @@ -package task - -import ( - "code.gitea.io/gitea/models" - "time" -) - -var LimiterMap = map[string]Limiter{ - models.TaskConfigRefreshRateNotCycle: new(NoCycleLimiter), - models.TaskConfigRefreshRateDaily: new(DailyLimiter), -} - -type Limiter interface { - GetCurrentPeriod() *models.LimiterPeriod -} - -type NoCycleLimiter struct { -} - -func (l *NoCycleLimiter) GetCurrentPeriod() *models.LimiterPeriod { - return nil -} - -type DailyLimiter struct { -} - -func (l *DailyLimiter) GetCurrentPeriod() *models.LimiterPeriod { - t := time.Now() - startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) - endTime := startTime.Add(24 * time.Hour) - return &models.LimiterPeriod{ - StartTime: startTime, - EndTime: endTime, - } -} - -func GetLimiter(refreshRateype string) Limiter { - return LimiterMap[refreshRateype] -} diff --git a/services/task/period/handler.go b/services/task/period/handler.go new file mode 100644 index 000000000..c3e5443d3 --- /dev/null +++ b/services/task/period/handler.go @@ -0,0 +1,50 @@ +package period + +import ( + "code.gitea.io/gitea/models" + "errors" + "time" +) + +var PeriodHandlerMap = map[string]PeriodHandler{ + models.PeriodNotCycle: new(NoCycleHandler), + models.PeriodDaily: new(DailyHandler), +} + +type PeriodHandler interface { + GetCurrentPeriod() *models.PeriodResult +} + +type NoCycleHandler struct { +} + +func (l *NoCycleHandler) GetCurrentPeriod() *models.PeriodResult { + return nil +} + +type DailyHandler struct { +} + +func (l *DailyHandler) GetCurrentPeriod() *models.PeriodResult { + t := time.Now() + startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) + endTime := startTime.Add(24 * time.Hour) + leftTime := endTime.Sub(t) + return &models.PeriodResult{ + StartTime: startTime, + EndTime: endTime, + LeftTime: leftTime, + } +} + +func getPeriodHandler(refreshRateype string) PeriodHandler { + return PeriodHandlerMap[refreshRateype] +} + +func GetPeriod(refreshRate string) (*models.PeriodResult, error) { + handler := getPeriodHandler(refreshRate) + if handler == nil { + return nil, errors.New("task config incorrect") + } + return handler.GetCurrentPeriod(), nil +} diff --git a/services/task/task.go b/services/task/task.go index c5f65240b..403a2ba8f 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -5,8 +5,9 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/reward" - "errors" + "code.gitea.io/gitea/services/task/period" "time" ) @@ -16,7 +17,7 @@ func Accomplish(userId int64, taskType models.TaskType, sourceId string) { func accomplish(userId int64, taskType models.TaskType, sourceId string) error { //lock - var taskLock = redis_lock.NewDistributeLock(redis_key.TaskAccomplishLock(userId, sourceId, taskType)) + var taskLock = redis_lock.NewDistributeLock(redis_key.TaskAccomplishLock(sourceId, taskType)) if !taskLock.Lock(3 * time.Second) { log.Info("duplicated task request,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) return nil @@ -57,12 +58,17 @@ func accomplish(userId int64, taskType models.TaskType, sourceId string) error { } //add log - models.InsertTaskAccomplishLog(&models.TaskAccomplishLog{ + logId := util.UUID() + _, err = models.InsertTaskAccomplishLog(&models.TaskAccomplishLog{ + LogId: logId, ConfigId: config.ID, TaskCode: config.TaskCode, UserId: userId, SourceId: sourceId, }) + if err != nil { + return err + } //reward reward.Send(reward.RewardOperateContext{ @@ -73,6 +79,7 @@ func accomplish(userId int64, taskType models.TaskType, sourceId string) error { Type: config.AwardType, }, TargetUserId: userId, + RequestId: logId, }) return nil @@ -91,11 +98,11 @@ func isHandled(taskType models.TaskType, sourceId string) (bool, error) { } func IsLimited(userId int64, config *models.TaskConfig) (bool, error) { - limiter := GetLimiter(config.RefreshRate) - if limiter == nil { - return false, errors.New("task config incorrect") + p, err := period.GetPeriod(config.RefreshRate) + if err != nil { + return false, err } - n, err := models.CountInTaskPeriod(config.ID, userId, limiter.GetCurrentPeriod()) + n, err := models.CountInTaskPeriod(config.ID, userId, p) if err != nil { return false, err } diff --git a/services/task/task_config.go b/services/task/task_config.go index 6812f5d67..22e0b1828 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -24,9 +24,9 @@ func GetTaskConfig(taskType models.TaskType) (*models.TaskConfig, error) { config, err := models.GetTaskConfigByTaskCode(taskType.String()) if err != nil { if models.IsErrRecordNotExist(err) { + redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) return nil, nil } - redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) return nil, err } jsonStr, _ := json.Marshal(config) From 0a4ab4dcaf60018a2e5dcbebc272ad50aea3fc89 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 2 Jun 2022 18:02:14 +0800 Subject: [PATCH 05/90] update --- models/limit_config.go | 30 +++++-- models/models.go | 7 ++ models/point_account.go | 6 +- models/repo_watch.go | 2 +- models/reward_operate_record.go | 31 ++++--- models/task_config.go | 18 ++-- modules/auth/wechat/access_token.go | 8 +- modules/notification/notification.go | 2 + modules/notification/task/task.go | 85 +++++++++++++++++++ modules/redis/redis_client/client.go | 2 +- modules/redis/redis_key/account_redis_key.go | 4 +- modules/redis/redis_key/limit_redis_key.go | 4 +- modules/redis/redis_key/task_redis_key.go | 12 +-- modules/redis/redis_lock/lock.go | 22 +++-- services/reward/limiter/limiter.go | 50 +++++++---- services/reward/operator.go | 61 +++++++------ .../reward/point/account/point_account.go | 8 +- services/reward/point/point_operate.go | 26 ++++-- services/reward/reward.go | 6 -- services/task/task.go | 51 +++++------ services/task/task_config.go | 4 +- 21 files changed, 292 insertions(+), 147 deletions(-) create mode 100644 modules/notification/task/task.go delete mode 100644 services/reward/reward.go diff --git a/models/limit_config.go b/models/limit_config.go index 273af0de1..2196b5b6d 100644 --- a/models/limit_config.go +++ b/models/limit_config.go @@ -2,6 +2,24 @@ package models import "code.gitea.io/gitea/modules/timeutil" +type LimitType string + +const ( + LimitTypeTask LimitType = "TASK" + LimitTypeReward LimitType = "REWARD" +) + +func (l LimitType) Name() string { + switch l { + case LimitTypeTask: + return "TASK" + case LimitTypeReward: + return "REWARD" + default: + return "" + } +} + type LimitConfig struct { ID int64 `xorm:"pk autoincr"` Tittle string @@ -9,14 +27,15 @@ type LimitConfig struct { Scope string `xorm:"NOT NULL"` LimitNum int64 `xorm:"NOT NULL"` LimitCode string `xorm:"NOT NULL"` + LimitType string `xorm:"NOT NULL"` Creator int64 `xorm:"NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` DeletedAt timeutil.TimeStamp `xorm:"deleted"` } -func findLimitConfig(tl *LimitConfig) ([]LimitConfig, error) { +func GetLimitConfigByLimitCode(limitCode string, limitType LimitType) ([]LimitConfig, error) { r := make([]LimitConfig, 0) - err := x.Find(r, tl) + err := x.Where("limit_code = ? and limit_type = ?", limitCode, limitType.Name()).Find(&r) if err != nil { return nil, err } else if len(r) == 0 { @@ -24,10 +43,3 @@ func findLimitConfig(tl *LimitConfig) ([]LimitConfig, error) { } return r, nil } - -func GetLimitConfigByLimitCode(limitCode string) ([]LimitConfig, error) { - t := &LimitConfig{ - LimitCode: limitCode, - } - return findLimitConfig(t) -} diff --git a/models/models.go b/models/models.go index 9d255c5e6..59e7a3a48 100755 --- a/models/models.go +++ b/models/models.go @@ -144,6 +144,13 @@ func init() { new(WechatBindLog), new(OrgStatistic), new(SearchRecord), + new(TaskConfig), + new(TaskAccomplishLog), + new(RewardOperateRecord), + new(LimitConfig), + new(PeriodicTask), + new(PointAccountLog), + new(PointAccount), ) tablesStatistic = append(tablesStatistic, diff --git a/models/point_account.go b/models/point_account.go index 7fa38cb7a..9a8032553 100644 --- a/models/point_account.go +++ b/models/point_account.go @@ -1,6 +1,8 @@ package models -import "code.gitea.io/gitea/modules/timeutil" +import ( + "code.gitea.io/gitea/modules/timeutil" +) type PointAccountStatus int @@ -87,7 +89,7 @@ func GetAccountByUserId(userId int64) (*PointAccount, error) { return nil, err } if !has { - return nil, nil + return nil, ErrRecordNotExist{} } return p, nil } diff --git a/models/repo_watch.go b/models/repo_watch.go index 31868fcae..2d01bde1f 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -287,7 +287,7 @@ func NotifyWatchers(actions ...*Action) error { func producer(actions ...*Action) { for _, action := range actions { - if !action.IsPrivate{ + if !action.IsPrivate { ActionChan <- action } } diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index ca1f52168..6f4e9a797 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -5,18 +5,12 @@ import ( "fmt" ) -type RewardSourceType string - const ( - SourceTypeAccomplishTask RewardSourceType = "ACCOMPLISH_TASK" - SourceTypeAdminOperate RewardSourceType = "ADMIN_OPERATE" - SourceTypeRunCloudbrainTask RewardSourceType = "RUN_CLOUBRAIN_TASK" + SourceTypeAccomplishTask string = "ACCOMPLISH_TASK" + SourceTypeAdminOperate = "ADMIN_OPERATE" + SourceTypeRunCloudbrainTask = "RUN_CLOUBRAIN_TASK" ) -func (r *RewardSourceType) String() string { - return fmt.Sprint(r) -} - type RewardType string const ( @@ -40,6 +34,7 @@ const ( type RewardOperateRecord struct { ID int64 `xorm:"pk autoincr"` + RecordId string `xorm:"INDEX NOT NULL"` UserId int64 `xorm:"INDEX NOT NULL"` Amount int64 `xorm:"NOT NULL"` RewardType string `xorm:"NOT NULL"` @@ -80,5 +75,21 @@ func UpdateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus r := &RewardOperateRecord{ Status: newStatus, } - return x.Cols("status").Where("source_type=? and requestId=? and status=?", sourceType, requestId, oldStatus).Update(r) + return x.Cols("status").Where("source_type=? and request_id=? and status=?", sourceType, requestId, oldStatus).Update(r) +} + +type RewardOperateContext struct { + SourceType string + SourceId string + Remark string + Reward Reward + TargetUserId int64 + RequestId string + OperateType string + CycleIntervalSeconds int64 +} + +type Reward struct { + Amount int64 + Type string } diff --git a/models/task_config.go b/models/task_config.go index 036f4e315..aa09ee603 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -2,19 +2,13 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" - "fmt" ) -type TaskType string - const ( - TaskTypeComment TaskType = "COMMENT" + TaskTypeCreateIssueComment string = "CREATE_IS" + TaskTypeNewIssue = "NEW_ISSUE" ) -func (t *TaskType) String() string { - return fmt.Sprint(t) -} - const ( PeriodNotCycle = "NOT_CYCLE" PeriodDaily = "DAILY" @@ -23,11 +17,9 @@ const ( //PointTaskConfig Only add and delete are allowed, edit is not allowed //so if you want to edit config for some task code,please delete first and add new one type TaskConfig struct { - ID int64 `xorm:"pk autoincr"` - TaskCode string `xorm:"NOT NULL"` - Tittle string `xorm:"NOT NULL"` - RefreshRate string `xorm:"NOT NULL"` - Times int64 `xorm:"NOT NULL"` + ID int64 `xorm:"pk autoincr"` + TaskCode string `xorm:"NOT NULL"` + Tittle string AwardType string `xorm:"NOT NULL"` AwardAmount int64 `xorm:"NOT NULL"` Creator int64 `xorm:"NOT NULL"` diff --git a/modules/auth/wechat/access_token.go b/modules/auth/wechat/access_token.go index af62c3e7b..e4e38ee30 100644 --- a/modules/auth/wechat/access_token.go +++ b/modules/auth/wechat/access_token.go @@ -26,14 +26,18 @@ func GetWechatAccessToken() string { } func refreshAccessToken() { - if ok := accessTokenLock.Lock(3 * time.Second); ok { + if ok, _ := accessTokenLock.Lock(3 * time.Second); ok { defer accessTokenLock.UnLock() callAccessTokenAndUpdateCache() } } func refreshAndGetAccessToken() string { - if ok := accessTokenLock.LockWithWait(3*time.Second, 3*time.Second); ok { + isOk, err := accessTokenLock.LockWithWait(3*time.Second, 3*time.Second) + if err != nil { + return "" + } + if isOk { defer accessTokenLock.UnLock() token, _ := redis_client.Get(redis_key.WechatAccessTokenKey()) if token != "" { diff --git a/modules/notification/notification.go b/modules/notification/notification.go index 0fd6fa471..8329ca903 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/notification/indexer" "code.gitea.io/gitea/modules/notification/mail" + "code.gitea.io/gitea/modules/notification/task" "code.gitea.io/gitea/modules/notification/ui" "code.gitea.io/gitea/modules/notification/webhook" "code.gitea.io/gitea/modules/repository" @@ -35,6 +36,7 @@ func NewContext() { RegisterNotifier(indexer.NewNotifier()) RegisterNotifier(webhook.NewNotifier()) RegisterNotifier(action.NewNotifier()) + RegisterNotifier(task.NewNotifier()) } // NotifyUploadAttachment notifies attachment upload message to notifiers diff --git a/modules/notification/task/task.go b/modules/notification/task/task.go new file mode 100644 index 000000000..ce3b023ba --- /dev/null +++ b/modules/notification/task/task.go @@ -0,0 +1,85 @@ +package task + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/notification/base" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/services/task" + "fmt" +) + +type taskNotifier struct { + base.NullNotifier +} + +var ( + _ base.Notifier = &taskNotifier{} +) + +// NewNotifier create a new actionNotifier notifier +func NewNotifier() base.Notifier { + return &taskNotifier{} +} + +func (t *taskNotifier) NotifyNewIssue(issue *models.Issue) { + task.Accomplish(issue.Poster.ID, models.TaskTypeNewIssue, fmt.Sprint(issue.ID)) +} + +// NotifyIssueChangeStatus notifies close or reopen issue to notifiers +func (t *taskNotifier) NotifyIssueChangeStatus(doer *models.User, issue *models.Issue, actionComment *models.Comment, closeOrReopen bool) { + return +} + +// NotifyCreateIssueComment notifies comment on an issue to notifiers +func (t *taskNotifier) NotifyCreateIssueComment(doer *models.User, repo *models.Repository, + issue *models.Issue, comment *models.Comment) { + task.Accomplish(doer.ID, models.TaskTypeCreateIssueComment, fmt.Sprint(comment.ID)) +} + +func (t *taskNotifier) NotifyNewPullRequest(pull *models.PullRequest) { + task.Accomplish(pull.Issue.Poster.ID, models.TaskTypeCreateIssueComment, fmt.Sprint(pull.ID)) +} + +func (t *taskNotifier) NotifyRenameRepository(doer *models.User, repo *models.Repository, oldRepoName string) { + return +} + +func (t *taskNotifier) NotifyAliasRepository(doer *models.User, repo *models.Repository, oldAlias string) { + return +} + +func (t *taskNotifier) NotifyTransferRepository(doer *models.User, repo *models.Repository, oldOwnerName string) { + return +} + +func (t *taskNotifier) NotifyCreateRepository(doer *models.User, u *models.User, repo *models.Repository) { + return +} + +func (t *taskNotifier) NotifyForkRepository(doer *models.User, oldRepo, repo *models.Repository) { + return +} + +func (t *taskNotifier) NotifyPullRequestReview(pr *models.PullRequest, review *models.Review, comment *models.Comment) { + return +} + +func (t *taskNotifier) NotifyMergePullRequest(pr *models.PullRequest, doer *models.User) { + return +} + +func (t *taskNotifier) NotifySyncPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { + return +} + +func (t *taskNotifier) NotifySyncCreateRef(doer *models.User, repo *models.Repository, refType, refFullName string) { + return +} + +func (t *taskNotifier) NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) { + return +} + +func (t *taskNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) { + return +} diff --git a/modules/redis/redis_client/client.go b/modules/redis/redis_client/client.go index 2c487a72c..21a6da9fb 100644 --- a/modules/redis/redis_client/client.go +++ b/modules/redis/redis_client/client.go @@ -103,7 +103,7 @@ func Expire(key string, expireSeconds int64) error { redisClient := labelmsg.Get() defer redisClient.Close() - _, err := redisClient.Do("EXPIRE ", key, expireSeconds) + _, err := redisClient.Do("EXPIRE", key, expireSeconds) if err != nil { return err } diff --git a/modules/redis/redis_key/account_redis_key.go b/modules/redis/redis_key/account_redis_key.go index f36a8ea5c..896ea4ff4 100644 --- a/modules/redis/redis_key/account_redis_key.go +++ b/modules/redis/redis_key/account_redis_key.go @@ -8,8 +8,8 @@ func PointAccountOperateLock(accountCode string) string { return KeyJoin(ACCOUNT_REDIS_PREFIX, accountCode, "operate", "lock") } -func PointAccountDetail(userId int64) string { - return KeyJoin(ACCOUNT_REDIS_PREFIX, fmt.Sprint(userId), "detail") +func PointAccountInfo(userId int64) string { + return KeyJoin(ACCOUNT_REDIS_PREFIX, fmt.Sprint(userId), "info") } func PointAccountInitLock(userId int64) string { diff --git a/modules/redis/redis_key/limit_redis_key.go b/modules/redis/redis_key/limit_redis_key.go index e9d8352a2..86a77e59e 100644 --- a/modules/redis/redis_key/limit_redis_key.go +++ b/modules/redis/redis_key/limit_redis_key.go @@ -21,6 +21,6 @@ func LimitCount(userId int64, limitCode string, period *models.PeriodResult) str } -func LimitConfig(limitCode string) string { - return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, "config") +func LimitConfig(limitCode string, limitType models.LimitType) string { + return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, limitType.Name(), "config") } diff --git a/modules/redis/redis_key/task_redis_key.go b/modules/redis/redis_key/task_redis_key.go index 2eb8c21d1..3427c8f7f 100644 --- a/modules/redis/redis_key/task_redis_key.go +++ b/modules/redis/redis_key/task_redis_key.go @@ -1,15 +1,11 @@ package redis_key -import ( - "code.gitea.io/gitea/models" -) - const TASK_REDIS_PREFIX = "task" -func TaskAccomplishLock(sourceId string, taskType models.TaskType) string { - return KeyJoin(TASK_REDIS_PREFIX, sourceId, taskType.String(), "accomplish") +func TaskAccomplishLock(sourceId string, taskType string) string { + return KeyJoin(TASK_REDIS_PREFIX, sourceId, taskType, "accomplish") } -func TaskConfig(taskType models.TaskType) string { - return KeyJoin(TASK_REDIS_PREFIX, "config", taskType.String()) +func TaskConfig(taskType string) string { + return KeyJoin(TASK_REDIS_PREFIX, "config", taskType) } diff --git a/modules/redis/redis_lock/lock.go b/modules/redis/redis_lock/lock.go index b8cd837f1..5723c379d 100644 --- a/modules/redis/redis_lock/lock.go +++ b/modules/redis/redis_lock/lock.go @@ -13,26 +13,32 @@ func NewDistributeLock(lockKey string) *DistributeLock { return &DistributeLock{lockKey: lockKey} } -func (lock *DistributeLock) Lock(expireTime time.Duration) bool { - isOk, _ := redis_client.Setnx(lock.lockKey, "", expireTime) - return isOk +func (lock *DistributeLock) Lock(expireTime time.Duration) (bool, error) { + isOk, err := redis_client.Setnx(lock.lockKey, "", expireTime) + if err != nil { + return false, err + } + return isOk, nil } -func (lock *DistributeLock) LockWithWait(expireTime time.Duration, waitTime time.Duration) bool { +func (lock *DistributeLock) LockWithWait(expireTime time.Duration, waitTime time.Duration) (bool, error) { start := time.Now().Unix() * 1000 duration := waitTime.Milliseconds() for { - isOk, _ := redis_client.Setnx(lock.lockKey, "", expireTime) + isOk, err := redis_client.Setnx(lock.lockKey, "", expireTime) + if err != nil { + return false, err + } if isOk { - return true + return true, nil } if time.Now().Unix()*1000-start > duration { - return false + return false, nil } time.Sleep(50 * time.Millisecond) } - return false + return false, nil } func (lock *DistributeLock) UnLock() error { diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go index aca8af22e..8117ba173 100644 --- a/services/reward/limiter/limiter.go +++ b/services/reward/limiter/limiter.go @@ -2,6 +2,7 @@ package limiter import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/services/task/period" @@ -17,25 +18,27 @@ type limiterRunner struct { userId int64 amount int64 limitCode string + limitType models.LimitType } -func newLimiterRunner(limitCode string, userId, amount int64) *limiterRunner { +func newLimiterRunner(limitCode string, limitType models.LimitType, userId, amount int64) *limiterRunner { return &limiterRunner{ userId: userId, amount: amount, limitCode: limitCode, + limitType: limitType, index: 0, } } func (l *limiterRunner) Run() error { - if err := l.LoadLimiters(l.limitCode); err != nil { + if err := l.LoadLimiters(); err != nil { return err } - //todo 验证未配置的情况 - for l.index <= len(l.limiters) { + for l.index < len(l.limiters) { err := l.limit(l.limiters[l.index]) if err != nil { + log.Info("limiter check failed,%v", err) return err } l.index += 1 @@ -62,32 +65,43 @@ func (l *limiterRunner) limit(r models.LimitConfig) error { return nil } -func (l *limiterRunner) LoadLimiters(limitCode string) error { - redisKey := redis_key.LimitConfig(limitCode) +func (l *limiterRunner) LoadLimiters() error { + limiters, err := GetLimiters(l.limitCode, l.limitType) + if err != nil { + return err + } + if limiters != nil { + l.limiters = limiters + } + return nil +} + +func CheckLimit(limitCode string, limitType models.LimitType, userId, amount int64) error { + r := newLimiterRunner(limitCode, limitType, userId, amount) + return r.Run() +} + +func GetLimiters(limitCode string, limitType models.LimitType) ([]models.LimitConfig, error) { + redisKey := redis_key.LimitConfig(limitCode, limitType) val, _ := redis_client.Get(redisKey) if val != "" { if val == redis_key.EMPTY_REDIS_VAL { - return nil + return nil, nil } limiters := make([]models.LimitConfig, 0) - json.Unmarshal([]byte(val), limiters) - return nil + json.Unmarshal([]byte(val), &limiters) + return limiters, nil } - limiters, err := models.GetLimitConfigByLimitCode(limitCode) + limiters, err := models.GetLimitConfigByLimitCode(limitCode, limitType) if err != nil { if models.IsErrRecordNotExist(err) { redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) - return nil + return nil, nil } - return err + return nil, err } jsonStr, _ := json.Marshal(limiters) redis_client.Setex(redisKey, string(jsonStr), 30*24*time.Hour) - return nil -} - -func CheckLimit(limitCode string, userId, amount int64) error { - r := newLimiterRunner(limitCode, userId, amount) - return r.Run() + return limiters, nil } diff --git a/services/reward/operator.go b/services/reward/operator.go index 321562474..b0bd53f8a 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/reward/point" "errors" "fmt" @@ -15,26 +16,25 @@ var RewardOperatorMap = map[string]RewardOperator{ fmt.Sprint(models.RewardTypePoint): new(point.PointOperator), } -type RewardOperateContext struct { - SourceType models.RewardSourceType - SourceId string - Remark string - Reward Reward - TargetUserId int64 - RequestId string - OperateType string - CycleIntervalSeconds int64 -} - type RewardOperator interface { - IsLimited(ctx RewardOperateContext) bool - Operate(ctx RewardOperateContext) error + IsLimited(ctx models.RewardOperateContext) bool + Operate(ctx models.RewardOperateContext) error } -func Send(ctx RewardOperateContext) error { +func Send(ctx models.RewardOperateContext) error { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:%v", combinedErr) + } + }() //add lock - var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardSendLock(ctx.RequestId, ctx.SourceType.String())) - if !rewardLock.Lock(3 * time.Second) { + var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardSendLock(ctx.RequestId, ctx.SourceType)) + isOk, err := rewardLock.Lock(3 * time.Second) + if err != nil { + return err + } + if !isOk { log.Info("duplicated reward request,targetUserId=%d requestId=%s", ctx.TargetUserId, ctx.RequestId) return nil } @@ -63,19 +63,22 @@ func Send(ctx RewardOperateContext) error { } //new reward operate record - if err := initAwardOperateRecord(ctx); err != nil { + recordId, err := initAwardOperateRecord(ctx) + if err != nil { return err } + ctx.SourceId = recordId + //operate if err := operator.Operate(ctx); err != nil { - updateAwardOperateRecordStatus(ctx.SourceType.String(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) + updateAwardOperateRecordStatus(ctx.SourceType, ctx.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) return err } //if not a cycle operate,update status to success - if ctx.CycleIntervalSeconds > 0 { - updateAwardOperateRecordStatus(ctx.SourceType.String(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusSucceeded) + if ctx.CycleIntervalSeconds == 0 { + updateAwardOperateRecordStatus(ctx.SourceType, ctx.RequestId, models.OperateStatusOperating, models.OperateStatusSucceeded) } return nil } @@ -84,8 +87,8 @@ func GetOperator(rewardType string) RewardOperator { return RewardOperatorMap[rewardType] } -func isHandled(sourceType models.RewardSourceType, requestId string) (bool, error) { - _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType.String(), requestId) +func isHandled(sourceType string, requestId string) (bool, error) { + _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId) if err != nil { if models.IsErrRecordNotExist(err) { return false, nil @@ -96,23 +99,25 @@ func isHandled(sourceType models.RewardSourceType, requestId string) (bool, erro } -func initAwardOperateRecord(ctx RewardOperateContext) error { - _, err := models.InsertAwardOperateRecord(&models.RewardOperateRecord{ +func initAwardOperateRecord(ctx models.RewardOperateContext) (string, error) { + record := &models.RewardOperateRecord{ + RecordId: util.UUID(), UserId: ctx.TargetUserId, Amount: ctx.Reward.Amount, RewardType: ctx.Reward.Type, - SourceType: ctx.SourceType.String(), + SourceType: ctx.SourceType, SourceId: ctx.SourceId, RequestId: ctx.RequestId, OperateType: ctx.OperateType, CycleIntervalSeconds: ctx.CycleIntervalSeconds, Status: models.OperateStatusOperating, Remark: ctx.Remark, - }) + } + _, err := models.InsertAwardOperateRecord(record) if err != nil { - return err + return "", err } - return nil + return record.RecordId, nil } func updateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus string) error { diff --git a/services/reward/point/account/point_account.go b/services/reward/point/account/point_account.go index 9ff5001fc..ea127e162 100644 --- a/services/reward/point/account/point_account.go +++ b/services/reward/point/account/point_account.go @@ -11,7 +11,7 @@ import ( ) func GetAccount(userId int64) (*models.PointAccount, error) { - redisKey := redis_key.PointAccountDetail(userId) + redisKey := redis_key.PointAccountInfo(userId) val, _ := redis_client.Get(redisKey) if val != "" { account := &models.PointAccount{} @@ -36,7 +36,11 @@ func GetAccount(userId int64) (*models.PointAccount, error) { func InitAccount(userId int64) (*models.PointAccount, error) { lock := redis_lock.NewDistributeLock(redis_key.PointAccountInitLock(userId)) - if lock.LockWithWait(3*time.Second, 3*time.Second) { + isOk, err := lock.LockWithWait(3*time.Second, 3*time.Second) + if err != nil { + return nil, err + } + if isOk { defer lock.UnLock() account, _ := models.GetAccountByUserId(userId) if account == nil { diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index ddcac515b..eeba83ac7 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -2,9 +2,9 @@ package point import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" - "code.gitea.io/gitea/services/reward" "code.gitea.io/gitea/services/reward/limiter" "code.gitea.io/gitea/services/reward/point/account" "errors" @@ -14,28 +14,36 @@ import ( type PointOperator struct { } -func (operator *PointOperator) IsLimited(ctx reward.RewardOperateContext) bool { - if err := limiter.CheckLimit(ctx.Reward.Type, ctx.TargetUserId, ctx.Reward.Amount); err != nil { - return false +func (operator *PointOperator) IsLimited(ctx models.RewardOperateContext) bool { + if err := limiter.CheckLimit(ctx.Reward.Type, models.LimitTypeReward, ctx.TargetUserId, ctx.Reward.Amount); err != nil { + return true } - return true + return false } -func (operator *PointOperator) Operate(ctx reward.RewardOperateContext) error { +func (operator *PointOperator) Operate(ctx models.RewardOperateContext) error { a, err := account.GetAccount(ctx.TargetUserId) if err != nil || a == nil { return errors.New("get account error") } lock := redis_lock.NewDistributeLock(redis_key.PointAccountOperateLock(a.AccountCode)) - if lock.LockWithWait(3*time.Second, 3*time.Second) { + isOk, err := lock.LockWithWait(3*time.Second, 3*time.Second) + if err != nil { + return err + } + if isOk { defer lock.UnLock() na, _ := account.GetAccount(ctx.TargetUserId) if ctx.OperateType == models.OperateTypeIncrease { - na.Increase(ctx.Reward.Amount, ctx.SourceId) + err = na.Increase(ctx.Reward.Amount, ctx.SourceId) } else if ctx.OperateType == models.OperateTypeDecrease { - na.Decrease(ctx.Reward.Amount, ctx.SourceId) + err = na.Decrease(ctx.Reward.Amount, ctx.SourceId) + } + if err != nil { + return err } + redis_client.Del(redis_key.PointAccountInfo(ctx.TargetUserId)) } else { return errors.New("Get account operate lock failed") diff --git a/services/reward/reward.go b/services/reward/reward.go deleted file mode 100644 index ca1c1f3cd..000000000 --- a/services/reward/reward.go +++ /dev/null @@ -1,6 +0,0 @@ -package reward - -type Reward struct { - Amount int64 - Type string -} diff --git a/services/task/task.go b/services/task/task.go index 403a2ba8f..f38793419 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -7,18 +7,30 @@ import ( "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/reward" - "code.gitea.io/gitea/services/task/period" + "code.gitea.io/gitea/services/reward/limiter" + "fmt" "time" ) -func Accomplish(userId int64, taskType models.TaskType, sourceId string) { +func Accomplish(userId int64, taskType string, sourceId string) { go accomplish(userId, taskType, sourceId) } -func accomplish(userId int64, taskType models.TaskType, sourceId string) error { +func accomplish(userId int64, taskType string, sourceId string) error { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:%v", combinedErr) + } + }() //lock var taskLock = redis_lock.NewDistributeLock(redis_key.TaskAccomplishLock(sourceId, taskType)) - if !taskLock.Lock(3 * time.Second) { + isOk, err := taskLock.Lock(3 * time.Second) + if err != nil { + log.Error("get taskLock error. %v", err) + return err + } + if !isOk { log.Info("duplicated task request,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) return nil } @@ -47,12 +59,7 @@ func accomplish(userId int64, taskType models.TaskType, sourceId string) error { } //is limited? - isLimited, err := IsLimited(userId, config) - if err != nil { - log.Error("get limited error,%v", err) - return err - } - if isLimited { + if isLimited(userId, config) { log.Info("task accomplish maximum times are reached,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) return nil } @@ -71,22 +78,23 @@ func accomplish(userId int64, taskType models.TaskType, sourceId string) error { } //reward - reward.Send(reward.RewardOperateContext{ + reward.Send(models.RewardOperateContext{ SourceType: models.SourceTypeAccomplishTask, - SourceId: sourceId, - Reward: reward.Reward{ + SourceId: logId, + Reward: models.Reward{ Amount: config.AwardAmount, Type: config.AwardType, }, TargetUserId: userId, RequestId: logId, + OperateType: models.OperateTypeIncrease, }) return nil } -func isHandled(taskType models.TaskType, sourceId string) (bool, error) { - _, err := models.GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskType.String()) +func isHandled(taskType string, sourceId string) (bool, error) { + _, err := models.GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskType) if err != nil { if models.IsErrRecordNotExist(err) { return false, nil @@ -97,15 +105,10 @@ func isHandled(taskType models.TaskType, sourceId string) (bool, error) { } -func IsLimited(userId int64, config *models.TaskConfig) (bool, error) { - p, err := period.GetPeriod(config.RefreshRate) - if err != nil { - return false, err - } - n, err := models.CountInTaskPeriod(config.ID, userId, p) - if err != nil { - return false, err +func isLimited(userId int64, config *models.TaskConfig) bool { + if err := limiter.CheckLimit(config.TaskCode, models.LimitTypeTask, userId, 1); err != nil { + return true } - return n >= config.Times, nil + return false } diff --git a/services/task/task_config.go b/services/task/task_config.go index 22e0b1828..6e7f22e14 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -10,7 +10,7 @@ import ( //GetTaskConfig get task config from redis cache first // if not exist in redis, find in db and refresh the redis key -func GetTaskConfig(taskType models.TaskType) (*models.TaskConfig, error) { +func GetTaskConfig(taskType string) (*models.TaskConfig, error) { redisKey := redis_key.TaskConfig(taskType) configStr, _ := redis_client.Get(redisKey) if configStr != "" { @@ -21,7 +21,7 @@ func GetTaskConfig(taskType models.TaskType) (*models.TaskConfig, error) { json.Unmarshal([]byte(configStr), config) return config, nil } - config, err := models.GetTaskConfigByTaskCode(taskType.String()) + config, err := models.GetTaskConfigByTaskCode(taskType) if err != nil { if models.IsErrRecordNotExist(err) { redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) From c5a35c5982f8de551fc8c5a3f50f13f01e856638 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 6 Jun 2022 18:08:14 +0800 Subject: [PATCH 06/90] #1249 update limiters --- models/limit_config.go | 38 +++++++++--- models/reward_operate_record.go | 10 ++- models/task_accomplish_log.go | 11 +--- models/task_config.go | 20 +++++- modules/notification/action/action.go | 4 ++ modules/notification/base/notifier.go | 1 + modules/notification/base/null.go | 4 ++ modules/notification/notification.go | 7 +++ modules/notification/task/task.go | 52 +++++++++++----- modules/redis/redis_key/limit_redis_key.go | 16 ++--- services/reward/limiter/limiter.go | 71 +++++++++++++++++++--- services/reward/operator.go | 2 +- services/reward/point/point_operate.go | 2 +- services/task/task.go | 49 ++------------- 14 files changed, 185 insertions(+), 102 deletions(-) diff --git a/models/limit_config.go b/models/limit_config.go index 2196b5b6d..aec26a036 100644 --- a/models/limit_config.go +++ b/models/limit_config.go @@ -5,16 +5,34 @@ import "code.gitea.io/gitea/modules/timeutil" type LimitType string const ( - LimitTypeTask LimitType = "TASK" - LimitTypeReward LimitType = "REWARD" + LimitTypeTask LimitType = "TASK" + LimitTypeRewardPoint LimitType = "REWARD_POINT" ) func (l LimitType) Name() string { switch l { case LimitTypeTask: return "TASK" - case LimitTypeReward: - return "REWARD" + case LimitTypeRewardPoint: + return "REWARD_POINT" + default: + return "" + } +} + +type LimitScope string + +const ( + LimitScopeAllUsers LimitScope = "ALL_USERS" + LimitScopeSingleUser LimitScope = "SINGLE_USER" +) + +func (l LimitScope) Name() string { + switch l { + case LimitScopeAllUsers: + return "ALL_USERS" + case LimitScopeSingleUser: + return "SINGLE_USER" default: return "" } @@ -23,19 +41,19 @@ func (l LimitType) Name() string { type LimitConfig struct { ID int64 `xorm:"pk autoincr"` Tittle string - RefreshRate string `xorm:"NOT NULL"` - Scope string `xorm:"NOT NULL"` - LimitNum int64 `xorm:"NOT NULL"` - LimitCode string `xorm:"NOT NULL"` + RefreshRate string `xorm:"NOT NULL"` + Scope string `xorm:"NOT NULL"` + LimitNum int64 `xorm:"NOT NULL"` + LimitCode string LimitType string `xorm:"NOT NULL"` Creator int64 `xorm:"NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` DeletedAt timeutil.TimeStamp `xorm:"deleted"` } -func GetLimitConfigByLimitCode(limitCode string, limitType LimitType) ([]LimitConfig, error) { +func GetLimitConfigByLimitType(limitType LimitType) ([]LimitConfig, error) { r := make([]LimitConfig, 0) - err := x.Where("limit_code = ? and limit_type = ?", limitCode, limitType.Name()).Find(&r) + err := x.Where(" limit_type = ?", limitType.Name()).Find(&r) if err != nil { return nil, err } else if len(r) == 0 { diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 6f4e9a797..1577fbaff 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -2,7 +2,6 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" - "fmt" ) const ( @@ -17,8 +16,13 @@ const ( RewardTypePoint RewardType = "POINT" ) -func (r *RewardType) String() string { - return fmt.Sprint(r) +func (r RewardType) Name() string { + switch r { + case RewardTypePoint: + return "POINT" + default: + return "" + } } const ( diff --git a/models/task_accomplish_log.go b/models/task_accomplish_log.go index 707c214f5..3736d1c41 100644 --- a/models/task_accomplish_log.go +++ b/models/task_accomplish_log.go @@ -11,7 +11,6 @@ type TaskAccomplishLog struct { ConfigId int64 `xorm:"NOT NULL"` TaskCode string `xorm:"NOT NULL"` UserId int64 `xorm:"INDEX NOT NULL"` - SourceId string `xorm:"INDEX NOT NULL"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } @@ -31,15 +30,7 @@ func getTaskAccomplishLog(tl *TaskAccomplishLog) (*TaskAccomplishLog, error) { return tl, nil } -func GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskCode string) (*TaskAccomplishLog, error) { - t := &TaskAccomplishLog{ - SourceId: sourceId, - TaskCode: taskCode, - } - return getTaskAccomplishLog(t) -} - -func CountInTaskPeriod(configId int64, userId int64, period *PeriodResult) (int64, error) { +func CountTaskAccomplishLogInTaskPeriod(configId int64, userId int64, period *PeriodResult) (int64, error) { if period == nil { return x.Where("config_id = ? and user_id = ?", configId, userId).Count(&TaskAccomplishLog{}) } else { diff --git a/models/task_config.go b/models/task_config.go index aa09ee603..fe2bb7721 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -5,8 +5,24 @@ import ( ) const ( - TaskTypeCreateIssueComment string = "CREATE_IS" - TaskTypeNewIssue = "NEW_ISSUE" + TaskTypeNewIssue = "NEW_ISSUE" + TaskTypeIssueChangeStatus = "ISSUE_CHANGE_STATUS" + TaskTypeCreateIssueComment = "CREATE_ISSUE_COMMENT" + TaskTypeNewPullRequest = "NEW_PULL_REQUEST" + TaskTypeRenameRepository = "RENAME_REPOSITORY" + TaskTypeAliasRepository = "ALIAS_REPOSITORY" + TaskTypeTransferRepository = "TRANSFER_REPOSITORY" + TaskTypeCreateRepository = "CREATE_REPOSITORY" + TaskTypeForkRepository = "FORK_REPOSITORY" + TaskTypePullRequestReview = "PULL_REQUEST_REVIEW" + TaskTypeCommentPull = "COMMENT_PULL" + TaskTypeApprovePullRequest = "APPROVE_PULL_REQUEST" + TaskTypeRejectPullRequest = "REJECT_PULL_REQUEST" + TaskTypeMergePullRequest = "MERGE_PULL_REQUEST" + TaskTypeSyncPushCommits = "SYNC_PUSH_COMMITS" + TaskTypeSyncCreateRef = "SYNC_CREATE_REF" + TaskTypeSyncDeleteRef = "SYNC_DELETE_REF" + TaskTypeBindWechat = "BIND_WECHAT" ) const ( diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index 2ac73c2c3..943678a0b 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -345,3 +345,7 @@ func (a *actionNotifier) NotifyOtherTask(doer *models.User, repo *models.Reposit log.Error("notifyWatchers: %v", err) } } + +func (a *actionNotifier) NotifyWechatBind(doer *models.User) { + return +} diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 8d6fdeb52..26cd1feb8 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -56,4 +56,5 @@ type Notifier interface { NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) + NotifyWechatBind(doer *models.User) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index 0d3489882..ecdebd7a3 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -158,3 +158,7 @@ func (*NullNotifier) NotifySyncDeleteRef(doer *models.User, repo *models.Reposit func (*NullNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) { } + +func (*NullNotifier) NotifyWechatBind(doer *models.User) { + +} diff --git a/modules/notification/notification.go b/modules/notification/notification.go index 8329ca903..2f0335c5a 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -271,3 +271,10 @@ func NotifySyncDeleteRef(pusher *models.User, repo *models.Repository, refType, notifier.NotifySyncDeleteRef(pusher, repo, refType, refFullName) } } + +// NotifyWechatBind notifies wechat bind +func NotifyWechatBind(doer *models.User) { + for _, notifier := range notifiers { + notifier.NotifyWechatBind(doer) + } +} diff --git a/modules/notification/task/task.go b/modules/notification/task/task.go index ce3b023ba..f68872c01 100644 --- a/modules/notification/task/task.go +++ b/modules/notification/task/task.go @@ -5,7 +5,7 @@ import ( "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/services/task" - "fmt" + "strings" ) type taskNotifier struct { @@ -22,64 +22,86 @@ func NewNotifier() base.Notifier { } func (t *taskNotifier) NotifyNewIssue(issue *models.Issue) { - task.Accomplish(issue.Poster.ID, models.TaskTypeNewIssue, fmt.Sprint(issue.ID)) + task.Accomplish(issue.Poster.ID, models.TaskTypeNewIssue) } // NotifyIssueChangeStatus notifies close or reopen issue to notifiers func (t *taskNotifier) NotifyIssueChangeStatus(doer *models.User, issue *models.Issue, actionComment *models.Comment, closeOrReopen bool) { - return + task.Accomplish(doer.ID, models.TaskTypeIssueChangeStatus) } // NotifyCreateIssueComment notifies comment on an issue to notifiers func (t *taskNotifier) NotifyCreateIssueComment(doer *models.User, repo *models.Repository, issue *models.Issue, comment *models.Comment) { - task.Accomplish(doer.ID, models.TaskTypeCreateIssueComment, fmt.Sprint(comment.ID)) + task.Accomplish(doer.ID, models.TaskTypeCreateIssueComment) } func (t *taskNotifier) NotifyNewPullRequest(pull *models.PullRequest) { - task.Accomplish(pull.Issue.Poster.ID, models.TaskTypeCreateIssueComment, fmt.Sprint(pull.ID)) + task.Accomplish(pull.Issue.Poster.ID, models.TaskTypeNewPullRequest) } func (t *taskNotifier) NotifyRenameRepository(doer *models.User, repo *models.Repository, oldRepoName string) { - return + task.Accomplish(doer.ID, models.TaskTypeRenameRepository) } func (t *taskNotifier) NotifyAliasRepository(doer *models.User, repo *models.Repository, oldAlias string) { - return + task.Accomplish(doer.ID, models.TaskTypeAliasRepository) } func (t *taskNotifier) NotifyTransferRepository(doer *models.User, repo *models.Repository, oldOwnerName string) { - return + task.Accomplish(doer.ID, models.TaskTypeTransferRepository) } func (t *taskNotifier) NotifyCreateRepository(doer *models.User, u *models.User, repo *models.Repository) { - return + task.Accomplish(doer.ID, models.TaskTypeCreateRepository) } func (t *taskNotifier) NotifyForkRepository(doer *models.User, oldRepo, repo *models.Repository) { - return + task.Accomplish(doer.ID, models.TaskTypeForkRepository) } func (t *taskNotifier) NotifyPullRequestReview(pr *models.PullRequest, review *models.Review, comment *models.Comment) { - return + for _, lines := range review.CodeComments { + for _, comments := range lines { + for _, _ = range comments { + task.Accomplish(review.Reviewer.ID, models.TaskTypePullRequestReview) + } + } + } + if review.Type != models.ReviewTypeComment || strings.TrimSpace(comment.Content) != "" { + + switch review.Type { + case models.ReviewTypeApprove: + task.Accomplish(review.Reviewer.ID, models.TaskTypeApprovePullRequest) + case models.ReviewTypeReject: + task.Accomplish(review.Reviewer.ID, models.TaskTypeRejectPullRequest) + default: + task.Accomplish(review.Reviewer.ID, models.TaskTypeCommentPull) + } + + } } func (t *taskNotifier) NotifyMergePullRequest(pr *models.PullRequest, doer *models.User) { - return + task.Accomplish(doer.ID, models.TaskTypeMergePullRequest) } func (t *taskNotifier) NotifySyncPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { - return + task.Accomplish(repo.OwnerID, models.TaskTypeSyncPushCommits) } func (t *taskNotifier) NotifySyncCreateRef(doer *models.User, repo *models.Repository, refType, refFullName string) { - return + task.Accomplish(repo.OwnerID, models.TaskTypeSyncCreateRef) } func (t *taskNotifier) NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) { - return + task.Accomplish(repo.OwnerID, models.TaskTypeSyncDeleteRef) } func (t *taskNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) { return } + +func (t *taskNotifier) NotifyWechatBind(doer *models.User) { + task.Accomplish(doer.ID, models.TaskTypeSyncDeleteRef) +} diff --git a/modules/redis/redis_key/limit_redis_key.go b/modules/redis/redis_key/limit_redis_key.go index 86a77e59e..a58a70fdb 100644 --- a/modules/redis/redis_key/limit_redis_key.go +++ b/modules/redis/redis_key/limit_redis_key.go @@ -7,20 +7,20 @@ import ( const LIMIT_REDIS_PREFIX = "limit" -func LimitCount(userId int64, limitCode string, period *models.PeriodResult) string { - if userId == 0 { +func LimitCount(userId int64, limitCode string, limitType string, scope string, period *models.PeriodResult) string { + if scope == models.LimitScopeAllUsers.Name() { if period == nil { - return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, "count") + return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, limitType, "count") } - return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, fmt.Sprint(period.StartTime.Unix()), fmt.Sprint(period.EndTime.Unix()), "count") + return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, limitType, fmt.Sprint(period.StartTime.Unix()), fmt.Sprint(period.EndTime.Unix()), "count") } if period == nil { - return KeyJoin(LIMIT_REDIS_PREFIX, "uid", fmt.Sprint(userId), limitCode, "count") + return KeyJoin(LIMIT_REDIS_PREFIX, "uid", fmt.Sprint(userId), limitCode, limitType, "count") } - return KeyJoin(LIMIT_REDIS_PREFIX, "uid", fmt.Sprint(userId), limitCode, fmt.Sprint(period.StartTime.Unix()), fmt.Sprint(period.EndTime.Unix()), "count") + return KeyJoin(LIMIT_REDIS_PREFIX, "uid", fmt.Sprint(userId), limitCode, limitType, fmt.Sprint(period.StartTime.Unix()), fmt.Sprint(period.EndTime.Unix()), "count") } -func LimitConfig(limitCode string, limitType models.LimitType) string { - return KeyJoin(LIMIT_REDIS_PREFIX, limitCode, limitType.Name(), "config") +func LimitConfig(limitType models.LimitType) string { + return KeyJoin(LIMIT_REDIS_PREFIX, limitType.Name(), "config") } diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go index 8117ba173..04cef2e2c 100644 --- a/services/reward/limiter/limiter.go +++ b/services/reward/limiter/limiter.go @@ -39,6 +39,7 @@ func (l *limiterRunner) Run() error { err := l.limit(l.limiters[l.index]) if err != nil { log.Info("limiter check failed,%v", err) + l.Rollback(l.index) return err } l.index += 1 @@ -46,20 +47,50 @@ func (l *limiterRunner) Run() error { return nil } +//Rollback rollback the usedNum from limiters[0] to limiters[index] +func (l *limiterRunner) Rollback(index int) error { + for i := index; i >= 0; i-- { + l.rollback(l.limiters[i]) + } + return nil +} + +func (l *limiterRunner) rollback(r models.LimitConfig) error { + p, err := period.GetPeriod(r.RefreshRate) + if err != nil { + return err + } + redisKey := redis_key.LimitCount(l.userId, r.LimitCode, r.LimitType, r.Scope, p) + redis_client.IncrBy(redisKey, -1*l.amount) + return nil +} + func (l *limiterRunner) limit(r models.LimitConfig) error { p, err := period.GetPeriod(r.RefreshRate) if err != nil { return err } - redisKey := redis_key.LimitCount(l.userId, r.LimitCode, p) + redisKey := redis_key.LimitCount(l.userId, r.LimitCode, r.LimitType, r.Scope, p) usedNum, err := redis_client.IncrBy(redisKey, l.amount) - //if it is the first time,set expire time - if usedNum == l.amount && p != nil { - //todo 验证浮点精确度 - redis_client.Expire(redisKey, int64(p.LeftTime.Seconds())) + if err != nil { + return err + } + //if usedNum equals amount,it is the first operation in period or redis cache deleted + //count in database to distinguish the two cases + if usedNum == l.amount { + n, err := l.countInPeriod(r, p) + if err != nil { + return err + } + if n > 0 { + //means redis cache deleted,incr the cache with real value + usedNum, err = redis_client.IncrBy(redisKey, n) + } + if p != nil { + redis_client.Expire(redisKey, int64(p.LeftTime.Seconds())) + } } if usedNum > r.LimitNum { - redis_client.IncrBy(redisKey, -1*l.amount) return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) } return nil @@ -76,13 +107,37 @@ func (l *limiterRunner) LoadLimiters() error { return nil } +func (l *limiterRunner) countInPeriod(r models.LimitConfig, p *models.PeriodResult) (int64, error) { + switch r.LimitType { + case models.LimitTypeTask.Name(): + return models.CountTaskAccomplishLogInTaskPeriod(r.ID, l.userId, p) + default: + return 0, nil + + } +} + func CheckLimit(limitCode string, limitType models.LimitType, userId, amount int64) error { r := newLimiterRunner(limitCode, limitType, userId, amount) return r.Run() } func GetLimiters(limitCode string, limitType models.LimitType) ([]models.LimitConfig, error) { - redisKey := redis_key.LimitConfig(limitCode, limitType) + limiters, err := GetLimitersByLimitType(limitType) + if err != nil { + return nil, err + } + result := make([]models.LimitConfig, 0) + for i, v := range limiters { + if v.LimitCode == "" || v.LimitCode == limitCode { + result = append(result, limiters[i]) + } + } + return result, nil +} + +func GetLimitersByLimitType(limitType models.LimitType) ([]models.LimitConfig, error) { + redisKey := redis_key.LimitConfig(limitType) val, _ := redis_client.Get(redisKey) if val != "" { if val == redis_key.EMPTY_REDIS_VAL { @@ -92,7 +147,7 @@ func GetLimiters(limitCode string, limitType models.LimitType) ([]models.LimitCo json.Unmarshal([]byte(val), &limiters) return limiters, nil } - limiters, err := models.GetLimitConfigByLimitCode(limitCode, limitType) + limiters, err := models.GetLimitConfigByLimitType(limitType) if err != nil { if models.IsErrRecordNotExist(err) { redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) diff --git a/services/reward/operator.go b/services/reward/operator.go index b0bd53f8a..8d24ed055 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -58,7 +58,7 @@ func Send(ctx models.RewardOperateContext) error { } //is limited? - if operator.IsLimited(ctx) { + if isLimited := operator.IsLimited(ctx); isLimited { return nil } diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index eeba83ac7..80b0b4fe9 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -15,7 +15,7 @@ type PointOperator struct { } func (operator *PointOperator) IsLimited(ctx models.RewardOperateContext) bool { - if err := limiter.CheckLimit(ctx.Reward.Type, models.LimitTypeReward, ctx.TargetUserId, ctx.Reward.Amount); err != nil { + if err := limiter.CheckLimit(ctx.SourceType, models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount); err != nil { return true } return false diff --git a/services/task/task.go b/services/task/task.go index f38793419..737094b4e 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -3,49 +3,23 @@ package task import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/redis/redis_key" - "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/reward" "code.gitea.io/gitea/services/reward/limiter" "fmt" - "time" ) -func Accomplish(userId int64, taskType string, sourceId string) { - go accomplish(userId, taskType, sourceId) +func Accomplish(userId int64, taskType string) { + go accomplish(userId, taskType) } -func accomplish(userId int64, taskType string, sourceId string) error { +func accomplish(userId int64, taskType string) error { defer func() { if err := recover(); err != nil { combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) log.Error("PANIC:%v", combinedErr) } }() - //lock - var taskLock = redis_lock.NewDistributeLock(redis_key.TaskAccomplishLock(sourceId, taskType)) - isOk, err := taskLock.Lock(3 * time.Second) - if err != nil { - log.Error("get taskLock error. %v", err) - return err - } - if !isOk { - log.Info("duplicated task request,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) - return nil - } - defer taskLock.UnLock() - - //is handled before? - isHandled, err := isHandled(taskType, sourceId) - if err != nil { - log.Error("Get isHandled error,%v", err) - return err - } - if isHandled { - log.Info("task has been handled,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) - return nil - } //get task config config, err := GetTaskConfig(taskType) @@ -54,13 +28,13 @@ func accomplish(userId int64, taskType string, sourceId string) error { return err } if config == nil { - log.Info("task config not exist,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) + log.Info("task config not exist,userId=%d taskType=%s", userId, taskType) return nil } //is limited? if isLimited(userId, config) { - log.Info("task accomplish maximum times are reached,userId=%d taskType=%s sourceId=%s", userId, taskType, sourceId) + log.Info("task accomplish maximum times are reached,userId=%d taskType=%s", userId, taskType) return nil } @@ -71,7 +45,6 @@ func accomplish(userId int64, taskType string, sourceId string) error { ConfigId: config.ID, TaskCode: config.TaskCode, UserId: userId, - SourceId: sourceId, }) if err != nil { return err @@ -93,18 +66,6 @@ func accomplish(userId int64, taskType string, sourceId string) error { return nil } -func isHandled(taskType string, sourceId string) (bool, error) { - _, err := models.GetTaskAccomplishLogBySourceIdAndTaskCode(sourceId, taskType) - if err != nil { - if models.IsErrRecordNotExist(err) { - return false, nil - } - return false, err - } - return true, nil - -} - func isLimited(userId int64, config *models.TaskConfig) bool { if err := limiter.CheckLimit(config.TaskCode, models.LimitTypeTask, userId, 1); err != nil { return true From 0562bee1c2fc68f34009577bb500cd3f5c758571 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 8 Jun 2022 15:07:44 +0800 Subject: [PATCH 07/90] #1249 update task trigger --- models/attachment.go | 8 ++ models/reward_operate_record.go | 19 +++- models/task_config.go | 45 ++++---- modules/auth/wechat/event_handle.go | 3 +- modules/cloudbrain/resty.go | 17 +-- modules/notification/action/action.go | 4 - modules/notification/base/notifier.go | 6 +- modules/notification/base/null.go | 14 ++- modules/notification/notification.go | 32 +++++- modules/notification/task/task.go | 56 +++++++++- routers/admin/dataset.go | 3 + routers/image/image.go | 2 + routers/repo/ai_model_manage.go | 1 - routers/repo/cloudbrain.go | 1 - routers/user/setting/profile.go | 2 + services/reward/limiter/limiter.go | 139 +++++++++++++++++++++---- services/reward/point/point_operate.go | 10 +- 17 files changed, 300 insertions(+), 62 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index ea8f1645f..0e4751ed2 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -653,3 +653,11 @@ func Attachments(opts *AttachmentsOptions) ([]*AttachmentInfo, int64, error) { return attachments, count, nil } + +func GetAllUserIdByDatasetId(datasetId int64) ([]int64, error) { + r := make([]int64, 0) + if err := x.Table("attachment").Where("dataset_id = ?", datasetId).Distinct("uploader_id").Find(&r); err != nil { + return nil, err + } + return r, nil +} diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 1577fbaff..b1b9983c3 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -2,6 +2,7 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" ) const ( @@ -26,8 +27,8 @@ func (r RewardType) Name() string { } const ( - OperateTypeIncrease = "INCREASE_POINT" - OperateTypeDecrease = "DECREASE_POINT" + OperateTypeIncrease = "INCREASE" + OperateTypeDecrease = "DECREASE" ) const ( @@ -82,6 +83,20 @@ func UpdateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus return x.Cols("status").Where("source_type=? and request_id=? and status=?", sourceType, requestId, oldStatus).Update(r) } +func SumRewardAmountInTaskPeriod(rewardType string, sourceType string, userId int64, period *PeriodResult) (int64, error) { + var cond = builder.NewCond() + if period != nil { + cond = cond.And(builder.Gte{"created_unix": period.StartTime.Unix()}) + cond = cond.And(builder.Lt{"created_unix": period.EndTime.Unix()}) + } + if sourceType != "" { + cond = cond.And(builder.Eq{"source_type": sourceType}) + } + cond = cond.And(builder.Eq{"reward_type": rewardType}) + cond = cond.And(builder.Eq{"user_id": userId}) + return x.Where(cond).SumInt(&RewardOperateRecord{}, "amount") +} + type RewardOperateContext struct { SourceType string SourceId string diff --git a/models/task_config.go b/models/task_config.go index fe2bb7721..c9e352ed0 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -5,24 +5,33 @@ import ( ) const ( - TaskTypeNewIssue = "NEW_ISSUE" - TaskTypeIssueChangeStatus = "ISSUE_CHANGE_STATUS" - TaskTypeCreateIssueComment = "CREATE_ISSUE_COMMENT" - TaskTypeNewPullRequest = "NEW_PULL_REQUEST" - TaskTypeRenameRepository = "RENAME_REPOSITORY" - TaskTypeAliasRepository = "ALIAS_REPOSITORY" - TaskTypeTransferRepository = "TRANSFER_REPOSITORY" - TaskTypeCreateRepository = "CREATE_REPOSITORY" - TaskTypeForkRepository = "FORK_REPOSITORY" - TaskTypePullRequestReview = "PULL_REQUEST_REVIEW" - TaskTypeCommentPull = "COMMENT_PULL" - TaskTypeApprovePullRequest = "APPROVE_PULL_REQUEST" - TaskTypeRejectPullRequest = "REJECT_PULL_REQUEST" - TaskTypeMergePullRequest = "MERGE_PULL_REQUEST" - TaskTypeSyncPushCommits = "SYNC_PUSH_COMMITS" - TaskTypeSyncCreateRef = "SYNC_CREATE_REF" - TaskTypeSyncDeleteRef = "SYNC_DELETE_REF" - TaskTypeBindWechat = "BIND_WECHAT" + TaskTypeNewIssue = "NEW_ISSUE" + TaskTypeIssueChangeStatus = "ISSUE_CHANGE_STATUS" + TaskTypeCreateIssueComment = "CREATE_ISSUE_COMMENT" + TaskTypeNewPullRequest = "NEW_PULL_REQUEST" + TaskTypeRenameRepository = "RENAME_REPOSITORY" + TaskTypeAliasRepository = "ALIAS_REPOSITORY" + TaskTypeTransferRepository = "TRANSFER_REPOSITORY" + TaskTypeCreateRepository = "CREATE_REPOSITORY" + TaskTypeCreatePublicRepository = "CREATE_PUBLIC_REPOSITORY" + TaskTypeForkRepository = "FORK_REPOSITORY" + TaskTypePullRequestReview = "PULL_REQUEST_REVIEW" + TaskTypeCommentPull = "COMMENT_PULL" + TaskTypeApprovePullRequest = "APPROVE_PULL_REQUEST" + TaskTypeRejectPullRequest = "REJECT_PULL_REQUEST" + TaskTypeMergePullRequest = "MERGE_PULL_REQUEST" + TaskTypeSyncPushCommits = "SYNC_PUSH_COMMITS" + TaskTypeSyncCreateRef = "SYNC_CREATE_REF" + TaskTypeSyncDeleteRef = "SYNC_DELETE_REF" + TaskTypeBindWechat = "BIND_WECHAT" + TaskTypeUploadAttachment = "UPLOAD_ATTACHMENT" + TaskTypeCreateCloudbrainTask = "CREATE_CLOUDBRAIN_TASK" + TaskTypeDatasetRecommended = "DATASET_RECOMMENDED" + TaskTypeCreateModel = "CREATE_MODEL" + TaskTypeCreatePublicImage = "CREATE_PUBLIC_IMAGE" + TaskTypeImageRecommend = "IMAGE_RECOMMEND" + TaskTypeChangeUserAvatar = "CHANGE_USER_AVATAR" + TaskTypePushCommits = "PUSH_COMMITS" ) const ( diff --git a/modules/auth/wechat/event_handle.go b/modules/auth/wechat/event_handle.go index b40ab3101..67c3a7265 100644 --- a/modules/auth/wechat/event_handle.go +++ b/modules/auth/wechat/event_handle.go @@ -1,6 +1,7 @@ package wechat import ( + "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "encoding/json" @@ -71,6 +72,6 @@ func HandleSubscribeEvent(we WechatEvent) string { jsonStr, _ := json.Marshal(qrCache) redis_client.Setex(redis_key.WechatBindingUserIdKey(sceneStr), string(jsonStr), 60*time.Second) } - + notification.NotifyWechatBind(qrCache.UserId, we.FromUserName) return BIND_REPLY_SUCCESS } diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index e70dbdd2b..75614e571 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -1,6 +1,7 @@ package cloudbrain import ( + "code.gitea.io/gitea/modules/notification" "encoding/json" "errors" "fmt" @@ -24,10 +25,10 @@ var ( ) const ( - JobHasBeenStopped = "S410" - Public = "public" - Custom = "custom" - LogPageSize = 500 + JobHasBeenStopped = "S410" + Public = "public" + Custom = "custom" + LogPageSize = 500 LogPageTokenExpired = "5m" pageSize = 15 ) @@ -313,6 +314,7 @@ sendjob: }) if err == nil { go updateImageStatus(image, isSetCreatedUnix, createTime) + notification.NotifyCreateImage(params.UID, image) } return err } @@ -354,6 +356,9 @@ func CommitAdminImage(params models.CommitImageParams) error { } return nil }) + if err == nil { + notification.NotifyCreateImage(params.UID, image) + } return err } @@ -474,7 +479,7 @@ func GetJobAllLog(scrollID string) (*models.GetJobLogResult, error) { client := getRestyClient() var result models.GetJobLogResult req := models.GetAllJobLogParams{ - Scroll: LogPageTokenExpired, + Scroll: LogPageTokenExpired, ScrollID: scrollID, } @@ -498,7 +503,7 @@ func GetJobAllLog(scrollID string) (*models.GetJobLogResult, error) { return &result, nil } -func DeleteJobLogToken(scrollID string) (error) { +func DeleteJobLogToken(scrollID string) error { checkSetting() client := getRestyClient() var result models.DeleteJobLogTokenResult diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index 943678a0b..2ac73c2c3 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -345,7 +345,3 @@ func (a *actionNotifier) NotifyOtherTask(doer *models.User, repo *models.Reposit log.Error("notifyWatchers: %v", err) } } - -func (a *actionNotifier) NotifyWechatBind(doer *models.User) { - return -} diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 26cd1feb8..c3c7f404a 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -56,5 +56,9 @@ type Notifier interface { NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) - NotifyWechatBind(doer *models.User) + NotifyWechatBind(userId int64, wechatOpenId string) + NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) + NotifyCreateImage(optUserId int64, image models.Image) + NotifyImageRecommend(optUser *models.User, imageId int64, action string) + NotifyChangeUserAvatar(user *models.User) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index ecdebd7a3..c0a224697 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -159,6 +159,18 @@ func (*NullNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, } -func (*NullNotifier) NotifyWechatBind(doer *models.User) { +func (*NullNotifier) NotifyWechatBind(userId int64, wechatOpenId string) { } + +func (*NullNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { +} + +func (*NullNotifier) NotifyCreateImage(optUserId int64, image models.Image) { +} + +func (*NullNotifier) NotifyImageRecommend(optUser *models.User, imageId int64, action string) { +} + +func (*NullNotifier) NotifyChangeUserAvatar(user *models.User) { +} diff --git a/modules/notification/notification.go b/modules/notification/notification.go index 2f0335c5a..118bdf994 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -273,8 +273,36 @@ func NotifySyncDeleteRef(pusher *models.User, repo *models.Repository, refType, } // NotifyWechatBind notifies wechat bind -func NotifyWechatBind(doer *models.User) { +func NotifyWechatBind(userId int64, wechatOpenId string) { for _, notifier := range notifiers { - notifier.NotifyWechatBind(doer) + notifier.NotifyWechatBind(userId, wechatOpenId) + } +} + +// NotifyDatasetRecommend +func NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { + for _, notifier := range notifiers { + notifier.NotifyDatasetRecommend(optUser, dataset, action) + } +} + +// NotifyDatasetRecommend +func NotifyCreateImage(optUserId int64, image models.Image) { + for _, notifier := range notifiers { + notifier.NotifyCreateImage(optUserId, image) + } +} + +// NotifyDatasetRecommend +func NotifyImageRecommend(optUser *models.User, imageId int64, action string) { + for _, notifier := range notifiers { + notifier.NotifyImageRecommend(optUser, imageId, action) + } +} + +// NotifyDatasetRecommend +func NotifyChangeUserAvatar(user *models.User) { + for _, notifier := range notifiers { + notifier.NotifyChangeUserAvatar(user) } } diff --git a/modules/notification/task/task.go b/modules/notification/task/task.go index f68872c01..077d6699b 100644 --- a/modules/notification/task/task.go +++ b/modules/notification/task/task.go @@ -53,7 +53,10 @@ func (t *taskNotifier) NotifyTransferRepository(doer *models.User, repo *models. } func (t *taskNotifier) NotifyCreateRepository(doer *models.User, u *models.User, repo *models.Repository) { - task.Accomplish(doer.ID, models.TaskTypeCreateRepository) + if !repo.IsPrivate { + task.Accomplish(doer.ID, models.TaskTypeCreatePublicRepository) + } + } func (t *taskNotifier) NotifyForkRepository(doer *models.User, oldRepo, repo *models.Repository) { @@ -99,9 +102,56 @@ func (t *taskNotifier) NotifySyncDeleteRef(doer *models.User, repo *models.Repos } func (t *taskNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) { + switch optype { + case models.ActionUploadAttachment: + task.Accomplish(doer.ID, models.TaskTypeUploadAttachment) + case models.ActionCreateDebugGPUTask, + models.ActionCreateDebugNPUTask, + models.ActionCreateTrainTask, + models.ActionCreateInferenceTask, + models.ActionCreateBenchMarkTask, + models.ActionCreateGPUTrainTask: + task.Accomplish(doer.ID, models.TaskTypeCreateCloudbrainTask) + case models.ActionCreateNewModelTask: + task.Accomplish(doer.ID, models.TaskTypeCreateModel) + } return } -func (t *taskNotifier) NotifyWechatBind(doer *models.User) { - task.Accomplish(doer.ID, models.TaskTypeSyncDeleteRef) +func (t *taskNotifier) NotifyWechatBind(userId int64, wechatOpenId string) { + task.Accomplish(userId, models.TaskTypeBindWechat) +} + +func (t *taskNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { + switch action { + case "recommend": + userIds, err := models.GetAllUserIdByDatasetId(dataset.ID) + if err != nil { + return + } + for _, userId := range userIds { + task.Accomplish(userId, models.TaskTypeDatasetRecommended) + } + } +} + +func (t *taskNotifier) NotifyCreateImage(optUserId int64, image models.Image) { + if !image.IsPrivate { + task.Accomplish(optUserId, models.TaskTypeCreatePublicImage) + } +} + +func (t *taskNotifier) NotifyImageRecommend(optUser *models.User, imageId int64, action string) { + switch action { + case "recommend": + task.Accomplish(optUser.ID, models.TaskTypeImageRecommend) + } +} + +func (t *taskNotifier) NotifyChangeUserAvatar(user *models.User) { + task.Accomplish(user.ID, models.TaskTypeChangeUserAvatar) +} + +func (t *taskNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { + task.Accomplish(pusher.ID, models.TaskTypePushCommits) } diff --git a/routers/admin/dataset.go b/routers/admin/dataset.go index 6b29b06ff..0eb5d27ab 100644 --- a/routers/admin/dataset.go +++ b/routers/admin/dataset.go @@ -1,6 +1,7 @@ package admin import ( + "code.gitea.io/gitea/modules/notification" "net/http" "strconv" "strings" @@ -106,6 +107,8 @@ func DatasetAction(ctx *context.Context) { if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action")))) } else { + d, _ := models.GetDatasetByID(datasetId) + notification.NotifyDatasetRecommend(ctx.User, d, ctx.Params(":action")) ctx.JSON(http.StatusOK, models.BaseOKMessage) } } diff --git a/routers/image/image.go b/routers/image/image.go index ae9912e3d..e238387ab 100644 --- a/routers/image/image.go +++ b/routers/image/image.go @@ -1,6 +1,7 @@ package image import ( + "code.gitea.io/gitea/modules/notification" "net/http" "strconv" @@ -25,6 +26,7 @@ func Action(ctx *context.Context) { if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action")))) } else { + notification.NotifyImageRecommend(ctx.User, imageId, ctx.Params(":action")) ctx.JSON(http.StatusOK, models.BaseOKMessage) } } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index e2040e0d2..3ad37f1f6 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -170,7 +170,6 @@ func SaveModel(ctx *context.Context) { ctx.Error(500, fmt.Sprintf("save model error. %v", err)) return } - log.Info("save model end.") } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a5dd52956..7ed6fa6ef 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -783,7 +783,6 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain return } - ctx.JSON(200, models.BaseOKMessage) } diff --git a/routers/user/setting/profile.go b/routers/user/setting/profile.go index 3333a8cc4..1c1e664d0 100755 --- a/routers/user/setting/profile.go +++ b/routers/user/setting/profile.go @@ -6,6 +6,7 @@ package setting import ( + "code.gitea.io/gitea/modules/notification" "errors" "fmt" "io/ioutil" @@ -165,6 +166,7 @@ func AvatarPost(ctx *context.Context, form auth.AvatarForm) { if err := UpdateAvatarSetting(ctx, form, ctx.User); err != nil { ctx.Flash.Error(err.Error()) } else { + notification.NotifyChangeUserAvatar(ctx.User) ctx.Flash.Success(ctx.Tr("settings.update_avatar_success")) } diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go index 04cef2e2c..fafaab9cb 100644 --- a/services/reward/limiter/limiter.go +++ b/services/reward/limiter/limiter.go @@ -12,56 +12,118 @@ import ( "time" ) +type limiterRejectPolicy string + +const ( + JustReject limiterRejectPolicy = "JUST_REJECT" + PermittedOnce limiterRejectPolicy = "PERMITTED_ONCE" + FillUp limiterRejectPolicy = "FillUp" +) + type limiterRunner struct { - limiters []models.LimitConfig - index int - userId int64 - amount int64 - limitCode string - limitType models.LimitType + limiters []models.LimitConfig + index int + userId int64 + amount int64 + limitCode string + limitType models.LimitType + rejectPolicy limiterRejectPolicy + resultMap map[int]limitResult + minRealAmount int64 +} + +type limitResult struct { + isLoss bool + planAmount int64 + realAmount int64 +} + +func newLimitResult(isLoss bool, planAmount int64, realAmount int64) limitResult { + return limitResult{ + isLoss: isLoss, + planAmount: planAmount, + realAmount: realAmount, + } } -func newLimiterRunner(limitCode string, limitType models.LimitType, userId, amount int64) *limiterRunner { +func newLimiterRunner(limitCode string, limitType models.LimitType, userId, amount int64, policy limiterRejectPolicy) *limiterRunner { return &limiterRunner{ - userId: userId, - amount: amount, - limitCode: limitCode, - limitType: limitType, - index: 0, + userId: userId, + amount: amount, + limitCode: limitCode, + limitType: limitType, + index: 0, + rejectPolicy: policy, + resultMap: make(map[int]limitResult, 0), } } +//Run run all limiters +//return real used amount(when choose the FillUp reject policy, amount may only be partially used) func (l *limiterRunner) Run() error { if err := l.LoadLimiters(); err != nil { return err } + l.minRealAmount = l.amount for l.index < len(l.limiters) { err := l.limit(l.limiters[l.index]) if err != nil { log.Info("limiter check failed,%v", err) - l.Rollback(l.index) + l.Rollback() return err } + result := l.resultMap[l.index] + if result.isLoss { + //find the minimum real amount + if l.minRealAmount > result.realAmount { + l.minRealAmount = result.realAmount + } + } l.index += 1 } + + //post process + l.PostProcess() return nil } //Rollback rollback the usedNum from limiters[0] to limiters[index] -func (l *limiterRunner) Rollback(index int) error { - for i := index; i >= 0; i-- { - l.rollback(l.limiters[i]) +func (l *limiterRunner) Rollback() error { + for i := l.index - 1; i >= 0; i-- { + l.rollback(l.limiters[i], l.resultMap[i]) + } + return nil +} + +func (l *limiterRunner) rollback(r models.LimitConfig, result limitResult) error { + p, err := period.GetPeriod(r.RefreshRate) + if err != nil { + return err + } + redisKey := redis_key.LimitCount(l.userId, r.LimitCode, r.LimitType, r.Scope, p) + redis_client.IncrBy(redisKey, -1*result.realAmount) + return nil +} + +//PostProcess process loss,if realAmount < planAmount +func (l *limiterRunner) PostProcess() error { + for i := l.index - 1; i >= 0; i-- { + l.postProcess(l.limiters[i], l.resultMap[i]) } return nil } -func (l *limiterRunner) rollback(r models.LimitConfig) error { +func (l *limiterRunner) postProcess(r models.LimitConfig, result limitResult) error { + if result.realAmount == l.minRealAmount { + return nil + } p, err := period.GetPeriod(r.RefreshRate) if err != nil { return err } + diff := result.realAmount - l.minRealAmount redisKey := redis_key.LimitCount(l.userId, r.LimitCode, r.LimitType, r.Scope, p) - redis_client.IncrBy(redisKey, -1*l.amount) + redis_client.IncrBy(redisKey, -1*diff) return nil } @@ -91,8 +153,25 @@ func (l *limiterRunner) limit(r models.LimitConfig) error { } } if usedNum > r.LimitNum { - return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) + if usedNum-r.LimitNum >= l.amount { + redis_client.IncrBy(redisKey, -1*l.amount) + return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) + } + switch l.rejectPolicy { + case FillUp: + exceed := usedNum - r.LimitNum + realAmount := l.amount - exceed + redis_client.IncrBy(redisKey, -1*exceed) + l.resultMap[l.index] = newLimitResult(true, l.amount, realAmount) + case JustReject: + redis_client.IncrBy(redisKey, -1*l.amount) + return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) + case PermittedOnce: + l.resultMap[l.index] = newLimitResult(false, l.amount, l.amount) + } + } + l.resultMap[l.index] = newLimitResult(false, l.amount, l.amount) return nil } @@ -111,15 +190,33 @@ func (l *limiterRunner) countInPeriod(r models.LimitConfig, p *models.PeriodResu switch r.LimitType { case models.LimitTypeTask.Name(): return models.CountTaskAccomplishLogInTaskPeriod(r.ID, l.userId, p) + case models.LimitTypeRewardPoint.Name(): + return models.SumRewardAmountInTaskPeriod(models.RewardTypePoint.Name(), r.LimitCode, l.userId, p) default: return 0, nil } } +func CheckLimitWithFillUp(limitCode string, limitType models.LimitType, userId, amount int64) (int64, error) { + r := newLimiterRunner(limitCode, limitType, userId, amount, FillUp) + err := r.Run() + if err != nil { + return 0, err + } + return r.minRealAmount, nil +} + +func CheckLimitWithPermittedOnce(limitCode string, limitType models.LimitType, userId, amount int64) error { + r := newLimiterRunner(limitCode, limitType, userId, amount, PermittedOnce) + err := r.Run() + return err +} + func CheckLimit(limitCode string, limitType models.LimitType, userId, amount int64) error { - r := newLimiterRunner(limitCode, limitType, userId, amount) - return r.Run() + r := newLimiterRunner(limitCode, limitType, userId, amount, JustReject) + err := r.Run() + return err } func GetLimiters(limitCode string, limitType models.LimitType) ([]models.LimitConfig, error) { diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 80b0b4fe9..eaebdf764 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -8,16 +8,24 @@ import ( "code.gitea.io/gitea/services/reward/limiter" "code.gitea.io/gitea/services/reward/point/account" "errors" + "fmt" "time" ) +const LossMsg = "达到奖励上限，应得%d积分，实得%d积分" + type PointOperator struct { } func (operator *PointOperator) IsLimited(ctx models.RewardOperateContext) bool { - if err := limiter.CheckLimit(ctx.SourceType, models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount); err != nil { + realAmount, err := limiter.CheckLimitWithFillUp(ctx.SourceType, models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount) + if err != nil { return true } + if realAmount < ctx.Reward.Amount { + ctx.Remark = ctx.Remark + ";" + fmt.Sprintf(LossMsg, ctx.Reward.Amount, realAmount) + ctx.Reward.Amount = realAmount + } return false } From f606783d568b7ec8d205468af757c46df1aaaa1d Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 8 Jun 2022 16:42:06 +0800 Subject: [PATCH 08/90] #1249 add task config query api add point account query api --- models/limit_config.go | 13 ++++- models/task_config.go | 28 ++++++++-- modules/redis/redis_key/task_redis_key.go | 7 ++- routers/response/response.go | 2 +- routers/reward/point/point.go | 31 +++++++++++ routers/routes/routes.go | 10 ++++ routers/task/config.go | 17 ++++++ services/task/task_config.go | 64 +++++++++++++++++++++-- 8 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 routers/reward/point/point.go create mode 100644 routers/task/config.go diff --git a/models/limit_config.go b/models/limit_config.go index aec26a036..75dccac9a 100644 --- a/models/limit_config.go +++ b/models/limit_config.go @@ -45,12 +45,21 @@ type LimitConfig struct { Scope string `xorm:"NOT NULL"` LimitNum int64 `xorm:"NOT NULL"` LimitCode string - LimitType string `xorm:"NOT NULL"` - Creator int64 `xorm:"NOT NULL"` + LimitType string `xorm:"NOT NULL"` + CreatorId int64 `xorm:"NOT NULL"` + CreatorName string CreatedUnix timeutil.TimeStamp `xorm:"created"` DeletedAt timeutil.TimeStamp `xorm:"deleted"` } +type LimitConfigVO struct { + RefreshRate string + Scope string + LimitNum int64 + Creator string + CreatedUnix timeutil.TimeStamp +} + func GetLimitConfigByLimitType(limitType LimitType) ([]LimitConfig, error) { r := make([]LimitConfig, 0) err := x.Where(" limit_type = ?", limitType.Name()).Find(&r) diff --git a/models/task_config.go b/models/task_config.go index c9e352ed0..eee5caea5 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -45,13 +45,24 @@ type TaskConfig struct { ID int64 `xorm:"pk autoincr"` TaskCode string `xorm:"NOT NULL"` Tittle string - AwardType string `xorm:"NOT NULL"` - AwardAmount int64 `xorm:"NOT NULL"` - Creator int64 `xorm:"NOT NULL"` + AwardType string `xorm:"NOT NULL"` + AwardAmount int64 `xorm:"NOT NULL"` + CreatorId int64 `xorm:"NOT NULL"` + CreatorName string CreatedUnix timeutil.TimeStamp `xorm:"created"` DeletedAt timeutil.TimeStamp `xorm:"deleted"` } +type TaskConfigWithLimit struct { + TaskCode string + Tittle string + AwardType string + AwardAmount int64 + Creator string + CreatedUnix timeutil.TimeStamp + Limiters []LimitConfigVO +} + func getTaskConfig(t *TaskConfig) (*TaskConfig, error) { has, err := x.Get(t) if err != nil { @@ -68,3 +79,14 @@ func GetTaskConfigByTaskCode(taskCode string) (*TaskConfig, error) { } return getTaskConfig(t) } +func GetTaskConfigList() ([]*TaskConfig, error) { + r := make([]*TaskConfig, 0) + err := x.Find(&r) + if err != nil { + return nil, err + } + if len(r) == 0 { + return nil, ErrRecordNotExist{} + } + return r, nil +} diff --git a/modules/redis/redis_key/task_redis_key.go b/modules/redis/redis_key/task_redis_key.go index 3427c8f7f..8d6fb3f6e 100644 --- a/modules/redis/redis_key/task_redis_key.go +++ b/modules/redis/redis_key/task_redis_key.go @@ -6,6 +6,9 @@ func TaskAccomplishLock(sourceId string, taskType string) string { return KeyJoin(TASK_REDIS_PREFIX, sourceId, taskType, "accomplish") } -func TaskConfig(taskType string) string { - return KeyJoin(TASK_REDIS_PREFIX, "config", taskType) +func TaskConfigList() string { + return KeyJoin(TASK_REDIS_PREFIX, "config", "list") +} +func TaskConfigWithLimiterList() string { + return KeyJoin(TASK_REDIS_PREFIX, "config", "limiter", "list") } diff --git a/routers/response/response.go b/routers/response/response.go index edd3b9cca..e87471d4c 100644 --- a/routers/response/response.go +++ b/routers/response/response.go @@ -25,7 +25,7 @@ func ServerError(msg string) *AiforgeResponse { } func SuccessWithData(data interface{}) *AiforgeResponse { - return &AiforgeResponse{Code: RESPONSE_CODE_ERROR_DEFAULT, Msg: RESPONSE_MSG_SUCCESS, Data: data} + return &AiforgeResponse{Code: RESPONSE_CODE_SUCCESS, Msg: RESPONSE_MSG_SUCCESS, Data: data} } func ErrorWithData(code int, msg string, data interface{}) *AiforgeResponse { return &AiforgeResponse{Code: code, Msg: msg, Data: data} diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go new file mode 100644 index 000000000..eaae76c4f --- /dev/null +++ b/routers/reward/point/point.go @@ -0,0 +1,31 @@ +package point + +import ( + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/reward/point/account" + "net/http" +) + +type AccountResponse struct { + AccountCode string + Balance int64 + TotalEarned int64 + TotalConsumed int64 +} + +func GetPointAccount(ctx *context.Context) { + userId := ctx.User.ID + a, err := account.GetAccount(userId) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + res := &AccountResponse{ + AccountCode: a.AccountCode, + Balance: a.Balance, + TotalEarned: a.TotalEarned, + TotalConsumed: a.TotalConsumed, + } + ctx.JSON(http.StatusOK, response.SuccessWithData(res)) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 4c3f5f472..b2393246c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,6 +6,8 @@ package routes import ( "bytes" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/routers/task" "encoding/gob" "net/http" "path" @@ -1314,6 +1316,14 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/purge", user.NotificationPurgePost) }, reqSignIn) + m.Group("/reward/point", func() { + m.Get("/account", point.GetPointAccount) + }, reqSignIn) + + m.Group("/task/config", func() { + m.Get("/list", task.GetTaskConfigList) + }, reqSignIn) + if setting.API.EnableSwagger { m.Get("/swagger.v1.json", templates.JSONRenderer(), routers.SwaggerV1Json) } diff --git a/routers/task/config.go b/routers/task/config.go new file mode 100644 index 000000000..95db0b7d8 --- /dev/null +++ b/routers/task/config.go @@ -0,0 +1,17 @@ +package task + +import ( + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/task" + "net/http" +) + +func GetTaskConfigList(ctx *context.Context) { + r, err := task.GetTaskConfigWithLimitList() + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ctx.JSON(http.StatusOK, response.SuccessWithData(r)) +} diff --git a/services/task/task_config.go b/services/task/task_config.go index 6e7f22e14..3f2225d2c 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -4,6 +4,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/services/reward/limiter" "encoding/json" "time" ) @@ -11,17 +12,30 @@ import ( //GetTaskConfig get task config from redis cache first // if not exist in redis, find in db and refresh the redis key func GetTaskConfig(taskType string) (*models.TaskConfig, error) { - redisKey := redis_key.TaskConfig(taskType) + list, err := GetTaskConfigList() + if err != nil { + return nil, err + } + for _, v := range list { + if v.TaskCode == taskType { + return v, nil + } + } + return nil, nil +} + +func GetTaskConfigList() ([]*models.TaskConfig, error) { + redisKey := redis_key.TaskConfigList() configStr, _ := redis_client.Get(redisKey) if configStr != "" { if configStr == redis_key.EMPTY_REDIS_VAL { return nil, nil } - config := new(models.TaskConfig) - json.Unmarshal([]byte(configStr), config) + config := make([]*models.TaskConfig, 0) + json.Unmarshal([]byte(configStr), &config) return config, nil } - config, err := models.GetTaskConfigByTaskCode(taskType) + config, err := models.GetTaskConfigList() if err != nil { if models.IsErrRecordNotExist(err) { redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) @@ -33,3 +47,45 @@ func GetTaskConfig(taskType string) (*models.TaskConfig, error) { redis_client.Setex(redisKey, string(jsonStr), 30*24*time.Hour) return config, nil } + +func GetTaskConfigWithLimitList() ([]*models.TaskConfigWithLimit, error) { + list, err := GetTaskConfigList() + if err != nil { + return nil, err + } + if len(list) == 0 { + return nil, nil + } + r := make([]*models.TaskConfigWithLimit, 0) + l, err := limiter.GetLimitersByLimitType(models.LimitTypeTask) + if err != nil { + return nil, err + } + for i := 0; i < len(list); i++ { + li := list[i] + t := &models.TaskConfigWithLimit{ + TaskCode: li.TaskCode, + Tittle: li.Tittle, + AwardType: li.AwardType, + AwardAmount: li.AwardAmount, + Creator: li.CreatorName, + CreatedUnix: li.CreatedUnix, + } + lv := make([]models.LimitConfigVO, 0) + for j := 0; j < len(l); j++ { + lj := l[j] + if lj.LimitCode == li.TaskCode { + lv = append(lv, models.LimitConfigVO{ + RefreshRate: lj.RefreshRate, + Scope: lj.Scope, + LimitNum: lj.LimitNum, + Creator: lj.CreatorName, + CreatedUnix: lj.CreatedUnix, + }) + } + } + t.Limiters = lv + r = append(r, t) + } + return r, nil +} From 40f7620a3413daa2cb6a030685643078eb71effb Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 9 Jun 2022 16:58:03 +0800 Subject: [PATCH 09/90] #1249 add task config edit api --- models/limit_config.go | 1 + models/task_config.go | 69 ++++++++++++++++++++++- modules/redis/redis_key/task_redis_key.go | 3 - routers/routes/routes.go | 1 + routers/task/config.go | 10 ++++ services/task/task_config.go | 13 +++++ 6 files changed, 92 insertions(+), 5 deletions(-) diff --git a/models/limit_config.go b/models/limit_config.go index 75dccac9a..154c13ed8 100644 --- a/models/limit_config.go +++ b/models/limit_config.go @@ -53,6 +53,7 @@ type LimitConfig struct { } type LimitConfigVO struct { + Tittle string RefreshRate string Scope string LimitNum int64 diff --git a/models/task_config.go b/models/task_config.go index eee5caea5..44a3bea32 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -54,10 +54,10 @@ type TaskConfig struct { } type TaskConfigWithLimit struct { - TaskCode string + TaskCode string `binding:"Required;MaxSize(256)"` Tittle string AwardType string - AwardAmount int64 + AwardAmount int64 `binding:"Required;MaxSize(256)"` Creator string CreatedUnix timeutil.TimeStamp Limiters []LimitConfigVO @@ -90,3 +90,68 @@ func GetTaskConfigList() ([]*TaskConfig, error) { } return r, nil } + +func AddTaskConfig(config TaskConfigWithLimit, doer *User) error { + sess := x.NewSession() + defer sess.Close() + + //delete old task config + p := &TaskConfig{ + TaskCode: config.TaskCode, + } + _, err := sess.Delete(p) + if err != nil { + sess.Rollback() + return err + } + + //add new config + t := &TaskConfig{ + TaskCode: config.TaskCode, + Tittle: config.Tittle, + AwardType: config.AwardType, + AwardAmount: config.AwardAmount, + CreatorId: doer.ID, + CreatorName: doer.Name, + } + _, err = sess.Insert(t) + if err != nil { + sess.Rollback() + return err + } + + //delete old limiter config + lp := &LimitConfig{ + LimitType: LimitTypeTask.Name(), + LimitCode: config.TaskCode, + } + _, err = sess.Delete(lp) + if err != nil { + sess.Rollback() + return err + } + + //add new limiter config + if config.Limiters != nil && len(config.Limiters) > 0 { + for _, v := range config.Limiters { + //add new config + l := &LimitConfig{ + Tittle: v.Tittle, + RefreshRate: v.RefreshRate, + Scope: v.Scope, + LimitNum: v.LimitNum, + LimitCode: config.TaskCode, + LimitType: LimitTypeTask.Name(), + CreatorId: doer.ID, + CreatorName: doer.Name, + } + _, err = sess.Insert(l) + if err != nil { + sess.Rollback() + return err + } + } + } + sess.Commit() + return nil +} diff --git a/modules/redis/redis_key/task_redis_key.go b/modules/redis/redis_key/task_redis_key.go index 8d6fb3f6e..4e30688e1 100644 --- a/modules/redis/redis_key/task_redis_key.go +++ b/modules/redis/redis_key/task_redis_key.go @@ -9,6 +9,3 @@ func TaskAccomplishLock(sourceId string, taskType string) string { func TaskConfigList() string { return KeyJoin(TASK_REDIS_PREFIX, "config", "list") } -func TaskConfigWithLimiterList() string { - return KeyJoin(TASK_REDIS_PREFIX, "config", "limiter", "list") -} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index b2393246c..135f4d702 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1322,6 +1322,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/task/config", func() { m.Get("/list", task.GetTaskConfigList) + m.Post("/add", bindIgnErr(models.TaskConfigWithLimit{}), task.AddTaskConfig) }, reqSignIn) if setting.API.EnableSwagger { diff --git a/routers/task/config.go b/routers/task/config.go index 95db0b7d8..d92d0fb51 100644 --- a/routers/task/config.go +++ b/routers/task/config.go @@ -1,6 +1,7 @@ package task import ( + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/routers/response" "code.gitea.io/gitea/services/task" @@ -15,3 +16,12 @@ func GetTaskConfigList(ctx *context.Context) { } ctx.JSON(http.StatusOK, response.SuccessWithData(r)) } + +func AddTaskConfig(ctx *context.Context, config models.TaskConfigWithLimit) { + err := task.AddTaskConfig(config, ctx.User) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ctx.JSON(http.StatusOK, response.Success()) +} diff --git a/services/task/task_config.go b/services/task/task_config.go index 3f2225d2c..fe50647e3 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -2,6 +2,7 @@ package task import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/services/reward/limiter" @@ -76,6 +77,7 @@ func GetTaskConfigWithLimitList() ([]*models.TaskConfigWithLimit, error) { lj := l[j] if lj.LimitCode == li.TaskCode { lv = append(lv, models.LimitConfigVO{ + Tittle: lj.Tittle, RefreshRate: lj.RefreshRate, Scope: lj.Scope, LimitNum: lj.LimitNum, @@ -89,3 +91,14 @@ func GetTaskConfigWithLimitList() ([]*models.TaskConfigWithLimit, error) { } return r, nil } + +func AddTaskConfig(config models.TaskConfigWithLimit, doer *models.User) error { + err := models.AddTaskConfig(config, doer) + if err != nil { + log.Error("add task config error,config:%v err:%v", config, err) + return err + } + redis_client.Del(redis_key.LimitConfig(models.LimitTypeTask)) + redis_client.Del(redis_key.TaskConfigList()) + return nil +} From 2122fbab89f2769fa17b907146d91ca3000638a9 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 10 Jun 2022 10:01:04 +0800 Subject: [PATCH 10/90] #1249 add limiter config edit api --- models/limit_config.go | 51 ++++++++++++++++++++++++++++++- models/task_config.go | 2 +- routers/reward/point/limit.go | 27 ++++++++++++++++ routers/routes/routes.go | 2 ++ services/reward/limiter/config.go | 41 +++++++++++++++++++++++++ services/task/task_config.go | 11 ++----- 6 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 routers/reward/point/limit.go create mode 100644 services/reward/limiter/config.go diff --git a/models/limit_config.go b/models/limit_config.go index 154c13ed8..62ff3bfbe 100644 --- a/models/limit_config.go +++ b/models/limit_config.go @@ -1,6 +1,9 @@ package models -import "code.gitea.io/gitea/modules/timeutil" +import ( + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" +) type LimitType string @@ -57,10 +60,23 @@ type LimitConfigVO struct { RefreshRate string Scope string LimitNum int64 + LimitCode string Creator string CreatedUnix timeutil.TimeStamp } +func (l *LimitConfig) ToLimitConfigVO() *LimitConfigVO { + return &LimitConfigVO{ + Tittle: l.Tittle, + RefreshRate: l.RefreshRate, + Scope: l.Scope, + LimitNum: l.LimitNum, + LimitCode: l.LimitCode, + Creator: l.CreatorName, + CreatedUnix: l.CreatedUnix, + } +} + func GetLimitConfigByLimitType(limitType LimitType) ([]LimitConfig, error) { r := make([]LimitConfig, 0) err := x.Where(" limit_type = ?", limitType.Name()).Find(&r) @@ -71,3 +87,36 @@ func GetLimitConfigByLimitType(limitType LimitType) ([]LimitConfig, error) { } return r, nil } + +func AddLimitConfig(l *LimitConfig) error { + sess := x.NewSession() + defer sess.Close() + + //delete old limit config + cond := builder.NewCond() + cond = cond.And(builder.Eq{"limit_type": l.LimitType}) + cond = cond.And(builder.Eq{"scope": l.Scope}) + if l.LimitCode == "" { + subCond := builder.NewCond() + subCond = subCond.Or(builder.IsNull{"limit_code"}) + subCond = subCond.Or(builder.Eq{"limit_code": ""}) + cond = cond.And(subCond) + } else { + cond = cond.And(builder.Eq{"limit_code": l.LimitCode}) + } + _, err := sess.Where(cond).Delete(&LimitConfig{}) + if err != nil { + sess.Rollback() + return err + } + + //add new config + _, err = sess.Insert(l) + if err != nil { + sess.Rollback() + return err + } + + sess.Commit() + return nil +} diff --git a/models/task_config.go b/models/task_config.go index 44a3bea32..922273c46 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -60,7 +60,7 @@ type TaskConfigWithLimit struct { AwardAmount int64 `binding:"Required;MaxSize(256)"` Creator string CreatedUnix timeutil.TimeStamp - Limiters []LimitConfigVO + Limiters []*LimitConfigVO } func getTaskConfig(t *TaskConfig) (*TaskConfig, error) { diff --git a/routers/reward/point/limit.go b/routers/reward/point/limit.go new file mode 100644 index 000000000..a831169f8 --- /dev/null +++ b/routers/reward/point/limit.go @@ -0,0 +1,27 @@ +package point + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/reward/limiter" + "net/http" +) + +func GetPointLimitConfigList(ctx *context.Context) { + r, err := limiter.GetLimitConfigList(models.LimitTypeRewardPoint) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ctx.JSON(http.StatusOK, response.SuccessWithData(r)) +} + +func AddPointLimitConfig(ctx *context.Context, config models.LimitConfigVO) { + err := limiter.AddLimitConfig(&config, ctx.User, models.LimitTypeRewardPoint) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ctx.JSON(http.StatusOK, response.Success()) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 135f4d702..89416ba16 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1318,6 +1318,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/reward/point", func() { m.Get("/account", point.GetPointAccount) + m.Get("/limiter/list", point.GetPointLimitConfigList) + m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) }, reqSignIn) m.Group("/task/config", func() { diff --git a/services/reward/limiter/config.go b/services/reward/limiter/config.go new file mode 100644 index 000000000..12204b2c5 --- /dev/null +++ b/services/reward/limiter/config.go @@ -0,0 +1,41 @@ +package limiter + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" +) + +func GetLimitConfigList(limitType models.LimitType) ([]*models.LimitConfigVO, error) { + r, err := GetLimitersByLimitType(limitType) + if err != nil { + return nil, err + } + result := make([]*models.LimitConfigVO, 0) + for _, v := range r { + result = append(result, v.ToLimitConfigVO()) + } + return result, nil +} + +func AddLimitConfig(config *models.LimitConfigVO, doer *models.User, limitType models.LimitType) error { + r := &models.LimitConfig{ + Tittle: config.Tittle, + RefreshRate: config.RefreshRate, + Scope: config.Scope, + LimitNum: config.LimitNum, + LimitCode: config.LimitCode, + LimitType: limitType.Name(), + CreatorId: doer.ID, + CreatorName: doer.Name, + } + err := models.AddLimitConfig(r) + + if err != nil { + log.Error("add limit config error,config:%v err:%v", config, err) + return err + } + redis_client.Del(redis_key.LimitConfig(limitType)) + return nil +} diff --git a/services/task/task_config.go b/services/task/task_config.go index fe50647e3..0001edc21 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -72,18 +72,11 @@ func GetTaskConfigWithLimitList() ([]*models.TaskConfigWithLimit, error) { Creator: li.CreatorName, CreatedUnix: li.CreatedUnix, } - lv := make([]models.LimitConfigVO, 0) + lv := make([]*models.LimitConfigVO, 0) for j := 0; j < len(l); j++ { lj := l[j] if lj.LimitCode == li.TaskCode { - lv = append(lv, models.LimitConfigVO{ - Tittle: lj.Tittle, - RefreshRate: lj.RefreshRate, - Scope: lj.Scope, - LimitNum: lj.LimitNum, - Creator: lj.CreatorName, - CreatedUnix: lj.CreatedUnix, - }) + lv = append(lv, lj.ToLimitConfigVO()) } } t.Limiters = lv From 5a9d544003671898333f672a3760f4d3ff58a658 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 10 Jun 2022 11:48:26 +0800 Subject: [PATCH 11/90] #1249 add batch config api --- models/task_config.go | 3 +++ routers/routes/routes.go | 18 +++++++++++------- routers/task/config.go | 16 ++++++++++++++++ services/task/task_config.go | 4 ++++ 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/models/task_config.go b/models/task_config.go index 922273c46..cd4329834 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -62,6 +62,9 @@ type TaskConfigWithLimit struct { CreatedUnix timeutil.TimeStamp Limiters []*LimitConfigVO } +type BatchLimitConfigVO struct { + ConfigList []TaskConfigWithLimit +} func getTaskConfig(t *TaskConfig) (*TaskConfig, error) { has, err := x.Get(t) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 89416ba16..32d0a55af 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -589,6 +589,17 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/delete", admin.DeleteNotices) m.Post("/empty", admin.EmptyNotices) }) + + m.Group("/reward/point", func() { + m.Get("/limiter/list", point.GetPointLimitConfigList) + m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) + }) + + m.Group("/task/config", func() { + m.Get("/list", task.GetTaskConfigList) + m.Post("/add", bindIgnErr(models.TaskConfigWithLimit{}), task.AddTaskConfig) + m.Post("/add/batch", bindIgnErr(models.BatchLimitConfigVO{}), task.BatchAddTaskConfig) + }) }, adminReq) // ***** END: Admin ***** @@ -1318,13 +1329,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/reward/point", func() { m.Get("/account", point.GetPointAccount) - m.Get("/limiter/list", point.GetPointLimitConfigList) - m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) - }, reqSignIn) - - m.Group("/task/config", func() { - m.Get("/list", task.GetTaskConfigList) - m.Post("/add", bindIgnErr(models.TaskConfigWithLimit{}), task.AddTaskConfig) }, reqSignIn) if setting.API.EnableSwagger { diff --git a/routers/task/config.go b/routers/task/config.go index d92d0fb51..0216ffea0 100644 --- a/routers/task/config.go +++ b/routers/task/config.go @@ -25,3 +25,19 @@ func AddTaskConfig(ctx *context.Context, config models.TaskConfigWithLimit) { } ctx.JSON(http.StatusOK, response.Success()) } +func BatchAddTaskConfig(ctx *context.Context, list models.BatchLimitConfigVO) { + successCount := 0 + failCount := 0 + for _, config := range list.ConfigList { + err := task.AddTaskConfig(config, ctx.User) + if err != nil { + failCount++ + } else { + successCount++ + } + } + r := make(map[string]int, 2) + r["successCount"] = successCount + r["failCount"] = failCount + ctx.JSON(http.StatusOK, response.SuccessWithData(r)) +} diff --git a/services/task/task_config.go b/services/task/task_config.go index 0001edc21..4e02b4972 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -7,6 +7,7 @@ import ( "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/services/reward/limiter" "encoding/json" + "errors" "time" ) @@ -86,6 +87,9 @@ func GetTaskConfigWithLimitList() ([]*models.TaskConfigWithLimit, error) { } func AddTaskConfig(config models.TaskConfigWithLimit, doer *models.User) error { + if config.TaskCode == "" || config.AwardType == "" { + return errors.New("param error") + } err := models.AddTaskConfig(config, doer) if err != nil { log.Error("add task config error,config:%v err:%v", config, err) From f605544640e0f9d157665558dd64b09ba20c8e42 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 16 Jun 2022 17:57:35 +0800 Subject: [PATCH 12/90] #2225 add reward operate notification --- models/reward_operate_record.go | 66 +++++++++++++++++++-- modules/eventsource/manager_run.go | 20 +++++++ modules/redis/redis_client/client.go | 37 ++++++++++++ modules/redis/redis_key/reward_redis_key.go | 4 ++ modules/setting/setting.go | 27 +++++---- routers/routes/routes.go | 1 + services/reward/limiter/limiter.go | 2 + services/reward/notify.go | 47 +++++++++++++++ services/reward/operator.go | 28 ++++++--- services/reward/point/point_operate.go | 7 ++- services/task/task.go | 4 +- web_src/js/features/notification.js | 7 +++ 12 files changed, 221 insertions(+), 29 deletions(-) create mode 100644 services/reward/notify.go diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index b1b9983c3..4c31df03c 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -2,6 +2,7 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" + "strings" "xorm.io/builder" ) @@ -25,10 +26,49 @@ func (r RewardType) Name() string { return "" } } +func (r RewardType) Show() string { + switch r { + case RewardTypePoint: + return "积分" + default: + return "" + } +} +func GetRewardTypeInstance(s string) RewardType { + switch s { + case RewardTypePoint.Name(): + return RewardTypePoint + default: + return "" + } +} + +type RewardOperateType string + +func (r RewardOperateType) Name() string { + switch r { + case OperateTypeIncrease: + return "INCREASE" + case OperateTypeDecrease: + return "DECREASE" + default: + return "" + } +} +func (r RewardOperateType) Show() string { + switch r { + case OperateTypeIncrease: + return "奖励" + case OperateTypeDecrease: + return "扣减" + default: + return "" + } +} const ( - OperateTypeIncrease = "INCREASE" - OperateTypeDecrease = "DECREASE" + OperateTypeIncrease RewardOperateType = "INCREASE" + OperateTypeDecrease RewardOperateType = "DECREASE" ) const ( @@ -37,6 +77,8 @@ const ( OperateStatusFailed = "FAILED" ) +const Semicolon = ";" + type RewardOperateRecord struct { ID int64 `xorm:"pk autoincr"` RecordId string `xorm:"INDEX NOT NULL"` @@ -104,11 +146,27 @@ type RewardOperateContext struct { Reward Reward TargetUserId int64 RequestId string - OperateType string + OperateType RewardOperateType CycleIntervalSeconds int64 } type Reward struct { Amount int64 - Type string + Type RewardType +} + +type UserRewardOperationRedis struct { + UserId int64 + Amount int64 + RewardType RewardType + OperateType RewardOperateType +} + +type UserRewardOperation struct { + UserId int64 + Msg string +} + +func AppendRemark(remark, appendStr string) string { + return strings.TrimPrefix(remark+Semicolon+appendStr, Semicolon) } diff --git a/modules/eventsource/manager_run.go b/modules/eventsource/manager_run.go index 75d3ee5b0..857eaee22 100644 --- a/modules/eventsource/manager_run.go +++ b/modules/eventsource/manager_run.go @@ -5,6 +5,7 @@ package eventsource import ( + "code.gitea.io/gitea/services/reward" "context" "time" @@ -24,8 +25,26 @@ func (m *Manager) Init() { func (m *Manager) Run(ctx context.Context) { then := timeutil.TimeStampNow().Add(-2) timer := time.NewTicker(setting.UI.Notification.EventSourceUpdateTime) + rewardThen := then + rewardTimer := time.NewTicker(setting.UI.Notification.RewardNotifyUpdateTime) loop: for { + select { + case <-rewardTimer.C: + now := timeutil.TimeStampNow().Add(-2) + list := reward.GetRewardOperation(rewardThen, now) + if list != nil { + for _, l := range list { + m.SendMessage(l.UserId, &Event{ + Name: "reward-operation", + Data: l.Msg, + }) + } + } + + rewardThen = now + } + select { case <-ctx.Done(): timer.Stop() @@ -44,6 +63,7 @@ loop: }) } then = now + default: } } m.UnregisterAll() diff --git a/modules/redis/redis_client/client.go b/modules/redis/redis_client/client.go index 21a6da9fb..c5cb936b3 100644 --- a/modules/redis/redis_client/client.go +++ b/modules/redis/redis_client/client.go @@ -130,3 +130,40 @@ func GetInt64(key string) (bool, int64, error) { return true, i, nil } + +func ZAdd(key, value string, score float64) error { + redisClient := labelmsg.Get() + defer redisClient.Close() + + _, err := redisClient.Do("ZADD", key, score, value) + if err != nil { + return err + } + return nil +} + +func ZRangeByScore(key string, min, max float64) ([]string, error) { + redisClient := labelmsg.Get() + defer redisClient.Close() + + reply, err := redisClient.Do("ZRANGEBYSCORE", key, min, max) + if err != nil { + return nil, err + } + if reply == nil { + return nil, err + } + s, _ := redis.Strings(reply, nil) + return s, nil +} + +func ZRemRangeByScore(key string, min, max float64) error { + redisClient := labelmsg.Get() + defer redisClient.Close() + + _, err := redisClient.Do("ZREMRANGEBYSCORE", key, min, max) + if err != nil { + return err + } + return nil +} diff --git a/modules/redis/redis_key/reward_redis_key.go b/modules/redis/redis_key/reward_redis_key.go index df8c0ca16..add304db4 100644 --- a/modules/redis/redis_key/reward_redis_key.go +++ b/modules/redis/redis_key/reward_redis_key.go @@ -5,3 +5,7 @@ const REWARD_REDIS_PREFIX = "reward" func RewardSendLock(requestId string, sourceType string) string { return KeyJoin(REWARD_REDIS_PREFIX, requestId, sourceType, "send") } + +func RewardOperateNotification() string { + return KeyJoin(REWARD_REDIS_PREFIX, "operate", "notification") +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5c87b68c5..595c51286 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -184,10 +184,11 @@ var ( UseServiceWorker bool Notification struct { - MinTimeout time.Duration - TimeoutStep time.Duration - MaxTimeout time.Duration - EventSourceUpdateTime time.Duration + MinTimeout time.Duration + TimeoutStep time.Duration + MaxTimeout time.Duration + EventSourceUpdateTime time.Duration + RewardNotifyUpdateTime time.Duration } `ini:"ui.notification"` Admin struct { @@ -221,15 +222,17 @@ var ( Themes: []string{`gitea`, `arc-green`}, Reactions: []string{`+1`, `-1`, `laugh`, `hooray`, `confused`, `heart`, `rocket`, `eyes`}, Notification: struct { - MinTimeout time.Duration - TimeoutStep time.Duration - MaxTimeout time.Duration - EventSourceUpdateTime time.Duration + MinTimeout time.Duration + TimeoutStep time.Duration + MaxTimeout time.Duration + EventSourceUpdateTime time.Duration + RewardNotifyUpdateTime time.Duration }{ - MinTimeout: 10 * time.Second, - TimeoutStep: 10 * time.Second, - MaxTimeout: 60 * time.Second, - EventSourceUpdateTime: 10 * time.Second, + MinTimeout: 10 * time.Second, + TimeoutStep: 10 * time.Second, + MaxTimeout: 60 * time.Second, + EventSourceUpdateTime: 10 * time.Second, + RewardNotifyUpdateTime: 3 * time.Second, }, Admin: struct { UserPagingNum int diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 32d0a55af..31075742c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -325,6 +325,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/dashboard", routers.Dashboard) go routers.SocketManager.Run() m.Get("/action/notification", routers.ActionNotification) + m.Get("/reward/notification", routers.ActionNotification) m.Get("/recommend/org", routers.RecommendOrgFromPromote) m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Get("/recommend/userrank/:index", routers.GetUserRankFromPromote) diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go index fafaab9cb..f094e3a43 100644 --- a/services/reward/limiter/limiter.go +++ b/services/reward/limiter/limiter.go @@ -163,11 +163,13 @@ func (l *limiterRunner) limit(r models.LimitConfig) error { realAmount := l.amount - exceed redis_client.IncrBy(redisKey, -1*exceed) l.resultMap[l.index] = newLimitResult(true, l.amount, realAmount) + return nil case JustReject: redis_client.IncrBy(redisKey, -1*l.amount) return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) case PermittedOnce: l.resultMap[l.index] = newLimitResult(false, l.amount, l.amount) + return nil } } diff --git a/services/reward/notify.go b/services/reward/notify.go new file mode 100644 index 000000000..aa18fbe39 --- /dev/null +++ b/services/reward/notify.go @@ -0,0 +1,47 @@ +package reward + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/timeutil" + "encoding/json" + "fmt" + "time" +) + +func NotifyRewardOperation(userId, amount int64, rewardType models.RewardType, operateType models.RewardOperateType) { + data := &models.UserRewardOperationRedis{ + UserId: userId, + Amount: amount, + RewardType: rewardType, + OperateType: operateType, + } + b, _ := json.Marshal(data) + redis_client.ZAdd(redis_key.RewardOperateNotification(), string(b), float64(time.Now().UnixMilli())) +} + +func GetRewardOperation(since, until timeutil.TimeStamp) []models.UserRewardOperation { + list, err := redis_client.ZRangeByScore(redis_key.RewardOperateNotification(), float64(since*1000), float64(until*1000)) + if err != nil { + return nil + } + if len(list) == 0 { + return nil + } + r := make([]models.UserRewardOperation, len(list)) + for _, v := range list { + t := models.UserRewardOperationRedis{} + json.Unmarshal([]byte(v), &t) + r = append(r, models.UserRewardOperation{ + UserId: t.UserId, + Msg: GetRewardOperateMsg(t), + }) + } + redis_client.ZRemRangeByScore(redis_key.RewardOperateNotification(), float64(since*1000), float64(until*1000)) + return r +} + +func GetRewardOperateMsg(u models.UserRewardOperationRedis) string { + return u.OperateType.Show() + fmt.Sprint(u.Amount) + u.RewardType.Show() +} diff --git a/services/reward/operator.go b/services/reward/operator.go index 8d24ed055..40c093b67 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -17,17 +17,21 @@ var RewardOperatorMap = map[string]RewardOperator{ } type RewardOperator interface { - IsLimited(ctx models.RewardOperateContext) bool - Operate(ctx models.RewardOperateContext) error + IsLimited(ctx *models.RewardOperateContext) bool + Operate(ctx *models.RewardOperateContext) error } -func Send(ctx models.RewardOperateContext) error { +func Send(ctx *models.RewardOperateContext) error { defer func() { if err := recover(); err != nil { combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) log.Error("PANIC:%v", combinedErr) } }() + if !checkRewardOperationParam(ctx) { + log.Error("send reward error,param incorrect") + return errors.New("param incorrect") + } //add lock var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardSendLock(ctx.RequestId, ctx.SourceType)) isOk, err := rewardLock.Lock(3 * time.Second) @@ -79,12 +83,20 @@ func Send(ctx models.RewardOperateContext) error { //if not a cycle operate,update status to success if ctx.CycleIntervalSeconds == 0 { updateAwardOperateRecordStatus(ctx.SourceType, ctx.RequestId, models.OperateStatusOperating, models.OperateStatusSucceeded) + NotifyRewardOperation(ctx.TargetUserId, ctx.Reward.Amount, ctx.Reward.Type, ctx.OperateType) } return nil } -func GetOperator(rewardType string) RewardOperator { - return RewardOperatorMap[rewardType] +func checkRewardOperationParam(ctx *models.RewardOperateContext) bool { + if ctx.Reward.Type == "" { + return false + } + return true +} + +func GetOperator(rewardType models.RewardType) RewardOperator { + return RewardOperatorMap[rewardType.Name()] } func isHandled(sourceType string, requestId string) (bool, error) { @@ -99,16 +111,16 @@ func isHandled(sourceType string, requestId string) (bool, error) { } -func initAwardOperateRecord(ctx models.RewardOperateContext) (string, error) { +func initAwardOperateRecord(ctx *models.RewardOperateContext) (string, error) { record := &models.RewardOperateRecord{ RecordId: util.UUID(), UserId: ctx.TargetUserId, Amount: ctx.Reward.Amount, - RewardType: ctx.Reward.Type, + RewardType: ctx.Reward.Type.Name(), SourceType: ctx.SourceType, SourceId: ctx.SourceId, RequestId: ctx.RequestId, - OperateType: ctx.OperateType, + OperateType: ctx.OperateType.Name(), CycleIntervalSeconds: ctx.CycleIntervalSeconds, Status: models.OperateStatusOperating, Remark: ctx.Remark, diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index eaebdf764..38b6b5384 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -17,19 +17,20 @@ const LossMsg = "达到奖励上限，应得%d积分，实得%d积分" type PointOperator struct { } -func (operator *PointOperator) IsLimited(ctx models.RewardOperateContext) bool { +func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) bool { realAmount, err := limiter.CheckLimitWithFillUp(ctx.SourceType, models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount) if err != nil { return true } if realAmount < ctx.Reward.Amount { - ctx.Remark = ctx.Remark + ";" + fmt.Sprintf(LossMsg, ctx.Reward.Amount, realAmount) + ctx.Remark = models.AppendRemark(ctx.Remark, fmt.Sprintf(LossMsg, ctx.Reward.Amount, realAmount)) + ctx.Reward.Amount = realAmount } return false } -func (operator *PointOperator) Operate(ctx models.RewardOperateContext) error { +func (operator *PointOperator) Operate(ctx *models.RewardOperateContext) error { a, err := account.GetAccount(ctx.TargetUserId) if err != nil || a == nil { return errors.New("get account error") diff --git a/services/task/task.go b/services/task/task.go index 737094b4e..cd6ca830e 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -51,12 +51,12 @@ func accomplish(userId int64, taskType string) error { } //reward - reward.Send(models.RewardOperateContext{ + reward.Send(&models.RewardOperateContext{ SourceType: models.SourceTypeAccomplishTask, SourceId: logId, Reward: models.Reward{ Amount: config.AwardAmount, - Type: config.AwardType, + Type: models.GetRewardTypeInstance(config.AwardType), }, TargetUserId: userId, RequestId: logId, diff --git a/web_src/js/features/notification.js b/web_src/js/features/notification.js index 8b843e980..6f362eee6 100644 --- a/web_src/js/features/notification.js +++ b/web_src/js/features/notification.js @@ -45,6 +45,13 @@ export function initNotificationCount() { console.error(error); } }); + source.addEventListener('reward-operation', async (e) => { + try { + console.log(e.data); + } catch (error) { + console.error(error); + } + }); source.addEventListener('logout', async (e) => { if (e.data !== 'here') { return; From dc58c5493e45832ddaaf5939b1ba468d4ea4e285 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 21 Jun 2022 16:50:15 +0800 Subject: [PATCH 13/90] #2225 add clloudbrain deduct task --- models/cloudbrain.go | 27 ++-- models/models.go | 2 +- models/point_periodic_task.go | 28 ---- models/reward_operate_record.go | 88 ++++++++---- models/reward_periodic_task.go | 114 ++++++++++++++++ modules/auth/modelarts.go | 3 + modules/context/point.go | 19 +++ modules/cron/tasks_basic.go | 26 ++++ modules/modelarts/modelarts.go | 2 + modules/redis/redis_key/reward_redis_key.go | 9 +- modules/setting/setting.go | 8 +- routers/repo/cloudbrain.go | 30 +++- routers/repo/modelarts.go | 32 ++++- routers/routes/routes.go | 12 +- services/reward/cloubrain_deduct.go | 128 ++++++++++++++++++ services/reward/operator.go | 143 ++++++++++++++++---- services/reward/period_task.go | 103 ++++++++++++++ services/reward/point/point_operate.go | 3 +- services/task/task.go | 2 +- 19 files changed, 675 insertions(+), 104 deletions(-) delete mode 100644 models/point_periodic_task.go create mode 100644 models/reward_periodic_task.go create mode 100644 modules/context/point.go create mode 100644 services/reward/cloubrain_deduct.go create mode 100644 services/reward/period_task.go diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 810e68d30..9b30c4200 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -531,11 +531,12 @@ type ResourceSpecs struct { } type ResourceSpec struct { - Id int `json:"id"` - CpuNum int `json:"cpu"` - GpuNum int `json:"gpu"` - MemMiB int `json:"memMiB"` - ShareMemMiB int `json:"shareMemMiB"` + Id int `json:"id"` + CpuNum int `json:"cpu"` + GpuNum int `json:"gpu"` + MemMiB int `json:"memMiB"` + ShareMemMiB int `json:"shareMemMiB"` + UnitPrice int64 `json:"unitPrice"` } type FlavorInfos struct { @@ -543,9 +544,10 @@ type FlavorInfos struct { } type FlavorInfo struct { - Id int `json:"id"` - Value string `json:"value"` - Desc string `json:"desc"` + Id int `json:"id"` + Value string `json:"value"` + Desc string `json:"desc"` + UnitPrice int64 `json:"unitPrice"` } type ImageInfosModelArts struct { @@ -1692,3 +1694,12 @@ func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { return cloudbrains, count, nil } + +func GetStartedCloudbrainTaskByUpdatedUnix(startTime, endTime time.Time) ([]Cloudbrain, error) { + r := make([]Cloudbrain, 0) + err := x.Where("updated_unix >= ? and updated_unix <= ? and start_time > 0", startTime.Unix(), endTime.Unix()).Find(&r) + if err != nil { + return nil, err + } + return r, nil +} diff --git a/models/models.go b/models/models.go index 59e7a3a48..c6c0d6610 100755 --- a/models/models.go +++ b/models/models.go @@ -148,7 +148,7 @@ func init() { new(TaskAccomplishLog), new(RewardOperateRecord), new(LimitConfig), - new(PeriodicTask), + new(RewardPeriodicTask), new(PointAccountLog), new(PointAccount), ) diff --git a/models/point_periodic_task.go b/models/point_periodic_task.go deleted file mode 100644 index 0d4297f2f..000000000 --- a/models/point_periodic_task.go +++ /dev/null @@ -1,28 +0,0 @@ -package models - -import "code.gitea.io/gitea/modules/timeutil" - -type PeriodicTaskStatus int - -// Possible PeriodicTaskStatus types. -const ( - PeriodicTaskStatusRunning PointAccountStatus = iota + 1 // 1 - PeriodicTaskStatusSuccess // 2 - PeriodicTaskStatusFailed // 3 -) - -type PeriodicTask struct { - ID int64 `xorm:"pk autoincr"` - Type string `xorm:"NOT NULL"` - OperateRecordId int64 `xorm:"INDEX NOT NULL"` - IntervalSecond int64 `xorm:"NOT NULL"` - PointsAmount int64 `xorm:"NOT NULL"` - NextExecuteTime timeutil.TimeStamp - SuccessCount int `xorm:"NOT NULL default 0"` - FailedCount int `xorm:"NOT NULL default 0"` - Status string `xorm:"NOT NULL"` - ExitCode string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - FinishedUnix timeutil.TimeStamp `xorm:"INDEX"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` -} diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 4c31df03c..d3b2e0a10 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -6,12 +6,27 @@ import ( "xorm.io/builder" ) +type SourceType string + const ( - SourceTypeAccomplishTask string = "ACCOMPLISH_TASK" - SourceTypeAdminOperate = "ADMIN_OPERATE" - SourceTypeRunCloudbrainTask = "RUN_CLOUBRAIN_TASK" + SourceTypeAccomplishTask SourceType = "ACCOMPLISH_TASK" + SourceTypeAdminOperate SourceType = "ADMIN_OPERATE" + SourceTypeRunCloudbrainTask SourceType = "RUN_CLOUDBRAIN_TASK" ) +func (r SourceType) Name() string { + switch r { + case SourceTypeAccomplishTask: + return "ACCOMPLISH_TASK" + case SourceTypeAdminOperate: + return "ADMIN_OPERATE" + case SourceTypeRunCloudbrainTask: + return "RUN_CLOUDBRAIN_TASK" + default: + return "" + } +} + type RewardType string const ( @@ -66,6 +81,17 @@ func (r RewardOperateType) Show() string { } } +func GetRewardOperateTypeInstance(s string) RewardOperateType { + switch s { + case OperateTypeIncrease.Name(): + return OperateTypeIncrease + case OperateTypeDecrease.Name(): + return OperateTypeDecrease + default: + return "" + } +} + const ( OperateTypeIncrease RewardOperateType = "INCREASE" OperateTypeDecrease RewardOperateType = "DECREASE" @@ -80,20 +106,19 @@ const ( const Semicolon = ";" type RewardOperateRecord struct { - ID int64 `xorm:"pk autoincr"` - RecordId string `xorm:"INDEX NOT NULL"` - UserId int64 `xorm:"INDEX NOT NULL"` - Amount int64 `xorm:"NOT NULL"` - RewardType string `xorm:"NOT NULL"` - SourceType string `xorm:"NOT NULL"` - SourceId string `xorm:"INDEX NOT NULL"` - RequestId string `xorm:"INDEX NOT NULL"` - OperateType string `xorm:"NOT NULL"` - CycleIntervalSeconds int64 `xorm:"NOT NULL default 0"` - Status string `xorm:"NOT NULL"` - Remark string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + ID int64 `xorm:"pk autoincr"` + RecordId string `xorm:"INDEX NOT NULL"` + UserId int64 `xorm:"INDEX NOT NULL"` + Amount int64 `xorm:"NOT NULL"` + RewardType string `xorm:"NOT NULL"` + SourceType string `xorm:"NOT NULL"` + SourceId string `xorm:"INDEX NOT NULL"` + RequestId string `xorm:"INDEX NOT NULL"` + OperateType string `xorm:"NOT NULL"` + Status string `xorm:"NOT NULL"` + Remark string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) { @@ -106,10 +131,18 @@ func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error return tl, nil } -func GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId string) (*RewardOperateRecord, error) { +func GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId, operateType string) (*RewardOperateRecord, error) { + t := &RewardOperateRecord{ + SourceType: sourceType, + RequestId: requestId, + OperateType: operateType, + } + return getPointOperateRecord(t) +} + +func GetPointOperateRecordByRecordId(recordId string) (*RewardOperateRecord, error) { t := &RewardOperateRecord{ - SourceType: sourceType, - RequestId: requestId, + RecordId: recordId, } return getPointOperateRecord(t) } @@ -140,14 +173,13 @@ func SumRewardAmountInTaskPeriod(rewardType string, sourceType string, userId in } type RewardOperateContext struct { - SourceType string - SourceId string - Remark string - Reward Reward - TargetUserId int64 - RequestId string - OperateType RewardOperateType - CycleIntervalSeconds int64 + SourceType SourceType + SourceId string + Remark string + Reward Reward + TargetUserId int64 + RequestId string + OperateType RewardOperateType } type Reward struct { diff --git a/models/reward_periodic_task.go b/models/reward_periodic_task.go new file mode 100644 index 000000000..e6ebd17c2 --- /dev/null +++ b/models/reward_periodic_task.go @@ -0,0 +1,114 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "time" +) + +type PeriodicTaskStatus int + +const ( + PeriodicTaskStatusRunning = iota + 1 // 1 + PeriodicTaskStatusFinished // 2 +) + +type PeriodType string + +const ( + PeriodType30MinutesFree1HourCost PeriodType = "30MF1HC" +) + +func (r PeriodType) Name() string { + switch r { + case PeriodType30MinutesFree1HourCost: + return "30MF1HC" + default: + return "" + } +} + +type RewardPeriodicTask struct { + ID int64 `xorm:"pk autoincr"` + OperateRecordId string `xorm:"INDEX NOT NULL"` + DelaySeconds int64 + IntervalSeconds int64 + Amount int64 `xorm:"NOT NULL"` + NextExecuteTime timeutil.TimeStamp `xorm:"INDEX NOT NULL"` + SuccessCount int `xorm:"NOT NULL default 0"` + Status int `xorm:"NOT NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + FinishedUnix timeutil.TimeStamp `xorm:"INDEX"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +type StartPeriodicTaskOpts struct { + SourceType SourceType + SourceId string + Remark string + TargetUserId int64 + RequestId string + OperateType RewardOperateType + Delay time.Duration + Interval time.Duration + UnitAmount int64 + RewardType RewardType + StartTime time.Time +} + +func InsertPeriodicTask(tl *RewardPeriodicTask) (int64, error) { + return x.Insert(tl) +} + +func GetRunningRewardTask(now time.Time) ([]RewardPeriodicTask, error) { + r := make([]RewardPeriodicTask, 0) + err := x.Where("next_execute_time <= ? and status = ?", now.Unix(), PeriodicTaskStatusRunning).Find(&r) + if err != nil { + return nil, err + } + return r, err +} + +func IncrRewardTaskSuccessCount(t RewardPeriodicTask, count int64, nextTime timeutil.TimeStamp) error { + sess := x.NewSession() + defer sess.Close() + _, err := sess.Exec("update reward_periodic_task set success_count = success_count + ? , next_execute_time = ?, updated_unix = ? where id = ?", count, nextTime, timeutil.TimeStampNow(), t.ID) + if err != nil { + sess.Rollback() + return err + } + _, err = sess.Exec("update reward_operate_record set amount = amount + ? ,updated_unix = ? where record_id = ?", count*t.Amount, timeutil.TimeStampNow(), t.OperateRecordId) + if err != nil { + sess.Rollback() + return err + } + sess.Commit() + return nil +} + +func GetPeriodicTaskBySourceIdAndType(sourceType SourceType, sourceId string, operateType RewardOperateType) (*RewardPeriodicTask, error) { + r := RewardPeriodicTask{} + _, err := x.SQL("select rpt.* from reward_periodic_task rpt "+ + "inner join reward_operate_record ror on rpt.operate_record_id = ror.record_id"+ + " where ror.source_type = ? and source_id = ? and operate_type = ? ", sourceType.Name(), sourceId, operateType.Name()).Get(&r) + if err != nil { + return nil, err + } + return &r, nil +} + +func StopPeriodicTask(taskId int64, operateRecordId string, stopTime time.Time) error { + sess := x.NewSession() + defer sess.Close() + _, err := sess.Where("id = ? and status = ?", taskId, PeriodicTaskStatusRunning).Update(&RewardPeriodicTask{Status: PeriodicTaskStatusFinished, FinishedUnix: timeutil.TimeStamp(stopTime.Unix())}) + if err != nil { + sess.Rollback() + return err + } + _, err = sess.Where("record_id = ? and status = ?", operateRecordId, OperateStatusOperating).Update(&RewardOperateRecord{Status: OperateStatusSucceeded}) + if err != nil { + sess.Rollback() + return err + } + sess.Commit() + return nil +} diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index ce41f5d1e..0cbed45a6 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -22,6 +22,7 @@ type CreateModelArtsNotebookForm struct { Description string `form:"description"` Flavor string `form:"flavor" binding:"Required"` ImageId string `form:"image_id" binding:"Required"` + ResourceSpecId int `form:"resource_spec_id"` } func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { @@ -46,6 +47,7 @@ type CreateModelArtsTrainJobForm struct { VersionName string `form:"version_name" binding:"Required"` FlavorName string `form:"flaver_names" binding:"Required"` EngineName string `form:"engine_names" binding:"Required"` + ResourceSpecId int `form:"resource_spec_id"` } type CreateModelArtsInferenceJobForm struct { @@ -71,6 +73,7 @@ type CreateModelArtsInferenceJobForm struct { ModelName string `form:"model_name" binding:"Required"` ModelVersion string `form:"model_version" binding:"Required"` CkptName string `form:"ckpt_name" binding:"Required"` + ResourceSpecId int `form:"resource_spec_id"` } func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { diff --git a/modules/context/point.go b/modules/context/point.go new file mode 100644 index 000000000..9fbff61be --- /dev/null +++ b/modules/context/point.go @@ -0,0 +1,19 @@ +package context + +import ( + "code.gitea.io/gitea/services/reward/point/account" + "gitea.com/macaron/macaron" +) + +// PointAccount returns a macaron to get request user's point account +func PointAccount() macaron.Handler { + return func(ctx *Context) { + a, err := account.GetAccount(ctx.User.ID) + if err != nil { + ctx.ServerError("GetPointAccount", err) + return + } + ctx.Data["PointAccount"] = a + ctx.Next() + } +} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index b3a6c02a1..39100594d 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,6 +5,7 @@ package cron import ( + "code.gitea.io/gitea/services/reward" "context" "time" @@ -207,6 +208,28 @@ func registerSyncCloudbrainStatus() { }) } +func registerRewardPeriodTask() { + RegisterTaskFatal("reward_period_task", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 5m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + reward.StartRewardTask() + return nil + }) +} + +func registerCloudbrainPointDeductTask() { + RegisterTaskFatal("cloudbrain_point_deduct_task", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 1m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + reward.StartCloudbrainPointDeductTask() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -227,4 +250,7 @@ func initBasicTasks() { registerSyncCloudbrainStatus() registerHandleOrgStatistic() + + registerRewardPeriodTask() + registerCloudbrainPointDeductTask() } diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 78b40fd56..de5c392cd 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -96,6 +96,7 @@ type GenerateTrainJobReq struct { VersionCount int EngineName string TotalVersionCount int + ResourceSpecId int } type GenerateInferenceJobReq struct { @@ -127,6 +128,7 @@ type GenerateInferenceJobReq struct { ModelVersion string CkptName string ResultUrl string + ResourceSpecId int } type VersionInfo struct { diff --git a/modules/redis/redis_key/reward_redis_key.go b/modules/redis/redis_key/reward_redis_key.go index add304db4..f6c9480a9 100644 --- a/modules/redis/redis_key/reward_redis_key.go +++ b/modules/redis/redis_key/reward_redis_key.go @@ -1,11 +1,16 @@ package redis_key +import "fmt" + const REWARD_REDIS_PREFIX = "reward" -func RewardSendLock(requestId string, sourceType string) string { - return KeyJoin(REWARD_REDIS_PREFIX, requestId, sourceType, "send") +func RewardOperateLock(requestId string, sourceType string, operateType string) string { + return KeyJoin(REWARD_REDIS_PREFIX, requestId, sourceType, operateType, "send") } func RewardOperateNotification() string { return KeyJoin(REWARD_REDIS_PREFIX, "operate", "notification") } +func RewardTaskRunningLock(taskId int64) string { + return KeyJoin(REWARD_REDIS_PREFIX, "periodic_task", fmt.Sprint(taskId), "lock") +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 595c51286..b5ffe6eab 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -548,6 +548,9 @@ var ( WechatQRCodeExpireSeconds int WechatAuthSwitch bool + //point config + CloudBrainTaskPointPaySwitch bool + //nginx proxy PROXYURL string RadarMap = struct { @@ -1374,7 +1377,10 @@ func NewContext() { WechatAppId = sec.Key("APP_ID").MustString("wxba77b915a305a57d") WechatAppSecret = sec.Key("APP_SECRET").MustString("e48e13f315adc32749ddc7057585f198") WechatQRCodeExpireSeconds = sec.Key("QR_CODE_EXPIRE_SECONDS").MustInt(120) - WechatAuthSwitch = sec.Key("AUTH_SWITCH").MustBool(true) + WechatAuthSwitch = sec.Key("AUTH_SWITCH").MustBool(false) + + sec = Cfg.Section("point") + CloudBrainTaskPointPaySwitch = sec.Key("CLOUDBRAIN_PAY_SWITCH").MustBool(false) SetRadarMapConfig() diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 7ed6fa6ef..b4d532ab0 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,6 +2,7 @@ package repo import ( "bufio" + "code.gitea.io/gitea/services/reward" "encoding/json" "errors" "fmt" @@ -229,6 +230,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { command = commandTrain } + if !reward.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("point balance not enough", tpl, &form) + return + } + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { @@ -308,6 +316,13 @@ func CloudBrainRestart(ctx *context.Context) { var status = string(models.JobWaiting) task := ctx.Cloudbrain for { + if !reward.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) + resultCode = "-1" + errorMsg = "insufficient points balance" + break + } + if task.Status != string(models.JobStopped) && task.Status != string(models.JobSucceeded) && task.Status != string(models.JobFailed) { log.Error("the job(%s) is not stopped", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" @@ -842,7 +857,6 @@ func CloudBrainStop(ctx *context.Context) { errorMsg = "system error" break } - status = task.Status break } @@ -1845,6 +1859,13 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo repo := ctx.Repo.Repository + if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("point balance not enough", tplCloudBrainBenchmarkNew, &form) + return + } + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName) if err == nil { if len(tasks) != 0 { @@ -2000,6 +2021,13 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) tpl := tplCloudBrainBenchmarkNew command := cloudbrain.Command + if !reward.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("point balance not enough", tpl, &form) + return + } + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 95ca8df62..dea996a50 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2,6 +2,7 @@ package repo import ( "archive/zip" + "code.gitea.io/gitea/services/reward" "encoding/json" "errors" "fmt" @@ -204,7 +205,14 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm flavor := form.Flavor imageId := form.ImageId repo := ctx.Repo.Repository + resourceSpecId := form.ResourceSpecId + if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeDebug), resourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("point balance not enough", tplModelArtsNotebookNew, &form) + return + } count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -418,6 +426,13 @@ func NotebookManage(ctx *context.Context) { errorMsg = "you have no right to restart the job" break } + if !reward.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) + resultCode = "-1" + errorMsg = "point balance not enough" + break + return + } count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) if err != nil { @@ -985,7 +1000,14 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) FlavorName := form.FlavorName VersionCount := modelarts.VersionCount EngineName := form.EngineName + resourceSpecId := form.ResourceSpecId + if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeTrain), resourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("point balance not enough", tplModelArtsTrainJobNew, &form) + return + } count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -1161,6 +1183,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) EngineName: EngineName, VersionCount: VersionCount, TotalVersionCount: modelarts.TotalVersionCount, + ResourceSpecId: resourceSpecId, } //将params转换Parameters.Parameter，出错时返回给前端 @@ -1716,7 +1739,6 @@ func TrainJobStop(ctx *context.Context) { ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) return } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType) } @@ -1825,9 +1847,16 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference modelName := form.ModelName modelVersion := form.ModelVersion ckptName := form.CkptName + resourceSpecId := form.ResourceSpecId ckptUrl := form.TrainUrl + form.CkptName + if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeInference), resourceSpecId) { + log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) + inferenceJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("point balance not enough", tplModelArtsInferenceJobNew, &form) + return + } count, err := models.GetCloudbrainInferenceJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainInferenceJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -1973,6 +2002,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference ModelVersion: modelVersion, CkptName: ckptName, ResultUrl: resultObsPath, + ResourceSpecId: resourceSpecId, } err = modelarts.GenerateInferenceJob(ctx, req) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 31075742c..3ce633f93 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1068,7 +1068,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel) }) - m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.CloudBrainNew) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) m.Group("/benchmark", func() { @@ -1079,7 +1079,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.BenchmarkDel) m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) }) - m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.CloudBrainBenchmarkNew) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainBenchmarkNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainBenchmarkCreate) m.Get("/get_child_types", repo.GetChildTypes) }) @@ -1093,7 +1093,7 @@ func RegisterRoutes(m *macaron.Macaron) { //m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobNewVersion) //m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) }) - m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.CloudBrainTrainJobNew) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainTrainJobNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }) }, context.RepoRef()) @@ -1141,7 +1141,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/:action", reqRepoCloudBrainWriter, repo.NotebookManage) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel) }) - m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.NotebookNew) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.NotebookNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.Notebook2Create) }) @@ -1155,7 +1155,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobNewVersion) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) }) - m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.TrainJobNew) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.TrainJobNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) @@ -1168,7 +1168,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ResultDownload) m.Get("/downloadall", repo.DownloadMultiResultFile) }) - m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.InferenceJobNew) + m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceJobNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsInferenceJobForm{}), repo.InferenceJobCreate) }) }, context.RepoRef()) diff --git a/services/reward/cloubrain_deduct.go b/services/reward/cloubrain_deduct.go new file mode 100644 index 000000000..61068a87a --- /dev/null +++ b/services/reward/cloubrain_deduct.go @@ -0,0 +1,128 @@ +package reward + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/reward/point/account" + "encoding/json" + "fmt" + "time" +) + +var ( + ResourceSpecs *models.ResourceSpecs + TrainResourceSpecs *models.ResourceSpecs +) + +//IsPointBalanceEnough check whether the user's point balance is bigger than task unit price +func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int) bool { + if !setting.CloudBrainTaskPointPaySwitch { + return true + } + spec := getResourceSpec(jobType, resourceSpecId) + if spec == nil { + return true + } + a, error := account.GetAccount(targetUserId) + if error != nil { + return false + } + return a.Balance >= spec.UnitPrice + +} + +func StartCloudBrainPointDeductTask(task models.Cloudbrain) { + if !setting.CloudBrainTaskPointPaySwitch { + return + } + + spec := getResourceSpec(task.JobType, task.ResourceSpecId) + if spec == nil || spec.UnitPrice == 0 { + return + } + + StartPeriodicTask(&models.StartPeriodicTaskOpts{ + SourceType: models.SourceTypeRunCloudbrainTask, + SourceId: getCloudBrainPointTaskSourceId(task), + TargetUserId: task.UserID, + RequestId: getCloudBrainPointTaskSourceId(task), + OperateType: models.OperateTypeDecrease, + Delay: 30 * time.Minute, + Interval: 60 * time.Minute, + UnitAmount: spec.UnitPrice, + RewardType: models.RewardTypePoint, + StartTime: time.Unix(int64(task.StartTime), 0), + }) +} + +func StopCloudBrainPointDeductTask(task models.Cloudbrain) { + StopPeriodicTask(models.SourceTypeRunCloudbrainTask, getCloudBrainPointTaskSourceId(task), models.OperateTypeDecrease) +} + +func getCloudBrainPointTaskSourceId(task models.Cloudbrain) string { + return models.SourceTypeRunCloudbrainTask.Name() + "_" + task.JobType + "_" + fmt.Sprint(task.Type) + "_" + fmt.Sprint(task.ID) +} + +func getResourceSpec(jobType string, resourceSpecId int) *models.ResourceSpec { + if jobType == string(models.JobTypeTrain) { + if TrainResourceSpecs == nil { + json.Unmarshal([]byte(setting.TrainResourceSpecs), &TrainResourceSpecs) + } + for _, spec := range TrainResourceSpecs.ResourceSpec { + if resourceSpecId == spec.Id { + return spec + } + } + } else { + if ResourceSpecs == nil { + json.Unmarshal([]byte(setting.ResourceSpecs), &ResourceSpecs) + } + for _, spec := range ResourceSpecs.ResourceSpec { + if resourceSpecId == spec.Id { + return spec + } + } + + } + return nil + +} + +var firstTimeFlag = true + +func StartCloudbrainPointDeductTask() { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:%v", combinedErr) + } + }() + log.Debug("try to run CloudbrainPointDeductTask") + end := time.Now() + start := end.Add(5 * time.Minute) + if firstTimeFlag { + //When it is executed for the first time, it needs to process the tasks of the last 1 hours. + //This is done to prevent the application from hanging for a long time + start = end.Add(1 * time.Hour) + firstTimeFlag = false + } + + taskList, err := models.GetStartedCloudbrainTaskByUpdatedUnix(start, end) + if err != nil { + log.Error("GetStartedCloudbrainTaskByUpdatedUnix error. %v", err) + return + } + if taskList == nil || len(taskList) == 0 { + log.Debug("No cloudbrain task need handled") + return + } + for _, t := range taskList { + if int64(t.StartTime) <= end.Unix() && int64(t.StartTime) >= start.Unix() { + StartCloudBrainPointDeductTask(t) + } + if int64(t.EndTime) <= end.Unix() && int64(t.EndTime) >= start.Unix() { + StopCloudBrainPointDeductTask(t) + } + } +} diff --git a/services/reward/operator.go b/services/reward/operator.go index 40c093b67..50ec01ff3 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -21,7 +21,7 @@ type RewardOperator interface { Operate(ctx *models.RewardOperateContext) error } -func Send(ctx *models.RewardOperateContext) error { +func Operate(ctx *models.RewardOperateContext) error { defer func() { if err := recover(); err != nil { combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) @@ -33,7 +33,7 @@ func Send(ctx *models.RewardOperateContext) error { return errors.New("param incorrect") } //add lock - var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardSendLock(ctx.RequestId, ctx.SourceType)) + var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardOperateLock(ctx.RequestId, ctx.SourceType.Name(), ctx.OperateType.Name())) isOk, err := rewardLock.Lock(3 * time.Second) if err != nil { return err @@ -45,7 +45,7 @@ func Send(ctx *models.RewardOperateContext) error { defer rewardLock.UnLock() //is handled before? - isHandled, err := isHandled(ctx.SourceType, ctx.RequestId) + isHandled, err := isHandled(ctx.SourceType.Name(), ctx.RequestId, ctx.OperateType.Name()) if err != nil { log.Error("reward is handled error,%v", err) return err @@ -61,9 +61,11 @@ func Send(ctx *models.RewardOperateContext) error { return errors.New("operator of reward type is not exist") } - //is limited? - if isLimited := operator.IsLimited(ctx); isLimited { - return nil + if ctx.OperateType == models.OperateTypeIncrease { + //is limited? + if isLimited := operator.IsLimited(ctx); isLimited { + return nil + } } //new reward operate record @@ -76,15 +78,12 @@ func Send(ctx *models.RewardOperateContext) error { //operate if err := operator.Operate(ctx); err != nil { - updateAwardOperateRecordStatus(ctx.SourceType, ctx.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) + updateAwardOperateRecordStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) return err } - //if not a cycle operate,update status to success - if ctx.CycleIntervalSeconds == 0 { - updateAwardOperateRecordStatus(ctx.SourceType, ctx.RequestId, models.OperateStatusOperating, models.OperateStatusSucceeded) - NotifyRewardOperation(ctx.TargetUserId, ctx.Reward.Amount, ctx.Reward.Type, ctx.OperateType) - } + updateAwardOperateRecordStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusSucceeded) + NotifyRewardOperation(ctx.TargetUserId, ctx.Reward.Amount, ctx.Reward.Type, ctx.OperateType) return nil } @@ -99,8 +98,8 @@ func GetOperator(rewardType models.RewardType) RewardOperator { return RewardOperatorMap[rewardType.Name()] } -func isHandled(sourceType string, requestId string) (bool, error) { - _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId) +func isHandled(sourceType string, requestId string, operateType string) (bool, error) { + _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId, operateType) if err != nil { if models.IsErrRecordNotExist(err) { return false, nil @@ -113,17 +112,36 @@ func isHandled(sourceType string, requestId string) (bool, error) { func initAwardOperateRecord(ctx *models.RewardOperateContext) (string, error) { record := &models.RewardOperateRecord{ - RecordId: util.UUID(), - UserId: ctx.TargetUserId, - Amount: ctx.Reward.Amount, - RewardType: ctx.Reward.Type.Name(), - SourceType: ctx.SourceType, - SourceId: ctx.SourceId, - RequestId: ctx.RequestId, - OperateType: ctx.OperateType.Name(), - CycleIntervalSeconds: ctx.CycleIntervalSeconds, - Status: models.OperateStatusOperating, - Remark: ctx.Remark, + RecordId: util.UUID(), + UserId: ctx.TargetUserId, + Amount: ctx.Reward.Amount, + RewardType: ctx.Reward.Type.Name(), + SourceType: ctx.SourceType.Name(), + SourceId: ctx.SourceId, + RequestId: ctx.RequestId, + OperateType: ctx.OperateType.Name(), + Status: models.OperateStatusOperating, + Remark: ctx.Remark, + } + _, err := models.InsertAwardOperateRecord(record) + if err != nil { + return "", err + } + return record.RecordId, nil +} + +func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (string, error) { + record := &models.RewardOperateRecord{ + RecordId: util.UUID(), + UserId: ctx.TargetUserId, + Amount: 0, + RewardType: ctx.RewardType.Name(), + SourceType: ctx.SourceType.Name(), + SourceId: ctx.SourceId, + RequestId: ctx.RequestId, + OperateType: ctx.OperateType.Name(), + Status: models.OperateStatusOperating, + Remark: ctx.Remark, } _, err := models.InsertAwardOperateRecord(record) if err != nil { @@ -139,3 +157,78 @@ func updateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus } return nil } + +func StartPeriodicTaskAsyn(opts *models.StartPeriodicTaskOpts) { + go StartPeriodicTask(opts) +} + +func StartPeriodicTask(opts *models.StartPeriodicTaskOpts) error { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:%v", combinedErr) + } + }() + //add lock + var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardOperateLock(opts.RequestId, opts.SourceType.Name(), opts.OperateType.Name())) + isOk, err := rewardLock.Lock(3 * time.Second) + if err != nil { + return err + } + if !isOk { + log.Info("duplicated operate request,targetUserId=%d requestId=%s", opts.TargetUserId, opts.RequestId) + return nil + } + defer rewardLock.UnLock() + + //is handled before? + isHandled, err := isHandled(opts.SourceType.Name(), opts.RequestId, opts.OperateType.Name()) + if err != nil { + log.Error("operate is handled error,%v", err) + return err + } + if isHandled { + log.Info("operate has been handled,opts=%+v", opts) + return nil + } + //new reward operate record + recordId, err := createPeriodicRewardOperateRecord(opts) + if err != nil { + return err + } + + if err = NewRewardPeriodicTask(recordId, opts); err != nil { + updateAwardOperateRecordStatus(opts.SourceType.Name(), opts.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) + return err + } + return nil +} + +func StopPeriodicTaskAsyn(sourceType models.SourceType, sourceId string, operateType models.RewardOperateType) { + go StopPeriodicTask(sourceType, sourceId, operateType) +} + +func StopPeriodicTask(sourceType models.SourceType, sourceId string, operateType models.RewardOperateType) error { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:%v", combinedErr) + } + }() + task, err := models.GetPeriodicTaskBySourceIdAndType(sourceType, sourceId, operateType) + if err != nil { + log.Error("StopPeriodicTask. GetPeriodicTaskBySourceIdAndType error. %v", err) + return err + } + if task == nil { + log.Info("Periodic task is not exist") + return nil + } + if task.Status == models.PeriodicTaskStatusFinished { + log.Info("Periodic task is finished") + return nil + } + now := time.Now() + RunRewardTask(*task, now) + return models.StopPeriodicTask(task.ID, task.OperateRecordId, now) +} diff --git a/services/reward/period_task.go b/services/reward/period_task.go new file mode 100644 index 000000000..d00e8d0c4 --- /dev/null +++ b/services/reward/period_task.go @@ -0,0 +1,103 @@ +package reward + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/modules/timeutil" + "fmt" + "time" +) + +func NewRewardPeriodicTask(operateRecordId string, opts *models.StartPeriodicTaskOpts) error { + task := &models.RewardPeriodicTask{} + task.DelaySeconds = int64(opts.Delay.Seconds()) + task.IntervalSeconds = int64(opts.Interval.Seconds()) + task.Amount = opts.UnitAmount + task.OperateRecordId = operateRecordId + task.Status = models.PeriodicTaskStatusRunning + task.NextExecuteTime = timeutil.TimeStamp(opts.StartTime.Add(opts.Delay).Unix()) + + _, err := models.InsertPeriodicTask(task) + return err +} + +func StartRewardTask() { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:%v", combinedErr) + } + }() + log.Debug("try to run reward tasks") + now := time.Now() + taskList, err := models.GetRunningRewardTask(now) + if err != nil { + log.Error("GetRunningRewardTask error. %v", err) + return + } + if taskList == nil || len(taskList) == 0 { + log.Debug("No GetRunningRewardTask need handled") + return + } + for _, t := range taskList { + RunRewardTask(t, now) + } +} + +func RunRewardTask(t models.RewardPeriodicTask, now time.Time) { + lock := redis_lock.NewDistributeLock(redis_key.RewardTaskRunningLock(t.ID)) + isOk, _ := lock.LockWithWait(3*time.Second, 3*time.Second) + if !isOk { + log.Error("get RewardTaskRunningLock failed,t=%+v", t) + return + } + defer lock.UnLock() + record, err := models.GetPointOperateRecordByRecordId(t.OperateRecordId) + if err != nil { + log.Error("RunRewardTask. GetPointOperateRecordByRecordId error. %v", err) + return + } + if record.Status != models.OperateStatusOperating { + log.Info("RunRewardTask. operate record is finished,record=%+v", record) + return + } + n, nextTime := countExecuteTimes(t, now) + if n == 0 { + return + } + //get operator + operator := GetOperator(models.GetRewardTypeInstance(record.RewardType)) + if operator == nil { + log.Error("RunRewardTask. operator of reward type is not exist") + return + } + err = operator.Operate(&models.RewardOperateContext{ + SourceType: models.SourceTypeRunCloudbrainTask, + SourceId: t.OperateRecordId, + Reward: models.Reward{ + Amount: n * t.Amount, + Type: models.GetRewardTypeInstance(record.RewardType), + }, + TargetUserId: record.UserId, + OperateType: models.GetRewardOperateTypeInstance(record.OperateType), + }) + if err != nil { + log.Error("RunRewardTask.operator operate error.%v", err) + return + } + models.IncrRewardTaskSuccessCount(t, n, nextTime) +} + +func countExecuteTimes(t models.RewardPeriodicTask, now time.Time) (int64, timeutil.TimeStamp) { + interval := t.IntervalSeconds + nextTime := int64(t.NextExecuteTime) + if nextTime > now.Unix() { + return 0, 0 + } + diff := now.Unix() - nextTime + n := diff/interval + 1 + newNextTime := timeutil.TimeStamp(nextTime + n*interval) + return n, newNextTime +} diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 38b6b5384..4b84cdd0c 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -18,13 +18,12 @@ type PointOperator struct { } func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) bool { - realAmount, err := limiter.CheckLimitWithFillUp(ctx.SourceType, models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount) + realAmount, err := limiter.CheckLimitWithFillUp(ctx.SourceType.Name(), models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount) if err != nil { return true } if realAmount < ctx.Reward.Amount { ctx.Remark = models.AppendRemark(ctx.Remark, fmt.Sprintf(LossMsg, ctx.Reward.Amount, realAmount)) - ctx.Reward.Amount = realAmount } return false diff --git a/services/task/task.go b/services/task/task.go index cd6ca830e..4c85ce52e 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -51,7 +51,7 @@ func accomplish(userId int64, taskType string) error { } //reward - reward.Send(&models.RewardOperateContext{ + reward.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeAccomplishTask, SourceId: logId, Reward: models.Reward{ From 7a3cc57f9f76d1c620a3816872043b32b7e2a6f7 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 24 Jun 2022 14:54:46 +0800 Subject: [PATCH 14/90] #2225 update --- models/action.go | 10 ++ models/attachment.go | 6 +- models/error.go | 12 ++ models/limit_config.go | 8 + models/models.go | 1 + models/repo_watch.go | 5 +- models/reward_admin_log.go | 46 ++++++ models/reward_operate_record.go | 84 ++++++++++- models/reward_periodic_task.go | 11 +- models/task_accomplish_log.go | 11 +- models/task_config.go | 30 ---- modules/auth/wechat/event_handle.go | 7 +- modules/cloudbrain/resty.go | 8 +- modules/cron/tasks_basic.go | 2 +- modules/notification/action/action.go | 103 +++++++++++++ modules/notification/base/notifier.go | 7 +- modules/notification/base/null.go | 8 +- modules/notification/notification.go | 15 +- modules/notification/task/task.go | 157 -------------------- modules/redis/redis_client/client.go | 4 +- modules/redis/redis_key/reward_redis_key.go | 5 +- modules/redis/redis_key/serial_redis_key.go | 10 ++ modules/setting/setting.go | 10 +- routers/repo/cloudbrain.go | 12 +- routers/repo/modelarts.go | 8 +- routers/reward/point/point.go | 46 ++++++ routers/routes/routes.go | 4 + routers/task/task.go | 15 ++ routers/user/setting/profile.go | 2 +- services/reward/admin_operate.go | 50 +++++++ services/reward/cloubrain_deduct.go | 17 ++- services/reward/limiter/limiter.go | 39 ++--- services/reward/operator.go | 54 +++++-- services/reward/period_task.go | 8 +- services/reward/point/point_operate.go | 2 +- services/reward/record.go | 20 +++ services/reward/serial.go | 21 +++ services/task/task.go | 31 +++- 38 files changed, 591 insertions(+), 298 deletions(-) create mode 100644 models/reward_admin_log.go delete mode 100644 modules/notification/task/task.go create mode 100644 modules/redis/redis_key/serial_redis_key.go create mode 100644 routers/task/task.go create mode 100644 services/reward/admin_operate.go create mode 100644 services/reward/record.go create mode 100644 services/reward/serial.go diff --git a/models/action.go b/models/action.go index 9b92b4192..456d5c6bc 100755 --- a/models/action.go +++ b/models/action.go @@ -58,6 +58,16 @@ const ( ActionCreateBenchMarkTask //29 ActionCreateNewModelTask //30 ActionCreateGPUTrainTask //31 + + ActionBindWechat //32issue_assignees + ActionCreateCloudbrainTask //33 + ActionDatasetRecommended //34 + ActionCreateImage //35 + ActionImageRecommend //36 + ActionChangeUserAvatar //37 + ActionPushCommits //38 + ActionForkRepo //39 + ) // Action represents user operation type and other information to diff --git a/models/attachment.go b/models/attachment.go index 0e4751ed2..453c819b1 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -654,9 +654,9 @@ func Attachments(opts *AttachmentsOptions) ([]*AttachmentInfo, int64, error) { return attachments, count, nil } -func GetAllUserIdByDatasetId(datasetId int64) ([]int64, error) { - r := make([]int64, 0) - if err := x.Table("attachment").Where("dataset_id = ?", datasetId).Distinct("uploader_id").Find(&r); err != nil { +func GetAllDatasetContributorByDatasetId(datasetId int64) ([]*User, error) { + r := make([]*User, 0) + if err := x.Select("distinct(user.*)").Table("attachment").Join("LEFT", "user", "user.ID = attachment.uploader_id").Where("attachment.dataset_id = ?", datasetId).Find(&r); err != nil { return nil, err } return r, nil diff --git a/models/error.go b/models/error.go index 19afa9d8b..7c7b0418b 100755 --- a/models/error.go +++ b/models/error.go @@ -2024,3 +2024,15 @@ func IsErrRecordNotExist(err error) bool { func (err ErrRecordNotExist) Error() string { return fmt.Sprintf("record not exist in database") } + +type ErrInsufficientPointsBalance struct { +} + +func IsErrInsufficientPointsBalance(err error) bool { + _, ok := err.(ErrInsufficientPointsBalance) + return ok +} + +func (err ErrInsufficientPointsBalance) Error() string { + return fmt.Sprintf("Insufficient points balance") +} diff --git a/models/limit_config.go b/models/limit_config.go index 62ff3bfbe..ce8d2cfc2 100644 --- a/models/limit_config.go +++ b/models/limit_config.go @@ -41,6 +41,14 @@ func (l LimitScope) Name() string { } } +type LimiterRejectPolicy string + +const ( + JustReject LimiterRejectPolicy = "JUST_REJECT" + PermittedOnce LimiterRejectPolicy = "PERMITTED_ONCE" + FillUp LimiterRejectPolicy = "FillUp" +) + type LimitConfig struct { ID int64 `xorm:"pk autoincr"` Tittle string diff --git a/models/models.go b/models/models.go index c6c0d6610..731b31960 100755 --- a/models/models.go +++ b/models/models.go @@ -151,6 +151,7 @@ func init() { new(RewardPeriodicTask), new(PointAccountLog), new(PointAccount), + new(RewardAdminLog), ) tablesStatistic = append(tablesStatistic, diff --git a/models/repo_watch.go b/models/repo_watch.go index 2d01bde1f..864aec254 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -25,6 +25,7 @@ const ( ) var ActionChan = make(chan *Action, 200) +var ActionChan4Task = make(chan Action, 200) // Watch is connection request for receiving repository notification. type Watch struct { @@ -199,6 +200,9 @@ func notifyWatchers(e Engine, actions ...*Action) error { if _, err = e.InsertOne(act); err != nil { return fmt.Errorf("insert new actioner: %v", err) } + // After InsertOne(act),the act has ID + // Send the act to task chan + ActionChan4Task <- *act if repoChanged { act.loadRepo() @@ -279,7 +283,6 @@ func notifyWatchers(e Engine, actions ...*Action) error { // NotifyWatchers creates batch of actions for every watcher. func NotifyWatchers(actions ...*Action) error { - error := notifyWatchers(x, actions...) producer(actions...) return error diff --git a/models/reward_admin_log.go b/models/reward_admin_log.go new file mode 100644 index 000000000..5e4258682 --- /dev/null +++ b/models/reward_admin_log.go @@ -0,0 +1,46 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" +) + +const ( + RewardAdminLogProcessing = 1 + RewardAdminLogSuccess = 2 + RewardAdminLogFailed = 3 +) + +type RewardAdminLog struct { + ID int64 `xorm:"pk autoincr"` + LogId string `xorm:"INDEX NOT NULL"` + Amount int64 `xorm:"NOT NULL"` + RewardType string + Remark string + Status int + TargetUserId int64 `xorm:"INDEX NOT NULL"` + CreatorId int64 `xorm:"NOT NULL"` + CreatorName string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + +func getRewardAdminLog(ra *RewardAdminLog) (*RewardAdminLog, error) { + has, err := x.Get(ra) + if err != nil { + return nil, err + } else if !has { + return nil, ErrRecordNotExist{} + } + return ra, nil +} + +func InsertRewardAdminLog(ra *RewardAdminLog) (int64, error) { + return x.Insert(ra) +} + +func UpdateRewardAdminLogStatus(logId string, oldStatus, newStatus int) error { + _, err := x.Where("log_id = ? and status = ?", logId, oldStatus).Update(&RewardAdminLog{Status: newStatus}) + if err != nil { + return err + } + return nil +} diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index d3b2e0a10..d58accfa5 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -95,6 +95,7 @@ func GetRewardOperateTypeInstance(s string) RewardOperateType { const ( OperateTypeIncrease RewardOperateType = "INCREASE" OperateTypeDecrease RewardOperateType = "DECREASE" + OperateTypeNull RewardOperateType = "NIL" ) const ( @@ -105,11 +106,18 @@ const ( const Semicolon = ";" +type RewardOperateOrderBy string + +const ( + RewardOrderByID RewardOperateOrderBy = "id" +) + type RewardOperateRecord struct { ID int64 `xorm:"pk autoincr"` - RecordId string `xorm:"INDEX NOT NULL"` + SerialNo string `xorm:"INDEX NOT NULL"` UserId int64 `xorm:"INDEX NOT NULL"` Amount int64 `xorm:"NOT NULL"` + Tittle string RewardType string `xorm:"NOT NULL"` SourceType string `xorm:"NOT NULL"` SourceId string `xorm:"INDEX NOT NULL"` @@ -121,6 +129,32 @@ type RewardOperateRecord struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } +type AdminRewardOperateReq struct { + TargetUserId int64 `binding:"Required"` + OperateType RewardOperateType `binding:"Required"` + Amount int64 `binding:"Required;Range(1,100000)"` + Remark string + RewardType RewardType +} + +func (r RewardOperateRecord) ToShow() RewardOperateRecordShow { + return RewardOperateRecordShow{ + SerialNo: r.SerialNo, + Date: r.CreatedUnix, + Tittle: r.Tittle, + OperateType: r.OperateType, + Amount: r.Amount, + } +} + +type RewardOperateRecordShow struct { + SerialNo string + Date timeutil.TimeStamp + Tittle string + OperateType string + Amount int64 +} + func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) { has, err := x.Get(tl) if err != nil { @@ -140,14 +174,14 @@ func GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId, operat return getPointOperateRecord(t) } -func GetPointOperateRecordByRecordId(recordId string) (*RewardOperateRecord, error) { +func GetPointOperateRecordBySerialNo(serialNo string) (*RewardOperateRecord, error) { t := &RewardOperateRecord{ - RecordId: recordId, + SerialNo: serialNo, } return getPointOperateRecord(t) } -func InsertAwardOperateRecord(tl *RewardOperateRecord) (int64, error) { +func InsertRewardOperateRecord(tl *RewardOperateRecord) (int64, error) { return x.Insert(tl) } @@ -175,11 +209,13 @@ func SumRewardAmountInTaskPeriod(rewardType string, sourceType string, userId in type RewardOperateContext struct { SourceType SourceType SourceId string + Tittle string Remark string Reward Reward TargetUserId int64 RequestId string OperateType RewardOperateType + RejectPolicy LimiterRejectPolicy } type Reward struct { @@ -202,3 +238,43 @@ type UserRewardOperation struct { func AppendRemark(remark, appendStr string) string { return strings.TrimPrefix(remark+Semicolon+appendStr, Semicolon) } + +type RewardRecordListOpts struct { + ListOptions + UserId int64 + OperateType RewardOperateType + RewardType RewardType + OrderBy RewardOperateOrderBy +} + +func GetRewardRecordList(opts RewardRecordListOpts) ([]RewardOperateRecord, int64, error) { + if opts.Page <= 0 { + opts.Page = 1 + } + + if len(opts.OrderBy) == 0 { + opts.OrderBy = RewardOrderByID + } + + r := make([]RewardOperateRecord, 0) + cond := builder.NewCond() + if opts.UserId > 0 { + cond = cond.And(builder.Eq{"user_id": opts.UserId}) + } + if opts.OperateType != OperateTypeNull { + cond = cond.And(builder.Eq{"operate_type": opts.OperateType.Name()}) + } + cond = cond.And(builder.Eq{"reward_type": opts.RewardType.Name()}) + cond = cond.And(builder.Gt{"amount": 0}) + + count, err := x.Where(cond).Count(&RewardOperateRecord{}) + if err != nil { + return nil, 0, err + } + + err = x.Where(cond).Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).OrderBy(string(opts.OrderBy)).Find(&r) + if err != nil { + return nil, 0, err + } + return r, count, nil +} diff --git a/models/reward_periodic_task.go b/models/reward_periodic_task.go index e6ebd17c2..5db5301b5 100644 --- a/models/reward_periodic_task.go +++ b/models/reward_periodic_task.go @@ -29,7 +29,7 @@ func (r PeriodType) Name() string { type RewardPeriodicTask struct { ID int64 `xorm:"pk autoincr"` - OperateRecordId string `xorm:"INDEX NOT NULL"` + OperateSerialNo string `xorm:"INDEX NOT NULL"` DelaySeconds int64 IntervalSeconds int64 Amount int64 `xorm:"NOT NULL"` @@ -45,6 +45,7 @@ type StartPeriodicTaskOpts struct { SourceType SourceType SourceId string Remark string + Tittle string TargetUserId int64 RequestId string OperateType RewardOperateType @@ -76,7 +77,7 @@ func IncrRewardTaskSuccessCount(t RewardPeriodicTask, count int64, nextTime time sess.Rollback() return err } - _, err = sess.Exec("update reward_operate_record set amount = amount + ? ,updated_unix = ? where record_id = ?", count*t.Amount, timeutil.TimeStampNow(), t.OperateRecordId) + _, err = sess.Exec("update reward_operate_record set amount = amount + ? ,updated_unix = ? where serial_no = ?", count*t.Amount, timeutil.TimeStampNow(), t.OperateSerialNo) if err != nil { sess.Rollback() return err @@ -88,7 +89,7 @@ func IncrRewardTaskSuccessCount(t RewardPeriodicTask, count int64, nextTime time func GetPeriodicTaskBySourceIdAndType(sourceType SourceType, sourceId string, operateType RewardOperateType) (*RewardPeriodicTask, error) { r := RewardPeriodicTask{} _, err := x.SQL("select rpt.* from reward_periodic_task rpt "+ - "inner join reward_operate_record ror on rpt.operate_record_id = ror.record_id"+ + "inner join reward_operate_record ror on rpt.operate_serial_no = ror.serial_no"+ " where ror.source_type = ? and source_id = ? and operate_type = ? ", sourceType.Name(), sourceId, operateType.Name()).Get(&r) if err != nil { return nil, err @@ -96,7 +97,7 @@ func GetPeriodicTaskBySourceIdAndType(sourceType SourceType, sourceId string, op return &r, nil } -func StopPeriodicTask(taskId int64, operateRecordId string, stopTime time.Time) error { +func StopPeriodicTask(taskId int64, operateSerialNo string, stopTime time.Time) error { sess := x.NewSession() defer sess.Close() _, err := sess.Where("id = ? and status = ?", taskId, PeriodicTaskStatusRunning).Update(&RewardPeriodicTask{Status: PeriodicTaskStatusFinished, FinishedUnix: timeutil.TimeStamp(stopTime.Unix())}) @@ -104,7 +105,7 @@ func StopPeriodicTask(taskId int64, operateRecordId string, stopTime time.Time) sess.Rollback() return err } - _, err = sess.Where("record_id = ? and status = ?", operateRecordId, OperateStatusOperating).Update(&RewardOperateRecord{Status: OperateStatusSucceeded}) + _, err = sess.Where("serial_no = ? and status = ?", operateSerialNo, OperateStatusOperating).Update(&RewardOperateRecord{Status: OperateStatusSucceeded}) if err != nil { sess.Rollback() return err diff --git a/models/task_accomplish_log.go b/models/task_accomplish_log.go index 3736d1c41..a1edb71ee 100644 --- a/models/task_accomplish_log.go +++ b/models/task_accomplish_log.go @@ -6,11 +6,12 @@ import ( ) type TaskAccomplishLog struct { - ID int64 `xorm:"pk autoincr"` - LogId string `xorm:"INDEX NOT NULL"` - ConfigId int64 `xorm:"NOT NULL"` - TaskCode string `xorm:"NOT NULL"` - UserId int64 `xorm:"INDEX NOT NULL"` + ID int64 `xorm:"pk autoincr"` + LogId string `xorm:"INDEX NOT NULL"` + ConfigId int64 `xorm:"NOT NULL"` + TaskCode string `xorm:"NOT NULL"` + UserId int64 `xorm:"INDEX NOT NULL"` + ActionId int64 CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } diff --git a/models/task_config.go b/models/task_config.go index cd4329834..eef3a6c33 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -4,36 +4,6 @@ import ( "code.gitea.io/gitea/modules/timeutil" ) -const ( - TaskTypeNewIssue = "NEW_ISSUE" - TaskTypeIssueChangeStatus = "ISSUE_CHANGE_STATUS" - TaskTypeCreateIssueComment = "CREATE_ISSUE_COMMENT" - TaskTypeNewPullRequest = "NEW_PULL_REQUEST" - TaskTypeRenameRepository = "RENAME_REPOSITORY" - TaskTypeAliasRepository = "ALIAS_REPOSITORY" - TaskTypeTransferRepository = "TRANSFER_REPOSITORY" - TaskTypeCreateRepository = "CREATE_REPOSITORY" - TaskTypeCreatePublicRepository = "CREATE_PUBLIC_REPOSITORY" - TaskTypeForkRepository = "FORK_REPOSITORY" - TaskTypePullRequestReview = "PULL_REQUEST_REVIEW" - TaskTypeCommentPull = "COMMENT_PULL" - TaskTypeApprovePullRequest = "APPROVE_PULL_REQUEST" - TaskTypeRejectPullRequest = "REJECT_PULL_REQUEST" - TaskTypeMergePullRequest = "MERGE_PULL_REQUEST" - TaskTypeSyncPushCommits = "SYNC_PUSH_COMMITS" - TaskTypeSyncCreateRef = "SYNC_CREATE_REF" - TaskTypeSyncDeleteRef = "SYNC_DELETE_REF" - TaskTypeBindWechat = "BIND_WECHAT" - TaskTypeUploadAttachment = "UPLOAD_ATTACHMENT" - TaskTypeCreateCloudbrainTask = "CREATE_CLOUDBRAIN_TASK" - TaskTypeDatasetRecommended = "DATASET_RECOMMENDED" - TaskTypeCreateModel = "CREATE_MODEL" - TaskTypeCreatePublicImage = "CREATE_PUBLIC_IMAGE" - TaskTypeImageRecommend = "IMAGE_RECOMMEND" - TaskTypeChangeUserAvatar = "CHANGE_USER_AVATAR" - TaskTypePushCommits = "PUSH_COMMITS" -) - const ( PeriodNotCycle = "NOT_CYCLE" PeriodDaily = "DAILY" diff --git a/modules/auth/wechat/event_handle.go b/modules/auth/wechat/event_handle.go index 67c3a7265..399537f1e 100644 --- a/modules/auth/wechat/event_handle.go +++ b/modules/auth/wechat/event_handle.go @@ -1,6 +1,7 @@ package wechat import ( + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" @@ -72,6 +73,10 @@ func HandleSubscribeEvent(we WechatEvent) string { jsonStr, _ := json.Marshal(qrCache) redis_client.Setex(redis_key.WechatBindingUserIdKey(sceneStr), string(jsonStr), 60*time.Second) } - notification.NotifyWechatBind(qrCache.UserId, we.FromUserName) + u, err := models.GetUserByID(qrCache.UserId) + if err == nil { + notification.NotifyWechatBind(u, we.FromUserName) + } + return BIND_REPLY_SUCCESS } diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index 75614e571..d45468ddb 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -212,7 +212,7 @@ func getQueryString(page int, size int, name string) string { return fmt.Sprintf("pageIndex=%d&pageSize=%d&name=%s", page, size, name) } -func CommitImage(jobID string, params models.CommitImageParams) error { +func CommitImage(jobID string, params models.CommitImageParams, doer *models.User) error { imageTag := strings.TrimSpace(params.ImageTag) dbImage, err := models.GetImageByTag(imageTag) @@ -314,12 +314,12 @@ sendjob: }) if err == nil { go updateImageStatus(image, isSetCreatedUnix, createTime) - notification.NotifyCreateImage(params.UID, image) + notification.NotifyCreateImage(doer, image) } return err } -func CommitAdminImage(params models.CommitImageParams) error { +func CommitAdminImage(params models.CommitImageParams, doer *models.User) error { imageTag := strings.TrimSpace(params.ImageTag) exist, err := models.IsImageExist(imageTag) @@ -357,7 +357,7 @@ func CommitAdminImage(params models.CommitImageParams) error { return nil }) if err == nil { - notification.NotifyCreateImage(params.UID, image) + notification.NotifyCreateImage(doer, image) } return err } diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 39100594d..5892699eb 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -212,7 +212,7 @@ func registerRewardPeriodTask() { RegisterTaskFatal("reward_period_task", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 5m", + Schedule: "@every 1m", }, func(ctx context.Context, _ *models.User, _ Config) error { reward.StartRewardTask() return nil diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index 2ac73c2c3..6a43c6e9a 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -5,6 +5,7 @@ package action import ( + "code.gitea.io/gitea/modules/auth" "encoding/json" "fmt" "path" @@ -345,3 +346,105 @@ func (a *actionNotifier) NotifyOtherTask(doer *models.User, repo *models.Reposit log.Error("notifyWatchers: %v", err) } } + +func (t *actionNotifier) NotifyWechatBind(user *models.User, wechatOpenId string) { + act := &models.Action{ + ActUserID: user.ID, + ActUser: user, + OpType: models.ActionBindWechat, + IsPrivate: true, + Content: wechatOpenId, + } + if err := models.NotifyWatchers(act); err != nil { + log.Error("notifyWatchers: %v", err) + } +} + +func (t *actionNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { + switch action { + case "recommend": + users, err := models.GetAllDatasetContributorByDatasetId(dataset.ID) + if err != nil { + return + } + var actions = make([]*models.Action, 0) + for _, user := range users { + actions = append(actions, &models.Action{ + OpType: models.ActionDatasetRecommended, + ActUserID: user.ID, + ActUser: user, + RepoID: dataset.RepoID, + Repo: dataset.Repo, + Content: fmt.Sprint(dataset.ID), + }) + } + if err := models.NotifyWatchers(actions...); err != nil { + log.Error("notifyWatchers: %v", err) + } + } +} + +func (t *actionNotifier) NotifyCreateImage(doer *models.User, image models.Image) { + act := &models.Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: models.ActionCreateImage, + IsPrivate: image.IsPrivate, + Content: fmt.Sprint(image.ID), + } + if err := models.NotifyWatchers(act); err != nil { + log.Error("notifyWatchers: %v", err) + } +} + +func (t *actionNotifier) NotifyImageRecommend(optUser *models.User, imageId int64, action string) { + image, err := models.GetImageByID(imageId) + if err != nil { + return + } + u, err := models.GetUserByID(image.UID) + if err != nil { + return + } + switch action { + case "recommend": + act := &models.Action{ + ActUserID: u.ID, + ActUser: u, + OpType: models.ActionImageRecommend, + IsPrivate: false, + Content: fmt.Sprint(imageId), + } + if err := models.NotifyWatchers(act); err != nil { + log.Error("notifyWatchers: %v", err) + } + } +} + +func (t *actionNotifier) NotifyChangeUserAvatar(user *models.User, form auth.AvatarForm) { + act := &models.Action{ + ActUserID: user.ID, + ActUser: user, + OpType: models.ActionChangeUserAvatar, + IsPrivate: true, + } + if err := models.NotifyWatchers(act); err != nil { + log.Error("notifyWatchers: %v", err) + } +} + +func (t *actionNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { + act := &models.Action{ + ActUserID: pusher.ID, + ActUser: pusher, + OpType: models.ActionPushCommits, + RepoID: repo.ID, + Repo: repo, + RefName: refName, + IsPrivate: repo.IsPrivate, + Content: fmt.Sprintf("%s|%s", oldCommitID, newCommitID), + } + if err := models.NotifyWatchers(act); err != nil { + log.Error("notifyWatchers: %v", err) + } +} diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index c3c7f404a..7673a5909 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -6,6 +6,7 @@ package base import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/repository" ) @@ -56,9 +57,9 @@ type Notifier interface { NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) - NotifyWechatBind(userId int64, wechatOpenId string) + NotifyWechatBind(user *models.User, wechatOpenId string) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) - NotifyCreateImage(optUserId int64, image models.Image) + NotifyCreateImage(doer *models.User, image models.Image) NotifyImageRecommend(optUser *models.User, imageId int64, action string) - NotifyChangeUserAvatar(user *models.User) + NotifyChangeUserAvatar(user *models.User, form auth.AvatarForm) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index c0a224697..eea5c5e77 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -6,6 +6,7 @@ package base import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/repository" ) @@ -159,18 +160,19 @@ func (*NullNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, } -func (*NullNotifier) NotifyWechatBind(userId int64, wechatOpenId string) { +func (*NullNotifier) NotifyWechatBind(user *models.User, wechatOpenId string) { } func (*NullNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { } -func (*NullNotifier) NotifyCreateImage(optUserId int64, image models.Image) { +func (*NullNotifier) NotifyCreateImage(doer *models.User, image models.Image) { } func (*NullNotifier) NotifyImageRecommend(optUser *models.User, imageId int64, action string) { } -func (*NullNotifier) NotifyChangeUserAvatar(user *models.User) { +func (*NullNotifier) NotifyChangeUserAvatar(user *models.User, form auth.AvatarForm) { + } diff --git a/modules/notification/notification.go b/modules/notification/notification.go index 118bdf994..d652dc043 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -6,11 +6,11 @@ package notification import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/notification/action" "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/notification/indexer" "code.gitea.io/gitea/modules/notification/mail" - "code.gitea.io/gitea/modules/notification/task" "code.gitea.io/gitea/modules/notification/ui" "code.gitea.io/gitea/modules/notification/webhook" "code.gitea.io/gitea/modules/repository" @@ -36,7 +36,6 @@ func NewContext() { RegisterNotifier(indexer.NewNotifier()) RegisterNotifier(webhook.NewNotifier()) RegisterNotifier(action.NewNotifier()) - RegisterNotifier(task.NewNotifier()) } // NotifyUploadAttachment notifies attachment upload message to notifiers @@ -273,9 +272,9 @@ func NotifySyncDeleteRef(pusher *models.User, repo *models.Repository, refType, } // NotifyWechatBind notifies wechat bind -func NotifyWechatBind(userId int64, wechatOpenId string) { +func NotifyWechatBind(user *models.User, wechatOpenId string) { for _, notifier := range notifiers { - notifier.NotifyWechatBind(userId, wechatOpenId) + notifier.NotifyWechatBind(user, wechatOpenId) } } @@ -287,9 +286,9 @@ func NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, actio } // NotifyDatasetRecommend -func NotifyCreateImage(optUserId int64, image models.Image) { +func NotifyCreateImage(doer *models.User, image models.Image) { for _, notifier := range notifiers { - notifier.NotifyCreateImage(optUserId, image) + notifier.NotifyCreateImage(doer, image) } } @@ -301,8 +300,8 @@ func NotifyImageRecommend(optUser *models.User, imageId int64, action string) { } // NotifyDatasetRecommend -func NotifyChangeUserAvatar(user *models.User) { +func NotifyChangeUserAvatar(user *models.User, form auth.AvatarForm) { for _, notifier := range notifiers { - notifier.NotifyChangeUserAvatar(user) + notifier.NotifyChangeUserAvatar(user, form) } } diff --git a/modules/notification/task/task.go b/modules/notification/task/task.go deleted file mode 100644 index 077d6699b..000000000 --- a/modules/notification/task/task.go +++ /dev/null @@ -1,157 +0,0 @@ -package task - -import ( - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/notification/base" - "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/services/task" - "strings" -) - -type taskNotifier struct { - base.NullNotifier -} - -var ( - _ base.Notifier = &taskNotifier{} -) - -// NewNotifier create a new actionNotifier notifier -func NewNotifier() base.Notifier { - return &taskNotifier{} -} - -func (t *taskNotifier) NotifyNewIssue(issue *models.Issue) { - task.Accomplish(issue.Poster.ID, models.TaskTypeNewIssue) -} - -// NotifyIssueChangeStatus notifies close or reopen issue to notifiers -func (t *taskNotifier) NotifyIssueChangeStatus(doer *models.User, issue *models.Issue, actionComment *models.Comment, closeOrReopen bool) { - task.Accomplish(doer.ID, models.TaskTypeIssueChangeStatus) -} - -// NotifyCreateIssueComment notifies comment on an issue to notifiers -func (t *taskNotifier) NotifyCreateIssueComment(doer *models.User, repo *models.Repository, - issue *models.Issue, comment *models.Comment) { - task.Accomplish(doer.ID, models.TaskTypeCreateIssueComment) -} - -func (t *taskNotifier) NotifyNewPullRequest(pull *models.PullRequest) { - task.Accomplish(pull.Issue.Poster.ID, models.TaskTypeNewPullRequest) -} - -func (t *taskNotifier) NotifyRenameRepository(doer *models.User, repo *models.Repository, oldRepoName string) { - task.Accomplish(doer.ID, models.TaskTypeRenameRepository) -} - -func (t *taskNotifier) NotifyAliasRepository(doer *models.User, repo *models.Repository, oldAlias string) { - task.Accomplish(doer.ID, models.TaskTypeAliasRepository) -} - -func (t *taskNotifier) NotifyTransferRepository(doer *models.User, repo *models.Repository, oldOwnerName string) { - task.Accomplish(doer.ID, models.TaskTypeTransferRepository) -} - -func (t *taskNotifier) NotifyCreateRepository(doer *models.User, u *models.User, repo *models.Repository) { - if !repo.IsPrivate { - task.Accomplish(doer.ID, models.TaskTypeCreatePublicRepository) - } - -} - -func (t *taskNotifier) NotifyForkRepository(doer *models.User, oldRepo, repo *models.Repository) { - task.Accomplish(doer.ID, models.TaskTypeForkRepository) -} - -func (t *taskNotifier) NotifyPullRequestReview(pr *models.PullRequest, review *models.Review, comment *models.Comment) { - for _, lines := range review.CodeComments { - for _, comments := range lines { - for _, _ = range comments { - task.Accomplish(review.Reviewer.ID, models.TaskTypePullRequestReview) - } - } - } - if review.Type != models.ReviewTypeComment || strings.TrimSpace(comment.Content) != "" { - - switch review.Type { - case models.ReviewTypeApprove: - task.Accomplish(review.Reviewer.ID, models.TaskTypeApprovePullRequest) - case models.ReviewTypeReject: - task.Accomplish(review.Reviewer.ID, models.TaskTypeRejectPullRequest) - default: - task.Accomplish(review.Reviewer.ID, models.TaskTypeCommentPull) - } - - } -} - -func (t *taskNotifier) NotifyMergePullRequest(pr *models.PullRequest, doer *models.User) { - task.Accomplish(doer.ID, models.TaskTypeMergePullRequest) -} - -func (t *taskNotifier) NotifySyncPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { - task.Accomplish(repo.OwnerID, models.TaskTypeSyncPushCommits) -} - -func (t *taskNotifier) NotifySyncCreateRef(doer *models.User, repo *models.Repository, refType, refFullName string) { - task.Accomplish(repo.OwnerID, models.TaskTypeSyncCreateRef) -} - -func (t *taskNotifier) NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) { - task.Accomplish(repo.OwnerID, models.TaskTypeSyncDeleteRef) -} - -func (t *taskNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) { - switch optype { - case models.ActionUploadAttachment: - task.Accomplish(doer.ID, models.TaskTypeUploadAttachment) - case models.ActionCreateDebugGPUTask, - models.ActionCreateDebugNPUTask, - models.ActionCreateTrainTask, - models.ActionCreateInferenceTask, - models.ActionCreateBenchMarkTask, - models.ActionCreateGPUTrainTask: - task.Accomplish(doer.ID, models.TaskTypeCreateCloudbrainTask) - case models.ActionCreateNewModelTask: - task.Accomplish(doer.ID, models.TaskTypeCreateModel) - } - return -} - -func (t *taskNotifier) NotifyWechatBind(userId int64, wechatOpenId string) { - task.Accomplish(userId, models.TaskTypeBindWechat) -} - -func (t *taskNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { - switch action { - case "recommend": - userIds, err := models.GetAllUserIdByDatasetId(dataset.ID) - if err != nil { - return - } - for _, userId := range userIds { - task.Accomplish(userId, models.TaskTypeDatasetRecommended) - } - } -} - -func (t *taskNotifier) NotifyCreateImage(optUserId int64, image models.Image) { - if !image.IsPrivate { - task.Accomplish(optUserId, models.TaskTypeCreatePublicImage) - } -} - -func (t *taskNotifier) NotifyImageRecommend(optUser *models.User, imageId int64, action string) { - switch action { - case "recommend": - task.Accomplish(optUser.ID, models.TaskTypeImageRecommend) - } -} - -func (t *taskNotifier) NotifyChangeUserAvatar(user *models.User) { - task.Accomplish(user.ID, models.TaskTypeChangeUserAvatar) -} - -func (t *taskNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { - task.Accomplish(pusher.ID, models.TaskTypePushCommits) -} diff --git a/modules/redis/redis_client/client.go b/modules/redis/redis_client/client.go index c5cb936b3..e795234df 100644 --- a/modules/redis/redis_client/client.go +++ b/modules/redis/redis_client/client.go @@ -99,11 +99,11 @@ func IncrBy(key string, n int64) (int64, error) { } -func Expire(key string, expireSeconds int64) error { +func Expire(key string, expireTime time.Duration) error { redisClient := labelmsg.Get() defer redisClient.Close() - _, err := redisClient.Do("EXPIRE", key, expireSeconds) + _, err := redisClient.Do("EXPIRE", key, int64(expireTime.Seconds())) if err != nil { return err } diff --git a/modules/redis/redis_key/reward_redis_key.go b/modules/redis/redis_key/reward_redis_key.go index f6c9480a9..05c10ce4f 100644 --- a/modules/redis/redis_key/reward_redis_key.go +++ b/modules/redis/redis_key/reward_redis_key.go @@ -1,6 +1,8 @@ package redis_key -import "fmt" +import ( + "fmt" +) const REWARD_REDIS_PREFIX = "reward" @@ -11,6 +13,7 @@ func RewardOperateLock(requestId string, sourceType string, operateType string) func RewardOperateNotification() string { return KeyJoin(REWARD_REDIS_PREFIX, "operate", "notification") } + func RewardTaskRunningLock(taskId int64) string { return KeyJoin(REWARD_REDIS_PREFIX, "periodic_task", fmt.Sprint(taskId), "lock") } diff --git a/modules/redis/redis_key/serial_redis_key.go b/modules/redis/redis_key/serial_redis_key.go new file mode 100644 index 000000000..c0ecf39eb --- /dev/null +++ b/modules/redis/redis_key/serial_redis_key.go @@ -0,0 +1,10 @@ +package redis_key + +import "time" + +const SERIAL_REDIS_PREFIX = "serial" + +func RewardSerialCounter(now time.Time) string { + h := now.Format("200601021504") + return KeyJoin(SERIAL_REDIS_PREFIX, "reward_operate", h, "counter") +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index b5ffe6eab..217388789 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -232,7 +232,7 @@ var ( TimeoutStep: 10 * time.Second, MaxTimeout: 60 * time.Second, EventSourceUpdateTime: 10 * time.Second, - RewardNotifyUpdateTime: 3 * time.Second, + RewardNotifyUpdateTime: 2 * time.Second, }, Admin: struct { UserPagingNum int @@ -549,7 +549,9 @@ var ( WechatAuthSwitch bool //point config - CloudBrainTaskPointPaySwitch bool + CloudBrainPaySwitch bool + CloudBrainPayDelay time.Duration + CloudBrainPayInterval time.Duration //nginx proxy PROXYURL string @@ -1380,7 +1382,9 @@ func NewContext() { WechatAuthSwitch = sec.Key("AUTH_SWITCH").MustBool(false) sec = Cfg.Section("point") - CloudBrainTaskPointPaySwitch = sec.Key("CLOUDBRAIN_PAY_SWITCH").MustBool(false) + CloudBrainPaySwitch = sec.Key("CLOUDBRAIN_PAY_SWITCH").MustBool(false) + CloudBrainPayDelay = sec.Key("CLOUDBRAIN_PAY_DELAY").MustDuration(30 * time.Minute) + CloudBrainPayInterval = sec.Key("CLOUDBRAIN_PAY_INTERVAL").MustDuration(60 * time.Minute) SetRadarMapConfig() diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index b4d532ab0..bcbc5ea6d 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -233,7 +233,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if !reward.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("point balance not enough", tpl, &form) + ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tpl, &form) return } @@ -319,7 +319,7 @@ func CloudBrainRestart(ctx *context.Context) { if !reward.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) resultCode = "-1" - errorMsg = "insufficient points balance" + errorMsg = models.ErrInsufficientPointsBalance{}.Error() break } @@ -737,7 +737,7 @@ func CloudBrainAdminCommitImage(ctx *context.Context, form auth.CommitAdminImage UID: ctx.User.ID, Type: models.GetRecommondType(form.IsRecommend), Place: form.Place, - }) + }, ctx.User) if err != nil { log.Error("CommitImagefailed") if models.IsErrImageTagExist(err) { @@ -784,7 +784,7 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain CloudBrainType: form.Type, Topics: validTopics, UID: ctx.User.ID, - }) + }, ctx.User) if err != nil { log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"]) if models.IsErrImageTagExist(err) { @@ -1862,7 +1862,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("point balance not enough", tplCloudBrainBenchmarkNew, &form) + ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplCloudBrainBenchmarkNew, &form) return } @@ -2024,7 +2024,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) if !reward.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("point balance not enough", tpl, &form) + ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tpl, &form) return } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index dea996a50..6e5175e15 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -210,7 +210,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeDebug), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("point balance not enough", tplModelArtsNotebookNew, &form) + ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsNotebookNew, &form) return } count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) @@ -429,7 +429,7 @@ func NotebookManage(ctx *context.Context) { if !reward.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) resultCode = "-1" - errorMsg = "point balance not enough" + errorMsg = models.ErrInsufficientPointsBalance{}.Error() break return } @@ -1005,7 +1005,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeTrain), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("point balance not enough", tplModelArtsTrainJobNew, &form) + ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsTrainJobNew, &form) return } count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) @@ -1854,7 +1854,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeInference), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) inferenceJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("point balance not enough", tplModelArtsInferenceJobNew, &form) + ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsInferenceJobNew, &form) return } count, err := models.GetCloudbrainInferenceJobCountByUserID(ctx.User.ID) diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index eaae76c4f..3140b4c38 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -1,8 +1,10 @@ package point import ( + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/reward" "code.gitea.io/gitea/services/reward/point/account" "net/http" ) @@ -29,3 +31,47 @@ func GetPointAccount(ctx *context.Context) { } ctx.JSON(http.StatusOK, response.SuccessWithData(res)) } + +func GetPointRecordList(ctx *context.Context) { + operateType := ctx.Query("operate") + page := ctx.QueryInt("page") + var orderBy models.RewardOperateOrderBy + switch ctx.Query("sort") { + default: + orderBy = models.RewardOrderByID + } + t := models.GetRewardOperateTypeInstance(operateType) + if t == "" { + ctx.JSON(http.StatusOK, response.ServerError("param error")) + return + } + + r, err := reward.GetRewardRecordList(models.RewardRecordListOpts{ + ListOptions: models.ListOptions{PageSize: 20, Page: page}, + UserId: ctx.User.ID, + OperateType: t, + RewardType: models.RewardTypePoint, + OrderBy: orderBy, + }) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + + ctx.JSON(http.StatusOK, response.SuccessWithData(r)) + return +} + +func OperatePointAccountBalance(ctx *context.Context, req models.AdminRewardOperateReq) { + req.RewardType = models.RewardTypePoint + if req.OperateType.Name() == "" { + ctx.JSON(http.StatusOK, "param error") + return + } + err := reward.AdminBalanceOperate(req, ctx.User) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ctx.JSON(http.StatusOK, response.Success()) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 3ce633f93..0658765ca 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -324,6 +324,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) go routers.SocketManager.Run() + go task.RunTask() m.Get("/action/notification", routers.ActionNotification) m.Get("/reward/notification", routers.ActionNotification) m.Get("/recommend/org", routers.RecommendOrgFromPromote) @@ -594,6 +595,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/reward/point", func() { m.Get("/limiter/list", point.GetPointLimitConfigList) m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) + m.Post("/operate", binding.Bind(models.AdminRewardOperateReq{}), point.OperatePointAccountBalance) }) m.Group("/task/config", func() { @@ -601,6 +603,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/add", bindIgnErr(models.TaskConfigWithLimit{}), task.AddTaskConfig) m.Post("/add/batch", bindIgnErr(models.BatchLimitConfigVO{}), task.BatchAddTaskConfig) }) + }, adminReq) // ***** END: Admin ***** @@ -1330,6 +1333,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/reward/point", func() { m.Get("/account", point.GetPointAccount) + m.Get("/record/list", point.GetPointRecordList) }, reqSignIn) if setting.API.EnableSwagger { diff --git a/routers/task/task.go b/routers/task/task.go new file mode 100644 index 000000000..1d3b8595b --- /dev/null +++ b/routers/task/task.go @@ -0,0 +1,15 @@ +package task + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/services/task" +) + +func RunTask() { + for { + select { + case action := <-models.ActionChan4Task: + task.Accomplish(action) + } + } +} diff --git a/routers/user/setting/profile.go b/routers/user/setting/profile.go index 1c1e664d0..0d788b422 100755 --- a/routers/user/setting/profile.go +++ b/routers/user/setting/profile.go @@ -166,7 +166,7 @@ func AvatarPost(ctx *context.Context, form auth.AvatarForm) { if err := UpdateAvatarSetting(ctx, form, ctx.User); err != nil { ctx.Flash.Error(err.Error()) } else { - notification.NotifyChangeUserAvatar(ctx.User) + notification.NotifyChangeUserAvatar(ctx.User, form) ctx.Flash.Success(ctx.Tr("settings.update_avatar_success")) } diff --git a/services/reward/admin_operate.go b/services/reward/admin_operate.go new file mode 100644 index 000000000..1eec0f414 --- /dev/null +++ b/services/reward/admin_operate.go @@ -0,0 +1,50 @@ +package reward + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/reward/limiter" +) + +func AdminBalanceOperate(req models.AdminRewardOperateReq, doer *models.User) error { + logId := util.UUID() + _, err := models.InsertRewardAdminLog(&models.RewardAdminLog{ + LogId: logId, + Amount: req.Amount, + RewardType: req.RewardType.Name(), + TargetUserId: req.TargetUserId, + CreatorId: doer.ID, + CreatorName: doer.Name, + Remark: req.Remark, + Status: models.RewardAdminLogProcessing, + }) + if err != nil { + log.Error("AdminBalanceOperate InsertRewardAdminLog error.%v", err) + return err + } + + //reward + err = Operate(&models.RewardOperateContext{ + SourceType: models.SourceTypeAdminOperate, + SourceId: logId, + Tittle: "管理员操作", + Reward: models.Reward{ + Amount: req.Amount, + Type: req.RewardType, + }, + TargetUserId: req.TargetUserId, + RequestId: logId, + OperateType: req.OperateType, + Remark: req.Remark, + RejectPolicy: limiter.JustReject, + }) + + if err != nil { + log.Error("AdminBalanceOperate operate error.%v", err) + models.UpdateRewardAdminLogStatus(logId, models.RewardAdminLogProcessing, models.RewardAdminLogFailed) + return err + } + models.UpdateRewardAdminLogStatus(logId, models.RewardAdminLogProcessing, models.RewardAdminLogSuccess) + return nil +} diff --git a/services/reward/cloubrain_deduct.go b/services/reward/cloubrain_deduct.go index 61068a87a..ce23e2dc7 100644 --- a/services/reward/cloubrain_deduct.go +++ b/services/reward/cloubrain_deduct.go @@ -15,9 +15,11 @@ var ( TrainResourceSpecs *models.ResourceSpecs ) +const RUN_CLOUDBRAIN_TASK_TITTLE = "运行云脑任务" + //IsPointBalanceEnough check whether the user's point balance is bigger than task unit price func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int) bool { - if !setting.CloudBrainTaskPointPaySwitch { + if !setting.CloudBrainPaySwitch { return true } spec := getResourceSpec(jobType, resourceSpecId) @@ -33,7 +35,7 @@ func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int } func StartCloudBrainPointDeductTask(task models.Cloudbrain) { - if !setting.CloudBrainTaskPointPaySwitch { + if !setting.CloudBrainPaySwitch { return } @@ -48,11 +50,12 @@ func StartCloudBrainPointDeductTask(task models.Cloudbrain) { TargetUserId: task.UserID, RequestId: getCloudBrainPointTaskSourceId(task), OperateType: models.OperateTypeDecrease, - Delay: 30 * time.Minute, - Interval: 60 * time.Minute, + Delay: setting.CloudBrainPayDelay, + Interval: setting.CloudBrainPayInterval, UnitAmount: spec.UnitPrice, RewardType: models.RewardTypePoint, StartTime: time.Unix(int64(task.StartTime), 0), + Tittle: RUN_CLOUDBRAIN_TASK_TITTLE, }) } @@ -61,7 +64,7 @@ func StopCloudBrainPointDeductTask(task models.Cloudbrain) { } func getCloudBrainPointTaskSourceId(task models.Cloudbrain) string { - return models.SourceTypeRunCloudbrainTask.Name() + "_" + task.JobType + "_" + fmt.Sprint(task.Type) + "_" + fmt.Sprint(task.ID) + return fmt.Sprint(task.ID) } func getResourceSpec(jobType string, resourceSpecId int) *models.ResourceSpec { @@ -100,11 +103,11 @@ func StartCloudbrainPointDeductTask() { }() log.Debug("try to run CloudbrainPointDeductTask") end := time.Now() - start := end.Add(5 * time.Minute) + start := end.Add(-5 * time.Minute) if firstTimeFlag { //When it is executed for the first time, it needs to process the tasks of the last 1 hours. //This is done to prevent the application from hanging for a long time - start = end.Add(1 * time.Hour) + start = end.Add(-1 * time.Hour) firstTimeFlag = false } diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go index f094e3a43..88e72a1a1 100644 --- a/services/reward/limiter/limiter.go +++ b/services/reward/limiter/limiter.go @@ -12,14 +12,6 @@ import ( "time" ) -type limiterRejectPolicy string - -const ( - JustReject limiterRejectPolicy = "JUST_REJECT" - PermittedOnce limiterRejectPolicy = "PERMITTED_ONCE" - FillUp limiterRejectPolicy = "FillUp" -) - type limiterRunner struct { limiters []models.LimitConfig index int @@ -27,7 +19,7 @@ type limiterRunner struct { amount int64 limitCode string limitType models.LimitType - rejectPolicy limiterRejectPolicy + rejectPolicy models.LimiterRejectPolicy resultMap map[int]limitResult minRealAmount int64 } @@ -46,7 +38,7 @@ func newLimitResult(isLoss bool, planAmount int64, realAmount int64) limitResult } } -func newLimiterRunner(limitCode string, limitType models.LimitType, userId, amount int64, policy limiterRejectPolicy) *limiterRunner { +func newLimiterRunner(limitCode string, limitType models.LimitType, userId, amount int64, policy models.LimiterRejectPolicy) *limiterRunner { return &limiterRunner{ userId: userId, amount: amount, @@ -149,7 +141,7 @@ func (l *limiterRunner) limit(r models.LimitConfig) error { usedNum, err = redis_client.IncrBy(redisKey, n) } if p != nil { - redis_client.Expire(redisKey, int64(p.LeftTime.Seconds())) + redis_client.Expire(redisKey, p.LeftTime) } } if usedNum > r.LimitNum { @@ -158,16 +150,16 @@ func (l *limiterRunner) limit(r models.LimitConfig) error { return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) } switch l.rejectPolicy { - case FillUp: + case models.FillUp: exceed := usedNum - r.LimitNum realAmount := l.amount - exceed redis_client.IncrBy(redisKey, -1*exceed) l.resultMap[l.index] = newLimitResult(true, l.amount, realAmount) return nil - case JustReject: + case models.JustReject: redis_client.IncrBy(redisKey, -1*l.amount) return errors.New(fmt.Sprintf("%s:over limit", r.Tittle)) - case PermittedOnce: + case models.PermittedOnce: l.resultMap[l.index] = newLimitResult(false, l.amount, l.amount) return nil } @@ -200,8 +192,11 @@ func (l *limiterRunner) countInPeriod(r models.LimitConfig, p *models.PeriodResu } } -func CheckLimitWithFillUp(limitCode string, limitType models.LimitType, userId, amount int64) (int64, error) { - r := newLimiterRunner(limitCode, limitType, userId, amount, FillUp) +func CheckLimit(limitCode string, limitType models.LimitType, userId, amount int64, rejectPolicy models.LimiterRejectPolicy) (int64, error) { + if rejectPolicy == "" { + rejectPolicy = models.JustReject + } + r := newLimiterRunner(limitCode, limitType, userId, amount, rejectPolicy) err := r.Run() if err != nil { return 0, err @@ -209,18 +204,6 @@ func CheckLimitWithFillUp(limitCode string, limitType models.LimitType, userId, return r.minRealAmount, nil } -func CheckLimitWithPermittedOnce(limitCode string, limitType models.LimitType, userId, amount int64) error { - r := newLimiterRunner(limitCode, limitType, userId, amount, PermittedOnce) - err := r.Run() - return err -} - -func CheckLimit(limitCode string, limitType models.LimitType, userId, amount int64) error { - r := newLimiterRunner(limitCode, limitType, userId, amount, JustReject) - err := r.Run() - return err -} - func GetLimiters(limitCode string, limitType models.LimitType) ([]models.LimitConfig, error) { limiters, err := GetLimitersByLimitType(limitType) if err != nil { diff --git a/services/reward/operator.go b/services/reward/operator.go index 50ec01ff3..865ac10d0 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -5,7 +5,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/reward/point" "errors" "fmt" @@ -69,7 +68,7 @@ func Operate(ctx *models.RewardOperateContext) error { } //new reward operate record - recordId, err := initAwardOperateRecord(ctx) + recordId, err := initRewardOperateRecord(ctx) if err != nil { return err } @@ -110,9 +109,12 @@ func isHandled(sourceType string, requestId string, operateType string) (bool, e } -func initAwardOperateRecord(ctx *models.RewardOperateContext) (string, error) { +func initRewardOperateRecord(ctx *models.RewardOperateContext) (string, error) { + sn, err := generateOperateSerialNo(ctx.OperateType, ctx.Reward.Type) + if err != nil { + return "", err + } record := &models.RewardOperateRecord{ - RecordId: util.UUID(), UserId: ctx.TargetUserId, Amount: ctx.Reward.Amount, RewardType: ctx.Reward.Type.Name(), @@ -122,17 +124,22 @@ func initAwardOperateRecord(ctx *models.RewardOperateContext) (string, error) { OperateType: ctx.OperateType.Name(), Status: models.OperateStatusOperating, Remark: ctx.Remark, + Tittle: ctx.Tittle, + SerialNo: sn, } - _, err := models.InsertAwardOperateRecord(record) + _, err = models.InsertRewardOperateRecord(record) if err != nil { return "", err } - return record.RecordId, nil + return record.SerialNo, nil } func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (string, error) { + sn, err := generateOperateSerialNo(ctx.OperateType, ctx.RewardType) + if err != nil { + return "", err + } record := &models.RewardOperateRecord{ - RecordId: util.UUID(), UserId: ctx.TargetUserId, Amount: 0, RewardType: ctx.RewardType.Name(), @@ -142,12 +149,14 @@ func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (strin OperateType: ctx.OperateType.Name(), Status: models.OperateStatusOperating, Remark: ctx.Remark, + Tittle: ctx.Tittle, + SerialNo: sn, } - _, err := models.InsertAwardOperateRecord(record) + _, err = models.InsertRewardOperateRecord(record) if err != nil { return "", err } - return record.RecordId, nil + return record.SerialNo, nil } func updateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus string) error { @@ -230,5 +239,30 @@ func StopPeriodicTask(sourceType models.SourceType, sourceId string, operateType } now := time.Now() RunRewardTask(*task, now) - return models.StopPeriodicTask(task.ID, task.OperateRecordId, now) + return models.StopPeriodicTask(task.ID, task.OperateSerialNo, now) +} + +func generateOperateSerialNo(operateType models.RewardOperateType, rewardType models.RewardType) (string, error) { + s, err := GetSerialNoByRedis() + if err != nil { + return "", err + } + + switch operateType { + case models.OperateTypeIncrease: + s += "1" + case models.OperateTypeDecrease: + s += "2" + default: + s += "9" + } + + switch rewardType { + case models.RewardTypePoint: + s += "1" + default: + s += "9" + } + + return s, nil } diff --git a/services/reward/period_task.go b/services/reward/period_task.go index d00e8d0c4..846989652 100644 --- a/services/reward/period_task.go +++ b/services/reward/period_task.go @@ -15,7 +15,7 @@ func NewRewardPeriodicTask(operateRecordId string, opts *models.StartPeriodicTas task.DelaySeconds = int64(opts.Delay.Seconds()) task.IntervalSeconds = int64(opts.Interval.Seconds()) task.Amount = opts.UnitAmount - task.OperateRecordId = operateRecordId + task.OperateSerialNo = operateRecordId task.Status = models.PeriodicTaskStatusRunning task.NextExecuteTime = timeutil.TimeStamp(opts.StartTime.Add(opts.Delay).Unix()) @@ -54,9 +54,9 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) { return } defer lock.UnLock() - record, err := models.GetPointOperateRecordByRecordId(t.OperateRecordId) + record, err := models.GetPointOperateRecordBySerialNo(t.OperateSerialNo) if err != nil { - log.Error("RunRewardTask. GetPointOperateRecordByRecordId error. %v", err) + log.Error("RunRewardTask. GetPointOperateRecordBySerialNo error. %v", err) return } if record.Status != models.OperateStatusOperating { @@ -75,7 +75,7 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) { } err = operator.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeRunCloudbrainTask, - SourceId: t.OperateRecordId, + SourceId: t.OperateSerialNo, Reward: models.Reward{ Amount: n * t.Amount, Type: models.GetRewardTypeInstance(record.RewardType), diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 4b84cdd0c..51a3657ad 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -18,7 +18,7 @@ type PointOperator struct { } func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) bool { - realAmount, err := limiter.CheckLimitWithFillUp(ctx.SourceType.Name(), models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount) + realAmount, err := limiter.CheckLimit(ctx.SourceType.Name(), models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount, ctx.RejectPolicy) if err != nil { return true } diff --git a/services/reward/record.go b/services/reward/record.go new file mode 100644 index 000000000..ac28b3565 --- /dev/null +++ b/services/reward/record.go @@ -0,0 +1,20 @@ +package reward + +import "code.gitea.io/gitea/models" + +type RecordResponse struct { + Records []models.RewardOperateRecordShow + Total int64 +} + +func GetRewardRecordList(opts models.RewardRecordListOpts) (*RecordResponse, error) { + l, n, err := models.GetRewardRecordList(opts) + if err != nil { + return nil, err + } + r := make([]models.RewardOperateRecordShow, 0) + for _, v := range l { + r = append(r, v.ToShow()) + } + return &RecordResponse{Records: r, Total: n}, nil +} diff --git a/services/reward/serial.go b/services/reward/serial.go new file mode 100644 index 000000000..e9509c403 --- /dev/null +++ b/services/reward/serial.go @@ -0,0 +1,21 @@ +package reward + +import ( + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" + "fmt" + "math/rand" + "time" +) + +func GetSerialNoByRedis() (string, error) { + now := time.Now() + n, err := redis_client.IncrBy(redis_key.RewardSerialCounter(now), 1) + if err != nil { + return "", err + } + if n == 1 { + redis_client.Expire(redis_key.RewardSerialCounter(now), 5*time.Minute) + } + return now.Format("200601021504") + fmt.Sprint(rand.Intn(10)) + fmt.Sprintf("%02d", n), nil +} diff --git a/services/task/task.go b/services/task/task.go index 4c85ce52e..c2c4861a3 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -9,17 +9,33 @@ import ( "fmt" ) -func Accomplish(userId int64, taskType string) { - go accomplish(userId, taskType) +func Accomplish(action models.Action) { + switch action.OpType { + case models.ActionCreateRepo, + models.ActionCreateImage: + if action.Repo.IsPrivate { + return + } + case models.ActionCreateDebugGPUTask, + models.ActionCreateDebugNPUTask, + models.ActionCreateTrainTask, + models.ActionCreateInferenceTask, + models.ActionCreateBenchMarkTask, + models.ActionCreateGPUTrainTask: + action.OpType = models.ActionCreateCloudbrainTask + } + go accomplish(action) } -func accomplish(userId int64, taskType string) error { +func accomplish(action models.Action) error { defer func() { if err := recover(); err != nil { combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) log.Error("PANIC:%v", combinedErr) } }() + userId := action.ActUserID + taskType := fmt.Sprint(action.OpType) //get task config config, err := GetTaskConfig(taskType) @@ -33,7 +49,7 @@ func accomplish(userId int64, taskType string) error { } //is limited? - if isLimited(userId, config) { + if isLimited(userId, config, limiter.JustReject) { log.Info("task accomplish maximum times are reached,userId=%d taskType=%s", userId, taskType) return nil } @@ -45,6 +61,7 @@ func accomplish(userId int64, taskType string) error { ConfigId: config.ID, TaskCode: config.TaskCode, UserId: userId, + ActionId: action.ID, }) if err != nil { return err @@ -54,6 +71,7 @@ func accomplish(userId int64, taskType string) error { reward.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeAccomplishTask, SourceId: logId, + Tittle: config.Tittle, Reward: models.Reward{ Amount: config.AwardAmount, Type: models.GetRewardTypeInstance(config.AwardType), @@ -61,13 +79,14 @@ func accomplish(userId int64, taskType string) error { TargetUserId: userId, RequestId: logId, OperateType: models.OperateTypeIncrease, + RejectPolicy: limiter.FillUp, }) return nil } -func isLimited(userId int64, config *models.TaskConfig) bool { - if err := limiter.CheckLimit(config.TaskCode, models.LimitTypeTask, userId, 1); err != nil { +func isLimited(userId int64, config *models.TaskConfig, rejectPolicy limiter.LimiterRejectPolicy) bool { + if _, err := limiter.CheckLimit(config.TaskCode, models.LimitTypeTask, userId, 1, rejectPolicy); err != nil { return true } return false From 06161c0d174d64232895da429b1e001ead3b6289 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 24 Jun 2022 17:56:59 +0800 Subject: [PATCH 15/90] #2225 update --- models/limit_config.go | 35 ++++++++++++++++++++++ models/reward_operate_record.go | 4 +-- modules/auth/wechat/client.go | 2 ++ modules/redis/redis_key/limit_redis_key.go | 4 +-- routers/authentication/wechat.go | 1 + routers/reward/point/limit.go | 9 ++++++ routers/reward/point/point.go | 2 +- routers/routes/routes.go | 1 + services/reward/admin_operate.go | 3 +- services/reward/limiter/config.go | 21 ++++++++++++- services/reward/limiter/limiter.go | 2 +- services/reward/operator.go | 7 +++-- services/reward/point/point_operate.go | 6 ++-- services/task/task.go | 6 ++-- services/task/task_config.go | 2 +- 15 files changed, 86 insertions(+), 19 deletions(-) diff --git a/models/limit_config.go b/models/limit_config.go index ce8d2cfc2..17f0c23a2 100644 --- a/models/limit_config.go +++ b/models/limit_config.go @@ -59,27 +59,33 @@ type LimitConfig struct { LimitType string `xorm:"NOT NULL"` CreatorId int64 `xorm:"NOT NULL"` CreatorName string + DeleterId int64 + DeleterName string CreatedUnix timeutil.TimeStamp `xorm:"created"` DeletedAt timeutil.TimeStamp `xorm:"deleted"` } type LimitConfigVO struct { + ID int64 Tittle string RefreshRate string Scope string LimitNum int64 LimitCode string + LimitType string Creator string CreatedUnix timeutil.TimeStamp } func (l *LimitConfig) ToLimitConfigVO() *LimitConfigVO { return &LimitConfigVO{ + ID: l.ID, Tittle: l.Tittle, RefreshRate: l.RefreshRate, Scope: l.Scope, LimitNum: l.LimitNum, LimitCode: l.LimitCode, + LimitType: l.LimitType, Creator: l.CreatorName, CreatedUnix: l.CreatedUnix, } @@ -128,3 +134,32 @@ func AddLimitConfig(l *LimitConfig) error { sess.Commit() return nil } + +func DeleteLimitConfig(config LimitConfig, deleterId int64, deleterName string) error { + sess := x.NewSession() + defer sess.Close() + + _, err := x.ID(config.ID).Update(&LimitConfig{DeleterName: deleterName, DeleterId: deleterId}) + if err != nil { + sess.Rollback() + return err + } + _, err = x.ID(config.ID).Delete(&LimitConfig{}) + if err != nil { + sess.Rollback() + return err + } + sess.Commit() + return nil +} + +func GetLimitConfigById(id int64) (*LimitConfig, error) { + r := &LimitConfig{} + isOk, err := x.ID(id).Get(r) + if err != nil { + return nil, err + } else if !isOk { + return nil, nil + } + return r, nil +} diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index d58accfa5..889d291fa 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -109,7 +109,7 @@ const Semicolon = ";" type RewardOperateOrderBy string const ( - RewardOrderByID RewardOperateOrderBy = "id" + RewardOrderByIDDesc RewardOperateOrderBy = "id desc" ) type RewardOperateRecord struct { @@ -253,7 +253,7 @@ func GetRewardRecordList(opts RewardRecordListOpts) ([]RewardOperateRecord, int6 } if len(opts.OrderBy) == 0 { - opts.OrderBy = RewardOrderByID + opts.OrderBy = RewardOrderByIDDesc } r := make([]RewardOperateRecord, 0) diff --git a/modules/auth/wechat/client.go b/modules/auth/wechat/client.go index 6734977a1..5a81aa808 100644 --- a/modules/auth/wechat/client.go +++ b/modules/auth/wechat/client.go @@ -66,6 +66,7 @@ func getWechatRestyClient() *resty.Client { func callAccessToken() *AccessTokenResponse { client := getWechatRestyClient() + log.Info("start to get wechat access token") var result AccessTokenResponse _, err := client.R(). SetQueryParam("grant_type", GRANT_TYPE). @@ -77,6 +78,7 @@ func callAccessToken() *AccessTokenResponse { log.Error("get wechat access token failed,e=%v", err) return nil } + log.Info("get wechat access token result=%v", result) return &result } diff --git a/modules/redis/redis_key/limit_redis_key.go b/modules/redis/redis_key/limit_redis_key.go index a58a70fdb..02c4b1b9a 100644 --- a/modules/redis/redis_key/limit_redis_key.go +++ b/modules/redis/redis_key/limit_redis_key.go @@ -21,6 +21,6 @@ func LimitCount(userId int64, limitCode string, limitType string, scope string, } -func LimitConfig(limitType models.LimitType) string { - return KeyJoin(LIMIT_REDIS_PREFIX, limitType.Name(), "config") +func LimitConfig(limitType string) string { + return KeyJoin(LIMIT_REDIS_PREFIX, limitType, "config") } diff --git a/routers/authentication/wechat.go b/routers/authentication/wechat.go index 72871afb3..152348125 100644 --- a/routers/authentication/wechat.go +++ b/routers/authentication/wechat.go @@ -29,6 +29,7 @@ func GetQRCode4Bind(ctx *context.Context) { r, err := createQRCode4Bind(userId) if err != nil { + log.Error("GetQRCode4Bind failed,error=%v", err) ctx.JSON(200, map[string]interface{}{ "code": "9999", "msg": "Get QR code failed", diff --git a/routers/reward/point/limit.go b/routers/reward/point/limit.go index a831169f8..6c5ec5827 100644 --- a/routers/reward/point/limit.go +++ b/routers/reward/point/limit.go @@ -25,3 +25,12 @@ func AddPointLimitConfig(ctx *context.Context, config models.LimitConfigVO) { } ctx.JSON(http.StatusOK, response.Success()) } +func DeletePointLimitConfig(ctx *context.Context) { + id := ctx.QueryInt64("id") + err := limiter.DeleteLimitConfig(id, ctx.User) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ctx.JSON(http.StatusOK, response.Success()) +} diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index 3140b4c38..e1a751495 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -38,7 +38,7 @@ func GetPointRecordList(ctx *context.Context) { var orderBy models.RewardOperateOrderBy switch ctx.Query("sort") { default: - orderBy = models.RewardOrderByID + orderBy = models.RewardOrderByIDDesc } t := models.GetRewardOperateTypeInstance(operateType) if t == "" { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 0658765ca..652560119 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -595,6 +595,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/reward/point", func() { m.Get("/limiter/list", point.GetPointLimitConfigList) m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) + m.Post("/limiter/delete", point.DeletePointLimitConfig) m.Post("/operate", binding.Bind(models.AdminRewardOperateReq{}), point.OperatePointAccountBalance) }) diff --git a/services/reward/admin_operate.go b/services/reward/admin_operate.go index 1eec0f414..1fdd942d2 100644 --- a/services/reward/admin_operate.go +++ b/services/reward/admin_operate.go @@ -4,7 +4,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/services/reward/limiter" ) func AdminBalanceOperate(req models.AdminRewardOperateReq, doer *models.User) error { @@ -37,7 +36,7 @@ func AdminBalanceOperate(req models.AdminRewardOperateReq, doer *models.User) er RequestId: logId, OperateType: req.OperateType, Remark: req.Remark, - RejectPolicy: limiter.JustReject, + RejectPolicy: models.JustReject, }) if err != nil { diff --git a/services/reward/limiter/config.go b/services/reward/limiter/config.go index 12204b2c5..8de00d178 100644 --- a/services/reward/limiter/config.go +++ b/services/reward/limiter/config.go @@ -18,6 +18,9 @@ func GetLimitConfigList(limitType models.LimitType) ([]*models.LimitConfigVO, er } return result, nil } +func GetLimitConfigById(id int64) (*models.LimitConfig, error) { + return models.GetLimitConfigById(id) +} func AddLimitConfig(config *models.LimitConfigVO, doer *models.User, limitType models.LimitType) error { r := &models.LimitConfig{ @@ -36,6 +39,22 @@ func AddLimitConfig(config *models.LimitConfigVO, doer *models.User, limitType m log.Error("add limit config error,config:%v err:%v", config, err) return err } - redis_client.Del(redis_key.LimitConfig(limitType)) + redis_client.Del(redis_key.LimitConfig(limitType.Name())) + return nil +} + +func DeleteLimitConfig(id int64, doer *models.User) error { + config, err := GetLimitConfigById(id) + if err != nil { + log.Error("GetLimitConfigById err,e=%v", err) + return err + } + err = models.DeleteLimitConfig(*config, doer.ID, doer.Name) + + if err != nil { + log.Error("add limit config error,config:%v err:%v", config, err) + return err + } + redis_client.Del(redis_key.LimitConfig(config.LimitType)) return nil } diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go index 88e72a1a1..a73779ac1 100644 --- a/services/reward/limiter/limiter.go +++ b/services/reward/limiter/limiter.go @@ -219,7 +219,7 @@ func GetLimiters(limitCode string, limitType models.LimitType) ([]models.LimitCo } func GetLimitersByLimitType(limitType models.LimitType) ([]models.LimitConfig, error) { - redisKey := redis_key.LimitConfig(limitType) + redisKey := redis_key.LimitConfig(limitType.Name()) val, _ := redis_client.Get(redisKey) if val != "" { if val == redis_key.EMPTY_REDIS_VAL { diff --git a/services/reward/operator.go b/services/reward/operator.go index 865ac10d0..79f82acbd 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -16,7 +16,7 @@ var RewardOperatorMap = map[string]RewardOperator{ } type RewardOperator interface { - IsLimited(ctx *models.RewardOperateContext) bool + IsLimited(ctx *models.RewardOperateContext) error Operate(ctx *models.RewardOperateContext) error } @@ -62,8 +62,9 @@ func Operate(ctx *models.RewardOperateContext) error { if ctx.OperateType == models.OperateTypeIncrease { //is limited? - if isLimited := operator.IsLimited(ctx); isLimited { - return nil + if err := operator.IsLimited(ctx); err != nil { + log.Info("operator IsLimited, err=%v", err) + return err } } diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 51a3657ad..1a4ff762b 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -17,16 +17,16 @@ const LossMsg = "达到奖励上限，应得%d积分，实得%d积分" type PointOperator struct { } -func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) bool { +func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) error { realAmount, err := limiter.CheckLimit(ctx.SourceType.Name(), models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount, ctx.RejectPolicy) if err != nil { - return true + return err } if realAmount < ctx.Reward.Amount { ctx.Remark = models.AppendRemark(ctx.Remark, fmt.Sprintf(LossMsg, ctx.Reward.Amount, realAmount)) ctx.Reward.Amount = realAmount } - return false + return nil } func (operator *PointOperator) Operate(ctx *models.RewardOperateContext) error { diff --git a/services/task/task.go b/services/task/task.go index c2c4861a3..0dfc38b6c 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -49,7 +49,7 @@ func accomplish(action models.Action) error { } //is limited? - if isLimited(userId, config, limiter.JustReject) { + if isLimited(userId, config, models.JustReject) { log.Info("task accomplish maximum times are reached,userId=%d taskType=%s", userId, taskType) return nil } @@ -79,13 +79,13 @@ func accomplish(action models.Action) error { TargetUserId: userId, RequestId: logId, OperateType: models.OperateTypeIncrease, - RejectPolicy: limiter.FillUp, + RejectPolicy: models.FillUp, }) return nil } -func isLimited(userId int64, config *models.TaskConfig, rejectPolicy limiter.LimiterRejectPolicy) bool { +func isLimited(userId int64, config *models.TaskConfig, rejectPolicy models.LimiterRejectPolicy) bool { if _, err := limiter.CheckLimit(config.TaskCode, models.LimitTypeTask, userId, 1, rejectPolicy); err != nil { return true } diff --git a/services/task/task_config.go b/services/task/task_config.go index 4e02b4972..0184ca15b 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -95,7 +95,7 @@ func AddTaskConfig(config models.TaskConfigWithLimit, doer *models.User) error { log.Error("add task config error,config:%v err:%v", config, err) return err } - redis_client.Del(redis_key.LimitConfig(models.LimitTypeTask)) + redis_client.Del(redis_key.LimitConfig(models.LimitTypeTask.Name())) redis_client.Del(redis_key.TaskConfigList()) return nil } From ee415d75aba7609b18272688ae9ac10b57804a28 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 27 Jun 2022 12:05:51 +0800 Subject: [PATCH 16/90] #2225 fix bug --- services/reward/notify.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/reward/notify.go b/services/reward/notify.go index aa18fbe39..f5b270d94 100644 --- a/services/reward/notify.go +++ b/services/reward/notify.go @@ -18,11 +18,11 @@ func NotifyRewardOperation(userId, amount int64, rewardType models.RewardType, o OperateType: operateType, } b, _ := json.Marshal(data) - redis_client.ZAdd(redis_key.RewardOperateNotification(), string(b), float64(time.Now().UnixMilli())) + redis_client.ZAdd(redis_key.RewardOperateNotification(), string(b), float64(time.Now().Unix())) } func GetRewardOperation(since, until timeutil.TimeStamp) []models.UserRewardOperation { - list, err := redis_client.ZRangeByScore(redis_key.RewardOperateNotification(), float64(since*1000), float64(until*1000)) + list, err := redis_client.ZRangeByScore(redis_key.RewardOperateNotification(), float64(since), float64(until)) if err != nil { return nil } @@ -38,7 +38,7 @@ func GetRewardOperation(since, until timeutil.TimeStamp) []models.UserRewardOper Msg: GetRewardOperateMsg(t), }) } - redis_client.ZRemRangeByScore(redis_key.RewardOperateNotification(), float64(since*1000), float64(until*1000)) + redis_client.ZRemRangeByScore(redis_key.RewardOperateNotification(), float64(since), float64(until)) return r } From 93317c7a81b102e4b8a7002f97827f40a0bb95a4 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 27 Jun 2022 17:44:14 +0800 Subject: [PATCH 17/90] #2225 add point page --- models/reward_operate_record.go | 27 +++++++++------- routers/reward/point/point.go | 8 ++++- routers/routes/routes.go | 1 + services/reward/admin_operate.go | 11 ++++--- services/reward/period_task.go | 45 +++++++++++++++++--------- services/reward/point/point_operate.go | 5 +++ services/reward/record.go | 8 +++-- 7 files changed, 70 insertions(+), 35 deletions(-) diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 889d291fa..394fba1cf 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -141,18 +141,22 @@ func (r RewardOperateRecord) ToShow() RewardOperateRecordShow { return RewardOperateRecordShow{ SerialNo: r.SerialNo, Date: r.CreatedUnix, - Tittle: r.Tittle, OperateType: r.OperateType, Amount: r.Amount, + Remark: r.Remark, } } type RewardOperateRecordShow struct { SerialNo string Date timeutil.TimeStamp - Tittle string + Status string OperateType string Amount int64 + Action Action + Cloudbrain Cloudbrain + SourceType SourceType + Remark string } func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) { @@ -207,15 +211,16 @@ func SumRewardAmountInTaskPeriod(rewardType string, sourceType string, userId in } type RewardOperateContext struct { - SourceType SourceType - SourceId string - Tittle string - Remark string - Reward Reward - TargetUserId int64 - RequestId string - OperateType RewardOperateType - RejectPolicy LimiterRejectPolicy + SourceType SourceType + SourceId string + Tittle string + Remark string + Reward Reward + TargetUserId int64 + RequestId string + OperateType RewardOperateType + RejectPolicy LimiterRejectPolicy + PermittedNegative bool } type Reward struct { diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index e1a751495..edf41cd72 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -2,6 +2,7 @@ package point import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/routers/response" "code.gitea.io/gitea/services/reward" @@ -9,6 +10,8 @@ import ( "net/http" ) +const tplPoint base.TplName = "/reward/point" + type AccountResponse struct { AccountCode string Balance int64 @@ -24,7 +27,6 @@ func GetPointAccount(ctx *context.Context) { return } res := &AccountResponse{ - AccountCode: a.AccountCode, Balance: a.Balance, TotalEarned: a.TotalEarned, TotalConsumed: a.TotalConsumed, @@ -75,3 +77,7 @@ func OperatePointAccountBalance(ctx *context.Context, req models.AdminRewardOper } ctx.JSON(http.StatusOK, response.Success()) } + +func GetPointPage(ctx *context.Context) { + ctx.HTML(200, tplPoint) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 946f9ddd2..ea2980364 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -595,6 +595,7 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Group("/reward/point", func() { + m.Get("", point.GetPointPage) m.Get("/limiter/list", point.GetPointLimitConfigList) m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) m.Post("/limiter/delete", point.DeletePointLimitConfig) diff --git a/services/reward/admin_operate.go b/services/reward/admin_operate.go index 1fdd942d2..b8490a3a8 100644 --- a/services/reward/admin_operate.go +++ b/services/reward/admin_operate.go @@ -32,11 +32,12 @@ func AdminBalanceOperate(req models.AdminRewardOperateReq, doer *models.User) er Amount: req.Amount, Type: req.RewardType, }, - TargetUserId: req.TargetUserId, - RequestId: logId, - OperateType: req.OperateType, - Remark: req.Remark, - RejectPolicy: models.JustReject, + TargetUserId: req.TargetUserId, + RequestId: logId, + OperateType: req.OperateType, + Remark: req.Remark, + RejectPolicy: models.JustReject, + PermittedNegative: true, }) if err != nil { diff --git a/services/reward/period_task.go b/services/reward/period_task.go index 846989652..3fa416dab 100644 --- a/services/reward/period_task.go +++ b/services/reward/period_task.go @@ -63,31 +63,41 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) { log.Info("RunRewardTask. operate record is finished,record=%+v", record) return } - n, nextTime := countExecuteTimes(t, now) + n, _ := countExecuteTimes(t, now) if n == 0 { return } + //get operator operator := GetOperator(models.GetRewardTypeInstance(record.RewardType)) if operator == nil { log.Error("RunRewardTask. operator of reward type is not exist") return } - err = operator.Operate(&models.RewardOperateContext{ - SourceType: models.SourceTypeRunCloudbrainTask, - SourceId: t.OperateSerialNo, - Reward: models.Reward{ - Amount: n * t.Amount, - Type: models.GetRewardTypeInstance(record.RewardType), - }, - TargetUserId: record.UserId, - OperateType: models.GetRewardOperateTypeInstance(record.OperateType), - }) - if err != nil { - log.Error("RunRewardTask.operator operate error.%v", err) - return + nextTime := t.NextExecuteTime + for i := 0; int64(i) <= n; i++ { + err = operator.Operate(&models.RewardOperateContext{ + SourceType: models.SourceTypeRunCloudbrainTask, + SourceId: t.OperateSerialNo, + Reward: models.Reward{ + Amount: t.Amount, + Type: models.GetRewardTypeInstance(record.RewardType), + }, + TargetUserId: record.UserId, + OperateType: models.GetRewardOperateTypeInstance(record.OperateType), + }) + if err != nil { + log.Error("RunRewardTask.operator operate error.%v", err) + if models.IsErrInsufficientPointsBalance(err) { + StopCloudbrainTask(record) + return + } + return + } + models.IncrRewardTaskSuccessCount(t, n, nextTime) + nextTime = timeutil.TimeStamp(int64(nextTime) + t.IntervalSeconds) } - models.IncrRewardTaskSuccessCount(t, n, nextTime) + } func countExecuteTimes(t models.RewardPeriodicTask, now time.Time) (int64, timeutil.TimeStamp) { @@ -101,3 +111,8 @@ func countExecuteTimes(t models.RewardPeriodicTask, now time.Time) (int64, timeu newNextTime := timeutil.TimeStamp(nextTime + n*interval) return n, newNextTime } + +func StopCloudbrainTask(r *models.RewardOperateRecord) { + //todo + +} diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 1a4ff762b..0115c288a 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -2,6 +2,7 @@ package point import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" @@ -46,6 +47,10 @@ func (operator *PointOperator) Operate(ctx *models.RewardOperateContext) error { if ctx.OperateType == models.OperateTypeIncrease { err = na.Increase(ctx.Reward.Amount, ctx.SourceId) } else if ctx.OperateType == models.OperateTypeDecrease { + if !ctx.PermittedNegative && na.Balance < ctx.Reward.Amount { + log.Info("account balance is not enough,ctx=%v", ctx) + return &models.ErrInsufficientPointsBalance{} + } err = na.Decrease(ctx.Reward.Amount, ctx.SourceId) } if err != nil { diff --git a/services/reward/record.go b/services/reward/record.go index ac28b3565..157e53b53 100644 --- a/services/reward/record.go +++ b/services/reward/record.go @@ -3,8 +3,10 @@ package reward import "code.gitea.io/gitea/models" type RecordResponse struct { - Records []models.RewardOperateRecordShow - Total int64 + Records []models.RewardOperateRecordShow + Total int64 + PageSize int + Page int } func GetRewardRecordList(opts models.RewardRecordListOpts) (*RecordResponse, error) { @@ -16,5 +18,5 @@ func GetRewardRecordList(opts models.RewardRecordListOpts) (*RecordResponse, err for _, v := range l { r = append(r, v.ToShow()) } - return &RecordResponse{Records: r, Total: n}, nil + return &RecordResponse{Records: r, Total: n, Page: opts.Page, PageSize: opts.PageSize}, nil } From 2280f0be44f365d698fdf8ee082f036736b03b53 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 27 Jun 2022 18:03:34 +0800 Subject: [PATCH 18/90] #2225 fix point page path --- routers/routes/routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index ea2980364..47ee3c50a 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -595,7 +595,6 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Group("/reward/point", func() { - m.Get("", point.GetPointPage) m.Get("/limiter/list", point.GetPointLimitConfigList) m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) m.Post("/limiter/delete", point.DeletePointLimitConfig) @@ -1337,6 +1336,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn) m.Group("/reward/point", func() { + m.Get("", point.GetPointPage) m.Get("/account", point.GetPointAccount) m.Get("/record/list", point.GetPointRecordList) }, reqSignIn) From 44586c567ba4842335b006559416bc1cb610a6a5 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 28 Jun 2022 18:14:23 +0800 Subject: [PATCH 19/90] #2225 update --- models/action.go | 15 ++ models/cloudbrain.go | 110 ++++++++++++- models/reward_admin_log.go | 15 ++ models/reward_operate_record.go | 147 +++++++++++++++--- models/reward_periodic_task.go | 2 +- models/task_accomplish_log.go | 1 - modules/cron/tasks_basic.go | 2 +- routers/repo/cloudbrain.go | 10 +- routers/repo/modelarts.go | 10 +- routers/reward/point/point.go | 1 - services/reward/cloubrain_deduct.go | 82 ++++------ services/reward/operator.go | 51 +++--- services/reward/period_task.go | 33 ++-- .../reward/point/account/point_account.go | 18 +++ services/reward/record.go | 18 ++- services/task/task.go | 7 +- 16 files changed, 386 insertions(+), 136 deletions(-) diff --git a/models/action.go b/models/action.go index 456d5c6bc..ff16dcd3f 100755 --- a/models/action.go +++ b/models/action.go @@ -412,3 +412,18 @@ func GetUnTransformedActions() ([]*Action, error) { Find(&actions) return actions, err } + +func GetActionByIds(ids []int64) ([]*Action, error) { + if len(ids) == 0 { + return nil, nil + } + actions := make([]*Action, 0) + err := x.In("id", ids).Find(&actions) + if err != nil { + return nil, err + } + if err := ActionList(actions).LoadAttributes(); err != nil { + return nil, fmt.Errorf("ActionList loadAttributes: %v", err) + } + return actions, nil +} diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 33b85de20..06cd42258 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -168,6 +168,72 @@ type Cloudbrain struct { EndTime timeutil.TimeStamp } +type CloudbrainShow struct { + JobID string `xorm:"INDEX NOT NULL"` + JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` + JobName string + DisplayJobName string + Status string + UserID int64 `xorm:"INDEX NOT NULL"` + RepoID int64 `xorm:"INDEX NOT NULL"` + SubTaskName string + ContainerID string + ContainerIp string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + Duration int64 `xorm:"DEFAULT 0"` //运行时长 单位秒 + TrainJobDuration string `xorm:"DEFAULT '00:00:00'"` + Image string //镜像名称 + GpuQueue string //GPU类型即GPU队列 + ResourceSpecId int //GPU规格id + DeletedAt time.Time `xorm:"deleted"` + CanDebug bool `xorm:"-"` + CanDel bool `xorm:"-"` + CanModify bool `xorm:"-"` + Type int + BenchmarkTypeID int + BenchmarkChildTypeID int + + VersionID int64 //版本id + VersionName string `xorm:"INDEX"` //当前版本 + Uuid string //数据集id + DatasetName string + VersionCount int //任务的当前版本数量，不包括删除的 + IsLatestVersion string //是否是最新版本，1是，0否 + CommitID string //提交的仓库代码id + PreVersionName string //父版本名称 + ComputeResource string //计算资源，例如npu + EngineID int64 //引擎id + + TrainUrl string //输出模型的obs路径 + BranchName string //分支名称 + Parameters string //传给modelarts的param参数 + BootFile string //启动文件 + DataUrl string //数据集的obs路径 + LogUrl string //日志输出的obs路径 + PreVersionId int64 //父版本的版本id + FlavorCode string //modelarts上的规格id + Description string `xorm:"varchar(256)"` //描述 + WorkServerNumber int //节点数 + FlavorName string //规格名称 + EngineName string //引擎名称 + TotalVersionCount int //任务的所有版本数量，包括删除的 + + LabelName string //标签名称 + ModelName string //模型名称 + ModelVersion string //模型版本 + CkptName string //权重文件名称 + ResultUrl string //推理结果的obs路径 + + User *User `xorm:"-"` + Repo *Repository `xorm:"-"` + BenchmarkType string `xorm:"-"` //算法评测，模型评测 + BenchmarkTypeName string `xorm:"-"` + BenchmarkTypeRankLink string `xorm:"-"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp +} + func (task *Cloudbrain) ComputeAndSetDuration() { var d int64 if task.StartTime == 0 { @@ -1844,9 +1910,51 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er func GetStartedCloudbrainTaskByUpdatedUnix(startTime, endTime time.Time) ([]Cloudbrain, error) { r := make([]Cloudbrain, 0) - err := x.Where("updated_unix >= ? and updated_unix <= ? and start_time > 0", startTime.Unix(), endTime.Unix()).Find(&r) + err := x.Where("updated_unix >= ? and updated_unix <= ? and start_time > 0", startTime.Unix(), endTime.Unix()).Unscoped().Find(&r) if err != nil { return nil, err } return r, nil } + +func GetCloudbrainByIds(ids []int64) ([]Cloudbrain, error) { + if len(ids) == 0 { + return nil, nil + } + cloudbrains := make([]Cloudbrain, 0) + err := x.In("id", ids).Unscoped().Find(&cloudbrains) + if err != nil { + return nil, err + } + return cloudbrains, nil +} + +var ( + DebugResourceSpecs *ResourceSpecs + TrainResourceSpecs *ResourceSpecs +) + +func GetResourceSpec(jobType string, resourceSpecId int) *ResourceSpec { + if jobType == string(JobTypeTrain) { + if TrainResourceSpecs == nil { + json.Unmarshal([]byte(setting.TrainResourceSpecs), &TrainResourceSpecs) + } + for _, spec := range TrainResourceSpecs.ResourceSpec { + if resourceSpecId == spec.Id { + return spec + } + } + } else { + if DebugResourceSpecs == nil { + json.Unmarshal([]byte(setting.ResourceSpecs), &DebugResourceSpecs) + } + for _, spec := range DebugResourceSpecs.ResourceSpec { + if resourceSpecId == spec.Id { + return spec + } + } + + } + return nil + +} diff --git a/models/reward_admin_log.go b/models/reward_admin_log.go index 5e4258682..b1a55af13 100644 --- a/models/reward_admin_log.go +++ b/models/reward_admin_log.go @@ -2,6 +2,7 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" + "strings" ) const ( @@ -44,3 +45,17 @@ func UpdateRewardAdminLogStatus(logId string, oldStatus, newStatus int) error { } return nil } + +func GetRewardAdminLogByLogIds(logIds []string) ([]RewardAdminLog, error) { + if len(logIds) == 0 { + return nil, nil + } + adminLogs := make([]RewardAdminLog, 0) + err := x.SQL("select rdl.id,rdl.log_id,rdl.amount,rdl.reward_type,rdl.remark,rdl.status,rdl.target_user_id,rdl.creator_id,u.name as creator_name "+ + "from reward_admin_log rdl left join public.user u on rdl.creator_id = u.id "+ + "where rdl.log_id in (?)", strings.Join(logIds, ",")).Find(&adminLogs) + if err != nil { + return nil, err + } + return adminLogs, nil +} diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 394fba1cf..04f43a8bd 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -2,6 +2,8 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" + "fmt" + "strconv" "strings" "xorm.io/builder" ) @@ -112,21 +114,120 @@ const ( RewardOrderByIDDesc RewardOperateOrderBy = "id desc" ) +type RewardRecordList []*RewardOperateRecord +type RewardRecordShowList []*RewardOperateRecordShow + +func (l *RewardRecordList) ToShow() (RewardRecordShowList, error) { + actionMap, err := l.GetRewardRecordAction() + adminLogMap, err := l.GetRewardRecordAdminLog() + CloudbrainMap, err := l.GetRewardRecordCloudbrainTask() + if err != nil { + return nil, err + } + result := make([]*RewardOperateRecordShow, 0) + for _, v := range *l { + temp := v.ToShow() + switch v.SourceType { + case SourceTypeAccomplishTask.Name(): + temp.Action = actionMap[v.SourceId] + case SourceTypeAdminOperate.Name(): + temp.AdminLog = adminLogMap[v.SourceId] + case SourceTypeRunCloudbrainTask.Name(): + temp.Cloudbrain = CloudbrainMap[v.SourceId] + } + result = append(result, &temp) + } + + return result, nil +} + +func (l *RewardRecordList) GetRewardRecordAction() (map[string]Action, error) { + if len(*l) == 0 { + return nil, nil + } + actionIds := make([]int64, 0) + for _, r := range *l { + if r.SourceType != SourceTypeAccomplishTask.Name() { + continue + } + i, _ := strconv.ParseInt(r.SourceId, 10, 64) + actionIds = append(actionIds, i) + } + actions, err := GetActionByIds(actionIds) + if err != nil { + return nil, err + } + result := make(map[string]Action, 0) + for _, v := range actions { + result[fmt.Sprint(v.ID)] = *v + } + return result, nil + +} + +func (l *RewardRecordList) GetRewardRecordAdminLog() (map[string]RewardAdminLog, error) { + if len(*l) == 0 { + return nil, nil + } + logIds := make([]string, 0) + for _, r := range *l { + if r.SourceType != SourceTypeAdminOperate.Name() { + continue + } + logIds = append(logIds, r.SourceId) + } + logs, err := GetRewardAdminLogByLogIds(logIds) + if err != nil { + return nil, err + } + result := make(map[string]RewardAdminLog, 0) + for _, v := range logs { + result[fmt.Sprint(v.LogId)] = v + } + return result, nil + +} + +func (l *RewardRecordList) GetRewardRecordCloudbrainTask() (map[string]Cloudbrain, error) { + if len(*l) == 0 { + return nil, nil + } + cloudbrainIds := make([]int64, 0) + for _, r := range *l { + if r.SourceType != SourceTypeRunCloudbrainTask.Name() { + continue + } + i, _ := strconv.ParseInt(r.SourceId, 10, 64) + cloudbrainIds = append(cloudbrainIds, i) + } + cloudbrains, err := GetCloudbrainByIds(cloudbrainIds) + if err != nil { + return nil, err + } + result := make(map[string]Cloudbrain, 0) + for _, v := range cloudbrains { + result[fmt.Sprint(v.ID)] = v + } + return result, nil + +} + type RewardOperateRecord struct { - ID int64 `xorm:"pk autoincr"` - SerialNo string `xorm:"INDEX NOT NULL"` - UserId int64 `xorm:"INDEX NOT NULL"` - Amount int64 `xorm:"NOT NULL"` - Tittle string - RewardType string `xorm:"NOT NULL"` - SourceType string `xorm:"NOT NULL"` - SourceId string `xorm:"INDEX NOT NULL"` - RequestId string `xorm:"INDEX NOT NULL"` - OperateType string `xorm:"NOT NULL"` - Status string `xorm:"NOT NULL"` - Remark string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + ID int64 `xorm:"pk autoincr"` + SerialNo string `xorm:"INDEX NOT NULL"` + UserId int64 `xorm:"INDEX NOT NULL"` + Amount int64 `xorm:"NOT NULL"` + Tittle string + RewardType string `xorm:"NOT NULL"` + SourceType string `xorm:"NOT NULL"` + SourceId string `xorm:"INDEX NOT NULL"` + RequestId string `xorm:"INDEX NOT NULL"` + OperateType string `xorm:"NOT NULL"` + Status string `xorm:"NOT NULL"` + Remark string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + FinishedUnix timeutil.TimeStamp `xorm:"INDEX"` } type AdminRewardOperateReq struct { @@ -144,6 +245,8 @@ func (r RewardOperateRecord) ToShow() RewardOperateRecordShow { OperateType: r.OperateType, Amount: r.Amount, Remark: r.Remark, + Status: r.Status, + SourceType: r.SourceType, } } @@ -153,10 +256,11 @@ type RewardOperateRecordShow struct { Status string OperateType string Amount int64 + Remark string + SourceType string Action Action Cloudbrain Cloudbrain - SourceType SourceType - Remark string + AdminLog RewardAdminLog } func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) { @@ -189,11 +293,12 @@ func InsertRewardOperateRecord(tl *RewardOperateRecord) (int64, error) { return x.Insert(tl) } -func UpdateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus string) (int64, error) { +func UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus string) (int64, error) { r := &RewardOperateRecord{ - Status: newStatus, + Status: newStatus, + FinishedUnix: timeutil.TimeStampNow(), } - return x.Cols("status").Where("source_type=? and request_id=? and status=?", sourceType, requestId, oldStatus).Update(r) + return x.Cols("status", "finished_unix").Where("source_type=? and request_id=? and status=?", sourceType, requestId, OperateStatusOperating).Update(r) } func SumRewardAmountInTaskPeriod(rewardType string, sourceType string, userId int64, period *PeriodResult) (int64, error) { @@ -252,7 +357,7 @@ type RewardRecordListOpts struct { OrderBy RewardOperateOrderBy } -func GetRewardRecordList(opts RewardRecordListOpts) ([]RewardOperateRecord, int64, error) { +func GetRewardRecordList(opts RewardRecordListOpts) (RewardRecordList, int64, error) { if opts.Page <= 0 { opts.Page = 1 } @@ -261,7 +366,7 @@ func GetRewardRecordList(opts RewardRecordListOpts) ([]RewardOperateRecord, int6 opts.OrderBy = RewardOrderByIDDesc } - r := make([]RewardOperateRecord, 0) + r := make([]*RewardOperateRecord, 0) cond := builder.NewCond() if opts.UserId > 0 { cond = cond.And(builder.Eq{"user_id": opts.UserId}) diff --git a/models/reward_periodic_task.go b/models/reward_periodic_task.go index 5db5301b5..a859676d6 100644 --- a/models/reward_periodic_task.go +++ b/models/reward_periodic_task.go @@ -77,7 +77,7 @@ func IncrRewardTaskSuccessCount(t RewardPeriodicTask, count int64, nextTime time sess.Rollback() return err } - _, err = sess.Exec("update reward_operate_record set amount = amount + ? ,updated_unix = ? where serial_no = ?", count*t.Amount, timeutil.TimeStampNow(), t.OperateSerialNo) + _, err = sess.Exec("update reward_operate_record set amount = amount + ? ,updated_unix = ? where serial_no = ?", t.Amount, timeutil.TimeStampNow(), t.OperateSerialNo) if err != nil { sess.Rollback() return err diff --git a/models/task_accomplish_log.go b/models/task_accomplish_log.go index a1edb71ee..75494bfa2 100644 --- a/models/task_accomplish_log.go +++ b/models/task_accomplish_log.go @@ -7,7 +7,6 @@ import ( type TaskAccomplishLog struct { ID int64 `xorm:"pk autoincr"` - LogId string `xorm:"INDEX NOT NULL"` ConfigId int64 `xorm:"NOT NULL"` TaskCode string `xorm:"NOT NULL"` UserId int64 `xorm:"INDEX NOT NULL"` diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 5892699eb..7d6c7df33 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -251,6 +251,6 @@ func initBasicTasks() { registerSyncCloudbrainStatus() registerHandleOrgStatistic() - registerRewardPeriodTask() + //registerRewardPeriodTask() registerCloudbrainPointDeductTask() } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f628a6f0a..29c8b97bb 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,7 +2,7 @@ package repo import ( "bufio" - "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/services/reward/point/account" "encoding/json" "errors" "fmt" @@ -230,7 +230,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { command = commandTrain } - if !reward.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tpl, &form) @@ -318,7 +318,7 @@ func CloudBrainRestart(ctx *context.Context) { var status = string(models.JobWaiting) task := ctx.Cloudbrain for { - if !reward.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) resultCode = "-1" errorMsg = models.ErrInsufficientPointsBalance{}.Error() @@ -1870,7 +1870,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo repo := ctx.Repo.Repository - if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplCloudBrainBenchmarkNew, &form) @@ -2032,7 +2032,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) tpl := tplCloudBrainBenchmarkNew command := cloudbrain.Command - if !reward.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tpl, &form) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 1fbf6c622..bff9ec525 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2,7 +2,7 @@ package repo import ( "archive/zip" - "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/services/reward/point/account" "encoding/json" "errors" "fmt" @@ -207,7 +207,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm repo := ctx.Repo.Repository resourceSpecId := form.ResourceSpecId - if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeDebug), resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeDebug), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsNotebookNew, &form) @@ -426,7 +426,7 @@ func NotebookManage(ctx *context.Context) { errorMsg = "you have no right to restart the job" break } - if !reward.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) resultCode = "-1" errorMsg = models.ErrInsufficientPointsBalance{}.Error() @@ -1002,7 +1002,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) EngineName := form.EngineName resourceSpecId := form.ResourceSpecId - if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeTrain), resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeTrain), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsTrainJobNew, &form) @@ -1851,7 +1851,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference ckptUrl := form.TrainUrl + form.CkptName - if !reward.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeInference), resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeInference), resourceSpecId) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsInferenceJobNew, &form) diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index edf41cd72..fa5e31afa 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -13,7 +13,6 @@ import ( const tplPoint base.TplName = "/reward/point" type AccountResponse struct { - AccountCode string Balance int64 TotalEarned int64 TotalConsumed int64 diff --git a/services/reward/cloubrain_deduct.go b/services/reward/cloubrain_deduct.go index ce23e2dc7..1e547a8a1 100644 --- a/services/reward/cloubrain_deduct.go +++ b/services/reward/cloubrain_deduct.go @@ -4,8 +4,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/services/reward/point/account" - "encoding/json" "fmt" "time" ) @@ -17,34 +15,17 @@ var ( const RUN_CLOUDBRAIN_TASK_TITTLE = "运行云脑任务" -//IsPointBalanceEnough check whether the user's point balance is bigger than task unit price -func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int) bool { +func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.RewardPeriodicTask, error) { if !setting.CloudBrainPaySwitch { - return true - } - spec := getResourceSpec(jobType, resourceSpecId) - if spec == nil { - return true - } - a, error := account.GetAccount(targetUserId) - if error != nil { - return false - } - return a.Balance >= spec.UnitPrice - -} - -func StartCloudBrainPointDeductTask(task models.Cloudbrain) { - if !setting.CloudBrainPaySwitch { - return + return nil, nil } - spec := getResourceSpec(task.JobType, task.ResourceSpecId) + spec := models.GetResourceSpec(task.JobType, task.ResourceSpecId) if spec == nil || spec.UnitPrice == 0 { - return + return nil, nil } - StartPeriodicTask(&models.StartPeriodicTaskOpts{ + return StartAndGetPeriodicTask(&models.StartPeriodicTaskOpts{ SourceType: models.SourceTypeRunCloudbrainTask, SourceId: getCloudBrainPointTaskSourceId(task), TargetUserId: task.UserID, @@ -67,31 +48,6 @@ func getCloudBrainPointTaskSourceId(task models.Cloudbrain) string { return fmt.Sprint(task.ID) } -func getResourceSpec(jobType string, resourceSpecId int) *models.ResourceSpec { - if jobType == string(models.JobTypeTrain) { - if TrainResourceSpecs == nil { - json.Unmarshal([]byte(setting.TrainResourceSpecs), &TrainResourceSpecs) - } - for _, spec := range TrainResourceSpecs.ResourceSpec { - if resourceSpecId == spec.Id { - return spec - } - } - } else { - if ResourceSpecs == nil { - json.Unmarshal([]byte(setting.ResourceSpecs), &ResourceSpecs) - } - for _, spec := range ResourceSpecs.ResourceSpec { - if resourceSpecId == spec.Id { - return spec - } - } - - } - return nil - -} - var firstTimeFlag = true func StartCloudbrainPointDeductTask() { @@ -107,10 +63,9 @@ func StartCloudbrainPointDeductTask() { if firstTimeFlag { //When it is executed for the first time, it needs to process the tasks of the last 1 hours. //This is done to prevent the application from hanging for a long time - start = end.Add(-1 * time.Hour) + start = end.Add(-3 * time.Hour) firstTimeFlag = false } - taskList, err := models.GetStartedCloudbrainTaskByUpdatedUnix(start, end) if err != nil { log.Error("GetStartedCloudbrainTaskByUpdatedUnix error. %v", err) @@ -121,11 +76,30 @@ func StartCloudbrainPointDeductTask() { return } for _, t := range taskList { - if int64(t.StartTime) <= end.Unix() && int64(t.StartTime) >= start.Unix() { - StartCloudBrainPointDeductTask(t) + //初始化 period_task 和 operate_record + if int64(t.StartTime) > end.Unix() || int64(t.StartTime) < start.Unix() { + continue + } + + task, err := StartAndGetCloudBrainPointDeductTask(t) + if err != nil { + log.Error("run cloubrain point deduct task error,err=%v", err) + continue } + if task == nil { + continue + } + if task.Status == models.PeriodicTaskStatusFinished { + log.Info("Periodic task is finished") + continue + } + if int64(t.EndTime) <= end.Unix() && int64(t.EndTime) >= start.Unix() { - StopCloudBrainPointDeductTask(t) + endTime := time.Unix(int64(t.EndTime), 0) + RunRewardTask(*task, endTime) + models.StopPeriodicTask(task.ID, task.OperateSerialNo, endTime) + } else { + RunRewardTask(*task, end) } } } diff --git a/services/reward/operator.go b/services/reward/operator.go index 79f82acbd..fc51aa1c5 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -78,11 +78,11 @@ func Operate(ctx *models.RewardOperateContext) error { //operate if err := operator.Operate(ctx); err != nil { - updateAwardOperateRecordStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) + UpdateRewardRecordToFinalStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusFailed) return err } - updateAwardOperateRecordStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusOperating, models.OperateStatusSucceeded) + UpdateRewardRecordToFinalStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusSucceeded) NotifyRewardOperation(ctx.TargetUserId, ctx.Reward.Amount, ctx.Reward.Type, ctx.OperateType) return nil } @@ -160,8 +160,8 @@ func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (strin return record.SerialNo, nil } -func updateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus string) error { - _, err := models.UpdateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus) +func UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus string) error { + _, err := models.UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus) if err != nil { return err } @@ -169,10 +169,10 @@ func updateAwardOperateRecordStatus(sourceType, requestId, oldStatus, newStatus } func StartPeriodicTaskAsyn(opts *models.StartPeriodicTaskOpts) { - go StartPeriodicTask(opts) + go StartAndGetPeriodicTask(opts) } -func StartPeriodicTask(opts *models.StartPeriodicTaskOpts) error { +func StartAndGetPeriodicTask(opts *models.StartPeriodicTaskOpts) (*models.RewardPeriodicTask, error) { defer func() { if err := recover(); err != nil { combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) @@ -183,35 +183,46 @@ func StartPeriodicTask(opts *models.StartPeriodicTaskOpts) error { var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardOperateLock(opts.RequestId, opts.SourceType.Name(), opts.OperateType.Name())) isOk, err := rewardLock.Lock(3 * time.Second) if err != nil { - return err + return nil, err } if !isOk { log.Info("duplicated operate request,targetUserId=%d requestId=%s", opts.TargetUserId, opts.RequestId) - return nil + return nil, nil } defer rewardLock.UnLock() - //is handled before? - isHandled, err := isHandled(opts.SourceType.Name(), opts.RequestId, opts.OperateType.Name()) - if err != nil { - log.Error("operate is handled error,%v", err) - return err + _, err = models.GetPointOperateRecordBySourceTypeAndRequestId(opts.SourceType.Name(), opts.RequestId, opts.OperateType.Name()) + if err == nil { + task, err := models.GetPeriodicTaskBySourceIdAndType(opts.SourceType, opts.SourceId, opts.OperateType) + if err != nil { + log.Error("GetPeriodicTaskBySourceIdAndType error,%v", err) + return nil, err + } + return task, nil } - if isHandled { - log.Info("operate has been handled,opts=%+v", opts) - return nil + + if err != nil && !models.IsErrRecordNotExist(err) { + log.Error("operate is handled error,%v", err) + return nil, err } + //new reward operate record recordId, err := createPeriodicRewardOperateRecord(opts) if err != nil { - return err + return nil, err } if err = NewRewardPeriodicTask(recordId, opts); err != nil { - updateAwardOperateRecordStatus(opts.SourceType.Name(), opts.RequestId, models.OperateStatusOperating, models.OperateStatusFailed) - return err + UpdateRewardRecordToFinalStatus(opts.SourceType.Name(), opts.RequestId, models.OperateStatusFailed) + return nil, err } - return nil + + task, err := models.GetPeriodicTaskBySourceIdAndType(opts.SourceType, opts.SourceId, opts.OperateType) + if err != nil { + log.Error("GetPeriodicTaskBySourceIdAndType error,%v", err) + return nil, err + } + return task, nil } func StopPeriodicTaskAsyn(sourceType models.SourceType, sourceId string, operateType models.RewardOperateType) { diff --git a/services/reward/period_task.go b/services/reward/period_task.go index 3fa416dab..c2808c4c0 100644 --- a/services/reward/period_task.go +++ b/services/reward/period_task.go @@ -6,6 +6,8 @@ import ( "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/routers/repo" + "errors" "fmt" "time" ) @@ -46,33 +48,33 @@ func StartRewardTask() { } } -func RunRewardTask(t models.RewardPeriodicTask, now time.Time) { +func RunRewardTask(t models.RewardPeriodicTask, now time.Time) error { lock := redis_lock.NewDistributeLock(redis_key.RewardTaskRunningLock(t.ID)) isOk, _ := lock.LockWithWait(3*time.Second, 3*time.Second) if !isOk { log.Error("get RewardTaskRunningLock failed,t=%+v", t) - return + return errors.New("get RewardTaskRunningLock failed") } defer lock.UnLock() record, err := models.GetPointOperateRecordBySerialNo(t.OperateSerialNo) if err != nil { log.Error("RunRewardTask. GetPointOperateRecordBySerialNo error. %v", err) - return + return errors.New("GetPointOperateRecordBySerialNo error") } if record.Status != models.OperateStatusOperating { log.Info("RunRewardTask. operate record is finished,record=%+v", record) - return + return nil } n, _ := countExecuteTimes(t, now) if n == 0 { - return + return nil } //get operator operator := GetOperator(models.GetRewardTypeInstance(record.RewardType)) if operator == nil { log.Error("RunRewardTask. operator of reward type is not exist") - return + return errors.New("operator of reward type is not exist") } nextTime := t.NextExecuteTime for i := 0; int64(i) <= n; i++ { @@ -89,14 +91,20 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) { if err != nil { log.Error("RunRewardTask.operator operate error.%v", err) if models.IsErrInsufficientPointsBalance(err) { - StopCloudbrainTask(record) - return + task, err := models.GetCloudbrainByID(record.SourceId) + if err != nil { + log.Error("RunRewardTask GetCloudbrainByID error. %v", err) + return err + } + repo.StopJobs([]*models.Cloudbrain{task}) + return nil } - return + return nil } - models.IncrRewardTaskSuccessCount(t, n, nextTime) + models.IncrRewardTaskSuccessCount(t, 1, nextTime) nextTime = timeutil.TimeStamp(int64(nextTime) + t.IntervalSeconds) } + return nil } @@ -111,8 +119,3 @@ func countExecuteTimes(t models.RewardPeriodicTask, now time.Time) (int64, timeu newNextTime := timeutil.TimeStamp(nextTime + n*interval) return n, newNextTime } - -func StopCloudbrainTask(r *models.RewardOperateRecord) { - //todo - -} diff --git a/services/reward/point/account/point_account.go b/services/reward/point/account/point_account.go index ea127e162..693694c76 100644 --- a/services/reward/point/account/point_account.go +++ b/services/reward/point/account/point_account.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "encoding/json" "time" @@ -60,3 +61,20 @@ func InitAccount(userId int64) (*models.PointAccount, error) { return nil, nil } + +//IsPointBalanceEnough check whether the user's point balance is bigger than task unit price +func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int) bool { + if !setting.CloudBrainPaySwitch { + return true + } + spec := models.GetResourceSpec(jobType, resourceSpecId) + if spec == nil { + return true + } + a, error := GetAccount(targetUserId) + if error != nil { + return false + } + return a.Balance >= spec.UnitPrice + +} diff --git a/services/reward/record.go b/services/reward/record.go index 157e53b53..b1ac86876 100644 --- a/services/reward/record.go +++ b/services/reward/record.go @@ -1,9 +1,11 @@ package reward -import "code.gitea.io/gitea/models" +import ( + "code.gitea.io/gitea/models" +) type RecordResponse struct { - Records []models.RewardOperateRecordShow + Records []*models.RewardOperateRecordShow Total int64 PageSize int Page int @@ -14,9 +16,13 @@ func GetRewardRecordList(opts models.RewardRecordListOpts) (*RecordResponse, err if err != nil { return nil, err } - r := make([]models.RewardOperateRecordShow, 0) - for _, v := range l { - r = append(r, v.ToShow()) + if len(l) == 0 { + return &RecordResponse{Records: make([]*models.RewardOperateRecordShow, 0), Total: n, Page: opts.Page, PageSize: opts.PageSize}, nil } - return &RecordResponse{Records: r, Total: n, Page: opts.Page, PageSize: opts.PageSize}, nil + result, err := l.ToShow() + if err != nil { + return nil, err + } + + return &RecordResponse{Records: result, Total: n, Page: opts.Page, PageSize: opts.PageSize}, nil } diff --git a/services/task/task.go b/services/task/task.go index 0dfc38b6c..e5b57ac3d 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -3,7 +3,6 @@ package task import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/reward" "code.gitea.io/gitea/services/reward/limiter" "fmt" @@ -55,9 +54,7 @@ func accomplish(action models.Action) error { } //add log - logId := util.UUID() _, err = models.InsertTaskAccomplishLog(&models.TaskAccomplishLog{ - LogId: logId, ConfigId: config.ID, TaskCode: config.TaskCode, UserId: userId, @@ -70,14 +67,14 @@ func accomplish(action models.Action) error { //reward reward.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeAccomplishTask, - SourceId: logId, + SourceId: fmt.Sprint(action.ID), Tittle: config.Tittle, Reward: models.Reward{ Amount: config.AwardAmount, Type: models.GetRewardTypeInstance(config.AwardType), }, TargetUserId: userId, - RequestId: logId, + RequestId: fmt.Sprint(action.ID), OperateType: models.OperateTypeIncrease, RejectPolicy: models.FillUp, }) From ec517ba3110078718fe8ca2d002e3c3ecb8f5675 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 28 Jun 2022 18:46:08 +0800 Subject: [PATCH 20/90] #2225 add admin log in reward record show --- models/reward_admin_log.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/models/reward_admin_log.go b/models/reward_admin_log.go index b1a55af13..24e3b8c47 100644 --- a/models/reward_admin_log.go +++ b/models/reward_admin_log.go @@ -2,7 +2,6 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" - "strings" ) const ( @@ -24,6 +23,11 @@ type RewardAdminLog struct { CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } +type AdminLogAndUser struct { + AdminRewardAdminLog RewardAdminLog `xorm:"extends"` + User User `xorm:"extends"` +} + func getRewardAdminLog(ra *RewardAdminLog) (*RewardAdminLog, error) { has, err := x.Get(ra) if err != nil { @@ -50,12 +54,16 @@ func GetRewardAdminLogByLogIds(logIds []string) ([]RewardAdminLog, error) { if len(logIds) == 0 { return nil, nil } - adminLogs := make([]RewardAdminLog, 0) - err := x.SQL("select rdl.id,rdl.log_id,rdl.amount,rdl.reward_type,rdl.remark,rdl.status,rdl.target_user_id,rdl.creator_id,u.name as creator_name "+ - "from reward_admin_log rdl left join public.user u on rdl.creator_id = u.id "+ - "where rdl.log_id in (?)", strings.Join(logIds, ",")).Find(&adminLogs) + adminLogs := make([]AdminLogAndUser, 0) + err := x.Table("reward_admin_log").Join("LEFT", "user", "reward_admin_log.creator_id = public.user.id").In("reward_admin_log.log_id", logIds).Find(&adminLogs) if err != nil { return nil, err } - return adminLogs, nil + r := make([]RewardAdminLog, len(adminLogs)) + for i, v := range adminLogs { + temp := v.AdminRewardAdminLog + temp.CreatorName = v.User.Name + r[i] = temp + } + return r, nil } From f7430122a57a6853d34b558394f3117bba707ed2 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 29 Jun 2022 12:00:16 +0800 Subject: [PATCH 21/90] #2225 update --- models/action.go | 17 ++++++ models/repo.go | 6 ++ models/reward_operate_record.go | 76 ++++++++++++++------------ models/reward_periodic_task.go | 2 +- models/user.go | 4 ++ services/reward/cloubrain_deduct.go | 53 ++++++++++-------- services/reward/notify.go | 2 +- services/reward/operator.go | 1 + services/reward/period_task.go | 3 +- services/reward/point/point_operate.go | 7 +-- 10 files changed, 104 insertions(+), 67 deletions(-) diff --git a/models/action.go b/models/action.go index ff16dcd3f..69ad797d6 100755 --- a/models/action.go +++ b/models/action.go @@ -91,6 +91,23 @@ type Action struct { CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } +type ActionShow struct { + UserID int64 + OpType ActionType + ActUserID int64 + ActUser *UserShow + RepoID int64 + Repo *RepositoryShow + CommentID int64 + Comment *Comment `xorm:"-"` + IsDeleted bool `xorm:"INDEX NOT NULL DEFAULT false"` + RefName string + IsPrivate bool `xorm:"INDEX NOT NULL DEFAULT false"` + IsTransformed bool `xorm:"INDEX NOT NULL DEFAULT false"` + Content string `xorm:"TEXT"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + // GetOpType gets the ActionType of this action. func (a *Action) GetOpType() ActionType { return a.OpType diff --git a/models/repo.go b/models/repo.go index db2694617..9845ffa82 100755 --- a/models/repo.go +++ b/models/repo.go @@ -237,6 +237,12 @@ type Repository struct { LowerAlias string `xorm:"INDEX"` } +type RepositoryShow struct { + Name string + RepoType RepoType + Alias string +} + // SanitizedOriginalURL returns a sanitized OriginalURL func (repo *Repository) SanitizedOriginalURL() string { if repo.OriginalURL == "" { diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 04f43a8bd..9c6b347a6 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -213,21 +213,22 @@ func (l *RewardRecordList) GetRewardRecordCloudbrainTask() (map[string]Cloudbrai } type RewardOperateRecord struct { - ID int64 `xorm:"pk autoincr"` - SerialNo string `xorm:"INDEX NOT NULL"` - UserId int64 `xorm:"INDEX NOT NULL"` - Amount int64 `xorm:"NOT NULL"` - Tittle string - RewardType string `xorm:"NOT NULL"` - SourceType string `xorm:"NOT NULL"` - SourceId string `xorm:"INDEX NOT NULL"` - RequestId string `xorm:"INDEX NOT NULL"` - OperateType string `xorm:"NOT NULL"` - Status string `xorm:"NOT NULL"` - Remark string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - FinishedUnix timeutil.TimeStamp `xorm:"INDEX"` + ID int64 `xorm:"pk autoincr"` + SerialNo string `xorm:"INDEX NOT NULL"` + UserId int64 `xorm:"INDEX NOT NULL"` + Amount int64 `xorm:"NOT NULL"` + LossAmount int64 + Tittle string + RewardType string `xorm:"NOT NULL"` + SourceType string `xorm:"NOT NULL"` + SourceId string `xorm:"INDEX NOT NULL"` + RequestId string `xorm:"INDEX NOT NULL"` + OperateType string `xorm:"NOT NULL"` + Status string `xorm:"NOT NULL"` + Remark string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + LastOperateUnix timeutil.TimeStamp `xorm:"INDEX"` } type AdminRewardOperateReq struct { @@ -240,27 +241,31 @@ type AdminRewardOperateReq struct { func (r RewardOperateRecord) ToShow() RewardOperateRecordShow { return RewardOperateRecordShow{ - SerialNo: r.SerialNo, - Date: r.CreatedUnix, - OperateType: r.OperateType, - Amount: r.Amount, - Remark: r.Remark, - Status: r.Status, - SourceType: r.SourceType, + SerialNo: r.SerialNo, + Date: r.CreatedUnix, + OperateType: r.OperateType, + Amount: r.Amount, + Remark: r.Remark, + Status: r.Status, + SourceType: r.SourceType, + LastOperateTask: r.LastOperateUnix, + LossAmount: r.LossAmount, } } type RewardOperateRecordShow struct { - SerialNo string - Date timeutil.TimeStamp - Status string - OperateType string - Amount int64 - Remark string - SourceType string - Action Action - Cloudbrain Cloudbrain - AdminLog RewardAdminLog + SerialNo string + Date timeutil.TimeStamp + Status string + OperateType string + Amount int64 + LossAmount int64 + Remark string + SourceType string + LastOperateTask timeutil.TimeStamp + Action Action + Cloudbrain Cloudbrain + AdminLog RewardAdminLog } func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) { @@ -295,10 +300,10 @@ func InsertRewardOperateRecord(tl *RewardOperateRecord) (int64, error) { func UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus string) (int64, error) { r := &RewardOperateRecord{ - Status: newStatus, - FinishedUnix: timeutil.TimeStampNow(), + Status: newStatus, + LastOperateUnix: timeutil.TimeStampNow(), } - return x.Cols("status", "finished_unix").Where("source_type=? and request_id=? and status=?", sourceType, requestId, OperateStatusOperating).Update(r) + return x.Cols("status", "last_operate_unix").Where("source_type=? and request_id=? and status=?", sourceType, requestId, OperateStatusOperating).Update(r) } func SumRewardAmountInTaskPeriod(rewardType string, sourceType string, userId int64, period *PeriodResult) (int64, error) { @@ -326,6 +331,7 @@ type RewardOperateContext struct { OperateType RewardOperateType RejectPolicy LimiterRejectPolicy PermittedNegative bool + LossAmount int64 } type Reward struct { diff --git a/models/reward_periodic_task.go b/models/reward_periodic_task.go index a859676d6..910f4fe8e 100644 --- a/models/reward_periodic_task.go +++ b/models/reward_periodic_task.go @@ -77,7 +77,7 @@ func IncrRewardTaskSuccessCount(t RewardPeriodicTask, count int64, nextTime time sess.Rollback() return err } - _, err = sess.Exec("update reward_operate_record set amount = amount + ? ,updated_unix = ? where serial_no = ?", t.Amount, timeutil.TimeStampNow(), t.OperateSerialNo) + _, err = sess.Exec("update reward_operate_record set amount = amount + ? ,updated_unix = ? ,last_operate_unix = ? where serial_no = ?", t.Amount, timeutil.TimeStampNow(), timeutil.TimeStampNow(), t.OperateSerialNo) if err != nil { sess.Rollback() return err diff --git a/models/user.go b/models/user.go index dd5a6f1d2..73537556a 100755 --- a/models/user.go +++ b/models/user.go @@ -186,6 +186,10 @@ type User struct { WechatBindUnix timeutil.TimeStamp } +type UserShow struct { + Name string +} + // SearchOrganizationsOptions options to filter organizations type SearchOrganizationsOptions struct { ListOptions diff --git a/services/reward/cloubrain_deduct.go b/services/reward/cloubrain_deduct.go index 1e547a8a1..39354b98c 100644 --- a/services/reward/cloubrain_deduct.go +++ b/services/reward/cloubrain_deduct.go @@ -59,7 +59,7 @@ func StartCloudbrainPointDeductTask() { }() log.Debug("try to run CloudbrainPointDeductTask") end := time.Now() - start := end.Add(-5 * time.Minute) + start := end.Add(-30 * time.Minute) if firstTimeFlag { //When it is executed for the first time, it needs to process the tasks of the last 1 hours. //This is done to prevent the application from hanging for a long time @@ -76,30 +76,35 @@ func StartCloudbrainPointDeductTask() { return } for _, t := range taskList { - //初始化 period_task 和 operate_record - if int64(t.StartTime) > end.Unix() || int64(t.StartTime) < start.Unix() { - continue - } + DeductPoint4Cloudbrain(t, end) + } +} - task, err := StartAndGetCloudBrainPointDeductTask(t) - if err != nil { - log.Error("run cloubrain point deduct task error,err=%v", err) - continue - } - if task == nil { - continue - } - if task.Status == models.PeriodicTaskStatusFinished { - log.Info("Periodic task is finished") - continue - } +func DeductPoint4Cloudbrain(t models.Cloudbrain, now time.Time) error { - if int64(t.EndTime) <= end.Unix() && int64(t.EndTime) >= start.Unix() { - endTime := time.Unix(int64(t.EndTime), 0) - RunRewardTask(*task, endTime) - models.StopPeriodicTask(task.ID, task.OperateSerialNo, endTime) - } else { - RunRewardTask(*task, end) - } + if t.StartTime == 0 { + return nil + } + + task, err := StartAndGetCloudBrainPointDeductTask(t) + if err != nil { + log.Error("run cloudbrain point deduct task error,err=%v", err) + return err + } + if task == nil { + return nil + } + if task.Status == models.PeriodicTaskStatusFinished { + log.Info("Periodic task is finished") + return nil + } + + if t.EndTime > 0 { + endTime := time.Unix(int64(t.EndTime), 0) + RunRewardTask(*task, endTime) + models.StopPeriodicTask(task.ID, task.OperateSerialNo, endTime) + } else { + RunRewardTask(*task, now) } + return nil } diff --git a/services/reward/notify.go b/services/reward/notify.go index f5b270d94..5cfe6ee77 100644 --- a/services/reward/notify.go +++ b/services/reward/notify.go @@ -35,7 +35,7 @@ func GetRewardOperation(since, until timeutil.TimeStamp) []models.UserRewardOper json.Unmarshal([]byte(v), &t) r = append(r, models.UserRewardOperation{ UserId: t.UserId, - Msg: GetRewardOperateMsg(t), + Msg: v, }) } redis_client.ZRemRangeByScore(redis_key.RewardOperateNotification(), float64(since), float64(until)) diff --git a/services/reward/operator.go b/services/reward/operator.go index fc51aa1c5..8a3ea12f2 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -118,6 +118,7 @@ func initRewardOperateRecord(ctx *models.RewardOperateContext) (string, error) { record := &models.RewardOperateRecord{ UserId: ctx.TargetUserId, Amount: ctx.Reward.Amount, + LossAmount: ctx.LossAmount, RewardType: ctx.Reward.Type.Name(), SourceType: ctx.SourceType.Name(), SourceId: ctx.SourceId, diff --git a/services/reward/period_task.go b/services/reward/period_task.go index c2808c4c0..f6bd45fc9 100644 --- a/services/reward/period_task.go +++ b/services/reward/period_task.go @@ -77,7 +77,7 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) error { return errors.New("operator of reward type is not exist") } nextTime := t.NextExecuteTime - for i := 0; int64(i) <= n; i++ { + for i := 1; int64(i) <= n; i++ { err = operator.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeRunCloudbrainTask, SourceId: t.OperateSerialNo, @@ -97,6 +97,7 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) error { return err } repo.StopJobs([]*models.Cloudbrain{task}) + models.StopPeriodicTask(task.ID, t.OperateSerialNo, time.Now()) return nil } return nil diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 0115c288a..101758db8 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -9,12 +9,9 @@ import ( "code.gitea.io/gitea/services/reward/limiter" "code.gitea.io/gitea/services/reward/point/account" "errors" - "fmt" "time" ) -const LossMsg = "达到奖励上限，应得%d积分，实得%d积分" - type PointOperator struct { } @@ -24,7 +21,7 @@ func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) error return err } if realAmount < ctx.Reward.Amount { - ctx.Remark = models.AppendRemark(ctx.Remark, fmt.Sprintf(LossMsg, ctx.Reward.Amount, realAmount)) + ctx.LossAmount = ctx.Reward.Amount - realAmount ctx.Reward.Amount = realAmount } return nil @@ -49,7 +46,7 @@ func (operator *PointOperator) Operate(ctx *models.RewardOperateContext) error { } else if ctx.OperateType == models.OperateTypeDecrease { if !ctx.PermittedNegative && na.Balance < ctx.Reward.Amount { log.Info("account balance is not enough,ctx=%v", ctx) - return &models.ErrInsufficientPointsBalance{} + return models.ErrInsufficientPointsBalance{} } err = na.Decrease(ctx.Reward.Amount, ctx.SourceId) } From 77e9692c2e13781810b2f54b8ded58f678d3bb30 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 29 Jun 2022 15:07:22 +0800 Subject: [PATCH 22/90] #2225 update --- routers/routes/routes.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 47ee3c50a..6cf87b527 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -594,6 +594,16 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/empty", admin.EmptyNotices) }) + }, adminReq) + // ***** END: Admin ***** + + operationReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, OperationRequired: true}) + + // ***** START: Operation ***** + m.Group("/operation", func() { + m.Get("/config/recommend_org", operation.Organizations) + m.Post("/config/recommend_org", bindIgnErr(operation.OrgInfos{}), operation.UpdateRecommendOrganizations) + m.Group("/reward/point", func() { m.Get("/limiter/list", point.GetPointLimitConfigList) m.Post("/limiter/add", bindIgnErr(models.LimitConfigVO{}), point.AddPointLimitConfig) @@ -606,16 +616,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/add", bindIgnErr(models.TaskConfigWithLimit{}), task.AddTaskConfig) m.Post("/add/batch", bindIgnErr(models.BatchLimitConfigVO{}), task.BatchAddTaskConfig) }) - - }, adminReq) - // ***** END: Admin ***** - - operationReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, OperationRequired: true}) - - // ***** START: Operation ***** - m.Group("/operation", func() { - m.Get("/config/recommend_org", operation.Organizations) - m.Post("/config/recommend_org", bindIgnErr(operation.OrgInfos{}), operation.UpdateRecommendOrganizations) }, operationReq) // ***** END: Operation ***** From 0efaa9b16c131b13a6b1d7346207d3ebc6345155 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 29 Jun 2022 15:40:49 +0800 Subject: [PATCH 23/90] #2225 add log --- services/reward/cloubrain_deduct.go | 1 + services/reward/notify.go | 2 ++ services/reward/operator.go | 15 +++++++++++++++ services/reward/period_task.go | 1 + services/reward/point/account/point_account.go | 9 +++++++-- services/reward/point/point_operate.go | 5 +++++ services/reward/record.go | 4 ++++ services/reward/serial.go | 2 ++ services/task/task.go | 4 +++- services/task/task_config.go | 4 ++++ 10 files changed, 44 insertions(+), 3 deletions(-) diff --git a/services/reward/cloubrain_deduct.go b/services/reward/cloubrain_deduct.go index 39354b98c..7d0c39028 100644 --- a/services/reward/cloubrain_deduct.go +++ b/services/reward/cloubrain_deduct.go @@ -22,6 +22,7 @@ func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.Rewar spec := models.GetResourceSpec(task.JobType, task.ResourceSpecId) if spec == nil || spec.UnitPrice == 0 { + log.Debug("GetResourceSpec failed,spec is nil or UnitPrice = 0") return nil, nil } diff --git a/services/reward/notify.go b/services/reward/notify.go index 5cfe6ee77..2cd27f007 100644 --- a/services/reward/notify.go +++ b/services/reward/notify.go @@ -2,6 +2,7 @@ package reward import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/timeutil" @@ -24,6 +25,7 @@ func NotifyRewardOperation(userId, amount int64, rewardType models.RewardType, o func GetRewardOperation(since, until timeutil.TimeStamp) []models.UserRewardOperation { list, err := redis_client.ZRangeByScore(redis_key.RewardOperateNotification(), float64(since), float64(until)) if err != nil { + log.Error("GetRewardOperation ZRangeByScore error. %v", err) return nil } if len(list) == 0 { diff --git a/services/reward/operator.go b/services/reward/operator.go index 8a3ea12f2..3a869d772 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -57,6 +57,7 @@ func Operate(ctx *models.RewardOperateContext) error { //get operator operator := GetOperator(ctx.Reward.Type) if operator == nil { + log.Error("operator of reward type is not exist,ctx=%v", ctx) return errors.New("operator of reward type is not exist") } @@ -71,6 +72,7 @@ func Operate(ctx *models.RewardOperateContext) error { //new reward operate record recordId, err := initRewardOperateRecord(ctx) if err != nil { + log.Error("initRewardOperateRecord error,err=%v", err) return err } @@ -78,6 +80,7 @@ func Operate(ctx *models.RewardOperateContext) error { //operate if err := operator.Operate(ctx); err != nil { + log.Error("operator Operate error,err=%v", err) UpdateRewardRecordToFinalStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusFailed) return err } @@ -101,9 +104,11 @@ func GetOperator(rewardType models.RewardType) RewardOperator { func isHandled(sourceType string, requestId string, operateType string) (bool, error) { _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType, requestId, operateType) if err != nil { + log.Error("operator isHandled error. %v", err) if models.IsErrRecordNotExist(err) { return false, nil } + log.Error("GetPointOperateRecordBySourceTypeAndRequestId ZRangeByScore error. %v", err) return false, err } return true, nil @@ -113,6 +118,7 @@ func isHandled(sourceType string, requestId string, operateType string) (bool, e func initRewardOperateRecord(ctx *models.RewardOperateContext) (string, error) { sn, err := generateOperateSerialNo(ctx.OperateType, ctx.Reward.Type) if err != nil { + log.Error("generateOperateSerialNo error. %v", err) return "", err } record := &models.RewardOperateRecord{ @@ -131,6 +137,7 @@ func initRewardOperateRecord(ctx *models.RewardOperateContext) (string, error) { } _, err = models.InsertRewardOperateRecord(record) if err != nil { + log.Error("InsertRewardOperateRecord error. %v", err) return "", err } return record.SerialNo, nil @@ -139,6 +146,7 @@ func initRewardOperateRecord(ctx *models.RewardOperateContext) (string, error) { func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (string, error) { sn, err := generateOperateSerialNo(ctx.OperateType, ctx.RewardType) if err != nil { + log.Error("createPeriodic generateOperateSerialNo error. %v", err) return "", err } record := &models.RewardOperateRecord{ @@ -156,6 +164,7 @@ func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (strin } _, err = models.InsertRewardOperateRecord(record) if err != nil { + log.Error("createPeriodic InsertRewardOperateRecord error. %v", err) return "", err } return record.SerialNo, nil @@ -164,6 +173,7 @@ func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (strin func UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus string) error { _, err := models.UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus) if err != nil { + log.Error("UpdateRewardRecord UpdateRewardRecordToFinalStatus error. %v", err) return err } return nil @@ -184,6 +194,7 @@ func StartAndGetPeriodicTask(opts *models.StartPeriodicTaskOpts) (*models.Reward var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardOperateLock(opts.RequestId, opts.SourceType.Name(), opts.OperateType.Name())) isOk, err := rewardLock.Lock(3 * time.Second) if err != nil { + log.Error("StartAndGetPeriodicTask RewardOperateLock error. %v", err) return nil, err } if !isOk { @@ -210,10 +221,12 @@ func StartAndGetPeriodicTask(opts *models.StartPeriodicTaskOpts) (*models.Reward //new reward operate record recordId, err := createPeriodicRewardOperateRecord(opts) if err != nil { + log.Error("StartAndGetPeriodicTask createPeriodicRewardOperateRecord error. %v", err) return nil, err } if err = NewRewardPeriodicTask(recordId, opts); err != nil { + log.Error("StartAndGetPeriodicTask NewRewardPeriodicTask error. %v", err) UpdateRewardRecordToFinalStatus(opts.SourceType.Name(), opts.RequestId, models.OperateStatusFailed) return nil, err } @@ -258,6 +271,8 @@ func StopPeriodicTask(sourceType models.SourceType, sourceId string, operateType func generateOperateSerialNo(operateType models.RewardOperateType, rewardType models.RewardType) (string, error) { s, err := GetSerialNoByRedis() if err != nil { + log.Error("generateOperateSerialNo error. %v", err) + return "", err } diff --git a/services/reward/period_task.go b/services/reward/period_task.go index f6bd45fc9..b6315b19f 100644 --- a/services/reward/period_task.go +++ b/services/reward/period_task.go @@ -67,6 +67,7 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) error { } n, _ := countExecuteTimes(t, now) if n == 0 { + log.Info("countExecuteTimes result is 0") return nil } diff --git a/services/reward/point/account/point_account.go b/services/reward/point/account/point_account.go index 693694c76..c1d5722c4 100644 --- a/services/reward/point/account/point_account.go +++ b/services/reward/point/account/point_account.go @@ -2,6 +2,7 @@ package account import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" @@ -24,10 +25,12 @@ func GetAccount(userId int64) (*models.PointAccount, error) { if models.IsErrRecordNotExist(err) { a, err := InitAccount(userId) if err != nil { + log.Error("InitAccount error,err=%v", err) return nil, err } return a, nil } + log.Error("GetAccountByUserId error,err=%v", err) return nil, err } jsonStr, _ := json.Marshal(account) @@ -39,6 +42,7 @@ func InitAccount(userId int64) (*models.PointAccount, error) { lock := redis_lock.NewDistributeLock(redis_key.PointAccountInitLock(userId)) isOk, err := lock.LockWithWait(3*time.Second, 3*time.Second) if err != nil { + log.Error("PointAccountInitLock error,err=%v", err) return nil, err } if isOk { @@ -71,8 +75,9 @@ func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int if spec == nil { return true } - a, error := GetAccount(targetUserId) - if error != nil { + a, err := GetAccount(targetUserId) + if err != nil { + log.Error("IsPointBalanceEnough GetAccount error,err=%v", err) return false } return a.Balance >= spec.UnitPrice diff --git a/services/reward/point/point_operate.go b/services/reward/point/point_operate.go index 101758db8..ccdf1f423 100644 --- a/services/reward/point/point_operate.go +++ b/services/reward/point/point_operate.go @@ -18,6 +18,7 @@ type PointOperator struct { func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) error { realAmount, err := limiter.CheckLimit(ctx.SourceType.Name(), models.LimitTypeRewardPoint, ctx.TargetUserId, ctx.Reward.Amount, ctx.RejectPolicy) if err != nil { + log.Error("PointOperator IsLimited error,err=%v", err) return err } if realAmount < ctx.Reward.Amount { @@ -30,12 +31,14 @@ func (operator *PointOperator) IsLimited(ctx *models.RewardOperateContext) error func (operator *PointOperator) Operate(ctx *models.RewardOperateContext) error { a, err := account.GetAccount(ctx.TargetUserId) if err != nil || a == nil { + log.Error("operator get account error error,err=%v", err) return errors.New("get account error") } lock := redis_lock.NewDistributeLock(redis_key.PointAccountOperateLock(a.AccountCode)) isOk, err := lock.LockWithWait(3*time.Second, 3*time.Second) if err != nil { + log.Error("Get PointAccountOperateLock error,err=%v", err) return err } if isOk { @@ -51,11 +54,13 @@ func (operator *PointOperator) Operate(ctx *models.RewardOperateContext) error { err = na.Decrease(ctx.Reward.Amount, ctx.SourceId) } if err != nil { + log.Error("operate account balance error,err=%v", err) return err } redis_client.Del(redis_key.PointAccountInfo(ctx.TargetUserId)) } else { + log.Error("Get account operate lock failed,ctx=%v", ctx) return errors.New("Get account operate lock failed") } return nil diff --git a/services/reward/record.go b/services/reward/record.go index b1ac86876..460a6fc81 100644 --- a/services/reward/record.go +++ b/services/reward/record.go @@ -2,6 +2,7 @@ package reward import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" ) type RecordResponse struct { @@ -14,6 +15,8 @@ type RecordResponse struct { func GetRewardRecordList(opts models.RewardRecordListOpts) (*RecordResponse, error) { l, n, err := models.GetRewardRecordList(opts) if err != nil { + log.Error("GetRewardRecordList error. %v", err) + return nil, err } if len(l) == 0 { @@ -21,6 +24,7 @@ func GetRewardRecordList(opts models.RewardRecordListOpts) (*RecordResponse, err } result, err := l.ToShow() if err != nil { + log.Error("GetRewardRecordList ToShow error. %v", err) return nil, err } diff --git a/services/reward/serial.go b/services/reward/serial.go index e9509c403..b6a47bbc3 100644 --- a/services/reward/serial.go +++ b/services/reward/serial.go @@ -1,6 +1,7 @@ package reward import ( + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "fmt" @@ -12,6 +13,7 @@ func GetSerialNoByRedis() (string, error) { now := time.Now() n, err := redis_client.IncrBy(redis_key.RewardSerialCounter(now), 1) if err != nil { + log.Error("GetSerialNoByRedis RewardSerialCounter error. %v", err) return "", err } if n == 1 { diff --git a/services/task/task.go b/services/task/task.go index e5b57ac3d..dcf7007c6 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -61,6 +61,7 @@ func accomplish(action models.Action) error { ActionId: action.ID, }) if err != nil { + log.Error("InsertTaskAccomplishLog error,%v", err) return err } @@ -78,12 +79,13 @@ func accomplish(action models.Action) error { OperateType: models.OperateTypeIncrease, RejectPolicy: models.FillUp, }) - + log.Debug("accomplish success,action=%v", action) return nil } func isLimited(userId int64, config *models.TaskConfig, rejectPolicy models.LimiterRejectPolicy) bool { if _, err := limiter.CheckLimit(config.TaskCode, models.LimitTypeTask, userId, 1, rejectPolicy); err != nil { + log.Error(" isLimited CheckLimit error. %v", err) return true } return false diff --git a/services/task/task_config.go b/services/task/task_config.go index 0184ca15b..0404a6f06 100644 --- a/services/task/task_config.go +++ b/services/task/task_config.go @@ -16,6 +16,7 @@ import ( func GetTaskConfig(taskType string) (*models.TaskConfig, error) { list, err := GetTaskConfigList() if err != nil { + log.Error(" GetTaskConfigList error. %v", err) return nil, err } for _, v := range list { @@ -39,6 +40,7 @@ func GetTaskConfigList() ([]*models.TaskConfig, error) { } config, err := models.GetTaskConfigList() if err != nil { + log.Error(" GetTaskConfigList from model error. %v", err) if models.IsErrRecordNotExist(err) { redis_client.Setex(redisKey, redis_key.EMPTY_REDIS_VAL, 5*time.Second) return nil, nil @@ -61,6 +63,7 @@ func GetTaskConfigWithLimitList() ([]*models.TaskConfigWithLimit, error) { r := make([]*models.TaskConfigWithLimit, 0) l, err := limiter.GetLimitersByLimitType(models.LimitTypeTask) if err != nil { + log.Error(" GetLimitersByLimitType from redis error. %v", err) return nil, err } for i := 0; i < len(list); i++ { @@ -88,6 +91,7 @@ func GetTaskConfigWithLimitList() ([]*models.TaskConfigWithLimit, error) { func AddTaskConfig(config models.TaskConfigWithLimit, doer *models.User) error { if config.TaskCode == "" || config.AwardType == "" { + log.Error(" AddTaskConfig param error") return errors.New("param error") } err := models.AddTaskConfig(config, doer) From 34bdcf070b0fa373aefce51a4b5d0abac0f5b3e1 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 29 Jun 2022 15:53:02 +0800 Subject: [PATCH 24/90] #2225 add log --- modules/eventsource/manager_run.go | 2 ++ services/reward/notify.go | 1 + 2 files changed, 3 insertions(+) diff --git a/modules/eventsource/manager_run.go b/modules/eventsource/manager_run.go index 857eaee22..252a0ec88 100644 --- a/modules/eventsource/manager_run.go +++ b/modules/eventsource/manager_run.go @@ -31,9 +31,11 @@ loop: for { select { case <-rewardTimer.C: + log.Debug("rewardTimer run") now := timeutil.TimeStampNow().Add(-2) list := reward.GetRewardOperation(rewardThen, now) if list != nil { + log.Debug("GetRewardOperation list=%v", list) for _, l := range list { m.SendMessage(l.UserId, &Event{ Name: "reward-operation", diff --git a/services/reward/notify.go b/services/reward/notify.go index 2cd27f007..875dde199 100644 --- a/services/reward/notify.go +++ b/services/reward/notify.go @@ -29,6 +29,7 @@ func GetRewardOperation(since, until timeutil.TimeStamp) []models.UserRewardOper return nil } if len(list) == 0 { + log.Debug("GetRewardOperation list length = 0") return nil } r := make([]models.UserRewardOperation, len(list)) From 6410dba541e43a72b0af8d7f3497de67a8c9aa88 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 29 Jun 2022 17:01:46 +0800 Subject: [PATCH 25/90] #2225 update --- modules/redis/redis_key/reward_redis_key.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/redis/redis_key/reward_redis_key.go b/modules/redis/redis_key/reward_redis_key.go index 05c10ce4f..fb294668a 100644 --- a/modules/redis/redis_key/reward_redis_key.go +++ b/modules/redis/redis_key/reward_redis_key.go @@ -1,7 +1,9 @@ package redis_key import ( + "code.gitea.io/gitea/modules/setting" "fmt" + "strings" ) const REWARD_REDIS_PREFIX = "reward" @@ -11,7 +13,7 @@ func RewardOperateLock(requestId string, sourceType string, operateType string) } func RewardOperateNotification() string { - return KeyJoin(REWARD_REDIS_PREFIX, "operate", "notification") + return KeyJoin(REWARD_REDIS_PREFIX, "operate", strings.ReplaceAll(setting.AppURL, "/", ""), "notification") } func RewardTaskRunningLock(taskId int64) string { From 2fabac1b536f838139e2a5b9c08dbb05009d5bab Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 29 Jun 2022 17:26:41 +0800 Subject: [PATCH 26/90] #2225 update --- models/reward_operate_record.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 9c6b347a6..444e477b6 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -242,27 +242,27 @@ type AdminRewardOperateReq struct { func (r RewardOperateRecord) ToShow() RewardOperateRecordShow { return RewardOperateRecordShow{ SerialNo: r.SerialNo, - Date: r.CreatedUnix, + CreateDate: r.CreatedUnix, OperateType: r.OperateType, Amount: r.Amount, Remark: r.Remark, Status: r.Status, SourceType: r.SourceType, - LastOperateTask: r.LastOperateUnix, + LastOperateDate: r.LastOperateUnix, LossAmount: r.LossAmount, } } type RewardOperateRecordShow struct { SerialNo string - Date timeutil.TimeStamp + CreateDate timeutil.TimeStamp Status string OperateType string Amount int64 LossAmount int64 Remark string SourceType string - LastOperateTask timeutil.TimeStamp + LastOperateDate timeutil.TimeStamp Action Action Cloudbrain Cloudbrain AdminLog RewardAdminLog From 186e6d1bd38d11e29ecee840a9f127300ccf1189 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 30 Jun 2022 12:00:59 +0800 Subject: [PATCH 27/90] #2225 update --- routers/reward/point/point.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index fa5e31afa..d912c1539 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -10,7 +10,7 @@ import ( "net/http" ) -const tplPoint base.TplName = "/reward/point" +const tplPoint base.TplName = "reward/point" type AccountResponse struct { Balance int64 From 39339a29e6a7791062d208c70a43d6eece269ffc Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 30 Jun 2022 16:08:59 +0800 Subject: [PATCH 28/90] #2225 update --- models/action.go | 61 +++-- models/action_list.go | 65 ++++++ models/cloudbrain.go | 211 +++++++++++------- models/helper.go | 8 + models/reward_admin_log.go | 8 +- models/reward_operate_record.go | 29 +-- modules/auth/modelarts.go | 3 - modules/modelarts/modelarts.go | 5 +- routers/repo/cloudbrain.go | 8 +- routers/repo/modelarts.go | 19 +- services/reward/cloubrain_deduct.go | 8 +- .../reward/point/account/point_account.go | 14 +- services/task/task.go | 8 +- 13 files changed, 295 insertions(+), 152 deletions(-) diff --git a/models/action.go b/models/action.go index 69ad797d6..33322a921 100755 --- a/models/action.go +++ b/models/action.go @@ -92,20 +92,14 @@ type Action struct { } type ActionShow struct { - UserID int64 - OpType ActionType - ActUserID int64 - ActUser *UserShow - RepoID int64 - Repo *RepositoryShow - CommentID int64 - Comment *Comment `xorm:"-"` - IsDeleted bool `xorm:"INDEX NOT NULL DEFAULT false"` - RefName string - IsPrivate bool `xorm:"INDEX NOT NULL DEFAULT false"` - IsTransformed bool `xorm:"INDEX NOT NULL DEFAULT false"` - Content string `xorm:"TEXT"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + OpType ActionType + RepoLink string + ShortRepoFullDisplayName string + Content string + RefName string + IssueInfos []string + CommentLink string + IssueTitle string } // GetOpType gets the ActionType of this action. @@ -243,6 +237,43 @@ func (a *Action) GetRepoLink() string { return "/" + a.GetRepoPath() } +func (a *Action) ToShow() *ActionShow { + actionShow := &ActionShow{} + actionShow.OpType = GetTaskOptType(*a) + actionShow.Content = a.Content + actionShow.RefName = a.RefName + + if strings.Contains(a.Content, "|") { + actionShow.IssueInfos = a.GetIssueInfos() + actionShow.IssueTitle = a.GetIssueTitle() + } + + if a.Repo != nil { + actionShow.RepoLink = a.GetRepoLink() + actionShow.ShortRepoFullDisplayName = a.ShortRepoFullDisplayName() + } + if a.Comment != nil { + actionShow.CommentLink = a.GetCommentLink() + } + + return actionShow +} + +func GetTaskOptType(action Action) ActionType { + switch action.OpType { + case ActionCreateDebugGPUTask, + ActionCreateDebugNPUTask, + ActionCreateTrainTask, + ActionCreateInferenceTask, + ActionCreateBenchMarkTask, + ActionCreateGPUTrainTask: + return ActionCreateCloudbrainTask + default: + return action.OpType + } + +} + // GetRepositoryFromMatch returns a *Repository from a username and repo strings func GetRepositoryFromMatch(ownerName string, repoName string) (*Repository, error) { var err error @@ -439,7 +470,7 @@ func GetActionByIds(ids []int64) ([]*Action, error) { if err != nil { return nil, err } - if err := ActionList(actions).LoadAttributes(); err != nil { + if err := ActionList(actions).LoadAllAttributes(); err != nil { return nil, fmt.Errorf("ActionList loadAttributes: %v", err) } return actions, nil diff --git a/models/action_list.go b/models/action_list.go index 6f726f4b3..a0987c20d 100644 --- a/models/action_list.go +++ b/models/action_list.go @@ -79,6 +79,48 @@ func (actions ActionList) LoadRepositories() ([]*Repository, error) { return actions.loadRepositories(x) } +func (actions ActionList) getCommentIDs() []int64 { + commentIDs := make(map[int64]struct{}, len(actions)) + for _, action := range actions { + if action.CommentID == 0 { + continue + } + if _, ok := commentIDs[action.CommentID]; !ok { + commentIDs[action.CommentID] = struct{}{} + } + } + return keysInt64(commentIDs) +} + +func (actions ActionList) loadComments(e Engine) ([]*Comment, error) { + if len(actions) == 0 { + return nil, nil + } + + commentIDs := actions.getCommentIDs() + + commentMaps := make(map[int64]*Comment, len(commentIDs)) + if len(commentIDs) == 0 { + return make([]*Comment, 0), nil + } + err := e. + In("id", commentIDs). + Find(&commentMaps) + if err != nil { + return nil, fmt.Errorf("find comment: %v", err) + } + + for _, action := range actions { + action.Comment = commentMaps[action.CommentID] + } + return valuesComment(commentMaps), nil +} + +// LoadComments loads actions' all comments +func (actions ActionList) LoadComments() ([]*Comment, error) { + return actions.loadComments(x) +} + // loadAttributes loads all attributes func (actions ActionList) loadAttributes(e Engine) (err error) { if _, err = actions.loadUsers(e); err != nil { @@ -96,3 +138,26 @@ func (actions ActionList) loadAttributes(e Engine) (err error) { func (actions ActionList) LoadAttributes() error { return actions.loadAttributes(x) } + +// LoadAllAttributes loads all attributes of the actions +// compare with LoadAttributes() ,LoadAllAttributes() loads Comment attribute +func (actions ActionList) LoadAllAttributes() error { + return actions.loadAllAttributes(x) +} + +// loadAllAttributes +func (actions ActionList) loadAllAttributes(e Engine) (err error) { + if _, err = actions.loadUsers(e); err != nil { + return + } + + if _, err = actions.loadRepositories(e); err != nil { + return + } + + if _, err = actions.loadComments(e); err != nil { + return + } + + return nil +} diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 06cd42258..2f82640b7 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -169,69 +169,23 @@ type Cloudbrain struct { } type CloudbrainShow struct { - JobID string `xorm:"INDEX NOT NULL"` - JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` - JobName string - DisplayJobName string - Status string - UserID int64 `xorm:"INDEX NOT NULL"` - RepoID int64 `xorm:"INDEX NOT NULL"` - SubTaskName string - ContainerID string - ContainerIp string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Duration int64 `xorm:"DEFAULT 0"` //运行时长 单位秒 - TrainJobDuration string `xorm:"DEFAULT '00:00:00'"` - Image string //镜像名称 - GpuQueue string //GPU类型即GPU队列 - ResourceSpecId int //GPU规格id - DeletedAt time.Time `xorm:"deleted"` - CanDebug bool `xorm:"-"` - CanDel bool `xorm:"-"` - CanModify bool `xorm:"-"` - Type int - BenchmarkTypeID int - BenchmarkChildTypeID int - - VersionID int64 //版本id - VersionName string `xorm:"INDEX"` //当前版本 - Uuid string //数据集id - DatasetName string - VersionCount int //任务的当前版本数量，不包括删除的 - IsLatestVersion string //是否是最新版本，1是，0否 - CommitID string //提交的仓库代码id - PreVersionName string //父版本名称 - ComputeResource string //计算资源，例如npu - EngineID int64 //引擎id - - TrainUrl string //输出模型的obs路径 - BranchName string //分支名称 - Parameters string //传给modelarts的param参数 - BootFile string //启动文件 - DataUrl string //数据集的obs路径 - LogUrl string //日志输出的obs路径 - PreVersionId int64 //父版本的版本id - FlavorCode string //modelarts上的规格id - Description string `xorm:"varchar(256)"` //描述 - WorkServerNumber int //节点数 - FlavorName string //规格名称 - EngineName string //引擎名称 - TotalVersionCount int //任务的所有版本数量，包括删除的 - - LabelName string //标签名称 - ModelName string //模型名称 - ModelVersion string //模型版本 - CkptName string //权重文件名称 - ResultUrl string //推理结果的obs路径 + ID int64 + JobType string + DisplayJobName string + Duration string + ResourceSpec *ResourceAndFlavor + ComputeResource string +} - User *User `xorm:"-"` - Repo *Repository `xorm:"-"` - BenchmarkType string `xorm:"-"` //算法评测，模型评测 - BenchmarkTypeName string `xorm:"-"` - BenchmarkTypeRankLink string `xorm:"-"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp +func (task *Cloudbrain) ToShow() *CloudbrainShow { + return &CloudbrainShow{ + ID: task.ID, + JobType: task.JobType, + DisplayJobName: task.DisplayJobName, + Duration: task.TrainJobDuration, + ResourceSpec: GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode), + ComputeResource: task.ComputeResource, + } } func (task *Cloudbrain) ComputeAndSetDuration() { @@ -1917,11 +1871,11 @@ func GetStartedCloudbrainTaskByUpdatedUnix(startTime, endTime time.Time) ([]Clou return r, nil } -func GetCloudbrainByIds(ids []int64) ([]Cloudbrain, error) { +func GetCloudbrainByIds(ids []int64) ([]*Cloudbrain, error) { if len(ids) == 0 { return nil, nil } - cloudbrains := make([]Cloudbrain, 0) + cloudbrains := make([]*Cloudbrain, 0) err := x.In("id", ids).Unscoped().Find(&cloudbrains) if err != nil { return nil, err @@ -1930,31 +1884,128 @@ func GetCloudbrainByIds(ids []int64) ([]Cloudbrain, error) { } var ( - DebugResourceSpecs *ResourceSpecs - TrainResourceSpecs *ResourceSpecs + SpecsMapInitFlag = false + CloudbrainDebugResourceSpecsMap map[int]*ResourceSpec + CloudbrainTrainResourceSpecsMap map[int]*ResourceSpec + CloudbrainBenchmarkResourceSpecsMap map[int]*ResourceSpec + ModelArtsDebugResourceSpecsMap map[string]*FlavorInfo + ModelArtsTrainResourceSpecsMap map[string]*FlavorInfo ) -func GetResourceSpec(jobType string, resourceSpecId int) *ResourceSpec { - if jobType == string(JobTypeTrain) { - if TrainResourceSpecs == nil { - json.Unmarshal([]byte(setting.TrainResourceSpecs), &TrainResourceSpecs) +type ModelArtsFlavor struct { + Info []struct { + Code string `json:"code"` + Value string `json:"value"` + UnitPrice int64 `json:"unitPrice"` + } `json:"flavor"` +} + +func InitResourceSpecMap() { + if CloudbrainDebugResourceSpecsMap == nil || len(CloudbrainDebugResourceSpecsMap) == 0 { + t := ResourceSpecs{} + json.Unmarshal([]byte(setting.ResourceSpecs), &t) + CloudbrainDebugResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) + for _, spec := range t.ResourceSpec { + CloudbrainDebugResourceSpecsMap[spec.Id] = spec } - for _, spec := range TrainResourceSpecs.ResourceSpec { - if resourceSpecId == spec.Id { - return spec - } + } + if CloudbrainTrainResourceSpecsMap == nil || len(CloudbrainTrainResourceSpecsMap) == 0 { + t := ResourceSpecs{} + json.Unmarshal([]byte(setting.TrainResourceSpecs), &t) + CloudbrainTrainResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) + for _, spec := range t.ResourceSpec { + CloudbrainTrainResourceSpecsMap[spec.Id] = spec } - } else { - if DebugResourceSpecs == nil { - json.Unmarshal([]byte(setting.ResourceSpecs), &DebugResourceSpecs) + } + if CloudbrainBenchmarkResourceSpecsMap == nil || len(CloudbrainBenchmarkResourceSpecsMap) == 0 { + t := ResourceSpecs{} + json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &t) + CloudbrainBenchmarkResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) + for _, spec := range t.ResourceSpec { + CloudbrainBenchmarkResourceSpecsMap[spec.Id] = spec + } + } + if ModelArtsDebugResourceSpecsMap == nil || len(ModelArtsDebugResourceSpecsMap) == 0 { + t := FlavorInfos{} + json.Unmarshal([]byte(setting.FlavorInfos), &t) + ModelArtsDebugResourceSpecsMap = make(map[string]*FlavorInfo, len(t.FlavorInfo)) + for _, spec := range t.FlavorInfo { + ModelArtsDebugResourceSpecsMap[spec.Value] = spec } - for _, spec := range DebugResourceSpecs.ResourceSpec { - if resourceSpecId == spec.Id { - return spec + } + if ModelArtsTrainResourceSpecsMap == nil || len(ModelArtsTrainResourceSpecsMap) == 0 { + t := ModelArtsFlavor{} + json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &t) + ModelArtsTrainResourceSpecsMap = make(map[string]*FlavorInfo, len(t.Info)) + for _, spec := range t.Info { + f := &FlavorInfo{ + Value: spec.Code, + Desc: spec.Value, + UnitPrice: spec.UnitPrice, } + ModelArtsTrainResourceSpecsMap[spec.Value] = f } + } + SpecsMapInitFlag = true +} +type ResourceAndFlavor struct { + ResourceSpec *ResourceSpec + FlavorInfo *FlavorInfo +} + +func NewResourceAndFlavor(resourceSpec *ResourceSpec, flavorInfo *FlavorInfo) *ResourceAndFlavor { + return &ResourceAndFlavor{ + ResourceSpec: resourceSpec, + FlavorInfo: flavorInfo, + } +} + +func GetCloudbrainResourceSpec(jobType string, clusterType int, resourceSpecId int, flavorCode string) *ResourceAndFlavor { + if !SpecsMapInitFlag { + InitResourceSpecMap() } + if clusterType == TypeCloudBrainOne { + switch jobType { + case string(JobTypeDebug): + return NewResourceAndFlavor(CloudbrainDebugResourceSpecsMap[resourceSpecId], nil) + case string(JobTypeTrain): + return NewResourceAndFlavor(CloudbrainTrainResourceSpecsMap[resourceSpecId], nil) + case string(JobTypeBenchmark): + return NewResourceAndFlavor(CloudbrainBenchmarkResourceSpecsMap[resourceSpecId], nil) + + } + } else if clusterType == TypeCloudBrainTwo { + switch jobType { + case string(JobTypeDebug): + return NewResourceAndFlavor(nil, ModelArtsDebugResourceSpecsMap[flavorCode]) + case string(JobTypeTrain): + return NewResourceAndFlavor(nil, ModelArtsTrainResourceSpecsMap[flavorCode]) + case string(JobTypeInference): + return NewResourceAndFlavor(nil, ModelArtsTrainResourceSpecsMap[flavorCode]) + + } + } + return nil } + +func GetCloudbrainTaskUnitPrice(task Cloudbrain) int64 { + spec := GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode) + if spec == nil { + return 0 + } + if task.Type == TypeCloudBrainOne { + if spec.ResourceSpec == nil { + return 0 + } + return spec.ResourceSpec.UnitPrice + } else if task.Type == TypeCloudBrainTwo { + if spec.FlavorInfo == nil { + return 0 + } + return spec.FlavorInfo.UnitPrice + } + return 0 +} diff --git a/models/helper.go b/models/helper.go index a284424bb..55d4cac31 100644 --- a/models/helper.go +++ b/models/helper.go @@ -27,3 +27,11 @@ func valuesUser(m map[int64]*User) []*User { } return values } + +func valuesComment(m map[int64]*Comment) []*Comment { + var values = make([]*Comment, 0, len(m)) + for _, v := range m { + values = append(values, v) + } + return values +} diff --git a/models/reward_admin_log.go b/models/reward_admin_log.go index 24e3b8c47..fd79c3ed9 100644 --- a/models/reward_admin_log.go +++ b/models/reward_admin_log.go @@ -50,18 +50,18 @@ func UpdateRewardAdminLogStatus(logId string, oldStatus, newStatus int) error { return nil } -func GetRewardAdminLogByLogIds(logIds []string) ([]RewardAdminLog, error) { +func GetRewardAdminLogByLogIds(logIds []string) ([]*RewardAdminLog, error) { if len(logIds) == 0 { return nil, nil } - adminLogs := make([]AdminLogAndUser, 0) + adminLogs := make([]*AdminLogAndUser, 0) err := x.Table("reward_admin_log").Join("LEFT", "user", "reward_admin_log.creator_id = public.user.id").In("reward_admin_log.log_id", logIds).Find(&adminLogs) if err != nil { return nil, err } - r := make([]RewardAdminLog, len(adminLogs)) + r := make([]*RewardAdminLog, len(adminLogs)) for i, v := range adminLogs { - temp := v.AdminRewardAdminLog + temp := &v.AdminRewardAdminLog temp.CreatorName = v.User.Name r[i] = temp } diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 444e477b6..6e4b15e9d 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -119,7 +119,6 @@ type RewardRecordShowList []*RewardOperateRecordShow func (l *RewardRecordList) ToShow() (RewardRecordShowList, error) { actionMap, err := l.GetRewardRecordAction() - adminLogMap, err := l.GetRewardRecordAdminLog() CloudbrainMap, err := l.GetRewardRecordCloudbrainTask() if err != nil { return nil, err @@ -129,11 +128,9 @@ func (l *RewardRecordList) ToShow() (RewardRecordShowList, error) { temp := v.ToShow() switch v.SourceType { case SourceTypeAccomplishTask.Name(): - temp.Action = actionMap[v.SourceId] - case SourceTypeAdminOperate.Name(): - temp.AdminLog = adminLogMap[v.SourceId] + temp.Action = actionMap[v.SourceId].ToShow() case SourceTypeRunCloudbrainTask.Name(): - temp.Cloudbrain = CloudbrainMap[v.SourceId] + temp.Cloudbrain = CloudbrainMap[v.SourceId].ToShow() } result = append(result, &temp) } @@ -141,7 +138,7 @@ func (l *RewardRecordList) ToShow() (RewardRecordShowList, error) { return result, nil } -func (l *RewardRecordList) GetRewardRecordAction() (map[string]Action, error) { +func (l *RewardRecordList) GetRewardRecordAction() (map[string]*Action, error) { if len(*l) == 0 { return nil, nil } @@ -157,15 +154,15 @@ func (l *RewardRecordList) GetRewardRecordAction() (map[string]Action, error) { if err != nil { return nil, err } - result := make(map[string]Action, 0) + result := make(map[string]*Action, 0) for _, v := range actions { - result[fmt.Sprint(v.ID)] = *v + result[fmt.Sprint(v.ID)] = v } return result, nil } -func (l *RewardRecordList) GetRewardRecordAdminLog() (map[string]RewardAdminLog, error) { +func (l *RewardRecordList) GetRewardRecordAdminLog() (map[string]*RewardAdminLog, error) { if len(*l) == 0 { return nil, nil } @@ -180,7 +177,7 @@ func (l *RewardRecordList) GetRewardRecordAdminLog() (map[string]RewardAdminLog, if err != nil { return nil, err } - result := make(map[string]RewardAdminLog, 0) + result := make(map[string]*RewardAdminLog, 0) for _, v := range logs { result[fmt.Sprint(v.LogId)] = v } @@ -188,7 +185,7 @@ func (l *RewardRecordList) GetRewardRecordAdminLog() (map[string]RewardAdminLog, } -func (l *RewardRecordList) GetRewardRecordCloudbrainTask() (map[string]Cloudbrain, error) { +func (l *RewardRecordList) GetRewardRecordCloudbrainTask() (map[string]*Cloudbrain, error) { if len(*l) == 0 { return nil, nil } @@ -204,7 +201,7 @@ func (l *RewardRecordList) GetRewardRecordCloudbrainTask() (map[string]Cloudbrai if err != nil { return nil, err } - result := make(map[string]Cloudbrain, 0) + result := make(map[string]*Cloudbrain, 0) for _, v := range cloudbrains { result[fmt.Sprint(v.ID)] = v } @@ -242,7 +239,6 @@ type AdminRewardOperateReq struct { func (r RewardOperateRecord) ToShow() RewardOperateRecordShow { return RewardOperateRecordShow{ SerialNo: r.SerialNo, - CreateDate: r.CreatedUnix, OperateType: r.OperateType, Amount: r.Amount, Remark: r.Remark, @@ -255,7 +251,6 @@ func (r RewardOperateRecord) ToShow() RewardOperateRecordShow { type RewardOperateRecordShow struct { SerialNo string - CreateDate timeutil.TimeStamp Status string OperateType string Amount int64 @@ -263,9 +258,9 @@ type RewardOperateRecordShow struct { Remark string SourceType string LastOperateDate timeutil.TimeStamp - Action Action - Cloudbrain Cloudbrain - AdminLog RewardAdminLog + Action *ActionShow + Cloudbrain *CloudbrainShow + AdminLog *RewardAdminLog } func getPointOperateRecord(tl *RewardOperateRecord) (*RewardOperateRecord, error) { diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index 0cbed45a6..ce41f5d1e 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -22,7 +22,6 @@ type CreateModelArtsNotebookForm struct { Description string `form:"description"` Flavor string `form:"flavor" binding:"Required"` ImageId string `form:"image_id" binding:"Required"` - ResourceSpecId int `form:"resource_spec_id"` } func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { @@ -47,7 +46,6 @@ type CreateModelArtsTrainJobForm struct { VersionName string `form:"version_name" binding:"Required"` FlavorName string `form:"flaver_names" binding:"Required"` EngineName string `form:"engine_names" binding:"Required"` - ResourceSpecId int `form:"resource_spec_id"` } type CreateModelArtsInferenceJobForm struct { @@ -73,7 +71,6 @@ type CreateModelArtsInferenceJobForm struct { ModelName string `form:"model_name" binding:"Required"` ModelVersion string `form:"model_version" binding:"Required"` CkptName string `form:"ckpt_name" binding:"Required"` - ResourceSpecId int `form:"resource_spec_id"` } func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index de5c392cd..9cc8c46c6 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -140,8 +140,9 @@ type VersionInfo struct { type Flavor struct { Info []struct { - Code string `json:"code"` - Value string `json:"value"` + Code string `json:"code"` + Value string `json:"value"` + UnitPrice int64 `json:"unitPrice"` } `json:"flavor"` } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 29c8b97bb..a075f3b70 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -230,7 +230,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { command = commandTrain } - if !account.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, jobType, models.TypeCloudBrainOne, resourceSpecId, "") { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tpl, &form) @@ -318,7 +318,7 @@ func CloudBrainRestart(ctx *context.Context) { var status = string(models.JobWaiting) task := ctx.Cloudbrain for { - if !account.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, task.JobType, models.TypeCloudBrainOne, task.ResourceSpecId, "") { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) resultCode = "-1" errorMsg = models.ErrInsufficientPointsBalance{}.Error() @@ -1870,7 +1870,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo repo := ctx.Repo.Repository - if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeBenchmark), models.TypeCloudBrainOne, cloudbrain.BenchMarkResourceID, "") { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplCloudBrainBenchmarkNew, &form) @@ -2032,7 +2032,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) tpl := tplCloudBrainBenchmarkNew command := cloudbrain.Command - if !account.IsPointBalanceEnough(ctx.User.ID, jobType, resourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, jobType, models.TypeCloudBrainOne, resourceSpecId, "") { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, jobType, resourceSpecId) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tpl, &form) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index bff9ec525..0b33a6dd7 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -205,10 +205,9 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm flavor := form.Flavor imageId := form.ImageId repo := ctx.Repo.Repository - resourceSpecId := form.ResourceSpecId - if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeDebug), resourceSpecId) { - log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeDebug), models.TypeCloudBrainTwo, 0, flavor) { + log.Error("point balance is not enough,userId=%d jobType=%s ", ctx.User.ID, string(models.JobTypeBenchmark)) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsNotebookNew, &form) return @@ -426,7 +425,7 @@ func NotebookManage(ctx *context.Context) { errorMsg = "you have no right to restart the job" break } - if !account.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.ResourceSpecId) { + if !account.IsPointBalanceEnough(ctx.User.ID, task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode) { log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, task.JobType, task.ResourceSpecId) resultCode = "-1" errorMsg = models.ErrInsufficientPointsBalance{}.Error() @@ -1000,10 +999,9 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) FlavorName := form.FlavorName VersionCount := modelarts.VersionCount EngineName := form.EngineName - resourceSpecId := form.ResourceSpecId - if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeTrain), resourceSpecId) { - log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeTrain), models.TypeCloudBrainTwo, 0, flavorCode) { + log.Error("point balance is not enough,userId=%d jobType=%s", ctx.User.ID, string(models.JobTypeBenchmark)) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsTrainJobNew, &form) return @@ -1183,7 +1181,6 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) EngineName: EngineName, VersionCount: VersionCount, TotalVersionCount: modelarts.TotalVersionCount, - ResourceSpecId: resourceSpecId, } //将params转换Parameters.Parameter，出错时返回给前端 @@ -1847,12 +1844,11 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference modelName := form.ModelName modelVersion := form.ModelVersion ckptName := form.CkptName - resourceSpecId := form.ResourceSpecId ckptUrl := form.TrainUrl + form.CkptName - if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeInference), resourceSpecId) { - log.Error("point balance is not enough,userId=%d jobType=%s resourceSpecId=%d", ctx.User.ID, string(models.JobTypeBenchmark), resourceSpecId) + if !account.IsPointBalanceEnough(ctx.User.ID, string(models.JobTypeInference), models.TypeCloudBrainTwo, 0, flavorCode) { + log.Error("point balance is not enough,userId=%d jobType=%s ", ctx.User.ID, string(models.JobTypeBenchmark)) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(models.ErrInsufficientPointsBalance{}.Error(), tplModelArtsInferenceJobNew, &form) return @@ -2002,7 +1998,6 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference ModelVersion: modelVersion, CkptName: ckptName, ResultUrl: resultObsPath, - ResourceSpecId: resourceSpecId, } err = modelarts.GenerateInferenceJob(ctx, req) diff --git a/services/reward/cloubrain_deduct.go b/services/reward/cloubrain_deduct.go index 7d0c39028..fdec1c0c1 100644 --- a/services/reward/cloubrain_deduct.go +++ b/services/reward/cloubrain_deduct.go @@ -20,9 +20,9 @@ func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.Rewar return nil, nil } - spec := models.GetResourceSpec(task.JobType, task.ResourceSpecId) - if spec == nil || spec.UnitPrice == 0 { - log.Debug("GetResourceSpec failed,spec is nil or UnitPrice = 0") + unitPrice := models.GetCloudbrainTaskUnitPrice(task) + if unitPrice == 0 { + log.Debug("finish StartAndGetCloudBrainPointDeductTask, UnitPrice = 0 task.ID=%d", task.ID) return nil, nil } @@ -34,7 +34,7 @@ func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.Rewar OperateType: models.OperateTypeDecrease, Delay: setting.CloudBrainPayDelay, Interval: setting.CloudBrainPayInterval, - UnitAmount: spec.UnitPrice, + UnitAmount: unitPrice, RewardType: models.RewardTypePoint, StartTime: time.Unix(int64(task.StartTime), 0), Tittle: RUN_CLOUDBRAIN_TASK_TITTLE, diff --git a/services/reward/point/account/point_account.go b/services/reward/point/account/point_account.go index c1d5722c4..79e98f2b2 100644 --- a/services/reward/point/account/point_account.go +++ b/services/reward/point/account/point_account.go @@ -67,12 +67,18 @@ func InitAccount(userId int64) (*models.PointAccount, error) { } //IsPointBalanceEnough check whether the user's point balance is bigger than task unit price -func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int) bool { +func IsPointBalanceEnough(targetUserId int64, jobType string, clusterType int, resourceSpecId int, flavorCode string) bool { if !setting.CloudBrainPaySwitch { return true } - spec := models.GetResourceSpec(jobType, resourceSpecId) - if spec == nil { + t := models.Cloudbrain{ + Type: clusterType, + JobType: jobType, + ResourceSpecId: resourceSpecId, + FlavorCode: flavorCode, + } + uniPrice := models.GetCloudbrainTaskUnitPrice(t) + if uniPrice == 0 { return true } a, err := GetAccount(targetUserId) @@ -80,6 +86,6 @@ func IsPointBalanceEnough(targetUserId int64, jobType string, resourceSpecId int log.Error("IsPointBalanceEnough GetAccount error,err=%v", err) return false } - return a.Balance >= spec.UnitPrice + return a.Balance >= uniPrice } diff --git a/services/task/task.go b/services/task/task.go index dcf7007c6..b53adb1f9 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -9,19 +9,13 @@ import ( ) func Accomplish(action models.Action) { + action.OpType = models.GetTaskOptType(action) switch action.OpType { case models.ActionCreateRepo, models.ActionCreateImage: if action.Repo.IsPrivate { return } - case models.ActionCreateDebugGPUTask, - models.ActionCreateDebugNPUTask, - models.ActionCreateTrainTask, - models.ActionCreateInferenceTask, - models.ActionCreateBenchMarkTask, - models.ActionCreateGPUTrainTask: - action.OpType = models.ActionCreateCloudbrainTask } go accomplish(action) } From 4332f2014ba7c62285a933c67e869fcbac7fd61c Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 30 Jun 2022 16:20:44 +0800 Subject: [PATCH 29/90] #2225 update --- models/cloudbrain.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 2f82640b7..ed52ad262 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -170,6 +170,7 @@ type Cloudbrain struct { type CloudbrainShow struct { ID int64 + Type int JobType string DisplayJobName string Duration string @@ -181,6 +182,7 @@ func (task *Cloudbrain) ToShow() *CloudbrainShow { return &CloudbrainShow{ ID: task.ID, JobType: task.JobType, + Type: task.Type, DisplayJobName: task.DisplayJobName, Duration: task.TrainJobDuration, ResourceSpec: GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode), From 0dbd239acce457cdea3c5f1e879a0aa738e94f14 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 30 Jun 2022 17:18:06 +0800 Subject: [PATCH 30/90] #2225 update --- models/cloudbrain.go | 2 ++ models/reward_operate_record.go | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index ed52ad262..75ab1dfd6 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -170,6 +170,7 @@ type Cloudbrain struct { type CloudbrainShow struct { ID int64 + RepoFullName string Type int JobType string DisplayJobName string @@ -181,6 +182,7 @@ type CloudbrainShow struct { func (task *Cloudbrain) ToShow() *CloudbrainShow { return &CloudbrainShow{ ID: task.ID, + RepoFullName: task.Repo.FullName(), JobType: task.JobType, Type: task.Type, DisplayJobName: task.DisplayJobName, diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index 6e4b15e9d..aea39a875 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -201,9 +201,17 @@ func (l *RewardRecordList) GetRewardRecordCloudbrainTask() (map[string]*Cloudbra if err != nil { return nil, err } + var ids []int64 + for _, task := range cloudbrains { + ids = append(ids, task.RepoID) + } + repositoryMap, err := GetRepositoriesMapByIDs(ids) result := make(map[string]*Cloudbrain, 0) - for _, v := range cloudbrains { - result[fmt.Sprint(v.ID)] = v + if err == nil { + for _, v := range cloudbrains { + v.Repo = repositoryMap[v.RepoID] + result[fmt.Sprint(v.ID)] = v + } } return result, nil From 260ab0963342de1a64ed66b5e727e4c4c8d5a103 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 30 Jun 2022 18:03:21 +0800 Subject: [PATCH 31/90] #2225 fix bug --- services/reward/{cloubrain_deduct.go => cloudbrain_deduct.go} | 0 services/reward/period_task.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename services/reward/{cloubrain_deduct.go => cloudbrain_deduct.go} (100%) diff --git a/services/reward/cloubrain_deduct.go b/services/reward/cloudbrain_deduct.go similarity index 100% rename from services/reward/cloubrain_deduct.go rename to services/reward/cloudbrain_deduct.go diff --git a/services/reward/period_task.go b/services/reward/period_task.go index b6315b19f..c4180db56 100644 --- a/services/reward/period_task.go +++ b/services/reward/period_task.go @@ -77,7 +77,7 @@ func RunRewardTask(t models.RewardPeriodicTask, now time.Time) error { log.Error("RunRewardTask. operator of reward type is not exist") return errors.New("operator of reward type is not exist") } - nextTime := t.NextExecuteTime + nextTime := timeutil.TimeStamp(int64(t.NextExecuteTime) + t.IntervalSeconds) for i := 1; int64(i) <= n; i++ { err = operator.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeRunCloudbrainTask, From fe4f394ab73fd365e77695731c8c62cccfc1a8cc Mon Sep 17 00:00:00 2001 From: chenshihai Date: Thu, 30 Jun 2022 19:51:20 +0800 Subject: [PATCH 32/90] =?UTF-8?q?=E7=AE=97=E5=8A=9B=E7=A7=AF=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/base/head_navbar.tmpl | 6 +- templates/base/head_navbar_fluid.tmpl | 2 + templates/base/head_navbar_home.tmpl | 6 +- templates/base/head_navbar_pro.tmpl | 6 +- templates/reward/point.tmpl | 6 + web_src/js/features/notification.js | 61 +++- web_src/vuepages/apis/modules/point.js | 41 +++ web_src/vuepages/apis/service.js | 26 ++ web_src/vuepages/pages/reward/point/const.js | 6 + .../vuepages/pages/reward/point/vp-point.js | 12 + .../vuepages/pages/reward/point/vp-point.vue | 293 ++++++++++++++++++ webpack.config.js | 6 + 12 files changed, 464 insertions(+), 7 deletions(-) create mode 100644 templates/reward/point.tmpl create mode 100644 web_src/vuepages/apis/modules/point.js create mode 100644 web_src/vuepages/apis/service.js create mode 100644 web_src/vuepages/pages/reward/point/const.js create mode 100644 web_src/vuepages/pages/reward/point/vp-point.js create mode 100644 web_src/vuepages/pages/reward/point/vp-point.vue diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 51761a7e5..0cf2f7484 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -27,7 +27,8 @@ {{.i18n.Tr "issues"}} {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} - {{.i18n.Tr "repo.cloudbrain.task"}} + {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 @@ -58,7 +59,8 @@ {{.i18n.Tr "issues"}} {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} - {{.i18n.Tr "repo.cloudbrain.task"}} + {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 diff --git a/templates/base/head_navbar_fluid.tmpl b/templates/base/head_navbar_fluid.tmpl index 6baeced54..74827e12c 100644 --- a/templates/base/head_navbar_fluid.tmpl +++ b/templates/base/head_navbar_fluid.tmpl @@ -28,6 +28,7 @@ {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 @@ -58,6 +59,7 @@ {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 diff --git a/templates/base/head_navbar_home.tmpl b/templates/base/head_navbar_home.tmpl index c9ea13b8a..1864bee6e 100644 --- a/templates/base/head_navbar_home.tmpl +++ b/templates/base/head_navbar_home.tmpl @@ -19,7 +19,8 @@ {{.i18n.Tr "issues"}} {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} - {{.i18n.Tr "repo.cloudbrain.task"}} + {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 @@ -49,7 +50,8 @@ {{.i18n.Tr "issues"}} {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} - {{.i18n.Tr "repo.cloudbrain.task"}} + {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 diff --git a/templates/base/head_navbar_pro.tmpl b/templates/base/head_navbar_pro.tmpl index e744508f0..ba50c88fe 100644 --- a/templates/base/head_navbar_pro.tmpl +++ b/templates/base/head_navbar_pro.tmpl @@ -28,7 +28,8 @@ {{.i18n.Tr "issues"}} {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} - {{.i18n.Tr "repo.cloudbrain.task"}} + {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 @@ -59,7 +60,8 @@ {{.i18n.Tr "issues"}} {{.i18n.Tr "pull_requests"}} {{.i18n.Tr "milestones"}} - {{.i18n.Tr "repo.cloudbrain.task"}} + {{.i18n.Tr "repo.cloudbrain.task"}} + 算力积分 diff --git a/templates/reward/point.tmpl b/templates/reward/point.tmpl new file mode 100644 index 000000000..359564b35 --- /dev/null +++ b/templates/reward/point.tmpl @@ -0,0 +1,6 @@ +{{template "base/head" .}} + +
+
+ +{{template "base/footer" .}} diff --git a/web_src/js/features/notification.js b/web_src/js/features/notification.js index 6f362eee6..aa8ed844e 100644 --- a/web_src/js/features/notification.js +++ b/web_src/js/features/notification.js @@ -47,7 +47,66 @@ export function initNotificationCount() { }); source.addEventListener('reward-operation', async (e) => { try { - console.log(e.data); + const data = JSON.parse(e.data); + const notice = $(` + + `); + $('body').append(notice); + notice.fadeIn(); + setTimeout(() => { + notice.fadeOut(); + }, 3000); + setTimeout(() => { + notice.remove(); + }, 5000); } catch (error) { console.error(error); } diff --git a/web_src/vuepages/apis/modules/point.js b/web_src/vuepages/apis/modules/point.js new file mode 100644 index 000000000..60b1df601 --- /dev/null +++ b/web_src/vuepages/apis/modules/point.js @@ -0,0 +1,41 @@ +import service from '../service'; + +// 算力积分概要 +export const getPointAccount = () => { + return service({ + url: '/reward/point/account', + method: 'get', + params: {}, + }); +} + +// 算力积分获取、消耗明细 +// operate-INCREASE 表示获取明细 DECREASE表示消耗明细, page-当前页, pageSize-每页条数 +export const getPointList = (params) => { + return service({ + url: '/reward/point/record/list', + method: 'get', + params, + }); +} + +// 管理员充值、扣减用户积分 +// TargetUserId, OperateType-INCREASE,DECREASE, Amount, Remark, RewardType-POINT +export const setPointOperate = (data) => { + return service({ + url: '/operation/reward/point/operate', + method: 'post', + data, + params: {} + }); +} + +// 算力积分页面 +export const getPoint = () => { + return service({ + url: '/reward/point', + method: 'get', + params: {}, + data: {}, + }); +} diff --git a/web_src/vuepages/apis/service.js b/web_src/vuepages/apis/service.js new file mode 100644 index 000000000..292b9ef78 --- /dev/null +++ b/web_src/vuepages/apis/service.js @@ -0,0 +1,26 @@ +import axios from 'axios'; + +const service = axios.create({ + baseURL: '/', + timeout: 20000, +}); + +service.interceptors.request.use((config) => { + config.data && Object.assign(config.data, { + _csrf: window.config ? window.config.csrf : '', + }); + config.params && Object.assign(config.params, { + _csrf: window.config ? window.config.csrf : '', + }); + return config; +}, (error) => { + return Promise.reject(error); +}); + +service.interceptors.response.use((response) => { + return response; +}, (error) => { + return Promise.reject(error); +}); + +export default service; diff --git a/web_src/vuepages/pages/reward/point/const.js b/web_src/vuepages/pages/reward/point/const.js new file mode 100644 index 000000000..af0332cc7 --- /dev/null +++ b/web_src/vuepages/pages/reward/point/const.js @@ -0,0 +1,6 @@ +export const SOURCE_TYPE = [{ k: 'ACCOMPLISH_TASK', v: '积分任务' }, { k: 'ADMIN_OPERATE', v: '管理员操作' }, { k: 'RUN_CLOUDBRAIN_TASK', v: '运行云脑任务' }]; +export const CONSUME_STATUS = [{ k: 'OPERATING', v: '进行中' }, { k: 'SUCCEEDED', v: '已完成' }]; +export const POINT_ACTIONS = [ + { k: 1, v: '创建公开项目' }, { k: 6, v: '每日提出任务' }, { k: 7, v: '每日提出PR' }, { k: 10, v: '发表评论' }, { k: 24, v: '上传数据集文件' }, { k: 30, v: '导入新模型' }, { k: 32, v: '完成微信扫码验证' }, + { k: 33, v: '每日运行云脑任务' }, { k: 34, v: '数据集被平台推荐' }, { k: 35, v: '提交新公开镜像' }, { k: 36, v: '镜像被平台推荐' }, { k: 37, v: '首次更换头像' }, { k: 38, v: '每日commit' }, { k: 39, v: '每日首次Fork项目' }, +]; diff --git a/web_src/vuepages/pages/reward/point/vp-point.js b/web_src/vuepages/pages/reward/point/vp-point.js new file mode 100644 index 000000000..8039d3f2c --- /dev/null +++ b/web_src/vuepages/pages/reward/point/vp-point.js @@ -0,0 +1,12 @@ +import Vue from 'vue'; +import ElementUI from 'element-ui'; +import 'element-ui/lib/theme-chalk/index.css'; + +Vue.use(ElementUI); +import App from './vp-point.vue'; +// import App from '../manage/vp-point-manage.vue'; + +new Vue({ + el: '#__vue-root', + render: (h) => h(App), +}); diff --git a/web_src/vuepages/pages/reward/point/vp-point.vue b/web_src/vuepages/pages/reward/point/vp-point.vue new file mode 100644 index 000000000..22bb1f47e --- /dev/null +++ b/web_src/vuepages/pages/reward/point/vp-point.vue @@ -0,0 +1,293 @@ + + + + + diff --git a/webpack.config.js b/webpack.config.js index cd3635427..8b8800150 100755 --- a/webpack.config.js +++ b/webpack.config.js @@ -20,6 +20,11 @@ for (const path of glob('web_src/less/themes/*.less')) { themes[parse(path).name] = [path]; } +const vuePages = {}; +for (const path of glob('web_src/vuepages/**/vp-*.js')) { + vuePages[parse(path).name] = [path]; +} + const isProduction = process.env.NODE_ENV !== 'development'; module.exports = { @@ -37,6 +42,7 @@ module.exports = { ], icons: glob('node_modules/@primer/octicons/build/svg/**/*.svg'), ...themes, + ...vuePages, }, devtool: false, output: { From 42d13cef3cf0c792d51475d40e6632a6c0552549 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 1 Jul 2022 11:59:14 +0800 Subject: [PATCH 33/90] #2225 update wechat bind rule --- models/wechat_bind.go | 4 ++++ modules/auth/wechat/bind.go | 6 +++--- routers/reward/point/point.go | 2 +- services/task/task.go | 11 +++++++++++ 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/models/wechat_bind.go b/models/wechat_bind.go index b100221f2..ea005e0a6 100644 --- a/models/wechat_bind.go +++ b/models/wechat_bind.go @@ -96,3 +96,7 @@ func UnbindWechatOpenId(userId int64, oldWechatOpenID string) error { sess.Insert(logParam) return sess.Commit() } + +func CountWechatBindLog(wechatOpenId string, action WechatBindAction) (int64, error) { + return x.Where("wechat_open_id = ? and action = ?", action, wechatOpenId).Count(&WechatBindLog{}) +} diff --git a/modules/auth/wechat/bind.go b/modules/auth/wechat/bind.go index 7b4bffc02..e166aceb4 100644 --- a/modules/auth/wechat/bind.go +++ b/modules/auth/wechat/bind.go @@ -38,7 +38,7 @@ func (err WechatBindError) Error() string { } func BindWechat(userId int64, wechatOpenId string) error { - if !IsWechatAccountAvailable(userId, wechatOpenId) { + if !IsWechatAccountUsed(userId, wechatOpenId) { log.Error("bind wechat failed, because user use wrong wechat account to bind,userId=%d wechatOpenId=%s", userId, wechatOpenId) return NewWechatBindError(BIND_REPLY_WECHAT_ACCOUNT_USED) } @@ -60,9 +60,9 @@ func IsUserAvailableForWechatBind(userId int64, wechatOpenId string) bool { return currentOpenId == "" || currentOpenId == wechatOpenId } -//IsWechatAccountAvailable if wechat account used by another account,return false +//IsWechatAccountUsed if wechat account used by another account,return false //if wechat account not used or used by the given user,return true -func IsWechatAccountAvailable(userId int64, wechatOpenId string) bool { +func IsWechatAccountUsed(userId int64, wechatOpenId string) bool { user := models.GetUserByWechatOpenId(wechatOpenId) if user != nil && user.WechatOpenId != "" && user.ID != userId { return false diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index d912c1539..7ef57c0f9 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -48,7 +48,7 @@ func GetPointRecordList(ctx *context.Context) { } r, err := reward.GetRewardRecordList(models.RewardRecordListOpts{ - ListOptions: models.ListOptions{PageSize: 20, Page: page}, + ListOptions: models.ListOptions{PageSize: 10, Page: page}, UserId: ctx.User.ID, OperateType: t, RewardType: models.RewardTypePoint, diff --git a/services/task/task.go b/services/task/task.go index b53adb1f9..78f188997 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -16,6 +16,17 @@ func Accomplish(action models.Action) { if action.Repo.IsPrivate { return } + case models.ActionBindWechat: + n, err := models.CountWechatBindLog(action.Content, models.WECHAT_BIND) + if err != nil { + log.Error("CountWechatBindLog error when accomplish task,err=%v", err) + return + } + //if wechatOpenId has been bound before,the action can not get reward + if n > 1 { + return + } + } go accomplish(action) } From a18c391ae94e659f5c5209f7821f28ae83e67268 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 1 Jul 2022 14:37:57 +0800 Subject: [PATCH 34/90] #2225 fix bug --- models/task_accomplish_log.go | 6 +++--- services/reward/limiter/limiter.go | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/models/task_accomplish_log.go b/models/task_accomplish_log.go index 75494bfa2..582411561 100644 --- a/models/task_accomplish_log.go +++ b/models/task_accomplish_log.go @@ -30,11 +30,11 @@ func getTaskAccomplishLog(tl *TaskAccomplishLog) (*TaskAccomplishLog, error) { return tl, nil } -func CountTaskAccomplishLogInTaskPeriod(configId int64, userId int64, period *PeriodResult) (int64, error) { +func CountTaskAccomplishLogInTaskPeriod(taskCode string, userId int64, period *PeriodResult) (int64, error) { if period == nil { - return x.Where("config_id = ? and user_id = ?", configId, userId).Count(&TaskAccomplishLog{}) + return x.Where("task_code = ? and user_id = ?", taskCode, userId).Count(&TaskAccomplishLog{}) } else { - return x.Where("config_id = ? and user_id = ? and created_unix >= ? and created_unix < ? ", configId, userId, period.StartTime.Unix(), period.EndTime.Unix()).Count(&TaskAccomplishLog{}) + return x.Where("task_code = ? and user_id = ? and created_unix >= ? and created_unix < ? ", taskCode, userId, period.StartTime.Unix(), period.EndTime.Unix()).Count(&TaskAccomplishLog{}) } } diff --git a/services/reward/limiter/limiter.go b/services/reward/limiter/limiter.go index a73779ac1..d357ceabd 100644 --- a/services/reward/limiter/limiter.go +++ b/services/reward/limiter/limiter.go @@ -142,6 +142,9 @@ func (l *limiterRunner) limit(r models.LimitConfig) error { } if p != nil { redis_client.Expire(redisKey, p.LeftTime) + } else { + //add default expire time if no period set + redis_client.Expire(redisKey, 24*time.Hour) } } if usedNum > r.LimitNum { @@ -183,7 +186,7 @@ func (l *limiterRunner) LoadLimiters() error { func (l *limiterRunner) countInPeriod(r models.LimitConfig, p *models.PeriodResult) (int64, error) { switch r.LimitType { case models.LimitTypeTask.Name(): - return models.CountTaskAccomplishLogInTaskPeriod(r.ID, l.userId, p) + return models.CountTaskAccomplishLogInTaskPeriod(r.LimitCode, l.userId, p) case models.LimitTypeRewardPoint.Name(): return models.SumRewardAmountInTaskPeriod(models.RewardTypePoint.Name(), r.LimitCode, l.userId, p) default: From 4d2f89ca8023063131a77ce39f04b5eb6763840a Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 1 Jul 2022 16:39:35 +0800 Subject: [PATCH 35/90] #2225 fix bug --- models/attachment.go | 2 +- models/repo_watch.go | 5 +++++ models/reward_periodic_task.go | 2 +- services/task/task.go | 13 +++++++++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index 3dc8eac49..e7051c632 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -667,7 +667,7 @@ func Attachments(opts *AttachmentsOptions) ([]*AttachmentInfo, int64, error) { func GetAllDatasetContributorByDatasetId(datasetId int64) ([]*User, error) { r := make([]*User, 0) - if err := x.Select("distinct(user.*)").Table("attachment").Join("LEFT", "user", "user.ID = attachment.uploader_id").Where("attachment.dataset_id = ?", datasetId).Find(&r); err != nil { + if err := x.Select("distinct(public.user.*)").Table("attachment").Join("LEFT", "user", "public.user.ID = attachment.uploader_id").Where("attachment.dataset_id = ?", datasetId).Find(&r); err != nil { return nil, err } return r, nil diff --git a/models/repo_watch.go b/models/repo_watch.go index 864aec254..485874301 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -204,6 +204,11 @@ func notifyWatchers(e Engine, actions ...*Action) error { // Send the act to task chan ActionChan4Task <- *act + // If it has nothing to do with repo, return directly + if act.Repo == nil && act.RepoID == 0 { + return nil + } + if repoChanged { act.loadRepo() repo = act.Repo diff --git a/models/reward_periodic_task.go b/models/reward_periodic_task.go index 910f4fe8e..5e5466e86 100644 --- a/models/reward_periodic_task.go +++ b/models/reward_periodic_task.go @@ -90,7 +90,7 @@ func GetPeriodicTaskBySourceIdAndType(sourceType SourceType, sourceId string, op r := RewardPeriodicTask{} _, err := x.SQL("select rpt.* from reward_periodic_task rpt "+ "inner join reward_operate_record ror on rpt.operate_serial_no = ror.serial_no"+ - " where ror.source_type = ? and source_id = ? and operate_type = ? ", sourceType.Name(), sourceId, operateType.Name()).Get(&r) + " where ror.source_type = ? and ror.source_id = ? and ror.operate_type = ? ", sourceType.Name(), sourceId, operateType.Name()).Get(&r) if err != nil { return nil, err } diff --git a/services/task/task.go b/services/task/task.go index 78f188997..5fc5e9bcb 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -9,13 +9,22 @@ import ( ) func Accomplish(action models.Action) { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:%v", combinedErr) + } + }() action.OpType = models.GetTaskOptType(action) switch action.OpType { - case models.ActionCreateRepo, - models.ActionCreateImage: + case models.ActionCreateRepo: if action.Repo.IsPrivate { return } + case models.ActionCreateImage: + if action.IsPrivate { + return + } case models.ActionBindWechat: n, err := models.CountWechatBindLog(action.Content, models.WECHAT_BIND) if err != nil { From 6730d4ca9370ad1d74da8f60d9480dc89d3fe4d2 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Fri, 1 Jul 2022 18:15:18 +0800 Subject: [PATCH 36/90] #2225 update --- models/action.go | 38 ++++++++++++----- models/action_list.go | 61 +++++++++++++++++++++++++-- models/cloudbrain.go | 8 ++++ models/helper.go | 7 +++ modules/notification/action/action.go | 12 ++---- modules/notification/base/notifier.go | 2 +- modules/notification/base/null.go | 2 +- modules/notification/notification.go | 4 +- routers/image/image.go | 5 ++- 9 files changed, 113 insertions(+), 26 deletions(-) diff --git a/models/action.go b/models/action.go index 33322a921..1a25c162a 100755 --- a/models/action.go +++ b/models/action.go @@ -89,6 +89,7 @@ type Action struct { IsTransformed bool `xorm:"INDEX NOT NULL DEFAULT false"` Content string `xorm:"TEXT"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + Cloudbrain *Cloudbrain `xorm:"-"` } type ActionShow struct { @@ -100,6 +101,7 @@ type ActionShow struct { IssueInfos []string CommentLink string IssueTitle string + Cloudbrain *CloudbrainShow4Action } // GetOpType gets the ActionType of this action. @@ -256,22 +258,25 @@ func (a *Action) ToShow() *ActionShow { actionShow.CommentLink = a.GetCommentLink() } + if a.Cloudbrain != nil { + c := &CloudbrainShow4Action{ + ID: a.Cloudbrain.ID, + Type: a.Cloudbrain.Type, + JobType: a.Cloudbrain.JobType, + DisplayJobName: a.Cloudbrain.DisplayJobName, + ComputeResource: a.Cloudbrain.ComputeResource, + } + actionShow.Cloudbrain = c + } + return actionShow } func GetTaskOptType(action Action) ActionType { - switch action.OpType { - case ActionCreateDebugGPUTask, - ActionCreateDebugNPUTask, - ActionCreateTrainTask, - ActionCreateInferenceTask, - ActionCreateBenchMarkTask, - ActionCreateGPUTrainTask: + if action.IsCloudbrainAction() { return ActionCreateCloudbrainTask - default: - return action.OpType } - + return action.OpType } // GetRepositoryFromMatch returns a *Repository from a username and repo strings @@ -371,6 +376,19 @@ func (a *Action) GetIssueContent() string { return issue.Content } +func (a *Action) IsCloudbrainAction() bool { + switch a.OpType { + case ActionCreateDebugGPUTask, + ActionCreateDebugNPUTask, + ActionCreateTrainTask, + ActionCreateInferenceTask, + ActionCreateBenchMarkTask, + ActionCreateGPUTrainTask: + return true + } + return false +} + // GetFeedsOptions options for retrieving feeds type GetFeedsOptions struct { RequestedUser *User // the user we want activity for diff --git a/models/action_list.go b/models/action_list.go index a0987c20d..17700edbd 100644 --- a/models/action_list.go +++ b/models/action_list.go @@ -4,7 +4,10 @@ package models -import "fmt" +import ( + "fmt" + "strconv" +) // ActionList defines a list of actions type ActionList []*Action @@ -111,7 +114,9 @@ func (actions ActionList) loadComments(e Engine) ([]*Comment, error) { } for _, action := range actions { - action.Comment = commentMaps[action.CommentID] + if action.CommentID > 0 { + action.Comment = commentMaps[action.CommentID] + } } return valuesComment(commentMaps), nil } @@ -121,6 +126,52 @@ func (actions ActionList) LoadComments() ([]*Comment, error) { return actions.loadComments(x) } +func (actions ActionList) getCloudbrainIDs() []int64 { + cloudbrainIDs := make(map[int64]struct{}, 0) + for _, action := range actions { + if !action.IsCloudbrainAction() { + continue + } + cloudbrainId, _ := strconv.ParseInt(action.Content, 10, 64) + if _, ok := cloudbrainIDs[cloudbrainId]; !ok { + cloudbrainIDs[cloudbrainId] = struct{}{} + } + } + return keysInt64(cloudbrainIDs) +} + +func (actions ActionList) loadCloudbrains(e Engine) ([]*Cloudbrain, error) { + if len(actions) == 0 { + return nil, nil + } + + cloudbrainIDs := actions.getCloudbrainIDs() + + cloudbrainMaps := make(map[int64]*Cloudbrain, len(cloudbrainIDs)) + if len(cloudbrainIDs) == 0 { + return make([]*Cloudbrain, 0), nil + } + err := e. + In("id", cloudbrainIDs).Unscoped(). + Find(&cloudbrainMaps) + if err != nil { + return nil, fmt.Errorf("find cloudbrain: %v", err) + } + + for _, action := range actions { + cloudbrainId, _ := strconv.ParseInt(action.Content, 10, 64) + if cloudbrainId > 0 { + action.Cloudbrain = cloudbrainMaps[cloudbrainId] + } + } + return valuesCloudbrain(cloudbrainMaps), nil +} + +// LoadComments loads actions' all comments +func (actions ActionList) LoadCloudbrains() ([]*Comment, error) { + return actions.loadComments(x) +} + // loadAttributes loads all attributes func (actions ActionList) loadAttributes(e Engine) (err error) { if _, err = actions.loadUsers(e); err != nil { @@ -140,7 +191,7 @@ func (actions ActionList) LoadAttributes() error { } // LoadAllAttributes loads all attributes of the actions -// compare with LoadAttributes() ,LoadAllAttributes() loads Comment attribute +// compare with LoadAttributes() ,LoadAllAttributes() loads Comment and Cloudbrain attribute func (actions ActionList) LoadAllAttributes() error { return actions.loadAllAttributes(x) } @@ -159,5 +210,9 @@ func (actions ActionList) loadAllAttributes(e Engine) (err error) { return } + if _, err = actions.loadCloudbrains(e); err != nil { + return + } + return nil } diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 75ab1dfd6..1d2e56476 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -179,6 +179,14 @@ type CloudbrainShow struct { ComputeResource string } +type CloudbrainShow4Action struct { + ID int64 + Type int + JobType string + DisplayJobName string + ComputeResource string +} + func (task *Cloudbrain) ToShow() *CloudbrainShow { return &CloudbrainShow{ ID: task.ID, diff --git a/models/helper.go b/models/helper.go index 55d4cac31..e381f1e37 100644 --- a/models/helper.go +++ b/models/helper.go @@ -35,3 +35,10 @@ func valuesComment(m map[int64]*Comment) []*Comment { } return values } +func valuesCloudbrain(m map[int64]*Cloudbrain) []*Cloudbrain { + var values = make([]*Cloudbrain, 0, len(m)) + for _, v := range m { + values = append(values, v) + } + return values +} diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index 6a43c6e9a..bfe574328 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -375,7 +375,7 @@ func (t *actionNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *m ActUser: user, RepoID: dataset.RepoID, Repo: dataset.Repo, - Content: fmt.Sprint(dataset.ID), + Content: fmt.Sprintf("%d|%s", dataset.ID, dataset.Title), }) } if err := models.NotifyWatchers(actions...); err != nil { @@ -390,18 +390,14 @@ func (t *actionNotifier) NotifyCreateImage(doer *models.User, image models.Image ActUser: doer, OpType: models.ActionCreateImage, IsPrivate: image.IsPrivate, - Content: fmt.Sprint(image.ID), + Content: fmt.Sprintf("%d|%s", image.ID, image.Tag), } if err := models.NotifyWatchers(act); err != nil { log.Error("notifyWatchers: %v", err) } } -func (t *actionNotifier) NotifyImageRecommend(optUser *models.User, imageId int64, action string) { - image, err := models.GetImageByID(imageId) - if err != nil { - return - } +func (t *actionNotifier) NotifyImageRecommend(optUser *models.User, image *models.Image, action string) { u, err := models.GetUserByID(image.UID) if err != nil { return @@ -413,7 +409,7 @@ func (t *actionNotifier) NotifyImageRecommend(optUser *models.User, imageId int6 ActUser: u, OpType: models.ActionImageRecommend, IsPrivate: false, - Content: fmt.Sprint(imageId), + Content: fmt.Sprintf("%d|%s", image.ID, image.Tag), } if err := models.NotifyWatchers(act); err != nil { log.Error("notifyWatchers: %v", err) diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 7673a5909..1429dc090 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -60,6 +60,6 @@ type Notifier interface { NotifyWechatBind(user *models.User, wechatOpenId string) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) NotifyCreateImage(doer *models.User, image models.Image) - NotifyImageRecommend(optUser *models.User, imageId int64, action string) + NotifyImageRecommend(optUser *models.User, image *models.Image, action string) NotifyChangeUserAvatar(user *models.User, form auth.AvatarForm) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index eea5c5e77..27ed24f15 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -170,7 +170,7 @@ func (*NullNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *model func (*NullNotifier) NotifyCreateImage(doer *models.User, image models.Image) { } -func (*NullNotifier) NotifyImageRecommend(optUser *models.User, imageId int64, action string) { +func (*NullNotifier) NotifyImageRecommend(optUser *models.User, image *models.Image, action string) { } func (*NullNotifier) NotifyChangeUserAvatar(user *models.User, form auth.AvatarForm) { diff --git a/modules/notification/notification.go b/modules/notification/notification.go index d652dc043..6c96d58da 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -293,9 +293,9 @@ func NotifyCreateImage(doer *models.User, image models.Image) { } // NotifyDatasetRecommend -func NotifyImageRecommend(optUser *models.User, imageId int64, action string) { +func NotifyImageRecommend(optUser *models.User, image *models.Image, action string) { for _, notifier := range notifiers { - notifier.NotifyImageRecommend(optUser, imageId, action) + notifier.NotifyImageRecommend(optUser, image, action) } } diff --git a/routers/image/image.go b/routers/image/image.go index e238387ab..35b6b943b 100644 --- a/routers/image/image.go +++ b/routers/image/image.go @@ -26,7 +26,10 @@ func Action(ctx *context.Context) { if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.star_fail", ctx.Params(":action")))) } else { - notification.NotifyImageRecommend(ctx.User, imageId, ctx.Params(":action")) + image, err := models.GetImageByID(imageId) + if err == nil { + notification.NotifyImageRecommend(ctx.User, image, ctx.Params(":action")) + } ctx.JSON(http.StatusOK, models.BaseOKMessage) } } From d03ec1ba7cf96d8861ad4cfade7f3b7e9911fda7 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Fri, 1 Jul 2022 18:47:11 +0800 Subject: [PATCH 37/90] =?UTF-8?q?=E7=AE=97=E5=8A=9B=E7=A7=AF=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web_src/vuepages/components/BaseDialog.vue | 113 ++++++++++++++++ web_src/vuepages/pages/reward/point/utils.js | 98 ++++++++++++++ .../vuepages/pages/reward/point/vp-point.vue | 126 ++++++++++-------- 3 files changed, 282 insertions(+), 55 deletions(-) create mode 100644 web_src/vuepages/components/BaseDialog.vue create mode 100644 web_src/vuepages/pages/reward/point/utils.js diff --git a/web_src/vuepages/components/BaseDialog.vue b/web_src/vuepages/components/BaseDialog.vue new file mode 100644 index 000000000..a95f99d18 --- /dev/null +++ b/web_src/vuepages/components/BaseDialog.vue @@ -0,0 +1,113 @@ + + + diff --git a/web_src/vuepages/pages/reward/point/utils.js b/web_src/vuepages/pages/reward/point/utils.js new file mode 100644 index 000000000..b19e41b1f --- /dev/null +++ b/web_src/vuepages/pages/reward/point/utils.js @@ -0,0 +1,98 @@ + +import { formatDate } from 'element-ui/lib/utils/date-util'; +import { SOURCE_TYPE, CONSUME_STATUS, POINT_ACTIONS } from './const'; + +const getSourceType = (key) => { + const find = SOURCE_TYPE.filter(item => item.k === key); + return find.length ? find[0].v : key; +}; +const getConsumeStatus = (key) => { + const find = CONSUME_STATUS.filter(item => item.k === key); + return find.length ? find[0].v : key; +}; +const getPointAction = (key) => { + const find = POINT_ACTIONS.filter(item => item.k === key); + return find.length ? find[0].v : key; +}; + +export const getRewardPointRecordInfo = (record) => { + const out = { + sn: record.SerialNo, + date: formatDate(new Date(record.LastOperateDate * 1000), 'yyyy-MM-DD HH:mm:ss'), + _status: record.Status, + status: getConsumeStatus(record.Status) || '--', + statusColor: record.Status === 'OPERATING' ? 'rgb(33, 186, 69)' : '', + _sourceType: record.SourceType, + sourceType: getSourceType(record.SourceType), + duration: record?.Cloudbrain?.Duration || '--', + taskName: record?.Cloudbrain?.DisplayJobName || '--', + taskId: record?.Cloudbrain?.ID, + action: record?.Action?.OpType ? getPointAction(record.Action.OpType) : '--', + remark: record.Remark, + amount: record.Amount, + }; + if (record.OperateType === 'INCREASE') { + if (record.SourceType === 'ADMIN_OPERATE') { + out.remark = record.Remark; + } else if (record.SourceType === 'ACCOMPLISH_TASK') { + switch (record?.Action?.OpType) { + case 1: // 创建公开项目 - 创建了项目OpenI/aiforge + out.remark = `创建了项目${record.Action.ShortRepoFullDisplayName}`; + break; + case 6: // 每日提出任务 - 创建了任务PCL-Platform.Intelligence/AISynergy#19 + out.remark = `创建了任务${record.Action.ShortRepoFullDisplayName}#${record.Action.IssueInfos[0]}`; + break; + case 7: // 每日提出PR - 创建了合并请求OpenI/aiforge#1 + out.remark = `创建了合并请求${record.Action.ShortRepoFullDisplayName}#${record.Action.IssueInfos[0]}`; + break; + case 10: // 发表评论 - 评论了任务PCL-Platform.Intelligence/AISynergy#19 + out.remark = `评论了任务${record.Action.ShortRepoFullDisplayName}#${record.Action.IssueInfos[0]}`; + break; + case 24: // 上传数据集文件 - 上传了数据集文件MMISTData.zip + out.remark = `上传了数据集文件${record.Action.RefName}`; + break; + case 30: // 导入新模型 - 导入了新模型resnet50_qx7l + break; + case 32: // 完成微信扫码验证 - 首次绑定微信奖励 + out.remark = '首次绑定微信奖励'; + break; + case 33: // 每日运行云脑任务 - 创建了(CPU/GPU/NPU)类型(调试/训练/推理/评测)任务tangl202204131431995 + out.remark = `创建了{{}}类型{{}}任务${record.Action.RefName}`; + break; + case 34: // 数据集被平台推荐 - 数据集XXX被设置为推荐数据集 + out.remark = `数据集${record.Action.RefName}被设置为推荐数据集`; + break; + case 35: // 提交新公开镜像 - 提交了镜像jiangxiang_ceshi_tang03 + out.remark = `提交了镜像${record.Action.RefName}`; + break; + case 36: // 镜像被平台推荐 - 镜像XXX被设置为推荐镜像 + out.remark = `镜像${record.Action.RefName}被设置为推荐镜像`; + break; + case 37: // 首次更换头像 - 更新了头像 + out.remark = '更新了头像'; + break; + case 38: // 每日commit - 推送了xxxx分支的代码到OpenI/aiforge + const words = record.Action.RefName.split('/'); + const branch = words[words.length - 1]; + out.remark = `推送了${branch}分支的代码到${record.Action.ShortRepoFullDisplayName}`; + break; + case 39: // 每日首次Fork项目 - 创建了项目OpenI/fork_aiforge + out.remark = `创建了项目${record.Action.ShortRepoFullDisplayName}`; + break; + default: + break; + } + } else if (record.SourceType === 'RUN_CLOUDBRAIN_TASK') { + + } + } else if (record.OperateType === 'DECREASE') { + if (record.SourceType === 'ADMIN_OPERATE') { + out.remark = record.Remark; + } else if (record.SourceType === 'ACCOMPLISH_TASK') { + + } else if (record.SourceType === 'RUN_CLOUDBRAIN_TASK') { + + } + } + return out; +}; diff --git a/web_src/vuepages/pages/reward/point/vp-point.vue b/web_src/vuepages/pages/reward/point/vp-point.vue index 22bb1f47e..65b48f1c9 100644 --- a/web_src/vuepages/pages/reward/point/vp-point.vue +++ b/web_src/vuepages/pages/reward/point/vp-point.vue @@ -41,30 +41,53 @@
-
- - - - +
+ + + + + + + + - - - + + + +
+
+ + - + - + + - + + + + - + + @@ -88,7 +111,7 @@ From 097c76d7f1f423209944a3379cffca2153895bc1 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 4 Jul 2022 17:41:25 +0800 Subject: [PATCH 38/90] #2225 update --- models/action.go | 25 ++++++++----------------- models/action_list.go | 6 ++++++ models/cloudbrain.go | 11 ++++++++--- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/models/action.go b/models/action.go index 3e6a58077..c3cf7e9f2 100755 --- a/models/action.go +++ b/models/action.go @@ -60,23 +60,14 @@ const ( ActionCreateGPUTrainTask //31 ActionCreateGrampusNPUTrainTask //32 ActionCreateGrampusGPUTrainTask //33 - ActionUploadAttachment //24 - ActionCreateDebugGPUTask //25 - ActionCreateDebugNPUTask //26 - ActionCreateTrainTask //27 - ActionCreateInferenceTask // 28 - ActionCreateBenchMarkTask //29 - ActionCreateNewModelTask //30 - ActionCreateGPUTrainTask //31 - - ActionBindWechat //32issue_assignees - ActionCreateCloudbrainTask //33 - ActionDatasetRecommended //34 - ActionCreateImage //35 - ActionImageRecommend //36 - ActionChangeUserAvatar //37 - ActionPushCommits //38 - ActionForkRepo //39 + ActionBindWechat //34 + ActionCreateCloudbrainTask //35 + ActionDatasetRecommended //36 + ActionCreateImage //37 + ActionImageRecommend //38 + ActionChangeUserAvatar //39 + ActionPushCommits //40 + ActionForkRepo //41 ) diff --git a/models/action_list.go b/models/action_list.go index 17700edbd..0a355d0ce 100644 --- a/models/action_list.go +++ b/models/action_list.go @@ -29,6 +29,9 @@ func (actions ActionList) loadUsers(e Engine) ([]*User, error) { userIDs := actions.getUserIDs() userMaps := make(map[int64]*User, len(userIDs)) + if len(userIDs) == 0 { + return make([]*User, 0), nil + } err := e. In("id", userIDs). Find(&userMaps) @@ -64,6 +67,9 @@ func (actions ActionList) loadRepositories(e Engine) ([]*Repository, error) { repoIDs := actions.getRepoIDs() repoMaps := make(map[int64]*Repository, len(repoIDs)) + if len(repoIDs) == 0 { + return make([]*Repository, 0), nil + } err := e. In("id", repoIDs). Find(&repoMaps) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index d2d4ac656..0f5707e3c 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -188,6 +188,7 @@ type CloudbrainShow struct { Duration string ResourceSpec *ResourceAndFlavor ComputeResource string + AiCenter string } type CloudbrainShow4Action struct { @@ -199,9 +200,9 @@ type CloudbrainShow4Action struct { } func (task *Cloudbrain) ToShow() *CloudbrainShow { - return &CloudbrainShow{ - ID: task.ID, - RepoFullName: task.Repo.FullName(), + c := &CloudbrainShow{ + ID: task.ID, + JobType: task.JobType, Type: task.Type, DisplayJobName: task.DisplayJobName, @@ -209,6 +210,10 @@ func (task *Cloudbrain) ToShow() *CloudbrainShow { ResourceSpec: GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode), ComputeResource: task.ComputeResource, } + if task.Repo != nil { + c.RepoFullName = task.Repo.FullName() + } + return c } func (task *Cloudbrain) ComputeAndSetDuration() { From 3a0aa205af9d029c6285698ee7b4a34e638c75d1 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 5 Jul 2022 09:32:49 +0800 Subject: [PATCH 39/90] #2225 remove issueTittle from ActionShow --- models/action.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/models/action.go b/models/action.go index c3cf7e9f2..b5b3381e3 100755 --- a/models/action.go +++ b/models/action.go @@ -101,7 +101,6 @@ type ActionShow struct { RefName string IssueInfos []string CommentLink string - IssueTitle string Cloudbrain *CloudbrainShow4Action } @@ -246,9 +245,8 @@ func (a *Action) ToShow() *ActionShow { actionShow.Content = a.Content actionShow.RefName = a.RefName - if strings.Contains(a.Content, "|") { + if strings.Contains(a.Content, "|") && a.IsIssueAction() { actionShow.IssueInfos = a.GetIssueInfos() - actionShow.IssueTitle = a.GetIssueTitle() } if a.Repo != nil { @@ -390,6 +388,24 @@ func (a *Action) IsCloudbrainAction() bool { return false } +func (a *Action) IsIssueAction() bool { + switch a.OpType { + case ActionCreateIssue, + ActionCloseIssue, + ActionClosePullRequest, + ActionReopenIssue, + ActionReopenPullRequest, + ActionCommentPull, + ActionCommentIssue, + ActionCreatePullRequest, + ActionApprovePullRequest, + ActionRejectPullRequest, + ActionMergePullRequest: + return true + } + return false +} + // GetFeedsOptions options for retrieving feeds type GetFeedsOptions struct { RequestedUser *User // the user we want activity for From 66fb71107f553238a3dac202ef7ac5ffb9da80d8 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 5 Jul 2022 16:22:19 +0800 Subject: [PATCH 40/90] #2225 add point rule route --- routers/reward/point/point.go | 5 +++++ routers/routes/routes.go | 1 + 2 files changed, 6 insertions(+) diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index 7ef57c0f9..3828a2900 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -11,6 +11,7 @@ import ( ) const tplPoint base.TplName = "reward/point" +const tplPointRule base.TplName = "reward/point/rule" type AccountResponse struct { Balance int64 @@ -80,3 +81,7 @@ func OperatePointAccountBalance(ctx *context.Context, req models.AdminRewardOper func GetPointPage(ctx *context.Context) { ctx.HTML(200, tplPoint) } + +func GetRulePage(ctx *context.Context) { + ctx.HTML(200, tplPointRule) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 00a820fc9..52504c388 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1355,6 +1355,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/reward/point", func() { m.Get("", point.GetPointPage) + m.Get("/rule", point.GetRulePage) m.Get("/account", point.GetPointAccount) m.Get("/record/list", point.GetPointRecordList) }, reqSignIn) From 77cdc92ef4f14917431171bedc9df1cbad4e160f Mon Sep 17 00:00:00 2001 From: chenshihai Date: Tue, 5 Jul 2022 17:15:10 +0800 Subject: [PATCH 41/90] =?UTF-8?q?=E7=AE=97=E5=8A=9B=E7=A7=AF=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/reward/point/rule.tmpl | 117 ++++++++++++++++++ web_src/vuepages/pages/reward/point/const.js | 5 +- web_src/vuepages/pages/reward/point/utils.js | 80 +++++++++--- .../vuepages/pages/reward/point/vp-point.js | 1 - .../vuepages/pages/reward/point/vp-point.vue | 28 ++--- 5 files changed, 191 insertions(+), 40 deletions(-) create mode 100644 templates/reward/point/rule.tmpl diff --git a/templates/reward/point/rule.tmpl b/templates/reward/point/rule.tmpl new file mode 100644 index 000000000..645a2c96c --- /dev/null +++ b/templates/reward/point/rule.tmpl @@ -0,0 +1,117 @@ +{{template "base/head_home" .}} + +
+

个人算力积分奖励规则

+
+

说明：单日用户积分的获取上限为50分。

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
奖励名称获取积分值上限值奖励细节澄清
完成微信扫码验证50累计积分获取上限501、首次完成微信扫码验证，即获取50积分。
2、同个账号，更换微信号码再验证不重复给积分。
3、同一个微信，绑定第一个账号时奖励50分，下次绑定其他账号时不再奖励。
每日首次Fork项目1每日积分获取上限1
创建公开项目1每日积分获取上限3请注意项目质量，请勿复制粘贴或者重复公开项目，任何非常规的以公开项目去获取积分的行为将被认定为积分舞弊，将扣除所有积分。
每日提出PR1每日积分获取上限3
每日commit1每日积分获取上限3通过前台界面和后台命令行方式commit，都可获得奖励积分。
每日提出任务1每日积分获取上限3
发表评论1每日积分获取上限2禁止空评论或评论后马上删除等非正常获取积分的方式，一经发现将扣除所有积分。
上传数据集文件1每日积分获取上限1请注意数据集质量，请勿复制粘贴或者重复公开数据集，任何非常规的以公开数据集去获取积分的行为将被认定为积分舞弊，将扣除所有积分。
数据集被平台推荐5每日积分获取上限15仅统计属于个人的数据集，属于组织的数据集暂不统计。
导入新模型1每日积分获取上限3请注意模型质量，请勿重复导入相同模型，任何非常规的以导入新模型去获取 积分的行为将被认定为积分舞弊，将扣除所有积分。
每日运行云脑任务10每日积分获取上限10 每日运行调试、训练、推理、评测中任何一种任务，即可获得。
提交新公开镜像1每日积分获取上限3
镜像被平台推荐5每日积分获取上限15
首次更换头像2累计积分获取上限2首次更换头像，积分+2。
+ +
+{{template "base/footer" .}} \ No newline at end of file diff --git a/web_src/vuepages/pages/reward/point/const.js b/web_src/vuepages/pages/reward/point/const.js index af0332cc7..a0fe162e5 100644 --- a/web_src/vuepages/pages/reward/point/const.js +++ b/web_src/vuepages/pages/reward/point/const.js @@ -1,6 +1,7 @@ export const SOURCE_TYPE = [{ k: 'ACCOMPLISH_TASK', v: '积分任务' }, { k: 'ADMIN_OPERATE', v: '管理员操作' }, { k: 'RUN_CLOUDBRAIN_TASK', v: '运行云脑任务' }]; export const CONSUME_STATUS = [{ k: 'OPERATING', v: '进行中' }, { k: 'SUCCEEDED', v: '已完成' }]; export const POINT_ACTIONS = [ - { k: 1, v: '创建公开项目' }, { k: 6, v: '每日提出任务' }, { k: 7, v: '每日提出PR' }, { k: 10, v: '发表评论' }, { k: 24, v: '上传数据集文件' }, { k: 30, v: '导入新模型' }, { k: 32, v: '完成微信扫码验证' }, - { k: 33, v: '每日运行云脑任务' }, { k: 34, v: '数据集被平台推荐' }, { k: 35, v: '提交新公开镜像' }, { k: 36, v: '镜像被平台推荐' }, { k: 37, v: '首次更换头像' }, { k: 38, v: '每日commit' }, { k: 39, v: '每日首次Fork项目' }, + { k: 1, v: '创建公开项目' }, { k: 6, v: '每日提出任务' }, { k: 7, v: '每日提出PR' }, { k: 10, v: '发表评论' }, { k: 24, v: '上传数据集文件' }, { k: 30, v: '导入新模型' }, { k: 34, v: '完成微信扫码验证' }, + { k: 35, v: '每日运行云脑任务' }, { k: 36, v: '数据集被平台推荐' }, { k: 37, v: '提交新公开镜像' }, { k: 38, v: '镜像被平台推荐' }, { k: 39, v: '首次更换头像' }, { k: 40, v: '每日commit' }, { k: 41, v: '每日首次Fork项目' }, ]; +export const JOB_TYPE = [{ k: 'DEBUG', v: '调试任务' }, { k: 'TRAIN', v: '训练任务' }, { k: 'INFERENCE', v: '推理任务' }, { k: 'BENCHMARK', v: '评测任务' }]; diff --git a/web_src/vuepages/pages/reward/point/utils.js b/web_src/vuepages/pages/reward/point/utils.js index b19e41b1f..d97fa4fc0 100644 --- a/web_src/vuepages/pages/reward/point/utils.js +++ b/web_src/vuepages/pages/reward/point/utils.js @@ -1,24 +1,63 @@ import { formatDate } from 'element-ui/lib/utils/date-util'; -import { SOURCE_TYPE, CONSUME_STATUS, POINT_ACTIONS } from './const'; +import { SOURCE_TYPE, CONSUME_STATUS, POINT_ACTIONS, JOB_TYPE } from './const'; const getSourceType = (key) => { const find = SOURCE_TYPE.filter(item => item.k === key); return find.length ? find[0].v : key; }; + const getConsumeStatus = (key) => { const find = CONSUME_STATUS.filter(item => item.k === key); return find.length ? find[0].v : key; }; + const getPointAction = (key) => { const find = POINT_ACTIONS.filter(item => item.k === key); return find.length ? find[0].v : key; }; +const getJobType = (key) => { + const find = JOB_TYPE.filter(item => item.k === key); + return find.length ? find[0].v : key; +}; + +const getJobTypeLink = (record, type) => { + let link = type === 'INCREASE' ? record.Action.RepoLink : '/' + record.Cloudbrain.RepoFullName; + const cloudbrain = type === 'INCREASE' ? record.Action?.Cloudbrain : record.Cloudbrain; + switch (cloudbrain?.JobType) { + case 'DEBUG': + if (cloudbrain.ComputeResource === 'CPU/GPU') { + link += `/cloudbrain/${cloudbrain.ID}`; + } else { + link += `/modelarts/notebook/${cloudbrain.ID}`; + } + break; + case 'TRAIN': + if (cloudbrain.Type === 1) { + link += `/modelarts/train-job/${cloudbrain.ID}`; + } else if (cloudbrain.Type === 0) { + link += `/cloudbrain/train-job/${cloudbrain.ID}`; + } else if (cloudbrain.Type === 2) { + link += `/grampus/train-job/${cloudbrain.ID}`; + } + break; + case 'INFERENCE': + link += `/modelarts/inference-job/${cloudbrain.ID}`; + break; + case 'BENCHMARK': + link += `/cloudbrain/benchmark/${cloudbrain.ID}`; + break; + default: + break; + }; + return link; +}; + export const getRewardPointRecordInfo = (record) => { const out = { sn: record.SerialNo, - date: formatDate(new Date(record.LastOperateDate * 1000), 'yyyy-MM-DD HH:mm:ss'), + date: formatDate(new Date(record.LastOperateDate * 1000), 'yyyy-MM-dd HH:mm:ss'), _status: record.Status, status: getConsumeStatus(record.Status) || '--', statusColor: record.Status === 'OPERATING' ? 'rgb(33, 186, 69)' : '', @@ -52,46 +91,53 @@ export const getRewardPointRecordInfo = (record) => { out.remark = `上传了数据集文件${record.Action.RefName}`; break; case 30: // 导入新模型 - 导入了新模型resnet50_qx7l + out.remark = '导入了新模型{{}}'; break; - case 32: // 完成微信扫码验证 - 首次绑定微信奖励 + case 34: // 完成微信扫码验证 - 首次绑定微信奖励 out.remark = '首次绑定微信奖励'; break; - case 33: // 每日运行云脑任务 - 创建了(CPU/GPU/NPU)类型(调试/训练/推理/评测)任务tangl202204131431995 - out.remark = `创建了{{}}类型{{}}任务${record.Action.RefName}`; + case 35: // 每日运行云脑任务 - 创建了(CPU/GPU/NPU)类型(调试/训练/推理/评测)任务tangl202204131431995 + out.remark = `创建了${record.Action?.Cloudbrain?.ComputeResource}类型${getJobType(record.Action?.Cloudbrain?.JobType)}${record.Action.RefName}`; break; - case 34: // 数据集被平台推荐 - 数据集XXX被设置为推荐数据集 - out.remark = `数据集${record.Action.RefName}被设置为推荐数据集`; + case 36: // 数据集被平台推荐 - 数据集XXX被设置为推荐数据集 + out.remark = `数据集${record.Action.Content && record.Action.Content.split('|')[1]}被设置为推荐数据集`; break; - case 35: // 提交新公开镜像 - 提交了镜像jiangxiang_ceshi_tang03 - out.remark = `提交了镜像${record.Action.RefName}`; + case 37: // 提交新公开镜像 - 提交了镜像jiangxiang_ceshi_tang03 + out.remark = `提交了镜像${record.Action.Content && record.Action.Content.split('|')[1]}`; break; - case 36: // 镜像被平台推荐 - 镜像XXX被设置为推荐镜像 - out.remark = `镜像${record.Action.RefName}被设置为推荐镜像`; + case 38: // 镜像被平台推荐 - 镜像XXX被设置为推荐镜像 + out.remark = `镜像${record.Action.Content && record.Action.Content.split('|')[1]}被设置为推荐镜像`; break; - case 37: // 首次更换头像 - 更新了头像 + case 39: // 首次更换头像 - 更新了头像 out.remark = '更新了头像'; break; - case 38: // 每日commit - 推送了xxxx分支的代码到OpenI/aiforge + case 40: // 每日commit - 推送了xxxx分支的代码到OpenI/aiforge const words = record.Action.RefName.split('/'); const branch = words[words.length - 1]; out.remark = `推送了${branch}分支的代码到${record.Action.ShortRepoFullDisplayName}`; break; - case 39: // 每日首次Fork项目 - 创建了项目OpenI/fork_aiforge + case 41: // 每日首次Fork项目 - 创建了项目OpenI/fork_aiforge out.remark = `创建了项目${record.Action.ShortRepoFullDisplayName}`; break; default: break; } } else if (record.SourceType === 'RUN_CLOUDBRAIN_TASK') { - + // } } else if (record.OperateType === 'DECREASE') { if (record.SourceType === 'ADMIN_OPERATE') { out.remark = record.Remark; } else if (record.SourceType === 'ACCOMPLISH_TASK') { - + // } else if (record.SourceType === 'RUN_CLOUDBRAIN_TASK') { - + out.taskName = `${record?.Cloudbrain?.DisplayJobName}`; + if (record?.Cloudbrain?.ComputeResource === 'CPU/GPU') { + const resourceSpec = record?.Cloudbrain?.ResourceSpec?.ResourceSpec; + out.remark = `【${getJobType(record?.Cloudbrain?.JobType)}】【${record?.Cloudbrain?.ComputeResource}】【GPU: ${resourceSpec?.gpu}, CPU: ${resourceSpec?.cpu}, 内存: ${(resourceSpec?.memMiB / 1024).toFixed(2)}GB, 共享内存: ${(resourceSpec?.shareMemMiB / 1024).toFixed(2)}GB】`; + } else { + out.remark = `【${getJobType(record?.Cloudbrain?.JobType)}】【${record?.Cloudbrain?.ComputeResource}】【${record?.Cloudbrain?.ResourceSpec.FlavorInfo.desc}】`; + } } } return out; diff --git a/web_src/vuepages/pages/reward/point/vp-point.js b/web_src/vuepages/pages/reward/point/vp-point.js index 8039d3f2c..8ef5bfa67 100644 --- a/web_src/vuepages/pages/reward/point/vp-point.js +++ b/web_src/vuepages/pages/reward/point/vp-point.js @@ -4,7 +4,6 @@ import 'element-ui/lib/theme-chalk/index.css'; Vue.use(ElementUI); import App from './vp-point.vue'; -// import App from '../manage/vp-point-manage.vue'; new Vue({ el: '#__vue-root', diff --git a/web_src/vuepages/pages/reward/point/vp-point.vue b/web_src/vuepages/pages/reward/point/vp-point.vue index 65b48f1c9..6a33ed6af 100644 --- a/web_src/vuepages/pages/reward/point/vp-point.vue +++ b/web_src/vuepages/pages/reward/point/vp-point.vue @@ -6,12 +6,10 @@

算力积分明细

@@ -78,13 +76,17 @@ + width="120"> - + + @@ -109,8 +111,7 @@ \ No newline at end of file diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index 295fe0435..955457eef 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -104,7 +104,26 @@ top: 14px; z-index: 2; */ } + .inline.field { + padding-left: 12rem !important; + } + + .inline.field>label { + width: 120px !important; + text-align: right; + } + .inline.field .dropdown.selection { + width: 60% !important; + } + + .width70 { + width: 60% !important; + } + + .inline.field input { + width: 40% !important; + }
@@ -119,170 +138,176 @@
{{template "repo/header" .}} -
-
- - {{template "base/alert" .}} -
-

-
+
+ + {{template "base/alert" .}} +

+ {{.i18n.Tr "repo.cloudbrain.new"}} +

+ +
{{.CsrfTokenHtml}} -

- {{.i18n.Tr "repo.cloudbrain.new"}} -

-
-
- -
- - - - - - CPU/GPU - - - - - - - Ascend NPU -
-
-
- - +
+ +
+ + + + + + CPU/GPU + + + + + + + Ascend NPU
+
+
+ + +
-
- - -
+
+ + +
-
- - -
- -
- -
-
- -
- - -
-
- - +
+ + +
+ +
+
+
+ +
+ + +
+
+ + +
-
+
-
-
+
+
+
+ +
+ + +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} +
- -
- - -
+
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
- +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ -
- - - {{.i18n.Tr "repo.cloudbrain.cancel"}} -
+
+ + + {{.i18n.Tr "repo.cloudbrain.cancel"}}
- +
-
+
{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index adf3adf9b..9b10897dd 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -25,6 +25,10 @@ margin-left: -2px; } + .width485 { + width: 48.5% !important; + } + .width85 { width: 85% !important; margin-left: 10.5rem !important; @@ -247,13 +251,22 @@
- {{range .train_resource_specs}} - + {{end}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
@@ -493,4 +506,19 @@ send_run_para() validate() }) + + ;(function() { + $('#cloudbrain_resource_spec').on('change', function(e) { + var cloudbrain_resource_spec_blance_tip_el = $('.cloudbrain_resource_spec_blance_tip'); + var val = $(this).val(); + var blance = $(this).attr('blance'); + var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (unitPrice == 0) { + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); + } else { + var canUseTime = Number(blance) / Number(unitPrice); + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').text(canUseTime.toFixed(2)).parent().show(); + } + }).trigger('change'); + })(); \ No newline at end of file diff --git a/templates/repo/modelarts/inferencejob/new.tmpl b/templates/repo/modelarts/inferencejob/new.tmpl index 90a7c900d..84b6d3951 100644 --- a/templates/repo/modelarts/inferencejob/new.tmpl +++ b/templates/repo/modelarts/inferencejob/new.tmpl @@ -222,11 +222,21 @@
        - {{range .flavor_infos}} - + {{end}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
@@ -463,4 +473,19 @@ get_name() validate() }) + + ;(function() { + $('#trainjob-flavor').on('change', function(e) { + var cloudbrain_resource_spec_blance_tip_el = $('.cloudbrain_resource_spec_blance_tip'); + var val = $(this).val(); + var blance = $(this).attr('blance'); + var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (unitPrice == 0) { + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); + } else { + var canUseTime = Number(blance) / Number(unitPrice); + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').text(canUseTime.toFixed(2)).parent().show(); + } + }).trigger('change'); + })(); diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index 4e2b3951d..38ac6b01f 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -3,6 +3,26 @@ .inline.required.field.cloudbrain_benchmark { display: none; } +.inline.field { + padding-left: 12rem !important; +} + +.inline.field>label { + width: 120px !important; + text-align: right; +} + +.inline.field .dropdown.selection { + width: 60% !important; +} + +.width70 { + width: 60% !important; +} + +.inline.field input { + width: 40% !important; +}
@@ -16,87 +36,92 @@
{{template "repo/header" .}} -
-
- - {{template "base/alert" .}} -
-

-
+
+ + {{template "base/alert" .}} +

+ {{.i18n.Tr "repo.cloudbrain.new"}} +

+ +
{{.CsrfTokenHtml}} -

- {{.i18n.Tr "repo.cloudbrain.new"}} -

-
- -
- -
- - - - - - CPU/GPU - - - - - - Ascend NPU -
-
-
- - +
+ +
+ + + + + + CPU/GPU + + + + + + Ascend NPU
+
+
+ + +
-
- - -
-
+
+ + +
+
-
+
- -
- - -
- -
- - -
-
- - - {{.i18n.Tr "repo.cloudbrain.cancel"}} + +
+ + +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} +
+ +
+ + +
+
+ + + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
@@ -158,4 +183,19 @@ } }); }); + + ;(function() { + $('#cloudbrain_flavor').on('change', function(e) { + var cloudbrain_resource_spec_blance_tip_el = $('.cloudbrain_resource_spec_blance_tip'); + var val = $(this).val(); + var blance = $(this).attr('blance'); + var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (unitPrice == 0) { + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); + } else { + var canUseTime = Number(blance) / Number(unitPrice); + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').text(canUseTime.toFixed(2)).parent().show(); + } + }).trigger('change'); + })(); diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 5022bd41b..ff9f35b51 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -247,11 +247,21 @@
- {{range .flavor_infos}} - + {{end}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
@@ -503,4 +513,19 @@ send_run_para() validate() }) + + ;(function() { + $('#trainjob-flavor').on('change', function(e) { + var cloudbrain_resource_spec_blance_tip_el = $('.cloudbrain_resource_spec_blance_tip'); + var val = $(this).val(); + var blance = $(this).attr('blance'); + var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (unitPrice == 0) { + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); + } else { + var canUseTime = Number(blance) / Number(unitPrice); + cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').text(canUseTime.toFixed(2)).parent().show(); + } + }).trigger('change'); + })(); \ No newline at end of file From 24afb0065ced24648411ec2dee9fe9643b68c897 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 13 Jul 2022 14:25:08 +0800 Subject: [PATCH 68/90] #2225 update --- modules/context/point.go | 2 ++ routers/reward/point/point.go | 2 +- services/reward/notify.go | 10 +++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/context/point.go b/modules/context/point.go index 9fbff61be..8fd4724ee 100644 --- a/modules/context/point.go +++ b/modules/context/point.go @@ -1,6 +1,7 @@ package context import ( + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/reward/point/account" "gitea.com/macaron/macaron" ) @@ -14,6 +15,7 @@ func PointAccount() macaron.Handler { return } ctx.Data["PointAccount"] = a + ctx.Data["CloudBrainPaySwitch"] = setting.CloudBrainPaySwitch ctx.Next() } } diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index a8ae00ce4..7b3e0fe49 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -71,7 +71,7 @@ func GetPointRecordList(ctx *context.Context) { func OperatePointAccountBalance(ctx *context.Context, req models.AdminRewardOperateReq) { req.RewardType = models.RewardTypePoint - if req.OperateType.Name() == "" { + if req.OperateType.Name() == "" || req.Remark == "" { ctx.JSON(http.StatusOK, "param error") return } diff --git a/services/reward/notify.go b/services/reward/notify.go index 875dde199..4f3190d67 100644 --- a/services/reward/notify.go +++ b/services/reward/notify.go @@ -11,7 +11,15 @@ import ( "time" ) -func NotifyRewardOperation(userId, amount int64, rewardType models.RewardType, operateType models.RewardOperateType) { +func NotifyRewardOperation(userId, amount int64, sourceType models.SourceType, rewardType models.RewardType, operateType models.RewardOperateType) { + switch sourceType { + case models.SourceTypeRunCloudbrainTask: + return + case models.SourceTypeAdminOperate: + if operateType == models.OperateTypeDecrease { + return + } + } data := &models.UserRewardOperationRedis{ UserId: userId, Amount: amount, From 252064d4100011fce03b2649df40625f763ee650 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 13 Jul 2022 14:39:38 +0800 Subject: [PATCH 69/90] #2225 update --- services/reward/notify.go | 4 ---- services/reward/operator.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/services/reward/notify.go b/services/reward/notify.go index 4f3190d67..4db218537 100644 --- a/services/reward/notify.go +++ b/services/reward/notify.go @@ -15,10 +15,6 @@ func NotifyRewardOperation(userId, amount int64, sourceType models.SourceType, r switch sourceType { case models.SourceTypeRunCloudbrainTask: return - case models.SourceTypeAdminOperate: - if operateType == models.OperateTypeDecrease { - return - } } data := &models.UserRewardOperationRedis{ UserId: userId, diff --git a/services/reward/operator.go b/services/reward/operator.go index f32024688..4e1d53b75 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -86,7 +86,7 @@ func Operate(ctx *models.RewardOperateContext) error { } UpdateRewardRecordToFinalStatus(ctx.SourceType.Name(), ctx.RequestId, models.OperateStatusSucceeded) - NotifyRewardOperation(ctx.TargetUserId, ctx.Reward.Amount, ctx.Reward.Type, ctx.OperateType) + NotifyRewardOperation(ctx.TargetUserId, ctx.Reward.Amount, ctx.SourceType, ctx.Reward.Type, ctx.OperateType) return nil } From 0e9b41328393a1c618c2742c71d51a8360636e7f Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 13 Jul 2022 17:25:39 +0800 Subject: [PATCH 70/90] #2225 update --- routers/repo/cloudbrain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 031ae2617..20360084c 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,7 +2,6 @@ package repo import ( "bufio" - "code.gitea.io/gitea/modules/grampus" "code.gitea.io/gitea/services/reward/point/account" "encoding/json" "errors" From 93ae27de092ff1d2eea925966c98e9533b392aa6 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 13 Jul 2022 17:32:10 +0800 Subject: [PATCH 71/90] #2225 update --- modules/templates/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 857e365f8..797ccdb2e 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -788,7 +788,7 @@ func GetRefName(ref string) string { return reg.ReplaceAllString(ref, "") } -func MB2GB(size int64) string { +func MB2GB(size int) string { s := strconv.FormatFloat(float64(size)/float64(1024), 'f', 2, 64) for strings.HasSuffix(s, "0") { s = strings.TrimSuffix(s, "0") From b5ead01ff8dc80e7f2526fa0e5b717c917451b31 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 13 Jul 2022 17:39:11 +0800 Subject: [PATCH 72/90] =?UTF-8?q?=E7=AE=97=E5=8A=9B=E7=A7=AF=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/repo/cloudbrain/benchmark/new.tmpl | 25 +++++++++++++------ templates/repo/cloudbrain/new.tmpl | 13 +++++++--- templates/repo/cloudbrain/trainjob/new.tmpl | 13 +++++++--- .../repo/modelarts/inferencejob/new.tmpl | 11 +++++--- templates/repo/modelarts/notebook/new.tmpl | 11 +++++--- templates/repo/modelarts/trainjob/new.tmpl | 13 +++++++--- web_src/js/features/notification.js | 5 +++- web_src/vuepages/langs/config/en-US.js | 4 +-- web_src/vuepages/langs/config/zh-CN.js | 4 +-- .../vuepages/pages/reward/point/vp-point.vue | 4 +-- 10 files changed, 70 insertions(+), 33 deletions(-) diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index d2dae1a50..ac0481b5f 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -126,22 +126,26 @@ {{template "custom/select_dataset_train" .}}
- + {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} {{$.i18n.Tr "points.points_acquisition_instructions"}} + {{end}}
@@ -226,15 +230,18 @@
+ {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -242,6 +249,7 @@ {{$.i18n.Tr "points.points_acquisition_instructions"}}
+ {{end}}
@@ -375,6 +383,7 @@ var val = $(this).val(); var blance = $(this).attr('blance'); var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (!blance || !unitPrice) return; if (unitPrice == 0) { cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); } else { diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index 955457eef..c3fccbf34 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -243,16 +243,19 @@
- + {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -260,6 +263,7 @@ {{$.i18n.Tr "points.points_acquisition_instructions"}}
+ {{end}}
@@ -405,6 +409,7 @@ var val = $(this).val(); var blance = $(this).attr('blance'); var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (!blance || !unitPrice) return; if (unitPrice == 0) { cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); } else { diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index 9b10897dd..de7ca766c 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -252,14 +252,17 @@
+ {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -267,6 +270,7 @@ {{$.i18n.Tr "points.points_acquisition_instructions"}}
+ {{end}}
@@ -513,6 +517,7 @@ var val = $(this).val(); var blance = $(this).attr('blance'); var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (!blance || !unitPrice) return; if (unitPrice == 0) { cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); } else { diff --git a/templates/repo/modelarts/inferencejob/new.tmpl b/templates/repo/modelarts/inferencejob/new.tmpl index 84b6d3951..f6fc6afd9 100644 --- a/templates/repo/modelarts/inferencejob/new.tmpl +++ b/templates/repo/modelarts/inferencejob/new.tmpl @@ -222,14 +222,17 @@
        - {{range .flavor_infos}} - {{end}} + {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -237,6 +240,7 @@ {{$.i18n.Tr "points.points_acquisition_instructions"}}
+ {{end}}
@@ -480,6 +484,7 @@ var val = $(this).val(); var blance = $(this).attr('blance'); var unitPrice = $(this).find('option:selected').attr('unitprice'); + if (!blance || !unitPrice) return; if (unitPrice == 0) { cloudbrain_resource_spec_blance_tip_el.find('.can-use-time').parent().hide(); } else { diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index 38ac6b01f..2d4a821c6 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -92,13 +92,16 @@
-->
- {{range .flavors}} - {{end}} + {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -106,6 +109,7 @@ {{$.i18n.Tr "points.points_acquisition_instructions"}}
+ {{end}}
- + {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}}
@@ -518,8 +527,8 @@ }) ;(function() { var SPECS = {{ .inference_specs }}; - var showPoint = true; - renderSpecsSelect($('#__specs__'), SPECS, showPoint, { + var showPoint = {{ .CloudBrainPaySwitch }}; + window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}}, diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl index 1d19627dd..157d29055 100644 --- a/templates/repo/cloudbrain/inference/show.tmpl +++ b/templates/repo/cloudbrain/inference/show.tmpl @@ -618,7 +618,7 @@ ;(function() { var SPEC = {{ .Spec }}; - var showPoint = true; + var showPoint = false; var specStr = window.renderSpecStr(SPEC, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index cac971eaf..d02f50d68 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -146,8 +146,18 @@ + {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}}
@@ -303,13 +313,13 @@ ;(function() { var SPECS = {{ .debug_specs }}; - var showPoint = true; + var showPoint = {{ .CloudBrainPaySwitch }}; window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}}, memory: {{$.i18n.Tr "cloudbrain.memory"}}, shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, - }); + }); })(); \ No newline at end of file diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index 8096e1e8a..b86a5b9bd 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -595,7 +595,7 @@ } ;(function() { var SPEC = {{ .Spec }}; - var showPoint = true; + var showPoint = false; var specStr = window.renderSpecStr(SPEC, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index 9d680cd70..1fba6df8e 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -259,8 +259,18 @@
+ {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}}
@@ -435,8 +445,8 @@ }) ;(function() { var SPECS = {{ .train_specs }}; - var showPoint = true; - renderSpecsSelect($('#__specs__'), SPECS, showPoint, { + var showPoint = {{ .CloudBrainPaySwitch }}; + window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}}, diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 1dba0b7f0..bcebf11ac 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -984,7 +984,7 @@ ;(function() { var SPEC = {{ .Spec }}; - var showPoint = true; + var showPoint = false; var specStr = window.renderSpecStr(SPEC, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, diff --git a/templates/repo/grampus/trainjob/gpu/new.tmpl b/templates/repo/grampus/trainjob/gpu/new.tmpl index 6eb2b49fd..5b3e3cf70 100755 --- a/templates/repo/grampus/trainjob/gpu/new.tmpl +++ b/templates/repo/grampus/trainjob/gpu/new.tmpl @@ -206,7 +206,16 @@
-->
- + + {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}}
@@ -386,8 +395,8 @@ ;(function() { var SPECS = {{ .Specs }}; - var showPoint = true; - renderSpecsSelect($('#__specs__'), SPECS, showPoint, { + var showPoint = {{ .CloudBrainPaySwitch }}; + window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}}, diff --git a/templates/repo/grampus/trainjob/npu/new.tmpl b/templates/repo/grampus/trainjob/npu/new.tmpl index 6849528dc..df40699b8 100755 --- a/templates/repo/grampus/trainjob/npu/new.tmpl +++ b/templates/repo/grampus/trainjob/npu/new.tmpl @@ -218,7 +218,16 @@
-->
- + + {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}}
@@ -407,8 +416,8 @@ ;(function() { var SPECS = {{ .Specs }}; - var showPoint = true; - renderSpecsSelect($('#__specs__'), SPECS, showPoint, { + var showPoint = {{ .CloudBrainPaySwitch }}; + window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, { gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}}, free: {{$.i18n.Tr "cloudbrain.free"}}, point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}}, diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 6e9e0d6dc..7e7fd86ea 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -633,7 +633,7 @@ \ No newline at end of file diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index b86a5b9bd..6cd977af2 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -346,9 +346,7 @@ -
- {{$.resource_type}} -
+
@@ -604,6 +602,6 @@ shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, }); $('td.ti-text-form-content.spec div').text(specStr); - $('td.ti-text-form-content.resorce_type div').text(getListValueWithKey(ACC_CARD_TYPE, SPEC.AccCardType)); + SPEC && $('td.ti-text-form-content.resorce_type div').text(getListValueWithKey(ACC_CARD_TYPE, SPEC.AccCardType)); })(); \ No newline at end of file diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index bcebf11ac..5ffce27cc 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -360,9 +360,7 @@ -
- {{$.resource_type}} -
+
@@ -371,9 +369,7 @@ -
- {{$.i18n.Tr "cloudbrain.gpu_num"}}：{{$.GpuNum}}，{{$.i18n.Tr "cloudbrain.cpu_num"}}：{{$.CpuNum}}，{{$.i18n.Tr "cloudbrain.memory"}}(MB)：{{$.MemMiB}}，{{$.i18n.Tr "cloudbrain.shared_memory"}}(MB)：{{$.ShareMemMiB}} -
+
@@ -993,6 +989,6 @@ shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, }); $('td.ti-text-form-content.spec div').text(specStr); - $('td.ti-text-form-content.resorce_type div').text(getListValueWithKey(ACC_CARD_TYPE, SPEC.AccCardType)); + SPEC && $('td.ti-text-form-content.resorce_type div').text(getListValueWithKey(ACC_CARD_TYPE, SPEC.AccCardType)); })(); \ No newline at end of file diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 7e7fd86ea..c76dbc940 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -358,9 +358,7 @@ -
- {{.FlavorName}} -
+
diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl index 3f8fda531..619a2acba 100644 --- a/templates/repo/modelarts/inferencejob/show.tmpl +++ b/templates/repo/modelarts/inferencejob/show.tmpl @@ -424,9 +424,7 @@ td, th { -
- {{.FlavorName}} -
+
@@ -543,6 +541,5 @@ $(document).ready(function(){ shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, }); $('td.ti-text-form-content.spec div').text(specStr); - // $('td.ti-text-form-content.resorce_type').text(getListValueWithKey(ACC_CARD_TYPE, SPEC.AccCardType)); })(); diff --git a/templates/repo/modelarts/notebook/show.tmpl b/templates/repo/modelarts/notebook/show.tmpl index 5353f7a52..2a22392cb 100755 --- a/templates/repo/modelarts/notebook/show.tmpl +++ b/templates/repo/modelarts/notebook/show.tmpl @@ -368,9 +368,7 @@ -
- {{$.resource_spec}} -
+
@@ -504,6 +502,5 @@ shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, }); $('td.ti-text-form-content.spec div').text(specStr); - $('td.ti-text-form-content.resorce_type div').text(getListValueWithKey(ACC_CARD_TYPE, SPEC.AccCardType)); })(); diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index e10afc38a..ddbbad3f0 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -398,9 +398,7 @@ -
- {{.FlavorName}} -
+
diff --git a/web_src/js/standalone/specsuse.js b/web_src/js/standalone/specsuse.js index 97a4647fe..99f5445a4 100644 --- a/web_src/js/standalone/specsuse.js +++ b/web_src/js/standalone/specsuse.js @@ -9,6 +9,7 @@ window.getListValueWithKey = (list, key, k = 'k', v = 'v', defaultV = '') => { }; window.renderSpecStr = (spec, showPoint, langObj) => { + if (!spec) return ''; var ngpu = `${spec.ComputeResource}: ${spec.AccCardsNum + '*' + getListValueWithKey(ACC_CARD_TYPE, spec.AccCardType)}`; var gpuMemStr = spec.GPUMemGiB != 0 ? `${langObj.gpu_memory}: ${spec.GPUMemGiB}GB, ` : ''; var sharedMemStr = spec.ShareMemGiB != 0 ? `, ${langObj.shared_memory}: ${spec.ShareMemGiB}GB` : ''; From 811e02e2eeddc601ff29d4e88ccffdaa0c09b55c Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 31 Aug 2022 15:29:57 +0800 Subject: [PATCH 83/90] #2701 update point serialNo --- services/reward/operator.go | 17 ----------------- services/reward/serial.go | 9 +++++---- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/services/reward/operator.go b/services/reward/operator.go index c9f00b1bf..b66810c70 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -276,22 +276,5 @@ func generateOperateSerialNo(operateType models.RewardOperateType, rewardType mo return "", err } - - switch operateType { - case models.OperateTypeIncrease: - s += "1" - case models.OperateTypeDecrease: - s += "2" - default: - s += "9" - } - - switch rewardType { - case models.RewardTypePoint: - s += "1" - default: - s += "9" - } - return s, nil } diff --git a/services/reward/serial.go b/services/reward/serial.go index b6a47bbc3..349da1266 100644 --- a/services/reward/serial.go +++ b/services/reward/serial.go @@ -11,13 +11,14 @@ import ( func GetSerialNoByRedis() (string, error) { now := time.Now() - n, err := redis_client.IncrBy(redis_key.RewardSerialCounter(now), 1) + r := int64(rand.Intn(4)) + 1 + n, err := redis_client.IncrBy(redis_key.RewardSerialCounter(now), r) if err != nil { log.Error("GetSerialNoByRedis RewardSerialCounter error. %v", err) return "", err } - if n == 1 { - redis_client.Expire(redis_key.RewardSerialCounter(now), 5*time.Minute) + if n == r { + redis_client.Expire(redis_key.RewardSerialCounter(now), 2*time.Minute) } - return now.Format("200601021504") + fmt.Sprint(rand.Intn(10)) + fmt.Sprintf("%02d", n), nil + return now.Format("200601021504") + fmt.Sprintf("%03d", n) + fmt.Sprint(rand.Intn(10)), nil } From 283d9045aa080a70ac3dfac1f04100bb10de6ae8 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 31 Aug 2022 15:34:43 +0800 Subject: [PATCH 84/90] #2701 update point serialNo --- services/reward/operator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/reward/operator.go b/services/reward/operator.go index b66810c70..b9d8c8d59 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -116,7 +116,7 @@ func isHandled(sourceType string, requestId string, operateType string) (bool, e } func initRewardOperateRecord(ctx *models.RewardOperateContext) (string, error) { - sn, err := generateOperateSerialNo(ctx.OperateType, ctx.Reward.Type) + sn, err := generateOperateSerialNo() if err != nil { log.Error("generateOperateSerialNo error. %v", err) return "", err @@ -145,7 +145,7 @@ func initRewardOperateRecord(ctx *models.RewardOperateContext) (string, error) { } func createPeriodicRewardOperateRecord(ctx *models.StartPeriodicTaskOpts) (string, error) { - sn, err := generateOperateSerialNo(ctx.OperateType, ctx.RewardType) + sn, err := generateOperateSerialNo() if err != nil { log.Error("createPeriodic generateOperateSerialNo error. %v", err) return "", err @@ -269,7 +269,7 @@ func StopPeriodicTask(sourceType models.SourceType, sourceId string, operateType return models.StopPeriodicTask(task.ID, task.OperateSerialNo, now) } -func generateOperateSerialNo(operateType models.RewardOperateType, rewardType models.RewardType) (string, error) { +func generateOperateSerialNo() (string, error) { s, err := GetSerialNoByRedis() if err != nil { log.Error("generateOperateSerialNo error. %v", err) From b226fe7b5c6040a15c2fdc0af2aa01f5b6c020c1 Mon Sep 17 00:00:00 2001 From: openihu Date: Tue, 13 Sep 2022 11:22:37 +0800 Subject: [PATCH 85/90] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=A1=B5=E8=84=9A?= =?UTF-8?q?=E5=9B=BE=E6=A0=87=E5=8F=8A=E8=A1=8C=E9=97=B4=E8=B7=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/base/footer_content.tmpl | 10 +++++----- templates/base/footer_content_fluid.tmpl | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) mode change 100644 => 100755 templates/base/footer_content_fluid.tmpl diff --git a/templates/base/footer_content.tmpl b/templates/base/footer_content.tmpl index bcf46f9f0..aec7b8d61 100755 --- a/templates/base/footer_content.tmpl +++ b/templates/base/footer_content.tmpl @@ -20,7 +20,7 @@
{{.i18n.Tr "custom.foot.help"}}
- +
{{.LangName}}
{{range .AllLangs}} @@ -29,12 +29,12 @@
-

{{.i18n.Tr "custom.Platform_Tutorial"}}

 - {{if .EnableSwagger}}

API

{{end}} + {{.i18n.Tr "custom.Platform_Tutorial"}} + {{if .EnableSwagger}} API{{end}} {{if .IsSigned}} -

{{.i18n.Tr "custom.foot.advice_feedback"}}

 + {{.i18n.Tr "custom.foot.advice_feedback"}} {{else}} -

{{.i18n.Tr "custom.foot.advice_feedback"}}

 + {{.i18n.Tr "custom.foot.advice_feedback"}} {{end}} {{template "custom/extra_links_footer" .}} diff --git a/templates/base/footer_content_fluid.tmpl b/templates/base/footer_content_fluid.tmpl old mode 100644 new mode 100755 index 6d3cada9e..29395a045 --- a/templates/base/footer_content_fluid.tmpl +++ b/templates/base/footer_content_fluid.tmpl @@ -18,7 +18,7 @@
{{.i18n.Tr "custom.foot.help"}}
- +
{{.LangName}}
{{range .AllLangs}} @@ -26,12 +26,12 @@ {{end}}
-

{{.i18n.Tr "custom.Platform_Tutorial"}}

 - {{if .EnableSwagger}}

API

{{end}} + {{.i18n.Tr "custom.Platform_Tutorial"}} + {{if .EnableSwagger}} API{{end}} {{if .IsSigned}} -

{{.i18n.Tr "custom.foot.advice_feedback"}}

 + {{.i18n.Tr "custom.foot.advice_feedback"}} {{else}} -

{{.i18n.Tr "custom.foot.advice_feedback"}}

 + {{.i18n.Tr "custom.foot.advice_feedback"}} {{end}} {{template "custom/extra_links_footer" .}}
From 55b24d613bec5a1f056682fd9dc7188474f88e95 Mon Sep 17 00:00:00 2001 From: openihu Date: Tue, 13 Sep 2022 11:26:21 +0800 Subject: [PATCH 86/90] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=A1=B5=E8=84=9A?= =?UTF-8?q?=E5=B8=AE=E5=8A=A9=E5=8C=BA=E5=9F=9F=E5=9B=BE=E6=A0=87=E5=8F=8A?= =?UTF-8?q?=E8=A1=8C=E9=97=B4=E8=B7=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/base/footer_content.tmpl | 4 ++-- templates/base/footer_content_fluid.tmpl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/base/footer_content.tmpl b/templates/base/footer_content.tmpl index aec7b8d61..cdc2cf549 100755 --- a/templates/base/footer_content.tmpl +++ b/templates/base/footer_content.tmpl @@ -30,11 +30,11 @@
{{.i18n.Tr "custom.Platform_Tutorial"}} - {{if .EnableSwagger}} API{{end}} + {{if .EnableSwagger}} API{{end}} {{if .IsSigned}} {{.i18n.Tr "custom.foot.advice_feedback"}} {{else}} - {{.i18n.Tr "custom.foot.advice_feedback"}} + {{.i18n.Tr "custom.foot.advice_feedback"}} {{end}} {{template "custom/extra_links_footer" .}} diff --git a/templates/base/footer_content_fluid.tmpl b/templates/base/footer_content_fluid.tmpl index 29395a045..24b18e94d 100755 --- a/templates/base/footer_content_fluid.tmpl +++ b/templates/base/footer_content_fluid.tmpl @@ -26,12 +26,12 @@ {{end}}
- {{.i18n.Tr "custom.Platform_Tutorial"}} + {{.i18n.Tr "custom.Platform_Tutorial"}} {{if .EnableSwagger}} API{{end}} {{if .IsSigned}} {{.i18n.Tr "custom.foot.advice_feedback"}} {{else}} - {{.i18n.Tr "custom.foot.advice_feedback"}} + {{.i18n.Tr "custom.foot.advice_feedback"}} {{end}} {{template "custom/extra_links_footer" .}}
From aea59980292e8de6d1ca043a2f2cb065e48865b9 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 14 Sep 2022 15:18:12 +0800 Subject: [PATCH 87/90] update tmpl --- templates/repo/modelarts/notebook/new.tmpl | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index 5d56a04f8..f9c4670a5 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -91,7 +91,6 @@
{{end}} -