You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

cloudbrain.go 82 kB

4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
5 years ago
5 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
4 years ago
4 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
5 years ago
4 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629
  1. package repo
  2. import (
  3. "bufio"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "net/http"
  9. "os"
  10. "regexp"
  11. "sort"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "unicode/utf8"
  16. "code.gitea.io/gitea/modules/grampus"
  17. "code.gitea.io/gitea/modules/timeutil"
  18. "github.com/unknwon/i18n"
  19. "code.gitea.io/gitea/models"
  20. "code.gitea.io/gitea/modules/auth"
  21. "code.gitea.io/gitea/modules/base"
  22. "code.gitea.io/gitea/modules/cloudbrain"
  23. "code.gitea.io/gitea/modules/context"
  24. "code.gitea.io/gitea/modules/git"
  25. "code.gitea.io/gitea/modules/log"
  26. "code.gitea.io/gitea/modules/modelarts"
  27. "code.gitea.io/gitea/modules/setting"
  28. "code.gitea.io/gitea/modules/storage"
  29. "code.gitea.io/gitea/modules/util"
  30. )
  31. const (
  32. tplCloudBrainNew base.TplName = "repo/cloudbrain/new"
  33. tplCloudBrainShow base.TplName = "repo/cloudbrain/show"
  34. tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index"
  35. tplCloudBrainBenchmarkIndex base.TplName = "repo/cloudbrain/benchmark/index"
  36. tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new"
  37. tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show"
  38. tplCloudBrainImageSubmit base.TplName = "repo/cloudbrain/image/submit"
  39. tplCloudBrainImageEdit base.TplName = "repo/cloudbrain/image/edit"
  40. tplCloudBrainTrainJobNew base.TplName = "repo/cloudbrain/trainjob/new"
  41. tplCloudBrainTrainJobShow base.TplName = "repo/cloudbrain/trainjob/show"
  42. tplCloudBrainInferenceJobNew base.TplName = "repo/cloudbrain/inference/new"
  43. tplCloudBrainInferenceJobShow base.TplName = "repo/cloudbrain/inference/show"
  44. )
  45. var (
  46. gpuInfos *models.GpuInfos
  47. categories *models.Categories
  48. benchmarkTypes *models.BenchmarkTypes
  49. benchmarkGpuInfos *models.GpuInfos
  50. benchmarkResourceSpecs *models.ResourceSpecs
  51. trainGpuInfos *models.GpuInfos
  52. inferenceGpuInfos *models.GpuInfos
  53. )
  54. const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types"
  55. const CLONE_FILE_PREFIX = "file:///"
  56. var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0)
  57. var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
  58. // MustEnableDataset check if repository enable internal cb
  59. func MustEnableCloudbrain(ctx *context.Context) {
  60. if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
  61. ctx.NotFound("MustEnableCloudbrain", nil)
  62. return
  63. }
  64. }
  65. func cutString(str string, lens int) string {
  66. if len(str) < lens {
  67. return str
  68. }
  69. return str[:lens]
  70. }
  71. func jobNamePrefixValid(s string) string {
  72. lowStr := strings.ToLower(s)
  73. re := regexp.MustCompile(`[^a-z0-9_\\-]+`)
  74. removeSpecial := re.ReplaceAllString(lowStr, "")
  75. re = regexp.MustCompile(`^[_\\-]+`)
  76. return re.ReplaceAllString(removeSpecial, "")
  77. }
  78. func cloudBrainNewDataPrepare(ctx *context.Context) error {
  79. ctx.Data["PageIsCloudBrain"] = true
  80. t := time.Now()
  81. var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
  82. ctx.Data["display_job_name"] = displayJobName
  83. ctx.Data["command"] = cloudbrain.GetCloudbrainDebugCommand()
  84. ctx.Data["code_path"] = cloudbrain.CodeMountPath
  85. ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
  86. ctx.Data["model_path"] = cloudbrain.ModelMountPath
  87. ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
  88. ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled
  89. if categories == nil {
  90. json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
  91. }
  92. ctx.Data["benchmark_categories"] = categories.Category
  93. ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType
  94. queuesDetail, _ := cloudbrain.GetQueuesDetail()
  95. if queuesDetail != nil {
  96. ctx.Data["QueuesDetail"] = queuesDetail
  97. }
  98. cloudbrain.InitSpecialPool()
  99. if gpuInfos == nil {
  100. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  101. }
  102. ctx.Data["gpu_types"] = gpuInfos.GpuInfo
  103. if trainGpuInfos == nil {
  104. json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos)
  105. }
  106. ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo
  107. if inferenceGpuInfos == nil && setting.InferenceGpuTypes != "" {
  108. json.Unmarshal([]byte(setting.InferenceGpuTypes), &inferenceGpuInfos)
  109. }
  110. if inferenceGpuInfos != nil {
  111. ctx.Data["inference_gpu_types"] = inferenceGpuInfos.GpuInfo
  112. }
  113. if benchmarkGpuInfos == nil {
  114. json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos)
  115. }
  116. ctx.Data["benchmark_gpu_types"] = benchmarkGpuInfos.GpuInfo
  117. if benchmarkResourceSpecs == nil {
  118. json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs)
  119. }
  120. ctx.Data["benchmark_resource_specs"] = benchmarkResourceSpecs.ResourceSpec
  121. if cloudbrain.ResourceSpecs == nil {
  122. json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
  123. }
  124. ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec
  125. if cloudbrain.TrainResourceSpecs == nil {
  126. json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs)
  127. }
  128. ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec
  129. if cloudbrain.InferenceResourceSpecs == nil && setting.InferenceResourceSpecs != "" {
  130. json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs)
  131. }
  132. if cloudbrain.InferenceResourceSpecs != nil {
  133. ctx.Data["inference_resource_specs"] = cloudbrain.InferenceResourceSpecs.ResourceSpec
  134. }
  135. if cloudbrain.SpecialPools != nil {
  136. var debugGpuTypes []*models.GpuInfo
  137. var trainGpuTypes []*models.GpuInfo
  138. for _, pool := range cloudbrain.SpecialPools.Pools {
  139. org, _ := models.GetOrgByName(pool.Org)
  140. if org != nil {
  141. isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID)
  142. if isOrgMember {
  143. for _, jobType := range pool.JobType {
  144. if jobType == string(models.JobTypeDebug) {
  145. debugGpuTypes = append(debugGpuTypes, pool.Pool...)
  146. if pool.ResourceSpec != nil {
  147. ctx.Data["resource_specs"] = pool.ResourceSpec
  148. }
  149. } else if jobType == string(models.JobTypeTrain) {
  150. trainGpuTypes = append(trainGpuTypes, pool.Pool...)
  151. if pool.ResourceSpec != nil {
  152. ctx.Data["train_resource_specs"] = pool.ResourceSpec
  153. }
  154. }
  155. }
  156. break
  157. }
  158. }
  159. }
  160. if len(debugGpuTypes) > 0 {
  161. ctx.Data["gpu_types"] = debugGpuTypes
  162. }
  163. if len(trainGpuTypes) > 0 {
  164. ctx.Data["train_gpu_types"] = trainGpuTypes
  165. }
  166. }
  167. ctx.Data["params"] = ""
  168. ctx.Data["branchName"] = ctx.Repo.BranchName
  169. ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
  170. ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled
  171. ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
  172. ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled
  173. ctx.Data["datasetType"] = models.TypeCloudBrainOne
  174. ctx.Data["benchmarkMode"] = ctx.Query("benchmarkMode")
  175. return nil
  176. }
  177. func CloudBrainNew(ctx *context.Context) {
  178. err := cloudBrainNewDataPrepare(ctx)
  179. if err != nil {
  180. ctx.ServerError("get new cloudbrain info failed", err)
  181. return
  182. }
  183. ctx.HTML(200, tplCloudBrainNew)
  184. }
  185. func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  186. ctx.Data["PageIsCloudBrain"] = true
  187. displayJobName := form.DisplayJobName
  188. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  189. image := strings.TrimSpace(form.Image)
  190. uuids := form.Attachment
  191. jobType := form.JobType
  192. gpuQueue := form.GpuType
  193. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  194. resourceSpecId := form.ResourceSpecId
  195. branchName := form.BranchName
  196. repo := ctx.Repo.Repository
  197. tpl := tplCloudBrainNew
  198. if jobType == string(models.JobTypeTrain) {
  199. tpl = tplCloudBrainTrainJobNew
  200. }
  201. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
  202. if err == nil {
  203. if len(tasks) != 0 {
  204. log.Error("the job name did already exist", ctx.Data["MsgID"])
  205. cloudBrainNewDataPrepare(ctx)
  206. ctx.RenderWithErr("the job name did already exist", tpl, &form)
  207. return
  208. }
  209. } else {
  210. if !models.IsErrJobNotExist(err) {
  211. log.Error("system error, %v", err, ctx.Data["MsgID"])
  212. cloudBrainNewDataPrepare(ctx)
  213. ctx.RenderWithErr("system error", tpl, &form)
  214. return
  215. }
  216. }
  217. if !jobNamePattern.MatchString(displayJobName) {
  218. cloudBrainNewDataPrepare(ctx)
  219. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
  220. return
  221. }
  222. if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) {
  223. log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
  224. cloudBrainNewDataPrepare(ctx)
  225. ctx.RenderWithErr("jobtype error", tpl, &form)
  226. return
  227. }
  228. count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
  229. if err != nil {
  230. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  231. cloudBrainNewDataPrepare(ctx)
  232. ctx.RenderWithErr("system error", tpl, &form)
  233. return
  234. } else {
  235. if count >= 1 {
  236. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  237. cloudBrainNewDataPrepare(ctx)
  238. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
  239. return
  240. }
  241. }
  242. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuids)
  243. if err != nil {
  244. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  245. cloudBrainNewDataPrepare(ctx)
  246. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
  247. return
  248. }
  249. command := cloudbrain.GetCloudbrainDebugCommand()
  250. if jobType == string(models.JobTypeTrain) {
  251. tpl = tplCloudBrainTrainJobNew
  252. commandTrain, err := getTrainJobCommand(form)
  253. if err != nil {
  254. log.Error("getTrainJobCommand failed: %v", err)
  255. ctx.RenderWithErr(err.Error(), tpl, &form)
  256. return
  257. }
  258. command = commandTrain
  259. }
  260. errStr := checkCloudBrainSpecialPool(ctx, jobType, gpuQueue, resourceSpecId)
  261. if errStr != "" {
  262. cloudBrainNewDataPrepare(ctx)
  263. ctx.RenderWithErr(errStr, tpl, &form)
  264. return
  265. }
  266. if branchName == "" {
  267. branchName = cloudbrain.DefaultBranchName
  268. }
  269. downloadCode(repo, codePath, branchName)
  270. uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
  271. modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
  272. mkModelPath(modelPath)
  273. uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")
  274. commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)
  275. req := cloudbrain.GenerateCloudBrainTaskReq{
  276. Ctx: ctx,
  277. DisplayJobName: displayJobName,
  278. JobName: jobName,
  279. Image: image,
  280. Command: command,
  281. Uuids: uuids,
  282. DatasetNames: datasetNames,
  283. DatasetInfos: datasetInfos,
  284. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  285. ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
  286. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  287. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  288. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  289. JobType: jobType,
  290. GpuQueue: gpuQueue,
  291. Description: form.Description,
  292. BranchName: branchName,
  293. BootFile: form.BootFile,
  294. Params: form.Params,
  295. CommitID: commitID,
  296. BenchmarkTypeID: 0,
  297. BenchmarkChildTypeID: 0,
  298. ResourceSpecId: resourceSpecId,
  299. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  300. }
  301. err = cloudbrain.GenerateTask(req)
  302. if err != nil {
  303. cloudBrainNewDataPrepare(ctx)
  304. ctx.RenderWithErr(err.Error(), tpl, &form)
  305. return
  306. }
  307. if jobType == string(models.JobTypeTrain) {
  308. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=all")
  309. } else {
  310. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all")
  311. }
  312. }
  313. func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBrainInferencForm) {
  314. ctx.Data["PageIsCloudBrain"] = true
  315. displayJobName := form.DisplayJobName
  316. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  317. image := strings.TrimSpace(form.Image)
  318. uuid := form.Attachment
  319. jobType := string(models.JobTypeInference)
  320. gpuQueue := form.GpuType
  321. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  322. resourceSpecId := form.ResourceSpecId
  323. branchName := form.BranchName
  324. labelName := form.LabelName
  325. repo := ctx.Repo.Repository
  326. ckptUrl := setting.Attachment.Minio.RealPath + form.TrainUrl + form.CkptName
  327. log.Info("ckpt url:" + ckptUrl)
  328. tpl := tplCloudBrainInferenceJobNew
  329. command, err := getInferenceJobCommand(form)
  330. if err != nil {
  331. log.Error("getTrainJobCommand failed: %v", err)
  332. ctx.RenderWithErr(err.Error(), tpl, &form)
  333. return
  334. }
  335. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
  336. if err == nil {
  337. if len(tasks) != 0 {
  338. log.Error("the job name did already exist", ctx.Data["MsgID"])
  339. cloudBrainNewDataPrepare(ctx)
  340. ctx.RenderWithErr("the job name did already exist", tpl, &form)
  341. return
  342. }
  343. } else {
  344. if !models.IsErrJobNotExist(err) {
  345. log.Error("system error, %v", err, ctx.Data["MsgID"])
  346. cloudBrainNewDataPrepare(ctx)
  347. ctx.RenderWithErr("system error", tpl, &form)
  348. return
  349. }
  350. }
  351. if !jobNamePattern.MatchString(displayJobName) {
  352. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
  353. return
  354. }
  355. count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
  356. if err != nil {
  357. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  358. cloudBrainNewDataPrepare(ctx)
  359. ctx.RenderWithErr("system error", tpl, &form)
  360. return
  361. } else {
  362. if count >= 1 {
  363. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  364. cloudBrainNewDataPrepare(ctx)
  365. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
  366. return
  367. }
  368. }
  369. if branchName == "" {
  370. branchName = cloudbrain.DefaultBranchName
  371. }
  372. downloadCode(repo, codePath, branchName)
  373. uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
  374. resultPath := setting.JobPath + jobName + cloudbrain.ResultPath + "/"
  375. mkResultPath(resultPath)
  376. uploadCodeToMinio(resultPath, jobName, cloudbrain.ResultPath+"/")
  377. commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)
  378. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
  379. if err != nil {
  380. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  381. cloudBrainNewDataPrepare(ctx)
  382. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
  383. return
  384. }
  385. req := cloudbrain.GenerateCloudBrainTaskReq{
  386. Ctx: ctx,
  387. DisplayJobName: displayJobName,
  388. JobName: jobName,
  389. Image: image,
  390. Command: command,
  391. Uuids: uuid,
  392. DatasetNames: datasetNames,
  393. DatasetInfos: datasetInfos,
  394. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  395. ModelPath: setting.Attachment.Minio.RealPath + form.TrainUrl,
  396. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  397. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  398. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  399. JobType: jobType,
  400. GpuQueue: gpuQueue,
  401. Description: form.Description,
  402. BranchName: branchName,
  403. BootFile: form.BootFile,
  404. Params: form.Params,
  405. CommitID: commitID,
  406. ResourceSpecId: resourceSpecId,
  407. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  408. ModelName: form.ModelName,
  409. ModelVersion: form.ModelVersion,
  410. CkptName: form.CkptName,
  411. TrainUrl: form.TrainUrl,
  412. LabelName: labelName,
  413. }
  414. err = cloudbrain.GenerateTask(req)
  415. if err != nil {
  416. cloudBrainNewDataPrepare(ctx)
  417. ctx.RenderWithErr(err.Error(), tpl, &form)
  418. return
  419. }
  420. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/inference-job")
  421. }
  422. /**
  423. 检查用户传输的参数是否符合专属资源池
  424. */
  425. func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string {
  426. if cloudbrain.SpecialPools != nil {
  427. var isInPoolOrg = false
  428. var matchSpecialPool = false
  429. for _, specialPool := range cloudbrain.SpecialPools.Pools {
  430. if cloudbrain.IsElementExist(specialPool.JobType, jobType) && cloudbrain.IsQueueInSpecialtPool(specialPool.Pool, queue) {
  431. if cloudbrain.IsResourceSpecInSpecialPool(specialPool.ResourceSpec, resourceSpecId) {
  432. matchSpecialPool = true
  433. org, _ := models.GetOrgByName(specialPool.Org)
  434. if org != nil {
  435. isInPoolOrg, _ = models.IsOrganizationMember(org.ID, ctx.User.ID)
  436. if isInPoolOrg {
  437. break //传入参数,和专属资源池匹配上了,检查通过
  438. }
  439. }
  440. }
  441. }
  442. }
  443. //资源池有匹配上,但是用户不在相应的组织中,返回错误信息。界面已经过滤了选择,界面操作不会到这个逻辑
  444. if matchSpecialPool && !isInPoolOrg {
  445. return ctx.Tr("repo.grampus.no_operate_right")
  446. }
  447. }
  448. //没有匹配到资源池或者没有设置专属资源池,检查通过; 获取和资源池完全匹配检查通过
  449. return ""
  450. }
  451. func CloudBrainRestart(ctx *context.Context) {
  452. var ID = ctx.Params(":id")
  453. var resultCode = "0"
  454. var errorMsg = ""
  455. var status = string(models.JobWaiting)
  456. task := ctx.Cloudbrain
  457. for {
  458. if task.Status != string(models.JobStopped) && task.Status != string(models.JobSucceeded) && task.Status != string(models.JobFailed) {
  459. log.Error("the job(%s) is not stopped", task.JobName, ctx.Data["MsgID"])
  460. resultCode = "-1"
  461. errorMsg = "the job is not stopped"
  462. break
  463. }
  464. if task.Image == "" || task.GpuQueue == "" || task.Type != models.TypeCloudBrainOne {
  465. log.Error("the job(%s) version is too old", task.JobName, ctx.Data["MsgID"])
  466. resultCode = "-1"
  467. errorMsg = "the job's version is too old and can not be restarted"
  468. break
  469. }
  470. if !ctx.IsSigned || (ctx.User.ID != task.UserID && !ctx.IsUserSiteAdmin()) {
  471. log.Error("the user has no right ro restart the job", task.JobName, ctx.Data["MsgID"])
  472. resultCode = "-1"
  473. errorMsg = "you have no right to restart the job"
  474. break
  475. }
  476. var hasSameResource bool
  477. if gpuInfos == nil {
  478. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  479. }
  480. for _, resourceType := range gpuInfos.GpuInfo {
  481. if resourceType.Queue == task.GpuQueue {
  482. hasSameResource = true
  483. break
  484. }
  485. }
  486. if !hasSameResource && cloudbrain.SpecialPools != nil {
  487. for _, specialPool := range cloudbrain.SpecialPools.Pools {
  488. cloudbrain.IsElementExist(specialPool.JobType, string(models.JobTypeDebug))
  489. for _, pool := range specialPool.Pool {
  490. if pool.Queue == task.GpuQueue {
  491. hasSameResource = true
  492. }
  493. }
  494. }
  495. }
  496. if !hasSameResource {
  497. log.Error("has no same resource, can not restart", ctx.Data["MsgID"])
  498. resultCode = "-1"
  499. errorMsg = "the job's version is too old and can not be restarted"
  500. break
  501. }
  502. count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug))
  503. if err != nil {
  504. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  505. resultCode = "-1"
  506. errorMsg = "system error"
  507. break
  508. } else {
  509. if count >= 1 {
  510. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  511. resultCode = "-1"
  512. errorMsg = ctx.Tr("repo.cloudbrain.morethanonejob")
  513. break
  514. }
  515. }
  516. err = cloudbrain.RestartTask(ctx, task, &ID)
  517. if err != nil {
  518. log.Error("RestartTask failed:%v", err.Error(), ctx.Data["MsgID"])
  519. resultCode = "-1"
  520. errorMsg = "system error"
  521. break
  522. }
  523. break
  524. }
  525. ctx.JSON(200, map[string]string{
  526. "result_code": resultCode,
  527. "error_msg": errorMsg,
  528. "status": status,
  529. "id": ID,
  530. })
  531. }
  532. func CloudBrainBenchMarkShow(ctx *context.Context) {
  533. cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark)
  534. }
  535. func CloudBrainShow(ctx *context.Context) {
  536. cloudBrainShow(ctx, tplCloudBrainShow, models.JobTypeDebug)
  537. }
  538. func CloudBrainTrainJobShow(ctx *context.Context) {
  539. cloudBrainShow(ctx, tplCloudBrainTrainJobShow, models.JobTypeTrain)
  540. }
  541. func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.JobType) {
  542. ctx.Data["PageIsCloudBrain"] = true
  543. debugListType := ctx.Query("debugListType")
  544. var task *models.Cloudbrain
  545. var err error
  546. if jobType == models.JobTypeTrain || jobType == models.JobTypeInference {
  547. task, err = models.GetCloudbrainByJobID(ctx.Params(":jobid"))
  548. } else {
  549. task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id"))
  550. }
  551. if err != nil {
  552. log.Info("error:" + err.Error())
  553. ctx.Data["error"] = err.Error()
  554. return
  555. }
  556. result, err := cloudbrain.GetJob(task.JobID)
  557. if err != nil {
  558. log.Info("error:" + err.Error())
  559. ctx.Data["error"] = err.Error()
  560. return
  561. }
  562. if task.JobType == string(models.JobTypeTrain) {
  563. if cloudbrain.TrainResourceSpecs == nil {
  564. json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs)
  565. }
  566. hasSpec := false
  567. for _, tmp := range cloudbrain.TrainResourceSpecs.ResourceSpec {
  568. if tmp.Id == task.ResourceSpecId {
  569. hasSpec = true
  570. ctx.Data["GpuNum"] = tmp.GpuNum
  571. ctx.Data["CpuNum"] = tmp.CpuNum
  572. ctx.Data["MemMiB"] = tmp.MemMiB
  573. ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
  574. }
  575. }
  576. if !hasSpec && cloudbrain.SpecialPools != nil {
  577. for _, specialPool := range cloudbrain.SpecialPools.Pools {
  578. if specialPool.ResourceSpec != nil {
  579. for _, spec := range specialPool.ResourceSpec {
  580. if task.ResourceSpecId == spec.Id {
  581. ctx.Data["GpuNum"] = spec.GpuNum
  582. ctx.Data["CpuNum"] = spec.CpuNum
  583. ctx.Data["MemMiB"] = spec.MemMiB
  584. ctx.Data["ShareMemMiB"] = spec.ShareMemMiB
  585. break
  586. }
  587. }
  588. }
  589. }
  590. }
  591. } else if task.JobType == string(models.JobTypeInference) {
  592. if cloudbrain.InferenceResourceSpecs == nil {
  593. json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs)
  594. }
  595. for _, tmp := range cloudbrain.InferenceResourceSpecs.ResourceSpec {
  596. if tmp.Id == task.ResourceSpecId {
  597. ctx.Data["GpuNum"] = tmp.GpuNum
  598. ctx.Data["CpuNum"] = tmp.CpuNum
  599. ctx.Data["MemMiB"] = tmp.MemMiB
  600. ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
  601. }
  602. }
  603. } else {
  604. if cloudbrain.ResourceSpecs == nil {
  605. json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
  606. }
  607. for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec {
  608. if tmp.Id == task.ResourceSpecId {
  609. ctx.Data["GpuNum"] = tmp.GpuNum
  610. ctx.Data["CpuNum"] = tmp.CpuNum
  611. ctx.Data["MemMiB"] = tmp.MemMiB
  612. ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
  613. }
  614. }
  615. }
  616. if result != nil {
  617. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  618. jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
  619. spec := "GPU数:" + strconv.Itoa(jobRes.Resource.NvidiaComGpu) + ",CPU数:" + strconv.Itoa(jobRes.Resource.CPU) + ",内存(MB):" + jobRes.Resource.Memory
  620. ctx.Data["resource_spec"] = spec
  621. if task.JobType == string(models.JobTypeTrain) {
  622. if trainGpuInfos == nil {
  623. json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos)
  624. }
  625. for _, resourceType := range trainGpuInfos.GpuInfo {
  626. if resourceType.Queue == jobRes.Config.GpuType {
  627. ctx.Data["resource_type"] = resourceType.Value
  628. }
  629. }
  630. for _, specialPool := range cloudbrain.SpecialPools.Pools {
  631. for _, resourceType := range specialPool.Pool {
  632. if resourceType.Queue == jobRes.Config.GpuType {
  633. ctx.Data["resource_type"] = resourceType.Value
  634. }
  635. }
  636. }
  637. } else if task.JobType == string(models.JobTypeInference) {
  638. if inferenceGpuInfos == nil {
  639. json.Unmarshal([]byte(setting.InferenceGpuTypes), &inferenceGpuInfos)
  640. }
  641. for _, resourceType := range inferenceGpuInfos.GpuInfo {
  642. if resourceType.Queue == jobRes.Config.GpuType {
  643. ctx.Data["resource_type"] = resourceType.Value
  644. }
  645. }
  646. } else if cloudbrain.IsBenchmarkJob(task.JobType) {
  647. if benchmarkGpuInfos == nil {
  648. json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos)
  649. }
  650. for _, resourceType := range benchmarkGpuInfos.GpuInfo {
  651. if resourceType.Queue == jobRes.Config.GpuType {
  652. ctx.Data["resource_type"] = resourceType.Value
  653. }
  654. }
  655. } else {
  656. if gpuInfos == nil {
  657. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  658. }
  659. for _, resourceType := range gpuInfos.GpuInfo {
  660. if resourceType.Queue == jobRes.Config.GpuType {
  661. ctx.Data["resource_type"] = resourceType.Value
  662. }
  663. }
  664. }
  665. taskRoles := jobRes.TaskRoles
  666. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  667. ctx.Data["taskRes"] = taskRes
  668. ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics
  669. task.Status = taskRes.TaskStatuses[0].State
  670. task.ContainerID = taskRes.TaskStatuses[0].ContainerID
  671. task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  672. models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)
  673. if task.DeletedAt.IsZero() { //normal record
  674. err = models.UpdateJob(task)
  675. if err != nil {
  676. ctx.Data["error"] = err.Error()
  677. return
  678. }
  679. } else { //deleted record
  680. }
  681. ctx.Data["result"] = jobRes
  682. } else {
  683. log.Info("error:" + err.Error())
  684. return
  685. }
  686. user, err := models.GetUserByID(task.UserID)
  687. if err == nil {
  688. task.User = user
  689. }
  690. if task.BenchmarkTypeID > 0 {
  691. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  692. if task.BenchmarkTypeID == benchmarkType.Id {
  693. ctx.Data["BenchmarkTypeName"] = benchmarkType.First
  694. task.BenchmarkTypeName = benchmarkType.First
  695. for _, benchmarkChildType := range benchmarkType.Second {
  696. if task.BenchmarkChildTypeID == benchmarkChildType.Id {
  697. ctx.Data["BenchmarkChildTypeName"] = benchmarkChildType.Value
  698. break
  699. }
  700. }
  701. break
  702. }
  703. }
  704. }
  705. if task.JobType == string(models.JobTypeBenchmark) {
  706. task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
  707. } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
  708. task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
  709. task.BenchmarkTypeName = task.JobType
  710. ctx.Data["BenchmarkTypeName"] = task.JobType
  711. if task.JobType == string(models.JobTypeBrainScore) {
  712. ctx.Data["BenchmarkChildTypeName"] = getBrainRegion(task.BenchmarkChildTypeID)
  713. }
  714. }
  715. if task.TrainJobDuration == "" {
  716. if task.Duration == 0 {
  717. var duration int64
  718. if task.Status == string(models.JobWaiting) {
  719. duration = 0
  720. } else if task.Status == string(models.JobRunning) {
  721. duration = time.Now().Unix() - int64(task.CreatedUnix)
  722. } else {
  723. duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix)
  724. }
  725. task.Duration = duration
  726. }
  727. task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
  728. }
  729. ctx.Data["duration"] = task.TrainJobDuration
  730. if len(task.Parameters) > 0 {
  731. var parameters models.Parameters
  732. err := json.Unmarshal([]byte(task.Parameters), &parameters)
  733. if err != nil {
  734. log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err)
  735. task.Parameters = ""
  736. } else {
  737. if len(parameters.Parameter) > 0 {
  738. paramTemp := ""
  739. for _, Parameter := range parameters.Parameter {
  740. param := Parameter.Label + " = " + Parameter.Value + "; "
  741. paramTemp = paramTemp + param
  742. }
  743. task.Parameters = paramTemp[:len(paramTemp)-2]
  744. } else {
  745. task.Parameters = ""
  746. }
  747. }
  748. }
  749. ctx.Data["task"] = task
  750. labelName := strings.Fields(task.LabelName)
  751. ctx.Data["LabelName"] = labelName
  752. ctx.Data["jobName"] = task.JobName
  753. ctx.Data["displayJobName"] = task.DisplayJobName
  754. version_list_task := make([]*models.Cloudbrain, 0)
  755. version_list_task = append(version_list_task, task)
  756. ctx.Data["version_list_task"] = version_list_task
  757. ctx.Data["debugListType"] = debugListType
  758. ctx.Data["code_path"] = cloudbrain.CodeMountPath
  759. ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
  760. ctx.Data["model_path"] = cloudbrain.ModelMountPath
  761. ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
  762. ctx.Data["branchName"] = task.BranchName
  763. ctx.HTML(200, tpName)
  764. }
  765. func CloudBrainDebug(ctx *context.Context) {
  766. task := ctx.Cloudbrain
  767. debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
  768. ctx.Redirect(debugUrl)
  769. }
  770. func CloudBrainCommitImageShow(ctx *context.Context) {
  771. ctx.Data["PageIsCloudBrain"] = true
  772. ctx.Data["Type"] = ctx.Cloudbrain.Type
  773. ctx.HTML(200, tplCloudBrainImageSubmit)
  774. }
  775. func CloudBrainImageEdit(ctx *context.Context) {
  776. ctx.Data["PageIsImageEdit"] = true
  777. ctx.Data["PageFrom"] = ctx.Params(":from")
  778. var ID = ctx.Params(":id")
  779. id, err := strconv.ParseInt(ID, 10, 64)
  780. if err != nil {
  781. log.Error("GetImageByID failed:%v", err.Error())
  782. ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
  783. }
  784. image, err := models.GetImageByID(id)
  785. if err != nil {
  786. log.Error("GetImageByID failed:%v", err.Error())
  787. ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
  788. }
  789. ctx.Data["Image"] = image
  790. ctx.HTML(http.StatusOK, tplCloudBrainImageEdit)
  791. }
  792. func CloudBrainImageEditPost(ctx *context.Context, form auth.EditImageCloudBrainForm) {
  793. if utf8.RuneCountInString(form.Description) > 255 {
  794. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
  795. return
  796. }
  797. validTopics, errMessage := checkTopics(form.Topics)
  798. if errMessage != "" {
  799. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
  800. return
  801. }
  802. image, err := models.GetImageByID(form.ID)
  803. if err != nil {
  804. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
  805. }
  806. image.IsPrivate = form.IsPrivate
  807. image.Description = form.Description
  808. err = models.WithTx(func(ctx models.DBContext) error {
  809. if err := models.UpdateLocalImage(image); err != nil {
  810. return err
  811. }
  812. if err := models.SaveImageTopics(image.ID, validTopics...); err != nil {
  813. return err
  814. }
  815. return nil
  816. })
  817. if err != nil {
  818. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
  819. } else {
  820. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  821. }
  822. }
  823. func CloudBrainImageDelete(ctx *context.Context) {
  824. var ID = ctx.Params(":id")
  825. id, err := strconv.ParseInt(ID, 10, 64)
  826. if err != nil {
  827. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
  828. return
  829. }
  830. err = models.DeleteLocalImage(id)
  831. if err != nil {
  832. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_delete_fail")))
  833. } else {
  834. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  835. }
  836. }
  837. func CloudBrainCommitImageCheck(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
  838. isExist, _ := models.IsImageExistByUser(form.Tag, ctx.User.ID)
  839. if isExist {
  840. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_overwrite")))
  841. } else {
  842. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  843. }
  844. }
  845. func CloudBrainAdminCommitImage(ctx *context.Context, form auth.CommitAdminImageCloudBrainForm) {
  846. if !NamePattern.MatchString(form.Tag) {
  847. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
  848. return
  849. }
  850. if utf8.RuneCountInString(form.Description) > 255 {
  851. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
  852. return
  853. }
  854. validTopics, errMessage := checkTopics(form.Topics)
  855. if errMessage != "" {
  856. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
  857. return
  858. }
  859. err := cloudbrain.CommitAdminImage(models.CommitImageParams{
  860. CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{
  861. ImageDescription: form.Description,
  862. ImageTag: form.Tag,
  863. },
  864. IsPrivate: form.IsPrivate,
  865. CloudBrainType: form.Type,
  866. Topics: validTopics,
  867. UID: ctx.User.ID,
  868. Type: models.GetRecommondType(form.IsRecommend),
  869. Place: form.Place,
  870. })
  871. if err != nil {
  872. log.Error("CommitImagefailed")
  873. if models.IsErrImageTagExist(err) {
  874. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist")))
  875. } else if models.IsErrorImageCommitting(err) {
  876. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing")))
  877. } else {
  878. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail")))
  879. }
  880. return
  881. }
  882. ctx.JSON(200, models.BaseOKMessage)
  883. }
  884. func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
  885. if !NamePattern.MatchString(form.Tag) {
  886. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
  887. return
  888. }
  889. if utf8.RuneCountInString(form.Description) > 255 {
  890. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
  891. return
  892. }
  893. validTopics, errMessage := checkTopics(form.Topics)
  894. if errMessage != "" {
  895. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
  896. return
  897. }
  898. err := cloudbrain.CommitImage(ctx.Cloudbrain.JobID, models.CommitImageParams{
  899. CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{
  900. Ip: ctx.Cloudbrain.ContainerIp,
  901. TaskContainerId: ctx.Cloudbrain.ContainerID,
  902. ImageDescription: form.Description,
  903. ImageTag: form.Tag,
  904. },
  905. IsPrivate: form.IsPrivate,
  906. CloudBrainType: form.Type,
  907. Topics: validTopics,
  908. UID: ctx.User.ID,
  909. })
  910. if err != nil {
  911. log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"])
  912. if models.IsErrImageTagExist(err) {
  913. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist")))
  914. } else if models.IsErrorImageCommitting(err) {
  915. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing")))
  916. } else {
  917. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail")))
  918. }
  919. return
  920. }
  921. ctx.JSON(200, models.BaseOKMessage)
  922. }
  923. func checkTopics(Topics string) ([]string, string) {
  924. var topics = make([]string, 0)
  925. var topicsStr = strings.TrimSpace(Topics)
  926. if len(topicsStr) > 0 {
  927. topics = strings.Split(topicsStr, ",")
  928. }
  929. validTopics, invalidTopics := models.SanitizeAndValidateImageTopics(topics)
  930. if len(validTopics) > 25 {
  931. return nil, "repo.topic.count_prompt"
  932. }
  933. if len(invalidTopics) > 0 {
  934. return nil, "repo.imagetopic.format_prompt"
  935. }
  936. return validTopics, ""
  937. }
  938. func CloudBrainStop(ctx *context.Context) {
  939. var ID = ctx.Params(":id")
  940. var resultCode = "0"
  941. var errorMsg = ""
  942. var status = ""
  943. task := ctx.Cloudbrain
  944. for {
  945. if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded) {
  946. log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
  947. resultCode = "-1"
  948. errorMsg = "cloudbrain.Already_stopped"
  949. break
  950. }
  951. err := cloudbrain.StopJob(task.JobID)
  952. if err != nil {
  953. log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
  954. resultCode = "-1"
  955. errorMsg = "cloudbrain.Stopped_failed"
  956. break
  957. }
  958. task.Status = string(models.JobStopped)
  959. if task.EndTime == 0 {
  960. task.EndTime = timeutil.TimeStampNow()
  961. }
  962. task.ComputeAndSetDuration()
  963. err = models.UpdateJob(task)
  964. if err != nil {
  965. log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
  966. resultCode = "-1"
  967. errorMsg = "cloudbrain.Stopped_success_update_status_fail"
  968. break
  969. }
  970. status = task.Status
  971. break
  972. }
  973. ctx.JSON(200, map[string]interface{}{
  974. "result_code": resultCode,
  975. "error_msg": ctx.Tr(errorMsg),
  976. "status": status,
  977. "id": ID,
  978. "StatusOK": 0,
  979. })
  980. }
  981. func StopJobsByUserID(userID int64) {
  982. cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
  983. if err != nil {
  984. log.Warn("Failed to get cloudBrain info", err)
  985. return
  986. }
  987. StopJobs(cloudBrains)
  988. }
  989. func StopJobsByRepoID(repoID int64) {
  990. cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
  991. if err != nil {
  992. log.Warn("Failed to get cloudBrain info", err)
  993. return
  994. }
  995. StopJobs(cloudBrains)
  996. }
  997. /**
  998. */
  999. func StopJobs(cloudBrains []*models.Cloudbrain) {
  1000. for _, taskInfo := range cloudBrains {
  1001. if taskInfo.Type == models.TypeCloudBrainOne {
  1002. err := retry(3, time.Second*30, func() error {
  1003. return cloudbrain.StopJob(taskInfo.JobID)
  1004. })
  1005. logErrorAndUpdateJobStatus(err, taskInfo)
  1006. } else {
  1007. if taskInfo.JobType == string(models.JobTypeTrain) {
  1008. err := retry(3, time.Second*30, func() error {
  1009. _, err := modelarts.StopTrainJob(taskInfo.JobID, strconv.FormatInt(taskInfo.VersionID, 10))
  1010. return err
  1011. })
  1012. logErrorAndUpdateJobStatus(err, taskInfo)
  1013. } else {
  1014. param := models.NotebookAction{
  1015. Action: models.ActionStop,
  1016. }
  1017. err := retry(3, time.Second*30, func() error {
  1018. _, err := modelarts.ManageNotebook2(taskInfo.JobID, param)
  1019. return err
  1020. })
  1021. logErrorAndUpdateJobStatus(err, taskInfo)
  1022. }
  1023. }
  1024. }
  1025. }
  1026. func retry(attempts int, sleep time.Duration, f func() error) (err error) {
  1027. for i := 0; i < attempts; i++ {
  1028. if i > 0 {
  1029. log.Warn("retrying after error:", err)
  1030. time.Sleep(sleep)
  1031. }
  1032. err = f()
  1033. if err == nil {
  1034. return nil
  1035. }
  1036. }
  1037. return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
  1038. }
  1039. func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
  1040. if err != nil {
  1041. log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
  1042. } else {
  1043. taskInfo.Status = string(models.JobStopped)
  1044. if taskInfo.EndTime == 0 {
  1045. taskInfo.EndTime = timeutil.TimeStampNow()
  1046. }
  1047. taskInfo.ComputeAndSetDuration()
  1048. err = models.UpdateJob(taskInfo)
  1049. if err != nil {
  1050. log.Warn("UpdateJob failed", err)
  1051. }
  1052. }
  1053. }
  1054. func CloudBrainDel(ctx *context.Context) {
  1055. var listType = ctx.Query("debugListType")
  1056. if err := deleteCloudbrainJob(ctx); err != nil {
  1057. log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
  1058. ctx.ServerError(err.Error(), err)
  1059. return
  1060. }
  1061. var isAdminPage = ctx.Query("isadminpage")
  1062. var isHomePage = ctx.Query("ishomepage")
  1063. if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
  1064. ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
  1065. } else if isHomePage == "true" {
  1066. ctx.Redirect(setting.AppSubURL + "/cloudbrains")
  1067. } else {
  1068. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=" + listType)
  1069. }
  1070. }
  1071. func deleteCloudbrainJob(ctx *context.Context) error {
  1072. task := ctx.Cloudbrain
  1073. if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) {
  1074. log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
  1075. return errors.New("the job has not been stopped")
  1076. }
  1077. err := models.DeleteJob(task)
  1078. if err != nil {
  1079. log.Error("DeleteJob failed: %v", err, ctx.Data["msgID"])
  1080. return err
  1081. }
  1082. deleteJobStorage(task.JobName, models.TypeCloudBrainOne)
  1083. return nil
  1084. }
  1085. func CloudBrainShowModels(ctx *context.Context) {
  1086. ctx.Data["PageIsCloudBrain"] = true
  1087. ID := ctx.Params(":id")
  1088. parentDir := ctx.Query("parentDir")
  1089. dirArray := strings.Split(parentDir, "/")
  1090. task, err := models.GetCloudbrainByID(ID)
  1091. if err != nil {
  1092. log.Error("no such job!", ctx.Data["msgID"])
  1093. ctx.ServerError("no such job:", err)
  1094. return
  1095. }
  1096. //get dirs
  1097. dirs, err := GetModelDirs(task.JobName, parentDir)
  1098. if err != nil {
  1099. log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  1100. ctx.ServerError("GetModelDirs failed:", err)
  1101. return
  1102. }
  1103. var fileInfos []storage.FileInfo
  1104. err = json.Unmarshal([]byte(dirs), &fileInfos)
  1105. if err != nil {
  1106. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  1107. ctx.ServerError("json.Unmarshal failed:", err)
  1108. return
  1109. }
  1110. for i, fileInfo := range fileInfos {
  1111. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  1112. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  1113. }
  1114. sort.Slice(fileInfos, func(i, j int) bool {
  1115. return fileInfos[i].ModTime > fileInfos[j].ModTime
  1116. })
  1117. ctx.Data["Path"] = dirArray
  1118. ctx.Data["Dirs"] = fileInfos
  1119. ctx.Data["task"] = task
  1120. ctx.Data["ID"] = ID
  1121. ctx.HTML(200, tplCloudBrainShowModels)
  1122. }
  1123. func GetPublicImages(ctx *context.Context) {
  1124. uid := getUID(ctx)
  1125. opts := models.SearchImageOptions{
  1126. IncludePublicOnly: true,
  1127. UID: uid,
  1128. Keyword: ctx.Query("q"),
  1129. Topics: ctx.Query("topic"),
  1130. IncludeOfficialOnly: ctx.QueryBool("recommend"),
  1131. SearchOrderBy: "type desc, num_stars desc,id desc",
  1132. Status: models.IMAGE_STATUS_SUCCESS,
  1133. CloudbrainType: ctx.QueryInt("cloudbrainType"),
  1134. }
  1135. getImages(ctx, &opts)
  1136. }
  1137. func GetCustomImages(ctx *context.Context) {
  1138. uid := getUID(ctx)
  1139. opts := models.SearchImageOptions{
  1140. UID: uid,
  1141. IncludeOwnerOnly: true,
  1142. Keyword: ctx.Query("q"),
  1143. Topics: ctx.Query("topic"),
  1144. Status: -1,
  1145. SearchOrderBy: "id desc",
  1146. }
  1147. getImages(ctx, &opts)
  1148. }
  1149. func GetStarImages(ctx *context.Context) {
  1150. uid := getUID(ctx)
  1151. opts := models.SearchImageOptions{
  1152. UID: uid,
  1153. IncludeStarByMe: true,
  1154. Keyword: ctx.Query("q"),
  1155. Topics: ctx.Query("topic"),
  1156. Status: models.IMAGE_STATUS_SUCCESS,
  1157. SearchOrderBy: "id desc",
  1158. }
  1159. getImages(ctx, &opts)
  1160. }
  1161. func getUID(ctx *context.Context) int64 {
  1162. var uid int64 = -1
  1163. if ctx.IsSigned {
  1164. uid = ctx.User.ID
  1165. }
  1166. return uid
  1167. }
  1168. func GetAllImages(ctx *context.Context) {
  1169. uid := getUID(ctx)
  1170. opts := models.SearchImageOptions{
  1171. UID: uid,
  1172. Keyword: ctx.Query("q"),
  1173. Topics: ctx.Query("topic"),
  1174. IncludeOfficialOnly: ctx.QueryBool("recommend"),
  1175. SearchOrderBy: "id desc",
  1176. Status: -1,
  1177. }
  1178. if ctx.Query("private") != "" {
  1179. if ctx.QueryBool("private") {
  1180. opts.IncludePrivateOnly = true
  1181. } else {
  1182. opts.IncludePublicOnly = true
  1183. }
  1184. }
  1185. getImages(ctx, &opts)
  1186. }
  1187. func getImages(ctx *context.Context, opts *models.SearchImageOptions) {
  1188. page := ctx.QueryInt("page")
  1189. if page <= 0 {
  1190. page = 1
  1191. }
  1192. pageSize := ctx.QueryInt("pageSize")
  1193. if pageSize <= 0 {
  1194. pageSize = 15
  1195. }
  1196. opts.ListOptions = models.ListOptions{
  1197. Page: page,
  1198. PageSize: pageSize,
  1199. }
  1200. imageList, total, err := models.SearchImage(opts)
  1201. if err != nil {
  1202. log.Error("Can not get images:%v", err)
  1203. ctx.JSON(http.StatusOK, models.ImagesPageResult{
  1204. Count: 0,
  1205. Images: []*models.Image{},
  1206. })
  1207. } else {
  1208. ctx.JSON(http.StatusOK, models.ImagesPageResult{
  1209. Count: total,
  1210. Images: imageList,
  1211. })
  1212. }
  1213. }
  1214. func GetModelDirs(jobName string, parentDir string) (string, error) {
  1215. var req string
  1216. modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/")
  1217. if parentDir == "" {
  1218. req = "baseDir=" + modelActualPath
  1219. } else {
  1220. req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
  1221. }
  1222. return getDirs(req)
  1223. }
  1224. func GetResultDirs(jobName string, parentDir string) (string, error) {
  1225. var req string
  1226. modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/")
  1227. if parentDir == "" {
  1228. req = "baseDir=" + modelActualPath
  1229. } else {
  1230. req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
  1231. }
  1232. return getDirs(req)
  1233. }
  1234. func CloudBrainDownloadModel(ctx *context.Context) {
  1235. parentDir := ctx.Query("parentDir")
  1236. fileName := ctx.Query("fileName")
  1237. jobName := ctx.Query("jobName")
  1238. filePath := "jobs/" + jobName + "/model/" + parentDir
  1239. url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
  1240. if err != nil {
  1241. log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
  1242. ctx.ServerError("PresignedGetURL", err)
  1243. return
  1244. }
  1245. ctx.Resp.Header().Set("Cache-Control", "max-age=0")
  1246. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  1247. }
  1248. func CloudBrainDownloadInferenceResult(ctx *context.Context) {
  1249. parentDir := ctx.Query("parentDir")
  1250. fileName := ctx.Query("fileName")
  1251. jobName := ctx.Query("jobName")
  1252. filePath := "jobs/" + jobName + "/result/" + parentDir
  1253. url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
  1254. if err != nil {
  1255. log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
  1256. ctx.ServerError("PresignedGetURL", err)
  1257. return
  1258. }
  1259. ctx.Resp.Header().Set("Cache-Control", "max-age=0")
  1260. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  1261. }
  1262. func GetRate(ctx *context.Context) {
  1263. isObjectDetcionAll := ctx.QueryBool("isObjectDetcionAll")
  1264. if isObjectDetcionAll {
  1265. ctx.Redirect(setting.BenchmarkServerHost + "?username=admin")
  1266. return
  1267. }
  1268. var ID = ctx.Params(":id")
  1269. job, err := models.GetCloudbrainByID(ID)
  1270. if err != nil {
  1271. ctx.ServerError("GetCloudbrainByJobID failed", err)
  1272. return
  1273. }
  1274. if job.JobType == string(models.JobTypeBenchmark) {
  1275. log.Info("url=" + setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
  1276. ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
  1277. } else if job.JobType == string(models.JobTypeSnn4imagenet) {
  1278. ctx.Redirect(setting.Snn4imagenetServerHost)
  1279. } else if job.JobType == string(models.JobTypeBrainScore) {
  1280. ctx.Redirect(setting.BrainScoreServerHost)
  1281. } else {
  1282. log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
  1283. }
  1284. }
  1285. func downloadCode(repo *models.Repository, codePath, branchName string) error {
  1286. //add "file:///" prefix to make the depth valid
  1287. if err := git.Clone(CLONE_FILE_PREFIX+repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName, Depth: 1}); err != nil {
  1288. log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
  1289. return err
  1290. }
  1291. configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
  1292. if err != nil {
  1293. log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
  1294. return err
  1295. }
  1296. defer configFile.Close()
  1297. pos := int64(0)
  1298. reader := bufio.NewReader(configFile)
  1299. for {
  1300. line, err := reader.ReadString('\n')
  1301. if err != nil {
  1302. if err == io.EOF {
  1303. log.Error("not find the remote-url")
  1304. return nil
  1305. } else {
  1306. log.Error("read error: %v", err)
  1307. return err
  1308. }
  1309. }
  1310. if strings.Contains(line, "url") && strings.Contains(line, ".git") {
  1311. originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
  1312. if len(line) > len(originUrl) {
  1313. originUrl += strings.Repeat(" ", len(line)-len(originUrl))
  1314. }
  1315. bytes := []byte(originUrl)
  1316. _, err := configFile.WriteAt(bytes, pos)
  1317. if err != nil {
  1318. log.Error("WriteAt failed:%v", err)
  1319. return err
  1320. }
  1321. break
  1322. }
  1323. pos += int64(len(line))
  1324. }
  1325. return nil
  1326. }
  1327. func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType, userName string) error {
  1328. err := os.MkdirAll(codePath, os.ModePerm)
  1329. if err != nil {
  1330. log.Error("mkdir codePath failed", err.Error())
  1331. return err
  1332. }
  1333. repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName)
  1334. if err != nil {
  1335. log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error())
  1336. return err
  1337. }
  1338. if err := git.Clone(CLONE_FILE_PREFIX+repoExt.RepoPath(), codePath, git.CloneRepoOptions{Depth: 1}); err != nil {
  1339. log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err)
  1340. return err
  1341. }
  1342. fileName := codePath + cloudbrain.TaskInfoName
  1343. f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
  1344. if err != nil {
  1345. log.Error("OpenFile failed", err.Error())
  1346. return err
  1347. }
  1348. defer f.Close()
  1349. data, err := json.Marshal(models.TaskInfo{
  1350. Username: userName,
  1351. TaskName: taskName,
  1352. CodeName: repo.Name,
  1353. BenchmarkCategory: strings.Split(benchmarkCategory, ","),
  1354. CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
  1355. GpuType: gpuType,
  1356. })
  1357. if err != nil {
  1358. log.Error("json.Marshal failed", err.Error())
  1359. return err
  1360. }
  1361. _, err = f.Write(data)
  1362. if err != nil {
  1363. log.Error("WriteString failed", err.Error())
  1364. return err
  1365. }
  1366. return nil
  1367. }
  1368. func uploadCodeToMinio(codePath, jobName, parentDir string) error {
  1369. files, err := readDir(codePath)
  1370. if err != nil {
  1371. log.Error("readDir(%s) failed: %s", codePath, err.Error())
  1372. return err
  1373. }
  1374. for _, file := range files {
  1375. if file.IsDir() {
  1376. if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil {
  1377. log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error())
  1378. return err
  1379. }
  1380. } else {
  1381. destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name()
  1382. sourceFile := codePath + file.Name()
  1383. err = storage.Attachments.UploadObject(destObject, sourceFile)
  1384. if err != nil {
  1385. log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error())
  1386. return err
  1387. }
  1388. }
  1389. }
  1390. return nil
  1391. }
  1392. func mkModelPath(modelPath string) error {
  1393. return mkPathAndReadMeFile(modelPath, "You can put the model file into this directory and download it by the web page.")
  1394. }
  1395. func mkResultPath(resultPath string) error {
  1396. return mkPathAndReadMeFile(resultPath, "You can put the result file into this directory and download it by the web page.")
  1397. }
  1398. func mkPathAndReadMeFile(path string, text string) error {
  1399. err := os.MkdirAll(path, os.ModePerm)
  1400. if err != nil {
  1401. log.Error("MkdirAll(%s) failed:%v", path, err)
  1402. return err
  1403. }
  1404. fileName := path + "README"
  1405. f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
  1406. if err != nil {
  1407. log.Error("OpenFile failed", err.Error())
  1408. return err
  1409. }
  1410. defer f.Close()
  1411. _, err = f.WriteString(text)
  1412. if err != nil {
  1413. log.Error("WriteString failed", err.Error())
  1414. return err
  1415. }
  1416. return nil
  1417. }
  1418. func deleteJobStorage(jobName string, cloudbrainType int) error {
  1419. //delete local
  1420. localJobPath := setting.JobPath + jobName
  1421. err := os.RemoveAll(localJobPath)
  1422. if err != nil {
  1423. log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
  1424. }
  1425. //delete oss
  1426. if cloudbrainType == models.TypeCloudBrainOne {
  1427. dirPath := setting.CBCodePathPrefix + jobName + "/"
  1428. err = storage.Attachments.DeleteDir(dirPath)
  1429. if err != nil {
  1430. log.Error("DeleteDir(%s) failed:%v", localJobPath, err)
  1431. }
  1432. } else if cloudbrainType == models.TypeCloudBrainTwo {
  1433. //dirPath := setting.CodePathPrefix + jobName + "/"
  1434. //err = storage.ObsRemoveObject(setting.Bucket, dirPath)
  1435. //if err != nil {
  1436. // log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err)
  1437. //}
  1438. log.Info("no need to delete")
  1439. } else {
  1440. log.Error("cloudbrainType(%d) error", cloudbrainType)
  1441. }
  1442. return nil
  1443. }
  1444. func SyncCloudbrainStatus() {
  1445. cloudBrains, err := models.GetCloudBrainUnStoppedJob()
  1446. if err != nil {
  1447. log.Error("GetCloudBrainUnStoppedJob failed:", err.Error())
  1448. return
  1449. }
  1450. for _, task := range cloudBrains {
  1451. if task.Type == models.TypeCloudBrainOne {
  1452. result, err := cloudbrain.GetJob(task.JobID)
  1453. if err != nil {
  1454. log.Error("GetJob(%s) failed:%v", task.JobName, err)
  1455. continue
  1456. }
  1457. if result != nil {
  1458. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  1459. taskRoles := jobRes.TaskRoles
  1460. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  1461. task.Status = taskRes.TaskStatuses[0].State
  1462. if task.Status != string(models.JobWaiting) {
  1463. models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)
  1464. err = models.UpdateJob(task)
  1465. if err != nil {
  1466. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1467. }
  1468. var maxDuration int64
  1469. if task.JobType == string(models.JobTypeBenchmark) {
  1470. maxDuration = setting.BenchmarkMaxDuration
  1471. } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
  1472. maxDuration = setting.ModelBenchmarkMaxDuration
  1473. } else {
  1474. maxDuration = setting.MaxDuration
  1475. }
  1476. if task.Duration >= maxDuration && task.JobType != string(models.JobTypeTrain) {
  1477. log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName)
  1478. err = cloudbrain.StopJob(task.JobID)
  1479. if err != nil {
  1480. log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err)
  1481. continue
  1482. }
  1483. task.Status = string(models.JobStopped)
  1484. if task.EndTime == 0 {
  1485. task.EndTime = timeutil.TimeStampNow()
  1486. }
  1487. task.ComputeAndSetDuration()
  1488. err = models.UpdateJob(task)
  1489. if err != nil {
  1490. log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
  1491. continue
  1492. }
  1493. }
  1494. }
  1495. }
  1496. } else if task.Type == models.TypeCloudBrainTwo {
  1497. if task.JobType == string(models.JobTypeDebug) {
  1498. err := modelarts.HandleNotebookInfo(task)
  1499. if err != nil {
  1500. log.Error("HandleNotebookInfo(%s) failed:%v", task.DisplayJobName, err)
  1501. continue
  1502. }
  1503. } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
  1504. err := modelarts.HandleTrainJobInfo(task)
  1505. if err != nil {
  1506. log.Error("HandleTrainJobInfo(%s) failed:%v", task.DisplayJobName, err)
  1507. continue
  1508. }
  1509. } else {
  1510. log.Error("task.JobType(%s) is error:%s", task.DisplayJobName, task.JobType)
  1511. }
  1512. } else if task.Type == models.TypeC2Net {
  1513. result, err := grampus.GetJob(task.JobID)
  1514. if err != nil {
  1515. log.Error("GetTrainJob(%s) failed:%v", task.DisplayJobName, err)
  1516. continue
  1517. }
  1518. if result != nil {
  1519. if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
  1520. task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
  1521. }
  1522. task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
  1523. task.Duration = result.JobInfo.RunSec
  1524. task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
  1525. if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
  1526. task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
  1527. }
  1528. if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
  1529. task.EndTime = task.StartTime.Add(task.Duration)
  1530. }
  1531. task.CorrectCreateUnix()
  1532. err = models.UpdateJob(task)
  1533. if err != nil {
  1534. log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
  1535. continue
  1536. }
  1537. }
  1538. } else {
  1539. log.Error("task.Type(%s) is error:%d", task.DisplayJobName, task.Type)
  1540. }
  1541. }
  1542. return
  1543. }
  1544. func HandleTaskWithNoDuration(ctx *context.Context) {
  1545. mode := ctx.Query("mode")
  1546. log.Info("HandleTaskWithNoDuration start")
  1547. count := 0
  1548. start := time.Now().Unix()
  1549. for {
  1550. var cloudBrains []*models.Cloudbrain
  1551. var err error
  1552. if mode == "1" {
  1553. cloudBrains, err = models.GetStoppedJobWithNoStartTimeEndTime()
  1554. } else {
  1555. cloudBrains, err = models.GetStoppedJobWithNoDurationJob()
  1556. }
  1557. if err != nil {
  1558. log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error())
  1559. break
  1560. }
  1561. if len(cloudBrains) == 0 {
  1562. log.Info("HandleTaskWithNoTrainJobDuration:no task need handle")
  1563. break
  1564. }
  1565. handleNoDurationTask(cloudBrains)
  1566. count += len(cloudBrains)
  1567. if len(cloudBrains) < 100 {
  1568. log.Info("HandleTaskWithNoTrainJobDuration:task less than 100")
  1569. break
  1570. }
  1571. if time.Now().Unix()-start > 600 {
  1572. log.Info("HandleTaskWithNoDuration : time out")
  1573. ctx.JSON(200, fmt.Sprintf("task stop for time out,count=%d", count))
  1574. return
  1575. }
  1576. }
  1577. log.Info("HandleTaskWithNoTrainJobDuration:count=%d", count)
  1578. ctx.JSON(200, fmt.Sprintf("success,count=%d", count))
  1579. }
  1580. func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
  1581. for _, task := range cloudBrains {
  1582. time.Sleep(time.Millisecond * 100)
  1583. log.Info("Handle job ,%+v", task)
  1584. if task.Type == models.TypeCloudBrainOne {
  1585. result, err := cloudbrain.GetJob(task.JobID)
  1586. if err != nil {
  1587. log.Error("GetJob(%s) failed:%v", task.JobName, err)
  1588. updateDefaultDuration(task)
  1589. continue
  1590. }
  1591. if result != nil {
  1592. if result.Msg != "success" {
  1593. updateDefaultDuration(task)
  1594. continue
  1595. }
  1596. jobRes, err := models.ConvertToJobResultPayload(result.Payload)
  1597. if err != nil || len(jobRes.TaskRoles) == 0 {
  1598. updateDefaultDuration(task)
  1599. continue
  1600. }
  1601. taskRoles := jobRes.TaskRoles
  1602. taskRes, err := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  1603. if err != nil || len(taskRes.TaskStatuses) == 0 {
  1604. updateDefaultDuration(task)
  1605. continue
  1606. }
  1607. task.Status = taskRes.TaskStatuses[0].State
  1608. log.Info("task startTime = %v endTime= %v ,jobId=%d", jobRes.JobStatus.StartTime, jobRes.JobStatus.EndTime, task.ID)
  1609. if jobRes.JobStatus.CreatedTime > 0 {
  1610. task.StartTime = timeutil.TimeStamp(jobRes.JobStatus.CreatedTime / 1000)
  1611. if jobRes.JobStatus.CompletedTime > 0 {
  1612. task.EndTime = timeutil.TimeStamp(jobRes.JobStatus.CompletedTime / 1000)
  1613. } else {
  1614. task.EndTime = task.UpdatedUnix
  1615. }
  1616. } else {
  1617. task.StartTime = 0
  1618. task.EndTime = 0
  1619. }
  1620. if task.EndTime < task.StartTime {
  1621. log.Info("endTime[%v] is less than starTime[%v],jobId=%d", task.EndTime, task.StartTime, task.ID)
  1622. st := task.StartTime
  1623. task.StartTime = task.EndTime
  1624. task.EndTime = st
  1625. }
  1626. task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
  1627. task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
  1628. err = models.UpdateJobDurationWithDeleted(task)
  1629. if err != nil {
  1630. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1631. }
  1632. }
  1633. } else if task.Type == models.TypeCloudBrainTwo {
  1634. if task.JobType == string(models.JobTypeDebug) {
  1635. //result, err := modelarts.GetJob(task.JobID)
  1636. result, err := modelarts.GetNotebook2(task.JobID)
  1637. if err != nil {
  1638. log.Error("GetJob(%s) failed:%v", task.JobName, err)
  1639. updateDefaultDuration(task)
  1640. continue
  1641. }
  1642. if result != nil {
  1643. task.Status = result.Status
  1644. startTime := result.Lease.CreateTime
  1645. duration := result.Lease.Duration / 1000
  1646. if startTime > 0 {
  1647. task.StartTime = timeutil.TimeStamp(startTime / 1000)
  1648. task.EndTime = task.StartTime.Add(duration)
  1649. }
  1650. task.CorrectCreateUnix()
  1651. task.ComputeAndSetDuration()
  1652. err = models.UpdateJobDurationWithDeleted(task)
  1653. if err != nil {
  1654. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1655. continue
  1656. }
  1657. }
  1658. } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
  1659. result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
  1660. if err != nil {
  1661. log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
  1662. updateDefaultDuration(task)
  1663. continue
  1664. }
  1665. if result != nil {
  1666. startTime := result.StartTime / 1000
  1667. if startTime > 0 {
  1668. task.StartTime = timeutil.TimeStamp(startTime)
  1669. task.EndTime = task.StartTime.Add(result.Duration / 1000)
  1670. }
  1671. task.ComputeAndSetDuration()
  1672. err = models.UpdateJobDurationWithDeleted(task)
  1673. if err != nil {
  1674. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1675. continue
  1676. }
  1677. }
  1678. } else {
  1679. log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType)
  1680. }
  1681. } else {
  1682. log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
  1683. }
  1684. }
  1685. }
  1686. func updateDefaultDuration(task *models.Cloudbrain) {
  1687. log.Info("updateDefaultDuration: taskId=%d", task.ID)
  1688. task.StartTime = task.CreatedUnix
  1689. task.EndTime = task.UpdatedUnix
  1690. task.ComputeAndSetDuration()
  1691. err := models.UpdateJobDurationWithDeleted(task)
  1692. if err != nil {
  1693. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1694. }
  1695. }
  1696. func CloudBrainBenchmarkIndex(ctx *context.Context) {
  1697. MustEnableCloudbrain(ctx)
  1698. repo := ctx.Repo.Repository
  1699. page := ctx.QueryInt("page")
  1700. if page <= 0 {
  1701. page = 1
  1702. }
  1703. var jobTypes []string
  1704. jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
  1705. ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
  1706. ListOptions: models.ListOptions{
  1707. Page: page,
  1708. PageSize: setting.UI.IssuePagingNum,
  1709. },
  1710. RepoID: repo.ID,
  1711. Type: models.TypeCloudBrainOne,
  1712. JobTypes: jobTypes,
  1713. })
  1714. if err != nil {
  1715. ctx.ServerError("Get debugjob faild:", err)
  1716. return
  1717. }
  1718. for i, task := range ciTasks {
  1719. ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
  1720. ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
  1721. if ciTasks[i].TrainJobDuration == "" {
  1722. if ciTasks[i].Duration == 0 {
  1723. var duration int64
  1724. if task.Status == string(models.JobRunning) {
  1725. duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix)
  1726. } else {
  1727. duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix)
  1728. }
  1729. ciTasks[i].Duration = duration
  1730. }
  1731. ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(ciTasks[i].Duration)
  1732. }
  1733. ciTasks[i].BenchmarkTypeName = ""
  1734. if ciTasks[i].JobType == string(models.JobTypeBenchmark) {
  1735. ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
  1736. } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) {
  1737. ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
  1738. ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType
  1739. if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) {
  1740. ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost
  1741. } else {
  1742. ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost
  1743. }
  1744. }
  1745. if task.BenchmarkTypeID > 0 {
  1746. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  1747. if task.BenchmarkTypeID == benchmarkType.Id {
  1748. ciTasks[i].BenchmarkTypeRankLink = benchmarkType.RankLink
  1749. ciTasks[i].BenchmarkTypeName = benchmarkType.First
  1750. break
  1751. }
  1752. }
  1753. }
  1754. }
  1755. pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
  1756. ctx.Data["Page"] = pager
  1757. ctx.Data["PageIsCloudBrain"] = true
  1758. ctx.Data["Tasks"] = ciTasks
  1759. ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx)
  1760. ctx.Data["RepoIsEmpty"] = repo.IsEmpty
  1761. ctx.HTML(200, tplCloudBrainBenchmarkIndex)
  1762. }
  1763. func GetChildTypes(ctx *context.Context) {
  1764. benchmarkTypeID := ctx.QueryInt("benchmark_type_id")
  1765. re := make(map[string]interface{})
  1766. for {
  1767. var isExist bool
  1768. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  1769. if benchmarkTypeID == benchmarkType.Id {
  1770. isExist = true
  1771. re["child_types"] = benchmarkType.Second
  1772. re["result_code"] = "0"
  1773. break
  1774. }
  1775. }
  1776. if !isExist {
  1777. re["result_code"] = "1"
  1778. log.Error("no such benchmark_type_id", ctx.Data["MsgID"])
  1779. re["errMsg"] = "system error"
  1780. break
  1781. }
  1782. break
  1783. }
  1784. ctx.JSON(200, re)
  1785. }
  1786. func CloudBrainBenchmarkNew(ctx *context.Context) {
  1787. ctx.Data["description"] = ""
  1788. ctx.Data["benchmarkTypeID"] = -1
  1789. ctx.Data["benchmark_child_types_id_hidden"] = -1
  1790. err := cloudBrainNewDataPrepare(ctx)
  1791. if err != nil {
  1792. ctx.ServerError("get new cloudbrain info failed", err)
  1793. return
  1794. }
  1795. ctx.HTML(200, tplCloudBrainBenchmarkNew)
  1796. }
  1797. func getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID int, ctx *context.Context) (*models.BenchmarkDataset, error) {
  1798. var childInfo *models.BenchmarkDataset
  1799. var isExist bool
  1800. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  1801. if benchmarkType.Id == benchmarkTypeID {
  1802. for _, childType := range benchmarkType.Second {
  1803. if childType.Id == benchmarkChildTypeID {
  1804. childInfo = childType
  1805. isExist = true
  1806. break
  1807. }
  1808. }
  1809. break
  1810. }
  1811. }
  1812. if !isExist {
  1813. log.Error("no such benchmark_type_id&benchmark_child_type_id")
  1814. return childInfo, errors.New("no such benchmark_type_id&benchmark_child_type_id")
  1815. }
  1816. return childInfo, nil
  1817. }
  1818. func getBenchmarkGpuQueue(gpuQueue string) (string, error) {
  1819. queue := ""
  1820. if benchmarkGpuInfos == nil {
  1821. if err := json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos); err != nil {
  1822. log.Error("json.Unmarshal BenchmarkGpuTypes(%s) failed:%v", setting.BenchmarkGpuTypes, err)
  1823. return queue, err
  1824. }
  1825. }
  1826. var isExist bool
  1827. for _, gpuInfo := range benchmarkGpuInfos.GpuInfo {
  1828. if gpuQueue == gpuInfo.Queue {
  1829. isExist = true
  1830. queue = gpuQueue
  1831. break
  1832. }
  1833. }
  1834. if !isExist {
  1835. log.Error("no such gpuQueue, %s", gpuQueue)
  1836. return queue, errors.New("no such gpuQueue")
  1837. }
  1838. return queue, nil
  1839. }
  1840. func getBenchmarkResourceSpec(resourceSpecID int) (int, error) {
  1841. var id int
  1842. if benchmarkResourceSpecs == nil {
  1843. if err := json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs); err != nil {
  1844. log.Error("json.Unmarshal BenchmarkResourceSpecs(%s) failed:%v", setting.BenchmarkResourceSpecs, err)
  1845. return id, err
  1846. }
  1847. }
  1848. var isExist bool
  1849. for _, resourceSpec := range benchmarkResourceSpecs.ResourceSpec {
  1850. if resourceSpecID == resourceSpec.Id {
  1851. isExist = true
  1852. id = resourceSpecID
  1853. break
  1854. }
  1855. }
  1856. if !isExist {
  1857. log.Error("no such resourceSpecID, %d", resourceSpecID)
  1858. return id, errors.New("no such resourceSpec")
  1859. }
  1860. return id, nil
  1861. }
  1862. func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  1863. jobType := form.JobType
  1864. if jobType == string(models.JobTypeBenchmark) {
  1865. BenchMarkAlgorithmCreate(ctx, form)
  1866. } else {
  1867. ModelBenchmarkCreate(ctx, form)
  1868. }
  1869. }
  1870. func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  1871. ctx.Data["PageIsCloudBrain"] = true
  1872. displayJobName := form.DisplayJobName
  1873. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  1874. image := strings.TrimSpace(form.Image)
  1875. gpuQueue := form.GpuType
  1876. command := cloudbrain.CommandBenchmark
  1877. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  1878. resourceSpecId := cloudbrain.BenchMarkResourceID
  1879. benchmarkTypeID := form.BenchmarkTypeID
  1880. benchmarkChildTypeID := form.BenchmarkChildTypeID
  1881. ctx.Data["description"] = form.Description
  1882. ctx.Data["benchmarkTypeID"] = benchmarkTypeID
  1883. ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID
  1884. repo := ctx.Repo.Repository
  1885. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName)
  1886. if err == nil {
  1887. if len(tasks) != 0 {
  1888. log.Error("the job name did already exist", ctx.Data["MsgID"])
  1889. cloudBrainNewDataPrepare(ctx)
  1890. ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, &form)
  1891. return
  1892. }
  1893. } else {
  1894. if !models.IsErrJobNotExist(err) {
  1895. log.Error("system error, %v", err, ctx.Data["MsgID"])
  1896. cloudBrainNewDataPrepare(ctx)
  1897. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1898. return
  1899. }
  1900. }
  1901. if !jobNamePattern.MatchString(jobName) {
  1902. cloudBrainNewDataPrepare(ctx)
  1903. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form)
  1904. return
  1905. }
  1906. childInfo, err := getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID, ctx)
  1907. if err != nil {
  1908. log.Error("getBenchmarkAttachment failed:%v", err, ctx.Data["MsgID"])
  1909. cloudBrainNewDataPrepare(ctx)
  1910. ctx.RenderWithErr("benchmark type error", tplCloudBrainBenchmarkNew, &form)
  1911. return
  1912. }
  1913. _, err = getBenchmarkGpuQueue(gpuQueue)
  1914. if err != nil {
  1915. log.Error("getBenchmarkGpuQueue failed:%v", err, ctx.Data["MsgID"])
  1916. cloudBrainNewDataPrepare(ctx)
  1917. ctx.RenderWithErr("gpu queue error", tplCloudBrainBenchmarkNew, &form)
  1918. return
  1919. }
  1920. _, err = getBenchmarkResourceSpec(resourceSpecId)
  1921. if err != nil {
  1922. log.Error("getBenchmarkResourceSpec failed:%v", err, ctx.Data["MsgID"])
  1923. cloudBrainNewDataPrepare(ctx)
  1924. ctx.RenderWithErr("resource spec error", tplCloudBrainBenchmarkNew, &form)
  1925. return
  1926. }
  1927. count, err := models.GetBenchmarkCountByUserID(ctx.User.ID)
  1928. if err != nil {
  1929. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  1930. cloudBrainNewDataPrepare(ctx)
  1931. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1932. return
  1933. } else {
  1934. if count >= 1 {
  1935. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  1936. cloudBrainNewDataPrepare(ctx)
  1937. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tplCloudBrainBenchmarkNew, &form)
  1938. return
  1939. }
  1940. }
  1941. os.RemoveAll(codePath)
  1942. if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil {
  1943. log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"])
  1944. cloudBrainNewDataPrepare(ctx)
  1945. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1946. return
  1947. }
  1948. if _, err := os.Stat(codePath + "/train.py"); err != nil {
  1949. if os.IsNotExist(err) {
  1950. // file does not exist
  1951. log.Error("train.py does not exist, %v", err, ctx.Data["MsgID"])
  1952. cloudBrainNewDataPrepare(ctx)
  1953. ctx.RenderWithErr("train.py does not exist", tplCloudBrainBenchmarkNew, &form)
  1954. } else {
  1955. log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
  1956. cloudBrainNewDataPrepare(ctx)
  1957. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1958. }
  1959. return
  1960. } else if _, err := os.Stat(codePath + "/test.py"); err != nil {
  1961. if os.IsNotExist(err) {
  1962. // file does not exist
  1963. log.Error("test.py does not exist, %v", err, ctx.Data["MsgID"])
  1964. cloudBrainNewDataPrepare(ctx)
  1965. ctx.RenderWithErr("test.py does not exist", tplCloudBrainBenchmarkNew, &form)
  1966. } else {
  1967. log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
  1968. cloudBrainNewDataPrepare(ctx)
  1969. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1970. }
  1971. return
  1972. }
  1973. if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil {
  1974. log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
  1975. cloudBrainNewDataPrepare(ctx)
  1976. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1977. return
  1978. }
  1979. benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
  1980. var gpuType string
  1981. for _, gpuInfo := range gpuInfos.GpuInfo {
  1982. if gpuInfo.Queue == gpuQueue {
  1983. gpuType = gpuInfo.Value
  1984. }
  1985. }
  1986. if err := downloadRateCode(repo, jobName, childInfo.Owner, childInfo.RepoName, benchmarkPath, form.BenchmarkCategory, gpuType, ctx.User.Name); err != nil {
  1987. log.Error("downloadRateCode failed, %v", err, ctx.Data["MsgID"])
  1988. //cloudBrainNewDataPrepare(ctx)
  1989. //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1990. //return
  1991. }
  1992. if err := uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/"); err != nil {
  1993. log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
  1994. //cloudBrainNewDataPrepare(ctx)
  1995. //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1996. //return
  1997. }
  1998. uuid := childInfo.Attachment
  1999. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
  2000. if err != nil {
  2001. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  2002. cloudBrainNewDataPrepare(ctx)
  2003. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplCloudBrainBenchmarkNew, &form)
  2004. return
  2005. }
  2006. req := cloudbrain.GenerateCloudBrainTaskReq{
  2007. Ctx: ctx,
  2008. DisplayJobName: displayJobName,
  2009. JobName: jobName,
  2010. Image: image,
  2011. Command: command,
  2012. Uuids: uuid,
  2013. DatasetNames: datasetNames,
  2014. DatasetInfos: datasetInfos,
  2015. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  2016. ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
  2017. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  2018. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  2019. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  2020. JobType: string(models.JobTypeBenchmark),
  2021. GpuQueue: gpuQueue,
  2022. Description: form.Description,
  2023. BranchName: cloudbrain.DefaultBranchName,
  2024. BootFile: "",
  2025. Params: "",
  2026. CommitID: "",
  2027. BenchmarkTypeID: benchmarkTypeID,
  2028. BenchmarkChildTypeID: benchmarkChildTypeID,
  2029. ResourceSpecId: resourceSpecId,
  2030. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  2031. }
  2032. err = cloudbrain.GenerateTask(req)
  2033. if err != nil {
  2034. cloudBrainNewDataPrepare(ctx)
  2035. ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form)
  2036. return
  2037. }
  2038. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
  2039. }
  2040. func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  2041. ctx.Data["PageIsCloudBrain"] = true
  2042. displayJobName := form.DisplayJobName
  2043. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  2044. image := form.Image
  2045. uuid := form.Attachment
  2046. jobType := form.JobType
  2047. gpuQueue := form.GpuType
  2048. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  2049. resourceSpecId := form.ResourceSpecId
  2050. branchName := cloudbrain.DefaultBranchName
  2051. repo := ctx.Repo.Repository
  2052. tpl := tplCloudBrainBenchmarkNew
  2053. command := cloudbrain.GetCloudbrainDebugCommand()
  2054. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
  2055. if err == nil {
  2056. if len(tasks) != 0 {
  2057. log.Error("the job name did already exist", ctx.Data["MsgID"])
  2058. cloudBrainNewDataPrepare(ctx)
  2059. ctx.RenderWithErr("the job name did already exist", tpl, &form)
  2060. return
  2061. }
  2062. } else {
  2063. if !models.IsErrJobNotExist(err) {
  2064. log.Error("system error, %v", err, ctx.Data["MsgID"])
  2065. cloudBrainNewDataPrepare(ctx)
  2066. ctx.RenderWithErr("system error", tpl, &form)
  2067. return
  2068. }
  2069. }
  2070. if !jobNamePattern.MatchString(displayJobName) {
  2071. cloudBrainNewDataPrepare(ctx)
  2072. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
  2073. return
  2074. }
  2075. if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
  2076. log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
  2077. cloudBrainNewDataPrepare(ctx)
  2078. ctx.RenderWithErr("jobtype error", tpl, &form)
  2079. return
  2080. }
  2081. count, err := models.GetBenchmarkCountByUserID(ctx.User.ID)
  2082. if err != nil {
  2083. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  2084. cloudBrainNewDataPrepare(ctx)
  2085. ctx.RenderWithErr("system error", tpl, &form)
  2086. return
  2087. } else {
  2088. if count >= 1 {
  2089. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  2090. cloudBrainNewDataPrepare(ctx)
  2091. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
  2092. return
  2093. }
  2094. }
  2095. downloadCode(repo, codePath, branchName)
  2096. uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
  2097. modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
  2098. mkModelPath(modelPath)
  2099. uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")
  2100. snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
  2101. if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
  2102. downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name)
  2103. uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/")
  2104. command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description))
  2105. }
  2106. benchmarkChildTypeID := 0
  2107. brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
  2108. if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
  2109. downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name)
  2110. uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/")
  2111. benchmarkChildTypeID = form.BenchmarkChildTypeID
  2112. command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
  2113. }
  2114. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
  2115. if err != nil {
  2116. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  2117. cloudBrainNewDataPrepare(ctx)
  2118. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
  2119. return
  2120. }
  2121. req := cloudbrain.GenerateCloudBrainTaskReq{
  2122. Ctx: ctx,
  2123. DisplayJobName: displayJobName,
  2124. JobName: jobName,
  2125. Image: image,
  2126. Command: command,
  2127. Uuids: uuid,
  2128. DatasetNames: datasetNames,
  2129. DatasetInfos: datasetInfos,
  2130. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  2131. ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
  2132. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  2133. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  2134. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  2135. JobType: jobType,
  2136. GpuQueue: gpuQueue,
  2137. Description: form.Description,
  2138. BranchName: branchName,
  2139. BootFile: form.BootFile,
  2140. Params: form.Params,
  2141. CommitID: "",
  2142. BenchmarkTypeID: 0,
  2143. BenchmarkChildTypeID: benchmarkChildTypeID,
  2144. ResourceSpecId: resourceSpecId,
  2145. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  2146. }
  2147. err = cloudbrain.GenerateTask(req)
  2148. if err != nil {
  2149. cloudBrainNewDataPrepare(ctx)
  2150. ctx.RenderWithErr(err.Error(), tpl, &form)
  2151. return
  2152. }
  2153. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
  2154. }
  2155. func getBrainRegion(benchmarkChildTypeID int) string {
  2156. values := []string{"V1", "V2", "V4", "IT"}
  2157. return values[benchmarkChildTypeID]
  2158. }
  2159. func trimSpaceNewlineInString(s string) string {
  2160. re := regexp.MustCompile(`\r?\n`)
  2161. return re.ReplaceAllString(s, " ")
  2162. }
  2163. func BenchmarkDel(ctx *context.Context) {
  2164. if err := deleteCloudbrainJob(ctx); err != nil {
  2165. log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
  2166. ctx.ServerError(err.Error(), err)
  2167. return
  2168. }
  2169. var isAdminPage = ctx.Query("isadminpage")
  2170. var isHomePage = ctx.Query("ishomepage")
  2171. if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
  2172. ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
  2173. } else if isHomePage == "true" {
  2174. ctx.Redirect(setting.AppSubURL + "/cloudbrains")
  2175. } else {
  2176. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
  2177. }
  2178. }
  2179. func CloudBrainTrainJobNew(ctx *context.Context) {
  2180. err := cloudBrainNewDataPrepare(ctx)
  2181. if err != nil {
  2182. ctx.ServerError("get new train-job info failed", err)
  2183. return
  2184. }
  2185. ctx.HTML(http.StatusOK, tplCloudBrainTrainJobNew)
  2186. }
  2187. func InferenceCloudBrainJobNew(ctx *context.Context) {
  2188. err := cloudBrainNewDataPrepare(ctx)
  2189. if err != nil {
  2190. ctx.ServerError("get new train-job info failed", err)
  2191. return
  2192. }
  2193. ctx.HTML(http.StatusOK, tplCloudBrainInferenceJobNew)
  2194. }
  2195. func InferenceCloudBrainJobShow(ctx *context.Context) {
  2196. cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference)
  2197. }
  2198. func DownloadInferenceResultFile(ctx *context.Context) {
  2199. var jobID = ctx.Params(":jobid")
  2200. var versionName = ctx.Query("version_name")
  2201. task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
  2202. if err != nil {
  2203. log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
  2204. return
  2205. }
  2206. allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, task.ResultUrl)
  2207. returnFileName := task.DisplayJobName + ".zip"
  2208. MinioDownloadManyFile(task.ResultUrl, ctx, returnFileName, allFile)
  2209. }
  2210. func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) {
  2211. var command string
  2212. bootFile := strings.TrimSpace(form.BootFile)
  2213. params := form.Params
  2214. if !strings.HasSuffix(bootFile, ".py") {
  2215. log.Error("bootFile(%s) format error", bootFile)
  2216. return command, errors.New("bootFile format error")
  2217. }
  2218. var parameters models.Parameters
  2219. var param string
  2220. if len(params) != 0 {
  2221. err := json.Unmarshal([]byte(params), &parameters)
  2222. if err != nil {
  2223. log.Error("Failed to Unmarshal params: %s (%v)", params, err)
  2224. return command, err
  2225. }
  2226. for _, parameter := range parameters.Parameter {
  2227. param += " --" + parameter.Label + "=" + parameter.Value
  2228. }
  2229. }
  2230. param += " --modelname" + "=" + form.CkptName
  2231. command += "python /code/" + bootFile + param + " > " + cloudbrain.ResultPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile
  2232. return command, nil
  2233. }
  2234. func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) {
  2235. var command string
  2236. bootFile := strings.TrimSpace(form.BootFile)
  2237. params := form.Params
  2238. if !strings.HasSuffix(bootFile, ".py") {
  2239. log.Error("bootFile(%s) format error", bootFile)
  2240. return command, errors.New("bootFile format error")
  2241. }
  2242. var parameters models.Parameters
  2243. var param string
  2244. if len(params) != 0 {
  2245. err := json.Unmarshal([]byte(params), &parameters)
  2246. if err != nil {
  2247. log.Error("Failed to Unmarshal params: %s (%v)", params, err)
  2248. return command, err
  2249. }
  2250. for _, parameter := range parameters.Parameter {
  2251. param += " --" + parameter.Label + "=" + parameter.Value
  2252. }
  2253. }
  2254. command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile
  2255. return command, nil
  2256. }
  2257. func CloudBrainTrainJobDel(ctx *context.Context) {
  2258. var listType = ctx.Query("listType")
  2259. if err := deleteCloudbrainJob(ctx); err != nil {
  2260. log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
  2261. ctx.ServerError(err.Error(), err)
  2262. return
  2263. }
  2264. var isAdminPage = ctx.Query("isadminpage")
  2265. var isHomePage = ctx.Query("ishomepage")
  2266. if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
  2267. ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
  2268. } else if isHomePage == "true" {
  2269. ctx.Redirect(setting.AppSubURL + "/cloudbrains")
  2270. } else {
  2271. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType)
  2272. }
  2273. }
  2274. func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes {
  2275. var lang = ctx.Locale.Language()
  2276. if benchmarkTypesMap[lang] == nil {
  2277. var val = i18n.Tr(lang, BENCHMARK_TYPE_CODE)
  2278. //use config
  2279. val = setting.BenchmarkTypes
  2280. var tempType *models.BenchmarkTypes
  2281. if err := json.Unmarshal([]byte(val), &tempType); err != nil {
  2282. log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", val, err, ctx.Data["MsgID"])
  2283. return &models.BenchmarkTypes{}
  2284. }
  2285. benchmarkTypesMap[lang] = tempType
  2286. }
  2287. return benchmarkTypesMap[lang]
  2288. }