You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

cloudbrain.go 82 kB

4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
5 years ago
5 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
4 years ago
4 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
3 years ago
5 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
5 years ago
4 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630
  1. package repo
  2. import (
  3. "bufio"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "net/http"
  9. "os"
  10. "regexp"
  11. "sort"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "unicode/utf8"
  16. "code.gitea.io/gitea/modules/grampus"
  17. "code.gitea.io/gitea/modules/timeutil"
  18. "github.com/unknwon/i18n"
  19. "code.gitea.io/gitea/models"
  20. "code.gitea.io/gitea/modules/auth"
  21. "code.gitea.io/gitea/modules/base"
  22. "code.gitea.io/gitea/modules/cloudbrain"
  23. "code.gitea.io/gitea/modules/context"
  24. "code.gitea.io/gitea/modules/git"
  25. "code.gitea.io/gitea/modules/log"
  26. "code.gitea.io/gitea/modules/modelarts"
  27. "code.gitea.io/gitea/modules/setting"
  28. "code.gitea.io/gitea/modules/storage"
  29. "code.gitea.io/gitea/modules/util"
  30. )
  31. const (
  32. tplCloudBrainNew base.TplName = "repo/cloudbrain/new"
  33. tplCloudBrainShow base.TplName = "repo/cloudbrain/show"
  34. tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index"
  35. tplCloudBrainBenchmarkIndex base.TplName = "repo/cloudbrain/benchmark/index"
  36. tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new"
  37. tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show"
  38. tplCloudBrainImageSubmit base.TplName = "repo/cloudbrain/image/submit"
  39. tplCloudBrainImageEdit base.TplName = "repo/cloudbrain/image/edit"
  40. tplCloudBrainTrainJobNew base.TplName = "repo/cloudbrain/trainjob/new"
  41. tplCloudBrainTrainJobShow base.TplName = "repo/cloudbrain/trainjob/show"
  42. tplCloudBrainInferenceJobNew base.TplName = "repo/cloudbrain/inference/new"
  43. tplCloudBrainInferenceJobShow base.TplName = "repo/cloudbrain/inference/show"
  44. )
  45. var (
  46. gpuInfos *models.GpuInfos
  47. categories *models.Categories
  48. benchmarkTypes *models.BenchmarkTypes
  49. benchmarkGpuInfos *models.GpuInfos
  50. benchmarkResourceSpecs *models.ResourceSpecs
  51. trainGpuInfos *models.GpuInfos
  52. inferenceGpuInfos *models.GpuInfos
  53. )
  54. const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types"
  55. const CLONE_FILE_PREFIX = "file:///"
  56. var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0)
  57. var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
  58. // MustEnableDataset check if repository enable internal cb
  59. func MustEnableCloudbrain(ctx *context.Context) {
  60. if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
  61. ctx.NotFound("MustEnableCloudbrain", nil)
  62. return
  63. }
  64. }
  65. func cutString(str string, lens int) string {
  66. if len(str) < lens {
  67. return str
  68. }
  69. return str[:lens]
  70. }
  71. func jobNamePrefixValid(s string) string {
  72. lowStr := strings.ToLower(s)
  73. re := regexp.MustCompile(`[^a-z0-9_\\-]+`)
  74. removeSpecial := re.ReplaceAllString(lowStr, "")
  75. re = regexp.MustCompile(`^[_\\-]+`)
  76. return re.ReplaceAllString(removeSpecial, "")
  77. }
  78. func cloudBrainNewDataPrepare(ctx *context.Context) error {
  79. ctx.Data["PageIsCloudBrain"] = true
  80. t := time.Now()
  81. var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
  82. ctx.Data["display_job_name"] = displayJobName
  83. ctx.Data["command"] = cloudbrain.GetCloudbrainDebugCommand()
  84. ctx.Data["code_path"] = cloudbrain.CodeMountPath
  85. ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
  86. ctx.Data["model_path"] = cloudbrain.ModelMountPath
  87. ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
  88. ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled
  89. if categories == nil {
  90. json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
  91. }
  92. ctx.Data["benchmark_categories"] = categories.Category
  93. ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType
  94. queuesDetail, _ := cloudbrain.GetQueuesDetail()
  95. if queuesDetail != nil {
  96. ctx.Data["QueuesDetail"] = queuesDetail
  97. }
  98. cloudbrain.InitSpecialPool()
  99. if gpuInfos == nil {
  100. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  101. }
  102. ctx.Data["gpu_types"] = gpuInfos.GpuInfo
  103. if trainGpuInfos == nil {
  104. json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos)
  105. }
  106. ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo
  107. if inferenceGpuInfos == nil {
  108. json.Unmarshal([]byte(setting.InferenceGpuTypes), &inferenceGpuInfos)
  109. }
  110. ctx.Data["inference_gpu_types"] = inferenceGpuInfos.GpuInfo
  111. if benchmarkGpuInfos == nil {
  112. json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos)
  113. }
  114. ctx.Data["benchmark_gpu_types"] = benchmarkGpuInfos.GpuInfo
  115. if benchmarkResourceSpecs == nil {
  116. json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs)
  117. }
  118. ctx.Data["benchmark_resource_specs"] = benchmarkResourceSpecs.ResourceSpec
  119. if cloudbrain.ResourceSpecs == nil {
  120. json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
  121. }
  122. ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec
  123. if cloudbrain.TrainResourceSpecs == nil {
  124. json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs)
  125. }
  126. ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec
  127. if cloudbrain.InferenceResourceSpecs == nil {
  128. json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs)
  129. }
  130. ctx.Data["inference_resource_specs"] = cloudbrain.InferenceResourceSpecs.ResourceSpec
  131. if cloudbrain.SpecialPools != nil {
  132. var debugGpuTypes []*models.GpuInfo
  133. var trainGpuTypes []*models.GpuInfo
  134. for _, pool := range cloudbrain.SpecialPools.Pools {
  135. org, _ := models.GetOrgByName(pool.Org)
  136. if org != nil {
  137. isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID)
  138. if isOrgMember {
  139. for _, jobType := range pool.JobType {
  140. if jobType == string(models.JobTypeDebug) {
  141. debugGpuTypes = append(debugGpuTypes, pool.Pool...)
  142. if pool.ResourceSpec != nil {
  143. ctx.Data["resource_specs"] = pool.ResourceSpec
  144. }
  145. } else if jobType == string(models.JobTypeTrain) {
  146. trainGpuTypes = append(trainGpuTypes, pool.Pool...)
  147. if pool.ResourceSpec != nil {
  148. ctx.Data["train_resource_specs"] = pool.ResourceSpec
  149. }
  150. }
  151. }
  152. break
  153. }
  154. }
  155. }
  156. if len(debugGpuTypes) > 0 {
  157. ctx.Data["gpu_types"] = debugGpuTypes
  158. }
  159. if len(trainGpuTypes) > 0 {
  160. ctx.Data["train_gpu_types"] = trainGpuTypes
  161. }
  162. }
  163. ctx.Data["params"] = ""
  164. ctx.Data["branchName"] = ctx.Repo.BranchName
  165. ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
  166. ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled
  167. ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
  168. ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled
  169. ctx.Data["datasetType"] = models.TypeCloudBrainOne
  170. ctx.Data["benchmarkMode"] = ctx.Query("benchmarkMode")
  171. return nil
  172. }
  173. func CloudBrainNew(ctx *context.Context) {
  174. err := cloudBrainNewDataPrepare(ctx)
  175. if err != nil {
  176. ctx.ServerError("get new cloudbrain info failed", err)
  177. return
  178. }
  179. ctx.HTML(200, tplCloudBrainNew)
  180. }
  181. func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  182. ctx.Data["PageIsCloudBrain"] = true
  183. displayJobName := form.DisplayJobName
  184. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  185. image := strings.TrimSpace(form.Image)
  186. uuids := form.Attachment
  187. jobType := form.JobType
  188. gpuQueue := form.GpuType
  189. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  190. resourceSpecId := form.ResourceSpecId
  191. branchName := form.BranchName
  192. repo := ctx.Repo.Repository
  193. tpl := tplCloudBrainNew
  194. if jobType == string(models.JobTypeTrain) {
  195. tpl = tplCloudBrainTrainJobNew
  196. }
  197. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
  198. if err == nil {
  199. if len(tasks) != 0 {
  200. log.Error("the job name did already exist", ctx.Data["MsgID"])
  201. cloudBrainNewDataPrepare(ctx)
  202. ctx.RenderWithErr("the job name did already exist", tpl, &form)
  203. return
  204. }
  205. } else {
  206. if !models.IsErrJobNotExist(err) {
  207. log.Error("system error, %v", err, ctx.Data["MsgID"])
  208. cloudBrainNewDataPrepare(ctx)
  209. ctx.RenderWithErr("system error", tpl, &form)
  210. return
  211. }
  212. }
  213. if !jobNamePattern.MatchString(displayJobName) {
  214. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
  215. return
  216. }
  217. if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) {
  218. log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
  219. cloudBrainNewDataPrepare(ctx)
  220. ctx.RenderWithErr("jobtype error", tpl, &form)
  221. return
  222. }
  223. count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
  224. if err != nil {
  225. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  226. cloudBrainNewDataPrepare(ctx)
  227. ctx.RenderWithErr("system error", tpl, &form)
  228. return
  229. } else {
  230. if count >= 1 {
  231. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  232. cloudBrainNewDataPrepare(ctx)
  233. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
  234. return
  235. }
  236. }
  237. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuids)
  238. if err != nil {
  239. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  240. cloudBrainNewDataPrepare(ctx)
  241. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
  242. return
  243. }
  244. command := cloudbrain.GetCloudbrainDebugCommand()
  245. if jobType == string(models.JobTypeTrain) {
  246. tpl = tplCloudBrainTrainJobNew
  247. commandTrain, err := getTrainJobCommand(form)
  248. if err != nil {
  249. log.Error("getTrainJobCommand failed: %v", err)
  250. ctx.RenderWithErr(err.Error(), tpl, &form)
  251. return
  252. }
  253. command = commandTrain
  254. }
  255. errStr := checkCloudBrainSpecialPool(ctx, jobType, gpuQueue, resourceSpecId)
  256. if errStr != "" {
  257. cloudBrainNewDataPrepare(ctx)
  258. ctx.RenderWithErr(errStr, tpl, &form)
  259. return
  260. }
  261. if branchName == "" {
  262. branchName = cloudbrain.DefaultBranchName
  263. }
  264. downloadCode(repo, codePath, branchName)
  265. uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
  266. modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
  267. mkModelPath(modelPath)
  268. uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")
  269. commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)
  270. req := cloudbrain.GenerateCloudBrainTaskReq{
  271. Ctx: ctx,
  272. DisplayJobName: displayJobName,
  273. JobName: jobName,
  274. Image: image,
  275. Command: command,
  276. Uuids: uuids,
  277. DatasetNames: datasetNames,
  278. DatasetInfos: datasetInfos,
  279. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  280. ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
  281. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  282. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  283. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  284. JobType: jobType,
  285. GpuQueue: gpuQueue,
  286. Description: form.Description,
  287. BranchName: branchName,
  288. BootFile: form.BootFile,
  289. Params: form.Params,
  290. CommitID: commitID,
  291. BenchmarkTypeID: 0,
  292. BenchmarkChildTypeID: 0,
  293. ResourceSpecId: resourceSpecId,
  294. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  295. }
  296. err = cloudbrain.GenerateTask(req)
  297. if err != nil {
  298. cloudBrainNewDataPrepare(ctx)
  299. ctx.RenderWithErr(err.Error(), tpl, &form)
  300. return
  301. }
  302. if jobType == string(models.JobTypeTrain) {
  303. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=all")
  304. } else {
  305. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all")
  306. }
  307. }
  308. func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBrainInferencForm) {
  309. ctx.Data["PageIsCloudBrain"] = true
  310. displayJobName := form.DisplayJobName
  311. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  312. image := strings.TrimSpace(form.Image)
  313. uuid := form.Attachment
  314. jobType := string(models.JobTypeInference)
  315. gpuQueue := form.GpuType
  316. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  317. resourceSpecId := form.ResourceSpecId
  318. branchName := form.BranchName
  319. labelName := form.LabelName
  320. repo := ctx.Repo.Repository
  321. ckptUrl := setting.Attachment.Minio.RealPath + form.TrainUrl + form.CkptName
  322. log.Info("ckpt url:" + ckptUrl)
  323. tpl := tplCloudBrainInferenceJobNew
  324. command, err := getInferenceJobCommand(form)
  325. if err != nil {
  326. log.Error("getTrainJobCommand failed: %v", err)
  327. ctx.RenderWithErr(err.Error(), tpl, &form)
  328. return
  329. }
  330. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
  331. if err == nil {
  332. if len(tasks) != 0 {
  333. log.Error("the job name did already exist", ctx.Data["MsgID"])
  334. cloudBrainNewDataPrepare(ctx)
  335. ctx.RenderWithErr("the job name did already exist", tpl, &form)
  336. return
  337. }
  338. } else {
  339. if !models.IsErrJobNotExist(err) {
  340. log.Error("system error, %v", err, ctx.Data["MsgID"])
  341. cloudBrainNewDataPrepare(ctx)
  342. ctx.RenderWithErr("system error", tpl, &form)
  343. return
  344. }
  345. }
  346. if !jobNamePattern.MatchString(displayJobName) {
  347. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
  348. return
  349. }
  350. count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
  351. if err != nil {
  352. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  353. cloudBrainNewDataPrepare(ctx)
  354. ctx.RenderWithErr("system error", tpl, &form)
  355. return
  356. } else {
  357. if count >= 1 {
  358. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  359. cloudBrainNewDataPrepare(ctx)
  360. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
  361. return
  362. }
  363. }
  364. if branchName == "" {
  365. branchName = cloudbrain.DefaultBranchName
  366. }
  367. downloadCode(repo, codePath, branchName)
  368. uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
  369. resultPath := setting.JobPath + jobName + cloudbrain.ResultPath + "/"
  370. mkResultPath(resultPath)
  371. uploadCodeToMinio(resultPath, jobName, cloudbrain.ResultPath+"/")
  372. commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)
  373. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
  374. if err != nil {
  375. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  376. cloudBrainNewDataPrepare(ctx)
  377. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
  378. return
  379. }
  380. req := cloudbrain.GenerateCloudBrainTaskReq{
  381. Ctx: ctx,
  382. DisplayJobName: displayJobName,
  383. JobName: jobName,
  384. Image: image,
  385. Command: command,
  386. Uuids: uuid,
  387. DatasetNames: datasetNames,
  388. DatasetInfos: datasetInfos,
  389. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  390. ModelPath: setting.Attachment.Minio.RealPath + form.TrainUrl,
  391. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  392. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  393. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  394. JobType: jobType,
  395. GpuQueue: gpuQueue,
  396. Description: form.Description,
  397. BranchName: branchName,
  398. BootFile: form.BootFile,
  399. Params: form.Params,
  400. CommitID: commitID,
  401. ResourceSpecId: resourceSpecId,
  402. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  403. ModelName: form.ModelName,
  404. ModelVersion: form.ModelVersion,
  405. CkptName: form.CkptName,
  406. TrainUrl: form.TrainUrl,
  407. LabelName: labelName,
  408. }
  409. err = cloudbrain.GenerateTask(req)
  410. if err != nil {
  411. cloudBrainNewDataPrepare(ctx)
  412. ctx.RenderWithErr(err.Error(), tpl, &form)
  413. return
  414. }
  415. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/inference-job")
  416. }
  417. /**
  418. 检查用户传输的参数是否符合专属资源池
  419. */
  420. func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string {
  421. if cloudbrain.SpecialPools != nil {
  422. var isInPoolOrg = false
  423. var matchSpecialPool = false
  424. for _, specialPool := range cloudbrain.SpecialPools.Pools {
  425. if cloudbrain.IsElementExist(specialPool.JobType, jobType) && cloudbrain.IsQueueInSpecialtPool(specialPool.Pool, queue) {
  426. if cloudbrain.IsResourceSpecInSpecialPool(specialPool.ResourceSpec, resourceSpecId) {
  427. matchSpecialPool = true
  428. org, _ := models.GetOrgByName(specialPool.Org)
  429. if org != nil {
  430. isInPoolOrg, _ = models.IsOrganizationMember(org.ID, ctx.User.ID)
  431. if isInPoolOrg {
  432. break //传入参数,和专属资源池匹配上了,检查通过
  433. }
  434. }
  435. }
  436. }
  437. }
  438. //资源池有匹配上,但是用户不在相应的组织中,返回错误信息。界面已经过滤了选择,界面操作不会到这个逻辑
  439. if matchSpecialPool && !isInPoolOrg {
  440. return ctx.Tr("repo.grampus.no_operate_right")
  441. }
  442. }
  443. //没有匹配到资源池或者没有设置专属资源池,检查通过; 获取和资源池完全匹配检查通过
  444. return ""
  445. }
  446. func CloudBrainRestart(ctx *context.Context) {
  447. var ID = ctx.Params(":id")
  448. var resultCode = "0"
  449. var errorMsg = ""
  450. var status = string(models.JobWaiting)
  451. task := ctx.Cloudbrain
  452. for {
  453. if task.Status != string(models.JobStopped) && task.Status != string(models.JobSucceeded) && task.Status != string(models.JobFailed) {
  454. log.Error("the job(%s) is not stopped", task.JobName, ctx.Data["MsgID"])
  455. resultCode = "-1"
  456. errorMsg = "the job is not stopped"
  457. break
  458. }
  459. if task.Image == "" || task.GpuQueue == "" || task.Type != models.TypeCloudBrainOne {
  460. log.Error("the job(%s) version is too old", task.JobName, ctx.Data["MsgID"])
  461. resultCode = "-1"
  462. errorMsg = "the job's version is too old and can not be restarted"
  463. break
  464. }
  465. if !ctx.IsSigned || (ctx.User.ID != task.UserID && !ctx.IsUserSiteAdmin()) {
  466. log.Error("the user has no right ro restart the job", task.JobName, ctx.Data["MsgID"])
  467. resultCode = "-1"
  468. errorMsg = "you have no right to restart the job"
  469. break
  470. }
  471. var hasSameResource bool
  472. if gpuInfos == nil {
  473. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  474. }
  475. for _, resourceType := range gpuInfos.GpuInfo {
  476. if resourceType.Queue == task.GpuQueue {
  477. hasSameResource = true
  478. break
  479. }
  480. }
  481. if !hasSameResource && cloudbrain.SpecialPools != nil {
  482. for _, specialPool := range cloudbrain.SpecialPools.Pools {
  483. cloudbrain.IsElementExist(specialPool.JobType, string(models.JobTypeDebug))
  484. for _, pool := range specialPool.Pool {
  485. if pool.Queue == task.GpuQueue {
  486. hasSameResource = true
  487. }
  488. }
  489. }
  490. }
  491. if !hasSameResource {
  492. log.Error("has no same resource, can not restart", ctx.Data["MsgID"])
  493. resultCode = "-1"
  494. errorMsg = "the job's version is too old and can not be restarted"
  495. break
  496. }
  497. count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug))
  498. if err != nil {
  499. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  500. resultCode = "-1"
  501. errorMsg = "system error"
  502. break
  503. } else {
  504. if count >= 1 {
  505. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  506. resultCode = "-1"
  507. errorMsg = ctx.Tr("repo.cloudbrain.morethanonejob")
  508. break
  509. }
  510. }
  511. err = cloudbrain.RestartTask(ctx, task, &ID)
  512. if err != nil {
  513. log.Error("RestartTask failed:%v", err.Error(), ctx.Data["MsgID"])
  514. resultCode = "-1"
  515. errorMsg = "system error"
  516. break
  517. }
  518. break
  519. }
  520. ctx.JSON(200, map[string]string{
  521. "result_code": resultCode,
  522. "error_msg": errorMsg,
  523. "status": status,
  524. "id": ID,
  525. })
  526. }
  527. func CloudBrainBenchMarkShow(ctx *context.Context) {
  528. cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark)
  529. }
  530. func CloudBrainShow(ctx *context.Context) {
  531. cloudBrainShow(ctx, tplCloudBrainShow, models.JobTypeDebug)
  532. }
  533. func CloudBrainTrainJobShow(ctx *context.Context) {
  534. cloudBrainShow(ctx, tplCloudBrainTrainJobShow, models.JobTypeTrain)
  535. }
  536. func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.JobType) {
  537. ctx.Data["PageIsCloudBrain"] = true
  538. debugListType := ctx.Query("debugListType")
  539. var task *models.Cloudbrain
  540. var err error
  541. if jobType == models.JobTypeTrain || jobType == models.JobTypeInference {
  542. task, err = models.GetCloudbrainByJobID(ctx.Params(":jobid"))
  543. } else {
  544. task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id"))
  545. }
  546. if err != nil {
  547. log.Info("error:" + err.Error())
  548. ctx.Data["error"] = err.Error()
  549. return
  550. }
  551. result, err := cloudbrain.GetJob(task.JobID)
  552. if err != nil {
  553. log.Info("error:" + err.Error())
  554. ctx.Data["error"] = err.Error()
  555. return
  556. }
  557. if task.JobType == string(models.JobTypeTrain) {
  558. if cloudbrain.TrainResourceSpecs == nil {
  559. json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs)
  560. }
  561. for _, tmp := range cloudbrain.TrainResourceSpecs.ResourceSpec {
  562. if tmp.Id == task.ResourceSpecId {
  563. ctx.Data["GpuNum"] = tmp.GpuNum
  564. ctx.Data["CpuNum"] = tmp.CpuNum
  565. ctx.Data["MemMiB"] = tmp.MemMiB
  566. ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
  567. }
  568. }
  569. } else if task.JobType == string(models.JobTypeInference) {
  570. if cloudbrain.InferenceResourceSpecs == nil {
  571. json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs)
  572. }
  573. for _, tmp := range cloudbrain.InferenceResourceSpecs.ResourceSpec {
  574. if tmp.Id == task.ResourceSpecId {
  575. ctx.Data["GpuNum"] = tmp.GpuNum
  576. ctx.Data["CpuNum"] = tmp.CpuNum
  577. ctx.Data["MemMiB"] = tmp.MemMiB
  578. ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
  579. }
  580. }
  581. } else {
  582. if cloudbrain.ResourceSpecs == nil {
  583. json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
  584. }
  585. for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec {
  586. if tmp.Id == task.ResourceSpecId {
  587. ctx.Data["GpuNum"] = tmp.GpuNum
  588. ctx.Data["CpuNum"] = tmp.CpuNum
  589. ctx.Data["MemMiB"] = tmp.MemMiB
  590. ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
  591. }
  592. }
  593. }
  594. if result != nil {
  595. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  596. jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
  597. spec := "GPU数:" + strconv.Itoa(jobRes.Resource.NvidiaComGpu) + ",CPU数:" + strconv.Itoa(jobRes.Resource.CPU) + ",内存(MB):" + jobRes.Resource.Memory
  598. ctx.Data["resource_spec"] = spec
  599. if task.JobType == string(models.JobTypeTrain) {
  600. if trainGpuInfos == nil {
  601. json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos)
  602. }
  603. for _, resourceType := range trainGpuInfos.GpuInfo {
  604. if resourceType.Queue == jobRes.Config.GpuType {
  605. ctx.Data["resource_type"] = resourceType.Value
  606. }
  607. }
  608. } else if task.JobType == string(models.JobTypeInference) {
  609. if inferenceGpuInfos == nil {
  610. json.Unmarshal([]byte(setting.InferenceGpuTypes), &inferenceGpuInfos)
  611. }
  612. for _, resourceType := range inferenceGpuInfos.GpuInfo {
  613. if resourceType.Queue == jobRes.Config.GpuType {
  614. ctx.Data["resource_type"] = resourceType.Value
  615. }
  616. }
  617. } else if cloudbrain.IsBenchmarkJob(task.JobType) {
  618. if benchmarkGpuInfos == nil {
  619. json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos)
  620. }
  621. for _, resourceType := range benchmarkGpuInfos.GpuInfo {
  622. if resourceType.Queue == jobRes.Config.GpuType {
  623. ctx.Data["resource_type"] = resourceType.Value
  624. }
  625. }
  626. } else {
  627. if gpuInfos == nil {
  628. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  629. }
  630. for _, resourceType := range gpuInfos.GpuInfo {
  631. if resourceType.Queue == jobRes.Config.GpuType {
  632. ctx.Data["resource_type"] = resourceType.Value
  633. }
  634. }
  635. }
  636. taskRoles := jobRes.TaskRoles
  637. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  638. ctx.Data["taskRes"] = taskRes
  639. ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics
  640. task.Status = taskRes.TaskStatuses[0].State
  641. task.ContainerID = taskRes.TaskStatuses[0].ContainerID
  642. task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  643. models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)
  644. if task.DeletedAt.IsZero() { //normal record
  645. err = models.UpdateJob(task)
  646. if err != nil {
  647. ctx.Data["error"] = err.Error()
  648. return
  649. }
  650. } else { //deleted record
  651. }
  652. ctx.Data["result"] = jobRes
  653. } else {
  654. log.Info("error:" + err.Error())
  655. return
  656. }
  657. user, err := models.GetUserByID(task.UserID)
  658. if err == nil {
  659. task.User = user
  660. }
  661. if task.BenchmarkTypeID > 0 {
  662. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  663. if task.BenchmarkTypeID == benchmarkType.Id {
  664. ctx.Data["BenchmarkTypeName"] = benchmarkType.First
  665. task.BenchmarkTypeName = benchmarkType.First
  666. for _, benchmarkChildType := range benchmarkType.Second {
  667. if task.BenchmarkChildTypeID == benchmarkChildType.Id {
  668. ctx.Data["BenchmarkChildTypeName"] = benchmarkChildType.Value
  669. break
  670. }
  671. }
  672. break
  673. }
  674. }
  675. }
  676. if task.JobType == string(models.JobTypeBenchmark) {
  677. task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
  678. } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
  679. task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
  680. task.BenchmarkTypeName = task.JobType
  681. ctx.Data["BenchmarkTypeName"] = task.JobType
  682. if task.JobType == string(models.JobTypeBrainScore) {
  683. ctx.Data["BenchmarkChildTypeName"] = getBrainRegion(task.BenchmarkChildTypeID)
  684. }
  685. }
  686. if task.TrainJobDuration == "" {
  687. if task.Duration == 0 {
  688. var duration int64
  689. if task.Status == string(models.JobWaiting) {
  690. duration = 0
  691. } else if task.Status == string(models.JobRunning) {
  692. duration = time.Now().Unix() - int64(task.CreatedUnix)
  693. } else {
  694. duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix)
  695. }
  696. task.Duration = duration
  697. }
  698. task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
  699. }
  700. ctx.Data["duration"] = task.TrainJobDuration
  701. if len(task.Parameters) > 0 {
  702. var parameters models.Parameters
  703. err := json.Unmarshal([]byte(task.Parameters), &parameters)
  704. if err != nil {
  705. log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err)
  706. task.Parameters = ""
  707. } else {
  708. if len(parameters.Parameter) > 0 {
  709. paramTemp := ""
  710. for _, Parameter := range parameters.Parameter {
  711. param := Parameter.Label + " = " + Parameter.Value + "; "
  712. paramTemp = paramTemp + param
  713. }
  714. task.Parameters = paramTemp[:len(paramTemp)-2]
  715. } else {
  716. task.Parameters = ""
  717. }
  718. }
  719. }
  720. ctx.Data["task"] = task
  721. ctx.Data["jobName"] = task.JobName
  722. ctx.Data["displayJobName"] = task.DisplayJobName
  723. version_list_task := make([]*models.Cloudbrain, 0)
  724. version_list_task = append(version_list_task, task)
  725. ctx.Data["version_list_task"] = version_list_task
  726. ctx.Data["debugListType"] = debugListType
  727. ctx.Data["code_path"] = cloudbrain.CodeMountPath
  728. ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
  729. ctx.Data["model_path"] = cloudbrain.ModelMountPath
  730. ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
  731. ctx.Data["branchName"] = task.BranchName
  732. ctx.HTML(200, tpName)
  733. }
  734. func CloudBrainDebug(ctx *context.Context) {
  735. task := ctx.Cloudbrain
  736. debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
  737. ctx.Redirect(debugUrl)
  738. }
  739. func CloudBrainCommitImageShow(ctx *context.Context) {
  740. ctx.Data["PageIsCloudBrain"] = true
  741. ctx.Data["Type"] = ctx.Cloudbrain.Type
  742. ctx.HTML(200, tplCloudBrainImageSubmit)
  743. }
  744. func CloudBrainImageEdit(ctx *context.Context) {
  745. ctx.Data["PageIsImageEdit"] = true
  746. ctx.Data["PageFrom"] = ctx.Params(":from")
  747. var ID = ctx.Params(":id")
  748. id, err := strconv.ParseInt(ID, 10, 64)
  749. if err != nil {
  750. log.Error("GetImageByID failed:%v", err.Error())
  751. ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
  752. }
  753. image, err := models.GetImageByID(id)
  754. if err != nil {
  755. log.Error("GetImageByID failed:%v", err.Error())
  756. ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
  757. }
  758. ctx.Data["Image"] = image
  759. ctx.HTML(http.StatusOK, tplCloudBrainImageEdit)
  760. }
  761. func CloudBrainImageEditPost(ctx *context.Context, form auth.EditImageCloudBrainForm) {
  762. if utf8.RuneCountInString(form.Description) > 255 {
  763. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
  764. return
  765. }
  766. validTopics, errMessage := checkTopics(form.Topics)
  767. if errMessage != "" {
  768. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
  769. return
  770. }
  771. image, err := models.GetImageByID(form.ID)
  772. if err != nil {
  773. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
  774. }
  775. image.IsPrivate = form.IsPrivate
  776. image.Description = form.Description
  777. err = models.WithTx(func(ctx models.DBContext) error {
  778. if err := models.UpdateLocalImage(image); err != nil {
  779. return err
  780. }
  781. if err := models.SaveImageTopics(image.ID, validTopics...); err != nil {
  782. return err
  783. }
  784. return nil
  785. })
  786. if err != nil {
  787. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
  788. } else {
  789. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  790. }
  791. }
  792. func CloudBrainImageDelete(ctx *context.Context) {
  793. var ID = ctx.Params(":id")
  794. id, err := strconv.ParseInt(ID, 10, 64)
  795. if err != nil {
  796. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
  797. return
  798. }
  799. err = models.DeleteLocalImage(id)
  800. if err != nil {
  801. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_delete_fail")))
  802. } else {
  803. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  804. }
  805. }
  806. func CloudBrainCommitImageCheck(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
  807. isExist, _ := models.IsImageExistByUser(form.Tag, ctx.User.ID)
  808. if isExist {
  809. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_overwrite")))
  810. } else {
  811. ctx.JSON(http.StatusOK, models.BaseOKMessage)
  812. }
  813. }
  814. func CloudBrainAdminCommitImage(ctx *context.Context, form auth.CommitAdminImageCloudBrainForm) {
  815. if !NamePattern.MatchString(form.Tag) {
  816. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
  817. return
  818. }
  819. if utf8.RuneCountInString(form.Description) > 255 {
  820. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
  821. return
  822. }
  823. validTopics, errMessage := checkTopics(form.Topics)
  824. if errMessage != "" {
  825. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
  826. return
  827. }
  828. err := cloudbrain.CommitAdminImage(models.CommitImageParams{
  829. CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{
  830. ImageDescription: form.Description,
  831. ImageTag: form.Tag,
  832. },
  833. IsPrivate: form.IsPrivate,
  834. CloudBrainType: form.Type,
  835. Topics: validTopics,
  836. UID: ctx.User.ID,
  837. Type: models.GetRecommondType(form.IsRecommend),
  838. Place: form.Place,
  839. })
  840. if err != nil {
  841. log.Error("CommitImagefailed")
  842. if models.IsErrImageTagExist(err) {
  843. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist")))
  844. } else if models.IsErrorImageCommitting(err) {
  845. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing")))
  846. } else {
  847. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail")))
  848. }
  849. return
  850. }
  851. ctx.JSON(200, models.BaseOKMessage)
  852. }
  853. func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
  854. if !NamePattern.MatchString(form.Tag) {
  855. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
  856. return
  857. }
  858. if utf8.RuneCountInString(form.Description) > 255 {
  859. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
  860. return
  861. }
  862. validTopics, errMessage := checkTopics(form.Topics)
  863. if errMessage != "" {
  864. ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
  865. return
  866. }
  867. err := cloudbrain.CommitImage(ctx.Cloudbrain.JobID, models.CommitImageParams{
  868. CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{
  869. Ip: ctx.Cloudbrain.ContainerIp,
  870. TaskContainerId: ctx.Cloudbrain.ContainerID,
  871. ImageDescription: form.Description,
  872. ImageTag: form.Tag,
  873. },
  874. IsPrivate: form.IsPrivate,
  875. CloudBrainType: form.Type,
  876. Topics: validTopics,
  877. UID: ctx.User.ID,
  878. })
  879. if err != nil {
  880. log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"])
  881. if models.IsErrImageTagExist(err) {
  882. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist")))
  883. } else if models.IsErrorImageCommitting(err) {
  884. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing")))
  885. } else {
  886. ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail")))
  887. }
  888. return
  889. }
  890. ctx.JSON(200, models.BaseOKMessage)
  891. }
  892. func checkTopics(Topics string) ([]string, string) {
  893. var topics = make([]string, 0)
  894. var topicsStr = strings.TrimSpace(Topics)
  895. if len(topicsStr) > 0 {
  896. topics = strings.Split(topicsStr, ",")
  897. }
  898. validTopics, invalidTopics := models.SanitizeAndValidateImageTopics(topics)
  899. if len(validTopics) > 25 {
  900. return nil, "repo.topic.count_prompt"
  901. }
  902. if len(invalidTopics) > 0 {
  903. return nil, "repo.imagetopic.format_prompt"
  904. }
  905. return validTopics, ""
  906. }
  907. func CloudBrainStop(ctx *context.Context) {
  908. var ID = ctx.Params(":id")
  909. var resultCode = "0"
  910. var errorMsg = ""
  911. var status = ""
  912. task := ctx.Cloudbrain
  913. for {
  914. if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded) {
  915. log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
  916. resultCode = "-1"
  917. errorMsg = "cloudbrain.Already_stopped"
  918. break
  919. }
  920. err := cloudbrain.StopJob(task.JobID)
  921. if err != nil {
  922. log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
  923. resultCode = "-1"
  924. errorMsg = "cloudbrain.Stopped_failed"
  925. break
  926. }
  927. task.Status = string(models.JobStopped)
  928. if task.EndTime == 0 {
  929. task.EndTime = timeutil.TimeStampNow()
  930. }
  931. task.ComputeAndSetDuration()
  932. err = models.UpdateJob(task)
  933. if err != nil {
  934. log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
  935. resultCode = "-1"
  936. errorMsg = "cloudbrain.Stopped_success_update_status_fail"
  937. break
  938. }
  939. status = task.Status
  940. break
  941. }
  942. ctx.JSON(200, map[string]interface{}{
  943. "result_code": resultCode,
  944. "error_msg": ctx.Tr(errorMsg),
  945. "status": status,
  946. "id": ID,
  947. "StatusOK": 0,
  948. })
  949. }
  950. func StopJobsByUserID(userID int64) {
  951. cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
  952. if err != nil {
  953. log.Warn("Failed to get cloudBrain info", err)
  954. return
  955. }
  956. StopJobs(cloudBrains)
  957. }
  958. func StopJobsByRepoID(repoID int64) {
  959. cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
  960. if err != nil {
  961. log.Warn("Failed to get cloudBrain info", err)
  962. return
  963. }
  964. StopJobs(cloudBrains)
  965. }
  966. /**
  967. */
  968. func StopJobs(cloudBrains []*models.Cloudbrain) {
  969. for _, taskInfo := range cloudBrains {
  970. if taskInfo.Type == models.TypeCloudBrainOne {
  971. err := retry(3, time.Second*30, func() error {
  972. return cloudbrain.StopJob(taskInfo.JobID)
  973. })
  974. logErrorAndUpdateJobStatus(err, taskInfo)
  975. } else {
  976. if taskInfo.JobType == string(models.JobTypeTrain) {
  977. err := retry(3, time.Second*30, func() error {
  978. _, err := modelarts.StopTrainJob(taskInfo.JobID, strconv.FormatInt(taskInfo.VersionID, 10))
  979. return err
  980. })
  981. logErrorAndUpdateJobStatus(err, taskInfo)
  982. } else {
  983. param := models.NotebookAction{
  984. Action: models.ActionStop,
  985. }
  986. err := retry(3, time.Second*30, func() error {
  987. _, err := modelarts.ManageNotebook(taskInfo.JobID, param)
  988. return err
  989. })
  990. logErrorAndUpdateJobStatus(err, taskInfo)
  991. }
  992. }
  993. }
  994. }
  995. func retry(attempts int, sleep time.Duration, f func() error) (err error) {
  996. for i := 0; i < attempts; i++ {
  997. if i > 0 {
  998. log.Warn("retrying after error:", err)
  999. time.Sleep(sleep)
  1000. }
  1001. err = f()
  1002. if err == nil {
  1003. return nil
  1004. }
  1005. }
  1006. return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
  1007. }
  1008. func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
  1009. if err != nil {
  1010. log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
  1011. } else {
  1012. taskInfo.Status = string(models.JobStopped)
  1013. if taskInfo.EndTime == 0 {
  1014. taskInfo.EndTime = timeutil.TimeStampNow()
  1015. }
  1016. taskInfo.ComputeAndSetDuration()
  1017. err = models.UpdateJob(taskInfo)
  1018. if err != nil {
  1019. log.Warn("UpdateJob failed", err)
  1020. }
  1021. }
  1022. }
  1023. func CloudBrainDel(ctx *context.Context) {
  1024. var listType = ctx.Query("debugListType")
  1025. if err := deleteCloudbrainJob(ctx); err != nil {
  1026. log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
  1027. ctx.ServerError(err.Error(), err)
  1028. return
  1029. }
  1030. var isAdminPage = ctx.Query("isadminpage")
  1031. var isHomePage = ctx.Query("ishomepage")
  1032. if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
  1033. ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
  1034. } else if isHomePage == "true" {
  1035. ctx.Redirect(setting.AppSubURL + "/cloudbrains")
  1036. } else {
  1037. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=" + listType)
  1038. }
  1039. }
  1040. func deleteCloudbrainJob(ctx *context.Context) error {
  1041. task := ctx.Cloudbrain
  1042. if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) {
  1043. log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
  1044. return errors.New("the job has not been stopped")
  1045. }
  1046. err := models.DeleteJob(task)
  1047. if err != nil {
  1048. log.Error("DeleteJob failed: %v", err, ctx.Data["msgID"])
  1049. return err
  1050. }
  1051. deleteJobStorage(task.JobName, models.TypeCloudBrainOne)
  1052. return nil
  1053. }
  1054. func CloudBrainShowModels(ctx *context.Context) {
  1055. ctx.Data["PageIsCloudBrain"] = true
  1056. ID := ctx.Params(":id")
  1057. parentDir := ctx.Query("parentDir")
  1058. dirArray := strings.Split(parentDir, "/")
  1059. task, err := models.GetCloudbrainByID(ID)
  1060. if err != nil {
  1061. log.Error("no such job!", ctx.Data["msgID"])
  1062. ctx.ServerError("no such job:", err)
  1063. return
  1064. }
  1065. //get dirs
  1066. dirs, err := GetModelDirs(task.JobName, parentDir)
  1067. if err != nil {
  1068. log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  1069. ctx.ServerError("GetModelDirs failed:", err)
  1070. return
  1071. }
  1072. var fileInfos []storage.FileInfo
  1073. err = json.Unmarshal([]byte(dirs), &fileInfos)
  1074. if err != nil {
  1075. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  1076. ctx.ServerError("json.Unmarshal failed:", err)
  1077. return
  1078. }
  1079. for i, fileInfo := range fileInfos {
  1080. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  1081. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  1082. }
  1083. sort.Slice(fileInfos, func(i, j int) bool {
  1084. return fileInfos[i].ModTime > fileInfos[j].ModTime
  1085. })
  1086. ctx.Data["Path"] = dirArray
  1087. ctx.Data["Dirs"] = fileInfos
  1088. ctx.Data["task"] = task
  1089. ctx.Data["ID"] = ID
  1090. ctx.HTML(200, tplCloudBrainShowModels)
  1091. }
  1092. func GetPublicImages(ctx *context.Context) {
  1093. uid := getUID(ctx)
  1094. opts := models.SearchImageOptions{
  1095. IncludePublicOnly: true,
  1096. UID: uid,
  1097. Keyword: ctx.Query("q"),
  1098. Topics: ctx.Query("topic"),
  1099. IncludeOfficialOnly: ctx.QueryBool("recommend"),
  1100. SearchOrderBy: "type desc, num_stars desc,id desc",
  1101. Status: models.IMAGE_STATUS_SUCCESS,
  1102. CloudbrainType: ctx.QueryInt("cloudbrainType"),
  1103. }
  1104. getImages(ctx, &opts)
  1105. }
  1106. func GetCustomImages(ctx *context.Context) {
  1107. uid := getUID(ctx)
  1108. opts := models.SearchImageOptions{
  1109. UID: uid,
  1110. IncludeOwnerOnly: true,
  1111. Keyword: ctx.Query("q"),
  1112. Topics: ctx.Query("topic"),
  1113. Status: -1,
  1114. SearchOrderBy: "id desc",
  1115. }
  1116. getImages(ctx, &opts)
  1117. }
  1118. func GetStarImages(ctx *context.Context) {
  1119. uid := getUID(ctx)
  1120. opts := models.SearchImageOptions{
  1121. UID: uid,
  1122. IncludeStarByMe: true,
  1123. Keyword: ctx.Query("q"),
  1124. Topics: ctx.Query("topic"),
  1125. Status: models.IMAGE_STATUS_SUCCESS,
  1126. SearchOrderBy: "id desc",
  1127. }
  1128. getImages(ctx, &opts)
  1129. }
  1130. func getUID(ctx *context.Context) int64 {
  1131. var uid int64 = -1
  1132. if ctx.IsSigned {
  1133. uid = ctx.User.ID
  1134. }
  1135. return uid
  1136. }
  1137. func GetAllImages(ctx *context.Context) {
  1138. uid := getUID(ctx)
  1139. opts := models.SearchImageOptions{
  1140. UID: uid,
  1141. Keyword: ctx.Query("q"),
  1142. Topics: ctx.Query("topic"),
  1143. IncludeOfficialOnly: ctx.QueryBool("recommend"),
  1144. SearchOrderBy: "id desc",
  1145. Status: -1,
  1146. }
  1147. if ctx.Query("private") != "" {
  1148. if ctx.QueryBool("private") {
  1149. opts.IncludePrivateOnly = true
  1150. } else {
  1151. opts.IncludePublicOnly = true
  1152. }
  1153. }
  1154. getImages(ctx, &opts)
  1155. }
  1156. func getImages(ctx *context.Context, opts *models.SearchImageOptions) {
  1157. page := ctx.QueryInt("page")
  1158. if page <= 0 {
  1159. page = 1
  1160. }
  1161. pageSize := ctx.QueryInt("pageSize")
  1162. if pageSize <= 0 {
  1163. pageSize = 15
  1164. }
  1165. opts.ListOptions = models.ListOptions{
  1166. Page: page,
  1167. PageSize: pageSize,
  1168. }
  1169. imageList, total, err := models.SearchImage(opts)
  1170. if err != nil {
  1171. log.Error("Can not get images:%v", err)
  1172. ctx.JSON(http.StatusOK, models.ImagesPageResult{
  1173. Count: 0,
  1174. Images: []*models.Image{},
  1175. })
  1176. } else {
  1177. ctx.JSON(http.StatusOK, models.ImagesPageResult{
  1178. Count: total,
  1179. Images: imageList,
  1180. })
  1181. }
  1182. }
  1183. func GetModelDirs(jobName string, parentDir string) (string, error) {
  1184. var req string
  1185. modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/")
  1186. if parentDir == "" {
  1187. req = "baseDir=" + modelActualPath
  1188. } else {
  1189. req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
  1190. }
  1191. return getDirs(req)
  1192. }
  1193. func GetResultDirs(jobName string, parentDir string) (string, error) {
  1194. var req string
  1195. modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/")
  1196. if parentDir == "" {
  1197. req = "baseDir=" + modelActualPath
  1198. } else {
  1199. req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
  1200. }
  1201. return getDirs(req)
  1202. }
  1203. func CloudBrainDownloadModel(ctx *context.Context) {
  1204. parentDir := ctx.Query("parentDir")
  1205. fileName := ctx.Query("fileName")
  1206. jobName := ctx.Query("jobName")
  1207. filePath := "jobs/" + jobName + "/model/" + parentDir
  1208. url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
  1209. if err != nil {
  1210. log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
  1211. ctx.ServerError("PresignedGetURL", err)
  1212. return
  1213. }
  1214. ctx.Resp.Header().Set("Cache-Control", "max-age=0")
  1215. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  1216. }
  1217. func CloudBrainDownloadInferenceResult(ctx *context.Context) {
  1218. parentDir := ctx.Query("parentDir")
  1219. fileName := ctx.Query("fileName")
  1220. jobName := ctx.Query("jobName")
  1221. filePath := "jobs/" + jobName + "/result/" + parentDir
  1222. url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
  1223. if err != nil {
  1224. log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
  1225. ctx.ServerError("PresignedGetURL", err)
  1226. return
  1227. }
  1228. ctx.Resp.Header().Set("Cache-Control", "max-age=0")
  1229. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  1230. }
  1231. func GetRate(ctx *context.Context) {
  1232. isObjectDetcionAll := ctx.QueryBool("isObjectDetcionAll")
  1233. if isObjectDetcionAll {
  1234. ctx.Redirect(setting.BenchmarkServerHost + "?username=admin")
  1235. return
  1236. }
  1237. var ID = ctx.Params(":id")
  1238. job, err := models.GetCloudbrainByID(ID)
  1239. if err != nil {
  1240. ctx.ServerError("GetCloudbrainByJobID failed", err)
  1241. return
  1242. }
  1243. if job.JobType == string(models.JobTypeBenchmark) {
  1244. log.Info("url=" + setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
  1245. ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
  1246. } else if job.JobType == string(models.JobTypeSnn4imagenet) {
  1247. ctx.Redirect(setting.Snn4imagenetServerHost)
  1248. } else if job.JobType == string(models.JobTypeBrainScore) {
  1249. ctx.Redirect(setting.BrainScoreServerHost)
  1250. } else {
  1251. log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
  1252. }
  1253. }
  1254. func downloadCode(repo *models.Repository, codePath, branchName string) error {
  1255. //add "file:///" prefix to make the depth valid
  1256. if err := git.Clone(CLONE_FILE_PREFIX+repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName, Depth: 1}); err != nil {
  1257. log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
  1258. return err
  1259. }
  1260. configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
  1261. if err != nil {
  1262. log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
  1263. return err
  1264. }
  1265. defer configFile.Close()
  1266. pos := int64(0)
  1267. reader := bufio.NewReader(configFile)
  1268. for {
  1269. line, err := reader.ReadString('\n')
  1270. if err != nil {
  1271. if err == io.EOF {
  1272. log.Error("not find the remote-url")
  1273. return nil
  1274. } else {
  1275. log.Error("read error: %v", err)
  1276. return err
  1277. }
  1278. }
  1279. if strings.Contains(line, "url") && strings.Contains(line, ".git") {
  1280. originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
  1281. if len(line) > len(originUrl) {
  1282. originUrl += strings.Repeat(" ", len(line)-len(originUrl))
  1283. }
  1284. bytes := []byte(originUrl)
  1285. _, err := configFile.WriteAt(bytes, pos)
  1286. if err != nil {
  1287. log.Error("WriteAt failed:%v", err)
  1288. return err
  1289. }
  1290. break
  1291. }
  1292. pos += int64(len(line))
  1293. }
  1294. return nil
  1295. }
  1296. func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType, userName string) error {
  1297. err := os.MkdirAll(codePath, os.ModePerm)
  1298. if err != nil {
  1299. log.Error("mkdir codePath failed", err.Error())
  1300. return err
  1301. }
  1302. repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName)
  1303. if err != nil {
  1304. log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error())
  1305. return err
  1306. }
  1307. if err := git.Clone(CLONE_FILE_PREFIX+repoExt.RepoPath(), codePath, git.CloneRepoOptions{Depth: 1}); err != nil {
  1308. log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err)
  1309. return err
  1310. }
  1311. fileName := codePath + cloudbrain.TaskInfoName
  1312. f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
  1313. if err != nil {
  1314. log.Error("OpenFile failed", err.Error())
  1315. return err
  1316. }
  1317. defer f.Close()
  1318. data, err := json.Marshal(models.TaskInfo{
  1319. Username: userName,
  1320. TaskName: taskName,
  1321. CodeName: repo.Name,
  1322. BenchmarkCategory: strings.Split(benchmarkCategory, ","),
  1323. CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
  1324. GpuType: gpuType,
  1325. })
  1326. if err != nil {
  1327. log.Error("json.Marshal failed", err.Error())
  1328. return err
  1329. }
  1330. _, err = f.Write(data)
  1331. if err != nil {
  1332. log.Error("WriteString failed", err.Error())
  1333. return err
  1334. }
  1335. return nil
  1336. }
  1337. func uploadCodeToMinio(codePath, jobName, parentDir string) error {
  1338. files, err := readDir(codePath)
  1339. if err != nil {
  1340. log.Error("readDir(%s) failed: %s", codePath, err.Error())
  1341. return err
  1342. }
  1343. for _, file := range files {
  1344. if file.IsDir() {
  1345. if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil {
  1346. log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error())
  1347. return err
  1348. }
  1349. } else {
  1350. destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name()
  1351. sourceFile := codePath + file.Name()
  1352. err = storage.Attachments.UploadObject(destObject, sourceFile)
  1353. if err != nil {
  1354. log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error())
  1355. return err
  1356. }
  1357. }
  1358. }
  1359. return nil
  1360. }
  1361. func mkModelPath(modelPath string) error {
  1362. return mkPathAndReadMeFile(modelPath, "You can put the model file into this directory and download it by the web page.")
  1363. }
  1364. func mkResultPath(resultPath string) error {
  1365. return mkPathAndReadMeFile(resultPath, "You can put the result file into this directory and download it by the web page.")
  1366. }
  1367. func mkPathAndReadMeFile(path string, text string) error {
  1368. err := os.MkdirAll(path, os.ModePerm)
  1369. if err != nil {
  1370. log.Error("MkdirAll(%s) failed:%v", path, err)
  1371. return err
  1372. }
  1373. fileName := path + "README"
  1374. f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
  1375. if err != nil {
  1376. log.Error("OpenFile failed", err.Error())
  1377. return err
  1378. }
  1379. defer f.Close()
  1380. _, err = f.WriteString(text)
  1381. if err != nil {
  1382. log.Error("WriteString failed", err.Error())
  1383. return err
  1384. }
  1385. return nil
  1386. }
  1387. func deleteJobStorage(jobName string, cloudbrainType int) error {
  1388. //delete local
  1389. localJobPath := setting.JobPath + jobName
  1390. err := os.RemoveAll(localJobPath)
  1391. if err != nil {
  1392. log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
  1393. }
  1394. //delete oss
  1395. if cloudbrainType == models.TypeCloudBrainOne {
  1396. dirPath := setting.CBCodePathPrefix + jobName + "/"
  1397. err = storage.Attachments.DeleteDir(dirPath)
  1398. if err != nil {
  1399. log.Error("DeleteDir(%s) failed:%v", localJobPath, err)
  1400. }
  1401. } else if cloudbrainType == models.TypeCloudBrainTwo {
  1402. //dirPath := setting.CodePathPrefix + jobName + "/"
  1403. //err = storage.ObsRemoveObject(setting.Bucket, dirPath)
  1404. //if err != nil {
  1405. // log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err)
  1406. //}
  1407. log.Info("no need to delete")
  1408. } else {
  1409. log.Error("cloudbrainType(%d) error", cloudbrainType)
  1410. }
  1411. return nil
  1412. }
  1413. func SyncCloudbrainStatus() {
  1414. cloudBrains, err := models.GetCloudBrainUnStoppedJob()
  1415. if err != nil {
  1416. log.Error("GetCloudBrainUnStoppedJob failed:", err.Error())
  1417. return
  1418. }
  1419. for _, task := range cloudBrains {
  1420. if task.Type == models.TypeCloudBrainOne {
  1421. result, err := cloudbrain.GetJob(task.JobID)
  1422. if err != nil {
  1423. log.Error("GetJob(%s) failed:%v", task.JobName, err)
  1424. continue
  1425. }
  1426. if result != nil {
  1427. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  1428. taskRoles := jobRes.TaskRoles
  1429. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  1430. task.Status = taskRes.TaskStatuses[0].State
  1431. if task.Status != string(models.JobWaiting) {
  1432. models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)
  1433. err = models.UpdateJob(task)
  1434. if err != nil {
  1435. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1436. }
  1437. var maxDuration int64
  1438. if task.JobType == string(models.JobTypeBenchmark) {
  1439. maxDuration = setting.BenchmarkMaxDuration
  1440. } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
  1441. maxDuration = setting.ModelBenchmarkMaxDuration
  1442. } else {
  1443. maxDuration = setting.MaxDuration
  1444. }
  1445. if task.Duration >= maxDuration && task.JobType != string(models.JobTypeTrain) {
  1446. log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName)
  1447. err = cloudbrain.StopJob(task.JobID)
  1448. if err != nil {
  1449. log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err)
  1450. continue
  1451. }
  1452. task.Status = string(models.JobStopped)
  1453. if task.EndTime == 0 {
  1454. task.EndTime = timeutil.TimeStampNow()
  1455. }
  1456. task.ComputeAndSetDuration()
  1457. err = models.UpdateJob(task)
  1458. if err != nil {
  1459. log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
  1460. continue
  1461. }
  1462. }
  1463. }
  1464. }
  1465. } else if task.Type == models.TypeCloudBrainTwo {
  1466. if task.JobType == string(models.JobTypeDebug) {
  1467. //result, err := modelarts.GetJob(task.JobID)
  1468. result, err := modelarts.GetNotebook2(task.JobID)
  1469. if err != nil {
  1470. log.Error("GetJob(%s) failed:%v", task.JobName, err)
  1471. continue
  1472. }
  1473. if result != nil {
  1474. task.Status = result.Status
  1475. if task.StartTime == 0 && result.Lease.UpdateTime > 0 {
  1476. task.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000)
  1477. }
  1478. if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
  1479. task.EndTime = timeutil.TimeStampNow()
  1480. }
  1481. task.CorrectCreateUnix()
  1482. task.ComputeAndSetDuration()
  1483. err = models.UpdateJob(task)
  1484. if err != nil {
  1485. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1486. continue
  1487. }
  1488. }
  1489. } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
  1490. result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
  1491. if err != nil {
  1492. log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
  1493. continue
  1494. }
  1495. if result != nil {
  1496. task.Status = modelarts.TransTrainJobStatus(result.IntStatus)
  1497. task.Duration = result.Duration / 1000
  1498. task.TrainJobDuration = result.TrainJobDuration
  1499. if task.StartTime == 0 && result.StartTime > 0 {
  1500. task.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
  1501. }
  1502. task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
  1503. if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
  1504. task.EndTime = task.StartTime.Add(task.Duration)
  1505. }
  1506. task.CorrectCreateUnix()
  1507. err = models.UpdateJob(task)
  1508. if err != nil {
  1509. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1510. continue
  1511. }
  1512. }
  1513. } else {
  1514. log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType)
  1515. }
  1516. } else if task.Type == models.TypeC2Net {
  1517. result, err := grampus.GetJob(task.JobID)
  1518. if err != nil {
  1519. log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
  1520. continue
  1521. }
  1522. if result != nil {
  1523. if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
  1524. task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
  1525. }
  1526. task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
  1527. task.Duration = result.JobInfo.RunSec
  1528. task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
  1529. if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
  1530. task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
  1531. }
  1532. if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
  1533. task.EndTime = task.StartTime.Add(task.Duration)
  1534. }
  1535. task.CorrectCreateUnix()
  1536. err = models.UpdateJob(task)
  1537. if err != nil {
  1538. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1539. continue
  1540. }
  1541. }
  1542. } else {
  1543. log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
  1544. }
  1545. }
  1546. return
  1547. }
  1548. func HandleTaskWithNoDuration(ctx *context.Context) {
  1549. mode := ctx.Query("mode")
  1550. log.Info("HandleTaskWithNoDuration start")
  1551. count := 0
  1552. start := time.Now().Unix()
  1553. for {
  1554. var cloudBrains []*models.Cloudbrain
  1555. var err error
  1556. if mode == "1" {
  1557. cloudBrains, err = models.GetStoppedJobWithNoStartTimeEndTime()
  1558. } else {
  1559. cloudBrains, err = models.GetStoppedJobWithNoDurationJob()
  1560. }
  1561. if err != nil {
  1562. log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error())
  1563. break
  1564. }
  1565. if len(cloudBrains) == 0 {
  1566. log.Info("HandleTaskWithNoTrainJobDuration:no task need handle")
  1567. break
  1568. }
  1569. handleNoDurationTask(cloudBrains)
  1570. count += len(cloudBrains)
  1571. if len(cloudBrains) < 100 {
  1572. log.Info("HandleTaskWithNoTrainJobDuration:task less than 100")
  1573. break
  1574. }
  1575. if time.Now().Unix()-start > 600 {
  1576. log.Info("HandleTaskWithNoDuration : time out")
  1577. ctx.JSON(200, fmt.Sprintf("task stop for time out,count=%d", count))
  1578. return
  1579. }
  1580. }
  1581. log.Info("HandleTaskWithNoTrainJobDuration:count=%d", count)
  1582. ctx.JSON(200, fmt.Sprintf("success,count=%d", count))
  1583. }
  1584. func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
  1585. for _, task := range cloudBrains {
  1586. time.Sleep(time.Millisecond * 100)
  1587. log.Info("Handle job ,%+v", task)
  1588. if task.Type == models.TypeCloudBrainOne {
  1589. result, err := cloudbrain.GetJob(task.JobID)
  1590. if err != nil {
  1591. log.Error("GetJob(%s) failed:%v", task.JobName, err)
  1592. updateDefaultDuration(task)
  1593. continue
  1594. }
  1595. if result != nil {
  1596. if result.Msg != "success" {
  1597. updateDefaultDuration(task)
  1598. continue
  1599. }
  1600. jobRes, err := models.ConvertToJobResultPayload(result.Payload)
  1601. if err != nil || len(jobRes.TaskRoles) == 0 {
  1602. updateDefaultDuration(task)
  1603. continue
  1604. }
  1605. taskRoles := jobRes.TaskRoles
  1606. taskRes, err := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  1607. if err != nil || len(taskRes.TaskStatuses) == 0 {
  1608. updateDefaultDuration(task)
  1609. continue
  1610. }
  1611. task.Status = taskRes.TaskStatuses[0].State
  1612. log.Info("task startTime = %v endTime= %v ,jobId=%d", jobRes.JobStatus.StartTime, jobRes.JobStatus.EndTime, task.ID)
  1613. if jobRes.JobStatus.CreatedTime > 0 {
  1614. task.StartTime = timeutil.TimeStamp(jobRes.JobStatus.CreatedTime / 1000)
  1615. if jobRes.JobStatus.CompletedTime > 0 {
  1616. task.EndTime = timeutil.TimeStamp(jobRes.JobStatus.CompletedTime / 1000)
  1617. } else {
  1618. task.EndTime = task.UpdatedUnix
  1619. }
  1620. } else {
  1621. task.StartTime = 0
  1622. task.EndTime = 0
  1623. }
  1624. if task.EndTime < task.StartTime {
  1625. log.Info("endTime[%v] is less than starTime[%v],jobId=%d", task.EndTime, task.StartTime, task.ID)
  1626. st := task.StartTime
  1627. task.StartTime = task.EndTime
  1628. task.EndTime = st
  1629. }
  1630. task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
  1631. task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
  1632. err = models.UpdateJobDurationWithDeleted(task)
  1633. if err != nil {
  1634. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1635. }
  1636. }
  1637. } else if task.Type == models.TypeCloudBrainTwo {
  1638. if task.JobType == string(models.JobTypeDebug) {
  1639. //result, err := modelarts.GetJob(task.JobID)
  1640. result, err := modelarts.GetNotebook2(task.JobID)
  1641. if err != nil {
  1642. log.Error("GetJob(%s) failed:%v", task.JobName, err)
  1643. updateDefaultDuration(task)
  1644. continue
  1645. }
  1646. if result != nil {
  1647. task.Status = result.Status
  1648. startTime := result.Lease.CreateTime
  1649. duration := result.Lease.Duration / 1000
  1650. if startTime > 0 {
  1651. task.StartTime = timeutil.TimeStamp(startTime / 1000)
  1652. task.EndTime = task.StartTime.Add(duration)
  1653. }
  1654. task.CorrectCreateUnix()
  1655. task.ComputeAndSetDuration()
  1656. err = models.UpdateJobDurationWithDeleted(task)
  1657. if err != nil {
  1658. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1659. continue
  1660. }
  1661. }
  1662. } else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
  1663. result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
  1664. if err != nil {
  1665. log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
  1666. updateDefaultDuration(task)
  1667. continue
  1668. }
  1669. if result != nil {
  1670. startTime := result.StartTime / 1000
  1671. if startTime > 0 {
  1672. task.StartTime = timeutil.TimeStamp(startTime)
  1673. task.EndTime = task.StartTime.Add(result.Duration / 1000)
  1674. }
  1675. task.ComputeAndSetDuration()
  1676. err = models.UpdateJobDurationWithDeleted(task)
  1677. if err != nil {
  1678. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1679. continue
  1680. }
  1681. }
  1682. } else {
  1683. log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType)
  1684. }
  1685. } else {
  1686. log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
  1687. }
  1688. }
  1689. }
  1690. func updateDefaultDuration(task *models.Cloudbrain) {
  1691. log.Info("updateDefaultDuration: taskId=%d", task.ID)
  1692. task.StartTime = task.CreatedUnix
  1693. task.EndTime = task.UpdatedUnix
  1694. task.ComputeAndSetDuration()
  1695. err := models.UpdateJobDurationWithDeleted(task)
  1696. if err != nil {
  1697. log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
  1698. }
  1699. }
  1700. func CloudBrainBenchmarkIndex(ctx *context.Context) {
  1701. MustEnableCloudbrain(ctx)
  1702. repo := ctx.Repo.Repository
  1703. page := ctx.QueryInt("page")
  1704. if page <= 0 {
  1705. page = 1
  1706. }
  1707. var jobTypes []string
  1708. jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
  1709. ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
  1710. ListOptions: models.ListOptions{
  1711. Page: page,
  1712. PageSize: setting.UI.IssuePagingNum,
  1713. },
  1714. RepoID: repo.ID,
  1715. Type: models.TypeCloudBrainOne,
  1716. JobTypes: jobTypes,
  1717. })
  1718. if err != nil {
  1719. ctx.ServerError("Get debugjob faild:", err)
  1720. return
  1721. }
  1722. for i, task := range ciTasks {
  1723. ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
  1724. ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
  1725. if ciTasks[i].TrainJobDuration == "" {
  1726. if ciTasks[i].Duration == 0 {
  1727. var duration int64
  1728. if task.Status == string(models.JobRunning) {
  1729. duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix)
  1730. } else {
  1731. duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix)
  1732. }
  1733. ciTasks[i].Duration = duration
  1734. }
  1735. ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(ciTasks[i].Duration)
  1736. }
  1737. ciTasks[i].BenchmarkTypeName = ""
  1738. if ciTasks[i].JobType == string(models.JobTypeBenchmark) {
  1739. ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
  1740. } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) {
  1741. ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
  1742. ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType
  1743. if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) {
  1744. ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost
  1745. } else {
  1746. ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost
  1747. }
  1748. }
  1749. if task.BenchmarkTypeID > 0 {
  1750. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  1751. if task.BenchmarkTypeID == benchmarkType.Id {
  1752. ciTasks[i].BenchmarkTypeRankLink = benchmarkType.RankLink
  1753. ciTasks[i].BenchmarkTypeName = benchmarkType.First
  1754. break
  1755. }
  1756. }
  1757. }
  1758. }
  1759. pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
  1760. ctx.Data["Page"] = pager
  1761. ctx.Data["PageIsCloudBrain"] = true
  1762. ctx.Data["Tasks"] = ciTasks
  1763. ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx)
  1764. ctx.Data["RepoIsEmpty"] = repo.IsEmpty
  1765. ctx.HTML(200, tplCloudBrainBenchmarkIndex)
  1766. }
  1767. func GetChildTypes(ctx *context.Context) {
  1768. benchmarkTypeID := ctx.QueryInt("benchmark_type_id")
  1769. re := make(map[string]interface{})
  1770. for {
  1771. var isExist bool
  1772. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  1773. if benchmarkTypeID == benchmarkType.Id {
  1774. isExist = true
  1775. re["child_types"] = benchmarkType.Second
  1776. re["result_code"] = "0"
  1777. break
  1778. }
  1779. }
  1780. if !isExist {
  1781. re["result_code"] = "1"
  1782. log.Error("no such benchmark_type_id", ctx.Data["MsgID"])
  1783. re["errMsg"] = "system error"
  1784. break
  1785. }
  1786. break
  1787. }
  1788. ctx.JSON(200, re)
  1789. }
  1790. func CloudBrainBenchmarkNew(ctx *context.Context) {
  1791. ctx.Data["description"] = ""
  1792. ctx.Data["benchmarkTypeID"] = -1
  1793. ctx.Data["benchmark_child_types_id_hidden"] = -1
  1794. err := cloudBrainNewDataPrepare(ctx)
  1795. if err != nil {
  1796. ctx.ServerError("get new cloudbrain info failed", err)
  1797. return
  1798. }
  1799. ctx.HTML(200, tplCloudBrainBenchmarkNew)
  1800. }
  1801. func getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID int, ctx *context.Context) (*models.BenchmarkDataset, error) {
  1802. var childInfo *models.BenchmarkDataset
  1803. var isExist bool
  1804. for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
  1805. if benchmarkType.Id == benchmarkTypeID {
  1806. for _, childType := range benchmarkType.Second {
  1807. if childType.Id == benchmarkChildTypeID {
  1808. childInfo = childType
  1809. isExist = true
  1810. break
  1811. }
  1812. }
  1813. break
  1814. }
  1815. }
  1816. if !isExist {
  1817. log.Error("no such benchmark_type_id&benchmark_child_type_id")
  1818. return childInfo, errors.New("no such benchmark_type_id&benchmark_child_type_id")
  1819. }
  1820. return childInfo, nil
  1821. }
  1822. func getBenchmarkGpuQueue(gpuQueue string) (string, error) {
  1823. queue := ""
  1824. if benchmarkGpuInfos == nil {
  1825. if err := json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos); err != nil {
  1826. log.Error("json.Unmarshal BenchmarkGpuTypes(%s) failed:%v", setting.BenchmarkGpuTypes, err)
  1827. return queue, err
  1828. }
  1829. }
  1830. var isExist bool
  1831. for _, gpuInfo := range benchmarkGpuInfos.GpuInfo {
  1832. if gpuQueue == gpuInfo.Queue {
  1833. isExist = true
  1834. queue = gpuQueue
  1835. break
  1836. }
  1837. }
  1838. if !isExist {
  1839. log.Error("no such gpuQueue, %s", gpuQueue)
  1840. return queue, errors.New("no such gpuQueue")
  1841. }
  1842. return queue, nil
  1843. }
  1844. func getBenchmarkResourceSpec(resourceSpecID int) (int, error) {
  1845. var id int
  1846. if benchmarkResourceSpecs == nil {
  1847. if err := json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs); err != nil {
  1848. log.Error("json.Unmarshal BenchmarkResourceSpecs(%s) failed:%v", setting.BenchmarkResourceSpecs, err)
  1849. return id, err
  1850. }
  1851. }
  1852. var isExist bool
  1853. for _, resourceSpec := range benchmarkResourceSpecs.ResourceSpec {
  1854. if resourceSpecID == resourceSpec.Id {
  1855. isExist = true
  1856. id = resourceSpecID
  1857. break
  1858. }
  1859. }
  1860. if !isExist {
  1861. log.Error("no such resourceSpecID, %d", resourceSpecID)
  1862. return id, errors.New("no such resourceSpec")
  1863. }
  1864. return id, nil
  1865. }
  1866. func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  1867. jobType := form.JobType
  1868. if jobType == string(models.JobTypeBenchmark) {
  1869. BenchMarkAlgorithmCreate(ctx, form)
  1870. } else {
  1871. ModelBenchmarkCreate(ctx, form)
  1872. }
  1873. }
  1874. func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  1875. ctx.Data["PageIsCloudBrain"] = true
  1876. displayJobName := form.DisplayJobName
  1877. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  1878. image := strings.TrimSpace(form.Image)
  1879. gpuQueue := form.GpuType
  1880. command := cloudbrain.CommandBenchmark
  1881. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  1882. resourceSpecId := cloudbrain.BenchMarkResourceID
  1883. benchmarkTypeID := form.BenchmarkTypeID
  1884. benchmarkChildTypeID := form.BenchmarkChildTypeID
  1885. ctx.Data["description"] = form.Description
  1886. ctx.Data["benchmarkTypeID"] = benchmarkTypeID
  1887. ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID
  1888. repo := ctx.Repo.Repository
  1889. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName)
  1890. if err == nil {
  1891. if len(tasks) != 0 {
  1892. log.Error("the job name did already exist", ctx.Data["MsgID"])
  1893. cloudBrainNewDataPrepare(ctx)
  1894. ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, &form)
  1895. return
  1896. }
  1897. } else {
  1898. if !models.IsErrJobNotExist(err) {
  1899. log.Error("system error, %v", err, ctx.Data["MsgID"])
  1900. cloudBrainNewDataPrepare(ctx)
  1901. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1902. return
  1903. }
  1904. }
  1905. if !jobNamePattern.MatchString(jobName) {
  1906. cloudBrainNewDataPrepare(ctx)
  1907. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form)
  1908. return
  1909. }
  1910. childInfo, err := getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID, ctx)
  1911. if err != nil {
  1912. log.Error("getBenchmarkAttachment failed:%v", err, ctx.Data["MsgID"])
  1913. cloudBrainNewDataPrepare(ctx)
  1914. ctx.RenderWithErr("benchmark type error", tplCloudBrainBenchmarkNew, &form)
  1915. return
  1916. }
  1917. _, err = getBenchmarkGpuQueue(gpuQueue)
  1918. if err != nil {
  1919. log.Error("getBenchmarkGpuQueue failed:%v", err, ctx.Data["MsgID"])
  1920. cloudBrainNewDataPrepare(ctx)
  1921. ctx.RenderWithErr("gpu queue error", tplCloudBrainBenchmarkNew, &form)
  1922. return
  1923. }
  1924. _, err = getBenchmarkResourceSpec(resourceSpecId)
  1925. if err != nil {
  1926. log.Error("getBenchmarkResourceSpec failed:%v", err, ctx.Data["MsgID"])
  1927. cloudBrainNewDataPrepare(ctx)
  1928. ctx.RenderWithErr("resource spec error", tplCloudBrainBenchmarkNew, &form)
  1929. return
  1930. }
  1931. count, err := models.GetBenchmarkCountByUserID(ctx.User.ID)
  1932. if err != nil {
  1933. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  1934. cloudBrainNewDataPrepare(ctx)
  1935. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1936. return
  1937. } else {
  1938. if count >= 1 {
  1939. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  1940. cloudBrainNewDataPrepare(ctx)
  1941. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tplCloudBrainBenchmarkNew, &form)
  1942. return
  1943. }
  1944. }
  1945. os.RemoveAll(codePath)
  1946. if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil {
  1947. log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"])
  1948. cloudBrainNewDataPrepare(ctx)
  1949. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1950. return
  1951. }
  1952. if _, err := os.Stat(codePath + "/train.py"); err != nil {
  1953. if os.IsNotExist(err) {
  1954. // file does not exist
  1955. log.Error("train.py does not exist, %v", err, ctx.Data["MsgID"])
  1956. cloudBrainNewDataPrepare(ctx)
  1957. ctx.RenderWithErr("train.py does not exist", tplCloudBrainBenchmarkNew, &form)
  1958. } else {
  1959. log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
  1960. cloudBrainNewDataPrepare(ctx)
  1961. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1962. }
  1963. return
  1964. } else if _, err := os.Stat(codePath + "/test.py"); err != nil {
  1965. if os.IsNotExist(err) {
  1966. // file does not exist
  1967. log.Error("test.py does not exist, %v", err, ctx.Data["MsgID"])
  1968. cloudBrainNewDataPrepare(ctx)
  1969. ctx.RenderWithErr("test.py does not exist", tplCloudBrainBenchmarkNew, &form)
  1970. } else {
  1971. log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
  1972. cloudBrainNewDataPrepare(ctx)
  1973. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1974. }
  1975. return
  1976. }
  1977. if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil {
  1978. log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
  1979. cloudBrainNewDataPrepare(ctx)
  1980. ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1981. return
  1982. }
  1983. benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
  1984. var gpuType string
  1985. for _, gpuInfo := range gpuInfos.GpuInfo {
  1986. if gpuInfo.Queue == gpuQueue {
  1987. gpuType = gpuInfo.Value
  1988. }
  1989. }
  1990. if err := downloadRateCode(repo, jobName, childInfo.Owner, childInfo.RepoName, benchmarkPath, form.BenchmarkCategory, gpuType, ctx.User.Name); err != nil {
  1991. log.Error("downloadRateCode failed, %v", err, ctx.Data["MsgID"])
  1992. //cloudBrainNewDataPrepare(ctx)
  1993. //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  1994. //return
  1995. }
  1996. if err := uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/"); err != nil {
  1997. log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
  1998. //cloudBrainNewDataPrepare(ctx)
  1999. //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
  2000. //return
  2001. }
  2002. uuid := childInfo.Attachment
  2003. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
  2004. if err != nil {
  2005. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  2006. cloudBrainNewDataPrepare(ctx)
  2007. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplCloudBrainBenchmarkNew, &form)
  2008. return
  2009. }
  2010. req := cloudbrain.GenerateCloudBrainTaskReq{
  2011. Ctx: ctx,
  2012. DisplayJobName: displayJobName,
  2013. JobName: jobName,
  2014. Image: image,
  2015. Command: command,
  2016. Uuids: uuid,
  2017. DatasetNames: datasetNames,
  2018. DatasetInfos: datasetInfos,
  2019. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  2020. ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
  2021. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  2022. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  2023. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  2024. JobType: string(models.JobTypeBenchmark),
  2025. GpuQueue: gpuQueue,
  2026. Description: form.Description,
  2027. BranchName: cloudbrain.DefaultBranchName,
  2028. BootFile: "",
  2029. Params: "",
  2030. CommitID: "",
  2031. BenchmarkTypeID: benchmarkTypeID,
  2032. BenchmarkChildTypeID: benchmarkChildTypeID,
  2033. ResourceSpecId: resourceSpecId,
  2034. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  2035. }
  2036. err = cloudbrain.GenerateTask(req)
  2037. if err != nil {
  2038. cloudBrainNewDataPrepare(ctx)
  2039. ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form)
  2040. return
  2041. }
  2042. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
  2043. }
  2044. func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  2045. ctx.Data["PageIsCloudBrain"] = true
  2046. displayJobName := form.DisplayJobName
  2047. jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
  2048. image := form.Image
  2049. uuid := form.Attachment
  2050. jobType := form.JobType
  2051. gpuQueue := form.GpuType
  2052. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  2053. resourceSpecId := form.ResourceSpecId
  2054. branchName := cloudbrain.DefaultBranchName
  2055. repo := ctx.Repo.Repository
  2056. tpl := tplCloudBrainBenchmarkNew
  2057. command := cloudbrain.GetCloudbrainDebugCommand()
  2058. tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
  2059. if err == nil {
  2060. if len(tasks) != 0 {
  2061. log.Error("the job name did already exist", ctx.Data["MsgID"])
  2062. cloudBrainNewDataPrepare(ctx)
  2063. ctx.RenderWithErr("the job name did already exist", tpl, &form)
  2064. return
  2065. }
  2066. } else {
  2067. if !models.IsErrJobNotExist(err) {
  2068. log.Error("system error, %v", err, ctx.Data["MsgID"])
  2069. cloudBrainNewDataPrepare(ctx)
  2070. ctx.RenderWithErr("system error", tpl, &form)
  2071. return
  2072. }
  2073. }
  2074. if !jobNamePattern.MatchString(displayJobName) {
  2075. cloudBrainNewDataPrepare(ctx)
  2076. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
  2077. return
  2078. }
  2079. if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
  2080. log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
  2081. cloudBrainNewDataPrepare(ctx)
  2082. ctx.RenderWithErr("jobtype error", tpl, &form)
  2083. return
  2084. }
  2085. count, err := models.GetBenchmarkCountByUserID(ctx.User.ID)
  2086. if err != nil {
  2087. log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
  2088. cloudBrainNewDataPrepare(ctx)
  2089. ctx.RenderWithErr("system error", tpl, &form)
  2090. return
  2091. } else {
  2092. if count >= 1 {
  2093. log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
  2094. cloudBrainNewDataPrepare(ctx)
  2095. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
  2096. return
  2097. }
  2098. }
  2099. downloadCode(repo, codePath, branchName)
  2100. uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
  2101. modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
  2102. mkModelPath(modelPath)
  2103. uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")
  2104. snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
  2105. if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
  2106. downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name)
  2107. uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/")
  2108. command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description))
  2109. }
  2110. benchmarkChildTypeID := 0
  2111. brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
  2112. if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
  2113. downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name)
  2114. uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/")
  2115. benchmarkChildTypeID = form.BenchmarkChildTypeID
  2116. command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
  2117. }
  2118. datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
  2119. if err != nil {
  2120. log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
  2121. cloudBrainNewDataPrepare(ctx)
  2122. ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
  2123. return
  2124. }
  2125. req := cloudbrain.GenerateCloudBrainTaskReq{
  2126. Ctx: ctx,
  2127. DisplayJobName: displayJobName,
  2128. JobName: jobName,
  2129. Image: image,
  2130. Command: command,
  2131. Uuids: uuid,
  2132. DatasetNames: datasetNames,
  2133. DatasetInfos: datasetInfos,
  2134. CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
  2135. ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
  2136. BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
  2137. Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
  2138. BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
  2139. JobType: jobType,
  2140. GpuQueue: gpuQueue,
  2141. Description: form.Description,
  2142. BranchName: branchName,
  2143. BootFile: form.BootFile,
  2144. Params: form.Params,
  2145. CommitID: "",
  2146. BenchmarkTypeID: 0,
  2147. BenchmarkChildTypeID: benchmarkChildTypeID,
  2148. ResourceSpecId: resourceSpecId,
  2149. ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
  2150. }
  2151. err = cloudbrain.GenerateTask(req)
  2152. if err != nil {
  2153. cloudBrainNewDataPrepare(ctx)
  2154. ctx.RenderWithErr(err.Error(), tpl, &form)
  2155. return
  2156. }
  2157. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
  2158. }
  2159. func getBrainRegion(benchmarkChildTypeID int) string {
  2160. values := []string{"V1", "V2", "V4", "IT"}
  2161. return values[benchmarkChildTypeID]
  2162. }
  2163. func trimSpaceNewlineInString(s string) string {
  2164. re := regexp.MustCompile(`\r?\n`)
  2165. return re.ReplaceAllString(s, " ")
  2166. }
  2167. func BenchmarkDel(ctx *context.Context) {
  2168. if err := deleteCloudbrainJob(ctx); err != nil {
  2169. log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
  2170. ctx.ServerError(err.Error(), err)
  2171. return
  2172. }
  2173. var isAdminPage = ctx.Query("isadminpage")
  2174. var isHomePage = ctx.Query("ishomepage")
  2175. if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
  2176. ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
  2177. } else if isHomePage == "true" {
  2178. ctx.Redirect(setting.AppSubURL + "/cloudbrains")
  2179. } else {
  2180. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
  2181. }
  2182. }
  2183. func CloudBrainTrainJobNew(ctx *context.Context) {
  2184. err := cloudBrainNewDataPrepare(ctx)
  2185. if err != nil {
  2186. ctx.ServerError("get new train-job info failed", err)
  2187. return
  2188. }
  2189. ctx.HTML(http.StatusOK, tplCloudBrainTrainJobNew)
  2190. }
  2191. func InferenceCloudBrainJobNew(ctx *context.Context) {
  2192. err := cloudBrainNewDataPrepare(ctx)
  2193. if err != nil {
  2194. ctx.ServerError("get new train-job info failed", err)
  2195. return
  2196. }
  2197. ctx.HTML(http.StatusOK, tplCloudBrainInferenceJobNew)
  2198. }
  2199. func InferenceCloudBrainJobShow(ctx *context.Context) {
  2200. cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference)
  2201. }
  2202. func DownloadInferenceResultFile(ctx *context.Context) {
  2203. var jobID = ctx.Params(":jobid")
  2204. var versionName = ctx.Query("version_name")
  2205. task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
  2206. if err != nil {
  2207. log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
  2208. return
  2209. }
  2210. allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, task.ResultUrl)
  2211. returnFileName := task.DisplayJobName + ".zip"
  2212. MinioDownloadManyFile(task.ResultUrl, ctx, returnFileName, allFile)
  2213. }
  2214. func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) {
  2215. var command string
  2216. bootFile := strings.TrimSpace(form.BootFile)
  2217. params := form.Params
  2218. if !strings.HasSuffix(bootFile, ".py") {
  2219. log.Error("bootFile(%s) format error", bootFile)
  2220. return command, errors.New("bootFile format error")
  2221. }
  2222. var parameters models.Parameters
  2223. var param string
  2224. if len(params) != 0 {
  2225. err := json.Unmarshal([]byte(params), &parameters)
  2226. if err != nil {
  2227. log.Error("Failed to Unmarshal params: %s (%v)", params, err)
  2228. return command, err
  2229. }
  2230. for _, parameter := range parameters.Parameter {
  2231. param += " --" + parameter.Label + "=" + parameter.Value
  2232. }
  2233. }
  2234. param += " --modelname" + "=" + form.CkptName
  2235. command += "python /code/" + bootFile + param + " > " + cloudbrain.ResultPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile
  2236. return command, nil
  2237. }
  2238. func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) {
  2239. var command string
  2240. bootFile := strings.TrimSpace(form.BootFile)
  2241. params := form.Params
  2242. if !strings.HasSuffix(bootFile, ".py") {
  2243. log.Error("bootFile(%s) format error", bootFile)
  2244. return command, errors.New("bootFile format error")
  2245. }
  2246. var parameters models.Parameters
  2247. var param string
  2248. if len(params) != 0 {
  2249. err := json.Unmarshal([]byte(params), &parameters)
  2250. if err != nil {
  2251. log.Error("Failed to Unmarshal params: %s (%v)", params, err)
  2252. return command, err
  2253. }
  2254. for _, parameter := range parameters.Parameter {
  2255. param += " --" + parameter.Label + "=" + parameter.Value
  2256. }
  2257. }
  2258. command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile
  2259. return command, nil
  2260. }
  2261. func CloudBrainTrainJobDel(ctx *context.Context) {
  2262. var listType = ctx.Query("listType")
  2263. if err := deleteCloudbrainJob(ctx); err != nil {
  2264. log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
  2265. ctx.ServerError(err.Error(), err)
  2266. return
  2267. }
  2268. var isAdminPage = ctx.Query("isadminpage")
  2269. var isHomePage = ctx.Query("ishomepage")
  2270. if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
  2271. ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
  2272. } else if isHomePage == "true" {
  2273. ctx.Redirect(setting.AppSubURL + "/cloudbrains")
  2274. } else {
  2275. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType)
  2276. }
  2277. }
  2278. func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes {
  2279. var lang = ctx.Locale.Language()
  2280. if benchmarkTypesMap[lang] == nil {
  2281. var val = i18n.Tr(lang, BENCHMARK_TYPE_CODE)
  2282. //use config
  2283. val = setting.BenchmarkTypes
  2284. var tempType *models.BenchmarkTypes
  2285. if err := json.Unmarshal([]byte(val), &tempType); err != nil {
  2286. log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", val, err, ctx.Data["MsgID"])
  2287. return &models.BenchmarkTypes{}
  2288. }
  2289. benchmarkTypesMap[lang] = tempType
  2290. }
  2291. return benchmarkTypesMap[lang]
  2292. }