Browse Source

Merge remote-tracking branch 'origin/feature_rzs' into feature_wq

# Conflicts:
#	go.mod
feature_wq
qiwang 8 months ago
parent
commit
e3f915c9ab
30 changed files with 1991 additions and 1009 deletions
  1. +28
    -28
      go.mod
  2. +52
    -81
      go.sum
  3. +63
    -0
      sdks/hpc/client.go
  4. +5
    -0
      sdks/hpc/config.go
  5. +73
    -0
      sdks/hpc/job.go
  6. +525
    -0
      sdks/hpc/models.go
  7. +16
    -0
      sdks/pcmscheduler/job.go
  8. +25
    -11
      sdks/pcmscheduler/jobset.go
  9. +49
    -28
      sdks/pcmscheduler/models.go
  10. +62
    -9
      sdks/scheduler/models.go
  11. +190
    -1
      sdks/sdks.go
  12. +42
    -94
      sdks/storage/cdsapi/bucket.go
  13. +12
    -25
      sdks/storage/cdsapi/cache.go
  14. +2
    -2
      sdks/storage/cdsapi/client.go
  15. +3
    -1
      sdks/storage/cdsapi/config.go
  16. +12
    -26
      sdks/storage/cdsapi/hub.go
  17. +5
    -5
      sdks/storage/cdsapi/hub_io.go
  18. +176
    -274
      sdks/storage/cdsapi/object.go
  19. +81
    -173
      sdks/storage/cdsapi/package.go
  20. +18
    -14
      sdks/storage/cdsapi/presigned.go
  21. +42
    -14
      sdks/storage/cdsapi/presigned_test.go
  22. +114
    -0
      sdks/storage/cdsapi/signer.go
  23. +34
    -80
      sdks/storage/cdsapi/storage.go
  24. +56
    -0
      sdks/storage/cdsapi/storage_test.go
  25. +21
    -45
      sdks/storage/cdsapi/user.go
  26. +159
    -0
      sdks/storage/cdsapi/utils.go
  27. +49
    -44
      sdks/uploader/models.go
  28. +53
    -54
      sdks/uploader/uploader.go
  29. +14
    -0
      utils/io2/io.go
  30. +10
    -0
      utils/serder/serder.go

+ 28
- 28
go.mod View File

@@ -1,6 +1,6 @@
module gitlink.org.cn/cloudream/common

go 1.22
go 1.23.0

toolchain go1.23.2

@@ -8,8 +8,7 @@ require (
github.com/antonfisher/nested-logrus-formatter v1.3.1
github.com/aws/aws-sdk-go-v2 v1.36.3
github.com/aws/aws-sdk-go-v2/credentials v1.17.62
github.com/google/go-querystring v1.1.0
github.com/google/uuid v1.3.0
github.com/google/uuid v1.6.0
github.com/hashicorp/go-multierror v1.1.1
github.com/imdario/mergo v0.3.15
github.com/json-iterator/go v1.1.12
@@ -17,40 +16,41 @@ require (
github.com/mitchellh/mapstructure v1.5.0
github.com/modern-go/reflect2 v1.0.2
github.com/otiai10/copy v1.12.0
github.com/samber/lo v1.36.0
github.com/sirupsen/logrus v1.9.2
github.com/smartystreets/goconvey v1.8.0
github.com/samber/lo v1.49.1
github.com/sirupsen/logrus v1.9.3
github.com/smartystreets/goconvey v1.8.1
github.com/streadway/amqp v1.1.0
github.com/zyedidia/generic v1.2.1
go.etcd.io/etcd/client/v3 v3.5.9
golang.org/x/exp v0.0.0-20230519143937-03e91628a987
go.etcd.io/etcd/client/v3 v3.5.12
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa
)

require github.com/aws/smithy-go v1.22.2 // indirect
require (
github.com/aws/smithy-go v1.22.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/smarty/assertions v1.15.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
)

require (
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-querystring v1.1.0
github.com/gopherjs/gopherjs v1.17.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/smartystreets/assertions v1.13.1 // indirect
github.com/stretchr/testify v1.8.2 // indirect
go.etcd.io/etcd/api/v3 v3.5.9 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.1.12 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect
google.golang.org/grpc v1.54.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
go.etcd.io/etcd/api/v3 v3.5.12 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/net v0.35.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.22.0 // indirect
google.golang.org/grpc v1.67.1 // indirect
google.golang.org/protobuf v1.35.1 // indirect
)

+ 52
- 81
go.sum View File

@@ -6,30 +6,27 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBa
github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8=
github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -45,122 +42,96 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw=
github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8=
github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU=
github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY=
github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w=
github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew=
github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM=
github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc=
github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis=
go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs=
go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k=
go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE=
go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4=
go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E=
go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c=
go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4=
go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A=
go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4=
go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg=
go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA=
golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4=
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU=
google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U=
google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

+ 63
- 0
sdks/hpc/client.go View File

@@ -0,0 +1,63 @@
package hpc

import (
"fmt"

"gitlink.org.cn/cloudream/common/sdks"
)

type response[T any] struct {
Code int `json:"code"`
Message string `json:"message"`
Data T `json:"data"`
}

type respons2[T any] struct {
Code int `json:"code"`
Message string `json:"msg"`
Data T `json:"data"`
}

const (
ResponseCodeOK int = 200
)

func (r *response[T]) ToError() *sdks.CodeMessageError {
return &sdks.CodeMessageError{
Code: fmt.Sprintf("%d", r.Code),
Message: r.Message,
}
}

type Client struct {
baseURL string
}

func NewClient(cfg *Config) *Client {
return &Client{
baseURL: cfg.URL,
}
}

type Pool interface {
Acquire() (*Client, error)
Release(cli *Client)
}

type pool struct {
cfg *Config
}

func NewPool(cfg *Config) Pool {
return &pool{
cfg: cfg,
}
}
func (p *pool) Acquire() (*Client, error) {
cli := NewClient(p.cfg)
return cli, nil
}

func (p *pool) Release(cli *Client) {

}

+ 5
- 0
sdks/hpc/config.go View File

@@ -0,0 +1,5 @@
package hpc

type Config struct {
URL string `json:"url"`
}

+ 73
- 0
sdks/hpc/job.go View File

@@ -0,0 +1,73 @@
package hpc

import (
"fmt"
schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder"
"net/url"
"strings"
)

type CreateJobReq struct {
Name string `json:"name"`
ClusterID schsdk.ClusterID `json:"clusterId"`
Backend string `json:"backend"`
App string `json:"app"`
OperateType string `json:"operateType"`
//Parameters HPCParameter `json:"parameters"`
Parameters map[string]string `json:"parameters"`
}

type HPCParameter struct {
JobName string `json:"jobName"`
Partition string `json:"partition"`
Ntasks string `json:"ntasks"`
Nodes string `json:"nodes"`
BamFile string `json:"bamFile"`
InputFile string `json:"inputFile"`
}

type CreateJobResp struct {
Backend string `json:"backend"`
JobInfo HPCJobInfo `json:"jobInfo"`
}

type HPCJobInfo struct {
JobDir string `json:"jobDir"`
JobID string `json:"jobId"`
}

func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) {
targetUrl, err := url.JoinPath(c.baseURL, "/hpc/commitHpcTask")
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(targetUrl, http2.RequestParam{
Body: req,
Header: map[string]string{
"Authorization": token,
},
})
if err != nil {
return nil, err
}

contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, http2.ContentTypeJSON) {
var codeResp respons2[CreateJobResp]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}

if codeResp.Code == ResponseCodeOK {
return &codeResp.Data, nil
}

return nil, fmt.Errorf("error: %s", codeResp.Message)
}

return nil, fmt.Errorf("unknow response content type: %s", contType)

}

+ 525
- 0
sdks/hpc/models.go View File

@@ -0,0 +1,525 @@
package hpc

import (
"gitlink.org.cn/cloudream/common/pkgs/types"
schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/serder"
"time"
)

type ResourceType string

const (
ResourceTypeCPU ResourceType = "CPU"
ResourceTypeNPU ResourceType = "NPU"
ResourceTypeGPU ResourceType = "GPU"
ResourceTypeMLU ResourceType = "MLU"
ResourceTypeStorage ResourceType = "STORAGE"
ResourceTypeMemory ResourceType = "MEMORY"

Split = "/"

CODE = "code"
DATASET = "dataset"
IMAGE = "image"
MODEL = "model"
RESULT = "result"

OrderByName = "name"
OrderBySize = "size"
OrderByTime = "time"

StorageTypeURL = "url"
StorageTypeJCS = "jcs"

RejectedStatus = "rejected"
PendingStatus = "pending"
ApprovedStatus = "approved"
RevokedStatus = "revoked"
CancelStatus = "cancel"
ExpiredStatus = "expired"

ApplyAccess = "apply"
PrivateAccess = "private"
PublicAccess = "public"

PreferencePriority = "preference"
SpecifyClusterPriority = "specify"

FailedStatus = "failed"
SuccessStatus = "success"

Query = "query"
Delete = "delete"

ChildrenType = "children"
ParentType = "parent"

PlatformSugon = "sugon"
PlatformOpenI = "OpenI"
PlatformModelArts = "ModelArts"

URL = "url"
ID = "id"

Startup = "startup"
)

type TaskID int64
type DataID int64

type ClusterDetail struct {
// 集群ID
ClusterId schsdk.ClusterID `json:"clusterID"`
// 集群功能类型:云算,智算,超算
ClusterType string `json:"clusterType"`
// 集群地区:华东地区、华南地区、华北地区、华中地区、西南地区、西北地区、东北地区
Region string `json:"region"`
// 资源类型
Resources2 []ResourceData `json:"resources1,omitempty"`
//Resources2 []ResourceData `json:"resources"`
Resources []ClusterResource `json:"resources"`
}

type ClusterResource struct {
Resource TmpResourceData `json:"resource"`
BaseResources []TmpResourceData `json:"baseResources"`
}

type TmpResourceData struct {
Type ResourceType `json:"type"`
Name string `json:"name"`
Total UnitValue[float64] `json:"total"`
Available UnitValue[float64] `json:"available"`
}

type ResourceData interface {
Noop()
}

var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData](
(*CPUResourceData)(nil),
(*NPUResourceData)(nil),
(*GPUResourceData)(nil),
(*MLUResourceData)(nil),
(*DCUResourceData)(nil),
(*GCUResourceData)(nil),
(*GPGPUResourceData)(nil),
(*StorageResourceData)(nil),
(*MemoryResourceData)(nil),
(*BalanceResourceData)(nil),
(*RateResourceData)(nil),
)
var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type")

type ResourceDataBase struct{}

func (d *ResourceDataBase) Noop() {}

type UnitValue[T any] struct {
Unit string `json:"unit"`
Value T `json:"value"`
}

type CPUResourceData struct {
serder.Metadata `union:"CPU"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[int64] `json:"total"`
Available UnitValue[int64] `json:"available"`
}

type NPUResourceData struct {
serder.Metadata `union:"NPU"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[int64] `json:"total"`
Available UnitValue[int64] `json:"available"`
}

type GPUResourceData struct {
serder.Metadata `union:"GPU"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[int64] `json:"total"`
Available UnitValue[int64] `json:"available"`
}

type MLUResourceData struct {
serder.Metadata `union:"MLU"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[int64] `json:"total"`
Available UnitValue[int64] `json:"available"`
}

type DCUResourceData struct {
serder.Metadata `union:"DCU"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[int64] `json:"total"`
Available UnitValue[int64] `json:"available"`
}

type GCUResourceData struct {
serder.Metadata `union:"GCU"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[int64] `json:"total"`
Available UnitValue[int64] `json:"available"`
}

type GPGPUResourceData struct {
serder.Metadata `union:"ILUVATAR-GPGPU"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[int64] `json:"total"`
Available UnitValue[int64] `json:"available"`
}

type StorageResourceData struct {
serder.Metadata `union:"STORAGE"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[float64] `json:"total"`
Available UnitValue[float64] `json:"available"`
}

type MemoryResourceData struct {
serder.Metadata `union:"MEMORY"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[float64] `json:"total"`
Available UnitValue[float64] `json:"available"`
}

type BalanceResourceData struct {
serder.Metadata `union:"BALANCE"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[float64] `json:"total"`
Available UnitValue[float64] `json:"available"`
}

type RateResourceData struct {
serder.Metadata `union:"RATE"`
ResourceDataBase
Type string `json:"type"`
Name ResourceType `json:"name"`
Total UnitValue[float64] `json:"total"`
Available UnitValue[float64] `json:"available"`
}

type ResourceRange struct {
UserID cdssdk.UserID `json:"userID"`
Type ResourceType `json:"type"`
GPU Range `json:"gpu"`
GPUNumber int `json:"gpuNumber"`
CPU Range `json:"cpu"`
Memory Range `json:"memory"`
Storage Range `json:"storage"`
}

type Range struct {
Min float64 `json:"min"`
Max float64 `json:"max"`
}

type ResourcePriority interface {
Noop()
}

type ResourcePriorityBase struct {
}

var ResourcePriorityTypeUnion = types.NewTypeUnion[ResourcePriority](
(*RegionPriority)(nil),
(*ChipPriority)(nil),
(*BiasPriority)(nil),
)

var _ = serder.UseTypeUnionInternallyTagged(&ResourcePriorityTypeUnion, "type")

func (d *ResourcePriorityBase) Noop() {}

type RegionPriority struct {
serder.Metadata `union:"region"`
ResourcePriorityBase
Type string `json:"type"`
Options []string `json:"options"`
}

type ChipPriority struct {
serder.Metadata `union:"chip"`
ResourcePriorityBase
Type string `json:"type"`
Options []string `json:"options"`
}

type BiasPriority struct {
serder.Metadata `union:"bias"`
ResourcePriorityBase
Type string `json:"type"`
Options []string `json:"options"`
}

type TaskMessage struct {
Status string `json:"status"`
Message string `json:"message"`
}

type ReportMessage struct {
TaskName string `json:"taskName"`
TaskID string `json:"taskID"`
Status bool `json:"status"`
Message string `json:"message"`
ClusterID schsdk.ClusterID `json:"clusterID"`
Output string `json:"output"`
}

type UploadParams struct {
DataType string `json:"dataType"`
UploadInfo UploadInfo `json:"uploadInfo"`
}

type UploadInfo interface {
Noop()
}

var UploadInfoTypeUnion = types.NewTypeUnion[UploadInfo](
(*LocalUploadInfo)(nil),
(*RemoteUploadInfo)(nil),
)

var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type")

type LocalUploadInfo struct {
serder.Metadata `union:"local"`
UploadInfoBase
Type string `json:"type"`
LocalPath string `json:"localPath"`
ObjectIDs []cdssdk.ObjectID `json:"objectIDs"`
}

type RemoteUploadInfo struct {
serder.Metadata `union:"url"`
UploadInfoBase
Type string `json:"type"`
Url string `json:"url"`
Branch string `json:"branch"`
DataName string `json:"dataName"`
Cluster schsdk.ClusterID `json:"clusterID"`
}

type UploadInfoBase struct{}

func (d *UploadInfoBase) Noop() {}

type UploadPriority interface {
Noop()
}

var UploadPriorityTypeUnion = types.NewTypeUnion[UploadPriority](
(*Preferences)(nil),
(*SpecifyCluster)(nil),
)

var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type")

type Preferences struct {
serder.Metadata `union:"preference"`
UploadPriorityBase
Type string `json:"type"`
ResourcePriorities []ResourcePriority `json:"priorities"`
}

type SpecifyCluster struct {
serder.Metadata `union:"specify"`
UploadPriorityBase
Type string `json:"type"`
Clusters []schsdk.ClusterID `json:"clusters"`
}

type UploadPriorityBase struct{}

func (d *UploadPriorityBase) Noop() {}

type QueryData struct {
DataType string `json:"dataType" binding:"required"`
UserID cdssdk.UserID `json:"userID" binding:"required"`
Path string `json:"path"`
PackageID cdssdk.PackageID `json:"packageID" binding:"required"`
CurrentPage int `json:"currentPage" binding:"required"`
PageSize int `json:"pageSize" binding:"required"`
OrderBy string `json:"orderBy" binding:"required"`
}

type DataBinding interface {
Noop()
}

var DataBindingTypeUnion = types.NewTypeUnion[DataBinding](
(*DatasetBinding)(nil),
(*ModelBinding)(nil),
(*CodeBinding)(nil),
(*ImageBinding)(nil),
)

var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type")

type DataBindingBase struct{}

func (d *DataBindingBase) Noop() {}

type DatasetBinding struct {
serder.Metadata `union:"dataset"`
DataBindingBase
Type string `json:"type"`
Name string `json:"name"`
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
Description string `json:"description"`
Category string `json:"category"`
PackageID cdssdk.PackageID `json:"packageID"`
RepositoryName string `json:"repositoryName"`
ConsumptionPoints float64 `json:"points"`
}

type ModelBinding struct {
serder.Metadata `union:"model"`
DataBindingBase
Type string `json:"type"`
Name string `json:"name"`
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
Description string `json:"description"`
Category string `json:"category"`
ModelType string `json:"modelType"`
Env string `json:"env"`
Version string `json:"version"`
PackageID cdssdk.PackageID `json:"packageID"`
RepositoryName string `json:"repositoryName"`
}

type CodeBinding struct {
serder.Metadata `union:"code"`
DataBindingBase
Type string `json:"type"`
Name string `json:"name"`
ClusterID schsdk.ClusterID `json:"clusterID"`
Description string `json:"description"`
ImageID schsdk.ImageID `json:"imageID"`
BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"`
PackageID cdssdk.PackageID `json:"packageID"`
Output string `json:"output"`
// 当集群为openi的时候,需要传入分支
Branch string `json:"branch"`
}

//type ImageBinding struct {
// serder.Metadata `union:"image"`
// DataBindingBase
// Type string `json:"type"`
// Name string `json:"name"`
// ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
// Description string `json:"description"`
// Architecture string `json:"architecture"`
// ResourceType string `json:"resourceType"`
// Tags []string `json:"tags"`
// PackageID cdssdk.PackageID `json:"packageID"`
//}

type ImageBinding struct {
serder.Metadata `union:"image"`
DataBindingBase
Type string `json:"type"`
ID int64 `json:"id"`
Name string `json:"name"`
IDType string `json:"idType"`
ImageID string `json:"imageID"`
ClusterID schsdk.ClusterID `json:"clusterID"`
}

type Image struct {
ImageID schsdk.ImageID `json:"imageID" gorm:"column:ImageID;primaryKey"`
Name string `json:"name" gorm:"column:Name"`
CreateTime time.Time `json:"createTime" gorm:"column:CreateTime"`
ClusterImage []ClusterImage `gorm:"foreignKey:image_id;references:ImageID" json:"clusterImages"`
}

type ClusterImage struct {
ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"`
ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"`
OriginImageType string `gorm:"column:origin_image_type" json:"originImageType"`
OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"`
OriginImageName string `gorm:"column:origin_image_name" json:"originImageName"`
ClusterImageCard []ClusterImageCard `gorm:"foreignKey:origin_image_id;references:origin_image_id" json:"cards"`
}

func (ClusterImage) TableName() string {
return "clusterImage"
}

type ClusterImageCard struct {
OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"`
Card string `gorm:"column:card" json:"card"`
}

func (ClusterImageCard) TableName() string {
return "clusterImageCard"
}

type QueryBindingFilters struct {
Status string `json:"status"`
Name string `json:"name"`
}

type QueryBindingDataParam interface {
Noop()
}

var QueryBindingDataParamTypeUnion = types.NewTypeUnion[QueryBindingDataParam](
(*PrivateLevel)(nil),
(*ApplyLevel)(nil),
(*PublicLevel)(nil),
)

var _ = serder.UseTypeUnionInternallyTagged(&QueryBindingDataParamTypeUnion, "type")

type QueryBindingDataParamBase struct{}

func (d *QueryBindingDataParamBase) Noop() {}

type PrivateLevel struct {
serder.Metadata `union:"private"`
QueryBindingDataParamBase
Type string `json:"type" binding:"required"`
UserID cdssdk.UserID `json:"userID" binding:"required"`
BindingID int64 `json:"bindingID" binding:"required"`
Info DataBinding `json:"info"` // 可选,用于精细筛选,功能暂未实现
}

type ApplyLevel struct {
serder.Metadata `union:"apply"`
QueryBindingDataParamBase
Type string `json:"type" binding:"required"`
UserID cdssdk.UserID `json:"userID" binding:"required"`
Info DataBinding `json:"info"` // 可选,用于精细筛选,功能暂未实现
}

type PublicLevel struct {
serder.Metadata `union:"public"`
QueryBindingDataParamBase
UserID cdssdk.UserID `json:"userID" binding:"required"`
Type string `json:"type" binding:"required"`
Info DataBinding `json:"info"` // 可选,用于精细筛选,功能暂未实现
}

+ 16
- 0
sdks/pcmscheduler/job.go View File

@@ -0,0 +1,16 @@
package sch

import (
schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"time"
)

type PCMJob struct {
ID string `gorm:"column:id" json:"ID"`
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"`
LocalJobID string `gorm:"column:local_job_id" json:"localJobID"`
Param string `gorm:"column:param" json:"param"`
CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"`
}

+ 25
- 11
sdks/pcmscheduler/jobset.go View File

@@ -58,10 +58,15 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error)
}

type CreateJobReq struct {
Name string `json:"name"`
Description string `json:"description"`
JobResources schsdk.JobResources `json:"jobResources"`
DataDistribute DataDistribute `json:"dataDistributes"`
Name string `json:"name"`
Description string `json:"description"`
JobResources schsdk.JobResources `json:"jobResources"`
DataDistributes DataDistribute `json:"dataDistributes"`
}

type CommonJsonData struct {
ID string `json:"id"`
Name string `json:"name"`
}

type DataDistribute struct {
@@ -73,8 +78,9 @@ type DataDistribute struct {

type DataDetail struct {
ClusterID schsdk.ClusterID `json:"clusterID"`
StorageID cdssdk.StorageID `json:"storageID"`
JsonData string `json:"jsonData"`
//StorageID cdssdk.StorageID `json:"storageID"`
StorageID cdssdk.StorageID
JsonData string `json:"jsonData"`
}

type DatasetDistribute struct {
@@ -86,13 +92,15 @@ type DatasetDistribute struct {
type CodeDistribute struct {
DataName string `json:"dataName"`
PackageID cdssdk.PackageID `json:"packageID"`
Output string `json:"output"`
Clusters []DataDetail `json:"clusters"`
}

type ImageDistribute struct {
DataName string `json:"dataName"`
PackageID cdssdk.PackageID `json:"packageID"`
Clusters []DataDetail `json:"clusters"`
DataName string `json:"dataName"`
//PackageID cdssdk.PackageID `json:"packageID"`
ImageID schsdk.ImageID `json:"packageID"`
Clusters []DataDetail `json:"clusters"`
}

type ModelDistribute struct {
@@ -113,7 +121,7 @@ type ScheduleData struct {
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
}

func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) {
func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) {
targetUrl, err := url.JoinPath(c.baseURL, "/createTask")
if err != nil {
return nil, err
@@ -128,6 +136,9 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) {

resp, err := http2.PostJSON(targetUrl, http2.RequestParam{
Body: req,
Header: map[string]string{
"Authorization": token,
},
})
if err != nil {
return nil, err
@@ -169,7 +180,7 @@ type DataScheduleResults struct {
Results []DataScheduleResult `json:"results"`
}

func (c *Client) RunJob(req RunJobReq) error {
func (c *Client) RunJob(req RunJobReq, token string) error {
targetUrl, err := url.JoinPath(c.baseURL, "runTask")
if err != nil {
return err
@@ -177,6 +188,9 @@ func (c *Client) RunJob(req RunJobReq) error {

resp, err := http2.PostJSON(targetUrl, http2.RequestParam{
Body: req,
Header: map[string]string{
"Authorization": token,
},
})
if err != nil {
return err


+ 49
- 28
sdks/pcmscheduler/models.go View File

@@ -26,6 +26,7 @@ const (
DATASET = "dataset"
IMAGE = "image"
MODEL = "model"
RESULT = "result"

OrderByName = "name"
OrderBySize = "size"
@@ -48,8 +49,9 @@ const (
PreferencePriority = "preference"
SpecifyClusterPriority = "specify"

FailedStatus = "failed"
SuccessStatus = "success"
FailedStatus = "failed"
SuccessStatus = "success"
UploadingStatus = "uploading"

Query = "query"
Delete = "delete"
@@ -58,11 +60,17 @@ const (
ParentType = "parent"

PlatformSugon = "sugon"
PlatformOpenI = "openi"
PlatformModelArts = "modelarts"
PlatformOpenI = "OpenI"
PlatformModelArts = "ModelArts"

URL = "url"
ID = "id"

Startup = "startup"
)

type TaskID int64
type DataID int64

type ClusterDetail struct {
// 集群ID
@@ -391,10 +399,18 @@ type TaskMessage struct {
Message string `json:"message"`
}

type ReportMessage struct {
TaskName string `json:"taskName"`
TaskID string `json:"taskID"`
Status bool `json:"status"`
Message string `json:"message"`
ClusterID schsdk.ClusterID `json:"clusterID"`
Output string `json:"output"`
}

type UploadParams struct {
DataType string `json:"dataType"`
UploadInfo UploadInfo `json:"uploadInfo"`
//UploadPriority UploadPriority `json:"uploadPriority"`
}

type UploadInfo interface {
@@ -489,39 +505,44 @@ func (d *DataBindingBase) Noop() {}
type DatasetBinding struct {
serder.Metadata `union:"dataset"`
DataBindingBase
Type string `json:"type"`
Name string `json:"name"`
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
Description string `json:"description"`
Category string `json:"category"`
PackageID cdssdk.PackageID `json:"packageID"`
Type string `json:"type"`
Name string `json:"name"`
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
Description string `json:"description"`
Category string `json:"category"`
PackageID cdssdk.PackageID `json:"packageID"`
RepositoryName string `json:"repositoryName"`
ConsumptionPoints float64 `json:"points"`
}

type ModelBinding struct {
serder.Metadata `union:"model"`
DataBindingBase
Type string `json:"type"`
Name string `json:"name"`
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
Description string `json:"description"`
Category string `json:"category"`
ModelType string `json:"modelType"`
Env string `json:"env"`
Version string `json:"version"`
PackageID cdssdk.PackageID `json:"packageID"`
Type string `json:"type"`
Name string `json:"name"`
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
Description string `json:"description"`
Category string `json:"category"`
ModelType string `json:"modelType"`
Env string `json:"env"`
Version string `json:"version"`
PackageID cdssdk.PackageID `json:"packageID"`
RepositoryName string `json:"repositoryName"`
}

type CodeBinding struct {
serder.Metadata `union:"code"`
DataBindingBase
Type string `json:"type"`
Name string `json:"name"`
ClusterIDs []schsdk.ClusterID `json:"clusterIDs"`
Description string `json:"description"`
ImageID int64 `json:"imageID"`
ObjectID cdssdk.ObjectID `json:"objectID"`
FilePath string `json:"filePath"`
PackageID cdssdk.PackageID `json:"packageID"`
Type string `json:"type"`
Name string `json:"name"`
ClusterID schsdk.ClusterID `json:"clusterID"`
Description string `json:"description"`
ImageID schsdk.ImageID `json:"imageID"`
BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"`
PackageID cdssdk.PackageID `json:"packageID"`
Output string `json:"output"`
// 当集群为openi的时候,需要传入分支
Branch string `json:"branch"`
}

//type ImageBinding struct {


+ 62
- 9
sdks/scheduler/models.go View File

@@ -49,6 +49,7 @@ type JobSetInfo struct {

type JobInfo interface {
GetLocalJobID() string
GetTargetLocalJobID() []string
}

var JobInfoTypeUnion = types.NewTypeUnion[JobInfo](
@@ -60,17 +61,23 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo](
(*FinetuningJobInfo)(nil),
(*DataPreprocessJobInfo)(nil),
(*PCMJobInfo)(nil),
(*HPCJobInfo)(nil),
)
var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type")

type JobInfoBase struct {
LocalJobID string `json:"localJobID"`
LocalJobID string `json:"localJobID"`
TargetLocalJobIDs []string `json:"targetLocalJobIDs"`
}

func (i *JobInfoBase) GetLocalJobID() string {
return i.LocalJobID
}

func (i *JobInfoBase) GetTargetLocalJobID() []string {
return i.TargetLocalJobIDs
}

type NormalJobInfo struct {
serder.Metadata `union:"Normal"`
JobInfoBase
@@ -92,6 +99,33 @@ type PCMJobInfo struct {
JobResources JobResources `json:"jobResources"`
}

type HPCJobInfo struct {
serder.Metadata `union:"HPC"`
JobInfoBase
Type string `json:"type"`
Name string `json:"name"`
ClusterID ClusterID `json:"clusterID"`
Backend string `json:"backend"`
App string `json:"app"`
OperateType string `json:"operateType"`
Parameters HPCParameter `json:"parameters"`
}

type HPCParameter struct {
JobName string `json:"jobName"`
JobDir string `json:"jobDir"`
Partition string `json:"partition"`
Ntasks string `json:"ntasks"`
Nodes string `json:"nodes"`
BamFile string `json:"bamFile"`
HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"`
}

type HPCBindingFile struct {
ParamName string `json:"paramName"`
BindingID int64 `json:"bindingID"`
}

type JobResources struct {
//任务分配策略:负载均衡、积分优先、随机分配等,dataLocality, leastLoadFirst
ScheduleStrategy string `json:"scheduleStrategy"`
@@ -99,9 +133,11 @@ type JobResources struct {
}

type ClusterInfo struct {
ClusterID ClusterID `json:"clusterID"`
Resources []JobResource `json:"resources"`
Runtime PCMJobRuntimeInfo `json:"runtime"`
ClusterID ClusterID `json:"clusterID"`
Resources []JobResource `json:"resources"`
//Files JobFilesInfo `json:"files"`
Code JobFileInfo `json:"code"`
Runtime PCMJobRuntimeInfo `json:"runtime"`
}

type PCMJobRuntimeInfo struct {
@@ -115,7 +151,7 @@ type PCMJobRuntimeInfo struct {
//}

type JobResource interface {
Noop2()
Noop()
}

var JobResourceTypeUnion = types.NewTypeUnion[JobResource](
@@ -124,20 +160,30 @@ var JobResourceTypeUnion = types.NewTypeUnion[JobResource](
(*NPU)(nil),
(*MLU)(nil),
(*DCU)(nil),
(*Memory)(nil),
(*MEMORY)(nil),
(*PRICE)(nil),
(*STORAGE)(nil),
)

var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type")

type JobResourceBase struct{}

func (d *JobResourceBase) Noop2() {}
func (d *JobResourceBase) Noop() {}

type CPU struct {
serder.Metadata `union:"CPU"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

type STORAGE struct {
serder.Metadata `union:"STORAGE"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

@@ -145,6 +191,7 @@ type GPU struct {
serder.Metadata `union:"GPU"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

@@ -152,13 +199,15 @@ type NPU struct {
serder.Metadata `union:"NPU"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

type Memory struct {
serder.Metadata `union:"Memory"`
type MEMORY struct {
serder.Metadata `union:"MEMORY"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

@@ -166,6 +215,7 @@ type DCU struct {
serder.Metadata `union:"DCU"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

@@ -173,6 +223,7 @@ type MLU struct {
serder.Metadata `union:"MLU"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

@@ -180,6 +231,7 @@ type PRICE struct {
serder.Metadata `union:"PRICE"`
JobResourceBase
Type string `json:"type"`
Name string `json:"name"`
Number int64 `json:"number"`
}

@@ -211,6 +263,7 @@ type DataReturnJobInfo struct {
JobInfoBase
Type string `json:"type"`
BucketID cdssdk.BucketID `json:"bucketID"`
BindingType string `json:"bindingType"`
TargetLocalJobID string `json:"targetLocalJobID"`
}



+ 190
- 1
sdks/sdks.go View File

@@ -1,6 +1,19 @@
package sdks

import "fmt"
import (
"bytes"
"fmt"
"io"
"net/http"
"net/url"
"strings"

"github.com/google/go-querystring/query"
"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/serder"
)

type CodeMessageError struct {
Code string
@@ -10,3 +23,179 @@ type CodeMessageError struct {
func (e *CodeMessageError) Error() string {
return fmt.Sprintf("code: %s, message: %s", e.Code, e.Message)
}

type RequestParam struct {
Method string
Path string
Query url.Values
Header http.Header
Body RequestBody
}

func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) {
var body io.ReadCloser
bodyLen := int64(0)
if p.Body != nil {
body = p.Body.IntoStream()
bodyLen = p.Body.Length()
}

req, err := http.NewRequest(p.Method, joinUrlUnsafe(baseURL, p.Path), body)
if err != nil {
return nil, err
}
req.ContentLength = bodyLen
req.URL.RawQuery = p.Query.Encode()

if p.Header != nil {
req.Header = p.Header
}

return req, nil
}

func (p *RequestParam) Do(baseURL string, cli *http.Client) (*http.Response, error) {
req, err := p.MakeRequest(baseURL)
if err != nil {
return nil, err
}

return cli.Do(req)
}

func joinUrlUnsafe(base string, path string) string {
if strings.HasSuffix(base, "/") {
if strings.HasPrefix(path, "/") {
return base + path[1:]
}

return base + path
}

if strings.HasPrefix(path, "/") {
return base + path
}

return base + "/" + path
}

type RequestBody interface {
// 请求体长度,如果长度未知,返回-1
Length() int64
// 将内部值变成一个流,用于发送请求
IntoStream() io.ReadCloser
}

type StringBody struct {
Value string
}

func (s *StringBody) Length() int64 {
return int64(len(s.Value))
}

func (s *StringBody) IntoStream() io.ReadCloser {
return io.NopCloser(bytes.NewReader([]byte(s.Value)))
}

type BytesBody struct {
Value []byte
}

func (b *BytesBody) Length() int64 {
return int64(len(b.Value))
}

func (b *BytesBody) IntoStream() io.ReadCloser {
return io.NopCloser(bytes.NewReader(b.Value))
}

type StreamBody struct {
Stream io.ReadCloser
LengthHint int64 // 长度提示,如果长度未知,可以设置为-1
}

func (s *StreamBody) Length() int64 {
return s.LengthHint
}

func (s *StreamBody) IntoStream() io.ReadCloser {
return s.Stream
}

type APIRequest interface {
MakeParam() *RequestParam
}

func MakeJSONParam(method string, path string, body any) *RequestParam {
data, err := serder.ObjectToJSONEx(body)
if err != nil {
// 开发人员应该保证param是可序列化的
panic(err)
}

return &RequestParam{
Method: method,
Path: path,
Body: &BytesBody{Value: data},
}
}

func MakeQueryParam(method string, path string, q any) *RequestParam {
values, err := query.Values(q)
if err != nil {
// 开发人员应该保证param是可序列化的
panic(err)
}

return &RequestParam{
Method: method,
Path: path,
Query: values,
}
}

type APIResponse interface {
ParseResponse(resp *http.Response) error
}

type CodeDataResponse[T any] struct {
Code string `json:"code"`
Message string `json:"message"`
Data T `json:"data"`
}

func (r *CodeDataResponse[T]) Unwarp() (T, error) {
if r.Code == errorcode.OK {
return r.Data, nil
}
var def T
return def, &CodeMessageError{Code: r.Code, Message: r.Message}
}

func ParseCodeDataJSONResponse[T any](resp *http.Response, ret T) error {
contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, http2.ContentTypeJSON) {
var err error
var r CodeDataResponse[T]
r.Data = ret

if err = serder.JSONToObjectStreamExRaw(resp.Body, &r); err != nil {
return fmt.Errorf("parsing response: %w", err)
}

if r.Code != errorcode.OK {
return &CodeMessageError{Code: r.Code, Message: r.Message}
}

return nil
}

cont, err := io2.ReadMost(resp.Body, 200)
if err != nil {
return fmt.Errorf("unknow response content type: %s, status: %d", contType, resp.StatusCode)
}
strCont := string(cont)

return fmt.Errorf("unknow response content type: %s, status: %d, body[:200]: %s", contType, resp.StatusCode, strCont)
}

+ 42
- 94
sdks/storage/cdsapi/bucket.go View File

@@ -1,11 +1,10 @@
package cdsapi

import (
"net/url"
"net/http"

"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/http2"
)

type BucketService struct {
@@ -19,36 +18,24 @@ func (c *Client) Bucket() *BucketService {
const BucketGetByNamePath = "/bucket/getByName"

type BucketGetByName struct {
UserID cdssdk.UserID `json:"userID" form:"userID" binding:"required"`
Name string `json:"name" form:"name" binding:"required"`
UserID cdssdk.UserID `url:"userID" form:"userID" binding:"required"`
Name string `url:"name" form:"name" binding:"required"`
}

func (r *BucketGetByName) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, BucketGetByNamePath, r)
}

type BucketGetByNameResp struct {
Bucket cdssdk.Bucket `json:"bucket"`
}

func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) {
url, err := url.JoinPath(c.baseURL, BucketGetByNamePath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[BucketGetByNameResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketGetByNameResp{})
}

const BucketCreatePath = "/bucket/create"
@@ -58,33 +45,20 @@ type BucketCreate struct {
Name string `json:"name" binding:"required"`
}

func (r *BucketCreate) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, BucketCreatePath, r)
}

type BucketCreateResp struct {
Bucket cdssdk.Bucket `json:"bucket"`
}

func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) {
url, err := url.JoinPath(c.baseURL, BucketCreatePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[BucketCreateResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *BucketCreateResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketCreateResp{})
}

const BucketDeletePath = "/bucket/delete"
@@ -94,64 +68,38 @@ type BucketDelete struct {
BucketID cdssdk.BucketID `json:"bucketID" binding:"required"`
}

type BucketDeleteResp struct{}

func (c *BucketService) Delete(req BucketDelete) error {
url, err := url.JoinPath(c.baseURL, BucketDeletePath)
if err != nil {
return err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return err
}
func (r *BucketDelete) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, BucketDeletePath, r)
}

codeResp, err := ParseJSONResponse[response[BucketDeleteResp]](resp)
if err != nil {
return err
}
type BucketDeleteResp struct{}

if codeResp.Code == errorcode.OK {
return nil
}
func (r *BucketDeleteResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return codeResp.ToError()
func (c *BucketService) Delete(req BucketDelete) error {
return JSONAPINoData(c.cfg, http.DefaultClient, &req)
}

const BucketListUserBucketsPath = "/bucket/listUserBuckets"

type BucketListUserBucketsReq struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
}

func (r *BucketListUserBucketsReq) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, BucketListUserBucketsPath, r)
}

type BucketListUserBucketsResp struct {
Buckets []cdssdk.Bucket `json:"buckets"`
}

func (r *BucketListUserBucketsResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *BucketService) ListUserBuckets(req BucketListUserBucketsReq) (*BucketListUserBucketsResp, error) {
url, err := url.JoinPath(c.baseURL, BucketListUserBucketsPath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[BucketListUserBucketsResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, codeResp.ToError()
return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketListUserBucketsResp{})
}

+ 12
- 25
sdks/storage/cdsapi/cache.go View File

@@ -1,11 +1,10 @@
package cdsapi

import (
"net/url"
"net/http"

"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/http2"
)

const CacheMovePackagePath = "/cache/movePackage"
@@ -15,29 +14,17 @@ type CacheMovePackageReq struct {
PackageID cdssdk.PackageID `json:"packageID"`
StorageID cdssdk.StorageID `json:"storageID"`
}

func (r *CacheMovePackageReq) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, CacheMovePackagePath, r)
}

type CacheMovePackageResp struct{}

func (r *CacheMovePackageResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageResp, error) {
url, err := url.JoinPath(c.baseURL, CacheMovePackagePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[CacheMovePackageResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}

return nil, jsonResp.ToError()
return JSONAPI(c.cfg, http.DefaultClient, &req, &CacheMovePackageResp{})
}

+ 2
- 2
sdks/storage/cdsapi/client.go View File

@@ -18,12 +18,12 @@ func (r *response[T]) ToError() *sdks.CodeMessageError {
}

type Client struct {
baseURL string
cfg *Config
}

func NewClient(cfg *Config) *Client {
return &Client{
baseURL: cfg.URL,
cfg: cfg,
}
}



+ 3
- 1
sdks/storage/cdsapi/config.go View File

@@ -1,5 +1,7 @@
package cdsapi

type Config struct {
URL string `json:"url"`
URL string `json:"url"`
AccessKey string `json:"accessKey"`
SecretKey string `json:"secretKey"`
}

+ 12
- 26
sdks/storage/cdsapi/hub.go View File

@@ -1,44 +1,30 @@
package cdsapi

import (
"net/url"
"net/http"

"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/http2"
)

var HubGetHubsPath = "/hub/getHubs"

type HubGetHubsReq struct {
HubIDs []cdssdk.HubID `json:"hubIDs"`
HubIDs []cdssdk.HubID `form:"hubIDs" url:"hubIDs"`
}

func (r *HubGetHubsReq) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, HubGetHubsPath, r)
}

type HubGetHubsResp struct {
Hubs []*cdssdk.Hub `json:"hubs"`
}

func (r *HubGetHubsResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *Client) HubGetHubs(req HubGetHubsReq) (*HubGetHubsResp, error) {
url, err := url.JoinPath(c.baseURL, HubGetHubsPath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[HubGetHubsResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}

return nil, jsonResp.ToError()
return JSONAPI(c.cfg, http.DefaultClient, &req, &HubGetHubsResp{})
}

+ 5
- 5
sdks/storage/cdsapi/hub_io.go View File

@@ -24,7 +24,7 @@ type GetStreamReq struct {
}

func (c *Client) GetStream(req GetStreamReq) (io.ReadCloser, error) {
targetUrl, err := url.JoinPath(c.baseURL, GetStreamPath)
targetUrl, err := url.JoinPath(c.cfg.URL, GetStreamPath)
if err != nil {
return nil, err
}
@@ -65,7 +65,7 @@ type SendStreamInfo struct {
}

func (c *Client) SendStream(req SendStreamReq) error {
targetUrl, err := url.JoinPath(c.baseURL, SendStreamPath)
targetUrl, err := url.JoinPath(c.cfg.URL, SendStreamPath)
if err != nil {
return err
}
@@ -133,7 +133,7 @@ type ExecuteIOPlanReq struct {
}

func (c *Client) ExecuteIOPlan(req ExecuteIOPlanReq) error {
targetUrl, err := url.JoinPath(c.baseURL, ExecuteIOPlanPath)
targetUrl, err := url.JoinPath(c.cfg.URL, ExecuteIOPlanPath)
if err != nil {
return err
}
@@ -171,7 +171,7 @@ type SendVarReq struct {
}

func (c *Client) SendVar(req SendVarReq) error {
targetUrl, err := url.JoinPath(c.baseURL, SendVarPath)
targetUrl, err := url.JoinPath(c.cfg.URL, SendVarPath)
if err != nil {
return err
}
@@ -214,7 +214,7 @@ type GetVarResp struct {
}

func (c *Client) GetVar(req GetVarReq) (*GetVarResp, error) {
targetUrl, err := url.JoinPath(c.baseURL, GetVarPath)
targetUrl, err := url.JoinPath(c.cfg.URL, GetVarPath)
if err != nil {
return nil, err
}


+ 176
- 274
sdks/storage/cdsapi/object.go View File

@@ -1,15 +1,20 @@
package cdsapi

import (
"context"
"fmt"
"io"
"mime"
"net/http"
"net/url"
"strings"
"time"

v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/credentials"
"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder"
@@ -28,76 +33,52 @@ func (c *Client) Object() *ObjectService {
const ObjectListPathByPath = "/object/listByPath"

type ObjectListByPath struct {
UserID cdssdk.UserID `form:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" binding:"required"`
Path string `form:"path"` // 允许为空字符串
IsPrefix bool `form:"isPrefix"`
NoRecursive bool `form:"noRecursive"` // 仅当isPrefix为true时有效,表示仅查询直接属于Prefix下的对象,对于更深的对象,返回它们的公共前缀
UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"`
PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" url:"path"` // 允许为空字符串
IsPrefix bool `form:"isPrefix" url:"isPrefix"`
NoRecursive bool `form:"noRecursive" url:"noRecursive"` // 仅当isPrefix为true时有效,表示仅查询直接属于Prefix下的对象,对于更深的对象,返回它们的公共前缀

}

func (r *ObjectListByPath) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, ObjectListPathByPath, r)
}

type ObjectListByPathResp struct {
CommonPrefixes []string `json:"commonPrefixes"` // 仅在IsPrefix为true且NoRecursive为true时有效,包含更深层对象的shared prefix
Objects []cdssdk.Object `json:"objects"` // 如果IsPrefix为true且NoRecursive为false,则返回所有匹配的对象,否则只返回直接属于Prefix下的对象
}

func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectListPathByPath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectListByPathResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectListByPathResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByPathResp{})
}

const ObjectListByIDsPath = "/object/listByIDs"

type ObjectListByIDs struct {
UserID cdssdk.UserID `json:"userID" binding:"required"`
ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"`
UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"`
ObjectIDs []cdssdk.ObjectID `form:"objectIDs" binding:"required" url:"objectIDs"`
}

func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r)
}

type ObjectListByIDsResp struct {
Objects []*cdssdk.Object `json:"object"` // 与ObjectIDs一一对应,如果某个ID不存在,则对应位置为nil
}

func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectListByIDsPath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectListByIDsResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectListByIDsResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByIDsResp{})
}

const ObjectUploadPath = "/object/upload"
@@ -127,7 +108,11 @@ type ObjectUploadResp struct {
}

func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectUploadPath)
type uploadInfo struct {
Info string `url:"info"`
}

url, err := url.JoinPath(c.cfg.URL, ObjectUploadPath)
if err != nil {
return nil, err
}
@@ -137,16 +122,15 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) {
return nil, fmt.Errorf("upload info to json: %w", err)
}

resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{
Form: map[string]string{"info": string(infoJSON)},
Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) {
resp, err := PostMultiPart(c.cfg, url,
uploadInfo{Info: string(infoJSON)},
iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) {
return &http2.IterMultiPartFile{
FieldName: "files",
FileName: src.Path,
File: src.File,
}, nil
}),
})
}))
if err != nil {
return nil, err
}
@@ -173,25 +157,42 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) {
const ObjectDownloadPath = "/object/download"

type ObjectDownload struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
ObjectID cdssdk.ObjectID `form:"objectID" json:"objectID" binding:"required"`
Offset int64 `form:"offset" json:"offset,omitempty"`
Length *int64 `form:"length" json:"length,omitempty"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
}

func (r *ObjectDownload) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadPath, r)
}

type DownloadingObject struct {
Path string
File io.ReadCloser
}

func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) {
url, err := url.JoinPath(c.baseURL, ObjectDownloadPath)
httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL)
if err != nil {
return nil, err
}

resp, err := http2.GetJSON(url, http2.RequestParam{
Query: req,
})
if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" {
prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "")
cred, err := prod.Retrieve(context.TODO())
if err != nil {
return nil, err
}

signer := v4.NewSigner()
err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now())
if err != nil {
return nil, err
}
}

resp, err := http.DefaultClient.Do(httpReq)
if err != nil {
return nil, err
}
@@ -221,22 +222,38 @@ func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error)
const ObjectDownloadByPathPath = "/object/downloadByPath"

type ObjectDownloadByPath struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"`
Path string `form:"path" json:"path" binding:"required"`
Offset int64 `form:"offset" json:"offset,omitempty"`
Length *int64 `form:"length" json:"length,omitempty"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
}

func (r *ObjectDownloadByPath) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadByPathPath, r)
}

func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) {
url, err := url.JoinPath(c.baseURL, ObjectDownloadByPathPath)
httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL)
if err != nil {
return nil, err
}

resp, err := http2.GetJSON(url, http2.RequestParam{
Query: req,
})
if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" {
prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "")
cred, err := prod.Retrieve(context.TODO())
if err != nil {
return nil, err
}

signer := v4.NewSigner()
err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now())
if err != nil {
return nil, err
}
}

resp, err := http.DefaultClient.Do(httpReq)
if err != nil {
return nil, err
}
@@ -279,33 +296,20 @@ type ObjectUpdateInfo struct {
Updatings []UpdatingObject `json:"updatings" binding:"required"`
}

func (r *ObjectUpdateInfo) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoPath, r)
}

type ObjectUpdateInfoResp struct {
Successes []cdssdk.ObjectID `json:"successes"`
}

func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoPath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectUpdateInfoResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoResp{})
}

const ObjectUpdateInfoByPathPath = "/object/updateInfoByPath"
@@ -317,31 +321,18 @@ type ObjectUpdateInfoByPath struct {
UpdateTime time.Time `json:"updateTime" binding:"required"`
}

type ObjectUpdateInfoByPathResp struct{}

func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoByPathPath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}
func (r *ObjectUpdateInfoByPath) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoByPathPath, r)
}

jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoByPathResp]](resp)
if err != nil {
return nil, err
}
type ObjectUpdateInfoByPathResp struct{}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectUpdateInfoByPathResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoByPathResp{})
}

const ObjectMovePath = "/object/move"
@@ -362,33 +353,20 @@ type ObjectMove struct {
Movings []MovingObject `json:"movings" binding:"required"`
}

func (r *ObjectMove) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectMovePath, r)
}

type ObjectMoveResp struct {
Successes []cdssdk.ObjectID `json:"successes"`
}

func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectMovePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectMoveResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectMoveResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectMoveResp{})
}

const ObjectDeletePath = "/object/delete"
@@ -398,31 +376,18 @@ type ObjectDelete struct {
ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"`
}

type ObjectDeleteResp struct{}

func (c *ObjectService) Delete(req ObjectDelete) error {
url, err := url.JoinPath(c.baseURL, ObjectDeletePath)
if err != nil {
return err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return err
}
func (r *ObjectDelete) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectDeletePath, r)
}

jsonResp, err := ParseJSONResponse[response[ObjectDeleteResp]](resp)
if err != nil {
return err
}
type ObjectDeleteResp struct{}

if jsonResp.Code == errorcode.OK {
return nil
}
func (r *ObjectDeleteResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return jsonResp.ToError()
func (c *ObjectService) Delete(req ObjectDelete) error {
return JSONAPINoData(c.cfg, http.DefaultClient, &req)
}

const ObjectDeleteByPathPath = "/object/deleteByPath"
@@ -432,31 +397,19 @@ type ObjectDeleteByPath struct {
PackageID cdssdk.PackageID `json:"packageID" binding:"required"`
Path string `json:"path" binding:"required"`
}
type ObjectDeleteByPathResp struct{}

func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error {
url, err := url.JoinPath(c.baseURL, ObjectDeleteByPathPath)
if err != nil {
return err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return err
}
func (r *ObjectDeleteByPath) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectDeleteByPathPath, r)
}

jsonResp, err := ParseJSONResponse[response[ObjectDeleteByPathResp]](resp)
if err != nil {
return err
}
type ObjectDeleteByPathResp struct{}

if jsonResp.Code == errorcode.OK {
return nil
}
func (r *ObjectDeleteByPathResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return jsonResp.ToError()
func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error {
return JSONAPINoData(c.cfg, http.DefaultClient, &req)
}

const ObjectClonePath = "/object/clone"
@@ -466,6 +419,10 @@ type ObjectClone struct {
Clonings []CloningObject `json:"clonings" binding:"required"`
}

func (r *ObjectClone) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectClonePath, r)
}

type CloningObject struct {
ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"`
NewPath string `json:"newPath" binding:"required"`
@@ -476,64 +433,35 @@ type ObjectCloneResp struct {
Objects []*cdssdk.Object `json:"objects"`
}

func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectClonePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectCloneResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectCloneResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCloneResp{})
}

const ObjectGetPackageObjectsPath = "/object/getPackageObjects"

type ObjectGetPackageObjects struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"`
}

func (r *ObjectGetPackageObjects) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, ObjectGetPackageObjectsPath, r)
}

type ObjectGetPackageObjectsResp struct {
Objects []cdssdk.Object `json:"objects"`
}

func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectGetPackageObjectsPath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectGetPackageObjectsResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectGetPackageObjectsResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectGetPackageObjectsResp{})
}

const ObjectNewMultipartUploadPath = "/v1/object/newMultipartUpload"
@@ -544,33 +472,20 @@ type ObjectNewMultipartUpload struct {
Path string `json:"path" binding:"required"`
}

func (r *ObjectNewMultipartUpload) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectNewMultipartUploadPath, r)
}

type ObjectNewMultipartUploadResp struct {
Object cdssdk.Object `json:"object"`
}

func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectNewMultipartUploadPath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectNewMultipartUploadResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectNewMultipartUploadResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectNewMultipartUploadResp{})
}

const ObjectUploadPartPath = "/v1/object/uploadPart"
@@ -589,7 +504,7 @@ type ObjectUploadPartInfo struct {
type ObjectUploadPartResp struct{}

func (c *ObjectService) UploadPart(req ObjectUploadPart) (*ObjectUploadPartResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectUploadPartPath)
url, err := url.JoinPath(c.cfg.URL, ObjectUploadPartPath)
if err != nil {
return nil, err
}
@@ -636,31 +551,18 @@ type ObjectCompleteMultipartUpload struct {
Indexes []int `json:"indexes" binding:"required"`
}

func (r *ObjectCompleteMultipartUpload) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, ObjectCompleteMultipartUploadPath, r)
}

type ObjectCompleteMultipartUploadResp struct {
Object cdssdk.Object `json:"object"`
}

func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectCompleteMultipartUploadPath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectCompleteMultipartUploadResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}
func (r *ObjectCompleteMultipartUploadResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, jsonResp.ToError()
func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCompleteMultipartUploadResp{})
}

+ 81
- 173
sdks/storage/cdsapi/package.go View File

@@ -2,11 +2,12 @@ package cdsapi

import (
"fmt"
"net/http"
"net/url"
"strings"

"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder"
@@ -23,72 +24,48 @@ func (c *Client) Package() *PackageService {
const PackageGetPath = "/package/get"

type PackageGetReq struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"`
}

func (r *PackageGetReq) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, PackageGetPath, r)
}

type PackageGetResp struct {
cdssdk.Package
}

func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) {
url, err := url.JoinPath(c.baseURL, PackageGetPath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageGetResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *PackageGetResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetResp{})
}

const PackageGetByFullNamePath = "/package/getByFullName"

type PackageGetByFullName struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
BucketName string `form:"bucketName" json:"bucketName" binding:"required"`
PackageName string `form:"packageName" json:"packageName" binding:"required"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
BucketName string `form:"bucketName" url:"bucketName" binding:"required"`
PackageName string `form:"packageName" url:"packageName" binding:"required"`
}

func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, PackageGetByFullNamePath, r)
}

type PackageGetByFullNameResp struct {
Package cdssdk.Package `json:"package"`
}

func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) {
url, err := url.JoinPath(c.baseURL, PackageGetByFullNamePath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageGetByFullNameResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetByFullNameResp{})
}

const PackageCreatePath = "/package/create"
@@ -99,33 +76,20 @@ type PackageCreate struct {
Name string `json:"name"`
}

func (r *PackageCreate) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, PackageCreatePath, r)
}

type PackageCreateResp struct {
Package cdssdk.Package `json:"package"`
}

func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) {
url, err := url.JoinPath(s.baseURL, PackageCreatePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageCreateResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *PackageCreateResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) {
return JSONAPI(s.cfg, http.DefaultClient, &req, &PackageCreateResp{})
}

const PackageCreateLoadPath = "/package/createLoad"
@@ -147,7 +111,7 @@ type PackageCreateLoadResp struct {
}

func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadResp, error) {
url, err := url.JoinPath(c.baseURL, PackageCreateLoadPath)
url, err := url.JoinPath(c.cfg.URL, PackageCreateLoadPath)
if err != nil {
return nil, err
}
@@ -157,16 +121,15 @@ func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadRe
return nil, fmt.Errorf("upload info to json: %w", err)
}

resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{
Form: map[string]string{"info": string(infoJSON)},
Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) {
resp, err := PostMultiPart(c.cfg, url,
map[string]string{"info": string(infoJSON)},
iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) {
return &http2.IterMultiPartFile{
FieldName: "files",
FileName: src.Path,
File: src.File,
}, nil
}),
})
}))
if err != nil {
return nil, err
}
@@ -190,35 +153,18 @@ type PackageDelete struct {
PackageID cdssdk.PackageID `json:"packageID" binding:"required"`
}

func (c *PackageService) Delete(req PackageDelete) error {
url, err := url.JoinPath(c.baseURL, PackageDeletePath)
if err != nil {
return err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return err
}

contType := resp.Header.Get("Content-Type")

if strings.Contains(contType, http2.ContentTypeJSON) {
var codeResp response[any]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return fmt.Errorf("parsing response: %w", err)
}
func (r *PackageDelete) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, PackageDeletePath, r)
}

if codeResp.Code == errorcode.OK {
return nil
}
type PackageDeleteResp struct{}

return codeResp.ToError()
}
func (r *PackageDeleteResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return fmt.Errorf("unknow response content type: %s", contType)
func (c *PackageService) Delete(req PackageDelete) error {
return JSONAPINoData(c.cfg, http.DefaultClient, &req)
}

const PackageClonePath = "/package/clone"
@@ -230,102 +176,64 @@ type PackageClone struct {
Name string `json:"name" binding:"required"`
}

func (r *PackageClone) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, PackageClonePath, r)
}

type PackageCloneResp struct {
Package cdssdk.Package `json:"package"`
}

func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) {
url, err := url.JoinPath(c.baseURL, PackageClonePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageCloneResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *PackageCloneResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageCloneResp{})
}

const PackageListBucketPackagesPath = "/package/listBucketPackages"

type PackageListBucketPackages struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
BucketID cdssdk.BucketID `form:"bucketID" json:"bucketID" binding:"required"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
BucketID cdssdk.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
}

func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, PackageListBucketPackagesPath, r)
}

type PackageListBucketPackagesResp struct {
Packages []cdssdk.Package `json:"packages"`
}

func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) {
url, err := url.JoinPath(c.baseURL, PackageListBucketPackagesPath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageListBucketPackagesResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageListBucketPackagesResp{})
}

const PackageGetCachedStoragesPath = "/package/getCachedStorages"

type PackageGetCachedStoragesReq struct {
PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"`
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
}

func (r *PackageGetCachedStoragesReq) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, PackageGetCachedStoragesPath, r)
}

type PackageGetCachedStoragesResp struct {
cdssdk.PackageCachingInfo
}

func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) {
url, err := url.JoinPath(c.baseURL, PackageGetCachedStoragesPath)
if err != nil {
return nil, err
}
resp, err := http2.GetJSON(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageGetCachedStoragesResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetCachedStoragesResp{})
}

+ 18
- 14
sdks/storage/cdsapi/presigned.go View File

@@ -13,22 +13,13 @@ import (
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
)

const (
AuthService = "jcs"
AuthRegion = "any"
)

type PresignedService struct {
*Client
accessKey string
secretKey string
}

func (c *Client) Presigned(accessKey string, secretKey string) *PresignedService {
func (c *Client) Presigned() *PresignedService {
return &PresignedService{
Client: c,
accessKey: accessKey,
secretKey: secretKey,
Client: c,
}
}

@@ -42,10 +33,23 @@ type PresignedObjectDownloadByPath struct {
Length *int64 `form:"length" url:"length,omitempty"`
}

func (c *PresignedService) ObjectDownload(req PresignedObjectDownloadByPath, expireIn int) (string, error) {
func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) {
return c.presign(req, PresignedObjectDownloadByPathPath, http.MethodGet, expireIn)
}

const PresignedObjectDownloadPath = "/v1/presigned/object/download"

type PresignedObjectDownload struct {
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
}

func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) {
return c.presign(req, PresignedObjectDownloadPath, http.MethodGet, expireIn)
}

const PresignedObjectUploadPath = "/v1/presigned/object/upload"

type PresignedObjectUpload struct {
@@ -112,7 +116,7 @@ func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectComp
}

func (c *PresignedService) presign(req any, path string, method string, expireIn int) (string, error) {
u, err := url.Parse(c.baseURL)
u, err := url.Parse(c.cfg.URL)
if err != nil {
return "", err
}
@@ -126,7 +130,7 @@ func (c *PresignedService) presign(req any, path string, method string, expireIn

u.RawQuery = us.Encode()

prod := credentials.NewStaticCredentialsProvider(c.accessKey, c.secretKey, "")
prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "")
cred, err := prod.Retrieve(context.TODO())
if err != nil {
return "", err


+ 42
- 14
sdks/storage/cdsapi/presigned_test.go View File

@@ -9,12 +9,14 @@ import (

func Test_Presigned(t *testing.T) {
cli := NewClient(&Config{
URL: "http://localhost:7890",
URL: "http://localhost:7890",
AccessKey: "123456",
SecretKey: "123456",
})

Convey("下载文件", t, func() {
pre := cli.Presigned("123456", "123456")
url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{
pre := cli.Presigned()
url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{
UserID: 1,
PackageID: 3,
Path: "example.java",
@@ -26,7 +28,7 @@ func Test_Presigned(t *testing.T) {
})

Convey("上传文件", t, func() {
pre := cli.Presigned("123456", "123456")
pre := cli.Presigned()
url, err := pre.ObjectUpload(PresignedObjectUpload{
UserID: 1,
PackageID: 3,
@@ -37,14 +39,16 @@ func Test_Presigned(t *testing.T) {
})
}

func Test_PresignedObjectDownload(t *testing.T) {
func Test_PresignedObjectDownloadByPath(t *testing.T) {
cli := NewClient(&Config{
URL: "http://localhost:7890",
URL: "http://localhost:7890",
AccessKey: "123456",
SecretKey: "123456",
})

Convey("下载文件", t, func() {
pre := cli.Presigned("123456", "123456")
url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{
pre := cli.Presigned()
url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{
UserID: 1,
PackageID: 3,
Path: "example.java",
@@ -56,13 +60,33 @@ func Test_PresignedObjectDownload(t *testing.T) {
})
}

func Test_PresignedObjectDownload(t *testing.T) {
cli := NewClient(&Config{
URL: "http://localhost:7890",
AccessKey: "123456",
SecretKey: "123456",
})

Convey("下载文件", t, func() {
pre := cli.Presigned()
url, err := pre.ObjectDownload(PresignedObjectDownload{
UserID: 1,
ObjectID: 1039,
// Offset: 1,
// Length: types.Ref(int64(100)),
}, 100)
So(err, ShouldEqual, nil)
t.Logf("url: %s", url)
})
}

func Test_PresignedObjectUpload(t *testing.T) {
cli := NewClient(&Config{
URL: "http://localhost:7890",
})

Convey("上传文件", t, func() {
pre := cli.Presigned("123456", "123456")
pre := cli.Presigned()
url, err := pre.ObjectUpload(PresignedObjectUpload{
UserID: 1,
PackageID: 3,
@@ -79,7 +103,7 @@ func Test_PresignedNewMultipartUpload(t *testing.T) {
})

Convey("启动分片上传", t, func() {
pre := cli.Presigned("123456", "123456")
pre := cli.Presigned()
url, err := pre.ObjectNewMultipartUpload(PresignedObjectNewMultipartUpload{
UserID: 1,
PackageID: 3,
@@ -92,11 +116,13 @@ func Test_PresignedNewMultipartUpload(t *testing.T) {

func Test_PresignedObjectUploadPart(t *testing.T) {
cli := NewClient(&Config{
URL: "http://localhost:7890",
URL: "http://localhost:7890",
AccessKey: "123456",
SecretKey: "123456",
})

Convey("上传分片", t, func() {
pre := cli.Presigned("123456", "123456")
pre := cli.Presigned()
url, err := pre.ObjectUploadPart(PresignedObjectUploadPart{
UserID: 1,
ObjectID: 7,
@@ -109,11 +135,13 @@ func Test_PresignedObjectUploadPart(t *testing.T) {

func Test_PresignedCompleteMultipartUpload(t *testing.T) {
cli := NewClient(&Config{
URL: "http://localhost:7890",
URL: "http://localhost:7890",
AccessKey: "123456",
SecretKey: "123456",
})

Convey("合并分片", t, func() {
pre := cli.Presigned("123456", "123456")
pre := cli.Presigned()
url, err := pre.ObjectCompleteMultipartUpload(PresignedObjectCompleteMultipartUpload{
UserID: 1,
ObjectID: 7,


+ 114
- 0
sdks/storage/cdsapi/signer.go View File

@@ -0,0 +1,114 @@
package cdsapi

import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"time"

v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/credentials"
)

const (
AuthService = "jcs"
AuthRegion = "any"
)

// 对一个请求进行签名,并将签名信息添加到请求头中。
//
// 会读取请求体计算sha256哈希值。如果hash值已知,可以使用SignWithPayloadHash方法。
func Sign(req *http.Request, accessKey, secretKey string) error {
prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
cred, err := prod.Retrieve(context.TODO())
if err != nil {
return err
}

payloadHash := ""
if req.Body != nil {
data, err := io.ReadAll(req.Body)
if err != nil {
return err
}
req.Body.Close()
req.Body = io.NopCloser(bytes.NewReader(data))

hasher := sha256.New()
hasher.Write(data)
payloadHash = hex.EncodeToString(hasher.Sum(nil))
} else {
hash := sha256.Sum256([]byte(""))
payloadHash = hex.EncodeToString(hash[:])
}

signer := v4.NewSigner()
err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now())
if err != nil {
return err
}

return nil
}

// 对一个请求进行签名,不计算请求体的哈希,适合上传文件接口。
func SignWithoutBody(req *http.Request, accessKey, secretKey string) error {
prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
cred, err := prod.Retrieve(context.TODO())
if err != nil {
return err
}

signer := v4.NewSigner()
err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now())
if err != nil {
return err
}

return nil
}

// 对一个请求进行签名,签名时使用指定的哈希值作为请求体的哈希值。
//
// 参数payloadHash必须为sha256哈希值的16进制字符串,全小写。
func SignWithPayloadHash(req *http.Request, payloadHash string, accessKey, secretKey string) error {
prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
cred, err := prod.Retrieve(context.TODO())
if err != nil {
return err
}

signer := v4.NewSigner()
err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now())
if err != nil {
return err
}

return nil
}

// 生成一个带签名的URL。
//
// expiration为签名过期时间,单位为秒。
//
// 签名时不会包含请求体的哈希值。注:不要设置任何额外的Header(除了自动添加的Host),以免签名校验不通过
func Presign(req *http.Request, accessKey, secretKey string, expiration int) (string, error) {
query := req.URL.Query()
query.Add("X-Expires", fmt.Sprintf("%v", expiration))

req.URL.RawQuery = query.Encode()

prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
cred, err := prod.Retrieve(context.TODO())
if err != nil {
return "", err
}

signer := v4.NewSigner()
signedURL, _, err := signer.PresignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now())
return signedURL, err
}

+ 34
- 80
sdks/storage/cdsapi/storage.go View File

@@ -1,14 +1,10 @@
package cdsapi

import (
"fmt"
"net/url"
"strings"
"net/http"

"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder"
)

const StorageLoadPackagePath = "/storage/loadPackage"
@@ -19,31 +15,19 @@ type StorageLoadPackageReq struct {
StorageID cdssdk.StorageID `json:"storageID" binding:"required"`
RootPath string `json:"rootPath"`
}

func (r *StorageLoadPackageReq) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, StorageLoadPackagePath, r)
}

type StorageLoadPackageResp struct{}

func (r *StorageLoadPackageResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) {
url, err := url.JoinPath(c.baseURL, StorageLoadPackagePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[StorageLoadPackageResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, codeResp.ToError()
return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageLoadPackageResp{})
}

const StorageCreatePackagePath = "/storage/createPackage"
@@ -57,71 +41,41 @@ type StorageCreatePackageReq struct {
StorageAffinity cdssdk.StorageID `json:"storageAffinity"`
}

func (r *StorageCreatePackageReq) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, StorageCreatePackagePath, r)
}

type StorageCreatePackageResp struct {
PackageID cdssdk.PackageID `json:"packageID"`
Package cdssdk.Package `json:"package"`
}

func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *Client) StorageCreatePackage(req StorageCreatePackageReq) (*StorageCreatePackageResp, error) {
url, err := url.JoinPath(c.baseURL, StorageCreatePackagePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, http2.ContentTypeJSON) {
var codeResp response[StorageCreatePackageResp]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, codeResp.ToError()
}

return nil, fmt.Errorf("unknow response content type: %s", contType)
return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageCreatePackageResp{})
}

const StorageGetPath = "/storage/get"

type StorageGet struct {
UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"`
StorageID cdssdk.StorageID `form:"storageID" json:"storageID" binding:"required"`
UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"`
StorageID cdssdk.StorageID `form:"storageID" url:"storageID" binding:"required"`
}

func (r *StorageGet) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, StorageGetPath, r)
}

type StorageGetResp struct {
cdssdk.Storage
}

func (r *StorageGetResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *Client) StorageGet(req StorageGet) (*StorageGetResp, error) {
url, err := url.JoinPath(c.baseURL, StorageGetPath)
if err != nil {
return nil, err
}

resp, err := http2.GetForm(url, http2.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[StorageGetResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, codeResp.ToError()
return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageGetResp{})
}

+ 56
- 0
sdks/storage/cdsapi/storage_test.go View File

@@ -245,3 +245,59 @@ func Test_Cache(t *testing.T) {
So(err, ShouldBeNil)
})
}

func Test_Sign(t *testing.T) {
Convey("签名接口", t, func() {
cli := NewClient(&Config{
URL: "http://localhost:7890/v1",
AccessKey: "123456",
SecretKey: "123456",
})

fileData := make([]byte, 4096)
for i := 0; i < len(fileData); i++ {
fileData[i] = byte(i)
}

pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreate{
UserID: 1,
BucketID: 1,
Name: pkgName,
})
So(err, ShouldBeNil)

_, err = cli.Object().Upload(ObjectUpload{
ObjectUploadInfo: ObjectUploadInfo{
UserID: 1,
PackageID: createResp.Package.PackageID,
},
Files: iterator.Array(
&UploadingObject{
Path: "abc/test",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
&UploadingObject{
Path: "test4",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
),
})
So(err, ShouldBeNil)

getResp, err := cli.Package().Get(PackageGetReq{
UserID: 1,
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)

So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID)
So(getResp.Package.Name, ShouldEqual, pkgName)

err = cli.Package().Delete(PackageDelete{
UserID: 1,
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)
})
}

+ 21
- 45
sdks/storage/cdsapi/user.go View File

@@ -1,11 +1,10 @@
package cdsapi

import (
"net/url"
"net/http"

"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/http2"
)

const UserCreatePath = "/v1/user/create"
@@ -13,32 +12,21 @@ const UserCreatePath = "/v1/user/create"
type UserCreate struct {
Name string `json:"name" binding:"required"`
}

func (r *UserCreate) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, UserCreatePath, r)
}

type UserCreateResp struct {
User cdssdk.User `json:"user"`
}

func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) {
url, err := url.JoinPath(c.baseURL, UserCreatePath)
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}
codeResp, err := ParseJSONResponse[response[UserCreateResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
func (r *UserCreateResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return nil, codeResp.ToError()
func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) {
return JSONAPI(c.cfg, http.DefaultClient, req, &UserCreateResp{})
}

const UserDeletePath = "/v1/user/delete"
@@ -47,28 +35,16 @@ type UserDelete struct {
UserID cdssdk.UserID `json:"userID" binding:"required"`
}

type UserDeleteResp struct{}

func (c *Client) UserDelete(req *UserDelete) error {
url, err := url.JoinPath(c.baseURL, UserDeletePath)
if err != nil {
return err
}
func (r *UserDelete) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, UserDeletePath, r)
}

resp, err := http2.PostJSON(url, http2.RequestParam{
Body: req,
})
if err != nil {
return err
}
codeResp, err := ParseJSONResponse[response[UserDeleteResp]](resp)
if err != nil {
return err
}
type UserDeleteResp struct{}

if codeResp.Code == errorcode.OK {
return nil
}
func (r *UserDeleteResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

return codeResp.ToError()
func (c *Client) UserDelete(req *UserDelete) error {
return JSONAPINoData(c.cfg, http.DefaultClient, req)
}

+ 159
- 0
sdks/storage/cdsapi/utils.go View File

@@ -1,12 +1,19 @@
package cdsapi

import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"mime/multipart"
"net/http"
ul "net/url"
"path/filepath"
"strings"

"github.com/google/go-querystring/query"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/common/sdks"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/serder"
@@ -36,3 +43,155 @@ func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) {

return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math2.Min(len(strCont), 200)])
}

func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req, resp Resp) (Resp, error) {

param := req.MakeParam()

httpReq, err := param.MakeRequest(cfg.URL)
if err != nil {
return resp, err
}

if cfg.AccessKey != "" && cfg.SecretKey != "" {
err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey)
if err != nil {
return resp, err
}
}

httpResp, err := cli.Do(httpReq)
if err != nil {
return resp, err
}

err = resp.ParseResponse(httpResp)
return resp, err
}

func JSONAPINoData[Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error {
param := req.MakeParam()

httpReq, err := param.MakeRequest(cfg.URL)
if err != nil {
return err
}

if cfg.AccessKey != "" && cfg.SecretKey != "" {
err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey)
if err != nil {
return err
}
}

resp, err := cli.Do(httpReq)
if err != nil {
return err
}

return sdks.ParseCodeDataJSONResponse(resp, any(nil))
}

func calcSha256(body sdks.RequestBody) string {
hasher := sha256.New()
switch body := body.(type) {
case *sdks.StringBody:
hasher.Write([]byte(body.Value))
return hex.EncodeToString(hasher.Sum(nil))

case *sdks.BytesBody:
hasher.Write(body.Value)
return hex.EncodeToString(hasher.Sum(nil))

case *sdks.StreamBody:
return ""

default:
hash := sha256.Sum256([]byte(""))
return hex.EncodeToString(hash[:])
}
}

func PostMultiPart(cfg *Config, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) {
req, err := http.NewRequest(http.MethodPost, url, nil)
if err != nil {
return nil, err
}

pr, pw := io.Pipe()
muWriter := multipart.NewWriter(pw)

req.Header.Set("Content-Type", fmt.Sprintf("%s;boundary=%s", http2.ContentTypeMultiPart, muWriter.Boundary()))

writeResult := make(chan error, 1)
go func() {
writeResult <- func() error {
defer pw.Close()
defer muWriter.Close()

if info != nil {
mp, err := query.Values(info)
if err != nil {
return fmt.Errorf("formValues object to map failed, err: %w", err)
}

for k, v := range mp {
err := muWriter.WriteField(k, v[0])
if err != nil {
return fmt.Errorf("write form field failed, err: %w", err)
}
}
}

for {
file, err := files.MoveNext()
if err == iterator.ErrNoMoreItem {
break
}
if err != nil {
return fmt.Errorf("opening file: %w", err)
}

err = sendFileOnePart(muWriter, file.FieldName, file.FileName, file.File)
file.File.Close()
if err != nil {
return err
}
}

return nil
}()
}()

req.Body = pr

if cfg.AccessKey != "" && cfg.SecretKey != "" {
err = SignWithoutBody(req, cfg.AccessKey, cfg.SecretKey)
if err != nil {
return nil, err
}
}

cli := http.Client{}
resp, err := cli.Do(req)
if err != nil {
return nil, err
}

writeErr := <-writeResult
if writeErr != nil {
return nil, writeErr
}

return resp, nil
}

func sendFileOnePart(muWriter *multipart.Writer, fieldName, fileName string, file io.ReadCloser) error {
w, err := muWriter.CreateFormFile(fieldName, ul.PathEscape(fileName))
if err != nil {
return fmt.Errorf("create form file failed, err: %w", err)
}

_, err = io.Copy(w, file)
return err
}

+ 49
- 44
sdks/uploader/models.go View File

@@ -25,14 +25,24 @@ func (BlockChain) TableName() string {
}

type Binding struct {
ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"`
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
Name string `gorm:"column:name" json:"Name"`
DataType string `gorm:"column:data_type" json:"dataType"`
//JsonData string `gorm:"column:json_data" json:"jsonData"`
Content string `gorm:"column:content" json:"Content"`
AccessLevel string `gorm:"column:access_level" json:"accessLevel"`
CreateTime time.Time `gorm:"column:created_at" json:"createTime"`
ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"`
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
Name string `gorm:"column:name" json:"Name"`
DataType string `gorm:"column:data_type" json:"dataType"`
Content string `gorm:"column:content" json:"Content"`
AccessLevel string `gorm:"column:access_level" json:"accessLevel"`
CreateTime time.Time `gorm:"column:created_at" json:"createTime"`
}

type BindingDAO struct {
ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"`
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
Name string `gorm:"column:name" json:"Name"`
DataType string `gorm:"column:data_type" json:"dataType"`
Content string `gorm:"column:content" json:"Content"`
AccessLevel string `gorm:"column:access_level" json:"accessLevel"`
CreateTime time.Time `gorm:"column:created_at" json:"createTime"`
BindingCluster []BindingCluster `gorm:"foreignKey:binding_id;references:id" json:"bindingCluster"`
}

type BindingAccessData struct {
@@ -56,7 +66,7 @@ type BindingDetail struct {
SSOId string `json:"ssoID"`
Name string `json:"Name"`
Info sch.DataBinding `json:"info"`
Packages []Package `json:"packages"`
Package Package `json:"package"`
Status string `json:"status"`
AccessLevel string `json:"accessLevel"`
CreateTime time.Time `json:"createTime"`
@@ -70,9 +80,14 @@ type BindingCluster struct {
BindingID DataID `gorm:"column:binding_id" json:"bindingID"`
ClusterID ClusterID `gorm:"column:cluster_id" json:"clusterID"`
Status string `gorm:"column:status" json:"status"`
Param string `gorm:"column:param" json:"Param"`
JsonData string `gorm:"column:json_data" json:"jsonData"`
}

func (BindingCluster) TableName() string {
return "bindingCluster" // 确保和数据库中的表名一致
}

type Folder struct {
PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"`
Path string `gorm:"column:path_name" json:"path"`
@@ -97,12 +112,12 @@ func (Cluster) TableName() string {
}

type Package struct {
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"`
PackageName string `gorm:"column:package_name" json:"packageName"`
BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"`
DataType string `gorm:"column:data_type" json:"dataType"`
JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"`
PackageName string `gorm:"column:package_name" json:"packageName"`
BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"`
DataType string `gorm:"column:data_type" json:"dataType"`
//JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段
BindingID DataID `gorm:"column:binding_id" json:"bindingID"`
CreateTime time.Time `gorm:"column:create_time" json:"createTime"`
Objects []cdssdk.Object `gorm:"column:objects" json:"objects"`
@@ -113,12 +128,12 @@ type Package struct {
}

type PackageDAO struct {
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"`
PackageName string `gorm:"column:package_name" json:"packageName"`
BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"`
DataType string `gorm:"column:data_type" json:"dataType"`
JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段
UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"`
PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"`
PackageName string `gorm:"column:package_name" json:"packageName"`
BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"`
DataType string `gorm:"column:data_type" json:"dataType"`
//JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段
BindingID DataID `gorm:"column:binding_id" json:"bindingID"`
CreateTime time.Time `gorm:"column:create_time" json:"createTime"`
UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据
@@ -134,9 +149,10 @@ type PackageCloneDAO struct {
Description string `gorm:"column:description" json:"description"`
BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"`
ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"`
ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"`
ImageID string `gorm:"column:image_id" json:"imageID"`
CreateTime time.Time `gorm:"column:created_at" json:"createTime"`
//ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"`
ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"`
BindingID DataID `gorm:"column:binding_id" json:"bindingID"`
CreateTime time.Time `gorm:"column:created_at" json:"createTime"`
}

func (PackageCloneDAO) TableName() string {
@@ -144,15 +160,14 @@ func (PackageCloneDAO) TableName() string {
}

type PackageCloneParam struct {
PackageID cdssdk.PackageID `json:"packageID" binding:"required"`
PackageName string `json:"packageName" binding:"required"`
//BucketID cdssdk.BucketID `json:"bucketID" binding:"required"`
PackageID cdssdk.PackageID `json:"packageID" binding:"required"`
PackageName string `json:"packageName" binding:"required"`
Name string `json:"name"`
Description string `json:"description"`
BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"`
ClusterID schsdk.ClusterID `json:"clusterID"`
ParentImageID schsdk.ImageID `json:"parentImageID"`
ImageID string `json:"imageID"`
Output string `json:"output"`
ImageID schsdk.ImageID `json:"imageID"`
}

type PackageCloneVO struct {
@@ -163,10 +178,11 @@ type PackageCloneVO struct {
Description string `gorm:"column:description" json:"description"`
BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"`
ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"`
ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"`
ImageID string `gorm:"column:image_id" json:"imageID"`
CreateTime time.Time `gorm:"column:created_at" json:"createTime"`
ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"`
//ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"`
ImageID string `gorm:"column:image_id" json:"imageID"`
BindingID DataID `gorm:"column:binding_id" json:"bindingID"`
CreateTime time.Time `gorm:"column:created_at" json:"createTime"`
ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"`
}

type ClusterMapping struct {
@@ -181,17 +197,6 @@ func (ClusterMapping) TableName() string {
return "ClusterMapping"
}

//type PackageCloneClusterDAO struct {
// ID DataID `gorm:"column:id" json:"ID"`
// ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"`
// ClusterName string `gorm:"column:cluster_name" json:"clusterName"`
//}
//
//type PackageCloneCluster struct {
// ClusterID schsdk.ClusterID `json:"clusterID"`
// ClusterName string `json:"clusterName"`
//}

type ScheduleTarget interface {
Noop()
}


+ 53
- 54
sdks/uploader/uploader.go View File

@@ -3,7 +3,6 @@ package uploadersdk
import (
"fmt"
"gitlink.org.cn/cloudream/common/pkgs/types"
sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
"gitlink.org.cn/cloudream/common/utils/http2"
@@ -12,65 +11,65 @@ import (
"strings"
)

type DataScheduleReq struct {
PackageID cdssdk.PackageID `json:"packageID"`
DataType string `json:"dataType"`
ScheduleTarget ScheduleTarget `json:"scheduleTarget"`
}
//type DataScheduleReq struct {
// PackageID cdssdk.PackageID `json:"packageID"`
// DataType string `json:"dataType"`
// ScheduleTarget ScheduleTarget `json:"scheduleTarget"`
//}

//type DataScheduleResp struct {
// Results []sch.DataScheduleResult `json:"data"`
//}

type TmpDataScheduleResult struct {
Cluster sch.DataDetail `json:"cluster"`
PackageID cdssdk.PackageID `json:"packageID"`
Status bool `json:"status"`
Msg string `json:"msg"`
}

func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) {
targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule")
if err != nil {
return nil, err
}

resp, err := http2.PostJSON(targetUrl, http2.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}
println(resp.Body)

contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, http2.ContentTypeJSON) {
var codeResp response[[]TmpDataScheduleResult]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}

if codeResp.Code == ResponseCodeOK {
var results []sch.DataScheduleResult
for _, tmpResult := range codeResp.Data {
result := sch.DataScheduleResult{
PackageID: tmpResult.PackageID,
Status: tmpResult.Status,
Msg: tmpResult.Msg,
Clusters: []sch.DataDetail{
tmpResult.Cluster,
},
}
results = append(results, result)
}
return results, nil
}

return nil, codeResp.ToError()
}
//type TmpDataScheduleResult struct {
// Cluster sch.DataDetail `json:"cluster"`
// PackageID cdssdk.PackageID `json:"packageID"`
// Status bool `json:"status"`
// Msg string `json:"msg"`
//}

return nil, fmt.Errorf("unknow response content type: %s", contType)
}
//func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) {
// targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule")
// if err != nil {
// return nil, err
// }
//
// resp, err := http2.PostJSON(targetUrl, http2.RequestParam{
// Body: req,
// })
// if err != nil {
// return nil, err
// }
// println(resp.Body)
//
// contType := resp.Header.Get("Content-Type")
// if strings.Contains(contType, http2.ContentTypeJSON) {
// var codeResp response[[]TmpDataScheduleResult]
// if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
// return nil, fmt.Errorf("parsing response: %w", err)
// }
//
// if codeResp.Code == ResponseCodeOK {
// var results []sch.DataScheduleResult
// for _, tmpResult := range codeResp.Data {
// result := sch.DataScheduleResult{
// PackageID: tmpResult.PackageID,
// Status: tmpResult.Status,
// Msg: tmpResult.Msg,
// Clusters: []sch.DataDetail{
// tmpResult.Cluster,
// },
// }
// results = append(results, result)
// }
// return results, nil
// }
//
// return nil, codeResp.ToError()
// }
//
// return nil, fmt.Errorf("unknow response content type: %s", contType)
//}

type UploadReq struct {
DataType string `json:"dataType"`


+ 14
- 0
utils/io2/io.go View File

@@ -196,3 +196,17 @@ func DropWithBuf(str io.Reader, buf []byte) {
}
}
}

func ReadMost(str io.Reader, n int) ([]byte, error) {
buf := make([]byte, n)
n, err := io.ReadFull(str, buf)
if err == nil {
return buf, nil
}

if err == io.EOF || err == io.ErrUnexpectedEOF {
return buf[:n], nil
}

return buf[:n], err
}

+ 10
- 0
utils/serder/serder.go View File

@@ -80,6 +80,16 @@ func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) {
return ret, nil
}

func JSONToObjectStreamExRaw(stream io.Reader, ret any) error {
dec := defaultAPI.NewDecoder(stream)
err := dec.Decode(ret)
if err != nil {
return err
}

return nil
}

// 将对象转为JSON字符串。如果需要支持解析TypeUnion类型,则使用"Ex"结尾的同名函数。
//
// 注:[]byte会被base64编码,如果要JSON内容要给外部解析,那么应该避免使用[]byte。


Loading…
Cancel
Save