You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

parser.go 21 kB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950
  1. package plans
  2. import (
  3. "fmt"
  4. "math"
  5. "gitlink.org.cn/cloudream/common/pkgs/ipfs"
  6. cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
  7. "gitlink.org.cn/cloudream/common/utils/lo2"
  8. "gitlink.org.cn/cloudream/common/utils/math2"
  9. )
  10. type FromToParser interface {
  11. Parse(ft FromTo, blder *PlanBuilder) error
  12. }
  13. type DefaultParser struct {
  14. EC cdssdk.ECRedundancy
  15. }
  16. func NewParser(ec cdssdk.ECRedundancy) *DefaultParser {
  17. return &DefaultParser{
  18. EC: ec,
  19. }
  20. }
  21. type ParseContext struct {
  22. Ft FromTo
  23. Nodes []*Node
  24. ToNodes []*Node
  25. // 为了产生所有To所需的数据范围,而需要From打开的范围。
  26. // 这个范围是基于整个文件的,且上下界都取整到条带大小的整数倍,因此上界是有可能超过文件大小的。
  27. StreamRange Range
  28. }
  29. func (p *DefaultParser) Parse(ft FromTo, blder *PlanBuilder) error {
  30. ctx := ParseContext{Ft: ft}
  31. // 分成两个阶段:
  32. // 1. 基于From和To生成更多指令,初步匹配to的需求
  33. // 计算一下打开流的范围
  34. p.calcStreamRange(&ctx)
  35. err := p.extend(&ctx, ft, blder)
  36. if err != nil {
  37. return err
  38. }
  39. // 2. 优化上一步生成的指令
  40. // 对于删除指令的优化,需要反复进行,直到没有变化为止。
  41. // 从目前实现上来说不会死循环
  42. for {
  43. opted := false
  44. if p.removeUnusedJoin(&ctx) {
  45. opted = true
  46. }
  47. if p.removeUnusedMultiplyOutput(&ctx) {
  48. opted = true
  49. }
  50. if p.removeUnusedSplit(&ctx) {
  51. opted = true
  52. }
  53. if p.omitSplitJoin(&ctx) {
  54. opted = true
  55. }
  56. if !opted {
  57. break
  58. }
  59. }
  60. // 确定指令执行位置的过程,也需要反复进行,直到没有变化为止。
  61. // 从目前实现上来说不会死循环
  62. for {
  63. opted := false
  64. if p.pinIPFSRead(&ctx) {
  65. opted = true
  66. }
  67. if p.pinJoin(&ctx) {
  68. opted = true
  69. }
  70. if p.pinMultiply(&ctx) {
  71. opted = true
  72. }
  73. if p.pinSplit(&ctx) {
  74. opted = true
  75. }
  76. if !opted {
  77. break
  78. }
  79. }
  80. // 下面这些只需要执行一次,但需要按顺序
  81. p.dropUnused(&ctx)
  82. p.storeIPFSWriteResult(&ctx)
  83. p.generateClone(&ctx)
  84. p.generateRange(&ctx)
  85. p.generateSend(&ctx)
  86. return p.buildPlan(&ctx, blder)
  87. }
  88. func (p *DefaultParser) findOutputStream(ctx *ParseContext, dataIndex int) *StreamVar {
  89. for _, op := range ctx.Nodes {
  90. for _, o := range op.OutputStreams {
  91. if o.DataIndex == dataIndex {
  92. return o
  93. }
  94. }
  95. }
  96. return nil
  97. }
  98. // 计算输入流的打开范围。会把流的范围按条带大小取整
  99. func (p *DefaultParser) calcStreamRange(ctx *ParseContext) {
  100. stripSize := int64(p.EC.ChunkSize * p.EC.K)
  101. rng := Range{
  102. Offset: math.MaxInt64,
  103. }
  104. for _, to := range ctx.Ft.Toes {
  105. if to.GetDataIndex() == -1 {
  106. toRng := to.GetRange()
  107. rng.ExtendStart(math2.Floor(toRng.Offset, stripSize))
  108. if toRng.Length != nil {
  109. rng.ExtendEnd(math2.Ceil(toRng.Offset+*toRng.Length, stripSize))
  110. } else {
  111. rng.Length = nil
  112. }
  113. } else {
  114. toRng := to.GetRange()
  115. blkStartIndex := math2.FloorDiv(toRng.Offset, int64(p.EC.ChunkSize))
  116. rng.ExtendStart(blkStartIndex * stripSize)
  117. if toRng.Length != nil {
  118. blkEndIndex := math2.CeilDiv(toRng.Offset+*toRng.Length, int64(p.EC.ChunkSize))
  119. rng.ExtendEnd(blkEndIndex * stripSize)
  120. } else {
  121. rng.Length = nil
  122. }
  123. }
  124. }
  125. ctx.StreamRange = rng
  126. }
  127. func (p *DefaultParser) extend(ctx *ParseContext, ft FromTo, blder *PlanBuilder) error {
  128. for _, f := range ft.Froms {
  129. n, err := p.buildFromNode(ctx, &ft, f)
  130. if err != nil {
  131. return err
  132. }
  133. ctx.Nodes = append(ctx.Nodes, n)
  134. // 对于完整文件的From,生成Split指令
  135. if f.GetDataIndex() == -1 {
  136. splitOp := &Node{
  137. Env: nil,
  138. Type: &ChunkedSplitType{ChunkSize: p.EC.ChunkSize, PaddingZeros: true},
  139. }
  140. splitOp.AddInputStream(n.OutputStreams[0])
  141. for i := 0; i < p.EC.K; i++ {
  142. splitOp.NewOutputStream(i)
  143. }
  144. ctx.Nodes = append(ctx.Nodes, splitOp)
  145. }
  146. }
  147. // 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令
  148. ecInputStrs := make(map[int]*StreamVar)
  149. loop:
  150. for _, o := range ctx.Nodes {
  151. for _, s := range o.OutputStreams {
  152. if s.DataIndex >= 0 && ecInputStrs[s.DataIndex] == nil {
  153. ecInputStrs[s.DataIndex] = s
  154. if len(ecInputStrs) == p.EC.K {
  155. break loop
  156. }
  157. }
  158. }
  159. }
  160. if len(ecInputStrs) == p.EC.K {
  161. mulOp := &Node{
  162. Env: nil,
  163. Type: &MultiplyOp{EC: p.EC},
  164. }
  165. for _, s := range ecInputStrs {
  166. mulOp.AddInputStream(s)
  167. }
  168. for i := 0; i < p.EC.N; i++ {
  169. mulOp.NewOutputStream(i)
  170. }
  171. ctx.Nodes = append(ctx.Nodes, mulOp)
  172. joinOp := &Node{
  173. Env: nil,
  174. Type: &ChunkedJoinType{p.EC.ChunkSize},
  175. }
  176. for i := 0; i < p.EC.K; i++ {
  177. // 不可能找不到流
  178. joinOp.AddInputStream(p.findOutputStream(ctx, i))
  179. }
  180. joinOp.NewOutputStream(-1)
  181. ctx.Nodes = append(ctx.Nodes, joinOp)
  182. }
  183. // 为每一个To找到一个输入流
  184. for _, t := range ft.Toes {
  185. n, err := p.buildToNode(&ft, t)
  186. if err != nil {
  187. return err
  188. }
  189. ctx.Nodes = append(ctx.Nodes, n)
  190. ctx.ToNodes = append(ctx.ToNodes, n)
  191. str := p.findOutputStream(ctx, t.GetDataIndex())
  192. if str == nil {
  193. return fmt.Errorf("no output stream found for data index %d", t.GetDataIndex())
  194. }
  195. n.AddInputStream(str)
  196. }
  197. return nil
  198. }
  199. func (p *DefaultParser) buildFromNode(ctx *ParseContext, ft *FromTo, f From) (*Node, error) {
  200. var repRange Range
  201. var blkRange Range
  202. repRange.Offset = ctx.StreamRange.Offset
  203. blkRange.Offset = ctx.StreamRange.Offset / int64(p.EC.ChunkSize*p.EC.K) * int64(p.EC.ChunkSize)
  204. if ctx.StreamRange.Length != nil {
  205. repRngLen := *ctx.StreamRange.Length
  206. repRange.Length = &repRngLen
  207. blkRngLen := *ctx.StreamRange.Length / int64(p.EC.ChunkSize*p.EC.K) * int64(p.EC.ChunkSize)
  208. blkRange.Length = &blkRngLen
  209. }
  210. switch f := f.(type) {
  211. case *FromNode:
  212. ty := &IPFSReadType{
  213. FileHash: f.FileHash,
  214. Option: ipfs.ReadOption{
  215. Offset: 0,
  216. Length: -1,
  217. },
  218. }
  219. if f.DataIndex == -1 {
  220. ty.Option.Offset = repRange.Offset
  221. if repRange.Length != nil {
  222. ty.Option.Length = *repRange.Length
  223. }
  224. } else {
  225. ty.Option.Offset = blkRange.Offset
  226. if blkRange.Length != nil {
  227. ty.Option.Length = *blkRange.Length
  228. }
  229. }
  230. n := &Node{
  231. Type: ty,
  232. }
  233. n.NewOutputStream(f.DataIndex)
  234. if f.Node != nil {
  235. n.Env = &AgentEnv{Node: *f.Node}
  236. }
  237. return n, nil
  238. case *FromExecutor:
  239. n := &Node{
  240. Env: &ExecutorEnv{},
  241. Type: &FromExecutorOp{Handle: f.Handle},
  242. }
  243. n.NewOutputStream(f.DataIndex)
  244. if f.DataIndex == -1 {
  245. f.Handle.RangeHint.Offset = repRange.Offset
  246. f.Handle.RangeHint.Length = repRange.Length
  247. } else {
  248. f.Handle.RangeHint.Offset = blkRange.Offset
  249. f.Handle.RangeHint.Length = blkRange.Length
  250. }
  251. return n, nil
  252. default:
  253. return nil, fmt.Errorf("unsupported from type %T", f)
  254. }
  255. }
  256. func (p *DefaultParser) buildToNode(ft *FromTo, t To) (*Node, error) {
  257. switch t := t.(type) {
  258. case *ToNode:
  259. n := &Node{
  260. Env: &AgentEnv{t.Node},
  261. Type: &IPFSWriteType{FileHashStoreKey: t.FileHashStoreKey, Range: t.Range},
  262. }
  263. n.NewOutputVar(StringValueVar)
  264. return n, nil
  265. case *ToExecutor:
  266. return &Node{
  267. Env: &ExecutorEnv{},
  268. Type: &ToExecutorOp{Handle: t.Handle, Range: t.Range},
  269. }, nil
  270. default:
  271. return nil, fmt.Errorf("unsupported to type %T", t)
  272. }
  273. }
  274. // 删除输出流未被使用的Join指令
  275. func (p *DefaultParser) removeUnusedJoin(ctx *ParseContext) bool {
  276. opted := false
  277. for i, op := range ctx.Nodes {
  278. _, ok := op.Type.(*ChunkedJoinType)
  279. if !ok {
  280. continue
  281. }
  282. if len(op.OutputStreams[0].Toes) > 0 {
  283. continue
  284. }
  285. for _, in := range op.InputStreams {
  286. in.RemoveTo(op)
  287. }
  288. ctx.Nodes[i] = nil
  289. opted = true
  290. }
  291. ctx.Nodes = lo2.RemoveAllDefault(ctx.Nodes)
  292. return opted
  293. }
  294. // 减少未使用的Multiply指令的输出流。如果减少到0,则删除该指令
  295. func (p *DefaultParser) removeUnusedMultiplyOutput(ctx *ParseContext) bool {
  296. opted := false
  297. for i, op := range ctx.Nodes {
  298. _, ok := op.Type.(*MultiplyOp)
  299. if !ok {
  300. continue
  301. }
  302. for i2, out := range op.OutputStreams {
  303. if len(out.Toes) > 0 {
  304. continue
  305. }
  306. op.OutputStreams[i2] = nil
  307. opted = true
  308. }
  309. op.OutputStreams = lo2.RemoveAllDefault(op.OutputStreams)
  310. if len(op.OutputStreams) == 0 {
  311. for _, in := range op.InputStreams {
  312. in.RemoveTo(op)
  313. }
  314. ctx.Nodes[i] = nil
  315. opted = true
  316. }
  317. }
  318. ctx.Nodes = lo2.RemoveAllDefault(ctx.Nodes)
  319. return opted
  320. }
  321. // 删除未使用的Split指令
  322. func (p *DefaultParser) removeUnusedSplit(ctx *ParseContext) bool {
  323. opted := false
  324. for i, op := range ctx.Nodes {
  325. _, ok := op.Type.(*ChunkedSplitType)
  326. if !ok {
  327. continue
  328. }
  329. // Split出来的每一个流都没有被使用,才能删除这个指令
  330. isAllUnused := true
  331. for _, out := range op.OutputStreams {
  332. if len(out.Toes) > 0 {
  333. isAllUnused = false
  334. break
  335. }
  336. }
  337. if isAllUnused {
  338. op.InputStreams[0].RemoveTo(op)
  339. ctx.Nodes[i] = nil
  340. opted = true
  341. }
  342. }
  343. ctx.Nodes = lo2.RemoveAllDefault(ctx.Nodes)
  344. return opted
  345. }
  346. // 如果Split的结果被完全用于Join,则省略Split和Join指令
  347. func (p *DefaultParser) omitSplitJoin(ctx *ParseContext) bool {
  348. opted := false
  349. loop:
  350. for iSplit, splitOp := range ctx.Nodes {
  351. // 进行合并操作时会删除多个指令,因此这里存在splitOp == nil的情况
  352. if splitOp == nil {
  353. continue
  354. }
  355. _, ok := splitOp.Type.(*ChunkedSplitType)
  356. if !ok {
  357. continue
  358. }
  359. // Split指令的每一个输出都有且只有一个目的地
  360. var joinOp *Node
  361. for _, out := range splitOp.OutputStreams {
  362. if len(out.Toes) != 1 {
  363. continue
  364. }
  365. if joinOp == nil {
  366. joinOp = out.Toes[0]
  367. } else if joinOp != out.Toes[0] {
  368. continue loop
  369. }
  370. }
  371. if joinOp == nil {
  372. continue
  373. }
  374. // 且这个目的地要是一个Join指令
  375. _, ok = joinOp.Type.(*ChunkedJoinType)
  376. if !ok {
  377. continue
  378. }
  379. // 同时这个Join指令的输入也必须全部来自Split指令的输出。
  380. // 由于上面判断了Split指令的输出目的地都相同,所以这里只要判断Join指令的输入数量是否与Split指令的输出数量相同即可
  381. if len(joinOp.InputStreams) != len(splitOp.OutputStreams) {
  382. continue
  383. }
  384. // 所有条件都满足,可以开始省略操作,将Join操作的目的地的输入流替换为Split操作的输入流:
  385. // F->Split->Join->T 变换为:F->T
  386. splitOp.InputStreams[0].RemoveTo(splitOp)
  387. for i := len(joinOp.OutputStreams[0].Toes) - 1; i >= 0; i-- {
  388. joinOp.OutputStreams[0].Toes[i].ReplaceInputStream(joinOp.OutputStreams[0], splitOp.InputStreams[0])
  389. }
  390. // 并删除这两个指令
  391. ctx.Nodes[iSplit] = nil
  392. lo2.Clear(ctx.Nodes, joinOp)
  393. opted = true
  394. }
  395. ctx.Nodes = lo2.RemoveAllDefault(ctx.Nodes)
  396. return opted
  397. }
  398. // 确定Split命令的执行位置
  399. func (p *DefaultParser) pinSplit(ctx *ParseContext) bool {
  400. opted := false
  401. for _, op := range ctx.Nodes {
  402. _, ok := op.Type.(*ChunkedSplitType)
  403. if !ok {
  404. continue
  405. }
  406. // 如果Split的每一个流的目的地都是同一个,则将Split固定在这个地方执行
  407. var toEnv OpEnv
  408. useToEnv := true
  409. for _, out := range op.OutputStreams {
  410. for _, to := range out.Toes {
  411. // 如果某个流的目的地也不确定,则将其视为与其他流的目的地相同
  412. if to.Env == nil {
  413. continue
  414. }
  415. if toEnv == nil {
  416. toEnv = to.Env
  417. } else if toEnv.Equals(to.Env) {
  418. useToEnv = false
  419. break
  420. }
  421. }
  422. if !useToEnv {
  423. break
  424. }
  425. }
  426. // 所有输出流的目的地都不确定,那么就不能根据输出流去固定
  427. if toEnv == nil {
  428. useToEnv = false
  429. }
  430. if useToEnv {
  431. if op.Env == nil || !op.Env.Equals(toEnv) {
  432. opted = true
  433. }
  434. op.Env = toEnv
  435. continue
  436. }
  437. // 此时查看输入流的始发地是否可以确定,可以的话使用这个位置
  438. fromEnv := op.InputStreams[0].From.Env
  439. if fromEnv != nil {
  440. if op.Env == nil || !op.Env.Equals(fromEnv) {
  441. opted = true
  442. }
  443. op.Env = fromEnv
  444. }
  445. }
  446. return opted
  447. }
  448. // 确定Join命令的执行位置,策略与固定Split类似
  449. func (p *DefaultParser) pinJoin(ctx *ParseContext) bool {
  450. opted := false
  451. for _, op := range ctx.Nodes {
  452. _, ok := op.Type.(*ChunkedJoinType)
  453. if !ok {
  454. continue
  455. }
  456. // 先查看输出流的目的地是否可以确定,可以的话使用这个位置
  457. var toEnv OpEnv
  458. for _, to := range op.OutputStreams[0].Toes {
  459. if to.Env == nil {
  460. continue
  461. }
  462. if toEnv == nil {
  463. toEnv = to.Env
  464. } else if !toEnv.Equals(to.Env) {
  465. toEnv = nil
  466. break
  467. }
  468. }
  469. if toEnv != nil {
  470. if op.Env == nil || !op.Env.Equals(toEnv) {
  471. opted = true
  472. }
  473. op.Env = toEnv
  474. continue
  475. }
  476. // 否则根据输入流的始发地来固定
  477. var fromEnv OpEnv
  478. for _, in := range op.InputStreams {
  479. if in.From.Env == nil {
  480. continue
  481. }
  482. if fromEnv == nil {
  483. fromEnv = in.From.Env
  484. } else if !fromEnv.Equals(in.From.Env) {
  485. // 输入流的始发地不同,那也必须选一个作为固定位置
  486. break
  487. }
  488. }
  489. // 所有输入流的始发地都不确定,那没办法了
  490. if fromEnv != nil {
  491. if op.Env == nil || !op.Env.Equals(fromEnv) {
  492. opted = true
  493. }
  494. op.Env = fromEnv
  495. continue
  496. }
  497. }
  498. return opted
  499. }
  500. // 确定Multiply命令的执行位置
  501. func (p *DefaultParser) pinMultiply(ctx *ParseContext) bool {
  502. opted := false
  503. for _, op := range ctx.Nodes {
  504. _, ok := op.Type.(*MultiplyOp)
  505. if !ok {
  506. continue
  507. }
  508. var toEnv OpEnv
  509. for _, out := range op.OutputStreams {
  510. for _, to := range out.Toes {
  511. if to.Env == nil {
  512. continue
  513. }
  514. if toEnv == nil {
  515. toEnv = to.Env
  516. } else if !toEnv.Equals(to.Env) {
  517. toEnv = nil
  518. break
  519. }
  520. }
  521. }
  522. if toEnv != nil {
  523. if op.Env == nil || !op.Env.Equals(toEnv) {
  524. opted = true
  525. }
  526. op.Env = toEnv
  527. continue
  528. }
  529. // 否则根据输入流的始发地来固定
  530. var fromEnv OpEnv
  531. for _, in := range op.InputStreams {
  532. if in.From.Env == nil {
  533. continue
  534. }
  535. if fromEnv == nil {
  536. fromEnv = in.From.Env
  537. } else if !fromEnv.Equals(in.From.Env) {
  538. // 输入流的始发地不同,那也必须选一个作为固定位置
  539. break
  540. }
  541. }
  542. // 所有输入流的始发地都不确定,那没办法了
  543. if fromEnv != nil {
  544. if op.Env == nil || !op.Env.Equals(fromEnv) {
  545. opted = true
  546. }
  547. op.Env = fromEnv
  548. continue
  549. }
  550. }
  551. return opted
  552. }
  553. // 确定IPFS读取指令的执行位置
  554. func (p *DefaultParser) pinIPFSRead(ctx *ParseContext) bool {
  555. opted := false
  556. for _, op := range ctx.Nodes {
  557. _, ok := op.Type.(*IPFSReadType)
  558. if !ok {
  559. continue
  560. }
  561. if op.Env != nil {
  562. continue
  563. }
  564. var toEnv OpEnv
  565. for _, to := range op.OutputStreams[0].Toes {
  566. if to.Env == nil {
  567. continue
  568. }
  569. if toEnv == nil {
  570. toEnv = to.Env
  571. } else if !toEnv.Equals(to.Env) {
  572. toEnv = nil
  573. break
  574. }
  575. }
  576. if toEnv != nil {
  577. if op.Env == nil || !op.Env.Equals(toEnv) {
  578. opted = true
  579. }
  580. op.Env = toEnv
  581. }
  582. }
  583. return opted
  584. }
  585. // 对于所有未使用的流,增加Drop指令
  586. func (p *DefaultParser) dropUnused(ctx *ParseContext) {
  587. for _, op := range ctx.Nodes {
  588. for _, out := range op.OutputStreams {
  589. if len(out.Toes) == 0 {
  590. dropOp := &Node{
  591. Env: op.Env,
  592. Type: &DropOp{},
  593. }
  594. dropOp.AddInputStream(out)
  595. ctx.Nodes = append(ctx.Nodes, dropOp)
  596. }
  597. }
  598. }
  599. }
  600. // 为IPFS写入指令存储结果
  601. func (p *DefaultParser) storeIPFSWriteResult(ctx *ParseContext) {
  602. for _, op := range ctx.Nodes {
  603. w, ok := op.Type.(*IPFSWriteType)
  604. if !ok {
  605. continue
  606. }
  607. if w.FileHashStoreKey == "" {
  608. continue
  609. }
  610. storeOp := &Node{
  611. Env: &ExecutorEnv{},
  612. Type: &StoreOp{
  613. StoreKey: w.FileHashStoreKey,
  614. },
  615. }
  616. storeOp.AddInputVar(op.OutputValues[0])
  617. ctx.Nodes = append(ctx.Nodes, storeOp)
  618. }
  619. }
  620. // 生成Range指令。StreamRange可能超过文件总大小,但Range指令会在数据量不够时不报错而是正常返回
  621. func (p *DefaultParser) generateRange(ctx *ParseContext) {
  622. for i, to := range ctx.ToNodes {
  623. toDataIdx := ctx.Ft.Toes[i].GetDataIndex()
  624. toRng := ctx.Ft.Toes[i].GetRange()
  625. if toDataIdx == -1 {
  626. rngType := &RangeType{Range: Range{Offset: toRng.Offset - ctx.StreamRange.Offset, Length: toRng.Length}}
  627. rngNode := &Node{
  628. Env: to.InputStreams[0].From.Env,
  629. Type: rngType,
  630. }
  631. rngNode.AddInputStream(to.InputStreams[0])
  632. to.ReplaceInputStream(to.InputStreams[0], rngNode.NewOutputStream(toDataIdx))
  633. ctx.Nodes = append(ctx.Nodes, rngNode)
  634. } else {
  635. stripSize := int64(p.EC.ChunkSize * p.EC.K)
  636. blkStartIdx := ctx.StreamRange.Offset / stripSize
  637. blkStart := blkStartIdx * int64(p.EC.ChunkSize)
  638. rngType := &RangeType{Range: Range{Offset: toRng.Offset - blkStart, Length: toRng.Length}}
  639. rngNode := &Node{
  640. Env: to.InputStreams[0].From.Env,
  641. Type: rngType,
  642. }
  643. rngNode.AddInputStream(to.InputStreams[0])
  644. to.ReplaceInputStream(to.InputStreams[0], rngNode.NewOutputStream(toDataIdx))
  645. ctx.Nodes = append(ctx.Nodes, rngNode)
  646. }
  647. }
  648. }
  649. // 生成Clone指令
  650. func (p *DefaultParser) generateClone(ctx *ParseContext) {
  651. for _, op := range ctx.Nodes {
  652. for _, out := range op.OutputStreams {
  653. if len(out.Toes) <= 1 {
  654. continue
  655. }
  656. cloneOp := &Node{
  657. Env: op.Env,
  658. Type: &CloneStreamType{},
  659. }
  660. for i := len(out.Toes) - 1; i >= 0; i-- {
  661. out.Toes[i].ReplaceInputStream(out, cloneOp.NewOutputStream(out.DataIndex))
  662. }
  663. out.Toes = nil
  664. cloneOp.AddInputStream(out)
  665. ctx.Nodes = append(ctx.Nodes, cloneOp)
  666. }
  667. for _, out := range op.OutputValues {
  668. if len(out.Toes) <= 1 {
  669. continue
  670. }
  671. cloneOp := &Node{
  672. Env: op.Env,
  673. Type: &CloneVarType{},
  674. }
  675. for i := len(out.Toes) - 1; i >= 0; i-- {
  676. out.Toes[i].ReplaceInputVar(out, cloneOp.NewOutputVar(out.Type))
  677. }
  678. out.Toes = nil
  679. cloneOp.AddInputVar(out)
  680. ctx.Nodes = append(ctx.Nodes, cloneOp)
  681. }
  682. }
  683. }
  684. // 生成Send指令
  685. func (p *DefaultParser) generateSend(ctx *ParseContext) {
  686. for _, op := range ctx.Nodes {
  687. for _, out := range op.OutputStreams {
  688. toOp := out.Toes[0]
  689. if toOp.Env.Equals(op.Env) {
  690. continue
  691. }
  692. switch toOp.Env.(type) {
  693. case *ExecutorEnv:
  694. // 如果是要送到Executor,则只能由Executor主动去拉取
  695. getStrOp := &Node{
  696. Env: &ExecutorEnv{},
  697. Type: &GetStreamOp{},
  698. }
  699. // 同时需要对此变量生成HoldUntil指令,避免Plan结束时Get指令还未到达
  700. holdOp := &Node{
  701. Env: op.Env,
  702. Type: &HoldUntilOp{},
  703. }
  704. holdOp.AddInputVar(getStrOp.NewOutputVar(SignalValueVar))
  705. holdOp.AddInputStream(out)
  706. getStrOp.AddInputStream(holdOp.NewOutputStream(out.DataIndex))
  707. toOp.ReplaceInputStream(out, getStrOp.NewOutputStream(out.DataIndex))
  708. ctx.Nodes = append(ctx.Nodes, holdOp)
  709. ctx.Nodes = append(ctx.Nodes, getStrOp)
  710. case *AgentEnv:
  711. // 如果是要送到Agent,则可以直接发送
  712. sendStrOp := &Node{
  713. Env: op.Env,
  714. Type: &SendStreamOp{},
  715. }
  716. out.Toes = nil
  717. sendStrOp.AddInputStream(out)
  718. toOp.ReplaceInputStream(out, sendStrOp.NewOutputStream(out.DataIndex))
  719. ctx.Nodes = append(ctx.Nodes, sendStrOp)
  720. }
  721. }
  722. for _, out := range op.OutputValues {
  723. toOp := out.Toes[0]
  724. if toOp.Env.Equals(op.Env) {
  725. continue
  726. }
  727. switch toOp.Env.(type) {
  728. case *ExecutorEnv:
  729. // 如果是要送到Executor,则只能由Executor主动去拉取
  730. getStrOp := &Node{
  731. Env: &ExecutorEnv{},
  732. Type: &GetVarOp{},
  733. }
  734. // 同时需要对此变量生成HoldUntil指令,避免Plan结束时Get指令还未到达
  735. holdOp := &Node{
  736. Env: op.Env,
  737. Type: &HoldUntilOp{},
  738. }
  739. holdOp.AddInputVar(getStrOp.NewOutputVar(SignalValueVar))
  740. holdOp.AddInputVar(out)
  741. getStrOp.AddInputVar(holdOp.NewOutputVar(out.Type))
  742. toOp.ReplaceInputVar(out, getStrOp.NewOutputVar(out.Type))
  743. ctx.Nodes = append(ctx.Nodes, holdOp)
  744. ctx.Nodes = append(ctx.Nodes, getStrOp)
  745. case *AgentEnv:
  746. // 如果是要送到Agent,则可以直接发送
  747. sendVarOp := &Node{
  748. Env: op.Env,
  749. Type: &SendVarOp{},
  750. }
  751. out.Toes = nil
  752. sendVarOp.AddInputVar(out)
  753. toOp.ReplaceInputVar(out, sendVarOp.NewOutputVar(out.Type))
  754. ctx.Nodes = append(ctx.Nodes, sendVarOp)
  755. }
  756. }
  757. }
  758. }
  759. // 生成Plan
  760. func (p *DefaultParser) buildPlan(ctx *ParseContext, blder *PlanBuilder) error {
  761. for _, op := range ctx.Nodes {
  762. for _, out := range op.OutputStreams {
  763. if out.Var != nil {
  764. continue
  765. }
  766. out.Var = blder.NewStreamVar()
  767. }
  768. for _, in := range op.InputStreams {
  769. if in.Var != nil {
  770. continue
  771. }
  772. in.Var = blder.NewStreamVar()
  773. }
  774. for _, out := range op.OutputValues {
  775. if out.Var != nil {
  776. continue
  777. }
  778. switch out.Type {
  779. case StringValueVar:
  780. out.Var = blder.NewStringVar()
  781. case SignalValueVar:
  782. out.Var = blder.NewSignalVar()
  783. }
  784. }
  785. for _, in := range op.InputValues {
  786. if in.Var != nil {
  787. continue
  788. }
  789. switch in.Type {
  790. case StringValueVar:
  791. in.Var = blder.NewStringVar()
  792. case SignalValueVar:
  793. in.Var = blder.NewSignalVar()
  794. }
  795. }
  796. if err := op.Type.GenerateOp(op, blder); err != nil {
  797. return err
  798. }
  799. }
  800. return nil
  801. }

本项目旨在将云际存储公共基础设施化,使个人及企业可低门槛使用高效的云际存储服务(安装开箱即用云际存储客户端即可,无需关注其他组件的部署),同时支持用户灵活便捷定制云际存储的功能细节。