From 9c310e193ea7578a299147937307c48f46b62734 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Mon, 24 Feb 2025 15:23:37 +0800 Subject: [PATCH 01/64] =?UTF-8?q?feat:=20.devops/pcm1=E7=8E=AF=E5=A2=83cic?= =?UTF-8?q?d.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .devops/pcm1环境cicd.yml diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml new file mode 100644 index 0000000..f04df5e --- /dev/null +++ b/.devops/pcm1环境cicd.yml @@ -0,0 +1,32 @@ +version: 2 +name: pcm1环境cicd +description: "" +global: + concurrent: 1 +workflow: + - ref: start + name: 开始 + task: start + - ref: ssh_cmd_0 + name: ssh执行命令 + task: ssh_cmd@1.1.0 + input: + ssh_private_key: ((SSH.pcm1)) + ssh_ip: '"123.60.146.162"' + ssh_port: '"22"' + ssh_user: '"pcm"' + ssh_cmd: '" + ls -l /home/pcm/deploy/cloudream/storage + && + cd /home/pcm/deploy/cloudream/storage + && + chmod +x build.sh + && + sh build.sh "' + needs: + - start + - ref: end + name: 结束 + task: end + needs: + - ssh_cmd_0 \ No newline at end of file From 5cc94047ecc1f4296ae0680f7747cc82ca98d883 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Tue, 25 Feb 2025 17:45:13 +0800 Subject: [PATCH 02/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index f04df5e..19cf28a 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -16,13 +16,15 @@ workflow: ssh_port: '"22"' ssh_user: '"pcm"' ssh_cmd: '" - ls -l /home/pcm/deploy/cloudream/storage + cd /home/pcm/deploy/cloudream/workspace/storage_yml && - cd /home/pcm/deploy/cloudream/storage + chmod +x 1_pull.sh 2_build_all.sh 3_deploy.sh && - chmod +x build.sh + sh 1_pull.sh && - sh build.sh "' + sh 2_build_all.sh + && + sh 3_deploy.sh "' needs: - start - ref: end From 32867185d20b92e3ce61b8f88358ffa0fb7626bc Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Tue, 25 Feb 2025 22:41:02 +0800 Subject: [PATCH 03/64] =?UTF-8?q?feat:=20.devops/=E6=9C=AA=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E9=A1=B9=E7=9B=AE.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/未命名项目.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .devops/未命名项目.yml diff --git a/.devops/未命名项目.yml b/.devops/未命名项目.yml new file mode 100644 index 0000000..31e48c8 --- /dev/null +++ b/.devops/未命名项目.yml @@ -0,0 +1,28 @@ +version: 2 +name: 未命名项目 +description: "" +global: + concurrent: 1 +workflow: + - ref: start + name: 开始 + task: start + - ref: ssh_cmd_0 + name: ssh执行命令 + task: ssh_cmd@1.1.1 + input: + ssh_pass: ((SSH.pcm1)) + ssh_ip: '"123.60.146.162"' + ssh_port: '"22"' + ssh_user: '"pcm"' + ssh_cmd: '"cd /home/pcm/deploy/cloudream/workspace/storage_yml&&chmod +x + 1_pull.sh 2_build_all.sh 3_deploy.sh&&sh 1_pull.sh&&sh + 2_build_all.sh&&sh 3_deploy.sh"' + needs: + - start + - ref: end + name: 结束 + task: end + needs: + - ssh_cmd_0 + From 072e2ece968447091f8989afbcc7fed5f833252d Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Tue, 25 Feb 2025 22:43:44 +0800 Subject: [PATCH 04/64] =?UTF-8?q?refactor:=20.devops/=E6=9C=AA=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E9=A1=B9=E7=9B=AE.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/未命名项目.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.devops/未命名项目.yml b/.devops/未命名项目.yml index 31e48c8..ad0d770 100644 --- a/.devops/未命名项目.yml +++ b/.devops/未命名项目.yml @@ -15,9 +15,7 @@ workflow: ssh_ip: '"123.60.146.162"' ssh_port: '"22"' ssh_user: '"pcm"' - ssh_cmd: '"cd /home/pcm/deploy/cloudream/workspace/storage_yml&&chmod +x - 1_pull.sh 2_build_all.sh 3_deploy.sh&&sh 1_pull.sh&&sh - 2_build_all.sh&&sh 3_deploy.sh"' + ssh_cmd: '"ls /home/pcm/deploy/cloudream/workspace/storage_yml"' needs: - start - ref: end From 6c4f26ba0156ef8036fc09aa7ba8573517854e8e Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Tue, 25 Feb 2025 23:29:00 +0800 Subject: [PATCH 05/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 19cf28a..caa1908 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -18,8 +18,6 @@ workflow: ssh_cmd: '" cd /home/pcm/deploy/cloudream/workspace/storage_yml && - chmod +x 1_pull.sh 2_build_all.sh 3_deploy.sh - && sh 1_pull.sh && sh 2_build_all.sh From 558b4c873257402b32cc91018b2ca9f4e8f824a3 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Tue, 25 Feb 2025 23:41:04 +0800 Subject: [PATCH 06/64] =?UTF-8?q?refactor:=20delete=20.devops/=E6=9C=AA?= =?UTF-8?q?=E5=91=BD=E5=90=8D=E9=A1=B9=E7=9B=AE.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/未命名项目.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .devops/未命名项目.yml diff --git a/.devops/未命名项目.yml b/.devops/未命名项目.yml deleted file mode 100644 index ad0d770..0000000 --- a/.devops/未命名项目.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 -name: 未命名项目 -description: "" -global: - concurrent: 1 -workflow: - - ref: start - name: 开始 - task: start - - ref: ssh_cmd_0 - name: ssh执行命令 - task: ssh_cmd@1.1.1 - input: - ssh_pass: ((SSH.pcm1)) - ssh_ip: '"123.60.146.162"' - ssh_port: '"22"' - ssh_user: '"pcm"' - ssh_cmd: '"ls /home/pcm/deploy/cloudream/workspace/storage_yml"' - needs: - - start - - ref: end - name: 结束 - task: end - needs: - - ssh_cmd_0 - From 91cd32409a34066c63b822a2a58a059a6c420fea Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Tue, 25 Feb 2025 23:41:59 +0800 Subject: [PATCH 07/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index caa1908..e8f2c2a 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -18,11 +18,11 @@ workflow: ssh_cmd: '" cd /home/pcm/deploy/cloudream/workspace/storage_yml && - sh 1_pull.sh + ./1_pull.sh && - sh 2_build_all.sh + ./2_build_all.sh && - sh 3_deploy.sh "' + ./3_deploy.sh "' needs: - start - ref: end From 208a1dfae23b7ec7df1611f10d994452a7e0374d Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Tue, 25 Feb 2025 23:46:04 +0800 Subject: [PATCH 08/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index e8f2c2a..0631e59 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -18,11 +18,11 @@ workflow: ssh_cmd: '" cd /home/pcm/deploy/cloudream/workspace/storage_yml && - ./1_pull.sh + sudo ./1_pull.sh && - ./2_build_all.sh + sudo ./2_build_all.sh && - ./3_deploy.sh "' + sudo ./3_deploy.sh "' needs: - start - ref: end From 2eaecc088474a6fb124290ce237bd56ebc695794 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 09:38:48 +0800 Subject: [PATCH 09/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 0631e59..320644d 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -18,6 +18,8 @@ workflow: ssh_cmd: '" cd /home/pcm/deploy/cloudream/workspace/storage_yml && + which go + && sudo ./1_pull.sh && sudo ./2_build_all.sh From 929518554e07287a0a6982d0fd5d1ce4dd574614 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 09:39:57 +0800 Subject: [PATCH 10/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 320644d..4d2a9da 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -22,9 +22,7 @@ workflow: && sudo ./1_pull.sh && - sudo ./2_build_all.sh - && - sudo ./3_deploy.sh "' + sudo ./2_build_all.sh"' needs: - start - ref: end From a34134e5bb8fd4244d7346e3e7cb608f1284058c Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 09:41:50 +0800 Subject: [PATCH 11/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 4d2a9da..6ba0971 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -22,7 +22,7 @@ workflow: && sudo ./1_pull.sh && - sudo ./2_build_all.sh"' + sudo ./2_build_all.sh "' needs: - start - ref: end From 5dd2ae7e5d9ca10cb521c5501e5889021a6d7c2b Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 09:44:43 +0800 Subject: [PATCH 12/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 6ba0971..a233b24 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -18,8 +18,6 @@ workflow: ssh_cmd: '" cd /home/pcm/deploy/cloudream/workspace/storage_yml && - which go - && sudo ./1_pull.sh && sudo ./2_build_all.sh "' From 9c24e57f6466c6d8f080592436fb6f7fe4cc6330 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 09:47:28 +0800 Subject: [PATCH 13/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index a233b24..9f2b329 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -20,6 +20,8 @@ workflow: && sudo ./1_pull.sh && + echo $PATH + && sudo ./2_build_all.sh "' needs: - start From 1ed073aab33e0136bf079d8956c4ecd0df59d576 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 09:49:50 +0800 Subject: [PATCH 14/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 9f2b329..74bf6d4 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -22,6 +22,8 @@ workflow: && echo $PATH && + export PATH=$PATH:/usr/local/go/bin + && sudo ./2_build_all.sh "' needs: - start From f7433655aa63592de1ede112250208d83e609364 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 09:52:22 +0800 Subject: [PATCH 15/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 74bf6d4..a8e765f 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -24,6 +24,8 @@ workflow: && export PATH=$PATH:/usr/local/go/bin && + echo $PATH + && sudo ./2_build_all.sh "' needs: - start From 93cb53f11ab8d3c85ef8156f52acdecf924dd8dc Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 10:03:44 +0800 Subject: [PATCH 16/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index a8e765f..0bac59c 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -26,6 +26,10 @@ workflow: && echo $PATH && + echo \"export PATH=\$PATH:/usr/local/go/bin\" >> ~/.bashrc + && + source ~/.bashrc + && sudo ./2_build_all.sh "' needs: - start From d6d3f1c56212eee693b17709ada071d4262822fe Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 10:09:18 +0800 Subject: [PATCH 17/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 0bac59c..4c81200 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -26,9 +26,7 @@ workflow: && echo $PATH && - echo \"export PATH=\$PATH:/usr/local/go/bin\" >> ~/.bashrc - && - source ~/.bashrc + ls /usr/local/go/bin && sudo ./2_build_all.sh "' needs: From 33125fc8f601bed47b37838134637ee84fa4c46f Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 10:14:30 +0800 Subject: [PATCH 18/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 4c81200..5020ca0 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -18,7 +18,7 @@ workflow: ssh_cmd: '" cd /home/pcm/deploy/cloudream/workspace/storage_yml && - sudo ./1_pull.sh + sudo sh 1_pull.sh && echo $PATH && @@ -28,7 +28,7 @@ workflow: && ls /usr/local/go/bin && - sudo ./2_build_all.sh "' + sudo sh 2_build_all.sh "' needs: - start - ref: end From 074dc66a95e6427be38b6ef9670cd8f179cb640d Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 10:18:45 +0800 Subject: [PATCH 19/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 5020ca0..cd323e9 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -18,7 +18,7 @@ workflow: ssh_cmd: '" cd /home/pcm/deploy/cloudream/workspace/storage_yml && - sudo sh 1_pull.sh + sh 1_pull.sh && echo $PATH && @@ -28,7 +28,7 @@ workflow: && ls /usr/local/go/bin && - sudo sh 2_build_all.sh "' + sh 2_build_all.sh "' needs: - start - ref: end From b1358f2a0e1f65ba234650bd1bf993c855a1abe2 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 26 Feb 2025 11:08:32 +0800 Subject: [PATCH 20/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index cd323e9..8dc0b55 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -20,12 +20,6 @@ workflow: && sh 1_pull.sh && - echo $PATH - && - export PATH=$PATH:/usr/local/go/bin - && - echo $PATH - && ls /usr/local/go/bin && sh 2_build_all.sh "' From 2631f61f7a941fd54361b8cb916d44ebb981e842 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 27 Feb 2025 09:13:11 +0800 Subject: [PATCH 21/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 8dc0b55..06e11bb 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -22,7 +22,9 @@ workflow: && ls /usr/local/go/bin && - sh 2_build_all.sh "' + sh 2_build_all.sh + && + sudo sh 3_deploy.sh "' needs: - start - ref: end From 830164a09bc138c34780923307a401a6e7ec6e4c Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 27 Feb 2025 09:15:28 +0800 Subject: [PATCH 22/64] =?UTF-8?q?feat:=20.devops/=E6=9C=AA=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E9=A1=B9=E7=9B=AE.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/未命名项目.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .devops/未命名项目.yml diff --git a/.devops/未命名项目.yml b/.devops/未命名项目.yml new file mode 100644 index 0000000..7f48910 --- /dev/null +++ b/.devops/未命名项目.yml @@ -0,0 +1,31 @@ +version: 2 +name: 未命名项目 +description: "" +global: + concurrent: 1 +trigger: + webhook: gitlink@1.0.0 + event: + - ref: create_tag + ruleset: + - param-ref: tag + operator: EQ + value: '"部署"' + ruleset-operator: AND +workflow: + - ref: start + name: 开始 + task: start + - ref: end + name: 结束 + task: end + needs: + - shell_0 + - ref: shell_0 + name: shell + image: docker.jianmuhub.com/library/alpine:3.17.0 + script: + - ls + needs: + - start + From d0665838b5a9ccc7100159dd65a7dc56808d2f8a Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 27 Feb 2025 09:17:10 +0800 Subject: [PATCH 23/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 06e11bb..2af2b07 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -3,6 +3,15 @@ name: pcm1环境cicd description: "" global: concurrent: 1 +trigger: + webhook: gitlink@1.0.0 + event: + - ref: create_tag + ruleset: + - param-ref: tag + operator: EQ + value: '"部署"' + ruleset-operator: AND workflow: - ref: start name: 开始 From 7bd34f45c6aff9180c3567b87c98422ea295449e Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 27 Feb 2025 09:32:23 +0800 Subject: [PATCH 24/64] =?UTF-8?q?refactor:=20delete=20.devops/=E6=9C=AA?= =?UTF-8?q?=E5=91=BD=E5=90=8D=E9=A1=B9=E7=9B=AE.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/未命名项目.yml | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 .devops/未命名项目.yml diff --git a/.devops/未命名项目.yml b/.devops/未命名项目.yml deleted file mode 100644 index 7f48910..0000000 --- a/.devops/未命名项目.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: 2 -name: 未命名项目 -description: "" -global: - concurrent: 1 -trigger: - webhook: gitlink@1.0.0 - event: - - ref: create_tag - ruleset: - - param-ref: tag - operator: EQ - value: '"部署"' - ruleset-operator: AND -workflow: - - ref: start - name: 开始 - task: start - - ref: end - name: 结束 - task: end - needs: - - shell_0 - - ref: shell_0 - name: shell - image: docker.jianmuhub.com/library/alpine:3.17.0 - script: - - ls - needs: - - start - From 6e5bd91d895336ced0396838ec0942938226b1e6 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 27 Feb 2025 15:06:58 +0800 Subject: [PATCH 25/64] =?UTF-8?q?=E8=A7=A3=E5=86=B3BucketPool=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pkgs/ioswitch2/ops2/ec.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/pkgs/ioswitch2/ops2/ec.go b/common/pkgs/ioswitch2/ops2/ec.go index 6c4e43c..8fa3bdc 100644 --- a/common/pkgs/ioswitch2/ops2/ec.go +++ b/common/pkgs/ioswitch2/ops2/ec.go @@ -69,7 +69,7 @@ func (o *ECMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error { fut := future.NewSetVoid() go func() { mul := ec.GaloisMultiplier().BuildGalois() - defer outputBufPool.WakeUpAll() + defer outputBufPool.Close() readLens := math2.SplitLessThan(o.ChunkSize, 64*1024) readLenIdx := 0 @@ -113,7 +113,7 @@ func (o *ECMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error { }() go func() { - defer outputBufPool.WakeUpAll() + defer outputBufPool.Close() for { outputChunks, ok := outputBufPool.GetFilled() From cf99e797e9c6176c35a7cb6a9da9a4a5de273f4b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 28 Feb 2025 09:52:18 +0800 Subject: [PATCH 26/64] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E5=88=97=E8=A1=A8=E6=8E=A5=E5=8F=A3=E6=94=AF=E6=8C=81=E5=8F=AA?= =?UTF-8?q?=E9=9D=9E=E9=80=92=E5=BD=92=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/http/object.go | 10 +++---- client/internal/services/object.go | 10 +++---- common/pkgs/db2/object.go | 39 ++++++++++++++++++++++++++++ common/pkgs/mq/coordinator/object.go | 28 +++++++++++--------- coordinator/internal/mq/object.go | 30 ++++++++++++++++----- 5 files changed, 88 insertions(+), 29 deletions(-) diff --git a/client/internal/http/object.go b/client/internal/http/object.go index 6d140f0..34acdfd 100644 --- a/client/internal/http/object.go +++ b/client/internal/http/object.go @@ -39,14 +39,14 @@ func (s *ObjectService) ListByPath(ctx *gin.Context) { return } - objs, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, req.IsPrefix) + coms, objs, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, req.IsPrefix, req.NoRecursive) if err != nil { log.Warnf("listing objects: %s", err.Error()) ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("listing objects: %v", err))) return } - ctx.JSON(http.StatusOK, OK(cdsapi.ObjectListByPathResp{Objects: objs})) + ctx.JSON(http.StatusOK, OK(cdsapi.ObjectListByPathResp{CommonPrefixes: coms, Objects: objs})) } func (s *ObjectService) ListByIDs(ctx *gin.Context) { @@ -191,7 +191,7 @@ func (s *ObjectService) DownloadByPath(ctx *gin.Context) { return } - obj, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, false) + _, obj, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, false, false) if err != nil { log.Warnf("getting object by path: %s", err.Error()) ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get object by path failed")) @@ -266,7 +266,7 @@ func (s *ObjectService) UpdateInfoByPath(ctx *gin.Context) { return } - obj, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, true) + _, obj, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, true, false) if err != nil { log.Warnf("getting object by path: %s", err.Error()) ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get object by path failed")) @@ -343,7 +343,7 @@ func (s *ObjectService) DeleteByPath(ctx *gin.Context) { return } - obj, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, false) + _, obj, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, false, false) if err != nil { log.Warnf("getting object by path: %s", err.Error()) ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get object by path failed")) diff --git a/client/internal/services/object.go b/client/internal/services/object.go index 4c2478d..1476d35 100644 --- a/client/internal/services/object.go +++ b/client/internal/services/object.go @@ -22,19 +22,19 @@ func (svc *Service) ObjectSvc() *ObjectService { return &ObjectService{Service: svc} } -func (svc *ObjectService) GetByPath(userID cdssdk.UserID, pkgID cdssdk.PackageID, path string, isPrefix bool) ([]cdssdk.Object, error) { +func (svc *ObjectService) GetByPath(userID cdssdk.UserID, pkgID cdssdk.PackageID, path string, isPrefix bool, noRecursive bool) ([]string, []cdssdk.Object, error) { coorCli, err := stgglb.CoordinatorMQPool.Acquire() if err != nil { - return nil, fmt.Errorf("new coordinator client: %w", err) + return nil, nil, fmt.Errorf("new coordinator client: %w", err) } defer stgglb.CoordinatorMQPool.Release(coorCli) - listResp, err := coorCli.GetObjectsByPath(coormq.ReqGetObjectsByPath(userID, pkgID, path, isPrefix)) + listResp, err := coorCli.GetObjectsByPath(coormq.ReqGetObjectsByPath(userID, pkgID, path, isPrefix, noRecursive)) if err != nil { - return nil, fmt.Errorf("requsting to coodinator: %w", err) + return nil, nil, fmt.Errorf("requsting to coodinator: %w", err) } - return listResp.Objects, nil + return listResp.CommonPrefixes, listResp.Objects, nil } func (svc *ObjectService) GetByIDs(userID cdssdk.UserID, objectIDs []cdssdk.ObjectID) ([]*cdssdk.Object, error) { diff --git a/common/pkgs/db2/object.go b/common/pkgs/db2/object.go index 6d86e30..9f0cef3 100644 --- a/common/pkgs/db2/object.go +++ b/common/pkgs/db2/object.go @@ -2,6 +2,7 @@ package db2 import ( "fmt" + "strings" "time" "gorm.io/gorm/clause" @@ -38,6 +39,44 @@ func (db *ObjectDB) GetWithPathPrefix(ctx SQLContext, packageID cdssdk.PackageID return ret, err } +func (db *ObjectDB) GetCommonPrefixes(ctx SQLContext, packageID cdssdk.PackageID, pathPrefix string) ([]string, error) { + var ret []string + + sepCnt := strings.Count(pathPrefix, cdssdk.ObjectPathSeparator) + 1 + + prefixStatm := fmt.Sprintf("Substring_Index(Path, '%s', %d)", cdssdk.ObjectPathSeparator, sepCnt) + + err := ctx.Table("Object").Select(prefixStatm+" as Prefix"). + Where("PackageID = ?", packageID). + Where("Path like ?", pathPrefix+"%"). + Where(prefixStatm + " <> Path"). + Group("Prefix").Find(&ret).Error + if err != nil { + return nil, err + } + + for i := range ret { + ret[i] = ret[i] + cdssdk.ObjectPathSeparator + } + + return ret, nil +} + +func (db *ObjectDB) GetDirectChildren(ctx SQLContext, packageID cdssdk.PackageID, pathPrefix string) ([]cdssdk.Object, error) { + var ret []cdssdk.Object + + sepCnt := strings.Count(pathPrefix, cdssdk.ObjectPathSeparator) + 1 + + prefixStatm := fmt.Sprintf("Substring_Index(Path, '%s', %d)", cdssdk.ObjectPathSeparator, sepCnt) + + err := ctx.Table("Object"). + Where("PackageID = ?", packageID). + Where("Path like ?", pathPrefix+"%"). + Where(prefixStatm + " = Path"). + Find(&ret).Error + return ret, err +} + func (db *ObjectDB) BatchTestObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) (map[cdssdk.ObjectID]bool, error) { if len(objectIDs) == 0 { return make(map[cdssdk.ObjectID]bool), nil diff --git a/common/pkgs/mq/coordinator/object.go b/common/pkgs/mq/coordinator/object.go index ad63e6e..4ee0005 100644 --- a/common/pkgs/mq/coordinator/object.go +++ b/common/pkgs/mq/coordinator/object.go @@ -67,27 +67,31 @@ var _ = Register(Service.GetObjectsByPath) type GetObjectsByPath struct { mq.MessageBodyBase - UserID cdssdk.UserID `json:"userID"` - PackageID cdssdk.PackageID `json:"packageID"` - Path string `json:"path"` - IsPrefix bool `json:"isPrefix"` + UserID cdssdk.UserID `json:"userID"` + PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` + IsPrefix bool `json:"isPrefix"` + NoRecursive bool `json:"noRecursive"` } type GetObjectsByPathResp struct { mq.MessageBodyBase - Objects []model.Object `json:"objects"` + CommonPrefixes []string `json:"commonPrefixes"` + Objects []model.Object `json:"objects"` } -func ReqGetObjectsByPath(userID cdssdk.UserID, packageID cdssdk.PackageID, path string, isPrefix bool) *GetObjectsByPath { +func ReqGetObjectsByPath(userID cdssdk.UserID, packageID cdssdk.PackageID, path string, isPrefix bool, noRecursive bool) *GetObjectsByPath { return &GetObjectsByPath{ - UserID: userID, - PackageID: packageID, - Path: path, - IsPrefix: isPrefix, + UserID: userID, + PackageID: packageID, + Path: path, + IsPrefix: isPrefix, + NoRecursive: noRecursive, } } -func RespGetObjectsByPath(objects []model.Object) *GetObjectsByPathResp { +func RespGetObjectsByPath(commonPrefixes []string, objects []model.Object) *GetObjectsByPathResp { return &GetObjectsByPathResp{ - Objects: objects, + CommonPrefixes: commonPrefixes, + Objects: objects, } } func (client *Client) GetObjectsByPath(msg *GetObjectsByPath) (*GetObjectsByPathResp, error) { diff --git a/coordinator/internal/mq/object.go b/coordinator/internal/mq/object.go index 1c2e256..fa5e012 100644 --- a/coordinator/internal/mq/object.go +++ b/coordinator/internal/mq/object.go @@ -56,6 +56,7 @@ func (svc *Service) GetObjects(msg *coormq.GetObjects) (*coormq.GetObjectsResp, } func (svc *Service) GetObjectsByPath(msg *coormq.GetObjectsByPath) (*coormq.GetObjectsByPathResp, *mq.CodeMessage) { + var coms []string var objs []cdssdk.Object err := svc.db2.DoTx(func(tx db2.SQLContext) error { var err error @@ -65,16 +66,31 @@ func (svc *Service) GetObjectsByPath(msg *coormq.GetObjectsByPath) (*coormq.GetO return fmt.Errorf("getting package by id: %w", err) } - if msg.IsPrefix { - objs, err = svc.db2.Object().GetWithPathPrefix(tx, msg.PackageID, msg.Path) - if err != nil { - return fmt.Errorf("getting objects with prefix: %w", err) - } - } else { + if !msg.IsPrefix { objs, err = svc.db2.Object().GetByPath(tx, msg.PackageID, msg.Path) if err != nil { return fmt.Errorf("getting object by path: %w", err) } + + return nil + } + + if !msg.NoRecursive { + objs, err = svc.db2.Object().GetWithPathPrefix(tx, msg.PackageID, msg.Path) + if err != nil { + return fmt.Errorf("getting objects with prefix: %w", err) + } + return nil + } + + coms, err = svc.db2.Object().GetCommonPrefixes(tx, msg.PackageID, msg.Path) + if err != nil { + return fmt.Errorf("getting common prefixes: %w", err) + } + + objs, err = svc.db2.Object().GetDirectChildren(tx, msg.PackageID, msg.Path) + if err != nil { + return fmt.Errorf("getting direct children: %w", err) } return nil @@ -84,7 +100,7 @@ func (svc *Service) GetObjectsByPath(msg *coormq.GetObjectsByPath) (*coormq.GetO return nil, mq.Failed(errorcode.OperationFailed, "get objects with prefix failed") } - return mq.ReplyOK(coormq.RespGetObjectsByPath(objs)) + return mq.ReplyOK(coormq.RespGetObjectsByPath(coms, objs)) } func (svc *Service) GetPackageObjects(msg *coormq.GetPackageObjects) (*coormq.GetPackageObjectsResp, *mq.CodeMessage) { From 7d7cf8b3f058869bfb863e01813d02e10839d5ab Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 3 Mar 2025 17:18:57 +0800 Subject: [PATCH 27/64] =?UTF-8?q?ObjectBlock=E8=A1=A8=E5=A2=9E=E5=8A=A0Siz?= =?UTF-8?q?e=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/models/models.go | 3 ++ common/pkgs/db2/object.go | 1 + common/pkgs/ioswitch2/ops2/bypass.go | 2 +- common/pkgs/ioswitch2/ops2/shard_store.go | 12 ++++--- common/pkgs/uploader/create_load.go | 2 +- common/pkgs/uploader/update.go | 2 +- .../event/check_package_redundancy.go | 34 ++++++++++++++----- scanner/internal/event/clean_pinned.go | 8 ++++- 8 files changed, 47 insertions(+), 17 deletions(-) diff --git a/common/models/models.go b/common/models/models.go index a86afbe..47d326e 100644 --- a/common/models/models.go +++ b/common/models/models.go @@ -11,6 +11,7 @@ type ObjectBlock struct { Index int `gorm:"column:Index; primaryKey; type:int" json:"index"` StorageID cdssdk.StorageID `gorm:"column:StorageID; primaryKey; type:bigint" json:"storageID"` // 这个块应该在哪个节点上 FileHash cdssdk.FileHash `gorm:"column:FileHash; type:char(68); not null" json:"fileHash"` + Size int64 `gorm:"column:Size; type:bigint" json:"size"` } func (ObjectBlock) TableName() string { @@ -77,6 +78,7 @@ type GrouppedObjectBlock struct { ObjectID cdssdk.ObjectID Index int FileHash cdssdk.FileHash + Size int64 StorageIDs []cdssdk.StorageID } @@ -89,6 +91,7 @@ func (o *ObjectDetail) GroupBlocks() []GrouppedObjectBlock { ObjectID: block.ObjectID, Index: block.Index, FileHash: block.FileHash, + Size: block.Size, } } grp.StorageIDs = append(grp.StorageIDs, block.StorageID) diff --git a/common/pkgs/db2/object.go b/common/pkgs/db2/object.go index 9f0cef3..14633ed 100644 --- a/common/pkgs/db2/object.go +++ b/common/pkgs/db2/object.go @@ -356,6 +356,7 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds [] Index: 0, StorageID: stgID, FileHash: add.FileHash, + Size: add.Size, }) } } diff --git a/common/pkgs/ioswitch2/ops2/bypass.go b/common/pkgs/ioswitch2/ops2/bypass.go index 8af1be7..496c0db 100644 --- a/common/pkgs/ioswitch2/ops2/bypass.go +++ b/common/pkgs/ioswitch2/ops2/bypass.go @@ -76,7 +76,7 @@ func (o *BypassToShardStore) Execute(ctx *exec.ExecContext, e *exec.Executor) er } e.PutVar(o.BypassCallback, &BypassHandleResultValue{Commited: true}) - e.PutVar(o.FileHash, &FileHashValue{Hash: fileInfo.Hash}) + e.PutVar(o.FileHash, &ShardInfoValue{Hash: fileInfo.Hash, Size: fileInfo.Size}) return nil } diff --git a/common/pkgs/ioswitch2/ops2/shard_store.go b/common/pkgs/ioswitch2/ops2/shard_store.go index 4fab018..5878884 100644 --- a/common/pkgs/ioswitch2/ops2/shard_store.go +++ b/common/pkgs/ioswitch2/ops2/shard_store.go @@ -19,15 +19,16 @@ import ( func init() { exec.UseOp[*ShardRead]() exec.UseOp[*ShardWrite]() - exec.UseVarValue[*FileHashValue]() + exec.UseVarValue[*ShardInfoValue]() } -type FileHashValue struct { +type ShardInfoValue struct { Hash cdssdk.FileHash `json:"hash"` + Size int64 `json:"size"` } -func (v *FileHashValue) Clone() exec.VarValue { - return &FileHashValue{Hash: v.Hash} +func (v *ShardInfoValue) Clone() exec.VarValue { + return &ShardInfoValue{Hash: v.Hash, Size: v.Size} } type ShardRead struct { @@ -105,8 +106,9 @@ func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error { return fmt.Errorf("writing file to shard store: %w", err) } - e.PutVar(o.FileHash, &FileHashValue{ + e.PutVar(o.FileHash, &ShardInfoValue{ Hash: fileInfo.Hash, + Size: fileInfo.Size, }) return nil } diff --git a/common/pkgs/uploader/create_load.go b/common/pkgs/uploader/create_load.go index 3827c16..1bc7fc7 100644 --- a/common/pkgs/uploader/create_load.go +++ b/common/pkgs/uploader/create_load.go @@ -68,7 +68,7 @@ func (u *CreateLoadUploader) Upload(pa string, size int64, stream io.Reader) err defer u.lock.Unlock() // 记录上传结果 - fileHash := ret["fileHash"].(*ops2.FileHashValue).Hash + fileHash := ret["fileHash"].(*ops2.ShardInfoValue).Hash u.successes = append(u.successes, coormq.AddObjectEntry{ Path: pa, Size: size, diff --git a/common/pkgs/uploader/update.go b/common/pkgs/uploader/update.go index 127b156..3514876 100644 --- a/common/pkgs/uploader/update.go +++ b/common/pkgs/uploader/update.go @@ -76,7 +76,7 @@ func (w *UpdateUploader) Upload(pat string, size int64, stream io.Reader) error w.successes = append(w.successes, coormq.AddObjectEntry{ Path: pat, Size: size, - FileHash: ret["fileHash"].(*ops2.FileHashValue).Hash, + FileHash: ret["fileHash"].(*ops2.ShardInfoValue).Hash, UploadTime: uploadTime, StorageIDs: []cdssdk.StorageID{w.targetStg.Storage.StorageID}, }) diff --git a/scanner/internal/event/check_package_redundancy.go b/scanner/internal/event/check_package_redundancy.go index 9ceee86..0883030 100644 --- a/scanner/internal/event/check_package_redundancy.go +++ b/scanner/internal/event/check_package_redundancy.go @@ -485,11 +485,13 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object var blocks []stgmod.ObjectBlock var blockChgs []stgmod.BlockChange for i, stg := range uploadStgs { + r := ret[fmt.Sprintf("%d", i)].(*ops2.ShardInfoValue) blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: 0, StorageID: stg.Storage.Storage.StorageID, - FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, + FileHash: r.Hash, + Size: r.Size, }) blockChgs = append(blockChgs, &stgmod.BlockChangeClone{ BlockType: stgmod.BlockTypeRaw, @@ -554,11 +556,13 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD var evtTargetBlocks []stgmod.Block var evtBlockTrans []stgmod.DataTransfer for i := 0; i < red.N; i++ { + r := ioRet[fmt.Sprintf("%d", i)].(*ops2.ShardInfoValue) blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: i, StorageID: uploadStgs[i].Storage.Storage.StorageID, - FileHash: ioRet[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, + FileHash: r.Hash, + Size: r.Size, }) evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ BlockType: stgmod.BlockTypeEC, @@ -635,11 +639,13 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object var evtTargetBlocks []stgmod.Block var evtBlockTrans []stgmod.DataTransfer for i := 0; i < red.N; i++ { + r := ioRet[fmt.Sprintf("%d", i)].(*ops2.ShardInfoValue) blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: i, StorageID: uploadStgs[i].Storage.Storage.StorageID, - FileHash: ioRet[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, + FileHash: r.Hash, + Size: r.Size, }) evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ BlockType: stgmod.BlockTypeEC, @@ -722,11 +728,13 @@ func (t *CheckPackageRedundancy) noneToSeg(ctx ExecuteContext, obj stgmod.Object var evtTargetBlocks []stgmod.Block var evtBlockTrans []stgmod.DataTransfer for i, stg := range uploadStgs { + r := ret[fmt.Sprintf("%d", i)].(*ops2.ShardInfoValue) blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: i, StorageID: stg.Storage.Storage.StorageID, - FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, + FileHash: r.Hash, + Size: r.Size, }) evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ BlockType: stgmod.BlockTypeSegment, @@ -807,11 +815,13 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD var blocks []stgmod.ObjectBlock var blockChgs []stgmod.BlockChange for i, stg := range uploadStgs { + r := ret[fmt.Sprintf("%d", i)].(*ops2.ShardInfoValue) blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: 0, StorageID: stg.Storage.Storage.StorageID, - FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, + FileHash: r.Hash, + Size: r.Size, }) blockChgs = append(blockChgs, &stgmod.BlockChangeClone{ BlockType: stgmod.BlockTypeRaw, @@ -904,11 +914,13 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe var blocks []stgmod.ObjectBlock for i := range uploadStgs { + r := ioRet[fmt.Sprintf("%d", i)].(*ops2.ShardInfoValue) blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: 0, StorageID: uploadStgs[i].Storage.Storage.StorageID, - FileHash: ioRet[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, + FileHash: r.Hash, + Size: r.Size, }) } @@ -1029,6 +1041,7 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet // 如果新选中的节点已经记录在Block表中，那么就不需要任何变更 if ok && lo.Contains(grp.StorageIDs, stg.Storage.Storage.StorageID) { newBlock.FileHash = grp.FileHash + newBlock.Size = grp.Size newBlocks = append(newBlocks, newBlock) continue } @@ -1071,7 +1084,9 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet return nil, fmt.Errorf("parsing result key %s as index: %w", k, err) } - newBlocks[idx].FileHash = v.(*ops2.FileHashValue).Hash + r := v.(*ops2.ShardInfoValue) + newBlocks[idx].FileHash = r.Hash + newBlocks[idx].Size = r.Size } var evtBlockTrans []stgmod.DataTransfer @@ -1268,6 +1283,7 @@ func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.O // 如果新选中的节点已经记录在Block表中，那么就不需要任何变更 if ok && lo.Contains(grp.StorageIDs, storage.Storage.Storage.StorageID) { newBlock.FileHash = grp.FileHash + newBlock.Size = grp.Size newBlocks = append(newBlocks, newBlock) continue } @@ -1311,7 +1327,9 @@ func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.O return nil, fmt.Errorf("parsing result key %s as index: %w", k, err) } - newBlocks[idx].FileHash = v.(*ops2.FileHashValue).Hash + r := v.(*ops2.ShardInfoValue) + newBlocks[idx].FileHash = r.Hash + newBlocks[idx].Size = r.Size } return &coormq.UpdatingObjectRedundancy{ diff --git a/scanner/internal/event/clean_pinned.go b/scanner/internal/event/clean_pinned.go index 277222d..693549a 100644 --- a/scanner/internal/event/clean_pinned.go +++ b/scanner/internal/event/clean_pinned.go @@ -249,6 +249,7 @@ type objectBlock struct { HasEntity bool // 节点拥有实际的文件数据块 HasShadow bool // 如果节点拥有完整文件数据，那么认为这个节点拥有所有块，这些块被称为影子块 FileHash cdssdk.FileHash // 只有在拥有实际文件数据块时，这个字段才有值 + Size int64 // 块大小 } type stgDist struct { @@ -590,6 +591,7 @@ func (t *CleanPinned) initBlockList(ctx *annealingState) { StorageID: b.StorageID, HasEntity: true, FileHash: b.FileHash, + Size: b.Size, }) blocksMap[b.StorageID] = blocks } @@ -774,6 +776,7 @@ func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*st Index: solu.blockList[i].Index, StorageID: solu.blockList[i].StorageID, FileHash: obj.Object.FileHash, + Size: solu.blockList[i].Size, }) } } @@ -859,6 +862,7 @@ func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stg Index: block.Index, StorageID: block.StorageID, FileHash: block.FileHash, + Size: block.Size, }) // 如果这个块是影子块，那么就要从完整对象里重建这个块 @@ -1024,7 +1028,9 @@ func (t *CleanPinned) populateECObjectEntry(entry *coormq.UpdatingObjectRedundan key := fmt.Sprintf("%d.%d", obj.Object.ObjectID, entry.Blocks[i].Index) // 不应该出现key不存在的情况 - entry.Blocks[i].FileHash = ioRets[key].(*ops2.FileHashValue).Hash + r := ioRets[key].(*ops2.ShardInfoValue) + entry.Blocks[i].FileHash = r.Hash + entry.Blocks[i].Size = r.Size } } From 65cd79ab5494dfc8cf85c72e1114cf086fe9e3b2 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 4 Mar 2025 16:38:43 +0800 Subject: [PATCH 28/64] =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=88=86=E5=9D=97?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/http/object.go | 73 +++++++++++ client/internal/http/server.go | 4 + client/internal/services/object.go | 121 ++++++++++++++++++ common/pkgs/db2/object_block.go | 8 +- .../ioswitch2/plans/complete_multipart.go | 49 +++++++ common/pkgs/ioswitch2/plans/utils.go | 20 +++ common/pkgs/mq/coordinator/object.go | 62 +++++++++ common/pkgs/uploader/uploader.go | 110 +++++++++++++++- coordinator/internal/mq/object.go | 118 ++++++++++++++++- .../event/check_package_redundancy.go | 16 +++ scanner/internal/event/clean_pinned.go | 4 + 11 files changed, 580 insertions(+), 5 deletions(-) create mode 100644 common/pkgs/ioswitch2/plans/complete_multipart.go create mode 100644 common/pkgs/ioswitch2/plans/utils.go diff --git a/client/internal/http/object.go b/client/internal/http/object.go index 34acdfd..0fc8593 100644 --- a/client/internal/http/object.go +++ b/client/internal/http/object.go @@ -403,3 +403,76 @@ func (s *ObjectService) GetPackageObjects(ctx *gin.Context) { ctx.JSON(http.StatusOK, OK(cdsapi.ObjectGetPackageObjectsResp{Objects: objs})) } + +func (s *ObjectService) NewMultipartUpload(ctx *gin.Context) { + log := logger.WithField("HTTP", "Object.NewMultipartUpload") + + var req cdsapi.ObjectNewMultipartUpload + if err := ctx.ShouldBindJSON(&req); err != nil { + log.Warnf("binding body: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + obj, err := s.svc.ObjectSvc().NewMultipartUploadObject(req.UserID, req.PackageID, req.Path) + if err != nil { + log.Warnf("new multipart upload object: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "new multipart upload object failed")) + return + } + + ctx.JSON(http.StatusOK, OK(cdsapi.ObjectNewMultipartUploadResp{Object: obj})) +} + +type ObjectUploadPartReq struct { + Info cdsapi.ObjectUploadPartInfo `form:"info" binding:"required"` + File *multipart.FileHeader `form:"file"` +} + +func (s *ObjectService) UploadPart(ctx *gin.Context) { + log := logger.WithField("HTTP", "Object.UploadPart") + + var req ObjectUploadPartReq + if err := ctx.ShouldBind(&req); err != nil { + log.Warnf("binding body: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + file, err := req.File.Open() + if err != nil { + log.Warnf("open file: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "open file failed")) + return + } + defer file.Close() + + err = s.svc.Uploader.UploadPart(req.Info.UserID, req.Info.ObjectID, req.Info.Index, file) + if err != nil { + log.Warnf("uploading part: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("upload part: %v", err))) + return + } + + ctx.JSON(http.StatusOK, OK(cdsapi.ObjectUploadPartResp{})) +} + +func (s *ObjectService) CompleteMultipartUpload(ctx *gin.Context) { + log := logger.WithField("HTTP", "Object.CompleteMultipartUpload") + + var req cdsapi.ObjectCompleteMultipartUpload + if err := ctx.ShouldBindJSON(&req); err != nil { + log.Warnf("binding body: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + obj, err := s.svc.ObjectSvc().CompleteMultipartUpload(req.UserID, req.ObjectID, req.Indexes) + if err != nil { + log.Warnf("completing multipart upload: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("complete multipart upload: %v", err))) + return + } + + ctx.JSON(http.StatusOK, OK(cdsapi.ObjectCompleteMultipartUploadResp{Object: obj})) +} diff --git a/client/internal/http/server.go b/client/internal/http/server.go index aca1faf..4597dca 100644 --- a/client/internal/http/server.go +++ b/client/internal/http/server.go @@ -119,4 +119,8 @@ func (s *Server) routeV1(eg *gin.Engine, rt gin.IRoutes) { rt.POST(cdsapi.UserCreatePath, s.User().Create) rt.POST(cdsapi.UserDeletePath, s.User().Delete) + + rt.POST(cdsapi.ObjectNewMultipartUploadPath, s.Object().NewMultipartUpload) + rt.POST(cdsapi.ObjectUploadPartPath, s.Object().UploadPart) + rt.POST(cdsapi.ObjectCompleteMultipartUploadPath, s.Object().CompleteMultipartUpload) } diff --git a/client/internal/services/object.go b/client/internal/services/object.go index 1476d35..780be64 100644 --- a/client/internal/services/object.go +++ b/client/internal/services/object.go @@ -1,14 +1,18 @@ package services import ( + "context" "fmt" + "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" stgglb "gitlink.org.cn/cloudream/storage/common/globals" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/db2/model" "gitlink.org.cn/cloudream/storage/common/pkgs/downloader" + "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/ops2" + "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/plans" coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" ) @@ -163,3 +167,120 @@ func (svc *ObjectService) GetObjectDetail(objectID cdssdk.ObjectID) (*stgmod.Obj return getResp.Objects[0], nil } + +func (svc *ObjectService) NewMultipartUploadObject(userID cdssdk.UserID, pkgID cdssdk.PackageID, path string) (cdssdk.Object, error) { + coorCli, err := stgglb.CoordinatorMQPool.Acquire() + if err != nil { + return cdssdk.Object{}, fmt.Errorf("new coordinator client: %w", err) + } + defer stgglb.CoordinatorMQPool.Release(coorCli) + + resp, err := coorCli.NewMultipartUploadObject(coormq.ReqNewMultipartUploadObject(userID, pkgID, path)) + if err != nil { + return cdssdk.Object{}, err + } + + return resp.Object, nil +} + +func (svc *ObjectService) CompleteMultipartUpload(userID cdssdk.UserID, objectID cdssdk.ObjectID, indexes []int) (cdssdk.Object, error) { + if len(indexes) == 0 { + return cdssdk.Object{}, fmt.Errorf("no block indexes specified") + } + + coorCli, err := stgglb.CoordinatorMQPool.Acquire() + if err != nil { + return cdssdk.Object{}, fmt.Errorf("new coordinator client: %w", err) + } + defer stgglb.CoordinatorMQPool.Release(coorCli) + + details, err := coorCli.GetObjectDetails(coormq.ReqGetObjectDetails([]cdssdk.ObjectID{objectID})) + if err != nil { + return cdssdk.Object{}, err + } + + if details.Objects[0] == nil { + return cdssdk.Object{}, fmt.Errorf("object %v not found", objectID) + } + + objDe := details.Objects[0] + + _, ok := objDe.Object.Redundancy.(*cdssdk.MultipartUploadRedundancy) + if !ok { + return cdssdk.Object{}, fmt.Errorf("object %v is not a multipart upload", objectID) + } + + if len(objDe.Blocks) == 0 { + return cdssdk.Object{}, fmt.Errorf("object %v has no blocks", objectID) + } + + objBlkMap := make(map[int]stgmod.ObjectBlock) + for _, blk := range objDe.Blocks { + objBlkMap[blk.Index] = blk + } + + var compBlks []stgmod.ObjectBlock + var compBlkStgs []stgmod.StorageDetail + var targetStg stgmod.StorageDetail + for i, idx := range indexes { + blk, ok := objBlkMap[idx] + if !ok { + return cdssdk.Object{}, fmt.Errorf("block %d not found in object %v", idx, objectID) + } + + stg := svc.StorageMeta.Get(blk.StorageID) + if stg == nil { + return cdssdk.Object{}, fmt.Errorf("storage %d not found", blk.StorageID) + } + + compBlks = append(compBlks, blk) + compBlkStgs = append(compBlkStgs, *stg) + if i == 0 { + targetStg = *stg + } + } + + bld := exec.NewPlanBuilder() + err = plans.CompleteMultipart(compBlks, compBlkStgs, targetStg, "shard", bld) + if err != nil { + return cdssdk.Object{}, err + } + + exeCtx := exec.NewExecContext() + ret, err := bld.Execute(exeCtx).Wait(context.Background()) + if err != nil { + return cdssdk.Object{}, err + } + + shardInfo := ret["shard"].(*ops2.ShardInfoValue) + _, err = coorCli.UpdateObjectRedundancy(coormq.ReqUpdateObjectRedundancy([]coormq.UpdatingObjectRedundancy{ + { + ObjectID: objectID, + FileHash: shardInfo.Hash, + Size: shardInfo.Size, + Redundancy: cdssdk.NewNoneRedundancy(), + Blocks: []stgmod.ObjectBlock{{ + ObjectID: objectID, + Index: 0, + StorageID: targetStg.Storage.StorageID, + FileHash: shardInfo.Hash, + Size: shardInfo.Size, + }}, + }, + })) + + if err != nil { + return cdssdk.Object{}, err + } + + getObj, err := coorCli.GetObjects(coormq.ReqGetObjects(userID, []cdssdk.ObjectID{objectID})) + if err != nil { + return cdssdk.Object{}, err + } + + if getObj.Objects[0] == nil { + return cdssdk.Object{}, fmt.Errorf("object %v not found", objectID) + } + + return *getObj.Objects[0], nil +} diff --git a/common/pkgs/db2/object_block.go b/common/pkgs/db2/object_block.go index 53550a6..b6e8511 100644 --- a/common/pkgs/db2/object_block.go +++ b/common/pkgs/db2/object_block.go @@ -43,8 +43,8 @@ func (*ObjectBlockDB) GetInPackageID(ctx SQLContext, packageID cdssdk.PackageID) return rets, err } -func (db *ObjectBlockDB) Create(ctx SQLContext, objectID cdssdk.ObjectID, index int, stgID cdssdk.StorageID, fileHash cdssdk.FileHash) error { - block := stgmod.ObjectBlock{ObjectID: objectID, Index: index, StorageID: stgID, FileHash: fileHash} +func (db *ObjectBlockDB) Create(ctx SQLContext, objectID cdssdk.ObjectID, index int, stgID cdssdk.StorageID, fileHash cdssdk.FileHash, size int64) error { + block := stgmod.ObjectBlock{ObjectID: objectID, Index: index, StorageID: stgID, FileHash: fileHash, Size: size} return ctx.Table("ObjectBlock").Create(&block).Error } @@ -60,6 +60,10 @@ func (db *ObjectBlockDB) DeleteByObjectID(ctx SQLContext, objectID cdssdk.Object return ctx.Table("ObjectBlock").Where("ObjectID = ?", objectID).Delete(&stgmod.ObjectBlock{}).Error } +func (db *ObjectBlockDB) DeleteByObjectIDIndex(ctx SQLContext, objectID cdssdk.ObjectID, index int) error { + return ctx.Table("ObjectBlock").Where("ObjectID = ? AND `Index` = ?", objectID, index).Delete(&stgmod.ObjectBlock{}).Error +} + func (db *ObjectBlockDB) BatchDeleteByObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) error { if len(objectIDs) == 0 { return nil diff --git a/common/pkgs/ioswitch2/plans/complete_multipart.go b/common/pkgs/ioswitch2/plans/complete_multipart.go new file mode 100644 index 0000000..1644a1d --- /dev/null +++ b/common/pkgs/ioswitch2/plans/complete_multipart.go @@ -0,0 +1,49 @@ +package plans + +import ( + "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" + "gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/ops2" + "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" +) + +func CompleteMultipart(blocks []stgmod.ObjectBlock, blockStgs []stgmod.StorageDetail, targetStg stgmod.StorageDetail, shardInfoKey string, blder *exec.PlanBuilder) error { + da := ops2.NewGraphNodeBuilder() + + sizes := make([]int64, len(blocks)) + for i, blk := range blocks { + sizes[i] = blk.Size + } + joinNode := da.NewSegmentJoin(sizes) + joinNode.Env().ToEnvWorker(getWorkerInfo(*targetStg.MasterHub)) + joinNode.Env().Pinned = true + + for i, blk := range blocks { + rd := da.NewShardRead(nil, blk.StorageID, types.NewOpen(blk.FileHash)) + rd.Env().ToEnvWorker(getWorkerInfo(*blockStgs[i].MasterHub)) + rd.Env().Pinned = true + + rd.Output().ToSlot(joinNode.InputSlot(i)) + } + + // TODO 应该采取更合理的方式同时支持Parser和直接生成DAG + wr := da.NewShardWrite(nil, targetStg, shardInfoKey) + wr.Env().ToEnvWorker(getWorkerInfo(*targetStg.MasterHub)) + wr.Env().Pinned = true + + joinNode.Joined().ToSlot(wr.Input()) + + if shardInfoKey != "" { + store := da.NewStore() + store.Env().ToEnvDriver() + store.Store(shardInfoKey, wr.FileHashVar()) + } + + err := plan.Compile(da.Graph, blder) + if err != nil { + return err + } + + return nil +} diff --git a/common/pkgs/ioswitch2/plans/utils.go b/common/pkgs/ioswitch2/plans/utils.go new file mode 100644 index 0000000..47e9003 --- /dev/null +++ b/common/pkgs/ioswitch2/plans/utils.go @@ -0,0 +1,20 @@ +package plans + +import ( + "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" +) + +func getWorkerInfo(hub cdssdk.Hub) exec.WorkerInfo { + switch addr := hub.Address.(type) { + case *cdssdk.HttpAddressInfo: + return &ioswitch2.HttpHubWorker{Hub: hub} + + case *cdssdk.GRPCAddressInfo: + return &ioswitch2.AgentWorker{Hub: hub, Address: *addr} + + default: + return nil + } +} diff --git a/common/pkgs/mq/coordinator/object.go b/common/pkgs/mq/coordinator/object.go index 4ee0005..6ac2248 100644 --- a/common/pkgs/mq/coordinator/object.go +++ b/common/pkgs/mq/coordinator/object.go @@ -33,6 +33,10 @@ type ObjectService interface { GetDatabaseAll(msg *GetDatabaseAll) (*GetDatabaseAllResp, *mq.CodeMessage) AddAccessStat(msg *AddAccessStat) + + NewMultipartUploadObject(msg *NewMultipartUploadObject) (*NewMultipartUploadObjectResp, *mq.CodeMessage) + + AddMultipartUploadPart(msg *AddMultipartUploadPart) (*AddMultipartUploadPartResp, *mq.CodeMessage) } var _ = Register(Service.GetObjects) @@ -190,6 +194,8 @@ type UpdateObjectRedundancyResp struct { } type UpdatingObjectRedundancy struct { ObjectID cdssdk.ObjectID `json:"objectID"` + FileHash cdssdk.FileHash `json:"fileHash"` + Size int64 `json:"size"` Redundancy cdssdk.Redundancy `json:"redundancy"` PinnedAt []cdssdk.StorageID `json:"pinnedAt"` Blocks []stgmod.ObjectBlock `json:"blocks"` @@ -346,3 +352,59 @@ func ReqAddAccessStat(entries []AddAccessStatEntry) *AddAccessStat { func (client *Client) AddAccessStat(msg *AddAccessStat) error { return mq.Send(Service.AddAccessStat, client.rabbitCli, msg) } + +var _ = Register(Service.NewMultipartUploadObject) + +type NewMultipartUploadObject struct { + mq.MessageBodyBase + UserID cdssdk.UserID `json:"userID"` + PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` +} +type NewMultipartUploadObjectResp struct { + mq.MessageBodyBase + Object cdssdk.Object `json:"object"` +} + +func ReqNewMultipartUploadObject(userID cdssdk.UserID, packageID cdssdk.PackageID, path string) *NewMultipartUploadObject { + return &NewMultipartUploadObject{ + UserID: userID, + PackageID: packageID, + Path: path, + } +} +func RespNewMultipartUploadObject(object cdssdk.Object) *NewMultipartUploadObjectResp { + return &NewMultipartUploadObjectResp{ + Object: object, + } +} +func (client *Client) NewMultipartUploadObject(msg *NewMultipartUploadObject) (*NewMultipartUploadObjectResp, error) { + return mq.Request(Service.NewMultipartUploadObject, client.rabbitCli, msg) +} + +var _ = Register(Service.AddMultipartUploadPart) + +type AddMultipartUploadPart struct { + mq.MessageBodyBase + UserID cdssdk.UserID `json:"userID"` + ObjectID cdssdk.ObjectID `json:"objectID"` + Block stgmod.ObjectBlock `json:"block"` +} + +type AddMultipartUploadPartResp struct { + mq.MessageBodyBase +} + +func ReqAddMultipartUploadPart(userID cdssdk.UserID, objectID cdssdk.ObjectID, blk stgmod.ObjectBlock) *AddMultipartUploadPart { + return &AddMultipartUploadPart{ + UserID: userID, + ObjectID: objectID, + Block: blk, + } +} +func RespAddMultipartUploadPart() *AddMultipartUploadPartResp { + return &AddMultipartUploadPartResp{} +} +func (client *Client) AddMultipartUploadPart(msg *AddMultipartUploadPart) (*AddMultipartUploadPartResp, error) { + return mq.Request(Service.AddMultipartUploadPart, client.rabbitCli, msg) +} diff --git a/common/pkgs/uploader/uploader.go b/common/pkgs/uploader/uploader.go index 245e36f..b98a05b 100644 --- a/common/pkgs/uploader/uploader.go +++ b/common/pkgs/uploader/uploader.go @@ -1,12 +1,15 @@ package uploader import ( + "context" "fmt" + "io" "math" "math/rand" "time" "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/sort2" stgglb "gitlink.org.cn/cloudream/storage/common/globals" @@ -14,6 +17,9 @@ import ( "gitlink.org.cn/cloudream/storage/common/pkgs/connectivity" "gitlink.org.cn/cloudream/storage/common/pkgs/distlock" "gitlink.org.cn/cloudream/storage/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" + "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/ops2" + "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/storage/common/pkgs/metacache" coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" @@ -25,8 +31,6 @@ type Uploader struct { connectivity *connectivity.Collector stgAgts *agtpool.AgentPool stgMeta *metacache.StorageMeta - loadTo []cdssdk.StorageID - loadToPath []string } func NewUploader(distlock *distlock.Service, connectivity *connectivity.Collector, stgAgts *agtpool.AgentPool, stgMeta *metacache.StorageMeta) *Uploader { @@ -177,3 +181,105 @@ func (u *Uploader) BeginCreateLoad(userID cdssdk.UserID, bktID cdssdk.BucketID, distlock: lock, }, nil } + +func (u *Uploader) UploadPart(userID cdssdk.UserID, objID cdssdk.ObjectID, index int, stream io.Reader) error { + coorCli, err := stgglb.CoordinatorMQPool.Acquire() + if err != nil { + return fmt.Errorf("new coordinator client: %w", err) + } + defer stgglb.CoordinatorMQPool.Release(coorCli) + + details, err := coorCli.GetObjectDetails(coormq.ReqGetObjectDetails([]cdssdk.ObjectID{objID})) + if err != nil { + return err + } + + if details.Objects[0] == nil { + return fmt.Errorf("object %v not found", objID) + } + + objDe := details.Objects[0] + _, ok := objDe.Object.Redundancy.(*cdssdk.MultipartUploadRedundancy) + if !ok { + return fmt.Errorf("object %v is not a multipart upload", objID) + } + + var stg stgmod.StorageDetail + if len(objDe.Blocks) > 0 { + cstg := u.stgMeta.Get(objDe.Blocks[0].StorageID) + if cstg == nil { + return fmt.Errorf("storage %v not found", objDe.Blocks[0].StorageID) + } + + stg = *cstg + + } else { + getUserStgsResp, err := coorCli.GetUserStorageDetails(coormq.ReqGetUserStorageDetails(userID)) + if err != nil { + return fmt.Errorf("getting user storages: %w", err) + } + + cons := u.connectivity.GetAll() + var userStgs []UploadStorageInfo + for _, stg := range getUserStgsResp.Storages { + if stg.MasterHub == nil { + continue + } + + delay := time.Duration(math.MaxInt64) + + con, ok := cons[stg.MasterHub.HubID] + if ok && con.Latency != nil { + delay = *con.Latency + } + + userStgs = append(userStgs, UploadStorageInfo{ + Storage: stg, + Delay: delay, + IsSameLocation: stg.MasterHub.LocationID == stgglb.Local.LocationID, + }) + } + + if len(userStgs) == 0 { + return fmt.Errorf("user no available storages") + } + + stg = u.chooseUploadStorage(userStgs, 0).Storage + } + + lock, err := reqbuilder.NewBuilder().Shard().Buzy(stg.Storage.StorageID).MutexLock(u.distlock) + if err != nil { + return fmt.Errorf("acquire distlock: %w", err) + } + defer lock.Unlock() + + ft := ioswitch2.NewFromTo() + fromDrv, hd := ioswitch2.NewFromDriver(ioswitch2.RawStream()) + ft.AddFrom(fromDrv). + AddTo(ioswitch2.NewToShardStore(*stg.MasterHub, stg, ioswitch2.RawStream(), "shard")) + + plans := exec.NewPlanBuilder() + err = parser.Parse(ft, plans) + if err != nil { + return fmt.Errorf("parse fromto: %w", err) + } + + exeCtx := exec.NewExecContext() + exec.SetValueByType(exeCtx, u.stgAgts) + exec := plans.Execute(exeCtx) + exec.BeginWrite(io.NopCloser(stream), hd) + ret, err := exec.Wait(context.TODO()) + if err != nil { + return fmt.Errorf("executing plan: %w", err) + } + + shardInfo := ret["shard"].(*ops2.ShardInfoValue) + _, err = coorCli.AddMultipartUploadPart(coormq.ReqAddMultipartUploadPart(userID, objID, stgmod.ObjectBlock{ + ObjectID: objID, + Index: index, + StorageID: stg.Storage.StorageID, + FileHash: shardInfo.Hash, + Size: shardInfo.Size, + })) + return err +} diff --git a/coordinator/internal/mq/object.go b/coordinator/internal/mq/object.go index fa5e012..9fb9ba2 100644 --- a/coordinator/internal/mq/object.go +++ b/coordinator/internal/mq/object.go @@ -235,13 +235,15 @@ func (svc *Service) UpdateObjectRedundancy(msg *coormq.UpdateObjectRedundancy) ( for _, obj := range objs { dummyObjs = append(dummyObjs, cdssdk.Object{ ObjectID: obj.ObjectID, + FileHash: obj.FileHash, + Size: obj.Size, Redundancy: obj.Redundancy, CreateTime: nowTime, // 实际不会更新，只因为不能是0值 UpdateTime: nowTime, }) } - err = db.Object().BatchUpdateColumns(ctx, dummyObjs, []string{"Redundancy", "UpdateTime"}) + err = db.Object().BatchUpdateColumns(ctx, dummyObjs, []string{"FileHash", "Size", "Redundancy", "UpdateTime"}) if err != nil { return fmt.Errorf("batch update object redundancy: %w", err) } @@ -764,3 +766,117 @@ func (svc *Service) CloneObjects(msg *coormq.CloneObjects) (*coormq.CloneObjects return mq.ReplyOK(coormq.RespCloneObjects(ret)) } + +func (svc *Service) NewMultipartUploadObject(msg *coormq.NewMultipartUploadObject) (*coormq.NewMultipartUploadObjectResp, *mq.CodeMessage) { + var obj cdssdk.Object + err := svc.db2.DoTx(func(tx db2.SQLContext) error { + oldObjs, err := svc.db2.Object().GetByPath(tx, msg.PackageID, msg.Path) + if err == nil && len(oldObjs) > 0 { + obj = oldObjs[0] + err := svc.db2.ObjectBlock().DeleteByObjectID(tx, obj.ObjectID) + if err != nil { + return fmt.Errorf("delete object blocks: %w", err) + } + + obj.FileHash = cdssdk.EmptyHash + obj.Size = 0 + obj.Redundancy = cdssdk.NewMultipartUploadRedundancy() + obj.UpdateTime = time.Now() + + err = svc.db2.Object().BatchUpdate(tx, []cdssdk.Object{obj}) + if err != nil { + return fmt.Errorf("update object: %w", err) + } + + return nil + } + + obj = cdssdk.Object{ + PackageID: msg.PackageID, + Path: msg.Path, + FileHash: cdssdk.EmptyHash, + Size: 0, + Redundancy: cdssdk.NewMultipartUploadRedundancy(), + CreateTime: time.Now(), + UpdateTime: time.Now(), + } + objID, err := svc.db2.Object().Create(tx, obj) + if err != nil { + return fmt.Errorf("create object: %w", err) + } + + obj.ObjectID = objID + return nil + }) + if err != nil { + logger.Warnf("new multipart upload object: %s", err.Error()) + return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("new multipart upload object: %v", err)) + } + + return mq.ReplyOK(coormq.RespNewMultipartUploadObject(obj)) +} + +func (svc *Service) AddMultipartUploadPart(msg *coormq.AddMultipartUploadPart) (*coormq.AddMultipartUploadPartResp, *mq.CodeMessage) { + err := svc.db2.DoTx(func(tx db2.SQLContext) error { + obj, err := svc.db2.Object().GetByID(tx, msg.ObjectID) + if err != nil { + return fmt.Errorf("getting object by id: %w", err) + } + + _, ok := obj.Redundancy.(*cdssdk.MultipartUploadRedundancy) + if !ok { + return fmt.Errorf("object is not a multipart upload object") + } + + blks, err := svc.db2.ObjectBlock().BatchGetByObjectID(tx, []cdssdk.ObjectID{obj.ObjectID}) + if err != nil { + return fmt.Errorf("batch getting object blocks: %w", err) + } + + blks = lo.Reject(blks, func(blk stgmod.ObjectBlock, idx int) bool { return blk.Index == msg.Block.Index }) + blks = append(blks, msg.Block) + + blks = sort2.Sort(blks, func(a, b stgmod.ObjectBlock) int { return a.Index - b.Index }) + + totalSize := int64(0) + var hashes [][]byte + for _, blk := range blks { + totalSize += blk.Size + hashes = append(hashes, blk.FileHash.GetHashBytes()) + } + + newObjHash := cdssdk.CalculateCompositeHash(hashes) + obj.Size = totalSize + obj.FileHash = newObjHash + obj.UpdateTime = time.Now() + + err = svc.db2.ObjectBlock().DeleteByObjectIDIndex(tx, msg.ObjectID, msg.Block.Index) + if err != nil { + return fmt.Errorf("delete object block: %w", err) + } + + err = svc.db2.ObjectBlock().Create(tx, msg.ObjectID, msg.Block.Index, msg.Block.StorageID, msg.Block.FileHash, msg.Block.Size) + if err != nil { + return fmt.Errorf("create object block: %w", err) + } + + err = svc.db2.Object().BatchUpdate(tx, []cdssdk.Object{obj}) + if err != nil { + return fmt.Errorf("update object: %w", err) + } + + return nil + }) + if err != nil { + logger.Warnf("add multipart upload part: %s", err.Error()) + + code := errorcode.OperationFailed + if errors.Is(err, gorm.ErrRecordNotFound) { + code = errorcode.DataNotFound + } + + return nil, mq.Failed(code, fmt.Sprintf("add multipart upload part: %v", err)) + } + + return mq.ReplyOK(coormq.RespAddMultipartUploadPart()) +} diff --git a/scanner/internal/event/check_package_redundancy.go b/scanner/internal/event/check_package_redundancy.go index 0883030..e66fabd 100644 --- a/scanner/internal/event/check_package_redundancy.go +++ b/scanner/internal/event/check_package_redundancy.go @@ -515,6 +515,8 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: red, Blocks: blocks, }, nil @@ -599,6 +601,8 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: red, Blocks: blocks, }, nil @@ -682,6 +686,8 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: red, Blocks: blocks, }, nil @@ -771,6 +777,8 @@ func (t *CheckPackageRedundancy) noneToSeg(ctx ExecuteContext, obj stgmod.Object return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: red, Blocks: blocks, }, nil @@ -845,6 +853,8 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: red, Blocks: blocks, }, nil @@ -975,6 +985,8 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: tarRed, Blocks: blocks, }, nil @@ -1126,6 +1138,8 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: tarRed, Blocks: newBlocks, }, nil @@ -1334,6 +1348,8 @@ func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.O return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: red, Blocks: newBlocks, }, nil diff --git a/scanner/internal/event/clean_pinned.go b/scanner/internal/event/clean_pinned.go index 693549a..7d7d6fc 100644 --- a/scanner/internal/event/clean_pinned.go +++ b/scanner/internal/event/clean_pinned.go @@ -750,6 +750,8 @@ func (t *CleanPinned) alwaysAccept(curTemp float64, dScore float64, coolingRate func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*stgmod.StorageDetail, solu annealingSolution, obj stgmod.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[cdssdk.StorageID]bool) coormq.UpdatingObjectRedundancy { entry := coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: obj.Object.Redundancy, } @@ -850,6 +852,8 @@ func (t *CleanPinned) generateSysEventForRepObject(solu annealingSolution, obj s func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stgmod.StorageDetail, solu annealingSolution, obj stgmod.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[cdssdk.StorageID]bool) coormq.UpdatingObjectRedundancy { entry := coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, + FileHash: obj.Object.FileHash, + Size: obj.Object.Size, Redundancy: obj.Object.Redundancy, } From d59ba9fd56435f1f4b8ea265b2a7e473332200af Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 5 Mar 2025 10:06:56 +0800 Subject: [PATCH 29/64] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E4=B8=8D=E8=83=BD=E5=90=8C=E6=97=B6=E8=A2=AB?= =?UTF-8?q?=E5=A4=9A=E4=B8=AA=E6=B6=88=E8=B4=B9=E8=80=85=E6=B6=88=E8=B4=B9?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pkgs/sysevent/publisher.go | 14 ++++---------- common/pkgs/sysevent/sysevent.go | 1 + common/pkgs/sysevent/watcher.go | 13 +++++++++++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/common/pkgs/sysevent/publisher.go b/common/pkgs/sysevent/publisher.go index dfb482b..bb95065 100644 --- a/common/pkgs/sysevent/publisher.go +++ b/common/pkgs/sysevent/publisher.go @@ -48,16 +48,10 @@ func NewPublisher(cfg Config, thisSource Source) (*Publisher, error) { return nil, fmt.Errorf("openning channel on connection: %w", err) } - _, err = channel.QueueDeclare( - SysEventQueueName, - false, - true, - false, - false, - nil, - ) + err = channel.ExchangeDeclare(ExchangeName, "fanout", false, true, false, false, nil) if err != nil { - return nil, fmt.Errorf("declare queue: %w", err) + connection.Close() + return nil, fmt.Errorf("declare exchange: %w", err) } pub := &Publisher{ @@ -94,7 +88,7 @@ func (p *Publisher) Start() *async.UnboundChannel[PublisherEvent] { continue } - err = p.channel.Publish("", SysEventQueueName, false, false, amqp.Publishing{ + err = p.channel.Publish(ExchangeName, "", false, false, amqp.Publishing{ ContentType: "text/plain", Body: eventData, Expiration: "60000", // 消息超时时间默认1分钟 diff --git a/common/pkgs/sysevent/sysevent.go b/common/pkgs/sysevent/sysevent.go index 93c2a62..00feb1f 100644 --- a/common/pkgs/sysevent/sysevent.go +++ b/common/pkgs/sysevent/sysevent.go @@ -6,6 +6,7 @@ import ( const ( SysEventQueueName = "SysEventQueue" + ExchangeName = "SysEventExchange" ) type SysEvent = stgmod.SysEvent diff --git a/common/pkgs/sysevent/watcher.go b/common/pkgs/sysevent/watcher.go index c93cca5..98e0b6d 100644 --- a/common/pkgs/sysevent/watcher.go +++ b/common/pkgs/sysevent/watcher.go @@ -45,6 +45,12 @@ func NewWatcherHost(cfg Config) (*WatcherHost, error) { return nil, fmt.Errorf("openning channel on connection: %w", err) } + err = channel.ExchangeDeclare(ExchangeName, "fanout", false, true, false, false, nil) + if err != nil { + connection.Close() + return nil, fmt.Errorf("declare exchange: %w", err) + } + _, err = channel.QueueDeclare( SysEventQueueName, false, @@ -59,6 +65,13 @@ func NewWatcherHost(cfg Config) (*WatcherHost, error) { return nil, fmt.Errorf("declare queue: %w", err) } + err = channel.QueueBind(SysEventQueueName, "", ExchangeName, false, nil) + if err != nil { + channel.Close() + connection.Close() + return nil, fmt.Errorf("bind queue: %w", err) + } + recvChan, err := channel.Consume(SysEventQueueName, "", true, false, true, false, nil) if err != nil { channel.Close() From a47b2c807e84d2360ff4bafe04805b44ce71fe4b Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 5 Mar 2025 11:10:02 +0800 Subject: [PATCH 30/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 2af2b07..236bc40 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -31,9 +31,9 @@ workflow: && ls /usr/local/go/bin && - sh 2_build_all.sh + echo "PATH:"$PATH && - sudo sh 3_deploy.sh "' + sh 2_build_all.sh "' needs: - start - ref: end From 0351fd6d7ce16b9087ad961a0e80e186f4120352 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 5 Mar 2025 11:11:25 +0800 Subject: [PATCH 31/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 236bc40..01fb31c 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -31,7 +31,7 @@ workflow: && ls /usr/local/go/bin && - echo "PATH:"$PATH + echo PATH:$PATH && sh 2_build_all.sh "' needs: From 4f68a4657037c32dda3bbba70b56d1e3f7c57f35 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 5 Mar 2025 11:14:47 +0800 Subject: [PATCH 32/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index 01fb31c..fcbdb51 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -31,7 +31,11 @@ workflow: && ls /usr/local/go/bin && - echo PATH:$PATH + echo PATH1:$PATH + && + export PATH=$PATH:/usr/local/go/bin + && + echo PATH2:$PATH && sh 2_build_all.sh "' needs: From 126721a5bc826a1e520ee955bd1b1086cee63076 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Wed, 5 Mar 2025 11:22:11 +0800 Subject: [PATCH 33/64] =?UTF-8?q?refactor:=20.devops/pcm1=E7=8E=AF?= =?UTF-8?q?=E5=A2=83cicd.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/pcm1环境cicd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devops/pcm1环境cicd.yml b/.devops/pcm1环境cicd.yml index fcbdb51..09295ce 100644 --- a/.devops/pcm1环境cicd.yml +++ b/.devops/pcm1环境cicd.yml @@ -37,7 +37,9 @@ workflow: && echo PATH2:$PATH && - sh 2_build_all.sh "' + sh 2_build_all.sh + && + sudo sh 3_deploy.sh "' needs: - start - ref: end From 5c2aeb5961db6ff77e523b33bb25108c20ab5471 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 10:11:53 +0800 Subject: [PATCH 34/64] =?UTF-8?q?feat:=20.devops/=E6=9C=AA=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E9=A1=B9=E7=9B=AE.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/未命名项目.yml | 109 ++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .devops/未命名项目.yml diff --git a/.devops/未命名项目.yml b/.devops/未命名项目.yml new file mode 100644 index 0000000..4b3b26c --- /dev/null +++ b/.devops/未命名项目.yml @@ -0,0 +1,109 @@ +version: 2 +name: 未命名项目 +description: "" +global: + concurrent: 1 +workflow: + - ref: start + name: 开始 + task: start + - ref: git_clone_0 + name: git clone + task: git_clone@1.2.9 + input: + remote_url: '""' + ref: '"refs/heads/master"' + commit_id: '""' + depth: 1 + needs: + - start + - ref: docker_image_build_2 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: docker_image_build_1 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: docker_image_build_3 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: ssh_cmd_0 + name: ssh执行命令 + task: ssh_cmd@1.1.1 + input: + ssh_ip: '""' + ssh_port: '"22"' + ssh_user: '"root"' + ssh_cmd: '""' + needs: + - docker_image_build_2 + - docker_image_build_0 + - docker_image_build_1 + - docker_image_build_3 + - docker_image_build_4 + - ref: end + name: 结束 + task: end + needs: + - ssh_cmd_0 + - ref: docker_image_build_4 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: docker_image_build_0 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + From 8259b37bb69fb4ecab0cf2c3ad60f2c5b04f14f9 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 10:44:00 +0800 Subject: [PATCH 36/64] =?UTF-8?q?refactor:=20delete=20.devops/=E6=9C=AA?= =?UTF-8?q?=E5=91=BD=E5=90=8D=E9=A1=B9=E7=9B=AE.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/未命名项目.yml | 109 ------------------------------------ 1 file changed, 109 deletions(-) delete mode 100644 .devops/未命名项目.yml diff --git a/.devops/未命名项目.yml b/.devops/未命名项目.yml deleted file mode 100644 index 4b3b26c..0000000 --- a/.devops/未命名项目.yml +++ /dev/null @@ -1,109 +0,0 @@ -version: 2 -name: 未命名项目 -description: "" -global: - concurrent: 1 -workflow: - - ref: start - name: 开始 - task: start - - ref: git_clone_0 - name: git clone - task: git_clone@1.2.9 - input: - remote_url: '""' - ref: '"refs/heads/master"' - commit_id: '""' - depth: 1 - needs: - - start - - ref: docker_image_build_2 - name: docker镜像构建 - task: docker_image_build@1.6.0 - input: - image_name: '""' - image_tag: '"latest"' - registry_address: '""' - docker_file: '"Dockerfile"' - docker_build_path: '"."' - workspace: '"."' - image_push: true - build_args: '""' - needs: - - git_clone_0 - - ref: docker_image_build_1 - name: docker镜像构建 - task: docker_image_build@1.6.0 - input: - image_name: '""' - image_tag: '"latest"' - registry_address: '""' - docker_file: '"Dockerfile"' - docker_build_path: '"."' - workspace: '"."' - image_push: true - build_args: '""' - needs: - - git_clone_0 - - ref: docker_image_build_3 - name: docker镜像构建 - task: docker_image_build@1.6.0 - input: - image_name: '""' - image_tag: '"latest"' - registry_address: '""' - docker_file: '"Dockerfile"' - docker_build_path: '"."' - workspace: '"."' - image_push: true - build_args: '""' - needs: - - git_clone_0 - - ref: ssh_cmd_0 - name: ssh执行命令 - task: ssh_cmd@1.1.1 - input: - ssh_ip: '""' - ssh_port: '"22"' - ssh_user: '"root"' - ssh_cmd: '""' - needs: - - docker_image_build_2 - - docker_image_build_0 - - docker_image_build_1 - - docker_image_build_3 - - docker_image_build_4 - - ref: end - name: 结束 - task: end - needs: - - ssh_cmd_0 - - ref: docker_image_build_4 - name: docker镜像构建 - task: docker_image_build@1.6.0 - input: - image_name: '""' - image_tag: '"latest"' - registry_address: '""' - docker_file: '"Dockerfile"' - docker_build_path: '"."' - workspace: '"."' - image_push: true - build_args: '""' - needs: - - git_clone_0 - - ref: docker_image_build_0 - name: docker镜像构建 - task: docker_image_build@1.6.0 - input: - image_name: '""' - image_tag: '"latest"' - registry_address: '""' - docker_file: '"Dockerfile"' - docker_build_path: '"."' - workspace: '"."' - image_push: true - build_args: '""' - needs: - - git_clone_0 - From 78bbc6fa0c424ae06c28721b60e3eb9ca3170ba0 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 10:44:01 +0800 Subject: [PATCH 37/64] =?UTF-8?q?feat:=20.devops/=E5=9F=BA=E4=BA=8Egitlink?= =?UTF-8?q?=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 109 ++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .devops/基于gitlink构建.yml diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml new file mode 100644 index 0000000..f59d347 --- /dev/null +++ b/.devops/基于gitlink构建.yml @@ -0,0 +1,109 @@ +version: 2 +name: 基于gitlink构建 +description: "" +global: + concurrent: 1 +workflow: + - ref: start + name: 开始 + task: start + - ref: git_clone_0 + name: git clone + task: git_clone@1.2.9 + input: + remote_url: '"https://gitlink.org.cn/JointCloud/storage.git"' + ref: '"refs/heads/master"' + commit_id: '""' + depth: 1 + needs: + - start + - ref: docker_image_build_2 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: docker_image_build_1 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: docker_image_build_3 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: ssh_cmd_0 + name: ssh执行命令 + task: ssh_cmd@1.1.1 + input: + ssh_ip: '""' + ssh_port: '"22"' + ssh_user: '"root"' + ssh_cmd: '""' + needs: + - docker_image_build_2 + - docker_image_build_0 + - docker_image_build_1 + - docker_image_build_3 + - docker_image_build_4 + - ref: end + name: 结束 + task: end + needs: + - ssh_cmd_0 + - ref: docker_image_build_4 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + - ref: docker_image_build_0 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - git_clone_0 + From 97347f760b695d0a789bf3de1d47ab50d1a82b95 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 6 Mar 2025 14:33:02 +0800 Subject: [PATCH 38/64] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=A2=84=E7=AD=BE?= =?UTF-8?q?=E5=90=8D=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/internal/task/create_package.go | 2 +- client/internal/cmdline/object.go | 6 +- client/internal/cmdline/put.go | 2 +- client/internal/http/aws_auth.go | 84 +++++++++++- client/internal/http/object.go | 2 +- client/internal/http/presigned.go | 180 ++++++++++++++++++++++++++ client/internal/http/server.go | 7 + common/pkgs/uploader/update.go | 9 +- 8 files changed, 277 insertions(+), 15 deletions(-) create mode 100644 client/internal/http/presigned.go diff --git a/agent/internal/task/create_package.go b/agent/internal/task/create_package.go index 6354938..f5c8ae3 100644 --- a/agent/internal/task/create_package.go +++ b/agent/internal/task/create_package.go @@ -113,7 +113,7 @@ func (t *CreatePackage) Execute(task *task.Task[TaskContext], ctx TaskContext, c path := filepath.ToSlash(obj.Path) // 上传对象 - err = up.Upload(path, obj.Size, obj.File) + err = up.Upload(path, obj.File) if err != nil { err = fmt.Errorf("uploading object: %w", err) log.Error(err.Error()) diff --git a/client/internal/cmdline/object.go b/client/internal/cmdline/object.go index c3791d5..8267162 100644 --- a/client/internal/cmdline/object.go +++ b/client/internal/cmdline/object.go @@ -48,17 +48,13 @@ var _ = MustAddCmd(func(ctx CommandContext, packageID cdssdk.PackageID, rootPath return nil } - info, err := fi.Info() - if err != nil { - return err - } file, err := os.Open(fname) if err != nil { return err } defer file.Close() - return up.Upload(fname, info.Size(), file) + return up.Upload(fname, file) }) if err != nil { return err diff --git a/client/internal/cmdline/put.go b/client/internal/cmdline/put.go index a540518..a8f9b82 100644 --- a/client/internal/cmdline/put.go +++ b/client/internal/cmdline/put.go @@ -100,7 +100,7 @@ func init() { } defer file.Close() - return up.Upload(fname, info.Size(), file) + return up.Upload(fname, file) }) if err != nil { fmt.Println(err.Error()) diff --git a/client/internal/http/aws_auth.go b/client/internal/http/aws_auth.go index 4a6d283..71a4a95 100644 --- a/client/internal/http/aws_auth.go +++ b/client/internal/http/aws_auth.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "time" @@ -93,7 +94,7 @@ func (a *AWSAuth) Auth(c *gin.Context) { return } - verifySig := a.getSignature(verifyReq) + verifySig := getSignatureFromAWSHeader(verifyReq) if !strings.EqualFold(verifySig, reqSig) { c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) return @@ -135,13 +136,14 @@ func (a *AWSAuth) AuthWithoutBody(c *gin.Context) { } err = a.signer.SignHTTP(context.TODO(), a.cred, verifyReq, "", AuthService, AuthRegion, timestamp) + if err != nil { logger.Warnf("sign request: %v", err) c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.OperationFailed, "sign request failed")) return } - verifySig := a.getSignature(verifyReq) + verifySig := getSignatureFromAWSHeader(verifyReq) if strings.EqualFold(verifySig, reqSig) { c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) return @@ -150,6 +152,68 @@ func (a *AWSAuth) AuthWithoutBody(c *gin.Context) { c.Next() } +func (a *AWSAuth) PresignedAuth(c *gin.Context) { + query := c.Request.URL.Query() + + signature := query.Get("X-Amz-Signature") + query.Del("X-Amz-Signature") + if signature == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing X-Amz-Signature query parameter")) + return + } + + // alg := c.Request.URL.Query().Get("X-Amz-Algorithm") + // cred := c.Request.URL.Query().Get("X-Amz-Credential") + + date := query.Get("X-Amz-Date") + expiresStr := query.Get("X-Expires") + expires, err := strconv.ParseInt(expiresStr, 10, 64) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "invalid X-Expires format")) + return + } + + signedHeaders := strings.Split(query.Get("X-Amz-SignedHeaders"), ";") + + c.Request.URL.RawQuery = query.Encode() + + verifyReq, err := http.NewRequest(c.Request.Method, c.Request.URL.String(), nil) + if err != nil { + c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.OperationFailed, err.Error())) + return + } + for _, h := range signedHeaders { + verifyReq.Header.Add(h, c.Request.Header.Get(h)) + } + verifyReq.Host = c.Request.Host + + timestamp, err := time.Parse("20060102T150405Z", date) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "invalid X-Amz-Date format")) + return + } + + if time.Now().After(timestamp.Add(time.Duration(expires) * time.Second)) { + c.AbortWithStatusJSON(http.StatusUnauthorized, Failed(errorcode.Unauthorized, "request expired")) + return + } + + signer := v4.NewSigner() + uri, _, err := signer.PresignHTTP(context.TODO(), a.cred, verifyReq, "", AuthService, AuthRegion, timestamp) + if err != nil { + c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.OperationFailed, "sign request failed")) + return + } + + verifySig := getSignatureFromAWSQuery(uri) + if !strings.EqualFold(verifySig, signature) { + c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) + return + } + + c.Next() +} + // 解析 Authorization 头部 func parseAuthorizationHeader(authorizationHeader string) (string, []string, string, error) { if !strings.HasPrefix(authorizationHeader, "AWS4-HMAC-SHA256 ") { @@ -186,7 +250,7 @@ func parseAuthorizationHeader(authorizationHeader string) (string, []string, str return credential, headers, signature, nil } -func (a *AWSAuth) getSignature(req *http.Request) string { +func getSignatureFromAWSHeader(req *http.Request) string { auth := req.Header.Get(AuthorizationHeader) idx := strings.Index(auth, "Signature=") if idx == -1 { @@ -195,3 +259,17 @@ func (a *AWSAuth) getSignature(req *http.Request) string { return auth[idx+len("Signature="):] } + +func getSignatureFromAWSQuery(uri string) string { + idx := strings.Index(uri, "X-Amz-Signature=") + if idx == -1 { + return "" + } + + andIdx := strings.Index(uri[idx:], "&") + if andIdx == -1 { + return uri[idx+len("X-Amz-Signature="):] + } + + return uri[idx+len("X-Amz-Signature=") : andIdx] +} diff --git a/client/internal/http/object.go b/client/internal/http/object.go index 0fc8593..4e44eac 100644 --- a/client/internal/http/object.go +++ b/client/internal/http/object.go @@ -109,7 +109,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) { } path = filepath.ToSlash(path) - err = up.Upload(path, file.Size, f) + err = up.Upload(path, f) if err != nil { log.Warnf("uploading file: %s", err.Error()) ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.Filename, err))) diff --git a/client/internal/http/presigned.go b/client/internal/http/presigned.go new file mode 100644 index 0000000..27cf389 --- /dev/null +++ b/client/internal/http/presigned.go @@ -0,0 +1,180 @@ +package http + +import ( + "fmt" + "io" + "net/http" + "net/url" + "path" + "path/filepath" + + "github.com/gin-gonic/gin" + "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" + "gitlink.org.cn/cloudream/common/utils/math2" + "gitlink.org.cn/cloudream/storage/client/internal/config" + "gitlink.org.cn/cloudream/storage/common/pkgs/downloader" +) + +type PresignedService struct { + *Server +} + +func (s *Server) Presigned() *PresignedService { + return &PresignedService{ + Server: s, + } +} + +func (s *PresignedService) ObjectDownloadByPath(ctx *gin.Context) { + log := logger.WithField("HTTP", "Presigned.ObjectDownloadByPath") + + var req cdsapi.PresignedObjectDownloadByPath + if err := ctx.ShouldBindQuery(&req); err != nil { + log.Warnf("binding query: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + _, obj, err := s.svc.ObjectSvc().GetByPath(req.UserID, req.PackageID, req.Path, false, false) + if err != nil { + log.Warnf("getting object by path: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get object by path failed")) + return + } + + if len(obj) == 0 { + log.Warnf("object not found: %s", req.Path) + ctx.JSON(http.StatusOK, Failed(errorcode.DataNotFound, "object not found")) + return + } + + off := req.Offset + len := int64(-1) + if req.Length != nil { + len = *req.Length + } + + file, err := s.svc.ObjectSvc().Download(req.UserID, downloader.DownloadReqeust{ + ObjectID: obj[0].ObjectID, + Offset: off, + Length: len, + }) + if err != nil { + log.Warnf("downloading object: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "download object failed")) + return + } + defer file.File.Close() + + ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(path.Base(file.Object.Path))) + ctx.Header("Content-Type", "application/octet-stream") + ctx.Header("Content-Transfer-Encoding", "binary") + + n, err := io.Copy(ctx.Writer, file.File) + if err != nil { + log.Warnf("copying file: %s", err.Error()) + } + + if config.Cfg().StorageID > 0 { + s.svc.AccessStat.AddAccessCounter(file.Object.ObjectID, file.Object.PackageID, config.Cfg().StorageID, math2.DivOrDefault(float64(n), float64(file.Object.Size), 1)) + } +} + +func (s *PresignedService) ObjectUpload(ctx *gin.Context) { + log := logger.WithField("HTTP", "Presigned.ObjectUpload") + + var req cdsapi.PresignedObjectUpload + if err := ctx.ShouldBindQuery(&req); err != nil { + log.Warnf("binding query: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + up, err := s.svc.Uploader.BeginUpdate(req.UserID, req.PackageID, req.Affinity, req.LoadTo, req.LoadToPath) + if err != nil { + log.Warnf("begin update: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("begin update: %v", err))) + return + } + defer up.Abort() + + path := filepath.ToSlash(req.Path) + + err = up.Upload(path, ctx.Request.Body) + if err != nil { + log.Warnf("uploading file: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("uploading file %v: %v", req.Path, err))) + return + } + + ret, err := up.Commit() + if err != nil { + log.Warnf("commit update: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("commit update: %v", err))) + return + } + + ctx.JSON(http.StatusOK, OK(cdsapi.PresignedObjectUploadResp{Object: ret.Objects[path]})) +} + +func (s *PresignedService) ObjectNewMultipartUpload(ctx *gin.Context) { + log := logger.WithField("HTTP", "Presigned.ObjectNewMultipartUpload") + + var req cdsapi.PresignedObjectNewMultipartUpload + if err := ctx.ShouldBindQuery(&req); err != nil { + log.Warnf("binding query: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + obj, err := s.svc.ObjectSvc().NewMultipartUploadObject(req.UserID, req.PackageID, req.Path) + if err != nil { + log.Warnf("new multipart upload: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("new multipart upload: %v", err))) + return + } + + ctx.JSON(http.StatusOK, OK(cdsapi.PresignedObjectUploadResp{Object: obj})) +} + +func (s *PresignedService) ObjectUploadPart(ctx *gin.Context) { + log := logger.WithField("HTTP", "Presigned.ObjectUploadPart") + + var req cdsapi.PresignedObjectUploadPart + if err := ctx.ShouldBindQuery(&req); err != nil { + log.Warnf("binding query: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + err := s.svc.Uploader.UploadPart(req.UserID, req.ObjectID, req.Index, ctx.Request.Body) + if err != nil { + log.Warnf("uploading part: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("upload part: %v", err))) + return + } + + ctx.JSON(http.StatusOK, OK(cdsapi.ObjectUploadPartResp{})) +} + +func (s *PresignedService) ObjectCompleteMultipartUpload(ctx *gin.Context) { + log := logger.WithField("HTTP", "Presigned.ObjectCompleteMultipartUpload") + + var req cdsapi.PresignedObjectCompleteMultipartUpload + if err := ctx.ShouldBindQuery(&req); err != nil { + log.Warnf("binding query: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + obj, err := s.svc.ObjectSvc().CompleteMultipartUpload(req.UserID, req.ObjectID, req.Indexes) + if err != nil { + log.Warnf("completing multipart upload: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("complete multipart upload: %v", err))) + return + } + + ctx.JSON(http.StatusOK, OK(cdsapi.ObjectCompleteMultipartUploadResp{Object: obj})) +} diff --git a/client/internal/http/server.go b/client/internal/http/server.go index 4597dca..123e66a 100644 --- a/client/internal/http/server.go +++ b/client/internal/http/server.go @@ -123,4 +123,11 @@ func (s *Server) routeV1(eg *gin.Engine, rt gin.IRoutes) { rt.POST(cdsapi.ObjectNewMultipartUploadPath, s.Object().NewMultipartUpload) rt.POST(cdsapi.ObjectUploadPartPath, s.Object().UploadPart) rt.POST(cdsapi.ObjectCompleteMultipartUploadPath, s.Object().CompleteMultipartUpload) + + rt.GET(cdsapi.PresignedObjectDownloadByPathPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectDownloadByPath) + rt.POST(cdsapi.PresignedObjectUploadPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectUpload) + + rt.POST(cdsapi.PresignedObjectNewMultipartUploadPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectNewMultipartUpload) + rt.POST(cdsapi.PresignedObjectUploadPartPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectUploadPart) + rt.POST(cdsapi.PresignedObjectCompleteMultipartUploadPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectCompleteMultipartUpload) } diff --git a/common/pkgs/uploader/update.go b/common/pkgs/uploader/update.go index 3514876..4000c6e 100644 --- a/common/pkgs/uploader/update.go +++ b/common/pkgs/uploader/update.go @@ -42,13 +42,13 @@ type UpdateResult struct { Objects map[string]cdssdk.Object } -func (w *UpdateUploader) Upload(pat string, size int64, stream io.Reader) error { +func (w *UpdateUploader) Upload(pat string, stream io.Reader) error { uploadTime := time.Now() ft := ioswitch2.NewFromTo() fromExec, hd := ioswitch2.NewFromDriver(ioswitch2.RawStream()) ft.AddFrom(fromExec). - AddTo(ioswitch2.NewToShardStore(*w.targetStg.MasterHub, w.targetStg, ioswitch2.RawStream(), "fileHash")) + AddTo(ioswitch2.NewToShardStore(*w.targetStg.MasterHub, w.targetStg, ioswitch2.RawStream(), "shardInfo")) for i, stg := range w.loadToStgs { ft.AddTo(ioswitch2.NewLoadToPublic(*stg.MasterHub, stg, path.Join(w.loadToPath[i], pat))) @@ -73,10 +73,11 @@ func (w *UpdateUploader) Upload(pat string, size int64, stream io.Reader) error defer w.lock.Unlock() // 记录上传结果 + shardInfo := ret["shardInfo"].(*ops2.ShardInfoValue) w.successes = append(w.successes, coormq.AddObjectEntry{ Path: pat, - Size: size, - FileHash: ret["fileHash"].(*ops2.ShardInfoValue).Hash, + Size: shardInfo.Size, + FileHash: shardInfo.Hash, UploadTime: uploadTime, StorageIDs: []cdssdk.StorageID{w.targetStg.Storage.StorageID}, }) From 2c15f161f6ad3cb9c6541a4931d51e65b05ba418 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 16:38:12 +0800 Subject: [PATCH 39/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index f59d347..84bca86 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -3,6 +3,9 @@ name: 基于gitlink构建 description: "" global: concurrent: 1 + cache: + - GOCACHE + - GOMODCACHE workflow: - ref: start name: 开始 @@ -11,6 +14,8 @@ workflow: name: git clone task: git_clone@1.2.9 input: + username: ((gitlink.songjc)) + password: ((gitlink.songjc)) remote_url: '"https://gitlink.org.cn/JointCloud/storage.git"' ref: '"refs/heads/master"' commit_id: '""' @@ -98,7 +103,7 @@ workflow: input: image_name: '""' image_tag: '"latest"' - registry_address: '""' + registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' workspace: '"."' From 6bef0840280cc2a6ee899244920930d0a8563d68 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 16:48:30 +0800 Subject: [PATCH 40/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 45 ++++++++++++++------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 84bca86..19304b0 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -23,12 +23,14 @@ workflow: needs: - start - ref: docker_image_build_2 - name: docker镜像构建 + name: scanner镜像构建 task: docker_image_build@1.6.0 input: - image_name: '""' + docker_username: ((docker_registry.registry_user)) + docker_password: ((docker_registry.registry_password)) + image_name: '"112.95.163.90:5010/scannerservice-x86"' image_tag: '"latest"' - registry_address: '""' + registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' workspace: '"."' @@ -37,12 +39,14 @@ workflow: needs: - git_clone_0 - ref: docker_image_build_1 - name: docker镜像构建 + name: coor镜像构建 task: docker_image_build@1.6.0 input: - image_name: '""' + docker_username: ((docker_registry.registry_user)) + docker_password: ((docker_registry.registry_password)) + image_name: '"112.95.163.90:5010/coordinatorservice-x86"' image_tag: '"latest"' - registry_address: '""' + registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' workspace: '"."' @@ -51,12 +55,14 @@ workflow: needs: - git_clone_0 - ref: docker_image_build_3 - name: docker镜像构建 + name: client镜像构建 task: docker_image_build@1.6.0 input: - image_name: '""' + docker_username: ((docker_registry.registry_user)) + docker_password: ((docker_registry.registry_password)) + image_name: '"112.95.163.90:5010/clientservice-x86"' image_tag: '"latest"' - registry_address: '""' + registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' workspace: '"."' @@ -77,31 +83,18 @@ workflow: - docker_image_build_0 - docker_image_build_1 - docker_image_build_3 - - docker_image_build_4 - ref: end name: 结束 task: end needs: - ssh_cmd_0 - - ref: docker_image_build_4 - name: docker镜像构建 - task: docker_image_build@1.6.0 - input: - image_name: '""' - image_tag: '"latest"' - registry_address: '""' - docker_file: '"Dockerfile"' - docker_build_path: '"."' - workspace: '"."' - image_push: true - build_args: '""' - needs: - - git_clone_0 - ref: docker_image_build_0 - name: docker镜像构建 + name: agent镜像构建 task: docker_image_build@1.6.0 input: - image_name: '""' + docker_username: ((docker_registry.registry_user)) + docker_password: ((docker_registry.registry_password)) + image_name: '"112.95.163.90:5010/agentservice-x86"' image_tag: '"latest"' registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' From 067e4af9e92ffacbb1a7c021a7b42368fd99d6a2 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 16:55:51 +0800 Subject: [PATCH 41/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 19304b0..41fffcc 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -24,6 +24,9 @@ workflow: - start - ref: docker_image_build_2 name: scanner镜像构建 + cache: + GOCACHE: /root/.cache/go-build + GOMODCACHE: /go/pkg/mod task: docker_image_build@1.6.0 input: docker_username: ((docker_registry.registry_user)) @@ -40,6 +43,9 @@ workflow: - git_clone_0 - ref: docker_image_build_1 name: coor镜像构建 + cache: + GOCACHE: /root/.cache/go-build + GOMODCACHE: /go/pkg/mod task: docker_image_build@1.6.0 input: docker_username: ((docker_registry.registry_user)) @@ -56,6 +62,9 @@ workflow: - git_clone_0 - ref: docker_image_build_3 name: client镜像构建 + cache: + GOCACHE: /root/.cache/go-build + GOMODCACHE: /go/pkg/mod task: docker_image_build@1.6.0 input: docker_username: ((docker_registry.registry_user)) @@ -90,6 +99,9 @@ workflow: - ssh_cmd_0 - ref: docker_image_build_0 name: agent镜像构建 + cache: + GOCACHE: /root/.cache/go-build + GOMODCACHE: /go/pkg/mod task: docker_image_build@1.6.0 input: docker_username: ((docker_registry.registry_user)) From df0ca8cd97fbdffa992687e39952980868634afa Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 17:00:28 +0800 Subject: [PATCH 42/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 41fffcc..0f267ec 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -36,7 +36,7 @@ workflow: registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' - workspace: '"."' + workspace: git_clone_0.git_path image_push: true build_args: '""' needs: @@ -55,7 +55,7 @@ workflow: registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' - workspace: '"."' + workspace: git_clone_0.git_path image_push: true build_args: '""' needs: @@ -74,7 +74,7 @@ workflow: registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' - workspace: '"."' + workspace: git_clone_0.git_path image_push: true build_args: '""' needs: @@ -83,9 +83,10 @@ workflow: name: ssh执行命令 task: ssh_cmd@1.1.1 input: - ssh_ip: '""' + ssh_private_key: ((SSH.pcm1)) + ssh_ip: '"123.60.146.162"' ssh_port: '"22"' - ssh_user: '"root"' + ssh_user: '"pcm"' ssh_cmd: '""' needs: - docker_image_build_2 @@ -111,7 +112,7 @@ workflow: registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' - workspace: '"."' + workspace: git_clone_0.git_path image_push: true build_args: '""' needs: From 2200b135793e0411902152b58f71a21a6925a459 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 17:04:52 +0800 Subject: [PATCH 43/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 0f267ec..746a991 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -87,7 +87,7 @@ workflow: ssh_ip: '"123.60.146.162"' ssh_port: '"22"' ssh_user: '"pcm"' - ssh_cmd: '""' + ssh_cmd: '"cd /home/pcm/deploy/cloudream/workspace/storage_yml&&ls"' needs: - docker_image_build_2 - docker_image_build_0 From 9291a1572921b70ba28fde82ed99cdcdcad2caf1 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 17:17:21 +0800 Subject: [PATCH 44/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 746a991..7febb07 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -11,7 +11,7 @@ workflow: name: 开始 task: start - ref: git_clone_0 - name: git clone + name: storage_clone task: git_clone@1.2.9 input: username: ((gitlink.songjc)) @@ -40,7 +40,7 @@ workflow: image_push: true build_args: '""' needs: - - git_clone_0 + - shell_0 - ref: docker_image_build_1 name: coor镜像构建 cache: @@ -59,7 +59,7 @@ workflow: image_push: true build_args: '""' needs: - - git_clone_0 + - shell_0 - ref: docker_image_build_3 name: client镜像构建 cache: @@ -78,7 +78,7 @@ workflow: image_push: true build_args: '""' needs: - - git_clone_0 + - shell_0 - ref: ssh_cmd_0 name: ssh执行命令 task: ssh_cmd@1.1.1 @@ -115,6 +115,24 @@ workflow: workspace: git_clone_0.git_path image_push: true build_args: '""' + needs: + - shell_0 + - ref: git_clone_1 + name: common_clone + task: git_clone@1.2.9 + input: + remote_url: '""' + ref: '"refs/heads/master"' + commit_id: '""' + depth: 1 + needs: + - start + - ref: shell_0 + name: shell + image: docker.jianmuhub.com/library/alpine:3.17.0 + script: + - ls needs: - git_clone_0 + - git_clone_1 From 03311fea524666d046f083b4f1e5148afd4e47a8 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 17:18:43 +0800 Subject: [PATCH 45/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 7febb07..55c4771 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -121,7 +121,7 @@ workflow: name: common_clone task: git_clone@1.2.9 input: - remote_url: '""' + remote_url: '"https://gitlink.org.cn/JointCloud/common.git"' ref: '"refs/heads/master"' commit_id: '""' depth: 1 From da794d034593d60b5e10f4ee58d4e0972d26f23e Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 6 Mar 2025 17:21:38 +0800 Subject: [PATCH 46/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 55c4771..ead91a4 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -14,8 +14,8 @@ workflow: name: storage_clone task: git_clone@1.2.9 input: - username: ((gitlink.songjc)) - password: ((gitlink.songjc)) + username: ((gitlink.user)) + password: ((gitlink.password)) remote_url: '"https://gitlink.org.cn/JointCloud/storage.git"' ref: '"refs/heads/master"' commit_id: '""' @@ -121,6 +121,8 @@ workflow: name: common_clone task: git_clone@1.2.9 input: + username: ((gitlink.user)) + password: ((gitlink.password)) remote_url: '"https://gitlink.org.cn/JointCloud/common.git"' ref: '"refs/heads/master"' commit_id: '""' From 880a578d3b84127b033cc846535cf7570c3e02ce Mon Sep 17 00:00:00 2001 From: JeshuaRen <1366929814@qq.com> Date: Fri, 7 Mar 2025 15:52:32 +0800 Subject: [PATCH 47/64] Add Dockerfile --- Dockerfile | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b507a05 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,64 @@ +FROM --platform=$BUILDPLATFORM docker-0.unsee.tech/golang:alpine AS builder +WORKDIR /app +ENV GOPROXY=https://goproxy.cn \ + GO111MODULE=on \ + CGO_ENABLED=0 \ + GOCACHE=/root/.cache/go-build \ + GOMODCACHE=/go/pkg/mod + +COPY ../common /app/common +COPY . /app/storage + +# 指定构建参数 +ARG TARGETOS +ARG TARGETARCH + +# 编译 +WORKDIR /app/storage +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + mage linux all + +# agent +FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS agentimage +WORKDIR /app +COPY --from=builder /app/storage/build/agent . +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN apk add --no-cache tzdata +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN chmod +x agent/agent +ENTRYPOINT ["./agent/agent"] + +# coordinator +FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS coorimage +WORKDIR /app +COPY --from=builder /app/storage/build/coordinator . +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN apk add --no-cache tzdata +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN chmod +x coordinator/coordinator +ENTRYPOINT ["./coordinator/coordinator"] + +# scanner +FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS scannerimage +WORKDIR /app +COPY --from=builder /app/storage/build/scanner . +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN apk add --no-cache tzdata +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN chmod +x scanner/scanner +ENTRYPOINT ["./scanner/scanner"] + +# client +FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS clientimage +WORKDIR /app +COPY --from=builder /app/storage/build/client . +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN apk add --no-cache tzdata +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN chmod +x client/client +ENTRYPOINT ["./client/client","serve","http"] From 43ca5ba2f530d1058cd665f096602f38d651e3cf Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Fri, 7 Mar 2025 16:01:55 +0800 Subject: [PATCH 48/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index ead91a4..1de0414 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -40,7 +40,7 @@ workflow: image_push: true build_args: '""' needs: - - shell_0 + - docker_image_build_4 - ref: docker_image_build_1 name: coor镜像构建 cache: @@ -59,7 +59,7 @@ workflow: image_push: true build_args: '""' needs: - - shell_0 + - docker_image_build_4 - ref: docker_image_build_3 name: client镜像构建 cache: @@ -78,7 +78,7 @@ workflow: image_push: true build_args: '""' needs: - - shell_0 + - docker_image_build_4 - ref: ssh_cmd_0 name: ssh执行命令 task: ssh_cmd@1.1.1 @@ -116,7 +116,7 @@ workflow: image_push: true build_args: '""' needs: - - shell_0 + - docker_image_build_4 - ref: git_clone_1 name: common_clone task: git_clone@1.2.9 @@ -137,4 +137,18 @@ workflow: needs: - git_clone_0 - git_clone_1 + - ref: docker_image_build_4 + name: docker镜像构建 + task: docker_image_build@1.6.0 + input: + image_name: '""' + image_tag: '"latest"' + registry_address: '""' + docker_file: '"Dockerfile"' + docker_build_path: '"."' + workspace: '"."' + image_push: true + build_args: '""' + needs: + - shell_0 From a126dce16fab7a48791b370ceb707da9e2c2000a Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Fri, 7 Mar 2025 16:07:59 +0800 Subject: [PATCH 49/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 1de0414..80ae5c8 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -138,15 +138,15 @@ workflow: - git_clone_0 - git_clone_1 - ref: docker_image_build_4 - name: docker镜像构建 + name: build task: docker_image_build@1.6.0 input: - image_name: '""' + image_name: '"storage-builder"' image_tag: '"latest"' registry_address: '""' docker_file: '"Dockerfile"' docker_build_path: '"."' - workspace: '"."' + workspace: git_clone_0.git_path image_push: true build_args: '""' needs: From 48f34201c0029dfb871e9871dfba8b9f61653262 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Fri, 7 Mar 2025 17:25:04 +0800 Subject: [PATCH 50/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 80ae5c8..e50640b 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -139,6 +139,9 @@ workflow: - git_clone_1 - ref: docker_image_build_4 name: build + cache: + GOCACHE: /root/.cache/go-build + GOMODCACHE: /go/pkg/mod task: docker_image_build@1.6.0 input: image_name: '"storage-builder"' From 6488b47e6a8f8382356524d3273c4f1826c43a40 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 10:32:23 +0800 Subject: [PATCH 51/64] =?UTF-8?q?=E6=9B=B4=E6=96=B0go.mod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 88 +++++++++++++++++-------------- go.sum | 164 ++++++++++++++++++++------------------------------------- 2 files changed, 104 insertions(+), 148 deletions(-) diff --git a/go.mod b/go.mod index 9dcf692..62b8529 100644 --- a/go.mod +++ b/go.mod @@ -1,48 +1,55 @@ module gitlink.org.cn/cloudream/storage -go 1.21 +go 1.23.0 toolchain go1.23.2 replace gitlink.org.cn/cloudream/common v0.0.0 => ../common require ( - github.com/aws/aws-sdk-go-v2 v1.32.6 - github.com/aws/aws-sdk-go-v2/credentials v1.17.47 + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/credentials v1.17.62 github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 - github.com/gin-gonic/gin v1.7.7 + github.com/gin-gonic/gin v1.10.0 github.com/go-co-op/gocron/v2 v2.15.0 github.com/go-sql-driver/mysql v1.8.1 - github.com/hashicorp/golang-lru/v2 v2.0.5 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.131 github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf - github.com/jedib0t/go-pretty/v6 v6.4.7 + github.com/jedib0t/go-pretty/v6 v6.5.8 github.com/klauspost/reedsolomon v1.11.8 github.com/magefile/mage v1.15.0 - github.com/samber/lo v1.38.1 + github.com/samber/lo v1.49.1 github.com/smartystreets/goconvey v1.8.1 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 gitlink.org.cn/cloudream/common v0.0.0 - golang.org/x/sync v0.7.0 - google.golang.org/grpc v1.62.1 - google.golang.org/protobuf v1.33.0 - gorm.io/gorm v1.25.11 + golang.org/x/sync v0.11.0 + google.golang.org/grpc v1.67.1 + google.golang.org/protobuf v1.35.1 + gorm.io/gorm v1.25.12 ) require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect - github.com/aws/smithy-go v1.22.1 // indirect + github.com/aws/smithy-go v1.22.2 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect @@ -51,6 +58,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -58,29 +66,30 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tjfoc/gmsm v1.4.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.mongodb.org/mongo-driver v1.12.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 // indirect - github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-playground/locales v0.13.0 // indirect - github.com/go-playground/universal-translator v0.17.0 // indirect - github.com/go-playground/validator/v10 v10.8.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -88,25 +97,24 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/leodido/go-urn v1.2.4 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/sirupsen/logrus v1.9.2 + github.com/rivo/uniseg v0.4.7 // indirect + github.com/sirupsen/logrus v1.9.3 github.com/smarty/assertions v1.15.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/streadway/amqp v1.1.0 - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/zyedidia/generic v1.2.1 // indirect go.etcd.io/etcd/api/v3 v3.5.12 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect go.etcd.io/etcd/client/v3 v3.5.12 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - go.uber.org/zap v1.24.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect gorm.io/driver/mysql v1.5.7 ) diff --git a/go.sum b/go.sum index 4262e4a..fce7032 100644 --- a/go.sum +++ b/go.sum @@ -4,40 +4,40 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= -github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= -github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= -github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= -github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBajbBWGqjNTCa79ocU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= -github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= -github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -49,24 +49,20 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= -github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/go-co-op/gocron/v2 v2.15.0 h1:Kpvo71VSihE+RImmpA+3ta5CcMhoRzMGw4dJawrj4zo= github.com/go-co-op/gocron/v2 v2.15.0/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k= -github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -81,18 +77,17 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -103,8 +98,7 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= -github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible h1:XQVXdk+WAJ4fSNB6mMRuYNvFWou7BZs6SZB925hPrnk= @@ -117,15 +111,13 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4= -github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE= -github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= +github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -133,60 +125,46 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= -github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= @@ -197,8 +175,7 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= @@ -210,13 +187,9 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -225,10 +198,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= @@ -246,27 +217,22 @@ go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE= go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= -golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -289,8 +255,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -299,28 +264,23 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -328,7 +288,6 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -336,8 +295,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -356,42 +315,31 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= -gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From bca3ca3e2e183072747e9b7bddd8ff8a04d8ea2f Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 14:35:21 +0800 Subject: [PATCH 52/64] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E6=88=B3=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/http/aws_auth.go | 40 +++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/client/internal/http/aws_auth.go b/client/internal/http/aws_auth.go index 71a4a95..6303175 100644 --- a/client/internal/http/aws_auth.go +++ b/client/internal/http/aws_auth.go @@ -66,6 +66,17 @@ func (a *AWSAuth) Auth(c *gin.Context) { return } + timestamp, err := time.Parse("20060102T150405Z", c.GetHeader("X-Amz-Date")) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "invalid X-Amz-Date header format")) + return + } + + if time.Now().After(timestamp.Add(5 * time.Minute)) { + c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "X-Amz-Date is expired")) + return + } + payloadHash := sha256.Sum256(body) hexPayloadHash := hex.EncodeToString(payloadHash[:]) @@ -79,12 +90,7 @@ func (a *AWSAuth) Auth(c *gin.Context) { verifyReq.Header.Add(h, c.Request.Header.Get(h)) } verifyReq.Host = c.Request.Host - - timestamp, err := time.Parse("20060102T150405Z", c.GetHeader("X-Amz-Date")) - if err != nil { - c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "invalid X-Amz-Date header format")) - return - } + verifyReq.ContentLength = c.Request.ContentLength signer := v4.NewSigner() err = signer.SignHTTP(context.TODO(), a.cred, verifyReq, hexPayloadHash, AuthService, AuthRegion, timestamp) @@ -96,6 +102,7 @@ func (a *AWSAuth) Auth(c *gin.Context) { verifySig := getSignatureFromAWSHeader(verifyReq) if !strings.EqualFold(verifySig, reqSig) { + logger.Warnf("signature mismatch, expect(%d): %s, actual(%d): %s", len(reqSig), reqSig, len(verifySig), verifySig) c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) return } @@ -118,6 +125,17 @@ func (a *AWSAuth) AuthWithoutBody(c *gin.Context) { return } + timestamp, err := time.Parse("20060102T150405Z", c.GetHeader("X-Amz-Date")) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "invalid X-Amz-Date header format")) + return + } + + if time.Now().After(timestamp.Add(5 * time.Minute)) { + c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "X-Amz-Date is expired")) + return + } + // 构造验签用的请求 verifyReq, err := http.NewRequest(c.Request.Method, c.Request.URL.String(), nil) if err != nil { @@ -128,12 +146,7 @@ func (a *AWSAuth) AuthWithoutBody(c *gin.Context) { verifyReq.Header.Add(h, c.Request.Header.Get(h)) } verifyReq.Host = c.Request.Host - - timestamp, err := time.Parse("20060102T150405Z", c.GetHeader("X-Amz-Date")) - if err != nil { - c.AbortWithStatusJSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "invalid X-Amz-Date header format")) - return - } + verifyReq.ContentLength = c.Request.ContentLength err = a.signer.SignHTTP(context.TODO(), a.cred, verifyReq, "", AuthService, AuthRegion, timestamp) @@ -144,7 +157,8 @@ func (a *AWSAuth) AuthWithoutBody(c *gin.Context) { } verifySig := getSignatureFromAWSHeader(verifyReq) - if strings.EqualFold(verifySig, reqSig) { + if !strings.EqualFold(verifySig, reqSig) { + logger.Warnf("signature mismatch, expect(%d): %s, actual(%d): %s", len(reqSig), reqSig, len(verifySig), verifySig) c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) return } From 2970041208fa888b2af65be414fbe48a8ff8d882 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 16:18:42 +0800 Subject: [PATCH 53/64] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=80=9A=E8=BF=87ID?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E6=96=87=E4=BB=B6=E7=9A=84=E9=A2=84=E7=AD=BE?= =?UTF-8?q?=E5=90=8D=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/http/aws_auth.go | 5 ++-- client/internal/http/presigned.go | 42 +++++++++++++++++++++++++++++++ client/internal/http/server.go | 1 + 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/client/internal/http/aws_auth.go b/client/internal/http/aws_auth.go index 6303175..fdfc7a5 100644 --- a/client/internal/http/aws_auth.go +++ b/client/internal/http/aws_auth.go @@ -102,7 +102,7 @@ func (a *AWSAuth) Auth(c *gin.Context) { verifySig := getSignatureFromAWSHeader(verifyReq) if !strings.EqualFold(verifySig, reqSig) { - logger.Warnf("signature mismatch, expect(%d): %s, actual(%d): %s", len(reqSig), reqSig, len(verifySig), verifySig) + logger.Warnf("signature mismatch, input header: %s, verify: %s", authorizationHeader, verifySig) c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) return } @@ -158,7 +158,7 @@ func (a *AWSAuth) AuthWithoutBody(c *gin.Context) { verifySig := getSignatureFromAWSHeader(verifyReq) if !strings.EqualFold(verifySig, reqSig) { - logger.Warnf("signature mismatch, expect(%d): %s, actual(%d): %s", len(reqSig), reqSig, len(verifySig), verifySig) + logger.Warnf("signature mismatch, input header: %s, verify: %s", authorizationHeader, verifySig) c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) return } @@ -221,6 +221,7 @@ func (a *AWSAuth) PresignedAuth(c *gin.Context) { verifySig := getSignatureFromAWSQuery(uri) if !strings.EqualFold(verifySig, signature) { + logger.Warnf("signature mismatch, input: %s, verify: %s", signature, verifySig) c.AbortWithStatusJSON(http.StatusOK, Failed(errorcode.Unauthorized, "signature mismatch")) return } diff --git a/client/internal/http/presigned.go b/client/internal/http/presigned.go index 27cf389..aef1c0e 100644 --- a/client/internal/http/presigned.go +++ b/client/internal/http/presigned.go @@ -82,6 +82,48 @@ func (s *PresignedService) ObjectDownloadByPath(ctx *gin.Context) { } } +func (s *PresignedService) ObjectDownload(ctx *gin.Context) { + log := logger.WithField("HTTP", "Presigned.ObjectDownloadByPath") + + var req cdsapi.PresignedObjectDownload + if err := ctx.ShouldBindQuery(&req); err != nil { + log.Warnf("binding query: %s", err.Error()) + ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) + return + } + + off := req.Offset + len := int64(-1) + if req.Length != nil { + len = *req.Length + } + + file, err := s.svc.ObjectSvc().Download(req.UserID, downloader.DownloadReqeust{ + ObjectID: req.ObjectID, + Offset: off, + Length: len, + }) + if err != nil { + log.Warnf("downloading object: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "download object failed")) + return + } + defer file.File.Close() + + ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(path.Base(file.Object.Path))) + ctx.Header("Content-Type", "application/octet-stream") + ctx.Header("Content-Transfer-Encoding", "binary") + + n, err := io.Copy(ctx.Writer, file.File) + if err != nil { + log.Warnf("copying file: %s", err.Error()) + } + + if config.Cfg().StorageID > 0 { + s.svc.AccessStat.AddAccessCounter(file.Object.ObjectID, file.Object.PackageID, config.Cfg().StorageID, math2.DivOrDefault(float64(n), float64(file.Object.Size), 1)) + } +} + func (s *PresignedService) ObjectUpload(ctx *gin.Context) { log := logger.WithField("HTTP", "Presigned.ObjectUpload") diff --git a/client/internal/http/server.go b/client/internal/http/server.go index 123e66a..3a12144 100644 --- a/client/internal/http/server.go +++ b/client/internal/http/server.go @@ -125,6 +125,7 @@ func (s *Server) routeV1(eg *gin.Engine, rt gin.IRoutes) { rt.POST(cdsapi.ObjectCompleteMultipartUploadPath, s.Object().CompleteMultipartUpload) rt.GET(cdsapi.PresignedObjectDownloadByPathPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectDownloadByPath) + rt.GET(cdsapi.PresignedObjectDownloadPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectDownload) rt.POST(cdsapi.PresignedObjectUploadPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectUpload) rt.POST(cdsapi.PresignedObjectNewMultipartUploadPath, s.awsAuth.PresignedAuth, s.Presigned().ObjectNewMultipartUpload) From 2144482eea46f9be6dca35f5fe5a5b3f8682fb9b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 14 Mar 2025 16:59:02 +0800 Subject: [PATCH 54/64] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BB=8EStorage?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0Package=E7=9A=84=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/internal/cmd/serve.go | 2 +- agent/internal/mq/service.go | 5 +- agent/internal/mq/storage.go | 97 +++++++++-------------- client/internal/cmdline/storage.go | 20 +---- client/internal/http/storage.go | 32 ++------ client/internal/services/storage.go | 42 ++-------- common/pkgs/mq/agent/storage.go | 58 +++----------- common/pkgs/storage/local/public_store.go | 59 ++++++++++++++ common/pkgs/storage/s3/public_store.go | 55 ++++++++++++- common/pkgs/storage/types/public_store.go | 3 + 10 files changed, 190 insertions(+), 183 deletions(-) diff --git a/agent/internal/cmd/serve.go b/agent/internal/cmd/serve.go index 654db1a..6731e29 100644 --- a/agent/internal/cmd/serve.go +++ b/agent/internal/cmd/serve.go @@ -166,7 +166,7 @@ func serve(configPath string) { // 启动命令服务器 // TODO 需要设计AgentID持久化机制 - agtSvr, err := agtmq.NewServer(cmdsvc.NewService(&taskMgr, stgAgts), config.Cfg().ID, config.Cfg().RabbitMQ) + agtSvr, err := agtmq.NewServer(cmdsvc.NewService(&taskMgr, stgAgts, uploader), config.Cfg().ID, config.Cfg().RabbitMQ) if err != nil { logger.Fatalf("new agent server failed, err: %s", err.Error()) } diff --git a/agent/internal/mq/service.go b/agent/internal/mq/service.go index b4688ed..95cd4e3 100644 --- a/agent/internal/mq/service.go +++ b/agent/internal/mq/service.go @@ -3,16 +3,19 @@ package mq import ( "gitlink.org.cn/cloudream/storage/agent/internal/task" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" + "gitlink.org.cn/cloudream/storage/common/pkgs/uploader" ) type Service struct { taskManager *task.Manager stgAgts *agtpool.AgentPool + uploader *uploader.Uploader } -func NewService(taskMgr *task.Manager, stgAgts *agtpool.AgentPool) *Service { +func NewService(taskMgr *task.Manager, stgAgts *agtpool.AgentPool, uplodaer *uploader.Uploader) *Service { return &Service{ taskManager: taskMgr, stgAgts: stgAgts, + uploader: uplodaer, } } diff --git a/agent/internal/mq/storage.go b/agent/internal/mq/storage.go index ff5b4ff..cdc3871 100644 --- a/agent/internal/mq/storage.go +++ b/agent/internal/mq/storage.go @@ -1,76 +1,57 @@ package mq import ( - "time" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" - mytask "gitlink.org.cn/cloudream/storage/agent/internal/task" + stgglb "gitlink.org.cn/cloudream/storage/common/globals" agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent" + coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" ) -func (svc *Service) StartStorageCreatePackage(msg *agtmq.StartStorageCreatePackage) (*agtmq.StartStorageCreatePackageResp, *mq.CodeMessage) { - return nil, mq.Failed(errorcode.OperationFailed, "not implemented") - // coorCli, err := stgglb.CoordinatorMQPool.Acquire() - // if err != nil { - // logger.Warnf("new coordinator client: %s", err.Error()) - - // return nil, mq.Failed(errorcode.OperationFailed, "new coordinator client failed") - // } - // defer stgglb.CoordinatorMQPool.Release(coorCli) - - // getStg, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{msg.StorageID})) - // if err != nil { - // return nil, mq.Failed(errorcode.OperationFailed, err.Error()) - // } - // if getStg.Storages[0] == nil { - // return nil, mq.Failed(errorcode.OperationFailed, "storage not found") - // } - // if getStg.Storages[0].Shared == nil { - // return nil, mq.Failed(errorcode.OperationFailed, "storage has no shared storage") - // } - - // fullPath := filepath.Clean(filepath.Join(getStg.Storages[0].Shared.LoadBase, msg.Path)) - - // var uploadFilePathes []string - // err = filepath.WalkDir(fullPath, func(fname string, fi os.DirEntry, err error) error { - // if err != nil { - // return nil - // } - - // if !fi.IsDir() { - // uploadFilePathes = append(uploadFilePathes, fname) - // } +func (svc *Service) StorageCreatePackage(msg *agtmq.StorageCreatePackage) (*agtmq.StorageCreatePackageResp, *mq.CodeMessage) { + coorCli, err := stgglb.CoordinatorMQPool.Acquire() + if err != nil { + logger.Warnf("new coordinator client: %s", err.Error()) - // return nil - // }) - // if err != nil { - // logger.Warnf("opening directory %s: %s", fullPath, err.Error()) - - // return nil, mq.Failed(errorcode.OperationFailed, "read directory failed") - // } + return nil, mq.Failed(errorcode.OperationFailed, "new coordinator client failed") + } + defer stgglb.CoordinatorMQPool.Release(coorCli) - // objIter := iterator.NewUploadingObjectIterator(fullPath, uploadFilePathes) - // tsk := svc.taskManager.StartNew(mytask.NewCreatePackage(msg.UserID, msg.BucketID, msg.Name, objIter, msg.StorageAffinity)) - // return mq.ReplyOK(agtmq.NewStartStorageCreatePackageResp(tsk.ID())) -} + pub, err := svc.stgAgts.GetPublicStore(msg.StorageID) + if err != nil { + return nil, mq.Failed(errorcode.OperationFailed, err.Error()) + } -func (svc *Service) WaitStorageCreatePackage(msg *agtmq.WaitStorageCreatePackage) (*agtmq.WaitStorageCreatePackageResp, *mq.CodeMessage) { - tsk := svc.taskManager.FindByID(msg.TaskID) - if tsk == nil { - return nil, mq.Failed(errorcode.TaskNotFound, "task not found") + createResp, err := coorCli.CreatePackage(coormq.NewCreatePackage(msg.UserID, msg.BucketID, msg.Name)) + if err != nil { + return nil, mq.Failed(errorcode.OperationFailed, err.Error()) } - if msg.WaitTimeoutMs == 0 { - tsk.Wait() - } else if !tsk.WaitTimeout(time.Duration(msg.WaitTimeoutMs) * time.Millisecond) { - return mq.ReplyOK(agtmq.NewWaitStorageCreatePackageResp(false, "", 0)) + uploader, err := svc.uploader.BeginUpdate(msg.UserID, createResp.Package.PackageID, msg.StorageAffinity, nil, nil) + if err != nil { + return nil, mq.Failed(errorcode.OperationFailed, err.Error()) } - if tsk.Error() != nil { - return mq.ReplyOK(agtmq.NewWaitStorageCreatePackageResp(true, tsk.Error().Error(), 0)) + objPathes, err := pub.List(msg.Path, true) + for _, p := range objPathes { + o, err := pub.Read(p) + if err != nil { + logger.Warnf("read object %s: %v", p, err) + continue + } + + err = uploader.Upload(p, o) + o.Close() + if err != nil { + logger.Warnf("upload object %s: %v", p, err) + continue + } + } + _, err = uploader.Commit() + if err != nil { + return nil, mq.Failed(errorcode.OperationFailed, err.Error()) } - taskBody := tsk.Body().(*mytask.CreatePackage) - return mq.ReplyOK(agtmq.NewWaitStorageCreatePackageResp(true, "", taskBody.Result.PackageID)) + return mq.ReplyOK(agtmq.RespStorageCreatePackage(createResp.Package)) } diff --git a/client/internal/cmdline/storage.go b/client/internal/cmdline/storage.go index bb6c09a..7263c58 100644 --- a/client/internal/cmdline/storage.go +++ b/client/internal/cmdline/storage.go @@ -22,27 +22,13 @@ func StorageCreatePackage(ctx CommandContext, bucketID cdssdk.BucketID, name str }() // 开始创建并上传包到存储系统 - hubID, taskID, err := ctx.Cmdline.Svc.StorageSvc().StartStorageCreatePackage(1, bucketID, name, storageID, path, 0) + pkg, err := ctx.Cmdline.Svc.StorageSvc().StorageCreatePackage(1, bucketID, name, storageID, path, 0) if err != nil { return fmt.Errorf("start storage uploading package: %w", err) } - // 循环等待上传完成 - for { - complete, packageID, err := ctx.Cmdline.Svc.StorageSvc().WaitStorageCreatePackage(hubID, taskID, time.Second*10) - if complete { - if err != nil { - return fmt.Errorf("uploading complete with: %w", err) - } - - fmt.Printf("%d\n", packageID) - return nil - } - - if err != nil { - return fmt.Errorf("wait uploading: %w", err) - } - } + fmt.Printf("%d\n", pkg.PackageID) + return nil } // 初始化函数，注册加载包和创建包的命令到命令行解析器。 diff --git a/client/internal/http/storage.go b/client/internal/http/storage.go index 96bb9a6..48b72c0 100644 --- a/client/internal/http/storage.go +++ b/client/internal/http/storage.go @@ -1,8 +1,8 @@ package http import ( + "fmt" "net/http" - "time" "github.com/gin-gonic/gin" "gitlink.org.cn/cloudream/common/consts/errorcode" @@ -50,35 +50,17 @@ func (s *StorageService) CreatePackage(ctx *gin.Context) { return } - hubID, taskID, err := s.svc.StorageSvc().StartStorageCreatePackage( + pkg, err := s.svc.StorageSvc().StorageCreatePackage( req.UserID, req.BucketID, req.Name, req.StorageID, req.Path, req.StorageAffinity) if err != nil { - log.Warnf("start storage create package: %s", err.Error()) - ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "storage create package failed")) + log.Warnf("storage create package: %s", err.Error()) + ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("storage create package: %v", err))) return } - for { - complete, packageID, err := s.svc.StorageSvc().WaitStorageCreatePackage(hubID, taskID, time.Second*10) - if complete { - if err != nil { - log.Warnf("creating complete with: %s", err.Error()) - ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "storage create package failed")) - return - } - - ctx.JSON(http.StatusOK, OK(cdsapi.StorageCreatePackageResp{ - PackageID: packageID, - })) - return - } - - if err != nil { - log.Warnf("wait creating: %s", err.Error()) - ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "storage create package failed")) - return - } - } + ctx.JSON(http.StatusOK, OK(cdsapi.StorageCreatePackageResp{ + Package: pkg, + })) } func (s *StorageService) Get(ctx *gin.Context) { diff --git a/client/internal/services/storage.go b/client/internal/services/storage.go index a9263eb..267e369 100644 --- a/client/internal/services/storage.go +++ b/client/internal/services/storage.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "path" - "time" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" @@ -161,57 +160,32 @@ func (svc *StorageService) LoadPackage(userID cdssdk.UserID, packageID cdssdk.Pa } // 请求节点启动从Storage中上传文件的任务。会返回节点ID和任务ID -func (svc *StorageService) StartStorageCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string, storageID cdssdk.StorageID, path string, storageAffinity cdssdk.StorageID) (cdssdk.HubID, string, error) { +func (svc *StorageService) StorageCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string, storageID cdssdk.StorageID, path string, storageAffinity cdssdk.StorageID) (cdssdk.Package, error) { coorCli, err := stgglb.CoordinatorMQPool.Acquire() if err != nil { - return 0, "", fmt.Errorf("new coordinator client: %w", err) + return cdssdk.Package{}, fmt.Errorf("new coordinator client: %w", err) } defer stgglb.CoordinatorMQPool.Release(coorCli) stgResp, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{storageID})) if err != nil { - return 0, "", fmt.Errorf("getting storage info: %w", err) + return cdssdk.Package{}, fmt.Errorf("getting storage info: %w", err) } if stgResp.Storages[0].Storage.ShardStore == nil { - return 0, "", fmt.Errorf("shard storage is not enabled") + return cdssdk.Package{}, fmt.Errorf("shard storage is not enabled") } agentCli, err := stgglb.AgentMQPool.Acquire(stgResp.Storages[0].MasterHub.HubID) if err != nil { - return 0, "", fmt.Errorf("new agent client: %w", err) + return cdssdk.Package{}, fmt.Errorf("new agent client: %w", err) } defer stgglb.AgentMQPool.Release(agentCli) - startResp, err := agentCli.StartStorageCreatePackage(agtmq.NewStartStorageCreatePackage(userID, bucketID, name, storageID, path, storageAffinity)) + createResp, err := agentCli.StorageCreatePackage(agtmq.ReqStorageCreatePackage(userID, bucketID, name, storageID, path, storageAffinity)) if err != nil { - return 0, "", fmt.Errorf("start storage upload package: %w", err) + return cdssdk.Package{}, err } - return stgResp.Storages[0].MasterHub.HubID, startResp.TaskID, nil -} - -func (svc *StorageService) WaitStorageCreatePackage(hubID cdssdk.HubID, taskID string, waitTimeout time.Duration) (bool, cdssdk.PackageID, error) { - agentCli, err := stgglb.AgentMQPool.Acquire(hubID) - if err != nil { - // TODO 失败是否要当做任务已经结束？ - return true, 0, fmt.Errorf("new agent client: %w", err) - } - defer stgglb.AgentMQPool.Release(agentCli) - - waitResp, err := agentCli.WaitStorageCreatePackage(agtmq.NewWaitStorageCreatePackage(taskID, waitTimeout.Milliseconds())) - if err != nil { - // TODO 请求失败是否要当做任务已经结束？ - return true, 0, fmt.Errorf("wait storage upload package: %w", err) - } - - if !waitResp.IsComplete { - return false, 0, nil - } - - if waitResp.Error != "" { - return true, 0, fmt.Errorf("%s", waitResp.Error) - } - - return true, waitResp.PackageID, nil + return createResp.Package, nil } diff --git a/common/pkgs/mq/agent/storage.go b/common/pkgs/mq/agent/storage.go index 1182752..bc200bf 100644 --- a/common/pkgs/mq/agent/storage.go +++ b/common/pkgs/mq/agent/storage.go @@ -6,15 +6,13 @@ import ( ) type StorageService interface { - StartStorageCreatePackage(msg *StartStorageCreatePackage) (*StartStorageCreatePackageResp, *mq.CodeMessage) - - WaitStorageCreatePackage(msg *WaitStorageCreatePackage) (*WaitStorageCreatePackageResp, *mq.CodeMessage) + StorageCreatePackage(msg *StorageCreatePackage) (*StorageCreatePackageResp, *mq.CodeMessage) } // 启动从Storage上传Package的任务 -var _ = Register(Service.StartStorageCreatePackage) +var _ = Register(Service.StorageCreatePackage) -type StartStorageCreatePackage struct { +type StorageCreatePackage struct { mq.MessageBodyBase UserID cdssdk.UserID `json:"userID"` BucketID cdssdk.BucketID `json:"bucketID"` @@ -23,13 +21,13 @@ type StartStorageCreatePackage struct { Path string `json:"path"` StorageAffinity cdssdk.StorageID `json:"storageAffinity"` } -type StartStorageCreatePackageResp struct { +type StorageCreatePackageResp struct { mq.MessageBodyBase - TaskID string `json:"taskID"` + Package cdssdk.Package `json:"package"` } -func NewStartStorageCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string, storageID cdssdk.StorageID, path string, stgAffinity cdssdk.StorageID) *StartStorageCreatePackage { - return &StartStorageCreatePackage{ +func ReqStorageCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string, storageID cdssdk.StorageID, path string, stgAffinity cdssdk.StorageID) *StorageCreatePackage { + return &StorageCreatePackage{ UserID: userID, BucketID: bucketID, Name: name, @@ -38,43 +36,11 @@ func NewStartStorageCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID StorageAffinity: stgAffinity, } } -func NewStartStorageCreatePackageResp(taskID string) *StartStorageCreatePackageResp { - return &StartStorageCreatePackageResp{ - TaskID: taskID, - } -} -func (client *Client) StartStorageCreatePackage(msg *StartStorageCreatePackage, opts ...mq.RequestOption) (*StartStorageCreatePackageResp, error) { - return mq.Request(Service.StartStorageCreatePackage, client.rabbitCli, msg, opts...) -} - -// 等待从Storage上传Package的任务 -var _ = Register(Service.WaitStorageCreatePackage) - -type WaitStorageCreatePackage struct { - mq.MessageBodyBase - TaskID string `json:"taskID"` - WaitTimeoutMs int64 `json:"waitTimeout"` -} -type WaitStorageCreatePackageResp struct { - mq.MessageBodyBase - IsComplete bool `json:"isComplete"` - Error string `json:"error"` - PackageID cdssdk.PackageID `json:"packageID"` -} - -func NewWaitStorageCreatePackage(taskID string, waitTimeoutMs int64) *WaitStorageCreatePackage { - return &WaitStorageCreatePackage{ - TaskID: taskID, - WaitTimeoutMs: waitTimeoutMs, - } -} -func NewWaitStorageCreatePackageResp(isComplete bool, err string, packageID cdssdk.PackageID) *WaitStorageCreatePackageResp { - return &WaitStorageCreatePackageResp{ - IsComplete: isComplete, - Error: err, - PackageID: packageID, +func RespStorageCreatePackage(pkg cdssdk.Package) *StorageCreatePackageResp { + return &StorageCreatePackageResp{ + Package: pkg, } } -func (client *Client) WaitStorageCreatePackage(msg *WaitStorageCreatePackage, opts ...mq.RequestOption) (*WaitStorageCreatePackageResp, error) { - return mq.Request(Service.WaitStorageCreatePackage, client.rabbitCli, msg, opts...) +func (client *Client) StorageCreatePackage(msg *StorageCreatePackage, opts ...mq.RequestOption) (*StorageCreatePackageResp, error) { + return mq.Request(Service.StorageCreatePackage, client.rabbitCli, msg, opts...) } diff --git a/common/pkgs/storage/local/public_store.go b/common/pkgs/storage/local/public_store.go index 344bda6..1d5fab3 100644 --- a/common/pkgs/storage/local/public_store.go +++ b/common/pkgs/storage/local/public_store.go @@ -2,6 +2,7 @@ package local import ( "io" + "io/fs" "os" "path/filepath" @@ -60,6 +61,64 @@ func (s *PublicStore) Write(objPath string, stream io.Reader) error { return nil } +func (s *PublicStore) Read(objPath string) (io.ReadCloser, error) { + fullPath := filepath.Join(s.cfg.LoadBase, objPath) + f, err := os.Open(fullPath) + if err != nil { + return nil, err + } + + return f, nil +} + +func (s *PublicStore) List(path string, recursive bool) ([]string, error) { + fullPath := filepath.Join(s.cfg.LoadBase, path) + + var pathes []string + if recursive { + err := filepath.WalkDir(fullPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + relPath, err := filepath.Rel(s.cfg.LoadBase, path) + if err != nil { + return err + } + + pathes = append(pathes, filepath.ToSlash(relPath)) + return nil + }) + if err != nil { + return nil, err + } + + } else { + files, err := os.ReadDir(fullPath) + if err != nil { + return nil, err + } + + for _, f := range files { + if f.IsDir() { + continue + } + + relPath, err := filepath.Rel(s.cfg.LoadBase, filepath.Join(fullPath, f.Name())) + if err != nil { + return nil, err + } + + pathes = append(pathes, filepath.ToSlash(relPath)) + } + } + + return pathes, nil +} + func (s *PublicStore) getLogger() logger.Logger { return logger.WithField("PublicStore", "Local").WithField("Storage", s.agt.Detail.Storage.String()) } diff --git a/common/pkgs/storage/s3/public_store.go b/common/pkgs/storage/s3/public_store.go index ab74b12..2ba5641 100644 --- a/common/pkgs/storage/s3/public_store.go +++ b/common/pkgs/storage/s3/public_store.go @@ -3,6 +3,7 @@ package s3 import ( "context" "io" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -52,7 +53,7 @@ func (s *PublicStore) Stop() { } func (s *PublicStore) Write(objPath string, stream io.Reader) error { - key := JoinKey(objPath, s.cfg.LoadBase) + key := JoinKey(s.cfg.LoadBase, objPath) _, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{ Bucket: aws.String(s.Bucket), @@ -63,6 +64,58 @@ func (s *PublicStore) Write(objPath string, stream io.Reader) error { return err } +func (s *PublicStore) Read(objPath string) (io.ReadCloser, error) { + key := JoinKey(s.cfg.LoadBase, objPath) + + resp, err := s.cli.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: aws.String(s.Bucket), + Key: aws.String(key), + }) + + if err != nil { + return nil, err + } + + return resp.Body, nil +} + +func (s *PublicStore) List(path string, recursive bool) ([]string, error) { + key := JoinKey(s.cfg.LoadBase, path) + // TODO 待测试 + + input := &s3.ListObjectsInput{ + Bucket: aws.String(s.Bucket), + Prefix: aws.String(key), + } + + if !recursive { + input.Delimiter = aws.String("/") + } + + var pathes []string + + var marker *string + for { + input.Marker = marker + resp, err := s.cli.ListObjects(context.Background(), input) + if err != nil { + return nil, err + } + + for _, obj := range resp.Contents { + pathes = append(pathes, strings.TrimPrefix(*obj.Key, s.cfg.LoadBase+"/")) + } + + if !*resp.IsTruncated { + break + } + + marker = resp.NextMarker + } + + return pathes, nil +} + func (s *PublicStore) getLogger() logger.Logger { return logger.WithField("PublicStore", "S3").WithField("Storage", s.Detail.Storage.String()) } diff --git a/common/pkgs/storage/types/public_store.go b/common/pkgs/storage/types/public_store.go index 782f2aa..2f022bd 100644 --- a/common/pkgs/storage/types/public_store.go +++ b/common/pkgs/storage/types/public_store.go @@ -9,4 +9,7 @@ type PublicStore interface { Stop() Write(objectPath string, stream io.Reader) error + Read(objectPath string) (io.ReadCloser, error) + // 返回指定路径下的所有文件 + List(path string, recursive bool) ([]string, error) } From 8fc55e3edf638ef5406d57de0dc608be3c3b050b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 14 Mar 2025 19:05:36 +0800 Subject: [PATCH 55/64] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E7=9A=84=E8=B7=AF=E7=94=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/http/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/internal/http/server.go b/client/internal/http/server.go index 3a12144..9726f25 100644 --- a/client/internal/http/server.go +++ b/client/internal/http/server.go @@ -48,7 +48,7 @@ func (s *Server) initRouters() { s.routeV1(s.engine, rt) rt.GET(cdsapi.ObjectListPathByPath, s.Object().ListByPath) - rt.POST(cdsapi.ObjectListByIDsPath, s.Object().ListByIDs) + rt.GET(cdsapi.ObjectListByIDsPath, s.Object().ListByIDs) rt.GET(cdsapi.ObjectDownloadPath, s.Object().Download) rt.GET(cdsapi.ObjectDownloadByPathPath, s.Object().DownloadByPath) rt.POST(cdsapi.ObjectUploadPath, s.Object().Upload) @@ -85,7 +85,7 @@ func (s *Server) routeV1(eg *gin.Engine, rt gin.IRoutes) { v1 := eg.Group("/v1") v1.GET(cdsapi.ObjectListPathByPath, s.awsAuth.Auth, s.Object().ListByPath) - v1.POST(cdsapi.ObjectListByIDsPath, s.awsAuth.Auth, s.Object().ListByIDs) + v1.GET(cdsapi.ObjectListByIDsPath, s.awsAuth.Auth, s.Object().ListByIDs) v1.GET(cdsapi.ObjectDownloadPath, s.awsAuth.Auth, s.Object().Download) v1.GET(cdsapi.ObjectDownloadByPathPath, s.awsAuth.Auth, s.Object().DownloadByPath) v1.POST(cdsapi.ObjectUploadPath, s.awsAuth.AuthWithoutBody, s.Object().Upload) From d64cf26f1d34bba6f3b49be31a3bf076e8a8eeb9 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 14 Mar 2025 19:42:21 +0800 Subject: [PATCH 56/64] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=82=E6=95=B0bind?= =?UTF-8?q?=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/http/object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/http/object.go b/client/internal/http/object.go index 4e44eac..39e84ab 100644 --- a/client/internal/http/object.go +++ b/client/internal/http/object.go @@ -53,7 +53,7 @@ func (s *ObjectService) ListByIDs(ctx *gin.Context) { log := logger.WithField("HTTP", "Object.ListByIDs") var req cdsapi.ObjectListByIDs - if err := ctx.ShouldBindJSON(&req); err != nil { + if err := ctx.ShouldBindQuery(&req); err != nil { log.Warnf("binding body: %s", err.Error()) ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument")) return From 0c6918a5df686b2eb96b9f6c55e4d61caba39338 Mon Sep 17 00:00:00 2001 From: songjc <1366929814@qq.com> Date: Sun, 16 Mar 2025 11:01:20 +0800 Subject: [PATCH 57/64] =?UTF-8?q?=E6=9B=B4=E6=96=B0dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index b507a05..a146aaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM docker-0.unsee.tech/golang:alpine AS builder +FROM docker-0.unsee.tech/golang:alpine AS builder WORKDIR /app ENV GOPROXY=https://goproxy.cn \ GO111MODULE=on \ @@ -19,46 +19,34 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ mage linux all -# agent -FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS agentimage +# 基础运行环境 +FROM docker-0.unsee.tech/alpine:latest AS base WORKDIR /app -COPY --from=builder /app/storage/build/agent . RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories RUN apk add --no-cache tzdata ENV TZ=Asia/Shanghai RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# agent +FROM base AS agentimage +COPY --from=builder /app/storage/build/agent . RUN chmod +x agent/agent ENTRYPOINT ["./agent/agent"] # coordinator -FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS coorimage -WORKDIR /app +FROM base AS coorimage COPY --from=builder /app/storage/build/coordinator . -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories -RUN apk add --no-cache tzdata -ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN chmod +x coordinator/coordinator ENTRYPOINT ["./coordinator/coordinator"] # scanner -FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS scannerimage -WORKDIR /app +FROM base AS scannerimage COPY --from=builder /app/storage/build/scanner . -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories -RUN apk add --no-cache tzdata -ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN chmod +x scanner/scanner ENTRYPOINT ["./scanner/scanner"] # client -FROM --platform=$TARGETPLATFORM docker-0.unsee.tech/alpine:latest AS clientimage -WORKDIR /app +FROM base AS clientimage COPY --from=builder /app/storage/build/client . -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories -RUN apk add --no-cache tzdata -ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN chmod +x client/client ENTRYPOINT ["./client/client","serve","http"] From b40b0bc393340fbe9e316ee29d1f66a2d6dd18ea Mon Sep 17 00:00:00 2001 From: songjc <1366929814@qq.com> Date: Sun, 16 Mar 2025 11:15:58 +0800 Subject: [PATCH 58/64] =?UTF-8?q?=E6=9B=B4=E6=96=B0dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a146aaa..c48175a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,6 @@ ENV GOPROXY=https://goproxy.cn \ COPY ../common /app/common COPY . /app/storage -# 指定构建参数 -ARG TARGETOS -ARG TARGETARCH - # 编译 WORKDIR /app/storage RUN --mount=type=cache,target=/go/pkg/mod \ From ee14099aec61b996e579a2a17eb529d4636c8a08 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Sun, 16 Mar 2025 11:16:22 +0800 Subject: [PATCH 59/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index e50640b..afc1e7a 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -150,7 +150,7 @@ workflow: docker_file: '"Dockerfile"' docker_build_path: '"."' workspace: git_clone_0.git_path - image_push: true + image_push: false build_args: '""' needs: - shell_0 From 1bd102d4b4296ff890792b2c53dde85ed00073a2 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Sun, 16 Mar 2025 11:52:21 +0800 Subject: [PATCH 60/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index afc1e7a..c2c398e 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -38,7 +38,7 @@ workflow: docker_build_path: '"."' workspace: git_clone_0.git_path image_push: true - build_args: '""' + build_args: '"TARGET=scannerimage"' needs: - docker_image_build_4 - ref: docker_image_build_1 @@ -57,7 +57,7 @@ workflow: docker_build_path: '"."' workspace: git_clone_0.git_path image_push: true - build_args: '""' + build_args: '"TARGET=coorimage"' needs: - docker_image_build_4 - ref: docker_image_build_3 @@ -76,7 +76,7 @@ workflow: docker_build_path: '"."' workspace: git_clone_0.git_path image_push: true - build_args: '""' + build_args: '"TARGET=clientimage"' needs: - docker_image_build_4 - ref: ssh_cmd_0 @@ -114,7 +114,7 @@ workflow: docker_build_path: '"."' workspace: git_clone_0.git_path image_push: true - build_args: '""' + build_args: '"TARGET=agentimage"' needs: - docker_image_build_4 - ref: git_clone_1 From a1c85b1aefb2b5054b9c1a8966b763769ad2305c Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Sun, 16 Mar 2025 15:00:44 +0800 Subject: [PATCH 61/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index c2c398e..b35ab68 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -40,7 +40,7 @@ workflow: image_push: true build_args: '"TARGET=scannerimage"' needs: - - docker_image_build_4 + - shell_0 - ref: docker_image_build_1 name: coor镜像构建 cache: @@ -59,7 +59,7 @@ workflow: image_push: true build_args: '"TARGET=coorimage"' needs: - - docker_image_build_4 + - shell_0 - ref: docker_image_build_3 name: client镜像构建 cache: @@ -78,7 +78,7 @@ workflow: image_push: true build_args: '"TARGET=clientimage"' needs: - - docker_image_build_4 + - shell_0 - ref: ssh_cmd_0 name: ssh执行命令 task: ssh_cmd@1.1.1 @@ -116,7 +116,7 @@ workflow: image_push: true build_args: '"TARGET=agentimage"' needs: - - docker_image_build_4 + - shell_0 - ref: git_clone_1 name: common_clone task: git_clone@1.2.9 @@ -137,21 +137,4 @@ workflow: needs: - git_clone_0 - git_clone_1 - - ref: docker_image_build_4 - name: build - cache: - GOCACHE: /root/.cache/go-build - GOMODCACHE: /go/pkg/mod - task: docker_image_build@1.6.0 - input: - image_name: '"storage-builder"' - image_tag: '"latest"' - registry_address: '""' - docker_file: '"Dockerfile"' - docker_build_path: '"."' - workspace: git_clone_0.git_path - image_push: false - build_args: '""' - needs: - - shell_0 From 5a3454b73ed04b9a3b52e78d4a64f5de3301d7d0 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Sun, 16 Mar 2025 16:22:29 +0800 Subject: [PATCH 62/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index b35ab68..824832d 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -109,7 +109,7 @@ workflow: docker_password: ((docker_registry.registry_password)) image_name: '"112.95.163.90:5010/agentservice-x86"' image_tag: '"latest"' - registry_address: '"123.60.146.162:5010"' + registry_address: '"http://123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' workspace: git_clone_0.git_path From 01d6b4be1111c70d4ecf8aa5afa9683ea6e723a4 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Sun, 16 Mar 2025 22:13:26 +0800 Subject: [PATCH 63/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index 824832d..b35ab68 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -109,7 +109,7 @@ workflow: docker_password: ((docker_registry.registry_password)) image_name: '"112.95.163.90:5010/agentservice-x86"' image_tag: '"latest"' - registry_address: '"http://123.60.146.162:5010"' + registry_address: '"123.60.146.162:5010"' docker_file: '"Dockerfile"' docker_build_path: '"."' workspace: git_clone_0.git_path From f9dbeb56e1180a7cc87ef58cafe1008c4eb10c92 Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Sun, 16 Mar 2025 22:58:22 +0800 Subject: [PATCH 64/64] =?UTF-8?q?refactor:=20.devops/=E5=9F=BA=E4=BA=8Egit?= =?UTF-8?q?link=E6=9E=84=E5=BB=BA.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devops/基于gitlink构建.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/基于gitlink构建.yml b/.devops/基于gitlink构建.yml index b35ab68..4d76609 100644 --- a/.devops/基于gitlink构建.yml +++ b/.devops/基于gitlink构建.yml @@ -133,7 +133,7 @@ workflow: name: shell image: docker.jianmuhub.com/library/alpine:3.17.0 script: - - ls + - "ls " needs: - git_clone_0 - git_clone_1