build: add native library splitter and adjust directory structure.

TensorFlow.NET.sln

@@ -37,6 +37,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.UnitTest.RedistH
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.CodeGen", "Tensorflow.CodeGen\Tensorflow.CodeGen.csproj", "{BADBB104-2F03-4824-A249-803A871D8122}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tensorflow.Redist.NativeLibrarySplitter", "NativeLibrarySplitter\Tensorflow.Redist.NativeLibrarySplitter.csproj", "{B85FA7C7-1E8D-4567-B3F4-605955557DAE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -302,6 +304,24 @@ Global
 		{BADBB104-2F03-4824-A249-803A871D8122}.Release|x64.Build.0 = Release|Any CPU
 		{BADBB104-2F03-4824-A249-803A871D8122}.Release|x86.ActiveCfg = Release|Any CPU
 		{BADBB104-2F03-4824-A249-803A871D8122}.Release|x86.Build.0 = Release|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Debug|x64.Build.0 = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Debug|x86.Build.0 = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.GPU|Any CPU.ActiveCfg = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.GPU|Any CPU.Build.0 = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.GPU|x64.ActiveCfg = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.GPU|x64.Build.0 = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.GPU|x86.ActiveCfg = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.GPU|x86.Build.0 = Debug|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Release|x64.ActiveCfg = Release|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Release|x64.Build.0 = Release|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Release|x86.ActiveCfg = Release|Any CPU
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -321,6 +341,7 @@ Global
 		{7DEA8760-E401-4872-81F3-405F185A13A0} = {1B0918B9-65AD-4F34-A287-AF4597B27DBD}
 		{62D543A2-8846-45A3-829B-5754B094A8E2} = {E1A5D2B7-10AF-4876-85C0-7714EF274214}
 		{BADBB104-2F03-4824-A249-803A871D8122} = {E1A5D2B7-10AF-4876-85C0-7714EF274214}
+		{B85FA7C7-1E8D-4567-B3F4-605955557DAE} = {E1A5D2B7-10AF-4876-85C0-7714EF274214}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {2DEAD3CC-486B-4918-A607-50B0DE7B114A}

tools/Tensorflow.Redist.NativeLibrarySplitter/Program.cs

@@ -0,0 +1,212 @@
+
+// =================================================================== //
+// This is a tool to split the native .so file of linux gpu library    //
+// =================================================================== //
+
+using System.Security.Cryptography;
+
+string filename = "libtensorflow.so";
+int count = 5;
+SplitFile(filename, count);
+
+static void SplitFile(string filename, int count)
+{
+    // 打开读取二进制文件的文件流
+    using (FileStream input = new FileStream(filename, FileMode.Open, FileAccess.Read))
+    {
+        long filesize = new FileInfo(filename).Length; // 获取文件大小
+        long fragmentSize = (long)(filesize / count + 1); // 计算每个分片的大小
+
+        byte[] buffer = new byte[fragmentSize]; // 设置缓冲区大小
+        int bytesRead; // 存储读取长度
+        int fragmentIndex = 1; // 分片计数器
+
+        // 使用循环遍历分片并写入相应的文件
+        while ((bytesRead = input.Read(buffer, 0, buffer.Length)) > 0)
+        {
+            string outputFileName = $"{filename}.fragment{fragmentIndex++}";
+            using (FileStream output = new FileStream(outputFileName, FileMode.Create, FileAccess.Write))
+            {
+                output.Write(buffer, 0, bytesRead);
+            }
+        }
+
+        // 计算整个文件的 SHA-256 哈希值并写入 .sha 文件
+        using (SHA256 sha256Hash = SHA256.Create())
+        {
+            input.Seek(0, SeekOrigin.Begin);
+            byte[] hashValue = sha256Hash.ComputeHash(input);
+
+            string shaFileName = $"{filename}.sha";
+            using (StreamWriter writer = new StreamWriter(shaFileName, false))
+            {
+                writer.Write(BitConverter.ToString(hashValue).Replace("-", ""));
+            }
+        }
+    }
+}
+
+// Resume the file from fregments. Thanks for the code in TorchSharp!
+static void Restitch(string RestitcherPackage)
+{
+    // !!!!!!!------------------------------NOTE------------------------------------!!!!!!
+    // !!!!!!! This code is manually copied into pkg\common\RestitchPackage.targets !!!!!!
+    // !!!!!!!------------------------------NOTE------------------------------------!!!!!!
+    //
+    // vvvvvvvvvvvvvvvvvvvvvvvvvvvvv START HERE vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
+    try
+    {
+        if (Directory.Exists(RestitcherPackage))
+        {
+            using (var writer = File.CreateText("obj/tensorflow_redist_build_log.txt"))
+            {
+                foreach (var p in Directory.EnumerateFiles(RestitcherPackage, "*", SearchOption.AllDirectories))
+                {
+
+                    var primaryFile = Path.GetFullPath(p);
+                    writer.WriteLine("Found primary file at {0}", primaryFile);
+
+                    // See if there are fragments in the parallel nuget packages. If the primary is 
+                    //        some-package-primary\runtimes\....\a.so 
+                    //        some-package-primary\runtimes\....\a.so.sha
+                    // then the expected fragments are
+                    //        some-package-fragment1\fragments\....\a.so 
+                    //        some-package-fragment2\fragments\....\a.so 
+                    //        some-package-fragment3\fragments\....\a.so 
+                    //        some-package-fragment4\fragments\....\a.so 
+                    //        some-package-fragment5\fragments\....\a.so 
+                    //        some-package-fragment6\fragments\....\a.so 
+                    //        some-package-fragment7\fragments\....\a.so 
+                    //        some-package-fragment8\fragments\....\a.so 
+                    //        some-package-fragment9\fragments\....\a.so 
+                    //        some-package-fragment10\fragments\....\a.so 
+                    var shaFile = primaryFile + ".sha";
+                    var fragmentFile1 = primaryFile.Replace("-primary", "-fragment1").Replace("runtimes", "fragments") + ".fragment1";
+                    var fragmentFile2 = primaryFile.Replace("-primary", "-fragment2").Replace("runtimes", "fragments") + ".fragment2";
+                    var fragmentFile3 = primaryFile.Replace("-primary", "-fragment3").Replace("runtimes", "fragments") + ".fragment3";
+                    var fragmentFile4 = primaryFile.Replace("-primary", "-fragment4").Replace("runtimes", "fragments") + ".fragment4";
+                    var fragmentFile5 = primaryFile.Replace("-primary", "-fragment5").Replace("runtimes", "fragments") + ".fragment5";
+
+
+                    if (File.Exists(fragmentFile1)) writer.WriteLine("Found fragment file at {0}", fragmentFile1);
+                    if (File.Exists(fragmentFile2)) writer.WriteLine("Found fragment file at {0}", fragmentFile2);
+                    if (File.Exists(fragmentFile3)) writer.WriteLine("Found fragment file at {0}", fragmentFile3);
+                    if (File.Exists(fragmentFile4)) writer.WriteLine("Found fragment file at {0}", fragmentFile4);
+                    if (File.Exists(fragmentFile5)) writer.WriteLine("Found fragment file at {0}", fragmentFile5);
+
+                    if (File.Exists(fragmentFile1))
+                    {
+                        var tmpFile = Path.GetTempFileName();
+
+                        {
+                            writer.WriteLine("Writing restored primary file at {0}", tmpFile);
+                            using (var os = File.OpenWrite(tmpFile))
+                            {
+
+                                //writer.WriteLine("Writing bytes from {0} to {1}", primaryFile, tmpFile);
+                                //var primaryBytes = File.ReadAllBytes(primaryFile);
+
+                                //os.Write(primaryBytes, 0, primaryBytes.Length);
+                                if (File.Exists(fragmentFile1))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile1, tmpFile);
+                                    var fragmentBytes1 = File.ReadAllBytes(fragmentFile1);
+                                    os.Write(fragmentBytes1, 0, fragmentBytes1.Length);
+                                }
+                                if (File.Exists(fragmentFile2))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile2, tmpFile);
+                                    var fragmentBytes2 = File.ReadAllBytes(fragmentFile2);
+                                    os.Write(fragmentBytes2, 0, fragmentBytes2.Length);
+                                }
+                                if (File.Exists(fragmentFile3))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile3, tmpFile);
+                                    var fragmentBytes3 = File.ReadAllBytes(fragmentFile3);
+                                    os.Write(fragmentBytes3, 0, fragmentBytes3.Length);
+                                }
+                                if (File.Exists(fragmentFile4))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile4, tmpFile);
+                                    var fragmentBytes4 = File.ReadAllBytes(fragmentFile4);
+                                    os.Write(fragmentBytes4, 0, fragmentBytes4.Length);
+                                }
+                                if (File.Exists(fragmentFile5))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile5, tmpFile);
+                                    var fragmentBytes5 = File.ReadAllBytes(fragmentFile5);
+                                    os.Write(fragmentBytes5, 0, fragmentBytes5.Length);
+                                }
+                            }
+                        }
+
+                        var shaExpected = File.Exists(shaFile) ? File.ReadAllText(shaFile).ToUpper() : "";
+                        writer.WriteLine($"real sha: {shaExpected}");
+
+                        using (var sha256Hash = System.Security.Cryptography.SHA256.Create())
+                        {
+                            using (var os2 = File.OpenRead(tmpFile))
+                            {
+
+                                byte[] bytes = sha256Hash.ComputeHash(os2);
+                                var builder = new System.Text.StringBuilder();
+                                for (int i = 0; i < bytes.Length; i++)
+                                {
+                                    builder.Append(bytes[i].ToString("x2"));
+                                }
+                                var shaReconstituted = builder.ToString().ToUpper();
+                                if (shaExpected != shaReconstituted)
+                                {
+                                    string msg =
+                                            $"Error downloading and reviving packages. Reconsituted file contents have incorrect SHA\n\tExpected SHA: ${shaExpected}\n\tActual SHA: ${shaReconstituted}\n\tFile was reconstituted from:"
+                                          + $"\n\t{primaryFile} (length ${new FileInfo(primaryFile).Length})"
+                                          + (File.Exists(fragmentFile1) ? $"\n\t{fragmentFile1} (length ${new FileInfo(fragmentFile1).Length})" : "")
+                                          + (File.Exists(fragmentFile2) ? $"\n\t{fragmentFile2} (length ${new FileInfo(fragmentFile2).Length})" : "")
+                                          + (File.Exists(fragmentFile3) ? $"\n\t{fragmentFile3} (length ${new FileInfo(fragmentFile3).Length})" : "")
+                                          + (File.Exists(fragmentFile4) ? $"\n\t{fragmentFile4} (length ${new FileInfo(fragmentFile4).Length})" : "")
+                                          + (File.Exists(fragmentFile5) ? $"\n\t{fragmentFile5} (length ${new FileInfo(fragmentFile5).Length})" : "");
+                                    writer.WriteLine(msg);
+                                    throw new Exception(msg);
+                                }
+                            }
+
+                        }
+
+                        writer.WriteLine("Deleting {0}", primaryFile);
+                        File.Delete(primaryFile);
+                        if (File.Exists(primaryFile))
+                            throw new Exception("wtf?");
+
+                        writer.WriteLine("Moving {0} --> {1}", tmpFile, primaryFile);
+                        File.Move(tmpFile, primaryFile);
+
+                        writer.WriteLine("Deleting {0}", fragmentFile1);
+                        File.Delete(fragmentFile1);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile2);
+                        if (File.Exists(fragmentFile2))
+                            File.Delete(fragmentFile2);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile3);
+                        if (File.Exists(fragmentFile3))
+                            File.Delete(fragmentFile3);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile4);
+                        if (File.Exists(fragmentFile4))
+                            File.Delete(fragmentFile4);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile5);
+                        if (File.Exists(fragmentFile5))
+                            File.Delete(fragmentFile5);  // free up space and prevent us doing this again 
+                    }
+                }
+            }
+        }
+    }
+    catch (Exception ex)
+    {
+        Console.Error.WriteLine(ex.ToString());
+        Console.Error.WriteLine(ex.StackTrace);
+    }
+    // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ END HERE^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+}

tools/Tensorflow.Redist.NativeLibrarySplitter/Tensorflow.Redist.NativeLibrarySplitter.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>