From 51b6c07f81ece5ed619affe4e3722632c55685ce Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Thu, 25 Jul 2019 07:26:19 -0500 Subject: [PATCH] Init TensorFlow.Text project. put all UnitTest and Example into one project. --- TensorFlow.NET.sln | 24 ++--- src/TensorFlowText/README.md | 9 ++ src/TensorFlowText/TensorFlowText.csproj | 7 ++ src/TensorFlowText/Tokenizer.cs | 8 ++ .../KerasNET.Example/{Program.cs => Keras.cs} | 0 test/KerasNET.Test/Keras.UnitTest.csproj | 1 - .../{BaseTests.cs => KerasTests.cs} | 0 test/TensorFlowNET.Examples/Keras.cs | 98 +++++++++++++++++++ .../TensorFlowNET.Examples.csproj | 2 + test/TensorFlowNET.UnitTest/KerasTests.cs | 26 +++++ .../TensorFlowNET.UnitTest.csproj | 2 + 11 files changed, 158 insertions(+), 19 deletions(-) create mode 100644 src/TensorFlowText/README.md create mode 100644 src/TensorFlowText/TensorFlowText.csproj create mode 100644 src/TensorFlowText/Tokenizer.cs rename test/KerasNET.Example/{Program.cs => Keras.cs} (100%) rename test/KerasNET.Test/{BaseTests.cs => KerasTests.cs} (100%) create mode 100644 test/TensorFlowNET.Examples/Keras.cs create mode 100644 test/TensorFlowNET.UnitTest/KerasTests.cs diff --git a/TensorFlow.NET.sln b/TensorFlow.NET.sln index 7046347d..5d6e5fe7 100644 --- a/TensorFlow.NET.sln +++ b/TensorFlow.NET.sln @@ -11,17 +11,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Core", "src\T EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Keras.Core", "src\KerasNET.Core\Keras.Core.csproj", "{902E188F-A953-43B4-9991-72BAB1697BC3}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Keras.Example", "test\KerasNET.Example\Keras.Example.csproj", "{17E1AC16-9E0E-4545-905A-E92C6300C7AF}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Keras.UnitTest", "test\KerasNET.Test\Keras.UnitTest.csproj", "{A5839A45-A117-4BEA-898B-DE1ED6E0D58F}" -EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "TensorFlowNET.Examples.FSharp", "test\TensorFlowNET.Examples.FSharp\TensorFlowNET.Examples.FSharp.fsproj", "{62BC3801-F0D3-44A9-A0AC-712F40C8F961}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowBenchmark", "src\TensorFlowNet.Benchmarks\TensorFlowBenchmark.csproj", "{68861442-971A-4196-876E-C9330F0B3C54}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TensorFlowHub", "src\TensorFlowHub\TensorFlowHub.csproj", "{8FD59A5A-97EB-457E-B9F1-D88B0C822C6E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowHub", "src\TensorFlowHub\TensorFlowHub.csproj", "{8FD59A5A-97EB-457E-B9F1-D88B0C822C6E}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TensorFlowHub.Examples", "test\TensorFlowHub.Examples\TensorFlowHub.Examples.csproj", "{5FF6CB67-DC99-4741-9545-E803DFCE5CDC}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TensorFlowText", "src\TensorFlowText\TensorFlowText.csproj", "{B598E5D5-BD2D-4191-8532-F2FBAC31AB81}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -45,14 +41,6 @@ Global {902E188F-A953-43B4-9991-72BAB1697BC3}.Debug|Any CPU.Build.0 = Debug|Any CPU {902E188F-A953-43B4-9991-72BAB1697BC3}.Release|Any CPU.ActiveCfg = Release|Any CPU {902E188F-A953-43B4-9991-72BAB1697BC3}.Release|Any CPU.Build.0 = Release|Any CPU - {17E1AC16-9E0E-4545-905A-E92C6300C7AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {17E1AC16-9E0E-4545-905A-E92C6300C7AF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {17E1AC16-9E0E-4545-905A-E92C6300C7AF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {17E1AC16-9E0E-4545-905A-E92C6300C7AF}.Release|Any CPU.Build.0 = Release|Any CPU - {A5839A45-A117-4BEA-898B-DE1ED6E0D58F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A5839A45-A117-4BEA-898B-DE1ED6E0D58F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A5839A45-A117-4BEA-898B-DE1ED6E0D58F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A5839A45-A117-4BEA-898B-DE1ED6E0D58F}.Release|Any CPU.Build.0 = Release|Any CPU {62BC3801-F0D3-44A9-A0AC-712F40C8F961}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {62BC3801-F0D3-44A9-A0AC-712F40C8F961}.Debug|Any CPU.Build.0 = Debug|Any CPU {62BC3801-F0D3-44A9-A0AC-712F40C8F961}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -65,10 +53,10 @@ Global {8FD59A5A-97EB-457E-B9F1-D88B0C822C6E}.Debug|Any CPU.Build.0 = Debug|Any CPU {8FD59A5A-97EB-457E-B9F1-D88B0C822C6E}.Release|Any CPU.ActiveCfg = Release|Any CPU {8FD59A5A-97EB-457E-B9F1-D88B0C822C6E}.Release|Any CPU.Build.0 = Release|Any CPU - {5FF6CB67-DC99-4741-9545-E803DFCE5CDC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5FF6CB67-DC99-4741-9545-E803DFCE5CDC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5FF6CB67-DC99-4741-9545-E803DFCE5CDC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5FF6CB67-DC99-4741-9545-E803DFCE5CDC}.Release|Any CPU.Build.0 = Release|Any CPU + {B598E5D5-BD2D-4191-8532-F2FBAC31AB81}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B598E5D5-BD2D-4191-8532-F2FBAC31AB81}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B598E5D5-BD2D-4191-8532-F2FBAC31AB81}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B598E5D5-BD2D-4191-8532-F2FBAC31AB81}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/TensorFlowText/README.md b/src/TensorFlowText/README.md new file mode 100644 index 00000000..2d82ea9d --- /dev/null +++ b/src/TensorFlowText/README.md @@ -0,0 +1,9 @@ +# TensorFlow Text - Text processing in Tensorflow + +TensorFlow Text provides a collection of text related classes and ops ready to use with TensorFlow 2.0. The library can perform the preprocessing regularly required by text-based models, and includes other features useful for sequence modeling not provided by core TensorFlow. + +The benefit of using these ops in your text preprocessing is that they are done in the TensorFlow graph. You do not need to worry about tokenization in training being different than the tokenization at inference, or managing preprocessing scripts. + + + +https://github.com/tensorflow/text \ No newline at end of file diff --git a/src/TensorFlowText/TensorFlowText.csproj b/src/TensorFlowText/TensorFlowText.csproj new file mode 100644 index 00000000..2bd48b21 --- /dev/null +++ b/src/TensorFlowText/TensorFlowText.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp2.2 + + + diff --git a/src/TensorFlowText/Tokenizer.cs b/src/TensorFlowText/Tokenizer.cs new file mode 100644 index 00000000..5621bcf7 --- /dev/null +++ b/src/TensorFlowText/Tokenizer.cs @@ -0,0 +1,8 @@ +using System; + +namespace TensorFlowText +{ + public class Tokenizer + { + } +} diff --git a/test/KerasNET.Example/Program.cs b/test/KerasNET.Example/Keras.cs similarity index 100% rename from test/KerasNET.Example/Program.cs rename to test/KerasNET.Example/Keras.cs diff --git a/test/KerasNET.Test/Keras.UnitTest.csproj b/test/KerasNET.Test/Keras.UnitTest.csproj index 980ad049..e5956a96 100644 --- a/test/KerasNET.Test/Keras.UnitTest.csproj +++ b/test/KerasNET.Test/Keras.UnitTest.csproj @@ -35,7 +35,6 @@ - diff --git a/test/KerasNET.Test/BaseTests.cs b/test/KerasNET.Test/KerasTests.cs similarity index 100% rename from test/KerasNET.Test/BaseTests.cs rename to test/KerasNET.Test/KerasTests.cs diff --git a/test/TensorFlowNET.Examples/Keras.cs b/test/TensorFlowNET.Examples/Keras.cs new file mode 100644 index 00000000..3bf46c00 --- /dev/null +++ b/test/TensorFlowNET.Examples/Keras.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using Tensorflow; +using Keras.Layers; +using NumSharp; +using Keras; + +namespace TensorFlowNET.Examples +{ + public class Keras : IExample + { + public bool Enabled { get; set; } = true; + public bool IsImportingGraph { get; set; } = false; + + public string Name => "Keras"; + + public bool Run() + { + Console.WriteLine("================================== Keras =================================="); + + #region data + var batch_size = 1000; + var (X, Y) = XOR(batch_size); + //var (X, Y, batch_size) = (np.array(new float[,]{{1, 0 },{1, 1 },{0, 0 },{0, 1 }}), np.array(new int[] { 0, 1, 1, 0 }), 4); + #endregion + + #region features + var (features, labels) = (new Tensor(X), new Tensor(Y)); + var num_steps = 10000; + #endregion + + #region model + var m = new Model(); + + //m.Add(new Dense(8, name: "Hidden", activation: tf.nn.relu())).Add(new Dense(1, name:"Output")); + + m.Add( + new ILayer[] { + new Dense(8, name: "Hidden_1", activation: tf.nn.relu()), + new Dense(1, name: "Output") + }); + + m.train(num_steps, (X, Y)); + #endregion + + return true; + } + + static (NDArray, NDArray) XOR(int samples) + { + var X = new List(); + var Y = new List(); + var r = new Random(); + for (int i = 0; i < samples; i++) + { + var x1 = (float)r.Next(0, 2); + var x2 = (float)r.Next(0, 2); + var y = 0.0f; + if (x1 == x2) + y = 1.0f; + X.Add(new float[] { x1, x2 }); + Y.Add(y); + } + + return (np.array(X.ToArray()), np.array(Y.ToArray())); + } + + public Graph BuildGraph() + { + throw new NotImplementedException(); + } + + public Graph ImportGraph() + { + throw new NotImplementedException(); + } + + public void Predict(Session sess) + { + throw new NotImplementedException(); + } + + public void PrepareData() + { + throw new NotImplementedException(); + } + + public void Test(Session sess) + { + throw new NotImplementedException(); + } + + public void Train(Session sess) + { + throw new NotImplementedException(); + } + } +} diff --git a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj index 83f4a3ba..149bd549 100644 --- a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj +++ b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj @@ -15,6 +15,8 @@ + + diff --git a/test/TensorFlowNET.UnitTest/KerasTests.cs b/test/TensorFlowNET.UnitTest/KerasTests.cs new file mode 100644 index 00000000..707a6083 --- /dev/null +++ b/test/TensorFlowNET.UnitTest/KerasTests.cs @@ -0,0 +1,26 @@ +using Tensorflow; +using Keras.Layers; +using NumSharp; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace TensorFlowNET.UnitTest +{ + [TestClass] + public class BaseTests + { + [TestMethod] + public void Dense_Tensor_ShapeTest() + { + var dense_1 = new Dense(1, name: "dense_1", activation: tf.nn.relu()); + var input = new Tensor(np.array(new int[] { 3 })); + dense_1.__build__(input.TensorShape); + var outputShape = dense_1.output_shape(input.TensorShape); + var a = (int[])(outputShape.Dimensions); + var b = (int[])(new int[] { 1 }); + var _a = np.array(a); + var _b = np.array(b); + + Assert.IsTrue(np.array_equal(_a, _b)); + } + } +} diff --git a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj index fed4dfc9..848512f0 100644 --- a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj @@ -23,8 +23,10 @@ + +