From c944f54b76adfc5199eb65c5eb0ab0ec521e59a2 Mon Sep 17 00:00:00 2001 From: haiping008 Date: Tue, 5 Mar 2019 11:32:05 -0600 Subject: [PATCH] VocabularyProcessor is DEPRECATED, change to another example. --- TensorFlow.NET.sln | 6 ++++++ .../Preprocessing/VocabularyProcessor.cs | 19 +++++++++++++++++++ .../TensorFlowNET.Core.csproj | 4 ++++ .../CnnTextClassification/CnnTextTrain.cs | 7 ++++++- .../CnnTextClassification/DataHelpers.cs | 7 +++---- .../TensorFlowNET.Examples.csproj | 1 + 6 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 src/TensorFlowNET.Core/Contrib/Learn/Preprocessing/VocabularyProcessor.cs diff --git a/TensorFlow.NET.sln b/TensorFlow.NET.sln index 880d886b..0f9960f0 100644 --- a/TensorFlow.NET.sln +++ b/TensorFlow.NET.sln @@ -13,6 +13,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Utility", "sr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Visualization", "TensorFlowNET.Visualization\TensorFlowNET.Visualization.csproj", "{4BB2ABD1-635E-41E4-B534-CB5B6A2D754D}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NumSharp.Core", "..\NumSharp\src\NumSharp.Core\NumSharp.Core.csproj", "{E8340C61-12C1-4BEE-A340-403E7C1ACD82}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -39,6 +41,10 @@ Global {4BB2ABD1-635E-41E4-B534-CB5B6A2D754D}.Debug|Any CPU.Build.0 = Debug|Any CPU {4BB2ABD1-635E-41E4-B534-CB5B6A2D754D}.Release|Any CPU.ActiveCfg = Release|Any CPU {4BB2ABD1-635E-41E4-B534-CB5B6A2D754D}.Release|Any CPU.Build.0 = Release|Any CPU + {E8340C61-12C1-4BEE-A340-403E7C1ACD82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E8340C61-12C1-4BEE-A340-403E7C1ACD82}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E8340C61-12C1-4BEE-A340-403E7C1ACD82}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E8340C61-12C1-4BEE-A340-403E7C1ACD82}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/TensorFlowNET.Core/Contrib/Learn/Preprocessing/VocabularyProcessor.cs b/src/TensorFlowNET.Core/Contrib/Learn/Preprocessing/VocabularyProcessor.cs new file mode 100644 index 00000000..00f2c9b0 --- /dev/null +++ b/src/TensorFlowNET.Core/Contrib/Learn/Preprocessing/VocabularyProcessor.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Contrib.Learn.Preprocessing +{ + public class VocabularyProcessor + { + private int _max_document_length; + private int _min_frequency; + + public VocabularyProcessor(int max_document_length, + int min_frequency) + { + _max_document_length = max_document_length; + _min_frequency = min_frequency; + } + } +} diff --git a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj index 267678b1..949166c1 100644 --- a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj +++ b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj @@ -51,4 +51,8 @@ Fixed import name scope issue. + + + + diff --git a/test/TensorFlowNET.Examples/CnnTextClassification/CnnTextTrain.cs b/test/TensorFlowNET.Examples/CnnTextClassification/CnnTextTrain.cs index 9bc19989..ef4b0749 100644 --- a/test/TensorFlowNET.Examples/CnnTextClassification/CnnTextTrain.cs +++ b/test/TensorFlowNET.Examples/CnnTextClassification/CnnTextTrain.cs @@ -1,6 +1,7 @@ using NumSharp.Core; using System; using System.Collections.Generic; +using System.Linq; using System.Text; using Tensorflow; @@ -46,7 +47,11 @@ namespace TensorFlowNET.Examples.CnnTextClassification public (NDArray, NDArray, NDArray, NDArray, NDArray) preprocess() { - DataHelpers.load_data_and_labels(positive_data_file, negative_data_file); + var (x_text, y) = DataHelpers.load_data_and_labels(positive_data_file, negative_data_file); + + // Build vocabulary + int max_document_length = x_text.Select(x => x.Split(' ').Length).Max(); + var vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length) throw new NotImplementedException(""); } } diff --git a/test/TensorFlowNET.Examples/CnnTextClassification/DataHelpers.cs b/test/TensorFlowNET.Examples/CnnTextClassification/DataHelpers.cs index 117ae187..043b6945 100644 --- a/test/TensorFlowNET.Examples/CnnTextClassification/DataHelpers.cs +++ b/test/TensorFlowNET.Examples/CnnTextClassification/DataHelpers.cs @@ -17,7 +17,7 @@ namespace TensorFlowNET.Examples.CnnTextClassification /// /// /// - public static (NDArray, NDArray) load_data_and_labels(string positive_data_file, string negative_data_file) + public static (string[], NDArray) load_data_and_labels(string positive_data_file, string negative_data_file) { Directory.CreateDirectory("CnnTextClassification"); Utility.Web.Download(positive_data_file, "CnnTextClassification/rt-polarity.pos"); @@ -39,9 +39,8 @@ namespace TensorFlowNET.Examples.CnnTextClassification var positive_labels = positive_examples.Select(x => new int[2] { 0, 1 }).ToArray(); var negative_labels = negative_examples.Select(x => new int[2] { 1, 0 }).ToArray(); - // var y = np. - // return (x_text, y); - throw new NotImplementedException("load_data_and_labels"); + var y = np.concatenate(new int[][][] { positive_labels, negative_labels }); + return (x_text.ToArray(), y); } private static string clean_str(string str) diff --git a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj index e32a8092..34a29361 100644 --- a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj +++ b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj @@ -11,6 +11,7 @@ +