From 0ad0fdbae77c72bc8279f6b2d3bd2a8cf66f19cc Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 15:12:26 +0300 Subject: [PATCH 01/16] Binding.print: revamped to print arrays properly --- src/TensorFlowNET.Core/Binding.Util.cs | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Binding.Util.cs b/src/TensorFlowNET.Core/Binding.Util.cs index ff383642..def78327 100644 --- a/src/TensorFlowNET.Core/Binding.Util.cs +++ b/src/TensorFlowNET.Core/Binding.Util.cs @@ -21,6 +21,7 @@ using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.Linq; +using NumSharp.Utilities; namespace Tensorflow { @@ -29,9 +30,37 @@ namespace Tensorflow /// public static partial class Binding { + private static string _tostring(object obj) + { + switch (obj) + { + case NDArray nd: + return nd.ToString(false); + case Array arr: + if (arr.Rank!=1 || arr.GetType().GetElementType()?.IsArray == true) + arr = Arrays.Flatten(arr); + var objs = toObjectArray(arr); + return $"[{string.Join(", ", objs.Select(_tostring))}]"; + default: + return obj?.ToString() ?? "null"; + } + + object[] toObjectArray(Array arr) + { + var len = arr.LongLength; + var ret = new object[len]; + for (long i = 0; i < len; i++) + { + ret[i] = arr.GetValue(i); + } + + return ret; + } + } + public static void print(object obj) { - Console.WriteLine(obj.ToString()); + Console.WriteLine(_tostring(obj)); } public static int len(object a) From cebc8b4bf58b386c5affee1dafe7db31a4b952cd Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 15:21:57 +0300 Subject: [PATCH 02/16] Binding: Added None --- src/TensorFlowNET.Core/Binding.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/TensorFlowNET.Core/Binding.cs b/src/TensorFlowNET.Core/Binding.cs index f443f2eb..e8355e32 100644 --- a/src/TensorFlowNET.Core/Binding.cs +++ b/src/TensorFlowNET.Core/Binding.cs @@ -7,5 +7,10 @@ namespace Tensorflow public static partial class Binding { public static tensorflow tf { get; } = New(); + + /// + /// Alias to null, similar to python's None. + /// + public static readonly object None = null; } } From 91a19c2dcf3d6339d1e1f7633faeb6bf64182fad Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 15:22:14 +0300 Subject: [PATCH 03/16] tf.placeholder: Added overload with object[] shape --- src/TensorFlowNET.Core/tensorflow.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index bdb2f537..3f141879 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -63,6 +63,11 @@ namespace Tensorflow return gen_array_ops.placeholder(dtype, shape, name); } + public unsafe Tensor placeholder(TF_DataType dtype, object[] shape = null, string name = null) + { + return placeholder(dtype, new TensorShape(shape), name); + } + public void enable_eager_execution() { // contex = new Context(); From 3cc4861a8f68365d2a74d8a4bb694646c66e6704 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 15:22:31 +0300 Subject: [PATCH 04/16] TensorShape: Added constructor of (params object[] dims) --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index 1fc95927..4de72c6c 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; +using NumSharp.Utilities; namespace Tensorflow { @@ -65,6 +66,30 @@ namespace Tensorflow } } + /// + /// An overload that can accept . + /// + public TensorShape(params object[] dims) + { + var intdims = new int[dims.Length]; + for (int i = 0; i < dims.Length; i++) + { + var val = dims[i]; + if (val == Binding.None) + intdims[i] = -1; + else + intdims[i] = Converts.ToInt32(val); + } + + switch (dims.Length) + { + case 0: shape = new Shape(new int[0]); break; + case 1: shape = Shape.Vector((int) intdims[0]); break; + case 2: shape = Shape.Matrix(intdims[0], intdims[1]); break; + default: shape = new Shape(intdims); break; + } + } + /// /// /// From a655882142d67c2bfa3287336c374d618e13bd99 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 15:26:47 +0300 Subject: [PATCH 05/16] Fixed ambiguous overloads. --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 11 ----------- src/TensorFlowNET.Core/tensorflow.cs | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index 4de72c6c..f434fba9 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -55,17 +55,6 @@ namespace Tensorflow } } - public TensorShape(params int[] dims) - { - switch (dims.Length) - { - case 0: shape = new Shape(new int[0]); break; - case 1: shape = Shape.Vector((int) dims[0]); break; - case 2: shape = Shape.Matrix(dims[0], dims[1]); break; - default: shape = new Shape(dims); break; - } - } - /// /// An overload that can accept . /// diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index 3f141879..494f2e89 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -63,7 +63,7 @@ namespace Tensorflow return gen_array_ops.placeholder(dtype, shape, name); } - public unsafe Tensor placeholder(TF_DataType dtype, object[] shape = null, string name = null) + public unsafe Tensor placeholder(TF_DataType dtype, object[] shape, string name = null) { return placeholder(dtype, new TensorShape(shape), name); } From db5cbd38484ee64fef89d3c6c601feacc1185507 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 15:29:37 +0300 Subject: [PATCH 06/16] Removed unnecessary doc --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index f434fba9..4f2accc3 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -55,9 +55,6 @@ namespace Tensorflow } } - /// - /// An overload that can accept . - /// public TensorShape(params object[] dims) { var intdims = new int[dims.Length]; From cc54fe19bf9cb109c66e95c80bef240b3c3ea782 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 17:01:37 +0300 Subject: [PATCH 07/16] TensorShape: Fixed construction when passing int[] or long[] --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 48 ++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index 4f2accc3..734e9bc5 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -57,10 +57,36 @@ namespace Tensorflow public TensorShape(params object[] dims) { - var intdims = new int[dims.Length]; - for (int i = 0; i < dims.Length; i++) + Array arr; + + if (dims.Length == 1) + { + switch (dims[0]) + { + case int[] intarr: + arr = intarr; + break; + case long[] longarr: + arr = longarr; + break; + case object[] objarr: + arr = objarr; + break; + case int _: + case long _: + arr = dims; + break; + default: + Binding.print(dims); + throw new ArgumentException(nameof(dims)); + } + } else + arr = dims; + + var intdims = new int[arr.Length]; + for (int i = 0; i < arr.Length; i++) { - var val = dims[i]; + var val = arr.GetValue(i); if (val == Binding.None) intdims[i] = -1; else @@ -69,10 +95,18 @@ namespace Tensorflow switch (dims.Length) { - case 0: shape = new Shape(new int[0]); break; - case 1: shape = Shape.Vector((int) intdims[0]); break; - case 2: shape = Shape.Matrix(intdims[0], intdims[1]); break; - default: shape = new Shape(intdims); break; + case 0: + shape = new Shape(new int[0]); + break; + case 1: + shape = Shape.Vector((int) intdims[0]); + break; + case 2: + shape = Shape.Matrix(intdims[0], intdims[1]); + break; + default: + shape = new Shape(intdims); + break; } } From 4db89af0499db1e9c0c8d65d652633c87e72b2d0 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 17:01:43 +0300 Subject: [PATCH 08/16] TensorShape: fixed size computation when None is used. --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index 734e9bc5..b2161d48 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -33,7 +33,23 @@ namespace Tensorflow /// /// Returns the size this shape represents. /// - public int size => shape.Size; + public int size + { + get + { + var dims = shape.Dimensions; + var computed = 1; + for (int i = 0; i < dims.Length; i++) + { + var val = dims[i]; + if (val <= 0) + continue; + computed *= val; + } + + return computed; + } + } public TensorShape(TensorShapeProto proto) { From 61c43175367f855d79f091d5d9b302f2c43feb50 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 17:02:00 +0300 Subject: [PATCH 09/16] tf.layers.flatten: Added support for None dimension --- src/TensorFlowNET.Core/APIs/tf.layers.cs | 23 +++++++++++++++++-- .../layers_test/flatten.cs | 9 ++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.layers.cs b/src/TensorFlowNET.Core/APIs/tf.layers.cs index 089dd8a5..39bacfde 100644 --- a/src/TensorFlowNET.Core/APIs/tf.layers.cs +++ b/src/TensorFlowNET.Core/APIs/tf.layers.cs @@ -15,6 +15,8 @@ ******************************************************************************/ using System.Collections.Generic; +using System.Linq; +using NumSharp; using Tensorflow.Keras.Layers; using Tensorflow.Operations.Activation; using static Tensorflow.Binding; @@ -182,6 +184,7 @@ namespace Tensorflow string name = null, string data_format = "channels_last") { + var input_shape = inputs.shape; if (inputs.shape.Length == 0) throw new ValueError($"Input 0 of layer flatten is incompatible with the layer: : expected min_ndim={1}, found ndim={0}. Full shape received: ()"); @@ -193,9 +196,25 @@ namespace Tensorflow inputs = array_ops.transpose(inputs, premutation.ToArray()); } - var ret = array_ops.reshape(inputs, new int[] {inputs.shape[0], -1}); - ret.set_shape(new int[] {inputs.shape[0], -1}); + var ret = array_ops.reshape(inputs, new int[] {input_shape[0], -1}); + ret.shape = ret.shape; + //ret.set_shape(compute_output_shape(ret.shape)); return ret; + + int[] compute_output_shape(int[] inputshape) + { + if (inputshape == null || inputshape.Length == 0) + inputshape = new int[] {1}; + + if (inputshape.Skip(1).All(d => d > 0)) + { + int[] output_shape = new int[2]; + output_shape[0] = inputshape[0]; + output_shape[1] = inputshape.Skip(1).Aggregate(1, (acc, rhs) => acc*rhs); //calculate size of all the rest dimensions + return output_shape; + } else + return new int[] {inputshape[0], -1}; //-1 == Binding.None + } } } } diff --git a/test/TensorFlowNET.UnitTest/layers_test/flatten.cs b/test/TensorFlowNET.UnitTest/layers_test/flatten.cs index d533f128..981af9a4 100644 --- a/test/TensorFlowNET.UnitTest/layers_test/flatten.cs +++ b/test/TensorFlowNET.UnitTest/layers_test/flatten.cs @@ -36,5 +36,14 @@ namespace TensorFlowNET.UnitTest.layers_test var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape()); new Action(() => sess.run(tf.layers.flatten(input), (input, NDArray.Scalar(6)))).Should().Throw(); } + + [TestMethod] + public void Case4() + { + var sess = tf.Session().as_default(); + + var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape(3, 4, None, 1, 2)); + sess.run(tf.layers.flatten(input), (input, np.arange(3 * 4 * 3 * 1 * 2).reshape(3, 4, 3, 1, 2))).Should().BeShaped(3, 24); + } } } \ No newline at end of file From f7364eb6e2be31e7894360fd432fc007da985975 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 17:04:39 +0300 Subject: [PATCH 10/16] TensorShape: fixed typo in constructor --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index b2161d48..6353aea6 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -109,7 +109,7 @@ namespace Tensorflow intdims[i] = Converts.ToInt32(val); } - switch (dims.Length) + switch (intdims.Length) { case 0: shape = new Shape(new int[0]); From cc65ae5ca2d5dcdacfc614b7de6daf1810e5355d Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 17:05:45 +0300 Subject: [PATCH 11/16] TensorShape: Fixed rare case when passing only None --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index 6353aea6..402ce3d8 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -92,6 +92,9 @@ namespace Tensorflow case long _: arr = dims; break; + case null: //==Binding.None + arr = dims; + break; default: Binding.print(dims); throw new ArgumentException(nameof(dims)); From 0c32b73fc11b8b75213c0450df68ec9ca0d03816 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 17:09:27 +0300 Subject: [PATCH 12/16] tf.layers.flatten: Added and fixed special unit-test case. --- src/TensorFlowNET.Core/APIs/tf.layers.cs | 3 +-- test/TensorFlowNET.UnitTest/layers_test/flatten.cs | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.layers.cs b/src/TensorFlowNET.Core/APIs/tf.layers.cs index 39bacfde..786469b5 100644 --- a/src/TensorFlowNET.Core/APIs/tf.layers.cs +++ b/src/TensorFlowNET.Core/APIs/tf.layers.cs @@ -196,8 +196,7 @@ namespace Tensorflow inputs = array_ops.transpose(inputs, premutation.ToArray()); } - var ret = array_ops.reshape(inputs, new int[] {input_shape[0], -1}); - ret.shape = ret.shape; + var ret = array_ops.reshape(inputs, compute_output_shape(input_shape)); //ret.set_shape(compute_output_shape(ret.shape)); return ret; diff --git a/test/TensorFlowNET.UnitTest/layers_test/flatten.cs b/test/TensorFlowNET.UnitTest/layers_test/flatten.cs index 981af9a4..8f97d5c2 100644 --- a/test/TensorFlowNET.UnitTest/layers_test/flatten.cs +++ b/test/TensorFlowNET.UnitTest/layers_test/flatten.cs @@ -45,5 +45,14 @@ namespace TensorFlowNET.UnitTest.layers_test var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape(3, 4, None, 1, 2)); sess.run(tf.layers.flatten(input), (input, np.arange(3 * 4 * 3 * 1 * 2).reshape(3, 4, 3, 1, 2))).Should().BeShaped(3, 24); } + + [TestMethod] + public void Case5() + { + var sess = tf.Session().as_default(); + + var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape(None, 4, 3, 1, 2)); + sess.run(tf.layers.flatten(input), (input, np.arange(3 * 4 * 3 * 1 * 2).reshape(3, 4, 3, 1, 2))).Should().BeShaped(3, 24); + } } } \ No newline at end of file From 074d06dc7f2672ffb57a334ce75720ea60019fbd Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Sep 2019 20:24:40 +0300 Subject: [PATCH 13/16] Added all keras activations --- src/TensorFlowNET.Core/APIs/tf.layers.cs | 17 +- src/TensorFlowNET.Core/APIs/tf.math.cs | 32 +++ src/TensorFlowNET.Core/APIs/tf.tensor.cs | 4 +- .../Activation/gen_nn_ops.activations.cs | 186 +++++++++++++++++- 4 files changed, 229 insertions(+), 10 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.layers.cs b/src/TensorFlowNET.Core/APIs/tf.layers.cs index 089dd8a5..dd74de2d 100644 --- a/src/TensorFlowNET.Core/APIs/tf.layers.cs +++ b/src/TensorFlowNET.Core/APIs/tf.layers.cs @@ -144,6 +144,20 @@ namespace Tensorflow return layer.apply(inputs); } + /// + /// Densely-connected layer class. aka fully-connected

+ /// `outputs = activation(inputs * kernel + bias)` + /// + /// + /// Python integer, dimensionality of the output space. + /// + /// Boolean, whether the layer uses a bias. + /// + /// + /// + /// + /// + /// public Tensor dense(Tensor inputs, int units, IActivation activation = null, @@ -160,7 +174,8 @@ namespace Tensorflow var layer = new Dense(units, activation, use_bias: use_bias, bias_initializer: bias_initializer, - kernel_initializer: kernel_initializer); + kernel_initializer: kernel_initializer, + trainable: trainable); return layer.apply(inputs); } diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index ec081cc4..985e3f73 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -14,6 +14,8 @@ limitations under the License. ******************************************************************************/ +using Tensorflow.Operations; + namespace Tensorflow { public partial class tensorflow @@ -211,6 +213,36 @@ namespace Tensorflow /// public Tensor _clip_by_value(Tensor t, Tensor clip_value_min, Tensor clip_value_max, string name = null) => gen_math_ops._clip_by_value(t, clip_value_min, clip_value_max); + + /// + /// Clips tensor values to a specified min and max. + /// + /// + /// A Tensor. + /// + /// + /// A 0-D (scalar) Tensor, or a Tensor with the same shape + /// as t. The minimum value to clip by. + /// + /// + /// A 0-D (scalar) Tensor, or a Tensor with the same shape + /// as t. The maximum value to clip by. + /// + /// + /// If specified, the created operation in the graph will be this one, otherwise it will be named 'ClipByValue'. + /// + /// + /// A clipped Tensor with the same shape as input 't'. + /// The Operation can be fetched from the resulting Tensor, by fetching the Operation property from the result. + /// + /// + /// Given a tensor t, this operation returns a tensor of the same type and + /// shape as t with its values clipped to clip_value_min and clip_value_max. + /// Any values less than clip_value_min are set to clip_value_min. Any values + /// greater than clip_value_max are set to clip_value_max. + /// + public Tensor clip_by_value (Tensor t, Tensor clip_value_min, Tensor clip_value_max, string name = "ClipByValue") + => gen_ops.clip_by_value(t, clip_value_min, clip_value_max, name); public Tensor sub(Tensor a, Tensor b) => gen_math_ops.sub(a, b); diff --git a/src/TensorFlowNET.Core/APIs/tf.tensor.cs b/src/TensorFlowNET.Core/APIs/tf.tensor.cs index b553095e..2052de93 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tensor.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tensor.cs @@ -18,8 +18,8 @@ namespace Tensorflow { public partial class tensorflow { - public Tensor convert_to_tensor(object value, - string name = null) => ops.convert_to_tensor(value, name: name); + public Tensor convert_to_tensor(object value, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, TF_DataType preferred_dtype = TF_DataType.DtInvalid) + => ops.convert_to_tensor(value, dtype, name, preferred_dtype); public Tensor strided_slice(Tensor input, Tensor begin, Tensor end, Tensor strides = null, int begin_mask = 0, diff --git a/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.activations.cs b/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.activations.cs index 80e1c305..788adda4 100644 --- a/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.activations.cs +++ b/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.activations.cs @@ -14,20 +14,192 @@ limitations under the License. ******************************************************************************/ +using System; +using static Tensorflow.Binding; + namespace Tensorflow.Operations.Activation { + public class sigmoid : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + return tf.sigmoid(x); + } + } + + public class tanh : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + return tf.tanh(x); + } + } + + public class leakyrelu : IActivation + { + private readonly float _alpha; + + public leakyrelu(float alpha = 0.3f) { + _alpha = alpha; + } + + public Tensor Activate(Tensor x, string name = null) + { + return nn_ops.leaky_relu(x, _alpha); + } + } + + public class elu : IActivation + { + private readonly float _alpha; + + public elu(float alpha = 0.1f) + { + _alpha = alpha; + } + + public Tensor Activate(Tensor x, string name = null) + { + var res = gen_ops.elu(x); + if (Math.Abs(_alpha - 0.1f) < 0.00001f) + { + return res; + } + + return array_ops.@where(x > 0, res, _alpha * res); + } + } + + public class softmax : IActivation + { + private readonly int _axis; + + /// Initializes a new instance of the class. + public softmax(int axis = -1) + { + _axis = axis; + } + + public Tensor Activate(Tensor x, string name = null) + { + return nn_ops.softmax(x, _axis); + } + } + + public class softplus : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + return gen_ops.softplus(x); + } + } + + public class softsign : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + return gen_ops.softsign(x); + } + } + + public class linear : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + return x; + } + } + + + public class exponential : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + return tf.exp(x, name: name); + } + } + + public class relu : IActivation { - public Tensor Activate(Tensor features, string name = null) + private readonly float _threshold; + private readonly float _alpha; + private readonly float? _maxValue; + + public relu(float threshold = 0f, float alpha = 0.2f, float? max_value = null) + { + _threshold = threshold; + _alpha = alpha; + _maxValue = max_value; + } + + public Tensor Activate(Tensor x, string name = null) { - OpDefLibrary _op_def_lib = new OpDefLibrary(); + //based on keras/backend.py + if (Math.Abs(_alpha) > 0.000001f) + { + if (!_maxValue.HasValue && Math.Abs(_threshold) < 0.0001) + { + return nn_ops.leaky_relu(x, _alpha); + } + } + + Tensor negative_part; + if (Math.Abs(_threshold) > 0.000001f) + { + negative_part = gen_ops.relu(-x + _threshold); + } else + { + negative_part = gen_ops.relu(-x + _threshold); + } + + if (Math.Abs(_threshold) > 0.000001f) + { + x = x * math_ops.cast(tf.greater(x, _threshold), TF_DataType.TF_FLOAT); + } else if (Math.Abs(_maxValue.Value - 6f) < 0.0001f) + { + x = gen_ops.relu6(x); + } else + { + x = gen_ops.relu(x); + } + + bool clip_max = _maxValue.HasValue; + if (clip_max) + { + Tensor maxval = constant_op.constant(_maxValue, x.dtype.as_base_dtype()); + var zero = constant_op.constant(0.0f, x.dtype.as_base_dtype()); + x = gen_ops.clip_by_value(x, zero, maxval); + } - var _op = _op_def_lib._apply_op_helper("Relu", name: name, args: new + if (Math.Abs(_alpha) > 0.00001) { - features - }); + var a = constant_op.constant(_alpha, x.dtype.as_base_dtype()); + x -= a * negative_part; + } - return _op.outputs[0]; + return x; + } + } + + public class selu : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + const float alpha = 1.6732632423543772848170429916717f; + const float scale = 1.0507009873554804934193349852946f; + return scale * new elu(alpha).Activate(x, name); + } + } + + public class hard_sigmoid : IActivation + { + public Tensor Activate(Tensor x, string name = null) + { + x = (0.2 * x) + 0.5; + var zero = tf.convert_to_tensor(0.0f, x.dtype.as_base_dtype()); + var one = tf.convert_to_tensor(1.0f, x.dtype.as_base_dtype()); + return tf.clip_by_value(x, zero, one); } } -} +} \ No newline at end of file From f70f0a9807f4daa8141576f3f99628089ad0db65 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Sep 2019 12:43:39 +0300 Subject: [PATCH 14/16] Added implicit conversion from object[] to TensorShape --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index 402ce3d8..fe8d391e 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -232,6 +232,8 @@ namespace Tensorflow public static implicit operator TensorShape(Shape shape) => new TensorShape((int[]) shape.Dimensions.Clone()); public static implicit operator Shape(TensorShape shape) => new Shape((int[]) shape.dims.Clone()); + public static implicit operator TensorShape(object[] dims) => new TensorShape(dims); + public static implicit operator int[](TensorShape shape) => (int[])shape.dims.Clone(); //we clone to avoid any changes public static implicit operator TensorShape(int[] dims) => new TensorShape(dims); From 420e195aca6a1821beb20fdfd9033b0bdd0c0430 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Sep 2019 13:59:14 +0300 Subject: [PATCH 15/16] TensorShape: Added implicit conversions for object type. --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 16 +++++ .../TensorFlowNET.UnitTest/TensorShapeTest.cs | 60 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 test/TensorFlowNET.UnitTest/TensorShapeTest.cs diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index fe8d391e..1edd634f 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -254,6 +254,22 @@ namespace Tensorflow public static explicit operator (int, int, int, int, int, int)(TensorShape shape) => shape.dims.Length == 6 ? (shape.dims[0], shape.dims[1], shape.dims[2], shape.dims[3], shape.dims[4], shape.dims[5]) : (0, 0, 0, 0, 0, 0); public static implicit operator TensorShape((int, int, int, int, int, int) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6); + + public static explicit operator (int, int, int, int, int, int, int)(TensorShape shape) => shape.dims.Length == 7 ? (shape.dims[0], shape.dims[1], shape.dims[2], shape.dims[3], shape.dims[4], shape.dims[5], shape.dims[6]) : (0, 0, 0, 0, 0, 0, 0); + public static implicit operator TensorShape((int, int, int, int, int, int, int) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6, dims.Item7); + + public static explicit operator (int, int, int, int, int, int, int, int)(TensorShape shape) => shape.dims.Length == 8 ? (shape.dims[0], shape.dims[1], shape.dims[2], shape.dims[3], shape.dims[4], shape.dims[5], shape.dims[6], shape.dims[7]) : (0, 0, 0, 0, 0, 0, 0, 0); + public static implicit operator TensorShape((int, int, int, int, int, int, int, int) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6, dims.Item7, dims.Item8); + + public static implicit operator TensorShape(int?[] dims) => new TensorShape(dims); + public static implicit operator TensorShape(int? dim) => new TensorShape(dim); + public static implicit operator TensorShape((object, object) dims) => new TensorShape(dims.Item1, dims.Item2); + public static implicit operator TensorShape((object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3); + public static implicit operator TensorShape((object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4); + public static implicit operator TensorShape((object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5); + public static implicit operator TensorShape((object, object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6); + public static implicit operator TensorShape((object, object, object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6, dims.Item7); + public static implicit operator TensorShape((object, object, object, object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6, dims.Item7, dims.Item8); } } diff --git a/test/TensorFlowNET.UnitTest/TensorShapeTest.cs b/test/TensorFlowNET.UnitTest/TensorShapeTest.cs new file mode 100644 index 00000000..90048aa6 --- /dev/null +++ b/test/TensorFlowNET.UnitTest/TensorShapeTest.cs @@ -0,0 +1,60 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NumSharp; +using Tensorflow; +using static Tensorflow.Binding; + +namespace TensorFlowNET.UnitTest +{ + [TestClass] + public class TensorShapeTest + { + [TestMethod] + public void Case1() + { + int? a = 2; + int? b = 3; + var dims = new object[] {(int?) None, a, b}; + new TensorShape(dims).GetPrivate("shape").Should().BeShaped(-1, 2, 3); + } + + [TestMethod] + public void Case2() + { + int? a = 2; + int? b = 3; + var dims = new object[] {(int?) None, a, b}; + new TensorShape(new object[] {dims}).GetPrivate("shape").Should().BeShaped(-1, 2, 3); + } + + [TestMethod] + public void Case3() + { + int? a = 2; + int? b = null; + var dims = new object[] {(int?) None, a, b}; + new TensorShape(new object[] {dims}).GetPrivate("shape").Should().BeShaped(-1, 2, -1); + } + + [TestMethod] + public void Case4() + { + TensorShape shape = (None, None); + shape.GetPrivate("shape").Should().BeShaped(-1, -1); + } + + [TestMethod] + public void Case5() + { + TensorShape shape = (1, None, 3); + shape.GetPrivate("shape").Should().BeShaped(1, -1, 3); + } + + [TestMethod] + public void Case6() + { + TensorShape shape = (None, 1, 2, 3, None); + shape.GetPrivate("shape").Should().BeShaped(-1, 1, 2, 3, -1); + } + } +} \ No newline at end of file From 269611bf8652bfb031cbc84ca4bd8b995dd903e4 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Mon, 9 Sep 2019 09:53:39 -0500 Subject: [PATCH 16/16] change None to Unknown for TensorShape. Remove object[] construction. --- src/TensorFlowNET.Core/Binding.cs | 6 ++ src/TensorFlowNET.Core/Tensors/TensorShape.cs | 80 +++++-------------- src/TensorFlowNET.Core/tensorflow.cs | 5 -- .../TensorFlowNET.UnitTest/TensorShapeTest.cs | 28 +++---- .../layers_test/flatten.cs | 4 +- 5 files changed, 41 insertions(+), 82 deletions(-) diff --git a/src/TensorFlowNET.Core/Binding.cs b/src/TensorFlowNET.Core/Binding.cs index e8355e32..e3136f83 100644 --- a/src/TensorFlowNET.Core/Binding.cs +++ b/src/TensorFlowNET.Core/Binding.cs @@ -10,7 +10,13 @@ namespace Tensorflow /// /// Alias to null, similar to python's None. + /// For TensorShape, please use Unknown /// public static readonly object None = null; + + /// + /// Used for TensorShape None + /// + public static readonly int Unknown = -1; } } diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index 1edd634f..853255cb 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -71,61 +71,32 @@ namespace Tensorflow } } - public TensorShape(params object[] dims) + public TensorShape(params int[] dims) { - Array arr; + switch (dims.Length) + { + case 0: shape = new Shape(new int[0]); break; + case 1: shape = Shape.Vector((int)dims[0]); break; + case 2: shape = Shape.Matrix(dims[0], dims[1]); break; + default: shape = new Shape(dims); break; + } + } - if (dims.Length == 1) + public TensorShape(int[][] dims) + { + if(dims.Length == 1) { - switch (dims[0]) + switch (dims[0].Length) { - case int[] intarr: - arr = intarr; - break; - case long[] longarr: - arr = longarr; - break; - case object[] objarr: - arr = objarr; - break; - case int _: - case long _: - arr = dims; - break; - case null: //==Binding.None - arr = dims; - break; - default: - Binding.print(dims); - throw new ArgumentException(nameof(dims)); + case 0: shape = new Shape(new int[0]); break; + case 1: shape = Shape.Vector((int)dims[0][0]); break; + case 2: shape = Shape.Matrix(dims[0][0], dims[1][2]); break; + default: shape = new Shape(dims[0]); break; } - } else - arr = dims; - - var intdims = new int[arr.Length]; - for (int i = 0; i < arr.Length; i++) - { - var val = arr.GetValue(i); - if (val == Binding.None) - intdims[i] = -1; - else - intdims[i] = Converts.ToInt32(val); } - - switch (intdims.Length) + else { - case 0: - shape = new Shape(new int[0]); - break; - case 1: - shape = Shape.Vector((int) intdims[0]); - break; - case 2: - shape = Shape.Matrix(intdims[0], intdims[1]); - break; - default: - shape = new Shape(intdims); - break; + throw new NotImplementedException("TensorShape int[][] dims"); } } @@ -232,8 +203,6 @@ namespace Tensorflow public static implicit operator TensorShape(Shape shape) => new TensorShape((int[]) shape.Dimensions.Clone()); public static implicit operator Shape(TensorShape shape) => new Shape((int[]) shape.dims.Clone()); - public static implicit operator TensorShape(object[] dims) => new TensorShape(dims); - public static implicit operator int[](TensorShape shape) => (int[])shape.dims.Clone(); //we clone to avoid any changes public static implicit operator TensorShape(int[] dims) => new TensorShape(dims); @@ -260,16 +229,5 @@ namespace Tensorflow public static explicit operator (int, int, int, int, int, int, int, int)(TensorShape shape) => shape.dims.Length == 8 ? (shape.dims[0], shape.dims[1], shape.dims[2], shape.dims[3], shape.dims[4], shape.dims[5], shape.dims[6], shape.dims[7]) : (0, 0, 0, 0, 0, 0, 0, 0); public static implicit operator TensorShape((int, int, int, int, int, int, int, int) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6, dims.Item7, dims.Item8); - - public static implicit operator TensorShape(int?[] dims) => new TensorShape(dims); - public static implicit operator TensorShape(int? dim) => new TensorShape(dim); - public static implicit operator TensorShape((object, object) dims) => new TensorShape(dims.Item1, dims.Item2); - public static implicit operator TensorShape((object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3); - public static implicit operator TensorShape((object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4); - public static implicit operator TensorShape((object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5); - public static implicit operator TensorShape((object, object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6); - public static implicit operator TensorShape((object, object, object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6, dims.Item7); - public static implicit operator TensorShape((object, object, object, object, object, object, object, object) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3, dims.Item4, dims.Item5, dims.Item6, dims.Item7, dims.Item8); - } } diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index 494f2e89..bdb2f537 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -63,11 +63,6 @@ namespace Tensorflow return gen_array_ops.placeholder(dtype, shape, name); } - public unsafe Tensor placeholder(TF_DataType dtype, object[] shape, string name = null) - { - return placeholder(dtype, new TensorShape(shape), name); - } - public void enable_eager_execution() { // contex = new Context(); diff --git a/test/TensorFlowNET.UnitTest/TensorShapeTest.cs b/test/TensorFlowNET.UnitTest/TensorShapeTest.cs index 90048aa6..efa7def3 100644 --- a/test/TensorFlowNET.UnitTest/TensorShapeTest.cs +++ b/test/TensorFlowNET.UnitTest/TensorShapeTest.cs @@ -12,48 +12,48 @@ namespace TensorFlowNET.UnitTest [TestMethod] public void Case1() { - int? a = 2; - int? b = 3; - var dims = new object[] {(int?) None, a, b}; + int a = 2; + int b = 3; + var dims = new [] { Unknown, a, b}; new TensorShape(dims).GetPrivate("shape").Should().BeShaped(-1, 2, 3); } [TestMethod] public void Case2() { - int? a = 2; - int? b = 3; - var dims = new object[] {(int?) None, a, b}; - new TensorShape(new object[] {dims}).GetPrivate("shape").Should().BeShaped(-1, 2, 3); + int a = 2; + int b = 3; + var dims = new[] { Unknown, a, b}; + new TensorShape(new [] {dims}).GetPrivate("shape").Should().BeShaped(-1, 2, 3); } [TestMethod] public void Case3() { - int? a = 2; - int? b = null; - var dims = new object[] {(int?) None, a, b}; - new TensorShape(new object[] {dims}).GetPrivate("shape").Should().BeShaped(-1, 2, -1); + int a = 2; + int b = Unknown; + var dims = new [] { Unknown, a, b}; + new TensorShape(new [] {dims}).GetPrivate("shape").Should().BeShaped(-1, 2, -1); } [TestMethod] public void Case4() { - TensorShape shape = (None, None); + TensorShape shape = (Unknown, Unknown); shape.GetPrivate("shape").Should().BeShaped(-1, -1); } [TestMethod] public void Case5() { - TensorShape shape = (1, None, 3); + TensorShape shape = (1, Unknown, 3); shape.GetPrivate("shape").Should().BeShaped(1, -1, 3); } [TestMethod] public void Case6() { - TensorShape shape = (None, 1, 2, 3, None); + TensorShape shape = (Unknown, 1, 2, 3, Unknown); shape.GetPrivate("shape").Should().BeShaped(-1, 1, 2, 3, -1); } } diff --git a/test/TensorFlowNET.UnitTest/layers_test/flatten.cs b/test/TensorFlowNET.UnitTest/layers_test/flatten.cs index 8f97d5c2..fa8ec792 100644 --- a/test/TensorFlowNET.UnitTest/layers_test/flatten.cs +++ b/test/TensorFlowNET.UnitTest/layers_test/flatten.cs @@ -42,7 +42,7 @@ namespace TensorFlowNET.UnitTest.layers_test { var sess = tf.Session().as_default(); - var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape(3, 4, None, 1, 2)); + var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape(3, 4, Unknown, 1, 2)); sess.run(tf.layers.flatten(input), (input, np.arange(3 * 4 * 3 * 1 * 2).reshape(3, 4, 3, 1, 2))).Should().BeShaped(3, 24); } @@ -51,7 +51,7 @@ namespace TensorFlowNET.UnitTest.layers_test { var sess = tf.Session().as_default(); - var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape(None, 4, 3, 1, 2)); + var input = tf.placeholder(TF_DataType.TF_INT32, new TensorShape(Unknown, 4, 3, 1, 2)); sess.run(tf.layers.flatten(input), (input, np.arange(3 * 4 * 3 * 1 * 2).reshape(3, 4, 3, 1, 2))).Should().BeShaped(3, 24); } }