diff --git a/src/TensorFlowNET.Core/APIs/tf.array.cs b/src/TensorFlowNET.Core/APIs/tf.array.cs
index 8757d653..c63fe7fe 100644
--- a/src/TensorFlowNET.Core/APIs/tf.array.cs
+++ b/src/TensorFlowNET.Core/APIs/tf.array.cs
@@ -19,5 +19,16 @@ namespace Tensorflow
///
public static Tensor expand_dims(Tensor input, int axis = -1, string name = null, int dim = -1)
=> array_ops.expand_dims(input, axis, name, dim);
+
+ ///
+ /// Transposes `a`. Permutes the dimensions according to `perm`.
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ public static Tensor transpose(Tensor a, int[] perm = null, string name = "transpose", bool conjugate = false)
+ => array_ops.transpose(a, perm, name, conjugate);
}
}
diff --git a/src/TensorFlowNET.Core/APIs/tf.layers.cs b/src/TensorFlowNET.Core/APIs/tf.layers.cs
index e9d9546d..8532a24d 100644
--- a/src/TensorFlowNET.Core/APIs/tf.layers.cs
+++ b/src/TensorFlowNET.Core/APIs/tf.layers.cs
@@ -46,6 +46,45 @@ namespace Tensorflow
return layer.apply(inputs);
}
+
+ ///
+ /// Functional interface for the batch normalization layer.
+ /// http://arxiv.org/abs/1502.03167
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ public static Tensor batch_normalization(Tensor inputs,
+ int axis = -1,
+ float momentum = 0.99f,
+ float epsilon = 0.001f,
+ bool center = true,
+ bool scale = true,
+ IInitializer beta_initializer = null,
+ IInitializer gamma_initializer = null,
+ IInitializer moving_mean_initializer = null,
+ IInitializer moving_variance_initializer = null,
+ Tensor training = null,
+ bool trainable = true,
+ string name = null,
+ bool renorm = false,
+ float renorm_momentum = 0.99f)
+ {
+ throw new NotImplementedException("batch_normalization");
+ }
}
}
}
diff --git a/src/TensorFlowNET.Core/Keras/Engine/Layer.cs b/src/TensorFlowNET.Core/Keras/Engine/Layer.cs
index b6603c28..7fb1a12d 100644
--- a/src/TensorFlowNET.Core/Keras/Engine/Layer.cs
+++ b/src/TensorFlowNET.Core/Keras/Engine/Layer.cs
@@ -30,6 +30,7 @@ namespace Tensorflow.Keras.Engine
VariableScope scope = null)
{
var input_list = new Tensor[] { inputs };
+ Tensor outputs = null;
// We will attempt to build a TF graph if & only if all inputs are symbolic.
// This is always the case in graph mode. It can also be the case in eager
@@ -45,9 +46,42 @@ namespace Tensorflow.Keras.Engine
_maybe_build(inputs);
built = true;
}
+
+ if (build_graph)
+ {
+ // Symbolic execution on symbolic tensors. We will attempt to build
+ // the corresponding TF subgraph inside `backend.get_graph()`
+ var graph = backend.get_graph();
+ outputs = call(inputs);
+ _handle_activity_regularization(inputs, outputs);
+ _set_mask_metadata(inputs, outputs, null);
+ }
});
- throw new NotImplementedException("");
+ return outputs;
+ }
+
+ private void _handle_activity_regularization(Tensor inputs, Tensor outputs)
+ {
+ //if(_activity_regularizer != null)
+ {
+
+ }
+ }
+
+ private void _set_mask_metadata(Tensor inputs, Tensor outputs, Tensor previous_mask)
+ {
+
+ }
+
+ private Tensor compute_mask(Tensor inputs, Tensor mask = null)
+ {
+ return null;
+ }
+
+ protected virtual Tensor call(Tensor inputs)
+ {
+ throw new NotImplementedException("Layer.call");
}
protected virtual string _name_scope()
diff --git a/src/TensorFlowNET.Core/Keras/Layers/Conv.cs b/src/TensorFlowNET.Core/Keras/Layers/Conv.cs
index e3c824e6..6661e43f 100644
--- a/src/TensorFlowNET.Core/Keras/Layers/Conv.cs
+++ b/src/TensorFlowNET.Core/Keras/Layers/Conv.cs
@@ -90,5 +90,26 @@ namespace Tensorflow.Keras.Layers
built = true;
}
+
+ protected override Tensor call(Tensor inputs)
+ {
+ var outputs = _convolution_op.__call__(inputs, kernel);
+ if (use_bias)
+ {
+ if (data_format == "channels_first")
+ {
+ throw new NotImplementedException("call channels_first");
+ }
+ else
+ {
+ outputs = nn_ops.bias_add(outputs, bias, data_format: "NHWC");
+ }
+ }
+
+ if (activation != null)
+ return activation.Activate(outputs);
+
+ return outputs;
+ }
}
}
diff --git a/src/TensorFlowNET.Core/Keras/backend.cs b/src/TensorFlowNET.Core/Keras/backend.cs
index 51d74c04..0196bfea 100644
--- a/src/TensorFlowNET.Core/Keras/backend.cs
+++ b/src/TensorFlowNET.Core/Keras/backend.cs
@@ -10,5 +10,10 @@ namespace Tensorflow.Keras
{
}
+
+ public static Graph get_graph()
+ {
+ return ops.get_default_graph();
+ }
}
}
diff --git a/src/TensorFlowNET.Core/Layers/Layer.cs b/src/TensorFlowNET.Core/Layers/Layer.cs
index 510b505b..3132048e 100644
--- a/src/TensorFlowNET.Core/Layers/Layer.cs
+++ b/src/TensorFlowNET.Core/Layers/Layer.cs
@@ -65,7 +65,10 @@ namespace Tensorflow.Layers
// Actually call layer
var outputs = base.__call__(inputs);
- throw new NotImplementedException("");
+ // Update global default collections.
+ //_add_elements_to_collection(updates, ops.GraphKeys.UPDATE_OPS);
+
+ return outputs;
}
protected virtual RefVariable add_weight(string name,
diff --git a/src/TensorFlowNET.Core/Operations/Activation/IActivation.cs b/src/TensorFlowNET.Core/Operations/Activation/IActivation.cs
index ed7c237c..8e1bef24 100644
--- a/src/TensorFlowNET.Core/Operations/Activation/IActivation.cs
+++ b/src/TensorFlowNET.Core/Operations/Activation/IActivation.cs
@@ -6,6 +6,6 @@ namespace Tensorflow.Operations.Activation
{
public interface IActivation
{
-
+ Tensor Activate(Tensor features, string name = null);
}
}
diff --git a/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.py.cs b/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.py.cs
index 4ee8bd00..af4df920 100644
--- a/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.py.cs
+++ b/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.py.cs
@@ -6,6 +6,16 @@ namespace Tensorflow.Operations.Activation
{
public class relu : IActivation
{
+ public Tensor Activate(Tensor features, string name = null)
+ {
+ OpDefLibrary _op_def_lib = new OpDefLibrary();
+ var _op = _op_def_lib._apply_op_helper("Relu", name: name, args: new
+ {
+ features
+ });
+
+ return _op.outputs[0];
+ }
}
}
diff --git a/src/TensorFlowNET.Core/Operations/NnOps/Convolution.cs b/src/TensorFlowNET.Core/Operations/NnOps/Convolution.cs
index a0d7887b..727b40b8 100644
--- a/src/TensorFlowNET.Core/Operations/NnOps/Convolution.cs
+++ b/src/TensorFlowNET.Core/Operations/NnOps/Convolution.cs
@@ -62,5 +62,10 @@ namespace Tensorflow.Operations
strides: strides,
name: name);
}
+
+ public Tensor __call__(Tensor inp, RefVariable filter)
+ {
+ return conv_op.__call__(inp, filter);
+ }
}
}
diff --git a/src/TensorFlowNET.Core/Operations/NnOps/_NonAtrousConvolution.cs b/src/TensorFlowNET.Core/Operations/NnOps/_NonAtrousConvolution.cs
index 38e77a4f..e6a3958a 100644
--- a/src/TensorFlowNET.Core/Operations/NnOps/_NonAtrousConvolution.cs
+++ b/src/TensorFlowNET.Core/Operations/NnOps/_NonAtrousConvolution.cs
@@ -52,5 +52,18 @@ namespace Tensorflow.Operations
throw new NotImplementedException("_NonAtrousConvolution conv_dims 3");
}
}
+
+ public Tensor __call__(Tensor inp, RefVariable filter)
+ {
+ return conv_op(new
+ {
+ input = inp,
+ filter,
+ strides,
+ padding,
+ data_format,
+ name
+ });
+ }
}
}
diff --git a/src/TensorFlowNET.Core/Operations/NnOps/_WithSpaceToBatch.cs b/src/TensorFlowNET.Core/Operations/NnOps/_WithSpaceToBatch.cs
index b144e95c..d86b5cb6 100644
--- a/src/TensorFlowNET.Core/Operations/NnOps/_WithSpaceToBatch.cs
+++ b/src/TensorFlowNET.Core/Operations/NnOps/_WithSpaceToBatch.cs
@@ -51,5 +51,10 @@ namespace Tensorflow.Operations
}
}
}
+
+ public Tensor __call__(Tensor inp, RefVariable filter)
+ {
+ return call.__call__(inp, filter);
+ }
}
}
diff --git a/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs b/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs
index 54d9242c..78346a8f 100644
--- a/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs
+++ b/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs
@@ -6,9 +6,51 @@ namespace Tensorflow.Operations
{
public class gen_nn_ops
{
+ public static OpDefLibrary _op_def_lib = new OpDefLibrary();
+
public static Tensor conv2d(object parameters)
{
- throw new NotImplementedException("gen_nn_op.conv2d");
+ var args = Python.ConvertToDict(parameters);
+
+ var input = args["input"];
+ var filter = args["filter"];
+ var strides = args["strides"];
+ var padding = args["padding"];
+ var name = args["name"];
+ var data_format = args.ContainsKey("data_format") ? args["data_format"] : "NHWC";
+ var use_cudnn_on_gpu = args.ContainsKey("use_cudnn_on_gpu") ? args["use_cudnn_on_gpu"] : true;
+ var dilations = args.ContainsKey("dilations") ? args["dilations"] : new int[] { 1, 1, 1, 1 };
+
+ var _op = _op_def_lib._apply_op_helper("Conv2D", name: name?.ToString(), args: new
+ {
+ input,
+ filter,
+ strides,
+ padding,
+ use_cudnn_on_gpu,
+ data_format,
+ dilations
+ });
+
+ return _op.outputs[0];
+ }
+
+ public static Tensor bias_add(Tensor value,
+ Tensor bias,
+ string data_format = null,
+ string name = null)
+ {
+ if (data_format == null)
+ data_format = "NHWC";
+
+ var _op = _op_def_lib._apply_op_helper("BiasAdd", name: name, args: new
+ {
+ value,
+ bias,
+ data_format
+ });
+
+ return _op.outputs[0];
}
}
}
diff --git a/src/TensorFlowNET.Core/Operations/array_ops.py.cs b/src/TensorFlowNET.Core/Operations/array_ops.py.cs
index c32bf397..0eaf6251 100644
--- a/src/TensorFlowNET.Core/Operations/array_ops.py.cs
+++ b/src/TensorFlowNET.Core/Operations/array_ops.py.cs
@@ -272,5 +272,14 @@ namespace Tensorflow
{
return gen_array_ops.gather_v2(@params, indices, axis, name: name);
}
+
+ public static Tensor transpose(Tensor a, int[] perm = null, string name = "transpose", bool conjugate = false)
+ {
+ return with(ops.name_scope(name, "transpose", new { a }), scope =>
+ {
+ name = scope;
+ return gen_array_ops.transpose(a, perm, name);
+ });
+ }
}
}
diff --git a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs
index 0ae4c3a2..59b77a1b 100644
--- a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs
+++ b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs
@@ -157,6 +157,12 @@ namespace Tensorflow
return _op.outputs[0];
}
+ public static Tensor transpose(Tensor x, int[] perm, string name = null)
+ {
+ var _op = _op_def_lib._apply_op_helper("Transpose", name, new { x, perm });
+ return _op.outputs[0];
+ }
+
public static Tensor zeros_like(Tensor x, string name = null)
{
var _op = _op_def_lib._apply_op_helper("ZerosLike", name, new { x });
diff --git a/src/TensorFlowNET.Core/Operations/nn_ops.cs b/src/TensorFlowNET.Core/Operations/nn_ops.cs
index 1c36819a..c7aec377 100644
--- a/src/TensorFlowNET.Core/Operations/nn_ops.cs
+++ b/src/TensorFlowNET.Core/Operations/nn_ops.cs
@@ -20,5 +20,26 @@ namespace Tensorflow
dilation_rate,
name: name,
data_format: data_format);
+
+ ///
+ /// Adds `bias` to `value`.
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ public static Tensor bias_add(Tensor value,
+ RefVariable bias,
+ string data_format = null,
+ string name = null)
+ {
+ return Python.with(ops.name_scope(name, "BiasAdd", new { value, bias }), scope =>
+ {
+ value = ops.convert_to_tensor(value, name: "input");
+ var bias_tensor = ops.convert_to_tensor(bias, dtype: value.dtype, name: "bias");
+ return gen_nn_ops.bias_add(value, bias_tensor, data_format: data_format, name: name);
+ });
+ }
}
}
diff --git a/src/TensorFlowNET.Core/ops.GraphKeys.cs b/src/TensorFlowNET.Core/ops.GraphKeys.cs
index 9c6c76e3..540f8d55 100644
--- a/src/TensorFlowNET.Core/ops.GraphKeys.cs
+++ b/src/TensorFlowNET.Core/ops.GraphKeys.cs
@@ -40,6 +40,10 @@ namespace Tensorflow
/// Key to collect BaseSaverBuilder.SaveableObject instances for checkpointing.
///
public static string SAVEABLE_OBJECTS = "saveable_objects";
+ ///
+ /// Key to collect update_ops
+ ///
+ public static string UPDATE_OPS = "update_ops";
}
}
}
diff --git a/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs b/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs
index 1900b8da..05929d3d 100644
--- a/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs
+++ b/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using System.Linq;
using System.Text;
using Tensorflow;
@@ -43,14 +44,61 @@ namespace TensorFlowNET.Examples.TextClassification
x_expanded = tf.expand_dims(x_emb, -1);
});
+ Tensor conv0 = null;
+ Tensor conv1 = null;
+
// First Convolution Layer
with(tf.variable_scope("conv-0"), delegate
{
- var conv0 = tf.layers.conv2d(x_expanded,
+ conv0 = tf.layers.conv2d(x_expanded,
filters: num_filters[0],
kernel_size: new int[] { filter_sizes[0], embedding_size },
kernel_initializer: cnn_initializer,
activation: tf.nn.relu);
+
+ conv0 = tf.transpose(conv0, new int[] { 0, 1, 3, 2 });
+ });
+
+ with(tf.name_scope("conv-block-1"), delegate {
+ conv1 = conv_block(conv0, 1);
+ });
+
+ }
+
+ private Tensor conv_block(Tensor input, int i, bool max_pool = true)
+ {
+ return with(tf.variable_scope($"conv-block-{i}"), delegate
+ {
+ Tensor conv = null;
+ // Two "conv-batch_norm-relu" layers.
+ foreach (var j in Enumerable.Range(0, 2))
+ {
+ with(tf.variable_scope($"conv-{j}"), delegate
+ {
+ // convolution
+ conv = tf.layers.conv2d(
+ input,
+ filters: num_filters[i],
+ kernel_size: new int[] { filter_sizes[i], num_filters[i - 1] },
+ kernel_initializer: cnn_initializer,
+ activation: null);
+ // batch normalization
+ conv = tf.layers.batch_normalization(conv, training: is_training);
+ // relu
+ conv = tf.nn.relu.Activate(conv);
+ conv = tf.transpose(conv, new int[] { 0, 1, 3, 2 });
+ });
+ }
+
+ if (max_pool)
+ {
+ // Max pooling
+ throw new NotImplementedException("conv_block");
+ }
+ else
+ {
+ return conv;
+ }
});
}
}