From 773a3bad6b335913bac890e4114cefa14e8c9704 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Sat, 8 May 2021 09:43:59 -0500 Subject: [PATCH] Add Preprocessing.timeseries_dataset_from_array --- src/TensorFlowNET.Core/Binding.Util.cs | 2 + .../Tensorflow.Binding.csproj | 9 ++-- .../Tensors/Tensor.Index.cs | 29 +++++++++++ src/TensorFlowNET.Core/Tensors/constant_op.cs | 2 - src/TensorFlowNET.Core/Tensors/tensor_util.cs | 33 +++++++++++++ ...processing.image_dataset_from_directory.cs | 49 ++++++++++++++++++- .../Tensorflow.Keras.csproj | 9 ++-- 7 files changed, 122 insertions(+), 11 deletions(-) diff --git a/src/TensorFlowNET.Core/Binding.Util.cs b/src/TensorFlowNET.Core/Binding.Util.cs index ed895ffc..ff02decd 100644 --- a/src/TensorFlowNET.Core/Binding.Util.cs +++ b/src/TensorFlowNET.Core/Binding.Util.cs @@ -155,6 +155,8 @@ namespace Tensorflow { switch (a) { + case Tensor tensor: + return tensor.shape[0]; case Tensors arr: return arr.Length; case Array arr: diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 92360a6d..1cd0ff46 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,7 +5,7 @@ TensorFlow.NET Tensorflow 2.2.0 - 0.40.1 + 0.50.0 8.0 Haiping Chen, Meinrad Recheis, Eli Belash SciSharp STACK @@ -19,7 +19,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.40.1.0 + 0.50.0.0 tf.net 0.20.x and above are based on tensorflow native 2.x. * Eager Mode is added finally. @@ -31,8 +31,9 @@ https://tensorflownet.readthedocs.io TensorFlow .NET v0.3x is focused on making more Keras API works. Keras API is a separate package released as TensorFlow.Keras. -tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library. - 0.40.1.0 +tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library. +tf.net 0.5x.x aligns with TensorFlow v2.5.x native library. + 0.50.0.0 LICENSE true true diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs index aa5df367..a12f1fb5 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs @@ -120,6 +120,35 @@ namespace Tensorflow }); } + public Tensor this[params Tensor[] slice] + { + get + { + var args = tensor_util.ParseSlices(slice); + + return tf_with(ops.name_scope(null, "strided_slice", args), scope => + { + string name = scope; + + var tensor = gen_array_ops.strided_slice( + this, + args.PackedBegin, + args.PackedEnd, + args.PackedStrides, + begin_mask: args.BeginMask, + end_mask: args.EndMask, + shrink_axis_mask: args.ShrinkAxisMask, + new_axis_mask: args.NewAxisMask, + ellipsis_mask: args.EllipsisMask, + name: name); + + tensor.OriginalVarSlice = args; + + return tensor; + }); + } + } + public Tensor slice(int start) { var slice_spec = new int[] { start }; diff --git a/src/TensorFlowNET.Core/Tensors/constant_op.cs b/src/TensorFlowNET.Core/Tensors/constant_op.cs index d35ed34e..2aa63883 100644 --- a/src/TensorFlowNET.Core/Tensors/constant_op.cs +++ b/src/TensorFlowNET.Core/Tensors/constant_op.cs @@ -146,9 +146,7 @@ namespace Tensorflow } if (dtype == TF_DataType.TF_STRING && value is byte[] bytes) - { return new EagerTensor(bytes, ctx.DeviceName, TF_DataType.TF_STRING); - } switch (value) { diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index 791306ca..ccc5c31c 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -673,5 +673,38 @@ would not be rank 1.", tensor.op.get_attr("axis"))); NewAxisMask = new_axis_mask }; } + + public static ParsedSliceArgs ParseSlices(Tensor[] slices) + { + var begin = new List(); + var end = new List(); + var strides = new List(); + + var index = 0; + var (new_axis_mask, shrink_axis_mask) = (0, 0); + var (begin_mask, end_mask) = (0, 0); + var ellipsis_mask = 0; + + foreach (var s in slices) + { + begin.Add(s); + end.Add(s + 1); + shrink_axis_mask |= (1 << index); + strides.Add(tf.constant(1, dtype: s.dtype)); + index += 1; + } + + return new ParsedSliceArgs + { + PackedBegin = array_ops.stack(begin), + PackedEnd = array_ops.stack(end), + PackedStrides = array_ops.stack(strides), + BeginMask = begin_mask, + EndMask = end_mask, + EllipsisMask = ellipsis_mask, + ShrinkAxisMask = shrink_axis_mask, + NewAxisMask = new_axis_mask + }; + } } } diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs index 8d7513a6..1fbcb3bd 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs @@ -1,4 +1,6 @@ using static Tensorflow.KerasApi; +using static Tensorflow.Binding; +using NumSharp; namespace Tensorflow.Keras { @@ -41,7 +43,7 @@ namespace Tensorflow.Keras int num_channels = 0; if (color_mode == "rgb") num_channels = 3; - + var (image_paths, label_list, class_name_list) = keras.preprocessing.dataset_utils.index_directory(directory, labels, formats: WHITELIST_FORMATS, @@ -90,5 +92,50 @@ namespace Tensorflow.Keras dataset.class_names = class_name_list; return dataset; } + + /// + /// Creates a dataset of sliding windows over a timeseries provided as array. + /// + /// + /// + /// + /// + /// + /// + /// + public IDatasetV2 timeseries_dataset_from_array(Tensor data, int sequence_length, + int sequence_stride = 1, int sampling_rate = 1, int batch_size = 128, + bool shuffle = false, int seed = (int)1e6, int start_index = 0, int? end_index = null) + { + if (!end_index.HasValue) + end_index = len(data); + + var num_seqs = end_index.Value - start_index - (sequence_length * sampling_rate) + 1; + var index_dtype = num_seqs < 2147483647 ? tf.int32 : tf.int64; + var start_positions = np.arange(0, num_seqs, sequence_stride); + if (shuffle) + { + var rng = np.random.RandomState(seed); + rng.shuffle(start_positions); + } + + var sequence_length_tensor = math_ops.cast(sequence_length, dtype: index_dtype); + var sampling_rate_tensor = math_ops.cast(sampling_rate, dtype: index_dtype); + + var start_positions_tensor = tf.constant(start_positions); + var positions_ds = tf.data.Dataset.from_tensor(start_positions_tensor).repeat(); + var z = tf.data.Dataset.zip(tf.data.Dataset.range(len(start_positions)), positions_ds); + var indices = z.map(m => + { + var (i, positions) = (m[0], m[1]); + return tf.range(positions[i], positions[i] + sequence_length_tensor * sampling_rate_tensor, sampling_rate_tensor); + }, num_parallel_calls: -1); + return null; + } + + IDatasetV2 sequences_from_indices(Tensor array, Tensor indices_ds, Tensor start_index, Tensor end_index) + { + return null; + } } } diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 6d246126..3eec3f2c 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -6,7 +6,7 @@ 8.0 Tensorflow.Keras AnyCPU;x64 - 0.5.1 + 0.5.2 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2021 @@ -24,7 +24,8 @@ * Support model.load_weights. * Add Subtract layer * Support YOLOv3 model. -* Text preprocessing +* Text preprocessing +* Preprocessing.timeseries_dataset_from_array Keras for .NET Keras is an API designed for human beings, not machines. Keras follows best practices for reducing cognitive load: it offers consistent & simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear & actionable error messages. @@ -35,8 +36,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.5.1.0 - 0.5.1.0 + 0.5.2.0 + 0.5.2.0 LICENSE