From c2d25ecb033dd002df7a7107236a5fbe871ffebf Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Wed, 1 Dec 2021 20:27:20 -0600 Subject: [PATCH] fix dataset is disposed when zip dataset. --- src/TensorFlowNET.Core/Data/ZipDataset.cs | 3 ++ src/TensorFlowNET.Core/NumPy/NDArrayRender.cs | 2 +- .../Tensorflow.Binding.csproj | 4 +- src/TensorFlowNET.Core/Tensors/tensor_util.cs | 2 +- ...eprocessing.paths_and_labels_to_dataset.cs | 37 +++---------------- .../Tensorflow.Keras.csproj | 4 +- 6 files changed, 15 insertions(+), 37 deletions(-) diff --git a/src/TensorFlowNET.Core/Data/ZipDataset.cs b/src/TensorFlowNET.Core/Data/ZipDataset.cs index 0d1c4f0e..888948f8 100644 --- a/src/TensorFlowNET.Core/Data/ZipDataset.cs +++ b/src/TensorFlowNET.Core/Data/ZipDataset.cs @@ -6,8 +6,11 @@ namespace Tensorflow { public class ZipDataset : DatasetV2 { + // keep all dataset references + IDatasetV2[] _inputs; public ZipDataset(params IDatasetV2[] ds) { + _inputs = ds; var input_datasets = ds.Select(x => x.variant_tensor).ToArray(); var _structure = new List(); foreach (var dataset in ds) diff --git a/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs b/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs index 3ab6cb34..741e2581 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs @@ -88,7 +88,7 @@ namespace Tensorflow.NumPy { if (array.rank == 0) return "'" + string.Join(string.Empty, array.StringBytes()[0] - .Take(25) + .Take(256) .Select(x => x < 32 || x > 127 ? "\\x" + x.ToString("x") : Convert.ToChar(x).ToString())) + "'"; else return $"'{string.Join("', '", array.StringData().Take(25))}'"; diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 1e0a8c10..e68bf2f1 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.60.4.0 + 0.60.5.0 tf.net 0.60.x and above are based on tensorflow native 2.6.0 * Eager Mode is added finally. @@ -35,7 +35,7 @@ Keras API is a separate package released as TensorFlow.Keras. tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library. tf.net 0.5x.x aligns with TensorFlow v2.5.x native library. tf.net 0.6x.x aligns with TensorFlow v2.6.x native library. - 0.60.4.0 + 0.60.5.0 LICENSE true true diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index ae2d4dd8..c9af0b2e 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -405,7 +405,7 @@ would not be rank 1.", tensor.op.get_attr("axis"))); var ret = tensor.shape.unknown_shape((int)shape.dims[0]); var value = constant_value(tensor); - if (!(value is null)) + if (value is not null) { var d_ = new int[value.size]; foreach (var (index, d) in enumerate(value.ToArray())) diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs index 80aaa98e..b4d58387 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs @@ -1,5 +1,4 @@ -using System; -using System.IO; +using System.IO; using static Tensorflow.Binding; using Tensorflow.NumPy; @@ -15,22 +14,8 @@ namespace Tensorflow.Keras int num_classes, string interpolation) { - // option 1: will load all images into memory, not efficient - var images = np.zeros((image_paths.Length, image_size[0], image_size[1], num_channels), np.float32); - for (int i = 0; i < len(images); i++) - { - var img = tf.io.read_file(image_paths[i]); - img = tf.image.decode_image( - img, channels: num_channels, expand_animations: false); - var resized_image = tf.image.resize_images_v2(img, image_size, method: interpolation); - images[i] = resized_image.numpy(); - tf_output_redirect.WriteLine(image_paths[i]); - }; - var img_ds = tf.data.Dataset.from_tensor_slices(images); - - // option 2: dynamic load, but has error, need to fix - // var path_ds = tf.data.Dataset.from_tensor_slices(image_paths); - // var img_ds = path_ds.map(x => path_to_image(x, image_size, num_channels, interpolation)); + var path_ds = tf.data.Dataset.from_tensor_slices(image_paths); + var img_ds = path_ds.map(x => path_to_image(x, image_size, num_channels, interpolation)); if (label_mode == "int") { @@ -43,7 +28,7 @@ namespace Tensorflow.Keras Tensor path_to_image(Tensor path, Shape image_size, int num_channels, string interpolation) { - tf.print(path); + // tf.print(path); var img = tf.io.read_file(path); img = tf.image.decode_image( img, channels: num_channels, expand_animations: false); @@ -58,18 +43,8 @@ namespace Tensorflow.Keras int num_classes, int max_length = -1) { - var text = new string[image_paths.Length]; - for (int i = 0; i < text.Length; i++) - { - text[i] = File.ReadAllText(image_paths[i]); - tf_output_redirect.WriteLine(image_paths[i]); - } - - var images = np.array(text); - var string_ds = tf.data.Dataset.from_tensor_slices(images); - - // var path_ds = tf.data.Dataset.from_tensor_slices(image_paths); - // var string_ds = path_ds.map(x => path_to_string_content(x, max_length)); + var path_ds = tf.data.Dataset.from_tensor_slices(image_paths); + var string_ds = path_ds.map(x => path_to_string_content(x, max_length)); if (label_mode == "int") { diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 03d9ad99..8f718041 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -37,8 +37,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.6.4.0 - 0.6.4.0 + 0.6.5.0 + 0.6.5.0 LICENSE