Browse Source

fix dataset is disposed when zip dataset.

tags/TimeSeries
Oceania2018 3 years ago
parent
commit
c2d25ecb03
6 changed files with 15 additions and 37 deletions
  1. +3
    -0
      src/TensorFlowNET.Core/Data/ZipDataset.cs
  2. +1
    -1
      src/TensorFlowNET.Core/NumPy/NDArrayRender.cs
  3. +2
    -2
      src/TensorFlowNET.Core/Tensorflow.Binding.csproj
  4. +1
    -1
      src/TensorFlowNET.Core/Tensors/tensor_util.cs
  5. +6
    -31
      src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs
  6. +2
    -2
      src/TensorFlowNET.Keras/Tensorflow.Keras.csproj

+ 3
- 0
src/TensorFlowNET.Core/Data/ZipDataset.cs View File

@@ -6,8 +6,11 @@ namespace Tensorflow
{
public class ZipDataset : DatasetV2
{
// keep all dataset references
IDatasetV2[] _inputs;
public ZipDataset(params IDatasetV2[] ds)
{
_inputs = ds;
var input_datasets = ds.Select(x => x.variant_tensor).ToArray();
var _structure = new List<TensorSpec>();
foreach (var dataset in ds)


+ 1
- 1
src/TensorFlowNET.Core/NumPy/NDArrayRender.cs View File

@@ -88,7 +88,7 @@ namespace Tensorflow.NumPy
{
if (array.rank == 0)
return "'" + string.Join(string.Empty, array.StringBytes()[0]
.Take(25)
.Take(256)
.Select(x => x < 32 || x > 127 ? "\\x" + x.ToString("x") : Convert.ToChar(x).ToString())) + "'";
else
return $"'{string.Join("', '", array.StringData().Take(25))}'";


+ 2
- 2
src/TensorFlowNET.Core/Tensorflow.Binding.csproj View File

@@ -20,7 +20,7 @@
<Description>Google's TensorFlow full binding in .NET Standard.
Building, training and infering deep learning models.
https://tensorflownet.readthedocs.io</Description>
<AssemblyVersion>0.60.4.0</AssemblyVersion>
<AssemblyVersion>0.60.5.0</AssemblyVersion>
<PackageReleaseNotes>tf.net 0.60.x and above are based on tensorflow native 2.6.0

* Eager Mode is added finally.
@@ -35,7 +35,7 @@ Keras API is a separate package released as TensorFlow.Keras.
tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library.
tf.net 0.5x.x aligns with TensorFlow v2.5.x native library.
tf.net 0.6x.x aligns with TensorFlow v2.6.x native library.</PackageReleaseNotes>
<FileVersion>0.60.4.0</FileVersion>
<FileVersion>0.60.5.0</FileVersion>
<PackageLicenseFile>LICENSE</PackageLicenseFile>
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
<SignAssembly>true</SignAssembly>


+ 1
- 1
src/TensorFlowNET.Core/Tensors/tensor_util.cs View File

@@ -405,7 +405,7 @@ would not be rank 1.", tensor.op.get_attr("axis")));

var ret = tensor.shape.unknown_shape((int)shape.dims[0]);
var value = constant_value(tensor);
if (!(value is null))
if (value is not null)
{
var d_ = new int[value.size];
foreach (var (index, d) in enumerate(value.ToArray<int>()))


+ 6
- 31
src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs View File

@@ -1,5 +1,4 @@
using System;
using System.IO;
using System.IO;
using static Tensorflow.Binding;
using Tensorflow.NumPy;

@@ -15,22 +14,8 @@ namespace Tensorflow.Keras
int num_classes,
string interpolation)
{
// option 1: will load all images into memory, not efficient
var images = np.zeros((image_paths.Length, image_size[0], image_size[1], num_channels), np.float32);
for (int i = 0; i < len(images); i++)
{
var img = tf.io.read_file(image_paths[i]);
img = tf.image.decode_image(
img, channels: num_channels, expand_animations: false);
var resized_image = tf.image.resize_images_v2(img, image_size, method: interpolation);
images[i] = resized_image.numpy();
tf_output_redirect.WriteLine(image_paths[i]);
};
var img_ds = tf.data.Dataset.from_tensor_slices(images);

// option 2: dynamic load, but has error, need to fix
// var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
// var img_ds = path_ds.map(x => path_to_image(x, image_size, num_channels, interpolation));
var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
var img_ds = path_ds.map(x => path_to_image(x, image_size, num_channels, interpolation));
if (label_mode == "int")
{
@@ -43,7 +28,7 @@ namespace Tensorflow.Keras

Tensor path_to_image(Tensor path, Shape image_size, int num_channels, string interpolation)
{
tf.print(path);
// tf.print(path);
var img = tf.io.read_file(path);
img = tf.image.decode_image(
img, channels: num_channels, expand_animations: false);
@@ -58,18 +43,8 @@ namespace Tensorflow.Keras
int num_classes,
int max_length = -1)
{
var text = new string[image_paths.Length];
for (int i = 0; i < text.Length; i++)
{
text[i] = File.ReadAllText(image_paths[i]);
tf_output_redirect.WriteLine(image_paths[i]);
}

var images = np.array(text);
var string_ds = tf.data.Dataset.from_tensor_slices(images);

// var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
// var string_ds = path_ds.map(x => path_to_string_content(x, max_length));
var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
var string_ds = path_ds.map(x => path_to_string_content(x, max_length));

if (label_mode == "int")
{


+ 2
- 2
src/TensorFlowNET.Keras/Tensorflow.Keras.csproj View File

@@ -37,8 +37,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac
<RepositoryType>Git</RepositoryType>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>Open.snk</AssemblyOriginatorKeyFile>
<AssemblyVersion>0.6.4.0</AssemblyVersion>
<FileVersion>0.6.4.0</FileVersion>
<AssemblyVersion>0.6.5.0</AssemblyVersion>
<FileVersion>0.6.5.0</FileVersion>
<PackageLicenseFile>LICENSE</PackageLicenseFile>
</PropertyGroup>



Loading…
Cancel
Save