Browse Source

fix dataset is disposed when zip dataset.

tags/TimeSeries
Oceania2018 3 years ago
parent
commit
c2d25ecb03
6 changed files with 15 additions and 37 deletions
  1. +3
    -0
      src/TensorFlowNET.Core/Data/ZipDataset.cs
  2. +1
    -1
      src/TensorFlowNET.Core/NumPy/NDArrayRender.cs
  3. +2
    -2
      src/TensorFlowNET.Core/Tensorflow.Binding.csproj
  4. +1
    -1
      src/TensorFlowNET.Core/Tensors/tensor_util.cs
  5. +6
    -31
      src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs
  6. +2
    -2
      src/TensorFlowNET.Keras/Tensorflow.Keras.csproj

+ 3
- 0
src/TensorFlowNET.Core/Data/ZipDataset.cs View File

@@ -6,8 +6,11 @@ namespace Tensorflow
{ {
public class ZipDataset : DatasetV2 public class ZipDataset : DatasetV2
{ {
// keep all dataset references
IDatasetV2[] _inputs;
public ZipDataset(params IDatasetV2[] ds) public ZipDataset(params IDatasetV2[] ds)
{ {
_inputs = ds;
var input_datasets = ds.Select(x => x.variant_tensor).ToArray(); var input_datasets = ds.Select(x => x.variant_tensor).ToArray();
var _structure = new List<TensorSpec>(); var _structure = new List<TensorSpec>();
foreach (var dataset in ds) foreach (var dataset in ds)


+ 1
- 1
src/TensorFlowNET.Core/NumPy/NDArrayRender.cs View File

@@ -88,7 +88,7 @@ namespace Tensorflow.NumPy
{ {
if (array.rank == 0) if (array.rank == 0)
return "'" + string.Join(string.Empty, array.StringBytes()[0] return "'" + string.Join(string.Empty, array.StringBytes()[0]
.Take(25)
.Take(256)
.Select(x => x < 32 || x > 127 ? "\\x" + x.ToString("x") : Convert.ToChar(x).ToString())) + "'"; .Select(x => x < 32 || x > 127 ? "\\x" + x.ToString("x") : Convert.ToChar(x).ToString())) + "'";
else else
return $"'{string.Join("', '", array.StringData().Take(25))}'"; return $"'{string.Join("', '", array.StringData().Take(25))}'";


+ 2
- 2
src/TensorFlowNET.Core/Tensorflow.Binding.csproj View File

@@ -20,7 +20,7 @@
<Description>Google's TensorFlow full binding in .NET Standard. <Description>Google's TensorFlow full binding in .NET Standard.
Building, training and infering deep learning models. Building, training and infering deep learning models.
https://tensorflownet.readthedocs.io</Description> https://tensorflownet.readthedocs.io</Description>
<AssemblyVersion>0.60.4.0</AssemblyVersion>
<AssemblyVersion>0.60.5.0</AssemblyVersion>
<PackageReleaseNotes>tf.net 0.60.x and above are based on tensorflow native 2.6.0 <PackageReleaseNotes>tf.net 0.60.x and above are based on tensorflow native 2.6.0


* Eager Mode is added finally. * Eager Mode is added finally.
@@ -35,7 +35,7 @@ Keras API is a separate package released as TensorFlow.Keras.
tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library. tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library.
tf.net 0.5x.x aligns with TensorFlow v2.5.x native library. tf.net 0.5x.x aligns with TensorFlow v2.5.x native library.
tf.net 0.6x.x aligns with TensorFlow v2.6.x native library.</PackageReleaseNotes> tf.net 0.6x.x aligns with TensorFlow v2.6.x native library.</PackageReleaseNotes>
<FileVersion>0.60.4.0</FileVersion>
<FileVersion>0.60.5.0</FileVersion>
<PackageLicenseFile>LICENSE</PackageLicenseFile> <PackageLicenseFile>LICENSE</PackageLicenseFile>
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance> <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
<SignAssembly>true</SignAssembly> <SignAssembly>true</SignAssembly>


+ 1
- 1
src/TensorFlowNET.Core/Tensors/tensor_util.cs View File

@@ -405,7 +405,7 @@ would not be rank 1.", tensor.op.get_attr("axis")));


var ret = tensor.shape.unknown_shape((int)shape.dims[0]); var ret = tensor.shape.unknown_shape((int)shape.dims[0]);
var value = constant_value(tensor); var value = constant_value(tensor);
if (!(value is null))
if (value is not null)
{ {
var d_ = new int[value.size]; var d_ = new int[value.size];
foreach (var (index, d) in enumerate(value.ToArray<int>())) foreach (var (index, d) in enumerate(value.ToArray<int>()))


+ 6
- 31
src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs View File

@@ -1,5 +1,4 @@
using System;
using System.IO;
using System.IO;
using static Tensorflow.Binding; using static Tensorflow.Binding;
using Tensorflow.NumPy; using Tensorflow.NumPy;


@@ -15,22 +14,8 @@ namespace Tensorflow.Keras
int num_classes, int num_classes,
string interpolation) string interpolation)
{ {
// option 1: will load all images into memory, not efficient
var images = np.zeros((image_paths.Length, image_size[0], image_size[1], num_channels), np.float32);
for (int i = 0; i < len(images); i++)
{
var img = tf.io.read_file(image_paths[i]);
img = tf.image.decode_image(
img, channels: num_channels, expand_animations: false);
var resized_image = tf.image.resize_images_v2(img, image_size, method: interpolation);
images[i] = resized_image.numpy();
tf_output_redirect.WriteLine(image_paths[i]);
};
var img_ds = tf.data.Dataset.from_tensor_slices(images);

// option 2: dynamic load, but has error, need to fix
// var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
// var img_ds = path_ds.map(x => path_to_image(x, image_size, num_channels, interpolation));
var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
var img_ds = path_ds.map(x => path_to_image(x, image_size, num_channels, interpolation));
if (label_mode == "int") if (label_mode == "int")
{ {
@@ -43,7 +28,7 @@ namespace Tensorflow.Keras


Tensor path_to_image(Tensor path, Shape image_size, int num_channels, string interpolation) Tensor path_to_image(Tensor path, Shape image_size, int num_channels, string interpolation)
{ {
tf.print(path);
// tf.print(path);
var img = tf.io.read_file(path); var img = tf.io.read_file(path);
img = tf.image.decode_image( img = tf.image.decode_image(
img, channels: num_channels, expand_animations: false); img, channels: num_channels, expand_animations: false);
@@ -58,18 +43,8 @@ namespace Tensorflow.Keras
int num_classes, int num_classes,
int max_length = -1) int max_length = -1)
{ {
var text = new string[image_paths.Length];
for (int i = 0; i < text.Length; i++)
{
text[i] = File.ReadAllText(image_paths[i]);
tf_output_redirect.WriteLine(image_paths[i]);
}

var images = np.array(text);
var string_ds = tf.data.Dataset.from_tensor_slices(images);

// var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
// var string_ds = path_ds.map(x => path_to_string_content(x, max_length));
var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
var string_ds = path_ds.map(x => path_to_string_content(x, max_length));


if (label_mode == "int") if (label_mode == "int")
{ {


+ 2
- 2
src/TensorFlowNET.Keras/Tensorflow.Keras.csproj View File

@@ -37,8 +37,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac
<RepositoryType>Git</RepositoryType> <RepositoryType>Git</RepositoryType>
<SignAssembly>true</SignAssembly> <SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>Open.snk</AssemblyOriginatorKeyFile> <AssemblyOriginatorKeyFile>Open.snk</AssemblyOriginatorKeyFile>
<AssemblyVersion>0.6.4.0</AssemblyVersion>
<FileVersion>0.6.4.0</FileVersion>
<AssemblyVersion>0.6.5.0</AssemblyVersion>
<FileVersion>0.6.5.0</FileVersion>
<PackageLicenseFile>LICENSE</PackageLicenseFile> <PackageLicenseFile>LICENSE</PackageLicenseFile>
</PropertyGroup> </PropertyGroup>




Loading…
Cancel
Save