From 2e3bc0630fd949b61a87c65d5ad6ab068bbedb11 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Mon, 12 Jul 2021 23:33:50 -0500 Subject: [PATCH] fix NumPy frombuffer. --- src/TensorFlowNET.Core/Data/MnistModelLoader.cs | 4 ++-- .../Eager/EagerTensor.Creation.cs | 2 +- .../NumPy/Implementation/NumPyImpl.Creation.cs | 4 ++-- .../NumPy/Numpy.Manipulation.cs | 14 ++++++++++++++ .../Numpy/NDArray.Creation.cs | 7 ++++--- src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs | 4 ++-- src/TensorFlowNET.Core/Numpy/Numpy.cs | 3 --- .../Operations/OpDefLibrary.cs | 3 ++- .../Tensors/Tensor.Creation.cs | 6 +++--- src/TensorFlowNET.Core/Tensors/c_api.tensor.cs | 17 +++++++++++++++-- src/TensorFlowNET.Core/Tensors/constant_op.cs | 2 +- src/TensorFlowNET.Core/Tensors/tensor_util.cs | 2 +- 12 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs diff --git a/src/TensorFlowNET.Core/Data/MnistModelLoader.cs b/src/TensorFlowNET.Core/Data/MnistModelLoader.cs index 5be88bba..f99c1e5d 100644 --- a/src/TensorFlowNET.Core/Data/MnistModelLoader.cs +++ b/src/TensorFlowNET.Core/Data/MnistModelLoader.cs @@ -123,7 +123,7 @@ namespace Tensorflow bytestream.Read(buf, 0, buf.Length); - var data = np.frombuffer(buf, np.@byte); + var data = np.frombuffer(buf, (rows, cols), np.@byte); data = data.reshape((num_images, rows, cols, 1)); return data; @@ -148,7 +148,7 @@ namespace Tensorflow bytestream.Read(buf, 0, buf.Length); - var labels = np.frombuffer(buf, np.uint8); + var labels = np.frombuffer(buf, new Shape(num_items), np.uint8); if (one_hot) return DenseToOneHot(labels, num_classes); diff --git a/src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs b/src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs index 81ae271d..9f40de5a 100644 --- a/src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs +++ b/src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs @@ -53,7 +53,7 @@ namespace Tensorflow.Eager public EagerTensor(Array array, Shape shape) : base(array, shape) => NewEagerTensorHandle(_handle); - public EagerTensor(byte[] bytes, TF_DataType dtype) : base(bytes, dtype) + public EagerTensor(byte[] bytes, Shape shape, TF_DataType dtype) : base(bytes, shape, dtype) => NewEagerTensorHandle(_handle); void NewEagerTensorHandle(IntPtr h) diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs index ca49e19f..618f4d70 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs @@ -33,9 +33,9 @@ namespace Tensorflow.NumPy return new NDArray(tensor); } - public NDArray frombuffer(byte[] bytes, TF_DataType dtype) + public NDArray frombuffer(byte[] bytes, Shape shape, TF_DataType dtype) { - throw new NotImplementedException(""); + return new NDArray(bytes, shape, dtype); } public NDArray linspace(T start, T stop, int num = 50, bool endpoint = true, bool retstep = false, diff --git a/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs b/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs new file mode 100644 index 00000000..81a5811f --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Numerics; +using System.Text; + +namespace Tensorflow.NumPy +{ + public partial class np + { + public static NDArray squeeze(NDArray x1, Axis? axis = null) + => new NDArray(array_ops.squeeze(x1, axis)); + } +} diff --git a/src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs b/src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs index 61fc9258..d222b3cd 100644 --- a/src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs +++ b/src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs @@ -18,7 +18,7 @@ namespace Tensorflow.NumPy public NDArray(Array value, Shape? shape = null) => Init(value, shape); public NDArray(Shape shape, TF_DataType dtype = TF_DataType.TF_DOUBLE) => Init(shape, dtype: dtype); public NDArray(Tensor value, Shape? shape = null) => Init(value, shape); - public NDArray(byte[] bytes, TF_DataType dtype) => Init(bytes, dtype); + public NDArray(byte[] bytes, Shape shape, TF_DataType dtype) => Init(bytes, shape, dtype); public static NDArray Scalar(T value) where T : unmanaged => value switch @@ -70,9 +70,10 @@ namespace Tensorflow.NumPy _tensor.SetReferencedByNDArray(); } - void Init(byte[] bytes, TF_DataType dtype) + void Init(byte[] bytes, Shape shape, TF_DataType dtype) { - + _tensor = new Tensor(bytes, shape, dtype); + _tensor.SetReferencedByNDArray(); } } } diff --git a/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs b/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs index aa1371db..237c3ea7 100644 --- a/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs +++ b/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs @@ -33,8 +33,8 @@ namespace Tensorflow.NumPy public static NDArray full(Shape shape, T fill_value) => new NDArray(tf.fill(tf.constant(shape), fill_value)); - public static NDArray frombuffer(byte[] bytes, TF_DataType dtype) - => tf.numpy.frombuffer(bytes, dtype); + public static NDArray frombuffer(byte[] bytes, Shape shape, TF_DataType dtype) + => tf.numpy.frombuffer(bytes, shape, dtype); public static NDArray linspace(T start, T stop, int num = 50, bool endpoint = true, bool retstep = false, TF_DataType dtype = TF_DataType.TF_DOUBLE, int axis = 0) where T : unmanaged diff --git a/src/TensorFlowNET.Core/Numpy/Numpy.cs b/src/TensorFlowNET.Core/Numpy/Numpy.cs index 174da10b..af9964df 100644 --- a/src/TensorFlowNET.Core/Numpy/Numpy.cs +++ b/src/TensorFlowNET.Core/Numpy/Numpy.cs @@ -59,9 +59,6 @@ namespace Tensorflow.NumPy public static NDArray frombuffer(byte[] bytes, string dtype) => throw new NotImplementedException(""); - public static NDArray squeeze(NDArray x1) - => throw new NotImplementedException(""); - public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8, bool equal_nan = false) => throw new NotImplementedException(""); diff --git a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs index 9dc876b5..7fc85dff 100644 --- a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs +++ b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs @@ -380,7 +380,8 @@ namespace Tensorflow attr_value.List.Type.AddRange((value as IList).Select(x => _MakeType(x, attr_def))); break; case "list(int)": - attr_value.List.I.AddRange((value as int[]).Select(x => Convert.ToInt64(x))); + if (value != null) + attr_value.List.I.AddRange((value as int[]).Select(x => Convert.ToInt64(x))); break; case "bool": attr_value.B = (bool)value; diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs index ba0d6009..6e9fa84d 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs @@ -96,7 +96,7 @@ namespace Tensorflow public Tensor(Shape shape, TF_DataType dtype) => InitTensor(shape, dtype); public Tensor(Array array, Shape? shape = null) => InitTensor(array, shape); - public Tensor(byte[] bytes, TF_DataType dtype) => InitTensor(bytes, dtype); + public Tensor(byte[] bytes, Shape shape, TF_DataType dtype) => InitTensor(shape, bytes, dtype); public Tensor(Operation op, int value_index, TF_DataType dtype) { @@ -113,12 +113,12 @@ namespace Tensorflow isCreatedInGraphMode = !tf.executing_eagerly(); } - protected unsafe void InitTensor(byte[] bytes, TF_DataType dtype) + protected unsafe void InitTensor(Shape shape, byte[] bytes, TF_DataType dtype) { if (dtype == TF_DataType.TF_STRING) _handle = StringTensor(new byte[][] { bytes }, Shape.Scalar); else - throw new NotImplementedException(""); + _handle = TF_NewTensor(bytes, shape, dtype); isCreatedInGraphMode = !tf.executing_eagerly(); } diff --git a/src/TensorFlowNET.Core/Tensors/c_api.tensor.cs b/src/TensorFlowNET.Core/Tensors/c_api.tensor.cs index c4e59aa7..07ef3c81 100644 --- a/src/TensorFlowNET.Core/Tensors/c_api.tensor.cs +++ b/src/TensorFlowNET.Core/Tensors/c_api.tensor.cs @@ -104,13 +104,26 @@ namespace Tensorflow return TF_NewTensor(dataType, dims, num_dims, data, len, EmptyDeallocator, DeallocatorArgs.Empty); } + public static unsafe IntPtr TF_NewTensor(byte[] data, Shape shape, TF_DataType dtype) + { + var length = data.Length; + var handle = TF_AllocateTensor(dtype, shape.dims, shape.ndim, (ulong)length); + var tensor = TF_TensorData(handle); + if (tensor == IntPtr.Zero) + throw new TensorflowException("AllocateTensor failed."); + fixed (void* addr = &data[0]) + System.Buffer.MemoryCopy(addr, tensor.ToPointer(), length, length); + return handle; + } + public static unsafe IntPtr TF_NewTensor(Shape shape, TF_DataType dtype, void* data) { var length = shape.size * dtype.get_datatype_size(); var handle = TF_AllocateTensor(dtype, shape.dims, shape.ndim, (ulong)length); var tensor = TF_TensorData(handle); - if (tensor != IntPtr.Zero) - System.Buffer.MemoryCopy(data, tensor.ToPointer(), length, length); + if (tensor == IntPtr.Zero) + throw new TensorflowException("AllocateTensor failed."); + System.Buffer.MemoryCopy(data, tensor.ToPointer(), length, length); return handle; } diff --git a/src/TensorFlowNET.Core/Tensors/constant_op.cs b/src/TensorFlowNET.Core/Tensors/constant_op.cs index 8547dd38..df060c05 100644 --- a/src/TensorFlowNET.Core/Tensors/constant_op.cs +++ b/src/TensorFlowNET.Core/Tensors/constant_op.cs @@ -100,7 +100,7 @@ namespace Tensorflow // non ascii char if (dtype == TF_DataType.TF_STRING && value is byte[] bytes) - return new EagerTensor(bytes, TF_DataType.TF_STRING); + return new EagerTensor(bytes, Shape.Scalar, TF_DataType.TF_STRING); switch (value) { diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index f93a27b5..d4e6c7f1 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -66,7 +66,7 @@ namespace Tensorflow if (shape.ndim > 0 && tensor.TensorContent.Length > 0) { - return np.frombuffer(tensor.TensorContent.ToByteArray(), tensor_dtype).reshape(shape); + return np.frombuffer(tensor.TensorContent.ToByteArray(), shape, tensor_dtype); } else if (tensor.Dtype == DataType.DtHalf || tensor.Dtype == DataType.DtBfloat16) {