Browse Source

fix NumPy frombuffer.

tags/v0.60-tf.numpy
Oceania2018 4 years ago
parent
commit
2e3bc0630f
12 changed files with 47 additions and 21 deletions
  1. +2
    -2
      src/TensorFlowNET.Core/Data/MnistModelLoader.cs
  2. +1
    -1
      src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs
  3. +2
    -2
      src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs
  4. +14
    -0
      src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs
  5. +4
    -3
      src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs
  6. +2
    -2
      src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs
  7. +0
    -3
      src/TensorFlowNET.Core/Numpy/Numpy.cs
  8. +2
    -1
      src/TensorFlowNET.Core/Operations/OpDefLibrary.cs
  9. +3
    -3
      src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs
  10. +15
    -2
      src/TensorFlowNET.Core/Tensors/c_api.tensor.cs
  11. +1
    -1
      src/TensorFlowNET.Core/Tensors/constant_op.cs
  12. +1
    -1
      src/TensorFlowNET.Core/Tensors/tensor_util.cs

+ 2
- 2
src/TensorFlowNET.Core/Data/MnistModelLoader.cs View File

@@ -123,7 +123,7 @@ namespace Tensorflow

bytestream.Read(buf, 0, buf.Length);

var data = np.frombuffer(buf, np.@byte);
var data = np.frombuffer(buf, (rows, cols), np.@byte);
data = data.reshape((num_images, rows, cols, 1));

return data;
@@ -148,7 +148,7 @@ namespace Tensorflow

bytestream.Read(buf, 0, buf.Length);

var labels = np.frombuffer(buf, np.uint8);
var labels = np.frombuffer(buf, new Shape(num_items), np.uint8);

if (one_hot)
return DenseToOneHot(labels, num_classes);


+ 1
- 1
src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs View File

@@ -53,7 +53,7 @@ namespace Tensorflow.Eager
public EagerTensor(Array array, Shape shape) : base(array, shape)
=> NewEagerTensorHandle(_handle);

public EagerTensor(byte[] bytes, TF_DataType dtype) : base(bytes, dtype)
public EagerTensor(byte[] bytes, Shape shape, TF_DataType dtype) : base(bytes, shape, dtype)
=> NewEagerTensorHandle(_handle);

void NewEagerTensorHandle(IntPtr h)


+ 2
- 2
src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs View File

@@ -33,9 +33,9 @@ namespace Tensorflow.NumPy
return new NDArray(tensor);
}

public NDArray frombuffer(byte[] bytes, TF_DataType dtype)
public NDArray frombuffer(byte[] bytes, Shape shape, TF_DataType dtype)
{
throw new NotImplementedException("");
return new NDArray(bytes, shape, dtype);
}

public NDArray linspace<T>(T start, T stop, int num = 50, bool endpoint = true, bool retstep = false,


+ 14
- 0
src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs View File

@@ -0,0 +1,14 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Numerics;
using System.Text;

namespace Tensorflow.NumPy
{
public partial class np
{
public static NDArray squeeze(NDArray x1, Axis? axis = null)
=> new NDArray(array_ops.squeeze(x1, axis));
}
}

+ 4
- 3
src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs View File

@@ -18,7 +18,7 @@ namespace Tensorflow.NumPy
public NDArray(Array value, Shape? shape = null) => Init(value, shape);
public NDArray(Shape shape, TF_DataType dtype = TF_DataType.TF_DOUBLE) => Init(shape, dtype: dtype);
public NDArray(Tensor value, Shape? shape = null) => Init(value, shape);
public NDArray(byte[] bytes, TF_DataType dtype) => Init(bytes, dtype);
public NDArray(byte[] bytes, Shape shape, TF_DataType dtype) => Init(bytes, shape, dtype);

public static NDArray Scalar<T>(T value) where T : unmanaged
=> value switch
@@ -70,9 +70,10 @@ namespace Tensorflow.NumPy
_tensor.SetReferencedByNDArray();
}

void Init(byte[] bytes, TF_DataType dtype)
void Init(byte[] bytes, Shape shape, TF_DataType dtype)
{

_tensor = new Tensor(bytes, shape, dtype);
_tensor.SetReferencedByNDArray();
}
}
}

+ 2
- 2
src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs View File

@@ -33,8 +33,8 @@ namespace Tensorflow.NumPy
public static NDArray full<T>(Shape shape, T fill_value)
=> new NDArray(tf.fill(tf.constant(shape), fill_value));

public static NDArray frombuffer(byte[] bytes, TF_DataType dtype)
=> tf.numpy.frombuffer(bytes, dtype);
public static NDArray frombuffer(byte[] bytes, Shape shape, TF_DataType dtype)
=> tf.numpy.frombuffer(bytes, shape, dtype);

public static NDArray linspace<T>(T start, T stop, int num = 50, bool endpoint = true, bool retstep = false,
TF_DataType dtype = TF_DataType.TF_DOUBLE, int axis = 0) where T : unmanaged


+ 0
- 3
src/TensorFlowNET.Core/Numpy/Numpy.cs View File

@@ -59,9 +59,6 @@ namespace Tensorflow.NumPy
public static NDArray frombuffer(byte[] bytes, string dtype)
=> throw new NotImplementedException("");

public static NDArray squeeze(NDArray x1)
=> throw new NotImplementedException("");

public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8,
bool equal_nan = false) => throw new NotImplementedException("");



+ 2
- 1
src/TensorFlowNET.Core/Operations/OpDefLibrary.cs View File

@@ -380,7 +380,8 @@ namespace Tensorflow
attr_value.List.Type.AddRange((value as IList<TF_DataType>).Select(x => _MakeType(x, attr_def)));
break;
case "list(int)":
attr_value.List.I.AddRange((value as int[]).Select(x => Convert.ToInt64(x)));
if (value != null)
attr_value.List.I.AddRange((value as int[]).Select(x => Convert.ToInt64(x)));
break;
case "bool":
attr_value.B = (bool)value;


+ 3
- 3
src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs View File

@@ -96,7 +96,7 @@ namespace Tensorflow

public Tensor(Shape shape, TF_DataType dtype) => InitTensor(shape, dtype);
public Tensor(Array array, Shape? shape = null) => InitTensor(array, shape);
public Tensor(byte[] bytes, TF_DataType dtype) => InitTensor(bytes, dtype);
public Tensor(byte[] bytes, Shape shape, TF_DataType dtype) => InitTensor(shape, bytes, dtype);

public Tensor(Operation op, int value_index, TF_DataType dtype)
{
@@ -113,12 +113,12 @@ namespace Tensorflow
isCreatedInGraphMode = !tf.executing_eagerly();
}

protected unsafe void InitTensor(byte[] bytes, TF_DataType dtype)
protected unsafe void InitTensor(Shape shape, byte[] bytes, TF_DataType dtype)
{
if (dtype == TF_DataType.TF_STRING)
_handle = StringTensor(new byte[][] { bytes }, Shape.Scalar);
else
throw new NotImplementedException("");
_handle = TF_NewTensor(bytes, shape, dtype);
isCreatedInGraphMode = !tf.executing_eagerly();
}



+ 15
- 2
src/TensorFlowNET.Core/Tensors/c_api.tensor.cs View File

@@ -104,13 +104,26 @@ namespace Tensorflow
return TF_NewTensor(dataType, dims, num_dims, data, len, EmptyDeallocator, DeallocatorArgs.Empty);
}

public static unsafe IntPtr TF_NewTensor(byte[] data, Shape shape, TF_DataType dtype)
{
var length = data.Length;
var handle = TF_AllocateTensor(dtype, shape.dims, shape.ndim, (ulong)length);
var tensor = TF_TensorData(handle);
if (tensor == IntPtr.Zero)
throw new TensorflowException("AllocateTensor failed.");
fixed (void* addr = &data[0])
System.Buffer.MemoryCopy(addr, tensor.ToPointer(), length, length);
return handle;
}

public static unsafe IntPtr TF_NewTensor(Shape shape, TF_DataType dtype, void* data)
{
var length = shape.size * dtype.get_datatype_size();
var handle = TF_AllocateTensor(dtype, shape.dims, shape.ndim, (ulong)length);
var tensor = TF_TensorData(handle);
if (tensor != IntPtr.Zero)
System.Buffer.MemoryCopy(data, tensor.ToPointer(), length, length);
if (tensor == IntPtr.Zero)
throw new TensorflowException("AllocateTensor failed.");
System.Buffer.MemoryCopy(data, tensor.ToPointer(), length, length);
return handle;
}



+ 1
- 1
src/TensorFlowNET.Core/Tensors/constant_op.cs View File

@@ -100,7 +100,7 @@ namespace Tensorflow

// non ascii char
if (dtype == TF_DataType.TF_STRING && value is byte[] bytes)
return new EagerTensor(bytes, TF_DataType.TF_STRING);
return new EagerTensor(bytes, Shape.Scalar, TF_DataType.TF_STRING);

switch (value)
{


+ 1
- 1
src/TensorFlowNET.Core/Tensors/tensor_util.cs View File

@@ -66,7 +66,7 @@ namespace Tensorflow

if (shape.ndim > 0 && tensor.TensorContent.Length > 0)
{
return np.frombuffer(tensor.TensorContent.ToByteArray(), tensor_dtype).reshape(shape);
return np.frombuffer(tensor.TensorContent.ToByteArray(), shape, tensor_dtype);
}
else if (tensor.Dtype == DataType.DtHalf || tensor.Dtype == DataType.DtBfloat16)
{


Loading…
Cancel
Save