scisharp
/
TensorFlow.NET
Not watched
1
0
Fork 0
Code
Releases 21 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Align with tensorflow v2.4.1

tags/v0.40-tf2.4-tstring
Oceania2018 4 years ago
parent
fb903d8703
commit
ad6ad8493c
8 changed files with 62 additions and 101 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +7
    -3
      src/TensorFlowNET.Console/MemoryBasicTest.cs
  2. +1
    -1
      src/TensorFlowNET.Console/Program.cs
  3. +1
    -1
      src/TensorFlowNET.Core/APIs/c_api.cs
  4. +6
    -4
      src/TensorFlowNET.Core/Tensorflow.Binding.csproj
  5. +22
    -85
      src/TensorFlowNET.Core/Tensors/Tensor.String.cs
  6. +16
    -2
      src/TensorFlowNET.Core/Tensors/Tensor.cs
  7. +4
    -1
      src/TensorFlowNET.Core/Tensors/c_api.tensor.cs
  8. +5
    -4
      test/TensorFlowNET.Native.UnitTest/Tensors/TensorTest.cs

+ 7
- 3
src/TensorFlowNET.Console/MemoryBasicTest.cs View File

@@ -4,6 +4,8 @@ using Tensorflow.Keras.ArgsDefinition;
using Tensorflow.Keras.Engine.DataAdapters;
using static Tensorflow.Binding;
using static Tensorflow.KerasApi;
using System.Linq;
using System.Collections.Generic;

namespace Tensorflow
{
@@ -35,13 +37,15 @@ namespace Tensorflow
public Action<int, int> ConstantString
=> (epoch, iterate) =>
{
var tensor = tf.constant(new string[]
var strList = new string[]
{
"Biden immigration bill would put millions of illegal immigrants on 8-year fast-track to citizenship",
"The Associated Press, which also reported that the eight-year path is in the bill.",
"The bill would also include provisions to stem the flow of migration by addressing root causes of migration from south of the border."
});
var data = tensor.numpy();
};

var tensor = tf.constant(strList, TF_DataType.TF_STRING);
var data = tensor.StringData();
};

public Action<int, int> Variable


+ 1
- 1
src/TensorFlowNET.Console/Program.cs View File

@@ -47,7 +47,7 @@ namespace Tensorflow
// explaination of constant
mm.Execute(10, 100 * batchSize, basic.Constant2x3);

mm.Execute(10, 100 * batchSize, basic.ConstantString);
mm.Execute(10, batchSize, basic.ConstantString);

// 100K float variable.
mm.Execute(10, batchSize, basic.Variable);


+ 1
- 1
src/TensorFlowNET.Core/APIs/c_api.cs View File

@@ -43,7 +43,7 @@ namespace Tensorflow
/// </summary>
public partial class c_api
{
public const string TensorFlowLibName = "tensorflow";
public const string TensorFlowLibName = @"D:\Projects\tensorflow-haiping\bazel-bin\tensorflow\tensorflow";

public static string StringPiece(IntPtr handle)
{


+ 6
- 4
src/TensorFlowNET.Core/Tensorflow.Binding.csproj View File

@@ -5,7 +5,7 @@
<AssemblyName>TensorFlow.NET</AssemblyName>
<RootNamespace>Tensorflow</RootNamespace>
<TargetTensorFlow>2.2.0</TargetTensorFlow>
<Version>0.33.0</Version>
<Version>0.40.0</Version>
<LangVersion>8.0</LangVersion>
<Authors>Haiping Chen, Meinrad Recheis, Eli Belash</Authors>
<Company>SciSharp STACK</Company>
@@ -19,7 +19,7 @@
<Description>Google's TensorFlow full binding in .NET Standard.
Building, training and infering deep learning models.
https://tensorflownet.readthedocs.io</Description>
<AssemblyVersion>0.33.0.0</AssemblyVersion>
<AssemblyVersion>0.40.0.0</AssemblyVersion>
<PackageReleaseNotes>tf.net 0.20.x and above are based on tensorflow native 2.x.

* Eager Mode is added finally.
@@ -29,8 +29,10 @@ https://tensorflownet.readthedocs.io</Description>
* Improve memory usage.

TensorFlow .NET v0.3x is focused on making more Keras API works.
Keras API is a separate package released as TensorFlow.Keras.</PackageReleaseNotes>
<FileVersion>0.33.0.0</FileVersion>
Keras API is a separate package released as TensorFlow.Keras.

tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library.</PackageReleaseNotes>
<FileVersion>0.40.0.0</FileVersion>
<PackageLicenseFile>LICENSE</PackageLicenseFile>
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
<SignAssembly>true</SignAssembly>


+ 22
- 85
src/TensorFlowNET.Core/Tensors/Tensor.String.cs View File

@@ -8,27 +8,7 @@ namespace Tensorflow
{
public partial class Tensor
{
const ulong TF_TSRING_SIZE = 24;

public IntPtr StringTensor25(string[] strings, TensorShape shape)
{
var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING,
shape.dims.Select(x => (long)x).ToArray(),
shape.ndim,
(ulong)shape.size * TF_TSRING_SIZE);

var data = c_api.TF_TensorData(handle);
var tstr = c_api.TF_StringInit(handle);
// AllocationHandle = tstr;
// AllocationType = AllocationType.Tensorflow;
for (int i = 0; i< strings.Length; i++)
{
c_api.TF_StringCopy(tstr, strings[i], strings[i].Length);
tstr += (int)TF_TSRING_SIZE;
}
// c_api.TF_StringDealloc(tstr);
return handle;
}
const int TF_TSRING_SIZE = 24;

public IntPtr StringTensor(string[] strings, TensorShape shape)
{
@@ -40,69 +20,28 @@ namespace Tensorflow
return StringTensor(buffer, shape);
}

public unsafe IntPtr StringTensor(byte[][] buffer, TensorShape shape)
public IntPtr StringTensor(byte[][] buffer, TensorShape shape)
{
ulong size = 0;
foreach (var b in buffer)
size += c_api.TF_StringEncodedSize((ulong)b.Length);

var src_size = size + (ulong)buffer.Length * sizeof(ulong);
var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING,
shape.dims.Select(x => (long)x).ToArray(),
shape.ndim == 0 ? null : shape.dims.Select(x => (long)x).ToArray(),
shape.ndim,
src_size);
AllocationType = AllocationType.Tensorflow;
(ulong)shape.size * TF_TSRING_SIZE);

IntPtr data_start = c_api.TF_TensorData(handle);
IntPtr string_start = data_start + buffer.Length * sizeof(ulong);
IntPtr limit = data_start + (int)src_size;
ulong offset = 0;
var tstr = c_api.TF_TensorData(handle);
#if TRACK_TENSOR_LIFE
print($"New TString 0x{handle.ToString("x16")} {AllocationType} Data: 0x{tstr.ToString("x16")}");
#endif
for (int i = 0; i < buffer.Length; i++)
{
Marshal.WriteInt64(data_start, i * sizeof(ulong), (long)offset);
if (buffer[i].Length == 0)
{
Marshal.WriteByte(string_start, 0);
break;
}

fixed (byte* src = &buffer[i][0])
{
/*Marshal.WriteByte(string_start, Convert.ToByte(buffer[i].Length));
tf.memcpy((string_start + 1).ToPointer(), src, (ulong)buffer[i].Length);
string_start += buffer[i].Length + 1;
offset += buffer[i].Length + 1;*/

var written = c_api.TF_StringEncode(src, (ulong)buffer[i].Length, (byte*)string_start, (ulong)(limit.ToInt64() - string_start.ToInt64()), tf.Status.Handle);
tf.Status.Check(true);
string_start += (int)written;
offset += written;
}
c_api.TF_StringInit(tstr);
c_api.TF_StringCopy(tstr, buffer[i], buffer[i].Length);
var data = c_api.TF_StringGetDataPointer(tstr);
tstr += TF_TSRING_SIZE;
}

return handle;
}

public string[] StringData25()
{
string[] strings = new string[c_api.TF_Dim(_handle, 0)];
var tstrings = TensorDataPointer;
for (int i = 0; i< strings.Length; i++)
{
var tstringData = c_api.TF_StringGetDataPointer(tstrings);
/*var size = c_api.TF_StringGetSize(tstrings);
var capacity = c_api.TF_StringGetCapacity(tstrings);
var type = c_api.TF_StringGetType(tstrings);*/
strings[i] = c_api.StringPiece(tstringData);
tstrings += (int)TF_TSRING_SIZE;
}
return strings;
}

/// <summary>
/// Extracts string array from current Tensor.
/// </summary>
/// <exception cref="InvalidOperationException">When <see cref="dtype"/> != TF_DataType.TF_STRING</exception>
public string[] StringData()
{
var buffer = StringBytes();
@@ -114,7 +53,7 @@ namespace Tensorflow
return _str;
}

public unsafe byte[][] StringBytes()
public byte[][] StringBytes()
{
if (dtype != TF_DataType.TF_STRING)
throw new InvalidOperationException($"Unable to call StringData when dtype != TF_DataType.TF_STRING (dtype is {dtype})");
@@ -123,24 +62,22 @@ namespace Tensorflow
// TF_STRING tensors are encoded with a table of 8-byte offsets followed by TF_StringEncode-encoded bytes.
// [offset1, offset2,...,offsetn, s1size, s1bytes, s2size, s2bytes,...,snsize,snbytes]
//
long size = 1;
int size = 1;
foreach (var s in TensorShape.dims)
size *= s;

var buffer = new byte[size][];
var data_start = c_api.TF_TensorData(_handle);
data_start += (int)(size * sizeof(ulong));
var tstrings = TensorDataPointer;
for (int i = 0; i < buffer.Length; i++)
{
IntPtr dst = IntPtr.Zero;
ulong dstLen = 0;
var read = c_api.TF_StringDecode((byte*)data_start, bytesize, (byte**)&dst, ref dstLen, tf.Status.Handle);
tf.Status.Check(true);
buffer[i] = new byte[(int)dstLen];
Marshal.Copy(dst, buffer[i], 0, buffer[i].Length);
data_start += (int)read;
var data = c_api.TF_StringGetDataPointer(tstrings);
var len = c_api.TF_StringGetSize(tstrings);
buffer[i] = new byte[len];
// var capacity = c_api.TF_StringGetCapacity(tstrings);
// var type = c_api.TF_StringGetType(tstrings);
Marshal.Copy(data, buffer[i], 0, Convert.ToInt32(len));
tstrings += TF_TSRING_SIZE;
}

return buffer;
}
}


+ 16
- 2
src/TensorFlowNET.Core/Tensors/Tensor.cs View File

@@ -15,7 +15,6 @@
******************************************************************************/

using NumSharp;
using NumSharp.Backends.Unmanaged;
using System;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
@@ -24,7 +23,6 @@ using System.Runtime.InteropServices;
using Tensorflow.Eager;
using Tensorflow.Framework;
using Tensorflow.Keras.Engine;
using Tensorflow.Variables;
using static Tensorflow.Binding;

namespace Tensorflow
@@ -287,6 +285,22 @@ namespace Tensorflow
throw new InvalidOperationException($"Tensor.AllocationHandle is not null ({AllocationHandle}) but AllocationType is not matched to a C# allocation type ({AllocationType}).");
}

if (dtype == TF_DataType.TF_STRING)
{
int size = 1;
foreach (var s in TensorShape.dims)
size *= s;
var tstr = TensorDataPointer;
#if TRACK_TENSOR_LIFE
print($"Delete TString 0x{handle.ToString("x16")} {AllocationType} Data: 0x{tstrings.ToString("x16")}");
#endif
for (int i = 0; i < size; i++)
{
c_api.TF_StringDealloc(tstr);
tstr += TF_TSRING_SIZE;
}
}

c_api.TF_DeleteTensor(handle);
}



+ 4
- 1
src/TensorFlowNET.Core/Tensors/c_api.tensor.cs View File

@@ -182,7 +182,10 @@ namespace Tensorflow
public static extern unsafe ulong TF_StringEncode(byte* src, ulong src_len, byte* dst, ulong dst_len, SafeStatusHandle status);

[DllImport(TensorFlowLibName)]
public static extern IntPtr TF_StringInit(IntPtr t);
public static extern void TF_StringInit(IntPtr t);

[DllImport(TensorFlowLibName)]
public static extern void TF_StringCopy(IntPtr dst, byte[] text, long size);

[DllImport(TensorFlowLibName)]
public static extern void TF_StringCopy(IntPtr dst, string text, long size);


+ 5
- 4
test/TensorFlowNET.Native.UnitTest/Tensors/TensorTest.cs View File

@@ -111,7 +111,7 @@ namespace Tensorflow.Native.UnitTest.Tensors
/// Port from c_api_test.cc
/// `TEST_F(CApiAttributesTest, StringTensor)`
/// </summary>
[TestMethod, Ignore("Waiting for PR https://github.com/tensorflow/tensorflow/pull/46804")]
[TestMethod]
public void StringTensor()
{
string text = "Hello world!.";
@@ -120,13 +120,14 @@ namespace Tensorflow.Native.UnitTest.Tensors
null,
0,
1 * 24);
var tstr = c_api.TF_StringInit(tensor);
var data = c_api.TF_StringGetDataPointer(tstr);
var tstr = c_api.TF_TensorData(tensor);
c_api.TF_StringInit(tstr);
c_api.TF_StringCopy(tstr, text, text.Length);
var data = c_api.TF_StringGetDataPointer(tstr);

Assert.AreEqual((ulong)text.Length, c_api.TF_StringGetSize(tstr));
Assert.AreEqual(text, c_api.StringPiece(data));
Assert.AreEqual((ulong)text.Length, c_api.TF_TensorByteSize(tensor));
Assert.AreEqual(TF_TString_Type.TF_TSTR_SMALL, c_api.TF_StringGetType(tensor));
Assert.AreEqual(0, c_api.TF_NumDims(tensor));

TF_DeleteTensor(tensor);


Write Preview
Loading…
Cancel
Save