Browse Source

ndarray mask set value.

tags/TimeSeries
Oceania2018 3 years ago
parent
commit
87fa204247
9 changed files with 79 additions and 146 deletions
  1. +19
    -2
      src/TensorFlowNET.Core/NumPy/NDArray.Index.cs
  2. +5
    -2
      src/TensorFlowNET.Core/NumPy/NDArray.Operators.cs
  3. +0
    -1
      src/TensorFlowNET.Keras/Engine/Model.Training.cs
  4. +1
    -1
      src/TensorFlowNET.Keras/Tensorflow.Keras.csproj
  5. +15
    -28
      test/TensorFlowNET.Graph.UnitTest/Basics/SessionTest.cs
  6. +6
    -9
      test/TensorFlowNET.Graph.UnitTest/Basics/TensorTest.cs
  7. +0
    -102
      test/TensorFlowNET.UnitTest/EnforcedSinglethreadingTests.cs
  8. +1
    -1
      test/TensorFlowNET.UnitTest/Hub/MnistModelLoaderTest.cs
  9. +32
    -0
      test/TensorFlowNET.UnitTest/NumPy/Array.Indexing.Test.cs

+ 19
- 2
src/TensorFlowNET.Core/NumPy/NDArray.Index.cs View File

@@ -43,7 +43,9 @@ namespace Tensorflow.NumPy
{
get
{
if(mask.dtype == TF_DataType.TF_INT32)
if (mask.dtype == TF_DataType.TF_BOOL)
return GetData(enumerate(mask.ToArray<bool>()).Where(x => x.Item2).Select(x => x.Item1).ToArray());
else if (mask.dtype == TF_DataType.TF_INT32)
return GetData(mask.ToArray<int>());
else if (mask.dtype == TF_DataType.TF_INT64)
return GetData(mask.ToArray<long>().Select(x => Convert.ToInt32(x)).ToArray());
@@ -55,7 +57,10 @@ namespace Tensorflow.NumPy

set
{
throw new NotImplementedException("");
if (mask.dtype == TF_DataType.TF_BOOL)
MaskData(mask, value);
else
throw new NotImplementedException("");
}
}

@@ -266,5 +271,17 @@ namespace Tensorflow.NumPy
// reset indices
indices[currentNDim] = 0;
}

unsafe void MaskData(NDArray mask, NDArray value)
{
var masks = mask.ToArray<bool>();
var s1 = new Shape(dims.Skip(mask.rank).ToArray());
var val = tf.fill(s1, value).numpy();
for (int i = 0; i < masks.Length; i++)
{
if (masks[i])
this[i] = val;
}
}
}
}

+ 5
- 2
src/TensorFlowNET.Core/NumPy/NDArray.Operators.cs View File

@@ -25,7 +25,10 @@ namespace Tensorflow.NumPy
[AutoNumPy]
public static NDArray operator -(NDArray lhs) => new NDArray(gen_math_ops.neg(lhs));
[AutoNumPy]
public static bool operator ==(NDArray lhs, NDArray rhs) => rhs is null ? false : (bool)math_ops.equal(lhs, rhs);
public static bool operator !=(NDArray lhs, NDArray rhs) => !(lhs == rhs);
public static NDArray operator ==(NDArray lhs, NDArray rhs)
=> rhs is null ? Scalar(false) : new NDArray(math_ops.equal(lhs, rhs));
[AutoNumPy]
public static NDArray operator !=(NDArray lhs, NDArray rhs)
=> new NDArray(math_ops.not_equal(lhs, rhs));
}
}

+ 0
- 1
src/TensorFlowNET.Keras/Engine/Model.Training.cs View File

@@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.Text;
using HDF.PInvoke;
using HDF5CSharp;
using Tensorflow.NumPy;
using static Tensorflow.Binding;
using Tensorflow.Keras.Saving;



+ 1
- 1
src/TensorFlowNET.Keras/Tensorflow.Keras.csproj View File

@@ -60,9 +60,9 @@ Keras is an API designed for human beings, not machines. Keras follows best prac
</PropertyGroup>

<ItemGroup>
<PackageReference Include="HDF5-CSharp" Version="1.12.4" />
<PackageReference Include="MethodBoundaryAspect.Fody" Version="2.0.144" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="SciSharp.Keras.HDF5" Version="1.1.10.500" />
<PackageReference Include="SharpZipLib" Version="1.3.3" />
</ItemGroup>



test/TensorFlowNET.UnitTest/Basics/SessionTest.cs → test/TensorFlowNET.Graph.UnitTest/Basics/SessionTest.cs View File

@@ -1,17 +1,15 @@
using FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Tensorflow.NumPy;
using System;
using System.Collections.Generic;
using System.Text;
using Tensorflow;
using Tensorflow.Util;
using static Tensorflow.Binding;

namespace TensorFlowNET.UnitTest
namespace TensorFlowNET.UnitTest.Basics
{
[TestClass, Ignore]
public class SessionTest
[TestClass]
public class SessionTest : GraphModeTestBase
{
[TestMethod]
public void EvalTensor()
@@ -32,16 +30,12 @@ namespace TensorFlowNET.UnitTest
[TestMethod]
public void Eval_SmallString_Scalar()
{
lock (this)
var a = constant_op.constant("123 heythere 123 ", TF_DataType.TF_STRING);
var c = tf.strings.substr(a, 4, 8);
using (var sess = tf.Session())
{
var a = constant_op.constant("123 heythere 123 ", TF_DataType.TF_STRING);
var c = tf.strings.substr(a, 4, 8);
using (var sess = tf.Session())
{
var result = UTF8Encoding.UTF8.GetString(c.eval(sess).ToByteArray());
Console.WriteLine(result);
result.Should().Be("heythere");
}
var result = c.eval(sess).StringData();
Assert.AreEqual(result[0], "heythere");
}
}

@@ -57,7 +51,6 @@ namespace TensorFlowNET.UnitTest
{
var result = UTF8Encoding.UTF8.GetString(c.eval(sess).ToByteArray());
Console.WriteLine(result);
result.Should().HaveLength(size - 5000).And.ContainAll("a");
}
}
}
@@ -69,21 +62,19 @@ namespace TensorFlowNET.UnitTest
ITensorOrOperation operation = tf.global_variables_initializer();
// the cast to ITensorOrOperation is essential for the test of this method signature
var ret = sess.run(operation);

ret.Should().BeNull();
}

[TestMethod]
public void Autocast_Case1()
{
var sess = tf.Session().as_default();
var input = tf.placeholder(tf.float32, shape: new Shape(6));
var input = tf.placeholder(tf.int32, shape: new Shape(6));
var op = tf.reshape(input, new int[] { 2, 3 });
sess.run(tf.global_variables_initializer());
var ret = sess.run(op, feed_dict: (input, np.array(1, 2, 3, 4, 5, 6)));

Assert.AreEqual(ret.shape, (2, 3));
Assert.AreEqual(ret, new[] { 1, 2, 3, 4, 5, 6 });
assertAllEqual(ret.ToArray<int>(), new[] { 1, 2, 3, 4, 5, 6 });
print(ret.dtype);
print(ret);
}
@@ -92,21 +83,17 @@ namespace TensorFlowNET.UnitTest
public void Autocast_Case2()
{
var sess = tf.Session().as_default();
var input = tf.placeholder(tf.float64, shape: new Shape(6));
var input = tf.placeholder(tf.float32, shape: new Shape(6));
var op = tf.reshape(input, new int[] { 2, 3 });
sess.run(tf.global_variables_initializer());
var ret = sess.run(op, feed_dict: (input, np.array(1, 2, 3, 4, 5, 6).astype(np.float32) + 0.1f));

ret.Should().BeShaped(2, 3).And.BeOfValuesApproximately(0.001d, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1);
print(ret.dtype);
print(ret);
}

[TestMethod]
[TestMethod, Ignore]
public void Autocast_Case3()
{
var sess = tf.Session().as_default();
var input = tf.placeholder(tf.int64, shape: new Shape(6));
var input = tf.placeholder(tf.float32, shape: new Shape(6));
var op = tf.reshape(input, new int[] { 2, 3 });
sess.run(tf.global_variables_initializer());
var ret = sess.run(op, feed_dict: (input, np.array(1, 2, 3, 4, 5, 6).astype(np.float32) + 0.1f));
@@ -117,7 +104,7 @@ namespace TensorFlowNET.UnitTest
print(ret);
}

[TestMethod]
[TestMethod, Ignore]
public void Autocast_Case4()
{
var sess = tf.Session().as_default();

test/TensorFlowNET.UnitTest/Basics/TensorTest.cs → test/TensorFlowNET.Graph.UnitTest/Basics/TensorTest.cs View File

@@ -1,18 +1,15 @@
using FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Tensorflow.NumPy;
using System;
using System.Linq;
using System.Runtime.InteropServices;
using Tensorflow;
using static Tensorflow.Binding;

namespace TensorFlowNET.UnitTest
namespace TensorFlowNET.UnitTest.Basics
{
[TestClass, Ignore]
public class TensorTest
[TestClass]
public class TensorTest : GraphModeTestBase
{
[TestMethod]
[TestMethod, Ignore]
public void sparse_to_dense()
{
var indices = tf.reshape(tf.range(0, 5), new int[] { 5, 1 });
@@ -30,7 +27,7 @@ namespace TensorFlowNET.UnitTest
};
}

[TestMethod]
[TestMethod, Ignore]
public void sparse_tensor_to_dense()
{
var decoded_list = tf.SparseTensor(new[,]

+ 0
- 102
test/TensorFlowNET.UnitTest/EnforcedSinglethreadingTests.cs View File

@@ -1,102 +0,0 @@
using FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Diagnostics;
using System.Threading;
using Tensorflow;
using static Tensorflow.Binding;

namespace TensorFlowNET.UnitTest
{
[TestClass]
public class EnforcedSinglethreadingTests
{
private static readonly object _singlethreadLocker = new object();

/// <summary>Initializes a new instance of the <see cref="T:System.Object" /> class.</summary>
public EnforcedSinglethreadingTests()
{
}

[TestMethod, Ignore("Has to be tested manually.")]
public void SessionCreation()
{
lock (_singlethreadLocker)
{
ops.uid(); //increment id by one

//the core method
tf.peak_default_graph().Should().BeNull();

using (var sess = tf.Session())
{
var default_graph = tf.peak_default_graph();
var sess_graph = sess.graph;
sess_graph.Should().NotBeNull();
default_graph.Should().NotBeNull()
.And.BeEquivalentTo(sess_graph);

var (graph, session) = Parallely(() => (tf.get_default_graph(), tf.get_default_session()));

graph.Should().BeEquivalentTo(default_graph);
session.Should().BeEquivalentTo(sess);
}
}
}

T Parallely<T>(Func<T> fnc)
{
var mrh = new ManualResetEventSlim();
T ret = default;
Exception e = default;
new Thread(() =>
{
try
{
ret = fnc();
}
catch (Exception ee)
{
e = ee;
throw;
}
finally
{
mrh.Set();
}
}).Start();

if (!Debugger.IsAttached)
mrh.Wait(10000).Should().BeTrue();
else
mrh.Wait(-1);
e.Should().BeNull(e?.ToString());
return ret;
}

void Parallely(Action fnc)
{
var mrh = new ManualResetEventSlim();
Exception e = default;
new Thread(() =>
{
try
{
fnc();
}
catch (Exception ee)
{
e = ee;
throw;
}
finally
{
mrh.Set();
}
}).Start();

mrh.Wait(10000).Should().BeTrue();
e.Should().BeNull(e.ToString());
}
}
}

+ 1
- 1
test/TensorFlowNET.UnitTest/Hub/MnistModelLoaderTest.cs View File

@@ -7,7 +7,7 @@ namespace TensorFlowNET.UnitTest
[TestClass]
public class MnistModelLoaderTest
{
[TestMethod, Ignore]
[TestMethod]
public async Task TestLoad()
{
var loader = new MnistModelLoader();


+ 32
- 0
test/TensorFlowNET.UnitTest/NumPy/Array.Indexing.Test.cs View File

@@ -37,6 +37,14 @@ namespace TensorFlowNET.UnitTest.NumPy
Assert.AreEqual(x[1], z);
}

[TestMethod]
public void slice_newaxis()
{
var x = np.arange(20).reshape((4, 5));
var y = x[np.newaxis, ":2"];
Assert.AreEqual(y.shape, (1, 2, 5));
}

[TestMethod]
public void slice_params()
{
@@ -143,5 +151,29 @@ namespace TensorFlowNET.UnitTest.NumPy
Assert.AreEqual(array[2], new[] { 16, 17, 100, 19, 200, 21, 22, 23 });
Assert.AreEqual(array[3], new[] { 24, 25, 100, 27, 200, 29, 30, 31 });
}

[TestMethod]
public void mask_2d_get_value()
{
var x = np.arange(25).reshape((5, 5));
var y = np.array(new[] { true, false, true, false, true });
var z = x[y];
Assert.AreEqual(z.shape, (3, 5));
Assert.AreEqual(z[0], new[] { 0, 1, 2, 3, 4 });
Assert.AreEqual(z[1], new[] { 10, 11, 12, 13, 14 });
Assert.AreEqual(z[2], new[] { 20, 21, 22, 23, 24 });
}

[TestMethod]
public void mask_2d_set_value()
{
var x = np.arange(25).reshape((5, 5));
var y = np.array(new[] {true, false, true, false, false});
x[y] = 0;
Assert.AreEqual(x[0], new[] { 0, 0, 0, 0, 0 });
Assert.AreEqual(x[1], new[] { 5, 6, 7, 8, 9 });
Assert.AreEqual(x[2], new[] { 0, 0, 0, 0, 0 });
Assert.AreEqual(x[3], new[] { 15, 16, 17, 18, 19 });
}
}
}

Loading…
Cancel
Save