using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json.Linq; using Tensorflow.NumPy; using System.Collections; using Tensorflow; using static Tensorflow.Binding; namespace TensorFlowNET.UnitTest { /// /// Use as base class for test classes to get additional assertions /// public class PythonTest { #region python compatibility layer protected PythonTest self { get => this; } protected int None => -1; #endregion #region pytest assertions public void assertItemsEqual(ICollection given, ICollection expected) { if (given is Hashtable && expected is Hashtable) { Assert.AreEqual(JObject.FromObject(expected).ToString(), JObject.FromObject(given).ToString()); return; } Assert.IsNotNull(expected); Assert.IsNotNull(given); var e = expected.OfType().ToArray(); var g = given.OfType().ToArray(); Assert.AreEqual(e.Length, g.Length, $"The collections differ in length expected {e.Length} but got {g.Length}"); for (int i = 0; i < e.Length; i++) { /*if (g[i] is NDArray && e[i] is NDArray) assertItemsEqual((g[i] as NDArray).GetData(), (e[i] as NDArray).GetData()); else*/ if (e[i] is ICollection && g[i] is ICollection) assertEqual(g[i], e[i]); else Assert.AreEqual(e[i], g[i], $"Items differ at index {i}, expected {e[i]} but got {g[i]}"); } } public void assertAllEqual(ICollection given, ICollection expected) { assertItemsEqual(given, expected); } public void assertFloat32Equal(float expected, float actual, string msg) { float eps = 1e-6f; Assert.IsTrue(Math.Abs(expected - actual) < eps * Math.Max(1.0f, Math.Abs(expected)), $"{msg}: expected {expected} vs actual {actual}"); } public void assertFloat64Equal(double expected, double actual, string msg) { double eps = 1e-16f; Assert.IsTrue(Math.Abs(expected - actual) < eps * Math.Max(1.0f, Math.Abs(expected)), $"{msg}: expected {expected} vs actual {actual}"); } public void AssetSequenceEqual(float[] expected, float[] actual) { float eps = 1e-5f; for (int i = 0; i < expected.Length; i++) Assert.IsTrue(Math.Abs(expected[i] - actual[i]) < eps * Math.Max(1.0f, Math.Abs(expected[i])), $"expected {expected} vs actual {actual}"); } public void AssetSequenceEqual(double[] expected, double[] actual) { double eps = 1e-5f; for (int i = 0; i < expected.Length; i++) Assert.IsTrue(Math.Abs(expected[i] - actual[i]) < eps * Math.Max(1.0f, Math.Abs(expected[i])), $"expected {expected} vs actual {actual}"); } public void assertEqual(object given, object expected) { /*if (given is NDArray && expected is NDArray) { assertItemsEqual((given as NDArray).GetData(), (expected as NDArray).GetData()); return; }*/ if (given is Hashtable && expected is Hashtable) { Assert.AreEqual(JObject.FromObject(expected).ToString(), JObject.FromObject(given).ToString()); return; } if (given is ICollection collectionGiven && expected is ICollection collectionExpected) { assertItemsEqual(collectionGiven, collectionExpected); return; } if (given is float && expected is float) { assertFloat32Equal((float)expected, (float)given, ""); return; } if (given is double && expected is double) { assertFloat64Equal((double)expected, (double)given, ""); return; } Assert.AreEqual(expected, given); } public void assertEquals(object given, object expected) { assertEqual(given, expected); } public void assert(object given) { if (given is bool) Assert.IsTrue((bool)given); Assert.IsNotNull(given); } public void assertIsNotNone(object given) { Assert.IsNotNull(given); } public void assertFalse(bool cond) { Assert.IsFalse(cond); } public void assertTrue(bool cond) { Assert.IsTrue(cond); } public void assertAllClose(NDArray array1, NDArray array2, double eps = 1e-5) { CollectionAssert.AreEqual(array1.ToArray(), array2.ToArray(), new CollectionComparer(eps)); //TODO: Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); } public void assertAllClose(double value, NDArray array2, double eps = 1e-5) { if (array2.shape.IsScalar) { double value2 = array2; Assert.AreEqual(value, value2, eps); return; } var array1 = np.ones_like(array2) * value; CollectionAssert.AreEqual(array1.ToArray(), array2.ToArray(), new CollectionComparer(eps)); //TODO: Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); } private class CollectionComparer : IComparer { private readonly double _epsilon; public CollectionComparer(double eps = 1e-06) { _epsilon = eps; } public int Compare(object? x, object? y) { if (x == null && y == null) { return 0; } else if (x == null) { return -1; } else if (y == null) { return 1; } var a = Convert.ToDouble(x); var b = Convert.ToDouble(y); double delta = Math.Abs(a - b); if (delta < _epsilon) { return 0; } return a.CompareTo(b); } } public void assertAllCloseAccordingToType( double[,] expected, T[,] given, double eps = 1e-6, float float_eps = 1e-6f) { Assert.AreEqual(expected.GetLength(0), given.GetLength(0)); Assert.AreEqual(expected.GetLength(1), given.GetLength(1)); var flattenGiven = given.Cast().ToArray(); assertAllCloseAccordingToType(expected, flattenGiven, eps, float_eps); } public void assertAllCloseAccordingToType( ICollection expected, ICollection given, double eps = 1e-6, float float_eps = 1e-6f) { // TODO: check if any of arguments is not double and change toletance // remove givenAsDouble and cast expected instead var givenAsDouble = given.Select(x => Convert.ToDouble(x)).ToArray(); CollectionAssert.AreEqual(expected, givenAsDouble, new CollectionComparer(eps)); } public void assertProtoEquals(object toProto, object o) { throw new NotImplementedException(); } #endregion #region tensor evaluation and test session private Session? _cached_session = null; private Graph? _cached_graph = null; private object? _cached_config = null; private bool _cached_force_gpu = false; private void _ClearCachedSession() { if (self._cached_session != null) { self._cached_session.Dispose(); self._cached_session = null; } } //protected object _eval_helper(Tensor[] tensors) //{ // if (tensors == null) // return null; // return nest.map_structure(self._eval_tensor, tensors); //} protected object? _eval_tensor(object tensor) { if (tensor == null) return None; //else if (callable(tensor)) // return self._eval_helper(tensor()) else { try { //TODO: // if sparse_tensor.is_sparse(tensor): // return sparse_tensor.SparseTensorValue(tensor.indices, tensor.values, // tensor.dense_shape) //return (tensor as Tensor).numpy(); } catch (Exception) { throw new ValueError("Unsupported type: " + tensor.GetType()); } return null; } } /// /// This function is used in many original tensorflow unit tests to evaluate tensors /// in a test session with special settings (for instance constant folding off) /// /// public T evaluate(Tensor tensor) { object? result = null; // if context.executing_eagerly(): // return self._eval_helper(tensors) // else: { var sess = tf.get_default_session(); var ndarray = tensor.eval(sess); if (typeof(T) == typeof(int)) { int i = ndarray; result = i; } else if (typeof(T) == typeof(float)) { float f = ndarray; result = f; } else if (typeof(T) == typeof(double)) { double d = ndarray; result = d; } else if ( typeof(T) == typeof(double[]) || typeof(T) == typeof(double[,])) { result = ndarray.ToMultiDimArray(); } else if (typeof(T) == typeof(float[]) || typeof(T) == typeof(float[,])) { result = ndarray.ToMultiDimArray(); } else if (typeof(T) == typeof(int[]) || typeof(T) == typeof(int[,])) { result = ndarray.ToMultiDimArray(); } else { result = ndarray; } return (T)result; } } ///Returns a TensorFlow Session for use in executing tests. public Session? cached_session( Graph? graph = null, object? config = null, bool use_gpu = false, bool force_gpu = false) { // This method behaves differently than self.session(): for performance reasons // `cached_session` will by default reuse the same session within the same // test.The session returned by this function will only be closed at the end // of the test(in the TearDown function). // Use the `use_gpu` and `force_gpu` options to control where ops are run.If // `force_gpu` is True, all ops are pinned to `/ device:GPU:0`. Otherwise, if // `use_gpu` is True, TensorFlow tries to run as many ops on the GPU as // possible.If both `force_gpu and `use_gpu` are False, all ops are pinned to // the CPU. // Example: // python // class MyOperatorTest(test_util.TensorFlowTestCase) : // def testMyOperator(self): // with self.cached_session() as sess: // valid_input = [1.0, 2.0, 3.0, 4.0, 5.0] // result = MyOperator(valid_input).eval() // self.assertEqual(result, [1.0, 2.0, 3.0, 5.0, 8.0] // invalid_input = [-1.0, 2.0, 7.0] // with self.assertRaisesOpError("negative input not supported"): // MyOperator(invalid_input).eval() // Args: // graph: Optional graph to use during the returned session. // config: An optional config_pb2.ConfigProto to use to configure the // session. // use_gpu: If True, attempt to run as many ops as possible on GPU. // force_gpu: If True, pin all ops to `/device:GPU:0`. // Yields: // A Session object that should be used as a context manager to surround // the graph building and execution code in a test case. // TODO: // if context.executing_eagerly(): // return self._eval_helper(tensors) // else: { var sess = self._get_cached_session( graph, config, force_gpu, crash_if_inconsistent_args: true); using var cached = self._constrain_devices_and_set_default(sess, use_gpu, force_gpu); return cached; } } //Returns a TensorFlow Session for use in executing tests. public Session session(Graph? graph = null, object? config = null, bool use_gpu = false, bool force_gpu = false) { //Note that this will set this session and the graph as global defaults. //Use the `use_gpu` and `force_gpu` options to control where ops are run.If //`force_gpu` is True, all ops are pinned to `/device:GPU:0`. Otherwise, if //`use_gpu` is True, TensorFlow tries to run as many ops on the GPU as //possible.If both `force_gpu and `use_gpu` are False, all ops are pinned to //the CPU. //Example: //```python //class MyOperatorTest(test_util.TensorFlowTestCase): // def testMyOperator(self): // with self.session(use_gpu= True): // valid_input = [1.0, 2.0, 3.0, 4.0, 5.0] // result = MyOperator(valid_input).eval() // self.assertEqual(result, [1.0, 2.0, 3.0, 5.0, 8.0] // invalid_input = [-1.0, 2.0, 7.0] // with self.assertRaisesOpError("negative input not supported"): // MyOperator(invalid_input).eval() //``` //Args: // graph: Optional graph to use during the returned session. // config: An optional config_pb2.ConfigProto to use to configure the // session. // use_gpu: If True, attempt to run as many ops as possible on GPU. // force_gpu: If True, pin all ops to `/device:GPU:0`. //Yields: // A Session object that should be used as a context manager to surround // the graph building and execution code in a test case. Session? s = null; //if (context.executing_eagerly()) // yield None //else //{ s = self._create_session(graph, config, force_gpu); //} return s.as_default(); } private Session? _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) { // Set the session and its graph to global default and constrain devices.""" if (tf.executing_eagerly()) return null; else { sess.graph.as_default(); sess.as_default(); { if (force_gpu) { // TODO: // Use the name of an actual device if one is detected, or // '/device:GPU:0' otherwise /* var gpu_name = gpu_device_name(); if (!gpu_name) gpu_name = "/device:GPU:0" using (sess.graph.device(gpu_name)) { yield return sess; }*/ return sess; } else if (use_gpu) return sess; else using (sess.graph.device("/device:CPU:0")) return sess; } } } // See session() for details. private Session _create_session(Graph? graph, object? cfg, bool forceGpu) { var prepare_config = new Func((config) => { // """Returns a config for sessions. // Args: // config: An optional config_pb2.ConfigProto to use to configure the // session. // Returns: // A config_pb2.ConfigProto object. //TODO: config // # use_gpu=False. Currently many tests rely on the fact that any device // # will be used even when a specific device is supposed to be used. // allow_soft_placement = not force_gpu // if config is None: // config = config_pb2.ConfigProto() // config.allow_soft_placement = allow_soft_placement // config.gpu_options.per_process_gpu_memory_fraction = 0.3 // elif not allow_soft_placement and config.allow_soft_placement: // config_copy = config_pb2.ConfigProto() // config_copy.CopyFrom(config) // config = config_copy // config.allow_soft_placement = False // # Don't perform optimizations for tests so we don't inadvertently run // # gpu ops on cpu // config.graph_options.optimizer_options.opt_level = -1 // # Disable Grappler constant folding since some tests & benchmarks // # use constant input and become meaningless after constant folding. // # DO NOT DISABLE GRAPPLER OPTIMIZERS WITHOUT CONSULTING WITH THE // # GRAPPLER TEAM. // config.graph_options.rewrite_options.constant_folding = ( // rewriter_config_pb2.RewriterConfig.OFF) // config.graph_options.rewrite_options.pin_to_host_optimization = ( // rewriter_config_pb2.RewriterConfig.OFF) return config; }); //TODO: use this instead of normal session //return new ErrorLoggingSession(graph = graph, config = prepare_config(config)) return new Session(graph);//, config = prepare_config(config)) } private Session _get_cached_session( Graph? graph = null, object? config = null, bool force_gpu = false, bool crash_if_inconsistent_args = true) { // See cached_session() for documentation. if (self._cached_session == null) { var sess = self._create_session(graph, config, force_gpu); self._cached_session = sess; self._cached_graph = graph; self._cached_config = config; self._cached_force_gpu = force_gpu; return sess; } else { if (crash_if_inconsistent_args && self._cached_graph != null && !self._cached_graph.Equals(graph)) throw new ValueError(@"The graph used to get the cached session is different than the one that was used to create the session. Maybe create a new session with self.session()"); if (crash_if_inconsistent_args && self._cached_config != null && !self._cached_config.Equals(config)) { throw new ValueError(@"The config used to get the cached session is different than the one that was used to create the session. Maybe create a new session with self.session()"); } if (crash_if_inconsistent_args && !self._cached_force_gpu.Equals(force_gpu)) { throw new ValueError(@"The force_gpu value used to get the cached session is different than the one that was used to create the session. Maybe create a new session with self.session()"); } return self._cached_session; } } [TestCleanup] public void Cleanup() { _ClearCachedSession(); } #endregion public void AssetSequenceEqual(T[] a, T[] b) { Assert.IsTrue(Enumerable.SequenceEqual(a, b)); } } }