From a36d0602e2b21c0e860d9a989238f95047c987ef Mon Sep 17 00:00:00 2001 From: Kerry Jiang Date: Wed, 7 Aug 2019 00:53:07 -0700 Subject: [PATCH 01/14] fixed issues about displaying progress in console when download/unzip resources --- src/TensorFlowHub/MnistModelLoader.cs | 21 +++---- src/TensorFlowHub/Utils.cs | 57 +++++++++++++------ .../BasicModels/KMeansClustering.cs | 3 +- .../BasicModels/LogisticRegression.cs | 2 +- .../BasicModels/NearestNeighbor.cs | 2 +- .../ImageProcessing/DigitRecognitionCNN.cs | 2 +- .../ImageProcessing/DigitRecognitionNN.cs | 2 +- .../ImageProcessing/DigitRecognitionRNN.cs | 2 +- 8 files changed, 57 insertions(+), 34 deletions(-) diff --git a/src/TensorFlowHub/MnistModelLoader.cs b/src/TensorFlowHub/MnistModelLoader.cs index 121c0961..3a9fabb2 100644 --- a/src/TensorFlowHub/MnistModelLoader.cs +++ b/src/TensorFlowHub/MnistModelLoader.cs @@ -15,14 +15,15 @@ namespace Tensorflow.Hub private const string TEST_IMAGES = "t10k-images-idx3-ubyte.gz"; private const string TEST_LABELS = "t10k-labels-idx1-ubyte.gz"; - public static async Task> LoadAsync(string trainDir, bool oneHot = false, int? trainSize = null, int? validationSize = null, int? testSize = null) + public static async Task> LoadAsync(string trainDir, bool oneHot = false, int? trainSize = null, int? validationSize = null, int? testSize = null, bool showProgressInConsole = false) { var loader = new MnistModelLoader(); var setting = new ModelLoadSetting { TrainDir = trainDir, - OneHot = oneHot + OneHot = oneHot, + ShowProgressInConsole = showProgressInConsole }; if (trainSize.HasValue) @@ -48,37 +49,37 @@ namespace Tensorflow.Hub sourceUrl = DEFAULT_SOURCE_URL; // load train images - await this.DownloadAsync(sourceUrl + TRAIN_IMAGES, setting.TrainDir, TRAIN_IMAGES) + await this.DownloadAsync(sourceUrl + TRAIN_IMAGES, setting.TrainDir, TRAIN_IMAGES, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); - await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_IMAGES), setting.TrainDir) + await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_IMAGES), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); var trainImages = ExtractImages(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TRAIN_IMAGES)), limit: setting.TrainSize); // load train labels - await this.DownloadAsync(sourceUrl + TRAIN_LABELS, setting.TrainDir, TRAIN_LABELS) + await this.DownloadAsync(sourceUrl + TRAIN_LABELS, setting.TrainDir, TRAIN_LABELS, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); - await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_LABELS), setting.TrainDir) + await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_LABELS), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); var trainLabels = ExtractLabels(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TRAIN_LABELS)), one_hot: setting.OneHot, limit: setting.TrainSize); // load test images - await this.DownloadAsync(sourceUrl + TEST_IMAGES, setting.TrainDir, TEST_IMAGES) + await this.DownloadAsync(sourceUrl + TEST_IMAGES, setting.TrainDir, TEST_IMAGES, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); - await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_IMAGES), setting.TrainDir) + await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_IMAGES), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); var testImages = ExtractImages(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TEST_IMAGES)), limit: setting.TestSize); // load test labels - await this.DownloadAsync(sourceUrl + TEST_LABELS, setting.TrainDir, TEST_LABELS) + await this.DownloadAsync(sourceUrl + TEST_LABELS, setting.TrainDir, TEST_LABELS, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); - await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_LABELS), setting.TrainDir) + await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_LABELS), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole) .ShowProgressInConsole(setting.ShowProgressInConsole); var testLabels = ExtractLabels(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TEST_LABELS)), one_hot: setting.OneHot, limit: setting.TestSize); diff --git a/src/TensorFlowHub/Utils.cs b/src/TensorFlowHub/Utils.cs index 3245071f..46f94f35 100644 --- a/src/TensorFlowHub/Utils.cs +++ b/src/TensorFlowHub/Utils.cs @@ -19,7 +19,7 @@ namespace Tensorflow.Hub await modelLoader.DownloadAsync(url, dir, fileName); } - public static async Task DownloadAsync(this IModelLoader modelLoader, string url, string dirSaveTo, string fileName) + public static async Task DownloadAsync(this IModelLoader modelLoader, string url, string dirSaveTo, string fileName, bool showProgressInConsole = false) where TDataSet : IDataSet { if (!Path.IsPathRooted(dirSaveTo)) @@ -27,18 +27,30 @@ namespace Tensorflow.Hub var fileSaveTo = Path.Combine(dirSaveTo, fileName); + if (showProgressInConsole) + { + Console.WriteLine($"Downloading {fileName}"); + } + if (File.Exists(fileSaveTo)) + { + if (showProgressInConsole) + { + Console.WriteLine($"The file {fileName} already exists"); + } + return; + } Directory.CreateDirectory(dirSaveTo); using (var wc = new WebClient()) { - await wc.DownloadFileTaskAsync(url, fileSaveTo); + await wc.DownloadFileTaskAsync(url, fileSaveTo).ConfigureAwait(false); } } - public static async Task UnzipAsync(this IModelLoader modelLoader, string zipFile, string saveTo) + public static async Task UnzipAsync(this IModelLoader modelLoader, string zipFile, string saveTo, bool showProgressInConsole = false) where TDataSet : IDataSet { if (!Path.IsPathRooted(saveTo)) @@ -49,67 +61,76 @@ namespace Tensorflow.Hub if (!Path.IsPathRooted(zipFile)) zipFile = Path.Combine(AppContext.BaseDirectory, zipFile); - var destFilePath = Path.Combine(saveTo, Path.GetFileNameWithoutExtension(zipFile)); + var destFileName = Path.GetFileNameWithoutExtension(zipFile); + var destFilePath = Path.Combine(saveTo, destFileName); + + if (showProgressInConsole) + Console.WriteLine($"Unzippinng {Path.GetFileName(zipFile)}"); if (File.Exists(destFilePath)) - File.Delete(destFilePath); + { + if (showProgressInConsole) + Console.WriteLine($"The file {destFileName} already exists"); + } using (GZipStream unzipStream = new GZipStream(File.OpenRead(zipFile), CompressionMode.Decompress)) { using (var destStream = File.Create(destFilePath)) { - await unzipStream.CopyToAsync(destStream); - await destStream.FlushAsync(); + await unzipStream.CopyToAsync(destStream).ConfigureAwait(false); + await destStream.FlushAsync().ConfigureAwait(false); destStream.Close(); } unzipStream.Close(); } - } - - public static async Task ShowProgressInConsole(this Task task) - { - await ShowProgressInConsole(task, true); - } + } public static async Task ShowProgressInConsole(this Task task, bool enable) { if (!enable) { await task; + return; } var cts = new CancellationTokenSource(); + var showProgressTask = ShowProgressInConsole(cts); try - { + { await task; } finally { - cts.Cancel(); + cts.Cancel(); } + + await showProgressTask; + Console.WriteLine("Done."); } private static async Task ShowProgressInConsole(CancellationTokenSource cts) { var cols = 0; + await Task.Delay(1000); + while (!cts.IsCancellationRequested) { await Task.Delay(1000); Console.Write("."); cols++; - if (cols >= 50) + if (cols % 50 == 0) { - cols = 0; Console.WriteLine(); } } - Console.WriteLine(); + if (cols > 0) + Console.WriteLine(); } } } diff --git a/test/TensorFlowNET.Examples/BasicModels/KMeansClustering.cs b/test/TensorFlowNET.Examples/BasicModels/KMeansClustering.cs index 7bacb28d..9221d68c 100644 --- a/test/TensorFlowNET.Examples/BasicModels/KMeansClustering.cs +++ b/test/TensorFlowNET.Examples/BasicModels/KMeansClustering.cs @@ -70,7 +70,8 @@ namespace TensorFlowNET.Examples OneHot = true, TrainSize = train_size, ValidationSize = validation_size, - TestSize = test_size + TestSize = test_size, + ShowProgressInConsole = true }; mnist = loader.LoadAsync(setting).Result; diff --git a/test/TensorFlowNET.Examples/BasicModels/LogisticRegression.cs b/test/TensorFlowNET.Examples/BasicModels/LogisticRegression.cs index ca691d40..263023ef 100644 --- a/test/TensorFlowNET.Examples/BasicModels/LogisticRegression.cs +++ b/test/TensorFlowNET.Examples/BasicModels/LogisticRegression.cs @@ -124,7 +124,7 @@ namespace TensorFlowNET.Examples public void PrepareData() { - mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: train_size, validationSize: validation_size, testSize: test_size).Result; + mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: train_size, validationSize: validation_size, testSize: test_size, showProgressInConsole: true).Result; } public void SaveModel(Session sess) diff --git a/test/TensorFlowNET.Examples/BasicModels/NearestNeighbor.cs b/test/TensorFlowNET.Examples/BasicModels/NearestNeighbor.cs index 8f761d00..7ae34364 100644 --- a/test/TensorFlowNET.Examples/BasicModels/NearestNeighbor.cs +++ b/test/TensorFlowNET.Examples/BasicModels/NearestNeighbor.cs @@ -84,7 +84,7 @@ namespace TensorFlowNET.Examples public void PrepareData() { - mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: TrainSize, validationSize: ValidationSize, testSize: TestSize).Result; + mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: TrainSize, validationSize: ValidationSize, testSize: TestSize, showProgressInConsole: true).Result; // In this example, we limit mnist data (Xtr, Ytr) = mnist.Train.GetNextBatch(TrainSize == null ? 5000 : TrainSize.Value / 100); // 5000 for training (nn candidates) (Xte, Yte) = mnist.Test.GetNextBatch(TestSize == null ? 200 : TestSize.Value / 100); // 200 for testing diff --git a/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionCNN.cs b/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionCNN.cs index 4b882a1a..dd2cc756 100644 --- a/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionCNN.cs +++ b/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionCNN.cs @@ -310,7 +310,7 @@ namespace TensorFlowNET.Examples public void PrepareData() { - mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true).Result; + mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, showProgressInConsole: true).Result; (x_train, y_train) = Reformat(mnist.Train.Data, mnist.Train.Labels); (x_valid, y_valid) = Reformat(mnist.Validation.Data, mnist.Validation.Labels); (x_test, y_test) = Reformat(mnist.Test.Data, mnist.Test.Labels); diff --git a/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionNN.cs b/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionNN.cs index 02feecb9..49bbc680 100644 --- a/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionNN.cs +++ b/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionNN.cs @@ -121,7 +121,7 @@ namespace TensorFlowNET.Examples public void PrepareData() { - mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true).Result; + mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, showProgressInConsole: true).Result; } public void Train(Session sess) diff --git a/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionRNN.cs b/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionRNN.cs index b91a19ca..07df8e6a 100644 --- a/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionRNN.cs +++ b/test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionRNN.cs @@ -143,7 +143,7 @@ namespace TensorFlowNET.Examples public void PrepareData() { - mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true).Result; + mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, showProgressInConsole: true).Result; (x_train, y_train) = (mnist.Train.Data, mnist.Train.Labels); (x_valid, y_valid) = (mnist.Validation.Data, mnist.Validation.Labels); (x_test, y_test) = (mnist.Test.Data, mnist.Test.Labels); From 0e99b605a5273554832031400a5033baed1d8537 Mon Sep 17 00:00:00 2001 From: Antonio Cifonelli Date: Wed, 7 Aug 2019 13:36:35 +0200 Subject: [PATCH 02/14] Adding `logical_or` operator (#344) Relative unit test in `OperationTest`. --- src/TensorFlowNET.Core/APIs/tf.math.cs | 3 +++ src/TensorFlowNET.Core/Operations/gen_math_ops.cs | 7 +++++++ test/TensorFlowNET.UnitTest/OperationsTest.cs | 9 +++++++++ 3 files changed, 19 insertions(+) diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index b787bf1d..e76c1e4b 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -195,6 +195,9 @@ namespace Tensorflow public static Tensor logical_not(Tensor x, string name = null) => gen_math_ops.logical_not(x, name); + public static Tensor logical_or(Tensor x, Tensor y, string name = null) + => gen_math_ops.logical_or(x, y, name); + /// /// Clips tensor values to a specified min and max. /// diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs index c3b30d8f..b526c1d8 100644 --- a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs @@ -364,6 +364,13 @@ namespace Tensorflow return _op.outputs[0]; } + public static Tensor logical_or(Tensor x, Tensor y, string name = null) + { + var _op = _op_def_lib._apply_op_helper("LogicalOr", name, args: new { x, y }); + + return _op.outputs[0]; + } + public static Tensor squared_difference(Tensor x, Tensor y, string name = null) { var _op = _op_def_lib._apply_op_helper("SquaredDifference", name, args: new { x, y, name }); diff --git a/test/TensorFlowNET.UnitTest/OperationsTest.cs b/test/TensorFlowNET.UnitTest/OperationsTest.cs index 68c44831..542505a3 100644 --- a/test/TensorFlowNET.UnitTest/OperationsTest.cs +++ b/test/TensorFlowNET.UnitTest/OperationsTest.cs @@ -153,6 +153,15 @@ namespace TensorFlowNET.UnitTest var o = sess.run(d); Assert.IsTrue(o.array_equal(check)); } + + d = tf.cast(tf.logical_or(b, c), tf.int32); + check = np.array(new[] { 1, 1, 1, 1, 1, 1, 1, 1 }); + + using (var sess = tf.Session()) + { + var o = sess.run(d); + Assert.IsTrue(o.array_equal(check)); + } } [TestMethod] From 0d6b287955630df34c81fe94ab92dcfa81192373 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Thu, 8 Aug 2019 07:32:21 -0500 Subject: [PATCH 03/14] Allocate tensor without memory copy --- .../Tensors/Tensor.Creation.cs | 144 ++++++++++++++++-- 1 file changed, 135 insertions(+), 9 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs index a104f066..896d6d4d 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs @@ -506,7 +506,7 @@ namespace Tensorflow IsMemoryOwner = true; } - private unsafe IntPtr Allocate(NDArray nd, TF_DataType? tensorDType = null) + private unsafe IntPtr AllocateWithMemoryCopy(NDArray nd, TF_DataType? tensorDType = null) { IntPtr dotHandle = IntPtr.Zero; int buffersize = 0; @@ -520,30 +520,30 @@ namespace Tensorflow var dataType = ToTFDataType(nd.dtype); // shape var dims = nd.shape.Select(x => (long)x).ToArray(); - var nd1 = nd.ravel(); + // var nd1 = nd.ravel(); switch (nd.dtype.Name) { case "Boolean": - var boolVals = Array.ConvertAll(nd1.Data(), x => Convert.ToByte(x)); + var boolVals = Array.ConvertAll(nd.Data(), x => Convert.ToByte(x)); Marshal.Copy(boolVals, 0, dotHandle, nd.size); break; case "Int16": - Marshal.Copy(nd1.Data(), 0, dotHandle, nd.size); + Marshal.Copy(nd.Data(), 0, dotHandle, nd.size); break; case "Int32": - Marshal.Copy(nd1.Data(), 0, dotHandle, nd.size); + Marshal.Copy(nd.Data(), 0, dotHandle, nd.size); break; case "Int64": - Marshal.Copy(nd1.Data(), 0, dotHandle, nd.size); + Marshal.Copy(nd.Data(), 0, dotHandle, nd.size); break; case "Single": - Marshal.Copy(nd1.Data(), 0, dotHandle, nd.size); + Marshal.Copy(nd.Data(), 0, dotHandle, nd.size); break; case "Double": - Marshal.Copy(nd1.Data(), 0, dotHandle, nd.size); + Marshal.Copy(nd.Data(), 0, dotHandle, nd.size); break; case "Byte": - Marshal.Copy(nd1.Data(), 0, dotHandle, nd.size); + Marshal.Copy(nd.Data(), 0, dotHandle, nd.size); break; case "String": return new Tensor(UTF8Encoding.UTF8.GetBytes(nd.Data(0)), TF_DataType.TF_STRING); @@ -559,6 +559,132 @@ namespace Tensorflow ref _deallocatorArgs); return tfHandle; + } + + private unsafe IntPtr Allocate(NDArray nd, TF_DataType? tensorDType = null) + { + IntPtr dotHandle = IntPtr.Zero; + IntPtr tfHandle = IntPtr.Zero; + int buffersize = nd.size * nd.dtypesize; + + var dataType = ToTFDataType(nd.dtype); + // shape + var dims = nd.shape.Select(x => (long)x).ToArray(); + switch (nd.dtype.Name) + { + case "Boolean": + { + var boolVals = Array.ConvertAll(nd.Data(), x => Convert.ToByte(x)); + var array = nd.Data(); + fixed (byte* h = &array[0]) + { + tfHandle = c_api.TF_NewTensor(dataType, + dims, + dims.Length, + new IntPtr(h), + (UIntPtr)buffersize, + _nothingDeallocator, + ref _deallocatorArgs); + } + } + break; + case "Int16": + { + var array = nd.Data(); + fixed (short* h = &array[0]) + { + tfHandle = c_api.TF_NewTensor(dataType, + dims, + dims.Length, + new IntPtr(h), + (UIntPtr)buffersize, + _nothingDeallocator, + ref _deallocatorArgs); + } + } + break; + case "Int32": + { + var array = nd.Data(); + fixed (int* h = &array[0]) + { + tfHandle = c_api.TF_NewTensor(dataType, + dims, + dims.Length, + new IntPtr(h), + (UIntPtr)buffersize, + _nothingDeallocator, + ref _deallocatorArgs); + } + } + break; + case "Int64": + { + var array = nd.Data(); + fixed (long* h = &array[0]) + { + tfHandle = c_api.TF_NewTensor(dataType, + dims, + dims.Length, + new IntPtr(h), + (UIntPtr)buffersize, + _nothingDeallocator, + ref _deallocatorArgs); + } + } + break; + case "Single": + { + var array = nd.Data(); + fixed (float* h = &array[0]) + { + tfHandle = c_api.TF_NewTensor(dataType, + dims, + dims.Length, + new IntPtr(h), + (UIntPtr)buffersize, + _nothingDeallocator, + ref _deallocatorArgs); + } + } + break; + case "Double": + { + var array = nd.Data(); + fixed (double* h = &array[0]) + { + tfHandle = c_api.TF_NewTensor(dataType, + dims, + dims.Length, + new IntPtr(h), + (UIntPtr)buffersize, + _nothingDeallocator, + ref _deallocatorArgs); + } + } + break; + case "Byte": + { + var array = nd.Data(); + fixed (byte* h = &array[0]) + { + tfHandle = c_api.TF_NewTensor(dataType, + dims, + dims.Length, + new IntPtr(h), + (UIntPtr)buffersize, + _nothingDeallocator, + ref _deallocatorArgs); + } + } + break; + case "String": + return new Tensor(UTF8Encoding.UTF8.GetBytes(nd.Data(0)), TF_DataType.TF_STRING); + default: + throw new NotImplementedException($"Marshal.Copy failed for {nd.dtype.Name}."); + } + + return tfHandle; } public unsafe Tensor(byte[][] buffer, long[] shape) From 03c1637748ce79fb93222d6c0b7fa732e74f473f Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Thu, 8 Aug 2019 17:30:31 -0500 Subject: [PATCH 04/14] remove ScopedTFGraph destructor. no carsh occurs any more. --- src/TensorFlowNET.Core/Framework/Models/ScopedTFGraph.cs | 5 ----- src/TensorFlowNET.Core/Graphs/Graph.cs | 1 - 2 files changed, 6 deletions(-) diff --git a/src/TensorFlowNET.Core/Framework/Models/ScopedTFGraph.cs b/src/TensorFlowNET.Core/Framework/Models/ScopedTFGraph.cs index badf52a8..d6d24875 100644 --- a/src/TensorFlowNET.Core/Framework/Models/ScopedTFGraph.cs +++ b/src/TensorFlowNET.Core/Framework/Models/ScopedTFGraph.cs @@ -6,10 +6,5 @@ { } - - ~ScopedTFGraph() - { - base.Dispose(); - } } } diff --git a/src/TensorFlowNET.Core/Graphs/Graph.cs b/src/TensorFlowNET.Core/Graphs/Graph.cs index 2949b5d4..6656862f 100644 --- a/src/TensorFlowNET.Core/Graphs/Graph.cs +++ b/src/TensorFlowNET.Core/Graphs/Graph.cs @@ -445,7 +445,6 @@ namespace Tensorflow protected override void DisposeUnManagedState(IntPtr handle) { - Console.WriteLine($"Destroy graph {handle}"); c_api.TF_DeleteGraph(handle); } From 924e1592afb68d88c5ea3e1ba57eb8b29123b771 Mon Sep 17 00:00:00 2001 From: Antonio Cifonelli Date: Fri, 9 Aug 2019 13:50:29 +0200 Subject: [PATCH 05/14] Adding `logical_xor` operator (#346) Relative unit test in `OperationTest`. --- src/TensorFlowNET.Core/APIs/tf.math.cs | 3 +++ src/TensorFlowNET.Core/Operations/gen_math_ops.cs | 8 ++++++++ test/TensorFlowNET.UnitTest/OperationsTest.cs | 9 +++++++++ 3 files changed, 20 insertions(+) diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index e76c1e4b..a8604483 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -198,6 +198,9 @@ namespace Tensorflow public static Tensor logical_or(Tensor x, Tensor y, string name = null) => gen_math_ops.logical_or(x, y, name); + public static Tensor logical_xor(Tensor x, Tensor y, string name = "LogicalXor") + => gen_math_ops.logical_xor(x, y, name); + /// /// Clips tensor values to a specified min and max. /// diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs index b526c1d8..5c3bcf72 100644 --- a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs @@ -371,6 +371,14 @@ namespace Tensorflow return _op.outputs[0]; } + public static Tensor logical_xor(Tensor x, Tensor y, string name = "LogicalXor") + { + return logical_and( + logical_or(x, y), + logical_not(logical_and(x, y)), + name); + } + public static Tensor squared_difference(Tensor x, Tensor y, string name = null) { var _op = _op_def_lib._apply_op_helper("SquaredDifference", name, args: new { x, y, name }); diff --git a/test/TensorFlowNET.UnitTest/OperationsTest.cs b/test/TensorFlowNET.UnitTest/OperationsTest.cs index 542505a3..a91e68c5 100644 --- a/test/TensorFlowNET.UnitTest/OperationsTest.cs +++ b/test/TensorFlowNET.UnitTest/OperationsTest.cs @@ -162,6 +162,15 @@ namespace TensorFlowNET.UnitTest var o = sess.run(d); Assert.IsTrue(o.array_equal(check)); } + + d = tf.cast(tf.logical_xor(b, c), tf.int32); + check = np.array(new[] { 1, 1, 1, 1, 1, 1, 1, 1 }); + + using (var sess = tf.Session()) + { + var o = sess.run(d); + Assert.IsTrue(o.array_equal(check)); + } } [TestMethod] From 1252ea8ef8ee4471eb197a22eca979ad4928c197 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Sat, 10 Aug 2019 22:42:42 -0500 Subject: [PATCH 07/14] Fixed crash due to free tensor too early. --- .../Tensors/Tensor.Creation.cs | 120 ++++-------------- src/TensorFlowNET.Core/Tensors/Tensor.cs | 5 + 2 files changed, 29 insertions(+), 96 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs index 896d6d4d..071e7d70 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs @@ -561,122 +561,50 @@ namespace Tensorflow return tfHandle; } + private GCHandle gcHandle; + private IntPtr NewTensor(T[] array, TF_DataType dataType, long[] dims, int len, int element_size) + { + gcHandle = GCHandle.Alloc(array, GCHandleType.Pinned); + IntPtr p = gcHandle.AddrOfPinnedObject(); + return c_api.TF_NewTensor(dataType, + dims, + dims.Length, + p, + (UIntPtr)(len * element_size), + _nothingDeallocator, + ref _deallocatorArgs); + // free gcHandle at DisposeManagedState(), free here causes random crash + // TF_NewTensor already copied data into native, still don't know why we can't free immediately. + } + private unsafe IntPtr Allocate(NDArray nd, TF_DataType? tensorDType = null) { - IntPtr dotHandle = IntPtr.Zero; IntPtr tfHandle = IntPtr.Zero; - int buffersize = nd.size * nd.dtypesize; - var dataType = ToTFDataType(nd.dtype); // shape var dims = nd.shape.Select(x => (long)x).ToArray(); switch (nd.dtype.Name) { case "Boolean": - { - var boolVals = Array.ConvertAll(nd.Data(), x => Convert.ToByte(x)); - var array = nd.Data(); - fixed (byte* h = &array[0]) - { - tfHandle = c_api.TF_NewTensor(dataType, - dims, - dims.Length, - new IntPtr(h), - (UIntPtr)buffersize, - _nothingDeallocator, - ref _deallocatorArgs); - } - } + tfHandle = NewTensor(nd.Data(), dataType, dims, nd.size, nd.dtypesize); break; case "Int16": - { - var array = nd.Data(); - fixed (short* h = &array[0]) - { - tfHandle = c_api.TF_NewTensor(dataType, - dims, - dims.Length, - new IntPtr(h), - (UIntPtr)buffersize, - _nothingDeallocator, - ref _deallocatorArgs); - } - } + tfHandle = NewTensor(nd.Data(), dataType, dims, nd.size, nd.dtypesize); break; case "Int32": - { - var array = nd.Data(); - fixed (int* h = &array[0]) - { - tfHandle = c_api.TF_NewTensor(dataType, - dims, - dims.Length, - new IntPtr(h), - (UIntPtr)buffersize, - _nothingDeallocator, - ref _deallocatorArgs); - } - } + tfHandle = NewTensor(nd.Data(), dataType, dims, nd.size, nd.dtypesize); break; case "Int64": - { - var array = nd.Data(); - fixed (long* h = &array[0]) - { - tfHandle = c_api.TF_NewTensor(dataType, - dims, - dims.Length, - new IntPtr(h), - (UIntPtr)buffersize, - _nothingDeallocator, - ref _deallocatorArgs); - } - } + tfHandle = NewTensor(nd.Data(), dataType, dims, nd.size, nd.dtypesize); break; case "Single": - { - var array = nd.Data(); - fixed (float* h = &array[0]) - { - tfHandle = c_api.TF_NewTensor(dataType, - dims, - dims.Length, - new IntPtr(h), - (UIntPtr)buffersize, - _nothingDeallocator, - ref _deallocatorArgs); - } - } + tfHandle = NewTensor(nd.Data(), dataType, dims, nd.size, nd.dtypesize); break; case "Double": - { - var array = nd.Data(); - fixed (double* h = &array[0]) - { - tfHandle = c_api.TF_NewTensor(dataType, - dims, - dims.Length, - new IntPtr(h), - (UIntPtr)buffersize, - _nothingDeallocator, - ref _deallocatorArgs); - } - } + tfHandle = NewTensor(nd.Data(), dataType, dims, nd.size, nd.dtypesize); break; case "Byte": - { - var array = nd.Data(); - fixed (byte* h = &array[0]) - { - tfHandle = c_api.TF_NewTensor(dataType, - dims, - dims.Length, - new IntPtr(h), - (UIntPtr)buffersize, - _nothingDeallocator, - ref _deallocatorArgs); - } - } + tfHandle = NewTensor(nd.Data(), dataType, dims, nd.size, nd.dtypesize); break; case "String": return new Tensor(UTF8Encoding.UTF8.GetBytes(nd.Data(0)), TF_DataType.TF_STRING); @@ -808,7 +736,7 @@ namespace Tensorflow if (args.deallocator_called || args.gc_handle == IntPtr.Zero) return; // note: since the ptr given to tensorflow is just the addr of the pinned object we can not directly free it! we need to free the gcHandle instead - GCHandle.FromIntPtr(args.gc_handle).Free(); + //args.gchandle.Free(); args.deallocator_called = true; } diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.cs b/src/TensorFlowNET.Core/Tensors/Tensor.cs index 50141be6..801ab233 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.cs @@ -392,6 +392,11 @@ namespace Tensorflow return $"tf.Tensor '{name}' shape=({string.Join(",", shape)}) dtype={dtype}"; } + protected override void DisposeManagedState() + { + if (gcHandle.IsAllocated) + gcHandle.Free(); + } protected override void DisposeUnManagedState(IntPtr handle) { if(handle != IntPtr.Zero) From 24f3020ddf9b63c75a8947752a21ab1f22379983 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Sun, 11 Aug 2019 07:04:56 -0500 Subject: [PATCH 08/14] fix TensorSharp.concatenate --- src/TensorFlowNET.Core/Tensors/TensorShape.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Tensors/TensorShape.cs b/src/TensorFlowNET.Core/Tensors/TensorShape.cs index c19ecae7..2490d6a1 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorShape.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorShape.cs @@ -67,7 +67,16 @@ namespace Tensorflow if (NDim < 0 || other.NDim < 0) return new TensorShape(); else - return new TensorShape(NDim + other.NDim); + { + var concatenate_dims = new int[NDim + other.NDim]; + for (int i = 0; i < NDim; i++) + concatenate_dims[i] = dims[i]; + + for (int i = 0; i < other.NDim; i++) + concatenate_dims[NDim + i] = other.dims[i]; + + return new TensorShape(concatenate_dims); + } } public static implicit operator TensorShape(int[] dims) => new TensorShape(dims); From defe9500b69e2ab6155e2bc6043d809960674db8 Mon Sep 17 00:00:00 2001 From: Meinrad Recheis Date: Sun, 11 Aug 2019 15:18:37 +0200 Subject: [PATCH 09/14] Updated SciSharp STACK badges in Readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9cf23da2..c3d4219f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# TensorFlow.NET ![logo](docs/assets/tf.net.logo.svg) + -TensorFlow.NET (TF.NET) provides a .NET Standard binding for [TensorFlow](https://www.tensorflow.org/). It aims to implement the complete Tensorflow API in CSharp which allows .NET developers to develop, train and deploy Machine Learning models with the cross-platform .NET Standard framework. +**TensorFlow.NET** (TF.NET) provides a .NET Standard binding for [TensorFlow](https://www.tensorflow.org/). It aims to implement the complete Tensorflow API in CSharp which allows .NET developers to develop, train and deploy Machine Learning models with the cross-platform .NET Standard framework. [![Join the chat at https://gitter.im/publiclab/publiclab](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/sci-sharp/community) [![Tensorflow.NET](https://ci.appveyor.com/api/projects/status/wx4td43v2d3f2xj6?svg=true)](https://ci.appveyor.com/project/Haiping-Chen/tensorflow-net) @@ -200,5 +200,6 @@ Scan QR code to join Tencent TIM group: ![SciSharp STACK](docs/TIM.jpg) -![SciSharp](https://avatars3.githubusercontent.com/u/44989469) TensorFlow.NET is a part of [SciSharp STACK](https://scisharp.github.io/SciSharp/) - +TensorFlow.NET is a part of [SciSharp STACK](https://scisharp.github.io/SciSharp/) +
+ From cee7864702ced62db8b737991794c277f450a849 Mon Sep 17 00:00:00 2001 From: Meinrad Recheis Date: Sun, 11 Aug 2019 15:25:23 +0200 Subject: [PATCH 10/14] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c3d4219f..55ecb454 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ ![logo](docs/assets/tf.net.logo.svg) - **TensorFlow.NET** (TF.NET) provides a .NET Standard binding for [TensorFlow](https://www.tensorflow.org/). It aims to implement the complete Tensorflow API in CSharp which allows .NET developers to develop, train and deploy Machine Learning models with the cross-platform .NET Standard framework. @@ -10,7 +9,8 @@ [![Documentation Status](https://readthedocs.org/projects/tensorflownet/badge/?version=latest)](https://tensorflownet.readthedocs.io/en/latest/?badge=latest) [![Badge](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu/#/en_US) -TF.NET is a member project of [SciSharp STACK](https://github.com/SciSharp). +TF.NET is a member project of [SciSharp STACK](https://github.com/SciSharp). + ![tensors_flowing](docs/assets/tensors_flowing.gif) From 42a476d06f0ac1a3cf5b6fba2629590169ea109d Mon Sep 17 00:00:00 2001 From: Meinrad Recheis Date: Sun, 11 Aug 2019 15:30:53 +0200 Subject: [PATCH 11/14] updated logo --- docs/assets/tf.net.logo.png | Bin 8771 -> 27248 bytes docs/assets/tf.net.logo.svg | 293 +++++++++++++++++------------------- 2 files changed, 138 insertions(+), 155 deletions(-) diff --git a/docs/assets/tf.net.logo.png b/docs/assets/tf.net.logo.png index 66a0a160cb6c07765a822d41973c97db6dcdd483..ceebc184d73fa1e9862eade5a287a0cacbc71db7 100644 GIT binary patch literal 27248 zcmeFZWmr{f)CEdOhf24KfC7SmfPjEf(%mWD-3=mwbV#!S>F(~3lJ4&A-Zb2IpX2$y z@BY4jug@d7_S$Q`@0xRtImVd$kdgX~g-(JF2M32GDk3Nc2Zz`KzCK1p1|N}#7O-KT z?D$0$P{BWURQ&+(J({(MsvR60H9YJ;eByIMYw$y2dm$Bjc`HMECmmY@I4373MiWaj zJ3Sq114b)bqr`&`Bye!g;6w#KDL5zX%{zRNznEz`wkoi!$js~rtgh+xmQ4j8h)?Ia zYe`8Z4i31@o$6WQ>RX9$r@qu+zB-U06Z<8TD)Z|ga;?G3-rDD7X5>4Q8~6JesYtuz zisw9o(#fCC?N^oYzdhzheGWbZkVsh^!u`*8dC$;F{`-yEr>9?Fhro`j{Q9!x-xr`h zH+>}P?f3VfG4x0{jK2rv;J<_?_D9`r<(#Ie3RuWQa z2e$#aPb3HLV z*vSf`V%i#BuzOVco~Kv{X=h}Y$ilh&!aDl#;k%esOZz0`i5IEJ<(O6un<-0Fx$0dj zjpLmyT*oU);(xD4@SFOC=K$fbjCnFu-q~|Th{0%3CfR`<7GLUCholPu1DOS0D$>oYjck6dF zVC`vpA_>WMn~lb;iRGfO#eUULYCm^JYQjtL8Yq zrn!OdgVk~xImKyCw0}bpAGRd6te@`8`0N?$9;kD)&C_dA<5pU@G*U*w_L6_Ca~0Fj zQZ}!<<&+=4B?=isJJH#r7ue!#VujmABQA)mj_~MBGe55Sd%iZhm+(dKmJG7VN1fKU zJCn7wf-ckCyCRJVp(|-r7J@i(U1$8mCUoxP+4;Fhj5EQr$3buvJLAEk<-)_y-l2Mz zyXmR27~w0Y;})JY{ddv?aMBHuYO9vD_;gEpxk~#3x&g{y#OT01io%-vA-KT1h5|>) zi=0y=&t_sx+s^(M>bMWgPo=e;f?cWVV=4648$$H6*VXnD{@XFtNH}GAULUPscjm9W z@^$HQ`4TgD`0RzBN5)W3)Y(dU&J|q|c@x8yn)Ok3zP^}i&?IKFc)W~r!C$Bt6&pX8 zfWuMlFia8bnl7pPUwS+VN2*Bx@pk4`Lzte8)7ddk>{04N9&}M^dryhChL?I_L+hZc z&2r=PQ8wS4QrZ>Uv)6IC@tlY*oxjo0A^x2mp2c+~!ZT(lm-uGU#u&7mJxu6te;npfKUYnSWwJavlqrPFXwg;i2o=iU_vI-%OM( zw6w7v_{r+a;D`Y1xk{?g2)1O*2-y~$IkJ2;6Ijhw#-^#uJ-#XV%bB>hRaXb9*7D}{fawm@#6AC+q#2sB*H|)5ky+Xh7;%h7ODx}(Vt!fm& z|M%$``w|Jq9neB^KXoFK=y#qFY5i z22kp06N|+@*@qVi6*uUpCw<=lt$|Secj@mU;bwn`6{dC7NJ+TsAtifKyb|`iqYpir zoi*!`>Tli3p!P6U9_c*q+)68Vs!`gse-q+O$-e4&h5+}gF_G0}7o(3MROmIYC<5S|$>K8op66}pW_g0roAN9$sW)B(>6#Vi-A z`3#}89wx9C-Oi~rb(P-foFW!DI&$E6eNX*2xy7uC-4oe;=}QH-a0%Wl>?a zqPmF@f|pTP3|ssBqT6Efce0KWXLPIgU>4QooB1C!%|}K1DSyw9_5vgg1P*jwEEgIw z$ms(*uf?T1eTSbbCPh(()RyG+(m<*bs8#jDhHEPqvr-M!6aoec##3BzJ>{QA)7*Nz zTcYJBRX{G<#MY>=%lKPjcSN3kA!V67p*5XH^K^52V=~Di^Ujo|eKv|^TMJWbJi9QW zkW*XarS-6(#$YGn5G-WNg;y7c{t4Nm-h$|i&g@xMiN6uW1|llto4s+%nrgbqOf55< zP1lMlmlJ-eKNEcTWEOLeJk=|6g~?KGNH%>qj&f? z)PN^xnnz%t+_W;4acs5TmGW=2-n-+nUfs?ATV*1`%KnY2HWpFu!nID!^`lLcrIiDB zyYII1v=Jahq}6PRkPiEfF*nNCleqb-?{CSKAF0 zI=|NWtc<4}@82bT{}!Ay)kBa--Y~4S_!D`me%KB;Mx-_pnGS!NN6qH$7d+mL3kcMm z!TmD>gr>#d)AhB)#-Fse9!OgE54)KSdyq=~%(W?Qg}=bVHv*~VDM@AFAP9a2rPl*i z1cE2&zORfa%m?2diW$flS3&Vylo@UJnc6?*S2~GgvXUH6+hRUwJEisvbWP9yXq7(K z+bUl+hE%K>td}pJT9{us5?;S(Y)t?zL0Y8pF7A zac?50n-K%m3=XZu6)%iG=y2m&JDewI(0 zkO4@lJ?xLDpd>>6IMRXV>0!o3`yzdxEcfriKLJ<>o22mjQEnsA`N%U39?iE~SmqBp zl=%xczPEb_cp=||TOp#2#W={yMqmTzF1|_dE2nfUZo%h8zPuX=9e*WE1&3_qRMhnT z`)!FZh(d{^rz1J_#uyZz_25x=N56`)ett5?^4OcTexOu;!1U<=_4I#_={JZdf`+h; zhuT{gsPWeX6u2x=MXmbBkrc#&vL1YH-(8e<_<7zaC#UHIl2mR$GY6_^S~%9KQ8#9K zafff-d#31Ch@ig28&A1*oKY}5*0Mv#k8;DtNSjFOnJ<-tZg>p^7mtzd<$~J z%p6Q)cXsmDhe=4zDf(fZhUVWua07whlUPlko`v<|c3BJLoD-JQ>7x$zD)(|+k5c&b zL<|!>*GTpT(!hz*SrV5N>8$-uac5&5F^`2zh?xv#7TOV(un~Drm~c4pLhHceDONdI)GwVD===mRH(Tmv>+%e%9i=+~LFf zTRKHVR=~^bn~6LlAugIq#iPkB`j(V8`5VEl#h&DNLeD`uifJql9n0T#3|08|zIDtZ z;TB(=BPN^B zl(R!xw^pVeHN(xN58&2VMtl4C@k$rS*dqplZg7#Sa;nD%$k@d^*xn11QM>Zj5#!nY{eeB9bk2uDY7`Rf z)3vuk`HcG-{<^OiC(ZCe62?j|!nsPNYm1e~$NXG(%s+fPJgk=`#FMA(PAo0P%mh^{ zM}ZEM0##-!L{JK*|0m+H#&(dVru#wQUi=MQybb))?R;_$BZLqswGn?uY6}gkU`B%! z(Mpssdg2N#Qs1yYi1_&8BTR>C{<7LlNSDaz{`6>GDg1{FuMQS^RfvPN_Z=pj`w>sY2Cx>DNi$nE~vcSVmHpvxQ~Imh?7_#k2eAY0Rh5zS<_qQ7?{B)cBU9Vy2#NF;qeL(cpYG z>Lv42NkfVQUP}pAyO%t)_RD<(cJ4<<98hf8zt0qPCWBVY$a0Ty@Chztea?Pm}|2VUa0UzzLh$< zI$K6hb4*lX+;7l`ZFBy@K(9147y0Mi0!WHGX{05mh?N+ynajt(x zx=JQa{ac!B6!C)3PtgnABRa3;oV>NjNb%ihqa!9qfprB}Y3A=X<2A+wrD^WmcR;BKQBMBEb+jJ=`w<`fV=n$Nk&4z%O0Ku->wmQRg00^Jt&W zyrTi=#V=%MhIOod1aH^m_~tcxr2$29n0wfQE^X8ZM6)!W^`u znz`-`gV{rl?Tg8Vw!W-Mx)7S`fQeq| zr$1MN#Wr0|YkDG$;t~F3UebhfWzkBUylFx5OB?)okTXX2UvtdF88G}s&}KM-Jq;yS z&-K?9oig+EM8iP#&~+Gwv)PB%+`t$^<<`FJKgPegPlzaP1|!eL-j=yll-&tg4Zmgi zPLO>%w5`!PqGSbow*a+b5T6y4Q!{Iq8&;0C$6E&6f*?1vk(df16pWi}U05>-H-A7S z#396VMVxJ3?D})%B@-?W2dnY|wg0qnWc55@?AhTZi_+13Ql=jc?-BCz>V8W{see53 zZ#E#deT{xnzJ1;>NIU8%pwVM%1S$73n!mp{Kmc8ZY4P_$-)*k4#-8K$-B7#Z z9CC!G{HQwkelutTA`*>`%hVgUnR1?+;Fl8dhOjS^bhGD*FHt>LzG0=&PR%WBs zDg;+mOp7MwtprQMZ|3K|i@(R3)>IF_yfqG%nz$2LAd~5C^r*En(TBY$NPhtdxid5^ zpZXDNU`2dex$GPJ|8{Y3TOMg)cM1 zBC-qjFoLC&2~;B0^$J+IbKxr1|M-i+YJcFh_W#*VN3;?^@c`oHnY$?yND{TgE(LeS z80v)ktvMWN|EC$8VTxzK_+K1=Jv`Cfg!qWAF`};7yvynC`Cll2&lM&u!D8_c(B3$< zK9KonWeSl7^bq_}QX%`~-zD8ot5dqA`b(XN_ycU~4j+U^QY-bU>R63>tGPLw)`J_{ zAE)(cshlC1Rt5qf(&T1>*72wVik+(gYuIa){Siuzd-s#3U5ey3gPhBl_L0A-QoT#8 zCxNtp$K8s1V=ZsB){3%RUa+dttp^^vz<$r;#W)T8gfZJ3Yc9Q>5oT-qKb3PJ1zN7V zAZ$Bg3*T)<$Uj2m`zp_B_AVnjlW}a+cKtO;R*tOMhkc0C!rH4-NJ5_mN&nrCw(yTD z+Vwb%($mN&Ni3(vH?t-?JM*c*y%6!$9NZQxbrf7zX;wFX9b73!E;s*Y-*UW#_jg9i zKFhoL%NqJmQdB#k^8XZ{;^~()6Nh$X(&{I?R0tht?+Pj5eFHaj;iix|NI3Q(b!HA@ zJ4piLFpn+46xYiIGHa37WgF6xR7L0HZe99Caqtvkih+4B-C2g!|nV(P^hJ+V_`XN(>s1K zq)IfoUOMVb1WEOddk$K&*ke{P?Rq|VH!9^AC^AmVi~ ze~#SvPnnP=m7~}TTgZRP>Z#a5%@d|XFO93R6ezZfdgGo?>!8E@NxQJiYh7z4TN+84 zV}_AC#H*B)j($Hy>Avs53J+19%mJ$chNMlY(-#8N^0TDEIY8dkT|UbMXn{SQuMYve)bK0WPw5U$4d)%>KN(;9(iD3N__}>h zbI616O%?`07c?H}U|`K6jT(tLMXeqWqwU?=H1`)_ghxe~jhscU@HFAG94Y{pM0B`DmR?D+go54aR=Try2l0{EcnNU+F4ShxGQ zqnb#EjUv6iChxiH75!7JD>Eikyn5<)1|F=3(3YaqEn;u^d#Vua(5!c&?W^ueCbK39 z_{SK!s*-Z;E0v%Bejqv*Zc2n=SQRCHFR#bNX<&=7Lt%_`;F(?cg(hneS4_7SU@0m* zC)N4nGzbwR*`J}Ts-&oe(@a6`Lgcx zsF`Mp^TqSL*qtc2#ZLxrWsL4VB}mEeIB`YC{gPrv^)-1vP`dU>vTtyga`bCDNW2i{LPCCe|9L8m*XqTR> zArDqhii}qCZ?8dm+`8Bo{d%e)n@e8*)Il}p9o=^x4)59s)-B#Fy+SeML-%YJs(%EB z&l#`K`5K_4AWyz2fW zk5^>Z4!I(9W+De6_0lT9m}VHas+OC{6hRS|IV zcW?RXy$gPlB{*R$Q|42@HjM)8jJ2_(DE^kAra*vmS{Q`tY1=WU zCxV7gU_;(l-P@S`v>y7!mD)ird^@%dSN>Rv^utIMkmbzwMwUofTT~~@ubSN++7oct z^@d2dZa(5{Si1Q=AN-a+uK19-V0d>$9L4*OmXpKi>3ZtK5+$DZF)W_>8}hGDU<**Y zHo!s|Je368TPtNz>hf&!5?+ej@+s0`)5k@o(Bb#r z`kmBzXa`aX2a|GVGkb76sdk!$Gd=fbx_|D(*u3K$4X#F zjz+$*HZ+uIZ!_{nC zm&?KIXHeJ8ie)TP*H~#-|GgP0sl#9$z;wXFIGu>m7kgw8k3;ikUbaL#7|p(n9=5ej z?|+>!k0X0F_w!qIXE}O$1z-B`JNTyMz9PE@wll<-0ZqwIQt_khvbL=wrH~Tu$IQUQ!W^pimo#7^QL4TF)kjzO8mqE{8Z4XpCC!UmV{%N zl7^Qt9ItX2>TJAseiX`<>0zwj$tJ~@am_zlMl?p{mm%wWYPQ}zkG$@`7SJ4Wpd=kh}!}L(0_~DQQ zfs;%;bUX6X3mFnn&%CI#Z%MCLR+fV&M8$Qqo^$5bYVf+?dz7{=JfCm{l>X(|Y1?=& zK6vmdxUf3>t|Ptp(Bi?>L@h==gUXO4Hc$5~HJ`ONM=-5E@Fu0)A3C}DQ-2f4tweik zU2YLM7;pA&3orCf>nsWWx_+eRF{aPH>&za>cRt)NlEgRN9ppiq&A7a<1QqOIfcmD6 zW0}%S>&xaKiVCL}RqeJ5d(gPr$!L@)pHpO7LnL~j1F1OG)y}JuAAIv&j%fyAS???t zVaXzBP%#)?S?}qOX*VCeL}@!ms{+Z$;;QAs9Nay-+0w5`{MbcYYFU$$W=gR|RNO;V zUi2j29;Mq=-9{G@;Xl=j;;cVJ_&kxawAX~;DE2QWT6-;{BHA6oQ z6nP&j*w=)SH1k9NeBuZ?nU2cO#(?-&QdKzF0fhGPW5A+j5NnoZ@6z0TP|`eM?{G+V zcYn$LMXFT^jzu3HO)!u94>Kd&H%$Zmv(Zulvm*}rZV8U2--7(FlA(`k{oJoF`E{Y% zC2D;u2RletLl!`Oa>Dv$bAx<|PYtA~B&ry_O_HP>K3X>jC3U4YiJ{dp0;GOaX z(ucKezC&Fasx0n%PXzOQwxJD1S#v?Z)had?0>TKJ*WdT@30pdQ%d7}m%Sw=D$U1UL zc4H9*3%OgnO<=->P_w@)>^7A8AAHIA_MHMqr@8SrHzOVIk*;z@KlVu8fR@^LmJjdM zwjTpoqF83yXi3$X*BXVIy_MWX-Z2P2kQ@QWt=pp4C@S6k%(m4MRNYw6oaT2-J`69= z;E#^tvK%D9RUgbog4|;~k!cfOoGF)&YWhS(Cg@`u<$(}blnIYAKScUaCDBhD(P~1Q zI{N_tAWL$y(Ov99=96H=X*f0(@olW9J_y`jFQgy-^8##6`kG%^a!luPvIo@I_t*Ok zJwzPM%JO6`$%(+ZQYe{fh@{x2^9Jf6qK)2=HGZwz9pQte%LI z^oGnsP*YrS+EaF|n713FgX&qOGBIDF?aJmV+LKTFxKVD&oCw~uh5~nxll!5RP!U!G`##S){svSVbn!v8+ z$Mf~+qu=alAOrm1%HGEBrR*E#vuW!JDYcWblj)-?7~?m(@GDRb#Gp5j(uNDnL^1~z zX}kT=tb)G`$zPNKO%1z{Q|k)u&CnKu7id;4PmjE4hRQyVifLzSmDhc^zsI{?SoF2h zx{MG0-6xhZ8c8sp2;)Ubs<5kOI2#c^5JTtSka-tkL$uBHRtaZcF;?7_oi8b!6J7j~ znfd%pb=_T!_6_8FRp#hW73hmXwHLM8$Blc-%O)}=;%KCwlari`(c&yEmbl~J=Fi6X z?BsyT+1bb*jI1-Uy}YxrFLv49xaH>M1ig2R!+)6n=%{9yIpFewWyTy~?s_4HUD}%) z+fwiTGIauc+3s5X*6kL+D)vT80}W1(kV+r`^aVc$Dfsy8#xhmS($Fp2L9W7N&VkK=|mv_#pyvewu6%he1x-g7EfLoV#RfJ4p#SQ(V=7x zo5o#+J-PMhI8_q$v|3l3jLPP938&@rgrilSJ-e+^_+q)V>Je9~lb{&6s6IaxGxThX zi(|Z6&TKr`%Ytas>ncbwYEhjN@REQIUr@P2`1A(B!WQe53sbr_MPK5sR)yM$TXsp) z8)v<93~&vyMT+eB4n}YMn{dvSDh&o>xUU|)?Y{WcU*l4LGRZ`-VTD0i4PHVYq9l;= zdOp6=g-i2uRqspodqZs zkRsBQ5;s0N-_9+i{*`gWqX`0E_x*Xce~X4*E*9g%W{%tUb6U@M;SG|vc-QKc0hEqW zP&2PAPVj0R?PMcYnu1|;WNGZuK9VU5h=pOpIjO@r$7m<8$_#EV=6c35oeNO2gS25x z&w!Fy*@p7btqZe6Eq&u2B#1*qKy+(bx>Bg-a;{~przz!IpW3v8k}u77QO|e>J-PTY zBQ1TLVv}NfCuf?ui1?Es$m9Cd-`KABDse>pwXok>4c-WD`K#{NAGI@p+n~LF7XJ{A z34#Bh>5vy>4KMBE58Aw^zT?VKm&i9qzx^*3Z+TyZq`k_}eDgXZCNVi^IA>{Z<6C_- zy^rMX%JQz(&Tb_a7L2iSnS9*jc~6^ofYanaEq&NC?|igxW3WJDot+owQ5*up%s+4r zyDmc2;5UGe1aQuxA=24zs~@M!wgMy(#&v*HiQ2H63dt_!mi9P#CdtL73;YOl6>QtL z%kM!_usN-}PqSHHs+b@+f>HZ$<0kbfe5rsxg0OCs+3zUyh=& zYpUvD1|K!{Ex-UE=J3$u5q z6BuN6_p{>RM?N8xbZn!R!JG4#Bf8S<;WYpt8|;=D-va+y+x&PMf`tVEP*FfkU@XAL zxY?!5llCqwV%wR14(|A^m?}CF6PtSp<^%d@$J#fR|b3e_$;8v7s$Q~EfH9>oDEQ<;rhX-*l4TcTKMPp-xfxL zwJsdqI563(q!2VfZ|oYJ9+cVZe>9awW(!@=d^j6$4(hQ2Kz$QOb7du>H-pwNGB!f` zljZXwEyfzxnx)0=*gpMTY$UX;w>fOmUeL#Ckj@o5SV5>}2!tUopxSg~ZUM9lih9L9 zrP$$3-O6J334_;fTn{wky}{VQTbYAH1fbr;LpHZqR-p0vkn$Li_H9k;Eer*@u}w91 zR>_eoH^Pb?r`Be4Jo*MLQ*vm8GJ!jz6fhe#4~3;QF107!UbH96V^;YY?B5=T{sFZv zK}xn;G?11DiuZk`jipI2_vn>p{D75=jEN#>Xz>~ynx3&nmZ=<_JgxF<_J3_BJer{= zI-9X@;h+Z;h~J;poeqaW?nbu>(`7L`0O#TweAxHfEBH5%ntT^N!jg&WA356$2R}LW`e>T{C&=`G-fJ*Jp8!c{|HanDS+Vgxe{r!d$K(m%>hh-#1B5Mh zS&x43n&0ZWWiGS}ScR3x-4bjNly0Q=3wl=P_wkhjTSurBZ~Jk-(dBuH8PV-nBc|() zR(bC?HZ?41uj^+IJ$phJ0W{&uHm3+(NMk4m)MjD6@Sjj=m%h1 zw$APsntMGY5V46K2O(>23mHrWO(Sq(hTc5x^DZapH~mFBKc>BWx*yeZR&7@Z5YQF$ zaA8}v%ct-fGhg)&GQu+Vbe*+RyrE)eB=%#Dn_K7; zoey`@0`6CRx|r7|9@UUJ$BCEl-y;o%sziY9gTGI85Cd0Ts)}H~y=di-m)GEN%sYdH znGULU+jUjyq5#h{v|S6sE3dDI>E}zQDusA{6(?M$ zJ@g0S2V#sYrsv#j{xLj>Ct++%M?>7JhjsQP1+cE-y-Jyv8Z#b@p}e=65Bk9F#gg1~ z00MPArJYZ|CrL5#RtXWtL?U>hBjZsZC`p@(a!f|TcPx7WW z-|ddX{dgdfJ+MT}J`AHya&aOia4_%dlB<=~5;& z2Yub5$>k}cL=02R=j>k=K(PUe>`FRr9FH-tmwRnv8zM5NNiD;}pzA#w6c?z-@gAY& ziMZ14_FlQHMoz2So2?Zc(y9`k#oA))e7L-;T^up7oL7yb(W8^j53R*lJGGQOz3=$U z$fka$5EAf)4s=|*4S;0^oU|!RJ;Z@0aWMHh`6{Y;4P+>Ax21p=alU5l?XfybRx$kC zXSBLGOIMMhB8vxm&gNPVOLrRi816l}F-`k@t-|OpuA~Kp6E&F)@;zE4xj7ge)^UEi zOS_G~x#PtDHIZjB-ELpP1Z+(~pNm!QT&sWy2(U-9{ci|y7l?FhtOWyRU@Rr6l>JOd z)7zBYVcD1D(*m+mS5^fI?2O+GE>AnB)NENr!4yBt z>+SR&%lg+O1YYMOQgFheGr1z5_4Xy6Jz~D;yWrdhI5?;r4j*{v0*;Gun)j#1w*(hR zu6aMRCl{PuT@=5E*_c>w;)47!JPr=Eex2sAnMT!E)mX2woF3eKtuULIR(HHwQ(S%2 z3#hWL=9FB0kx^7l&Ru>=R{mnc&V1A)Ut4+CYTq9bcYN9WW?gSUu-BMi+a8tEzse z*jS39YOtEz?xK1HlSrq17gvG7uI8qO*<|Atz>n43G0HLbo>sRS#pJ;+`Pk4vZvuAf zEv+pwPnT0x%|~GQV?Q;#tY=@eV<<42D!((CpRDMX1I#+5)Lm%Y-3IPN;Hy6Rl~Gy( zH3>P2%gy&eH)g%N#}~AVRfB8?Q}`RwuyV^#%`oLqe!U5CRgG(HOmsx#?wdkI z*nFpdj*kxtXv4$OT4D1_9DseA61!hpXW`uYsXBZR9USg|hyi3ggQSkO3Rt1}FdlvrtyM zmJdo^C8Bw8N~{i~H_R&?s8OqdX-D%F+U9bl_0Ouqt+MaTtA-@L;Cda@7U#z+M=uuc zyUP`291l}1sy338=qg+SV$P5f!;)C#fKm5NIrR1yfUO^b89tZs$&V$?0wBvQ-t<@MgGd+xW>_z@B(*8L7CH>Zm!TLB9OthVud>*65Q ze|~+TEg9nF5jV|a8-d}dZV+EW4Ssty-xjh3+ji53i4}exZJzom!$NDAUJVR#eY_KS zu_oHR^(H!oOi5|4_B0)9<~HfVfEn|B#YW9^*9??N8{xz9VSE3;8sQqj$+|Y7e7N3h z9h~ZGPWRI`X+sPgm&-cVo%L2Fm^}4lvEeA{WwT?3-2}`1xsD-ay(4=>hZRHhmB)Zz zjyOTJ@Vi3-{)=JqUiYG)uA*>AjG66+0_h-I>Jg_=o$&3wLvYk@VNI4R7ZXTD#evgB zmAS$C%CasyXM^(@3t&u?>ZKHWd;b^>=Ql^&-wx%u*x&0Xjy-$a7QyL`prJ_|h%Wkr zm!E&k5uIhUz|EEvuXTiwP;M$U!=f!)nsKeyh#-o@>&Cp`4HP<;3%t^7l;(yLEiPnQ zZe)Pf>+BPndtWa$-B~wnQ%peIkE=X{0KVx>l#xsa}MARDPuS7n8+JW!a2_QbWCuIBg%{Vp$ANGdYV} zgKBKJEu^bQ(Wpw<#L%{?-_Xmd(yI1qAETd3-I7iJWR5Z9&44Zwq&{Ko{p{J3M)|o? z=Lye>jy=6bS|@sW>8TTy2ho>5N;hI7g7Ss|VRgxtqh~vp%5ij#!Iub$uk^DBf~xJM z^Q`#!7-*em65&f(aqxus7juWqASu&zsynch-(FpAu8ANG9^1rF7eq9ty7g3=imEU+iStx?IOXU2)Rma_01I)zrEac8#W%>yEWmnf_lXkE*_Su zz$+`@_@=NoGnw4cM&RcKs%e3CZj^(&pVzR^BcaD8*ZjsaOmRRl;S^=Hj{?V%*ce(v z4Gg)4>huOLm6}wfRg48F&5E~uVfB@ELt94y{IK{q!5l#ghmDVKHX zxCzniStEGdq|_MI^Ox*s805{yU@T&Jn8lqZ&g4K$^NO3$>%cbPcI>l!jcTqIJJ2pM zTc&Djo9O4!a-n7vK9rrz5i&F0ydNiP)lmCz*^N9J{c#`pyHXz4ZDE_~$D8z#+v`C# z4is0a#-_{ zc;n--U`_846Ze&Vu9sWeMxXPQqS&D;zOVdWNDg zIa)hNE~5i|WfOp=0Am@ce^#dI_e!RzJz;(TgU4H7@@>cX%r{JPx7=1mk7E*J>EF1j&_-ae$-iP^<(yrqirmS1TO8oW8l(G;2ymkxxU3?l z11I^%1gld1urTT z+}v)BsyXKu{AdYpj6mCf1AGxV(^twTHEi@}pW_?wR3$i=^i4OG;`2Izp&S4ja?Jw_(wNx8 zi4rAzhf-%lx$C)qjhpp06;>U5e7k69JUNK6rxf9Km#x0dRvIMwm3XF})AM{cP7`9C9jR3lYcj#MR<$Zw|E?3* zzuM-~#5#;E#}^CFDM879;zi?wtW)h(?Lah^z*xPeNdt^_0Z8v^sY^2WUW% zji=L1GM_H(2VE#_J1I%!T}3}BN@g@Ps_lQ>_vQuno5>8ZQegZ7t|XE7DH_}F53^W0 zs<>39_{&$j?(lhy42aK#p?4GpRdfyjh0#Ocv_II^ib0KHJLj4v^ENam?T0e*04;{P ztLOQ8KQv|UVp<&NuRutMdJzlYIB1f4KYX#aomGuOQpq13$}?gOwaUGGrrjfPuwVK* zPo#<8wA=EFXu}3le&_cl|2ihW5i}Q*?QSCI z8ox{I2@Dg;S7g0px^p($4d$E8m~s?N<-dYflZ3by)vlNC_f-83HI>g=yHIl3=dU?C z>=N;O`BfJZT8QCvYWUzZeKqR&j)cuOr75<$Y9MtIX>?d$J%BH&rB3<<^;Mw{NMVZ; zFSfn|Y_l)df3vrsQqCcZrG^sk;y6uyx2lk-rL7fneOl*Q8?*Z%R{D1$&w^Q+!;HXm zAP9-)`;z&!1cW=Bb)}|WgibT%W48;d6CP>`by$$hyT!DfioM~ny_g?>;I(LS-LYiw zfX@6(A4(~_hOZ7oWqth--)ZkpKa__fZWhJd^3d8PUfdhY6%kNn$~z6ABH9_7vxJ^a z)A##6)m_W$hdP@%-%1KinGxNyfgSYZ;P>D*a}(~WHmTI;K*-h2pXO4to=-|m^}qU9 zN$?|5lKEZt&;o76plOw1sH;t0!7o!qIWI;Pg;g)|UZZcXS(|DQ)t=LFAQTzD%fQ(e za~yno>pA9p4_`$APFvbDzA~~#Z^P=R-z&-8rVVwKxcZ)4 z^!T1>{g;Mvv|G?SHz!6~#^OA!yy#d1&-lzlrz@x%Si_3x#8n^ZGawhVYXphr$+%Y$ z%+%LT>z~~E1QB?#D^dgMCj(;=&D_<^0^@?qatx$iF{iF!$&Q-D(;uailBVtU zWMc{9UG$EWX>z%!Z1Qlsb3X5be1DA3SH``Z@T`dk)vsf865@Z)<^%YLg$14HX+Woc z)LtVJPu;)c5c(-uDdGJ~|8-xJfh^g)b>I^w;kQer3uleZ+?wC3^~P@!r<$CL73tv# zfzW7kj9J@5{YD&k>8%(0WGk*t#d*O*7RU`x@2#0E+4CzS$_jRR-uzka^S;igDzz-7 z1EU3^(%95`A3q~BFDkGQ*(?&;Q_eg2wRCoM(eWp$Vl`}aCwFK7pcAcXQ-06i-2YLol)VgV3L0;k zi^`Pv*M|%2^vb&i_iTcvleTzDY0%z&Os_gOlW2v^+mANCBgH>~%~pPmcs z&O6S5?CcE?QNIj*U>m5dXUTK>RjV*Qe{m@GC;hq=Vy@jpf@JEO!io1~%Si3Xq}H>g zp}w;vU^s5QiT$HY@5hGzF_N6u<0)@o-h7Z9(g3{+M9l;@8pp~h;lbggc~W(1@)v^r zvKF8zLbt_G1#_E@_Z2B%!e+S2>lGLvp#lDW21xlI-RAl-7Lqf4%^%dKwwQs#0idP} zq@WosMR|Y;z5f~NFr)>Q>W&q@imjJpUP$QbccuO6O1LFKiXN7qx?J<&4neVuZ~mzrA}PQ`D~@0D2+AM6iXvbR zn!%QNBX^tcaUu?6j*RGrT%8I9(!39jf?*vFXLllARue8B3I(Sz!SzYo_ zokiR7ucFaV>q!C{Q{a*c1ap|Z3XdfNcCuz@TR4Er<>*)s!}wnhZSG_NBNKz`S0Kd0 z2Ab9xk$d~*3mHOFA`wZE z{X#UW858qrj> zFJe7rYL{5fBU?`Z~It+%Cw8HeuiWr4Bcm8Nio(OLy+UC4Gf@QIq;QsT(EY*-SE z78y)Ah(=_-oa|jJpP+7mOKe*B1P~WmV|ILx9xxBU^nh#WM*nWV`swBs)Q(3`%IMw0YBbl4agt<#+-DFfpOO+GbT+fz!uM(0r0ptJk=6WNBFIz*jeD|Z6 zB9KNqPrM3OeY`^3JKFua_~P0cdbHn@VGcTK3GC+PqCv>ix zJLzcq$dKBCdbq7%2jb=`E*~!60!9^Re06_$B*ePq2LL;W$G2LfnlrA)pC8@}{f_dm z;-1ivNwlQ}9;xpH^8yl~DRZt>8k@NTj<+@ctl2Ez&y9J0qDvJ@cCY%;1syaUMK(r2 z%Ku5;~6*64fw>N%%^T&cm(;p~$y=z_E8*c+klln$HLcAy!9@3PSSvR`H ztA{jKn^Sge10L3C8(y~<=vK@{xs$1S1xV=|igF%z{G3%JfXScA7*{iO=Q_?7Pq7YA z^J;quZ#j|4tmVMdQ>O)u<^oK2TSvS7vnrY@zA`~sJWpp2Vpfi(Z@@|p#K%A2I`*2P zKcSlCDOrU7q2E_S*5#i}0x9DhYG6boc5YL$QoJ=*`>=d#+4AtV32k zZ8Zdn2wBo8xmy$2Jf!!|G!eMRf#(+<%sw;9i?cXUa&X~TbpghHwa}vY=&;div-B#e z#zszt+%dWt98CJj^ZqwpjEt{n47EFMnv_2SFX!zCJ8C#U z;Dz<4XzV_0{7A?Y1_=<9o?dk-Uj1Xz-X783Ovz@YzL_%?@OKtEJK7(4e4-$>879I) z2JQ}Kw^MuJhPRI)L7^Z$zBL+h1Lpi*&5N@6t(P)C7IqCHDeQvVkh+R6^G4{A47ZO8 z)XN0RSg^K*_Lt=5WAKze4VrzR+Mi;w% z3m+h$j-;zjK90(4iMBCfhLDmK>Kgxw|AbU2{DuQE9@jCb^+^`wxcJ&Za_@m!_ewz^ z#YEUn+q2g;+yQ?Gi==n&NWQu(UU~ja-##s4NUd}F#n=FPJBl*-`cQ%%+&f@8IBLeK z09>z%bfIdy(HJQVABS4L@0n8eA90i8K>je12>yZrOyLv>ZX5u^6~+^6z@b1}1ar~o z;&en`ktjy5=AH@PD($rz^*(ue&z>;nxS`%PB@UZMO8O(TxyEmLzV(!f?4!lJygDw- z=5%@3t8#UKd_A}5o4m@cUeI7J3g*@UPRlHctOcFFH|*hIO-qfdNOaT{b{RH*tWk7XXsIZ$;m%seAS;+ z!bP9w?~RdxhT{4Gdcew%$GVSr=|!T5gr^`Q`1q&pv1?Svd%9@tnsD?V@l3kVw;B2e6e=Njg$FB7AtZnmE*D6{j* zn}};f+6|qS4$APpK)4DU{Qjl$tToGekWpxALGW^G7G7$_FF1h2K)>nrz5dgS@Y!E| z`JCMi3BHso7O@4%dj@9IiXYdMBB1l?)EddcIl%x0PQM0&sFDg3!e zs;#kp>hFp6t(Z!r5P~9eMfvXB__p1^`U0L4881%)7~_d}8uwMl&4^vz%N8nJcwOEJQ5jmbr@q4ADgQi-1)iyHGYK-!efYDWUB4pYNI+{sjC_MNqvA> zgrJGNn;LW|#E9t}Qan{SyD++1iZ zR!hS`$1ZKTwz|-0;j@MTMhlNO&Uw4G>bg$uQmGU>_4X9<7d@iLQUbQvUfZo0ff@Gd zIO}ZX(s{D^$$a@D$E#&cTQ3{z#ndy#JCRdg;Im4W)gW-XwEXm}f%EKdxKeXM#VXI> zVauw6*HoQ!5t{UT>Av=p6!&}XjfYGzxmS7nCdi_-gbO0`Pv4eYz1tQvsEg*b8O+kI2MNHgzLHE3KM9!Q z*?#iYE{3ZXzojIy9ey^o9E@V)=;U8KLW+qOj(ldA_}MBZ+GezxIf6;5Y-8fDJ9u9{ z8?Op>!ogo!YN>-GQPcLm%u0m}41FU+BH|Hn@&rID@(3bf${9JKYy|bLIVzq6~c9AVeC}ZjXoi{6%Dv@ANixKZaRv#KrRD7&b=V3tuCNG@fCs({Gab zF5D31(b<{}$&}?yY^$o-UfO!Oop9SUFsOUFK|g8OUvAaC8Y?fsoWV9XOxkvG5FOH@ z40odVa=vW8Q>p!3C+KEeB6w}3)+%)U@ZfW-sG#F~M#0FcHAbY0!Mp=8KQ_LOakJ*M zWyf8McGlPTflYx6R1{)bK5QVP?M4J*u7;L)S*dX@%m9i2jjWKP>kf3Un-)i7;cY_im ztrfpASvn4=0kpV8cE#~>+uI|Zls@+~j4-Aahe{^tp*BsH2UYm9pQP9F5c@-)>iP^2 z$5pMAtI^0Xx53i!jgRjycTYMA4K%cvryX!iAco#`h?2X1zW#r`}wOsE_5UQyXZ1bVYY9Z-Ke0lPmNl!+w9n<16q zdDe5GODI7J$4_k$paw7AYcO?}gv-c9<8y`B5k&l|VTC7q_S%%mJWW)cV?ambEu9Vd zY2%f-70OOLzRkn6<}4KhqqHX*im%ub+AF-Q`|D?BNRaNcV~#YGrokrNLm`YAy;mi; z{(@}DZ#Xe~tRS+bQ^ znWTx^{0jr2couNu(5G@O$|o~;b|r**OEb*i(y6dznYrF#fDJf zRDtzR+y8F8*7X-N)A}nD@hTi8xP;Zy$fK@~OecO^)&uV<5uac`m1t0kp*%%8p3eTb}aFaWqA-lVPE>9j#O3@rk>< zH66hD6njBn!6;pn)ZkU0xs>h5{3SoST_?J&5+0rw$^nLULnsG*?eT*Q;~3!=!^(Xhuzi znQ5lB)2=pEH-FIRnJxKsvL3_&%Qn~cS(+1r0=Hfk0~ouLZZ zQt`6a{Zh1&LDb8k3AN%m#fYGOFMIhSm1w+bWKku&I3mEUqL77@E@ncyu4Vb?(Gj2dQ80k*XX)0y z7|y+uINT1Tk$I;YQ^NNFM`BXj@O@yV5PL+L-t{$k79g(qX-C!y+4RV=XLTW}g-2LS zC4-6dHG;=6tX_wu;7Fa!kqliGoME45O=#$E{WESvYYXU@Sj;LCrHv}_Z9;?y| z5$+aj@eBy%8->7uJM;52Q+^*fRIvRAL7h&|?P$KIOY*oshWU8h13UaM$px7+6boY(6rdu8 zhWT?cPqQ0GkQ+T!N;fh8yXLF@SN5Ykz-r{~Q}^Lq|1xn{i+TCC?Zq3{ogEuT{jxFB zzPZ(dw4P>yY@)SG6$`1Mft$TftsRX0^}^POp&yZ26xy3VR+qIQ17Oi6Wp>^0Ki=7* z;FxVI$9eLJbTTIPLaE-6>kxuDI5-1X=YZ9Uv{9s6uHucgzUCj&rKT!^AZjt%_~kqQ zg};YbW|hRC=W$Acr?-q|FNq1&5CX2)l_2>H9`AiHk#c2L4`GCe`FnF$yNh|!-ZeFD zT{H5XBn9$5Q+j8tX+PMsd&(~-bTYS0o>6TZ~sCf%>P&T~S3Pjp+{x#7ty zGBzg&TX*QJhv7fVtXbLeKa{mfGp6AyP>g^yH>cnbE-KsxLVO< z;%&1DJfH!#6QLS6A?}hVy3|-IN9wv=ZZz@*ucFG%y8besp8UfHzLB-(!w4SDymNq- z;Xy!&VYXiliy+$mw{N)_RA}Rt0Ua}GN6KPxzsHS4U5gt!3e-Q@x_bkV%i_y(wj$V3 z)#nc)LiF`d=NI!Rv`YWXnY_LQTug<<#7KaYN)1uzu{RgG3+;Wl(%~LZ1I8^d=7-62 zP~brGNA-RmXWC5>KIwCPduN22CdkB6O>zBiQqjgntf0l?`?!=m=CoTUlzJGaKZ-#} zB{}odyg}LJSjpf){ioL-=i%N(514yY>z!VTx>ErRZKXy!;d6j7EPH5|8MmR6PB$%9 z{0@3**0rT;h8bR&lYL< z8wA3A-1=VW*!6jD|7>O|Pw@9!8Sm%e)_BQ8y!(ble%ViKEJ zzv|O;1XHnh)p>u8Op6it8K4o@JQBq?sI^*@=Z9w?@Pm1`Fmce^IC*B)UZ6o>4q2c{5IwL zS3JK#aBy~FAT9M&_tAb7>7AwB7%5adf+xdT0I$1*UJda1w=BX|Qc;&|Xl+bEwoh0P zmCSfY;3nl~9Ur37s9dI08*7e8FcV3w-Z`;^O9i!6UK>vYZgIF@aRi?QQ5&t1HN^U` zHFd$tJ;<~Vr0ksoxBnda)|*`X+T!%;BCgb^#-gH#(1wXb8J~4&n-=wsPaj`L6MhgC zVS19Hb%1x<${Z)h0kA7iFZlCqfoaEdBbW0eoX`IIzO$CImC8jOwL=av$|lWkVKnfQ zK8sK%aD%6V5s`;Q@ujW6-RV-pNQGV>^S8crF2y>3dc(PU%pkOI(T)x2_)C)w*hf|$ zhFnIgu1#Bi`dnik%Ze`q6|M)H==*a7>~_B7w|P<-25262RW9P&!_;po4iUb*=bp~C znM)bp#=N=S-~%26IeUz6hjmt&$$Xil<NaGHU$Q+< zF6)0u0fb2u4AbeZ^--XEpA%|?qwChf!>)HQO=9SY52SsVfz4!?iG<-!9>8wiG^Y}o z(6aQKCq%5OH=(u()aaLoU97QxbLvbFK_gR`NekEjZY}SZD)3jFkON#!NhEFK4( zIE>-W0|{ywz+vTh5hqu^ej!kI+(!_(UsJmNaRW=W`;fL-6D)=<*=5B`d0o$DP)tH zy&5OJD}H#~l19B}d3ObINSSh*d_S_ac|oe^WKlkE!uV`=f&aSW=!d7Cdmeor{=M$0 z=*>k6+6vLE&N&aoO%umjczEJTr_^@>4eB%NXIy2Q8I9$LyQkI~+LK~T3GEXdDE0>i z!;C#dKu~9dr&u8iUQ}JJ3qV5FOvD8F-6p+8FVj4{&E(b3t|JP88X{T4>n!&BLDp+#)tbKVX%GVVq!M*vi$>-6begMacCTR9f{O@W6%S?K%C+mZ&&nx-(Ns{X9`kcR=R=`Ek92%~b&9DGES5W;fg8{E<4-dqb z&h|>F<@4#jMyo^@U9v)6fwYbcEz@sx8gxWK_lxfKmZ8qrnOjinYGva%bmIll@-2!D z$H9nG;%P!f?d|aNJ8%glWHP<`Fa}#f)A~JY;;t4)3;6V<{xn&zbXor=zype0P$(N) z>&$hTzFTBhQnqs_J@3=;bh(Bk!uRnNeJowGnCn=7z0)$jCz3s$yH5=;>{|-a*y+`m6MdH~PPXcISM=nY$ zAduHz6qz~2oocX|VYOnsjh@GHpXv7N=34O$2bb&1Q>I9RIr6VV7RzfJHYLKC^(aV2 zLNDT(hiS60-`sd-&@-Z1=WM`EiRsY68fJNtNk2g1d4b=G5T8>Lhv=v%2LTSR^#m#T z6_g%&2aBEPma_}3(TZ$Jr)(cN@`sGeb{p@X8atkG3-^hsXLVf(CpZDoB&exf=C&_q zo%1V7Izp`WPBht$mIO~-R|QL(YW@~D7xez1-F5 z89PXk%qebg|etx^Q3KufsGFh3OEP>Lhfn@*~K8@BLqkju^;DkRD#GQHoD zJ%&Ruv{GHeh0@TQv>a4*gf~@vzCkBaBnPYCtmoj9+IlngvWWXx|N1+%bl;`+;0uko z$To{UnA^7_;~3XcR4pvYIW6;TX%G- znm3MV&uK2>0o&k>E*>Hu80ZJcp=xW8uMXDmK@_CNeMrIN6P6Ag6NAjF{ zyw(aZmBug#8m>So(dR+30DuEH!kleRd4QqShYO=r$Ub-U8zRv7k>AhT3_wsm?u@pU z-a>tdYANL7TKgh1DrDfo5Wp`fJO0RgEDPM}T4vOt{ZJslgdTPtsxjI45y>2dpr`I@ z2y^I~6A(Jo@uyVM{Hh*tQDzIN@$he3PV(inl%e{BiDf+*THA|9P%6W)GGD3?CiDY6 zRm-B{MBTDHtI;Ix5!>(x3Ha@BdTQ*;>xOdfvUwAjVuLHiz3(=eC)~?; z`UYW}kC8m^7AZMCP8EpQ^`Vum&X@MdHMlOAnA$}ba`E`-5t*ybTm!aHA{dQi6GP){ zTLy;UspKVZ%x<|G1Z5CC*w*pU1j)Ms{E#FD{y$gb&q{^uPcPW_BLg*<+AgN|yMd$B z5nKw#YkhMZj$i|!(pAMVQEbDQftnP=Ag>Zt+rh+H>w$qIb!FO@P>0xJOrO8J&K7e4 zD;Q`iZ81Mq&!Lt+XY(&2=Ovl6GvM1vGD8@G6un&E)ZB$}gPM0Ln+%29mnX8kyw9zf z<<-98MedH8fZXg94q{gMm@jo&)Y-fY^3GF^++v;ooKde}jPav;UZ<_>Btl+DrZ z=&9QuQXb9fd?sx=z|r$!4Czr|3(zwCv)mac&>+;tp8fhpLa=RN4hvX~9PU_vum!Ld z0Hq}j45Kpp@yt5SvbcwMF>cIqRTc5bqTka5lAO7yy|7CfO3!EUq{ zF(P)082(F$SXLp&RdSGWC;-ggKk5f(jETQUFb47j99uJtz?u*UIFl<5$|h5?Xe?U! zpTi!m{2XFXtrOc~^Dz}yvteN2CLVQ0AvId>cYaHoanOT(y}$1Mnc^$9rb2H#w!2XB z?!PxO5QBt3;ZhOMg`kzi0%rMlYpU@HD8Is$B5jzyzj_-Xt>k4OXtibZfyK*?_gWDh zNp(us_)O0m$HoL(nD*bL@ZfOsj()49N&5EMrvvp@wUu!kNW>#(D@bY$}&k!MO#lf6jv|B7;9AsH3snewr}`ne(3@-v)W%P(g1 z#@Plt{NDujuw+kCis_l9@MNUOy{pChNs`d9obd0jS$rURLSK=&!IY2KS`yAQ|O5AOTLm2pPEvCdE zJ;Y$y$A42gK+`{G#cuP-q|58j5+ADqlA(_+tm6=a)O~G(AHUh*1HFLYTRsW{RP+Bj zVT%CO7u%IVt)gf_d8}9s@LlBEZz~^f<&At0l~?ZHw{gIT{y*WU#R6({?GB7_{tbx^ z&^Z};lCLw0)UxPVDK))x0+8!WmHk=3<}Vs=8>!I|BPzd7Qqn$7 zg3enl)aFiW*^l&b@*K8pVk~}cyl({SB=aU#rm^!X&X&+rfgIoIMUFE{eN5WSY8jEM z;CJ&ES^HpGQdN!s zxm*ynXbHOB(qb^+i75H0JSq+}2%($+pd4MJnl88i&k3k+S0`JV4hJG4QGN#ph(_YW zS+qhFIXh2=ZFn4Qjv{tqq9EC!vXY1NvH**Nx9~?u?kS7u74htOKPu*LH_U*N{H1kA z7Cm8XiUzPG4)V{dE83c-1|opH%d~XA2kC46L^F40;Rc?KD?g58YKIV7=VG;N##*eIBnUu+|B2u=>F5&L(f&JvV5BqS31z-q(&CY8Yz>@ zYd~KcZ{Kjd&(n| zU+)Phw#V!k6J6TtBi^|n#e?+6t*`pQ&Uil;QdNs4mwReoR=`?9RkI4KH}z;9m6j2x zZ#c@Z=`1bX{dKd(sm=IbFs zImc1PmiGR5)b$?>tHv6BVRbn8%vFY|u#^}%{7W3PTPWw?=fd}OFjvQ=?s1xdP;EH{ zIvGKbdC_J&5EBN8;ULS=<9#R6G=YS|ovi!3(7|`akI-fxicp>!`_~-khFrhbN3Qk3 zrw@b}>IX&#vci*;q>naYU<9(Yn1zE|f%jRU>_|^iKK;+{aeCgM@i!YG1h3eO%b^^P zO1mFDK8r;CIGWY~@A?nS)BGV^d-MSDGr4KrD$pK;x2{a4#l;E7Kh^Q1xhI{!GpVdpg@AQxJz)C;0}Re z^ZK7JGhb%1R^+{v_wKzXCwrf>_YME3CXbKv3I`1h4PQ|~MiUL~KNH|H2n!Q<%|WZi z1>P`Rr4+TXfQKKJ`A^^&+eyK|6%CCb=HK%_?tpJuz)1=>S$#JxM@u))&n^~do}Qi@ zHV$^KW}lrbI2>K9AV*@a(9r166lEl}y)q7$JoOXx+B!~;U-)naf5Res@v#PFVy0Lj zqZ-A?IR3WB7|hS;dwgUmIH&5tKg_2({>#P|ku^(idjQucDE~>P>J!F-ij5D7Py76` z?C3@;cC=gQI0FUaI)|KRor@wBlCR;pazq%C*4z__s&8~yU4fL(y`D-E8Kz>vg6Xi> zRWM8>iN2FDKB=a|lENhk(j&si4Gr&L!wMejl367MF8oY?f)h;F!1e#Q%y$*NTAPu< z&5wvo_Y6@LL5CHrNVC&HDb#@L_%$$mnZ&2YWkJ(N{a}1vhZsY$WCnds#qHF^vS&cP z#=T+fyeWX8S>9^CTH?l2~**})dCup`Rw zyNFGjA^@5g+}x!^g$m%gFv=&}Sf) zJ`Hmn`v<#y`GdV1+xuPW$6+Ls#gyJ8x4{knZ3Nl!6Cl^I#e71WBg5XrcOU`}7&zaz z->`CE1%HecAotoL1n8{0Am|r<(p)^qLwBQ=^dd^_Tggbs` zWn-@W%JW511Y<=r5O14#?Nxkg+TUi?YOCwZ3IW2IQh)EbwXYlP>dx3*aBP|m-|~ux z@>pAft}3vZ&YZr2lQ^-0O(O)Jd%;_Lh^~);NDg(sqgwrm5T?VEazT zA%^`EsX4>eFMEkUZ$(K-$ncS4O-w>f8#uP^z z=l=LLUkM?52;Vvme9&&xwM;_N(#VI=s(&;vQ1d3Q#%wpSXDKewvsyo(ZoV)Ox|fEh zHGZL|P&C-Ic#*2ZBuRu}W?1oOf9-hWVun_dK8W3p0L<55)jip5>#Q`8hYT`5V5jO+ zjmYD=#8hGuoiKqY8lVU)XC{wah&#hL>sPnp6Uh>550~ICwq8I!WdZ-N`}FrGhI0ly z&Zo@5KRhk=%g~KIJdqBwPW48=5Z;9t$3Js3yU60u;vg?nKkvC8au;?|D+bz=;rQCs zkG$4kb{2OF)i6jKG8Xxvd`mA3d%%cbEi~BrpRj)o(xXq}vbLn{kq`|wlgQjMbz`K! z?a`X+T!Nh?+KL=Uc1p+w89S7oFWv`{qm_OS9?vy@m-Y3Cfo_zo$85%?=ysml%g50p z&@0ZacYT;coe1N<^wQX-g`5$w$6b;{a=nvb`7j~gw}}V6!`T>X?jmUct&Xc2mf_ll zf*ZN-1S2E5#FielLoSC^j0uc$>fmfi;`oHutdz?ID4m5zSG9`>i7xnTg|Y)AiDS|kfo(~rmml~#-oGzJIIfLx(DTI zf;$f4X9L<~eVdxK18Hn&j7Xz82!&5l<-;{zJ?$FUw~p?L@R)4;Rd7 zf^`P)ue?pYmxZf`#UGVb2|`4S*XyweW(vYSkJo&~a7R~X(I|0x9K|*cXjPd%+f97x zJ|&LyeH--6jzM34ehT|eb9|k2Z*A{zC@&WI#llW@C>+D1QhW>m)Oou4C)MRmyrX_X zDV!@ljB5Jl~5WO&mMTR0WVsQ^=05 zvv<-dPV}3_Qb)C1v^Jw`Au;bfMz^tl#;!mKFB(s?Hoy>^zz9Q9a)!ntEL%zum~Mh> zJ4)q=7zZ&uf?B~U4lCatf{Uz3JRvcyUuw>|#80JIlX5guXV#O925x5N9=kiIVFAas z#Z4xgtw~0dIzMugh|sS&@GHp33RCsC?;?Pdsi+aRuhjM`WnjZf%X>D+ zuU|~Wr78mw>i5r)SMHuwHa{bM>2ECO)T3GRHCpkE3e7iADaY3B4uofS#9$E8{V(R5 zc34DU3znI(fb5vDQ|tF^D8!~Tr^!TX~;cUOLM2Ix+?&3 z?YNKhRQ$(H0mNZi+~(|5@`~9U{7fvB^gC_uf{Jtvs#ZgUK|I#Fw9AXNiPHPAt`11j z^fGx}_VJ5T)?q4REfyU==!Ja9X!Z5YQ~Qu#7)8#aYG@k$zD1J*xriciQDF;hf^mk zZPOFry6cNwM{I!X%OyCrC)yMU{QVa23tcv`xai-bc#6GXMGa>Ubithku(WRmeQHD zk|*50uLLiLHG8^Qj75oK;1hOC_fk;$+6_bu_Nxvu9k~(zi}R8F{k`KfW4jpHiYe#O zTgy7D;n!$Ceq34&@k%?R+zaVif095(oUa$aa)UMw$pU0}sR)S427jRRGd6ja&y2gS$vooHZ?g$M}>si>us65zR=?E8C&ajNqG^$i844^ z)%k5H?{gf2P3HTgYkyNIn@$D4hBcpCcvLQ{4=Hw2Z$?D$;5hu@bxPn2vHVf%IvtSj zIE5e*%cI2A3tL++6jid@HxFs8X6t;@a@rS}4}cJ(gK=A7Qxq`Ai$J~?-^?Vk{`^@; zvzi&|E-w!oaURKv!N5ECT_cQsZN7%7bf$cnj9lLus*4_->c^{buj{M~!Oh(WdCAjC zOFMF_bi7~a!osVk_T`|yJAS3<9$X2=cM#|JcZJKD&vp|9Z9C(1@nLXAjPuPEs~u_^&oieS={WK?NJ7S=3wMEcD=MEo z;zziFn_fW5ZfoQJU}C{AI1LQ~18$FR{PyIE9)+dnXk~K0<}HFTolQRtZKq{xeAiiC{tN($sb6ki z)EPB1jKGXL0^fdKy{=dXEXx4B|=G6_(!mZ z^2=9d&d+&AWlZeE4KHLhe$9cCp=~?|WHy+)-XpEeH%)SZ>*PrK8 z#n}xRI;i+jq(sil+4owIJ7VCA^uj@^hmi{#PZF?3i(3g)yrcPG7O+4>dtErduKINz z;xjQ!PNqv5!xIU@f@!s1-g%5PR)8AAGa4G3?(*x4->A1g{t^&6^m!B{sJQ*a^z-0) zMm#Iv(HT=^KRPLs0aq{dbh%+PDp_WJui@Q|7Yd5L1PI2hHz6L2A4(;7nI&e4xCA<)y*n zl}to4JHWbI;hsfyMrPgH1ayC`VdF-hef30(DpRpO$avuyJ+wd;MQ^-39b>`r=jO9h zKsN$rTFET6XO{Wid(utYwC`r7XrcLIl)7WFT~*qFr5B#hrViuHbWFCNpwaSbO5N;R zaNDa)J6kq$qceD{g5XcOf0NK{Tu7T93ZVY>#T6cCUQ6c(^Ue>}oS^-Uiej zKvm_Zg*e99s3%;+?~IvaYbx?$X@n{zXws3x~k}e1|Kia${_;+e%<_91=5-0{=PYY zd(O0(IQ;wfdco`Un`1W*LtSl_xx1OA?@{=_|0siJX+$`FsjXfTUQR#wdGv0HNRrd- zO7SR??;$#SI@3$5qh`dHA4dIsTG0RinmgGeJNR^f^(xmUN!36AVZEBPVid}8_VFF4 z?K(stZY6w_D;=FRUSW?quJT78nu&u*`XKa#aDVgoV~B-3^0 zY4v2x(U!SYLP$aP?9-w1^R?Rf4oaRP(e{08SWtHh;-)<-zjfa~dxi$B)#t=8q*^um zZb+)DU3esakUwxkkuzp{Gx^-TMeUXCY{q{VFD<+}swzq{MzfbnQ1IFLAspCK16jZ8 z-!nsA_s``xsk*i$j_F&oI%8~YlM=e7D1~Op#_hf{6*nyl1i-TT%H>pZ>r(BsS~WM3 zI3K)cod^n$&X<|!Vu;rFj$eJlK{MYeGo1j061v(j0QjR|W$L4NYW>KLskR+>pa9Pw z)_!&RFuISqWUI5Kld<3($}-sVWXx4tFh2;T=7(uV9$4(@>zgF?C(S-3^=v>0&F{&= zjBwG_Kn8xA0YU%6Xw80; zggBi!bqtIo0NuUSWVWY3nF%3~nsHN#CI)#L702tf-Pbug@f4kj}ZG zV}7p*ek^x7Yl?sQcI4$lo7`~o1XMy@Uu?o!c{5-sY)YlExq-8687++}zHBmcsxUiy zd_%Au3YiGz0q>i!`lkF)|BvwHEJdmDdF`Om$hgPu%RJ38w9D~uzRQaVe;{KX;=&8V|;jmJMPD*e5TawVEPe3HO-PW zEjL6tyHnZ(WZz}{)vHvW7=@BKSDhSR}m*Cofu9cM;6C^A#@f}sC;?i zVHZ#AXQW5DgBBG_nzeR>TUtFnzFT*Snw#9f7rxBKjWXiHt432-#TA6vBpBnpXrUzE)*140R9&h7i+CO`9b{g8O zDz=ADax~*C^^_J5KEA+ziLWzgzNxx5divjz;@t_Cdelx|$%Q>r;R z^jg4$VFX84^Vzs62Pn3_GKxzGJ!g6-Ks$Pc^V3Z-27Q$HS0Ef?(DN$n3L)`S7>Dn@FD4@*BoR3%1oHA`kstM6g3 zH3~F9Ak{dY?d~p9Jog%m9Gts4GJ&=DoLn2l#zWG>z+rC+pB7epy316Hh{A5YM9V2* zmiOObzM|gTj6Isp`Kype)+#`4%_BLT zjc*P2#k1XBrzOBN4~)M$pStw4wldi`r>{7>KOS8UHZgN@FugC$Xk}^s<>^+qj~4iq zs)B`$Rh|`0EG7j;8tIr!e`QGo%I)y?X(g322NG(Q?z1hH!g@xCcO=aJdK$MiOdtS~ zlDf|YtBd%`#1v&ovp|Qv?lTkhtqesVBhc*`xNefB?8Uej=I{Z!bqmqNI*-%2b8cofV^h=B zLDP=eX_N^UMw+qQYE6}~Fi~_MC+NVK&r`dW3A~3l#$T9OenleGz(!p99iU=t3d+7{ zw&=*`gz1)X|>HMc=&%sXant>k8;?Tb?d zQg~)BUd=6uYC%H_Ny8kM-)Ip2^Xc!+XXIDqRW8+4cFxkV{;Z$xlM06I@hK?3%g~P( zrIhOIkr|wOBlqs--{r-;_s629)gEP|;jlLwsG)d^!*JM0l|NVqbz5)=*I4Oo>YD&p zvfS!&cxv(4e9~-oP;)d&afa)1Z240sarfFa{@8ubtuLz49akIq!$y8jYhwa>pGFw` z`&=`V*`-@7~JK8Lpo=E7g*xiTd38WuqcT3MX{=wrtmFh_3hyfF=wiBZBFblWErh#W zJE79hCB;zB$hiyTC%HlRE1GVYXVIH@1vi(rjSF*T8JZZ2BT(^wt4nZ8#&wCCr_lOxjo8>Q^4Mx~za682W!;V&4w{rjW7h6F>cV z|021xq{Lvgu&u+mHQ;Z|CsY^8c#-6@(|D>VcG^GLF5qS-r_+_nFfbU^jd%Kul|4uDE;O0ai^G%EZ`;6w%~R*R=fL{p-gcToqbP3n@w0#-h`Lc#H*MGr=HcX z19=%=t^%FEH!=Ci^tg9_!h>e6_{$8T(?Be;IETBd;f$LDWj`1a$QWYf)2kc$!G)DDDXkGp9 z@ui@M1f0wlReyzA7R)THWC8VMuDPP31Px+vXH-zYT^U$e5JX0c#OC(ZarWC~C-MBGIMV-~jx$Uq2T2#&*( z9pSjHg&9D$8T<71o<=%I&p<=&p%UYXk1)^8*7$+3`?pjQZ1)%vz%^E^MD((aH_5~P z(@dkXdOVu-y^eEzpI%3`5y=ZQ!8nr+rtE`TX{e%)8Ae^j>VXSISIQ(9Ywa`W{J$8R zpuN22T)>a9$tau4Jh^nM9GyNyCJ1^C}?e1#5rb?)J(S5gKBL54cva}cY4->=cdh3$r?(FGN*h($*>X@!8Q zb<3Gm;6czck?Q*sJ^)gt9_fKd2;@u`;iZr!oiFVho#GyLxY}i{UO`2HIyJV;rMng9 z!!VBdK;D8%pgrF|gnrB(-YiIO{Llzk|90U2u~rckA0#bbTQ!2ZmWN|PA93>YU4z;_0v@wXsHZu3T-kR8<#_4q-J9g{E+_kDRQjHYbyD|6#aO^hW$nK>sa9&bdfZxw z`<&U`N&9(U_9AUo3Nx>~Wo0SmoKtL&>3`0m&p&E2Upp=9z0GQ$i-3}0NOGx{+uxf- z0m{(!-4?f|+2kC28|%<71N-*;MIHFKZa-ut-5ufrtZ^H`h|X!|KH8<|K9{L b`-rX@zM-WQ)5-#D_&`&XRg @@ -28,9 +28,9 @@ borderopacity="1.0" inkscape:pageopacity="0.0" inkscape:pageshadow="2" - inkscape:zoom="2.8" - inkscape:cx="156.72376" - inkscape:cy="62.579314" + inkscape:zoom="1.979899" + inkscape:cx="192.14333" + inkscape:cy="12.853681" inkscape:document-units="mm" inkscape:current-layer="layer1" showgrid="false" @@ -40,7 +40,7 @@ inkscape:guide-bbox="true" inkscape:window-width="1920" inkscape:window-height="1017" - inkscape:window-x="-8" + inkscape:window-x="1912" inkscape:window-y="-8" inkscape:window-maximized="1" fit-margin-top="0" @@ -63,165 +63,148 @@ inkscape:label="Layer 1" inkscape:groupmode="layer" id="layer1" - transform="translate(-15.426685,-284.39917)"> + transform="translate(-14.85972,-284.39917)"> + - + transform="matrix(1.5952381,0,0,1.5952381,17.392161,-166.33938)" + id="g836"> + transform="matrix(0.17545692,0,0,0.17545692,7.4983819,273.23227)" + id="g1731-8"> + style="opacity:1;vector-effect:none;fill:#661f76;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.10341848;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m -34.982113,87.648437 8.031994,4.677455 v 9.307668 l -8.031994,-4.677459 z" + id="rect1512-5" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + sodipodi:nodetypes="ccccccc" + inkscape:connector-curvature="0" + style="opacity:1;vector-effect:none;fill:#3fc6ee;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.39087299;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="M -8.6445312,259.66211 -132.2168,331.26953 -101.85938,348.94922 -8.9550781,295.11328 52.369141,330.64062 82.84305,312.46039 Z" + transform="scale(0.26458333)" + id="rect1512-9-4" /> + style="opacity:1;vector-effect:none;fill:#3fc6ee;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.10341848;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m -2.3294599,97.000255 8.031993,4.677455 -8.273187,4.68055 -8.0319941,-4.67746 z" + id="rect1512-9-8-8" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + sodipodi:nodetypes="ccccccccc" + inkscape:connector-curvature="0" + style="opacity:1;vector-effect:none;fill:#661f76;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.39087299;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m -40.429688,348.70508 -30.357421,17.32031 v 107.0293 l 30.357421,17.67969 v -71.57227 l 30.7128911,18 V 401.98438 L -39.857203,384.30047 Z" + transform="scale(0.26458333)" + id="rect1512-94-56" /> + style="opacity:1;vector-effect:none;fill:#661f76;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.10341848;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m -2.3696748,78.082315 16.2254758,9.399559 v 9.30767 L -2.3279139,87.434868 Z" + id="rect1512-94-5-5" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + style="opacity:1;vector-effect:none;fill:#865fc5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.10341848;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m 21.909986,82.679259 -8.054185,4.802615 v 9.319569 l 8.031994,-4.677459 z" + id="rect1512-3-7" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + style="opacity:1;vector-effect:none;fill:#865fc5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.10341848;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m 5.7025331,101.67771 -8.273187,4.68055 v 9.30767 l 8.273187,-4.68056 z" + id="rect1512-3-6-1" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + sodipodi:nodetypes="ccccccccc" + inkscape:connector-curvature="0" + style="opacity:1;vector-effect:none;fill:#865fc5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.39087299;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m -8.95767,295.11376 -92.90171,53.83546 v 35.17773 l 61.429692,-35.42383 0.491071,35.51284 31.1339295,-17.60073 V 330.4668 l 0.00586,-0.004 z" + transform="scale(0.26458333)" + id="rect1512-3-65-1" /> - - - - - - - - - - - - - - - - - - + style="opacity:1;vector-effect:none;fill:#865fc5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.10341848;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" + d="m -2.5706539,115.66593 -8.1264881,-4.77196 v 18.94606 l 8.1264881,-4.58296 z" + id="rect1512-3-6-8-8-9" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + + + + + + + + + + + + + + + + From db8c088f19e282629b3020e295445d141064dac6 Mon Sep 17 00:00:00 2001 From: Meinrad Recheis Date: Sun, 11 Aug 2019 15:32:49 +0200 Subject: [PATCH 12/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55ecb454..d10dd86a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![logo](docs/assets/tf.net.logo.svg) +![logo](docs/assets/tf.net.logo.png) **TensorFlow.NET** (TF.NET) provides a .NET Standard binding for [TensorFlow](https://www.tensorflow.org/). It aims to implement the complete Tensorflow API in CSharp which allows .NET developers to develop, train and deploy Machine Learning models with the cross-platform .NET Standard framework. From 879067deb4bb53eaa21ffcfd55b9ecc1f1fb6839 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Tue, 13 Aug 2019 22:07:37 -0500 Subject: [PATCH 13/14] tf.WhileContext() --- src/TensorFlowHub/MnistDataSet.cs | 6 + src/TensorFlowNET.Core/APIs/tf.control.cs | 19 ++ src/TensorFlowNET.Core/APIs/tf.math.cs | 4 +- .../Graphs/Graph.Control.cs | 2 +- .../ControlFlows/ControlFlowContext.cs | 36 ++++ .../Operations/ControlFlows/WhileContext.cs | 177 ++++++++++++++++-- .../Operations/NnOps/rnn.cs | 41 +++- .../Operations/control_flow_ops.py.cs | 94 +++++++++- .../Operations/gen_control_flow_ops.py.cs | 87 +++++++++ src/TensorFlowNET.Core/Operations/math_ops.cs | 3 + src/TensorFlowNET.Core/Tensors/Tensor.cs | 1 + .../WhileContextTestCase.cs | 14 +- 12 files changed, 458 insertions(+), 26 deletions(-) diff --git a/src/TensorFlowHub/MnistDataSet.cs b/src/TensorFlowHub/MnistDataSet.cs index accc57e1..f1a73349 100644 --- a/src/TensorFlowHub/MnistDataSet.cs +++ b/src/TensorFlowHub/MnistDataSet.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Text; using NumSharp; using Tensorflow; @@ -21,7 +22,12 @@ namespace Tensorflow.Hub images = images.reshape(images.shape[0], images.shape[1] * images.shape[2]); images.astype(dataType); + // for debug np.multiply performance + var sw = new Stopwatch(); + sw.Start(); images = np.multiply(images, 1.0f / 255.0f); + sw.Stop(); + Console.WriteLine($"{sw.ElapsedMilliseconds}ms"); Data = images; labels.astype(dataType); diff --git a/src/TensorFlowNET.Core/APIs/tf.control.cs b/src/TensorFlowNET.Core/APIs/tf.control.cs index ce5da031..d744a141 100644 --- a/src/TensorFlowNET.Core/APIs/tf.control.cs +++ b/src/TensorFlowNET.Core/APIs/tf.control.cs @@ -14,10 +14,29 @@ limitations under the License. ******************************************************************************/ +using System; + namespace Tensorflow { public static partial class tf { + public static Tensor while_loop(Func cond, Func body, Tensor[] loop_vars, + TensorShape shape_invariants = null, + int parallel_iterations = 10, + bool back_prop = true, + bool swap_memory = false, + string name = null, + int? maximum_iterations = null, + bool return_same_structure = false) + => control_flow_ops.while_loop(cond, body, loop_vars, + shape_invariants: shape_invariants, + parallel_iterations: parallel_iterations, + back_prop: back_prop, + swap_memory: swap_memory, + name: name, + maximum_iterations: maximum_iterations, + return_same_structure: return_same_structure); + public static _ControlDependenciesController control_dependencies(Operation[] control_inputs) => ops.control_dependencies(control_inputs); } diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index a8604483..c2686812 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -39,8 +39,8 @@ namespace Tensorflow public static Tensor asin(Tensor x, string name = null) => gen_math_ops.asin(x, name); - public static Tensor add(Tx a, Ty b) - => gen_math_ops.add(a, b); + public static Tensor add(Tx a, Ty b, string name = null) + => gen_math_ops.add(a, b, name: name); /// /// Computes atan of x element-wise. diff --git a/src/TensorFlowNET.Core/Graphs/Graph.Control.cs b/src/TensorFlowNET.Core/Graphs/Graph.Control.cs index 70057113..4a3ac793 100644 --- a/src/TensorFlowNET.Core/Graphs/Graph.Control.cs +++ b/src/TensorFlowNET.Core/Graphs/Graph.Control.cs @@ -33,7 +33,7 @@ namespace Tensorflow /// /// The data input ops for an op to be created. /// A list of control inputs for the op to be created. - private ITensorOrOperation[] _control_dependencies_for_inputs(ITensorOrOperation[] input_ops) + public ITensorOrOperation[] _control_dependencies_for_inputs(ITensorOrOperation[] input_ops) { var ret = new List(); diff --git a/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs b/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs index d2a1f628..5cfecc49 100644 --- a/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs +++ b/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs @@ -53,6 +53,11 @@ namespace Tensorflow.Operations protected Stack _context_stack; protected ControlFlowContext _outer_context; + /// + /// The keys are the names of tensors referenced by but external to this + /// context. Each value is the Tensor that should be used by this context to + /// access the key value (e.g. a switch output guarding a cond input value). + /// protected Dictionary _external_values; public ControlFlowContext() @@ -68,6 +73,12 @@ namespace Tensorflow.Operations _outer_context = ops.get_default_graph()._get_control_flow_context(); if (values_def != null) _init_values_from_proto(values_def, import_scope: import_scope); + else + { + _values = new HashSet(); + _external_values = new Dictionary(); + } + } public void __enter__() @@ -114,6 +125,27 @@ namespace Tensorflow.Operations graph._set_control_flow_context(this); } + protected virtual Tensor _Enter(Tensor data, string frame_name, + bool is_constant = false, + int parallel_iterations = 10, + bool use_ref = true, + bool use_input_shape = true, + string name = null) + { + Tensor result; + data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref: true); + if (data.dtype.is_ref_dtype() && use_ref) + throw new NotImplementedException("_Enter"); + else + result = gen_control_flow_ops.enter( + data, frame_name, is_constant, parallel_iterations, name: name); + + if (use_input_shape) + result.SetShape(data.TensorShape); + + return result; + } + /// /// Exit this control flow context. /// @@ -184,6 +216,10 @@ namespace Tensorflow.Operations return true; } + protected virtual bool _IsInOuterContext(Operation op) + { + throw new NotImplementedException("_IsInOuterContext"); + } protected virtual void _RemoveExternalControlEdges(Operation op) { diff --git a/src/TensorFlowNET.Core/Operations/ControlFlows/WhileContext.cs b/src/TensorFlowNET.Core/Operations/ControlFlows/WhileContext.cs index 69329b21..c5e7121c 100644 --- a/src/TensorFlowNET.Core/Operations/ControlFlows/WhileContext.cs +++ b/src/TensorFlowNET.Core/Operations/ControlFlows/WhileContext.cs @@ -15,8 +15,12 @@ ******************************************************************************/ using System; +using System.Collections.Generic; +using System.Linq; using Tensorflow.Operations.ControlFlows; +using Tensorflow.Util; using static Tensorflow.Python; +using static Tensorflow.control_flow_ops; namespace Tensorflow.Operations { @@ -32,10 +36,14 @@ namespace Tensorflow.Operations bool _swap_memory; Tensor _pivot_for_pred; Tensor _pivot_for_body; - Tensor[] _loop_exits; - Tensor[] _loop_enters; + List _loop_exits; + List _loop_enters; + Graph _graph; + public override GradLoopState grad_state => _grad_state; + public override bool back_prop => _back_prop; - public WhileContext(int parallel_iterations = 10, + public WhileContext(int? maximum_iterations = null, + int parallel_iterations = 10, bool back_prop = true, bool swap_memory = false, string name = "while_context", @@ -49,12 +57,27 @@ namespace Tensorflow.Operations } else { - + __init__(); + _init_from_args(maximum_iterations, parallel_iterations, back_prop, swap_memory, name); } _grad_state = grad_state; } + private void _init_from_args(int? maximum_iterations, + int parallel_iterations, + bool back_prop, + bool swap_memory, + string name) + { + _name = ops.get_default_graph().unique_name(name); + _back_prop = back_prop; + _swap_memory = swap_memory; + _loop_exits = new List(); + _loop_enters = new List(); + _graph = ops.get_default_graph(); + } + private void _init_from_proto(WhileContextDef context_def, string import_scope = null) { var g = ops.get_default_graph(); @@ -70,26 +93,156 @@ namespace Tensorflow.Operations // The boolean tensor for loop termination condition. _pivot = g.as_graph_element(ops.prepend_name_scope(context_def.PivotName, import_scope)) as Tensor; // The list of exit tensors for loop variables. - _loop_exits = new Tensor[context_def.LoopExitNames.Count]; + _loop_exits = new List(); foreach (var (i, exit_name) in enumerate(context_def.LoopExitNames)) - _loop_exits[i] = g.as_graph_element(ops.prepend_name_scope(exit_name, import_scope)) as Tensor; + _loop_exits.Add(g.as_graph_element(ops.prepend_name_scope(exit_name, import_scope)) as Tensor); // The list of enter tensors for loop variables. - _loop_enters = new Tensor[context_def.LoopEnterNames.Count]; + _loop_enters = new List(); foreach (var (i, enter_name) in enumerate(context_def.LoopEnterNames)) - _loop_enters[i] = g.as_graph_element(ops.prepend_name_scope(enter_name, import_scope)) as Tensor; + _loop_enters.Add(g.as_graph_element(ops.prepend_name_scope(enter_name, import_scope)) as Tensor); __init__(values_def: context_def.ValuesDef, import_scope: import_scope); } - public override WhileContext GetWhileContext() + /// + /// Add the loop termination condition and body to the graph. + /// + public Tensor[] BuildLoop(Func pred, + Func body, + Tensor[] loop_vars, + TensorShape shape_invariants, + bool return_same_structure) { - return this; + // Keep original_loop_vars to identify which are TensorArrays + var original_loop_vars = loop_vars; + // Convert TensorArrays to their flow variables + Enter(); + var(original_body_result, exit_vars) = _BuildLoop( + pred, body, original_loop_vars, loop_vars, shape_invariants); + Exit(); + + var flat_result = original_body_result; + + var exit_vars_with_tensor_arrays = _convert_flows_to_tensorarrays(flat_result, exit_vars); + var packed_exit_vars = nest.pack_sequence_as( + structure: original_body_result, + flat_sequence: exit_vars_with_tensor_arrays); + + return packed_exit_vars as Tensor[]; } + private (Tensor[], Tensor[]) _BuildLoop(Func pred, + Func body, + Tensor[] original_loop_vars, + Tensor[] loop_vars, + TensorShape shape_invariants) + { + var flat_loop_vars = original_loop_vars; - public override GradLoopState grad_state => _grad_state; + // Let the context know the loop variables so the loop variables + // would be added in the outer contexts properly. + _InitializeValues(loop_vars); + var real_vars = loop_vars; + Tensor[] enter_vars = null; + tf_with(ops.control_dependencies(null), delegate + { + enter_vars = real_vars.Select(x => _Enter(x, + _name, + is_constant: false, + parallel_iterations: _parallel_iterations, + use_input_shape: shape_invariants == null)) + .ToArray(); - public override bool back_prop => _back_prop; + foreach(var x in enter_vars) + { + x.graph.prevent_feeding(x); + if (_outer_context != null) + _outer_context.AddInnerOp(x.op); + } + }); + + // Finds the closest enclosing non-None control pivot. + var outer_context = _outer_context; + while (outer_context != null) + { + + } + + _SetShapeInvariants(real_vars, enter_vars, shape_invariants); + + // Fix the control inputs and control flow context of these enter ops. + _FixControlInputsAndContext(enter_vars); + _InitializeValues(enter_vars); + _loop_enters = enter_vars.ToList(); + + var merge_vars = enter_vars + .Select(x => merge(new[] { x, x })) + .ToArray(); + + _pivot_for_pred = merge_vars[0]; + + // Build the graph for pred. + var merge_vars_with_tensor_arrays = _convert_flows_to_tensorarrays(flat_loop_vars, merge_vars); + // var packed_vars = nest.pack_sequence_as(original_loop_vars, merge_vars_with_tensor_arrays); + var c = ops.convert_to_tensor(pred(merge_vars_with_tensor_arrays[0])); + _pivot = gen_control_flow_ops.loop_cond(c, name: "LoopCond"); + var switch_vars = merge_vars.Select(x => _SwitchRefOrTensor(x, _pivot)) + .ToArray(); + + // Build the graph for body. + var vars_for_body = switch_vars.Select(x => _Identity(x[1])).ToArray(); + // Convert TensorArray flow variables inside the context back into + // their associated TensorArrays for calling the body. + var packed_vars_for_body = _convert_flows_to_tensorarrays(flat_loop_vars, vars_for_body); + var body_result = body(packed_vars_for_body[0]); + var post_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION); + + // Store body_result to keep track of TensorArrays returned by body + var original_body_result = new[] { body_result }; + // Convert TensorArrays returned by body into their flow variables + var result = new[] { body_result }; + + var next_vars = new List(); + foreach (var (m, v) in zip(merge_vars, result)) + next_vars.Add(_AddNextAndBackEdge(m, v)); + + // Add the exit ops. + var exit_vars = switch_vars.Select(x => exit(x[0])).ToList(); + _loop_exits = exit_vars; + + // Exit the loop. + // ExitResult(exit_vars); + return (original_body_result, exit_vars.ToArray()); + } + + private void _FixControlInputsAndContext(Tensor[] enters) + { + var graph = ops.get_default_graph(); + foreach(var e in enters) + { + var inp_op = e.op.inputs[0].op; + var control_inputs = graph._control_dependencies_for_inputs(new[] { inp_op }); + // op for op in control_inputs if self._IsInOuterContext(op) + var outer_control_inputs = control_inputs.Where(x => _IsInOuterContext(x.op)) + .Select(x => x.op) + .ToArray(); + e.op._set_control_flow_context(this); + e.op._add_control_inputs(outer_control_inputs); + graph._record_op_seen_by_control_dependencies(e.op); + } + } + + private void _InitializeValues(Tensor[] values) + { + _values = new HashSet(); + foreach(var x in values) + _values.Add(x.name); + } + + public override WhileContext GetWhileContext() + { + return this; + } public WhileContext from_proto(WhileContextDef proto, string import_scope) { diff --git a/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs b/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs index 5b820b3a..1c7466f7 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs @@ -141,30 +141,57 @@ namespace Tensorflow.Operations string base_name = null; tf_with(ops.name_scope("dynamic_rnn"), scope => base_name = scope); - Func _create_ta = (name, element_shape, dtype_) => + Func _create_ta = (name, element_shape, dtype_) => { - new TensorArray(dtype: dtype_, + var ta = new TensorArray(dtype: dtype_, size: time_steps, element_shape: element_shape, tensor_array_name: base_name + name); - throw new NotImplementedException(""); + return ta; }; bool in_graph_mode = true; + var output_ta = new List(); + var input_ta = new List(); if (in_graph_mode) { - foreach(var (i, out_size) in enumerate(flat_output_size)) + foreach (var (i, out_size) in enumerate(flat_output_size)) { - _create_ta($"output_{i}", + output_ta.Add(_create_ta($"output_{i}", new TensorShape(const_batch_size).concatenate( _maybe_tensor_shape_from_tensor(out_size)), - _infer_state_dtype(dtype, state)); + _infer_state_dtype(dtype, state))); + } + foreach (var (i, flat_input_i) in enumerate(flat_input)) + { + input_ta.Add(_create_ta($"input_{i}", + new TensorShape(flat_input_i.dims.Skip(1).ToArray()), + flat_input_i.dtype)); + } - + for (int i = 0; i < input_ta.Count; i++) + { + var (ta, input_) = (input_ta[0], flat_input[0]); } } + // Make sure that we run at least 1 step, if necessary, to ensure + // the TensorArrays pick up the dynamic shape. + Tensor loop_bound; + if (in_graph_mode) + loop_bound = math_ops.minimum( + time_steps, math_ops.maximum(1, max_sequence_length)); + + /*Func cond = (ctime) => + { + return null; + }; + + control_flow_ops.while_loop( + cond: cond, + body = );*/ + throw new NotImplementedException(""); } diff --git a/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs b/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs index 2717fd3e..c8c711e7 100644 --- a/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs +++ b/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs @@ -26,6 +26,44 @@ namespace Tensorflow { public class control_flow_ops { + public static Tensor _AddNextAndBackEdge(Tensor m, Tensor v, bool enforce_shape_invariant = true) + { + v = ops.convert_to_tensor(v); + v = _NextIteration(v); + if (enforce_shape_invariant) + _EnforceShapeInvariant(m, v); + m.op._update_input(1, v); + return v; + } + + /// + /// Check if the shapes of the loops variables are invariants. + /// + /// + /// + public static void _EnforceShapeInvariant(Tensor merge_var, Tensor next_var) + { + + } + + public static Tensor exit(Tensor data, string name = null) + { + data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref: true); + if (data.dtype.is_ref_dtype()) + return gen_control_flow_ops.ref_exit(data, name: name); + else + return gen_control_flow_ops._exit(data, name: name); + } + + public static Tensor _NextIteration(Tensor data, string name = null) + { + data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref: true); + if (data.dtype.is_ref_dtype()) + return gen_control_flow_ops.ref_next_iteration(data, name: name); + else + return gen_control_flow_ops.next_iteration(data, name: name); + } + public static Operation Assert(Tensor condition, object[] data, int? summarize = null, string name = null) { return tf_with(ops.name_scope(name, "Assert", new { condition, data }), scope => @@ -213,6 +251,14 @@ namespace Tensorflow return gen_array_ops.identity(data, name: name); } + public static void _SetShapeInvariants(Tensor[] input_vars, Tensor[] enter_vars, TensorShape shapes = null) + { + if (shapes == null) + return; + + throw new NotImplementedException("_SetShapeInvariants"); + } + /// /// Forwards `data` to an output determined by `pred`. /// If `pred` is false, the `data` input is forwarded to the first output. @@ -516,10 +562,52 @@ namespace Tensorflow throw new NotImplementedException("ZerosLikeOutsideLoop"); } - // TODO - public static void while_loop(Func func, Func func1, Tensor[] tensors, int? i) + /// + /// Repeat `body` while the condition `cond` is true. + /// + /// + /// + /// + /// + public static Tensor while_loop(Func cond, Func body, Tensor[] loop_vars, + TensorShape shape_invariants = null, + int parallel_iterations = 10, + bool back_prop = true, + bool swap_memory = false, + string name = null, + int? maximum_iterations = null, + bool return_same_structure = false) { - throw new NotImplementedException(); + tf_with(ops.name_scope(name, "while", loop_vars), scope => + { + if (loop_vars == null || loop_vars.Length == 0) + throw new ValueError("No loop variables provided"); + if (cond == null) + throw new ValueError("cond must be callable."); + if (body == null) + throw new ValueError("body must be callable."); + if (parallel_iterations < 1) + throw new ValueError("parallel_iterations must be a positive integer."); + + var loop_context = new WhileContext( + maximum_iterations: maximum_iterations, + parallel_iterations: parallel_iterations, + back_prop: back_prop, + swap_memory: swap_memory); + + if (loop_context.outer_context == null) + ops.add_to_collection(ops.GraphKeys.WHILE_CONTEXT, loop_context); + + var results = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants, + return_same_structure); + + if (maximum_iterations != null) + return results[1]; + else + return results[0]; + }); + + throw new NotImplementedException("while_loop"); } } diff --git a/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs b/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs index 580da2b7..bfbf3413 100644 --- a/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs +++ b/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs @@ -20,6 +20,93 @@ namespace Tensorflow { public static OpDefLibrary _op_def_lib = new OpDefLibrary(); + /// + /// Creates or finds a child frame, and makes `data` available to the child frame. + /// + /// + /// + /// + /// + /// + /// + public static Tensor enter(Tensor data, string frame_name = "frame_name", bool is_constant = false, int parallel_iterations = 10, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Enter", name, new + { + data, + frame_name, + is_constant, + parallel_iterations + }); + + return _op.output; + } + + /// + /// Forwards the input to the output. + /// + /// + /// + /// + public static Tensor loop_cond(Tensor input, string name = null) + { + var _op = _op_def_lib._apply_op_helper("LoopCond", name, new { input }); + + return _op.output; + } + + /// + /// Makes its input available to the next iteration. + /// + /// + /// + /// + public static Tensor ref_next_iteration(Tensor data, string name = null) + { + var _op = _op_def_lib._apply_op_helper("RefNextIteration", name, new { data }); + + return _op; + } + + /// + /// Makes its input available to the next iteration. + /// + /// + /// + /// + public static Tensor next_iteration(Tensor data, string name = null) + { + var _op = _op_def_lib._apply_op_helper("NextIteration", name, new { data }); + + return _op; + } + + /// + /// Exits the current frame to its parent frame. + /// + /// + /// + /// + public static Tensor ref_exit(Tensor data, string name = null) + { + var _op = _op_def_lib._apply_op_helper("RefExit", name, new { data }); + + return _op; + } + + /// + /// Exits the current frame to its parent frame. + /// + /// + /// + /// + public static Tensor _exit(Tensor data, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Exit", name, new { data }); + + return _op; + } + public static Operation no_op(string name = null) { var _op = _op_def_lib._apply_op_helper("NoOp", name, null); diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index a5d26b23..1e2363e4 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -516,6 +516,9 @@ namespace Tensorflow }); } + public static Tensor minimum(Tx x, Ty y, string name = null) + => gen_math_ops.minimum(x, y, name: name); + public static Tensor maximum(Tx x, Ty y, string name = null) => gen_math_ops.maximum(x, y, name: name); diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.cs b/src/TensorFlowNET.Core/Tensors/Tensor.cs index 801ab233..d17e1f59 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.cs @@ -416,5 +416,6 @@ namespace Tensorflow } } + public int tensor_int_val { get; set; } } } diff --git a/test/TensorFlowNET.UnitTest/control_flow_ops_test/WhileContextTestCase.cs b/test/TensorFlowNET.UnitTest/control_flow_ops_test/WhileContextTestCase.cs index 682b826f..31109f0a 100644 --- a/test/TensorFlowNET.UnitTest/control_flow_ops_test/WhileContextTestCase.cs +++ b/test/TensorFlowNET.UnitTest/control_flow_ops_test/WhileContextTestCase.cs @@ -8,6 +8,18 @@ namespace TensorFlowNET.UnitTest.control_flow_ops_test [TestClass] public class WhileContextTestCase : PythonTest { + /// + /// https://www.tensorflow.org/api_docs/python/tf/while_loop + /// + [TestMethod] + public void SimpleWhileLoop() + { + var i = constant_op.constant(0, name: "i"); + var c = new Func(x => tf.less(x, 10, name: "c")); + var b = new Func(x => tf.add(x, 1, name: "c")); + var r = control_flow_ops.while_loop(c, b, new[] { i }); + } + private void _testWhileContextHelper(int? maximum_iterations = null) { // TODO: implement missing code dependencies @@ -17,7 +29,7 @@ namespace TensorFlowNET.UnitTest.control_flow_ops_test var c = new Func(x => gen_math_ops.less(x, 10, name: "c")); var b = new Func(x => gen_math_ops.add(x, 1, name: "c")); control_flow_ops.while_loop( - c, b, new[] { i }, maximum_iterations); + c, b, new[] { i }, maximum_iterations: maximum_iterations); foreach (Operation op in sess.graph.get_operations()) { var control_flow_context = op._get_control_flow_context(); From 174241d5a7e14ed78d96fffcc1bd9b2655aa1927 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Tue, 13 Aug 2019 22:11:03 -0500 Subject: [PATCH 14/14] update notes. --- src/TensorFlowNET.Core/TensorFlowNET.Core.csproj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj index 2afe9057..175486f1 100644 --- a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj +++ b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj @@ -5,7 +5,7 @@ TensorFlow.NET Tensorflow 1.14.0 - 0.10.10 + 0.10.11 Haiping Chen, Meinrad Recheis SciSharp STACK true @@ -17,7 +17,7 @@ TensorFlow, NumSharp, SciSharp, MachineLearning, TensorFlow.NET, C# Google's TensorFlow full binding in .NET Standard. Docs: https://tensorflownet.readthedocs.io - 0.10.10.0 + 0.10.11.0 Changes since v0.9.0: 1. Added full connected Convolution Neural Network example. @@ -35,9 +35,10 @@ Docs: https://tensorflownet.readthedocs.io 13. Fix default graph and operation issue when import model. 14. Fix TF_String endcode and decode. 15. Fix Tensor memory leak. -16. Rename with to tf_with that is only used to build graph purpose. +16. Rename with to tf_with that is only used to build graph purpose. +17. Graph inherit from DisposableObject. 7.2 - 0.10.10.0 + 0.10.11.0 LICENSE true true