From 448246b96bd451c20215a8b2b51c13ea9e8dc5fa Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Wed, 4 Dec 2019 16:09:03 +0200 Subject: [PATCH] Supported for forced singlethreading - Separated multithreading related methods to classname.threading.cs partial file - ops: Added enforce_singlethreading(), enforce_multithreading() --- src/TensorFlowNET.Core/Sessions/Session.cs | 3 +- src/TensorFlowNET.Core/ops.cs | 60 ------- src/TensorFlowNET.Core/ops.threading.cs | 152 ++++++++++++++++++ src/TensorFlowNET.Core/tensorflow.cs | 6 +- .../tensorflow.threading.cs | 53 ++++++ .../EnforcedSinglethreadingTests.cs | 107 ++++++++++++ .../MultithreadingTests.cs | 5 +- test/TensorFlowNET.UnitTest/UnitTest.csproj | 3 + .../Utilities/models/example1/saved_model.pb | Bin 0 -> 31263 bytes 9 files changed, 319 insertions(+), 70 deletions(-) create mode 100644 src/TensorFlowNET.Core/ops.threading.cs create mode 100644 src/TensorFlowNET.Core/tensorflow.threading.cs create mode 100644 test/TensorFlowNET.UnitTest/EnforcedSinglethreadingTests.cs create mode 100644 test/TensorFlowNET.UnitTest/Utilities/models/example1/saved_model.pb diff --git a/src/TensorFlowNET.Core/Sessions/Session.cs b/src/TensorFlowNET.Core/Sessions/Session.cs index caa669d3..c60a49c1 100644 --- a/src/TensorFlowNET.Core/Sessions/Session.cs +++ b/src/TensorFlowNET.Core/Sessions/Session.cs @@ -37,8 +37,7 @@ namespace Tensorflow public Session as_default() { - tf._defaultSessionFactory.Value = this; - return this; + return ops.set_default_session(this); } [MethodImpl(MethodImplOptions.NoOptimization)] diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index 02417594..633a9bf7 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -28,10 +28,6 @@ namespace Tensorflow { public partial class ops { - private static readonly ThreadLocal _defaultGraphFactory = new ThreadLocal(() => new DefaultGraphStack()); - - public static DefaultGraphStack default_graph_stack => _defaultGraphFactory.Value; - public static int tensor_id(Tensor tensor) { return tensor.Id; @@ -78,53 +74,6 @@ namespace Tensorflow return get_default_graph().get_collection_ref(key); } - /// - /// Returns the default graph for the current thread. - /// - /// The returned graph will be the innermost graph on which a - /// `Graph.as_default()` context has been entered, or a global default - /// graph if none has been explicitly created. - /// - /// NOTE: The default graph is a property of the current thread.If you - /// create a new thread, and wish to use the default graph in that - /// thread, you must explicitly add a `with g.as_default():` in that - /// thread's function. - /// - /// - public static Graph get_default_graph() - { - //TODO: original source indicates there should be a _default_graph_stack! - //return _default_graph_stack.get_default() - return default_graph_stack.get_controller(); - } - - public static Graph set_default_graph(Graph graph) - { - //TODO: original source does not have a 'set_default_graph' and indicates there should be a _default_graph_stack! - default_graph_stack.set_controller(graph); - return default_graph_stack.get_controller(); - } - - /// - /// Clears the default graph stack and resets the global default graph. - /// - /// NOTE: The default graph is a property of the current thread.This - /// function applies only to the current thread.Calling this function while - /// a `tf.Session` or `tf.InteractiveSession` is active will result in undefined - /// behavior. Using any previously created `tf.Operation` or `tf.Tensor` objects - /// after calling this function will result in undefined behavior. - /// - /// - public static void reset_default_graph() - { - //TODO: original source indicates there should be a _default_graph_stack! - //if (!_default_graph_stack.is_cleared()) - // throw new InvalidOperationException("Do not use tf.reset_default_graph() to clear " + - // "nested graphs. If you need a cleared graph, " + - // "exit the nesting and create a new graph."); - default_graph_stack.reset(); - } - public static Graph _get_graph_from_inputs(params Tensor[] op_input_list) => _get_graph_from_inputs(op_input_list: op_input_list, graph: null); @@ -399,15 +348,6 @@ namespace Tensorflow return session.run(tensor, feed_dict); } - /// - /// Returns the default session for the current thread. - /// - /// The default `Session` being used in the current thread. - public static Session get_default_session() - { - return tf.defaultSession; - } - /// /// Prepends name scope to a name. /// diff --git a/src/TensorFlowNET.Core/ops.threading.cs b/src/TensorFlowNET.Core/ops.threading.cs new file mode 100644 index 00000000..f8796596 --- /dev/null +++ b/src/TensorFlowNET.Core/ops.threading.cs @@ -0,0 +1,152 @@ +using System.Threading; +using Tensorflow.Util; +using static Tensorflow.Binding; + +namespace Tensorflow +{ + public partial class ops + { + private static readonly ThreadLocal _defaultGraphFactory = new ThreadLocal(() => new DefaultGraphStack()); + private static volatile Session _singleSesson; + private static volatile DefaultGraphStack _singleGraphStack; + private static readonly object _threadingLock = new object(); + + public static DefaultGraphStack default_graph_stack + { + get + { + if (!isSingleThreaded) + return _defaultGraphFactory.Value; + + if (_singleGraphStack == null) + { + lock (_threadingLock) + { + if (_singleGraphStack == null) + _singleGraphStack = new DefaultGraphStack(); + } + } + + return _singleGraphStack; + } + } + + private static bool isSingleThreaded = false; + + /// + /// Does this library ignore different thread accessing. + /// + /// https://github.com/SciSharp/TensorFlow.NET/wiki/Multithreading + public static bool IsSingleThreaded + { + get => isSingleThreaded; + set + { + if (value) + enforce_singlethreading(); + else + enforce_multithreading(); + } + } + + /// + /// Forces the library to ignore different thread accessing. + /// + /// https://github.com/SciSharp/TensorFlow.NET/wiki/Multithreading

Note that this discards any sessions and graphs used in a multithreaded manner + public static void enforce_singlethreading() + { + isSingleThreaded = true; + } + + /// + /// Forces the library to provide a separate and to every different thread accessing. + /// + /// https://github.com/SciSharp/TensorFlow.NET/wiki/Multithreading

Note that this discards any sessions and graphs used in a singlethreaded manner + public static void enforce_multithreading() + { + isSingleThreaded = false; + } + + /// + /// Returns the default session for the current thread. + /// + /// The default `Session` being used in the current thread. + public static Session get_default_session() + { + if (!isSingleThreaded) + return tf.defaultSession; + + if (_singleSesson == null) + { + lock (_threadingLock) + { + if (_singleSesson == null) + _singleSesson = new Session(); + } + } + + return _singleSesson; + } + + /// + /// Returns the default session for the current thread. + /// + /// The default `Session` being used in the current thread. + public static Session set_default_session(Session sess) + { + if (!isSingleThreaded) + return tf.defaultSession = sess; + + lock (_threadingLock) + { + _singleSesson = sess; + } + + return _singleSesson; + } + + /// + /// Returns the default graph for the current thread. + /// + /// The returned graph will be the innermost graph on which a + /// `Graph.as_default()` context has been entered, or a global default + /// graph if none has been explicitly created. + /// + /// NOTE: The default graph is a property of the current thread.If you + /// create a new thread, and wish to use the default graph in that + /// thread, you must explicitly add a `with g.as_default():` in that + /// thread's function. + /// + /// + public static Graph get_default_graph() + { + //return _default_graph_stack.get_default() + return default_graph_stack.get_controller(); + } + + public static Graph set_default_graph(Graph graph) + { + default_graph_stack.set_controller(graph); + return default_graph_stack.get_controller(); + } + + /// + /// Clears the default graph stack and resets the global default graph. + /// + /// NOTE: The default graph is a property of the current thread.This + /// function applies only to the current thread.Calling this function while + /// a `tf.Session` or `tf.InteractiveSession` is active will result in undefined + /// behavior. Using any previously created `tf.Operation` or `tf.Tensor` objects + /// after calling this function will result in undefined behavior. + /// + /// + public static void reset_default_graph() + { + //if (!_default_graph_stack.is_cleared()) + // throw new InvalidOperationException("Do not use tf.reset_default_graph() to clear " + + // "nested graphs. If you need a cleared graph, " + + // "exit the nesting and create a new graph."); + default_graph_stack.reset(); + } + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index 6ccf55f5..a42297b2 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -21,8 +21,6 @@ namespace Tensorflow { public partial class tensorflow : IObjectLife { - protected internal readonly ThreadLocal _defaultSessionFactory; - public TF_DataType @byte = TF_DataType.TF_UINT8; public TF_DataType @sbyte = TF_DataType.TF_INT8; public TF_DataType int16 = TF_DataType.TF_INT16; @@ -40,10 +38,10 @@ namespace Tensorflow public tensorflow() { - _defaultSessionFactory = new ThreadLocal(() => new Session()); + _constructThreadingObjects(); } - public Session defaultSession => _defaultSessionFactory.Value; + public RefVariable Variable(T data, bool trainable = true, diff --git a/src/TensorFlowNET.Core/tensorflow.threading.cs b/src/TensorFlowNET.Core/tensorflow.threading.cs new file mode 100644 index 00000000..33e925fd --- /dev/null +++ b/src/TensorFlowNET.Core/tensorflow.threading.cs @@ -0,0 +1,53 @@ +/***************************************************************************** + Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Tensorflow +{ + public partial class tensorflow : IObjectLife + { + protected ThreadLocal _defaultSessionFactory; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void _constructThreadingObjects() + { + _defaultSessionFactory = new ThreadLocal(() => new Session()); + } + + public Session defaultSession + { + get + { + if (!ops.IsSingleThreaded) + return _defaultSessionFactory.Value; + + return ops.get_default_session(); + } + internal set + { + if (!ops.IsSingleThreaded) + { + _defaultSessionFactory.Value = value; + return; + } + + ops.set_default_session(value); + } + } + } +} \ No newline at end of file diff --git a/test/TensorFlowNET.UnitTest/EnforcedSinglethreadingTests.cs b/test/TensorFlowNET.UnitTest/EnforcedSinglethreadingTests.cs new file mode 100644 index 00000000..b7efc116 --- /dev/null +++ b/test/TensorFlowNET.UnitTest/EnforcedSinglethreadingTests.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NumSharp; +using Tensorflow; +using Tensorflow.Util; +using static Tensorflow.Binding; + +namespace TensorFlowNET.UnitTest +{ + [TestClass] + public class EnforcedSinglethreadingTests : CApiTest + { + private static readonly object _singlethreadLocker = new object(); + + /// Initializes a new instance of the class. + public EnforcedSinglethreadingTests() + { + ops.IsSingleThreaded = true; + } + + [TestMethod, Ignore("Has to be tested manually.")] + public void SessionCreation() + { + lock (_singlethreadLocker) + { + ops.IsSingleThreaded.Should().BeTrue(); + + ops.uid(); //increment id by one + + //the core method + tf.peak_default_graph().Should().BeNull(); + + using (var sess = tf.Session()) + { + var default_graph = tf.peak_default_graph(); + var sess_graph = sess.GetPrivate("_graph"); + sess_graph.Should().NotBeNull(); + default_graph.Should().NotBeNull() + .And.BeEquivalentTo(sess_graph); + + var (graph, session) = Parallely(() => (tf.get_default_graph(), tf.get_default_session())); + + graph.Should().BeEquivalentTo(default_graph); + session.Should().BeEquivalentTo(sess); + } + } + } + + T Parallely(Func fnc) + { + var mrh = new ManualResetEventSlim(); + T ret = default; + Exception e = default; + new Thread(() => + { + try + { + ret = fnc(); + } catch (Exception ee) + { + e = ee; + throw; + } finally + { + mrh.Set(); + } + }).Start(); + + if (!Debugger.IsAttached) + mrh.Wait(10000).Should().BeTrue(); + else + mrh.Wait(-1); + e.Should().BeNull(e?.ToString()); + return ret; + } + + void Parallely(Action fnc) + { + var mrh = new ManualResetEventSlim(); + Exception e = default; + new Thread(() => + { + try + { + fnc(); + } catch (Exception ee) + { + e = ee; + throw; + } finally + { + mrh.Set(); + } + }).Start(); + + mrh.Wait(10000).Should().BeTrue(); + e.Should().BeNull(e.ToString()); + } + } +} \ No newline at end of file diff --git a/test/TensorFlowNET.UnitTest/MultithreadingTests.cs b/test/TensorFlowNET.UnitTest/MultithreadingTests.cs index f4f3f141..adae4fad 100644 --- a/test/TensorFlowNET.UnitTest/MultithreadingTests.cs +++ b/test/TensorFlowNET.UnitTest/MultithreadingTests.cs @@ -283,14 +283,11 @@ namespace TensorFlowNET.UnitTest } } - private static string modelPath = "./model/"; + private static readonly string modelPath = Path.GetFullPath("./Utilities/models/example1/"); [TestMethod] public void TF_GraphOperationByName_FromModel() { - if (!Directory.Exists(modelPath)) - return; - MultiThreadedUnitTestExecuter.Run(8, Core); //the core method diff --git a/test/TensorFlowNET.UnitTest/UnitTest.csproj b/test/TensorFlowNET.UnitTest/UnitTest.csproj index 58420b0a..6ff87f9d 100644 --- a/test/TensorFlowNET.UnitTest/UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/UnitTest.csproj @@ -43,6 +43,9 @@ PreserveNewest + + Always + diff --git a/test/TensorFlowNET.UnitTest/Utilities/models/example1/saved_model.pb b/test/TensorFlowNET.UnitTest/Utilities/models/example1/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..f37debb5a91fbd7c159d6f465425cea5eb6c2b35 GIT binary patch literal 31263 zcmeHQTZ|l6TAu3e>FNG^E|XbD&vy5CokW6gk#@Q4g1874gv3P%fdsrD1kth&JfQG^v>+q|;vyj-A;bd@`v6by zpZle5r^b`zC5i0uRMk2E_5A1j-#PVH#q+=REf4nlAzYLCKpkgm;c2KL6TR;KcyiF2`Q_OB}txm^rC76I=U2cy{~Z?mB5 zA<+o14f9B8Itx~$K=XZS+7+&5zdS=v7*85Y!0s?h!GEIfVAK2U7YyG)hV}=;SvZMk zX-(RtQ-N~3w6^RO8mC}^=%OD^{pxTqon1oipJ<$iI4v@p|_Lgy7!{CwIcz?HhN)y?cja^u4Kn2G;iEuiZn8D}x8&bYsC^ z2Or&ZHW*Pi;deF`Fv70%!(oUaW<2b7`-90;>eTx(dd{by$gPzlK(EpW{j=B{!{saE z?>>Z^P;U>hI2F&dDdN&JpJ zKpZ_LR+Xk<*q72Z>i9HXXsat{jOe9BpjGL&V&Wl2dl5>ng~KC%2`c+xZ+0{ZY1pqr zt+zKD+-H%3472m9`6{fmm)HzgU0%h>uLj=_3$KMtFbs@5km4xA;L4;azfOOeR@CA~ z0Q`xV$T5a0c0L2y&rRE9 z8uX_)8u`C`py8NI1yA5;q@Fy4@~z(eV^YIaMLudT!OE?>y-7dpe`YWYi6l7pRd#z* z`fn7oN|iY2ZxqqpV03gq=J-i}x>3YXSixC{KM2pl!maWC>;RWJXocbU&R|BZi2-xU zt`kef%}?+OHvPg;7-HPgR!E8Fx&Vt)_BaLtoF~+r78GFal861lEjssqCq^6et}qdlO9@IOnO1VpChZkTSvP&SAQ(Hz!cziZ!+lZ4$-7I zsOTDtjaOV}HLM|N&Hp%!RQJZCS#K~34VD^bp+@Ww_Nn*j+~UA$;JBRJ51Zw!%Uf42 zZ?!jeAHUE3`Sj+~cdlRGYWI8F&tBVleB1N?`2!Dr7#_JZ9Pjpq-RUemY`r`h%m%ok zdncT<@XvVav*kC6PW@Bh`*mFDb)M5YYt}?aC~Q`n z>#*GAOG!FXFo;8;GY9vernT`kjXAX^57GX7>@15T_z4eujarK@fB7;v&^Zax_0n7* zaft|)rG+HHKM!@Cl}Xs^`*NjXu=4PgLw&U)?9VqP|g~Jb@P#0i&*xhO|%X1ffOt|RA&U&TjmyqFNW0COx*4*5jhktaT zi%m>kERV$1Xoog~H=sla_+oYmP=A3&Q<%hSJ{jrdES%|_j;Ic~@!)qo*u;KuT;Jbn z-3uq9aA*uUTu-P9jJKzT(k84x8DCoUhcfakdc=&oR-o!`}E0i_V*ra=g^tC0y7t6L2gG8K3U8! zPb1>Qh-GvZ(P!t0-{PMN{6ju}nBuHSrp_Rj6ZIbCdwwtyQhQEPCk5mG$|u|bW``u{ z2##&-TwF%pO0l{)HjXKKrd|q3+2>DYUtXM)CON^WmlQ@|@X+%a! z6H_5JG1)`t4?I}6xUrWYHn%|z8#yt@*DX%N1%A|nQ#QlaLd9nI zhKyC}Suyj}ca|}1k-5CmEaB8R*Ssj_#&tO5`{vw8devt}FJ*C1fiJ?6!H?llCr76# zl31UuN-FS+x;dyOtOcyeSCFMPWvgtilUGX>n1OF7q-FVt$ySqfJsA_%RRG zP<3|S#&}Rsl*mT)X+KW}sjE-D#Ro)1Y6E`FqXuNXiVX>|GZDlwi2lfT9hgrS&>ei` zq6>VxDC+#=K}YA`6pX?<8ha5BCssB%mvXsUnGi~F@`9eaw@)s0Cuw^l4O2N&sk=7kBzBJhiqH5Gc|i74^LxPs zXsNrnmd2S=>h97J%@l;Irl#(qH&fH;2}uQAx~`}0qLVb)BB{GeBLx3ty65yfrh5s| zP-r7HM-enYOWnn_G~pttyGtX(+PTDZP2B|-895xYr|AV1L{L!CQT`#MrS77Wq|{y9m#MqBkYXbjjpQ2^TIw$8 zTWOg{-NpTrXF^Ne+pXNGdp$dKXT6%{Or9C*TI$YP8Xjfk+x0ZeATMDpb?2H$NZr|8 znY#0bBl~j78B$Vr&SaWGRO-$tSw12boxCft)OJu!8da6Lw{gkJ&INr~se8MXn!0Q2 ziafcj2fv_F_ZSk!%3Uo@`Knc>@i8sw8eZzzag9vUc)EPm()gIZ%{1QF(`6aDK9UmE zp{Mb&L=53Y8ZV*@1&3rBAKTtkL3TdRW<@QHm%S#W@$$Zcb7UGHbFaGJd90juczNPe4_B48$NUvzvE#w}LfD?$$z$>9t8 zRD`2Q#b);L2|R6AW+aq6^uR}8P*%+6Iver(9<*uibhd1A05DEsWu(E(zW?FDQ+Yg?fiO8Og(oO-w$Oo?lN0u5JXnngYoyL@w2ZLlFp82- zQv9(88xf9dKqks_kjrKwNO+Dx!a*Xy4d3-B<5togW&Fcn<>x$jOk*WCJR?q4#NlmL zzvirvJgZjVU0AW&8h&O$G?iGe&+5RPIPQPx#-SRqU>cjdAX|mXb}pJqQ_eeZ%971K zX<@T>a@%D2(AwUeomRhsK9|$rHX9ehRvma zZU%F1cZO{E;si#j(Si)w(twwThIWV~y#fbCkH#sg9@5u_V zarFuYej6Bq1f2EF6YJ=M!wV$j4GVcpBXylC2IZV#d z4TQsqyEF`ep2Rpo?hFjUg*!(K!KEv9XhWNfvkr!{ooODlGC&8x02mm8xSHl2F&KhN zGZRB_YT=w`ir$jWdYBl3csfnU7!1LsnPGh`t#)bajv!(he05ctlYKHX-$ogt2Rq-Mta?12$(lhfq7fg!NAhM%1zt)>M? z(9JiMJaLUAVhHTMh9U3<5fv&Iap68qIXZ^Gsac+*mb2{s$3+5Gi_H>C(Ux zyefj;MrzC&Zi;QK41xAMf`0@dSFB1ok!fM?aq-W{~t_Ag(w- z0W0w_n-ko`J~5&TDaMq-dUUR&ZF8I&xd}CdA+Zu(ixj;Raz&>F$C>e6D6{T-Szs?9 z2cDE*fh}pEGoVDkT=^Bhuu*W9rHm!y>ix<15Ub3zWW_gCQH3++-k;=!KyQ~>RinJe zQsNzCt~fVWAaDJ|o!vDExw{a)TKSaGG$kp~kD^3jr1wAgqd&{XSu%u_kE=BJ2)X*y zNpbZ}53W1w$^Z`RqbE7tLby~huOt^D$H11L)~Wh<@~e!?g#!Fd9`j3_D2K=3Ig=bu z>li||lP4`Q|1FPD5*IqAX>*8Y;6d5qWg74AJhFLVG2A%lH=)ohRc-fM?6?E zBr`JcY}IM$px4Zz)|5pQjiK>dST(`m!Nzz@4g+UXWHTgosj`PVybV~Bf)vgZQ;e0B zn@oy8;$`=o$tC=x2aj;txetL9_&YjrRnW9P^?591*VQ#5R+}U2{)ib-Sil{HDY}%w`8)w}BR=FNvtw${0OKRVqffe#w zw>KPiXQW;wR%fI)usAlovBiJC3UFsKK0567!$bZ|F#kF7NU%VEbhm;!`-kca-?&(t zVqMdgC>h$KMLC5D3?y&CpWB%~7-PX-J!C8-ZXb>Cq`ls?pHMIts~6~xF%Ad`wgXIx zM{09TGQAKh50Z`$WSPzqiJ3e(^=$A$@RkiqM-?+=EKjj0Aul@P!f*4F0w%q(vsEF= zeG;AN;IO;9w}17@_07HQ{*~>k?f$jhuzzi{y?@OVyH935?3rPe674H z7DSY)2ol*P&dD;lq-Easa4M?8tPt_odFVYXt3!CZWfr(4Z+lGoy!XaZs~_ISQoWrU zufEaQj5{`^TQ2!^xSu9vnPRB`9m?b&p) zBVLZ;ng_IeuM)uIoGu<;K(J-J@Z%$Cu6GBfHcOI2mn%jx_ z-<_3j6tJK#bpy(Z&uc&ju|6fnS=6-n0;B4Lz1F13!(kT$XRz`pSyTj3Q)$paY6QDGC==Mgw<;qAc@S7MYyKcmArN3M~*sSB2#{vYG#UwgPvCWBF0 z%~y5%b=M2hjEfZ)j6Na})N~u z9PL0bnE_Jf#|q$a9X2c(2e|4KMbcyDjk>A2&Q``z6MF1h@tS(-Ma&oE-D-oQw&X~y zQuthA^V3$e9`^Wzt(Pm33#At^g<;`2!=*Y`6d$QFiw-i>$acF)HBv%`8|tcLNMCeC zSh*@#Ier2AcTK|A6k#o8j}X2l2z#-W^R=ca{#hUik|FkWN!?n8*fzMX=x4)^S%@Q! zQ-h0V6c@4(B3*3RTs)J`h0!=g5A?MIgM)3!fml$90Ti*sw&EdfR7f7!(mcZ#*_#}e zI-Zrxh;@C!0DV@m5@!_73TvFAjyeJISV$CMYg4iy^dSaUuxgFYtxd&6ib6#D427`4 z`IiVRQZf_vAsO}s6$YjReC^3t4VKw5Mlc?%n7?mY545e{uUNldwSK>5{eIp0{Tb`` zZR_`E;c2YCNCJ%z1pyu|$Qol++@MKzqfj{)d=d+Wk|s9@TGH9{S^g_ldYw(~lcFK^ zcya!ccWv&u^Pkx+H2%E;Yxs#R{2&w}Qr-LfQz}#cn-%}BDhL#>S}uwNM12(D^gJqR zO^W0SH)0bok`(oA*abn*x=IjSB=?vkN1}p3jGmOFG?ZY% z)ilAxB|%3BEU$A)LNIy#W$nu@=1UNkU~H6Wg0TyNpb(5-wFKi=bz(}ecVSIif-plM z9d(4swpz>`PVaK0BSV=n0)F0jg_)?IXjSWmT7-|q$aPNkWA^jef(io|f{KemsY+0J zCnBi4qtna~RN7JrWLh(FJtAnTMeo51pRlDhW4GD@>nf+xG#Y1F@UwE0f?H2)Md+3@ zu5mIKv_*wMM7n0f`U=qx_@BjZuPtfc=Rxzud^spEt_gBlvp8~%KdC}hMYG?l`oB~C z@Th$hC&?F1%v zjt}wkfcQaEwK3TINAlSf{0<#{LKeYT_QlE%u-pXw4B0vIZLQ8`-3y{07=w@VPl!Fj zN=+p^=5NCieu{Osw>uajo48&1I4(}vrv~|EX?)9q{|Q*2->kzn=b*y2h4I;HU~a<( X0oyj&ACAA+;-a+h1Caf}-t7MX()Fs8 literal 0 HcmV?d00001