From 675b93a9d752b300313c007069518dc75bf9784a Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Sat, 17 Jun 2023 23:10:37 +0800 Subject: [PATCH] fix: none gradient error when training LSTM. --- src/TensorFlowNET.Core/APIs/tf.tensor.cs | 6 +- src/TensorFlowNET.Core/Common/Types/Nest.cs | 18 +- .../Eager/EagerRunner.TFE_FastPathExecute.cs | 6 +- .../Eager/EagerRunner.TFE_TapeGradient.cs | 8 +- .../Gradients/array_grad.cs | 5 +- .../Keras/ArgsDefinition/Rnn/LSTMArgs.cs | 2 - .../Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 26 +- .../ArgsDefinition/Rnn/StackedRNNCellsArgs.cs | 3 +- .../Keras/Layers/ILayersApi.cs | 2 +- .../Operations/NnOps/BasicLSTMCell.cs | 2 +- .../Operations/OpDefLibrary.cs | 9 +- .../Operations/_GraphTensorArray.cs | 3 +- .../Operations/array_ops.cs | 33 +- .../Operations/gen_resource_variable_ops.cs | 1573 +++++++++++++++-- .../Operations/image_ops_impl.cs | 6 +- src/TensorFlowNET.Core/Operations/while_v2.cs | 4 +- .../Variables/BaseResourceVariable.cs | 23 +- src/TensorFlowNET.Keras/Engine/Layer.Apply.cs | 2 + src/TensorFlowNET.Keras/Layers/LayersApi.cs | 15 +- src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs | 102 +- .../Layers/Rnn/LSTMCell.cs | 17 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 54 +- .../Layers/Rnn/SimpleRNN.cs | 7 +- .../Layers/Rnn/SimpleRNNCell.cs | 5 - .../Layers/Rnn/StackedRNNCells.cs | 12 +- .../Layers/Rnn.Test.cs | 74 +- tools/Tensorflow.CodeGen/OpClassifier.cs | 2 +- tools/Tensorflow.CodeGen/Utils.cs | 17 +- 29 files changed, 1743 insertions(+), 295 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.tensor.cs b/src/TensorFlowNET.Core/APIs/tf.tensor.cs index be8c2ab2..45aebc0c 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tensor.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tensor.cs @@ -71,15 +71,15 @@ namespace Tensorflow public Tensor[] split(Tensor value, int num_split, Tensor axis, string name = null) => array_ops.split( value: value, - num_split: num_split, + num_or_size_splits: num_split, axis: axis, name: name); public Tensor[] split(Tensor value, int num_split, int axis, string name = null) => array_ops.split( value: value, - num_split: num_split, - axis: axis, + num_or_size_splits: num_split, + axis: ops.convert_to_tensor(axis), name: name); public Tensor ensure_shape(Tensor x, Shape shape, string name = null) diff --git a/src/TensorFlowNET.Core/Common/Types/Nest.cs b/src/TensorFlowNET.Core/Common/Types/Nest.cs index 4de7d1fa..89ce29f2 100644 --- a/src/TensorFlowNET.Core/Common/Types/Nest.cs +++ b/src/TensorFlowNET.Core/Common/Types/Nest.cs @@ -197,25 +197,11 @@ namespace Tensorflow.Common.Types } else if(NestType is NestType.List) { - foreach(var item in ListValue!) - { - if(item.NestType is NestType.List or NestType.Dictionary) - { - return true; - } - } - return false; + return ListValue!.Count > 0; } else { - foreach (var item in DictValue!.Values) - { - if (item.NestType is NestType.List or NestType.Dictionary) - { - return true; - } - } - return false; + return DictValue!.Count > 0; } } diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs index 5f156fd9..0ce55841 100644 --- a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs +++ b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs @@ -352,7 +352,11 @@ namespace Tensorflow.Eager c_api.TFE_OpSetAttrFloat(op, key, Convert.ToSingle(value)); break; case TF_AttrType.TF_ATTR_SHAPE: - var dims = (value as long[]).ToArray(); + long[] dims; + if (value is Shape shape) dims = shape.dims.ToArray(); + else if (value is long[] longs) dims = longs.ToArray(); + else if (value is int[] ints) dims = ints.Select(x => (long)x).ToArray(); + else dims = ((long[])value).ToArray(); c_api.TFE_OpSetAttrShape(op, key, dims, dims.Length, status); status.Check(true); break; diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs index 1f7b3ae6..849dcb3f 100644 --- a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs +++ b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs @@ -137,7 +137,6 @@ namespace Tensorflow.Eager { dims[i] = c_api.TFE_TensorHandleDim(handle, i, status); } - Shape tensor_shape = new(dims); if(status.Code != TF_Code.TF_OK) { @@ -145,6 +144,7 @@ namespace Tensorflow.Eager } else { + Shape tensor_shape = new(dims); return new TapeTensor(id, dtype, tensor_shape); } } @@ -173,8 +173,12 @@ namespace Tensorflow.Eager return dtype == dtypes.variant || dtype == dtypes.resource; } - bool ListContainNone(long[] list) + bool ListContainNone(long[]? list) { + if(list is null) + { + return true; + } int len = list.Length; if(len == 0) { diff --git a/src/TensorFlowNET.Core/Gradients/array_grad.cs b/src/TensorFlowNET.Core/Gradients/array_grad.cs index f939f7b6..1b6bc95e 100644 --- a/src/TensorFlowNET.Core/Gradients/array_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/array_grad.cs @@ -90,8 +90,7 @@ namespace Tensorflow.Gradients ? input_values[0].rank + dim_int : dim_int % input_values[0].rank; var sizes = input_values.Select(x => x.shape[non_neg_concat_dim]).ToArray(); - var sizes_tensor = constant_op.constant(sizes); - out_grads = array_ops.split(grad, sizes_tensor, non_neg_concat_dim).ToList(); + out_grads = array_ops.split(grad, sizes.Select(x => (int)x).ToArray(), ops.convert_to_tensor(non_neg_concat_dim)).ToList(); } else if (constant_op.is_constant(concat_dim)) { @@ -127,7 +126,7 @@ namespace Tensorflow.Gradients new Tensor[] { non_neg_concat_dim, tf.constant(0) }, new Tensor[] { tf.constant(1), tf.constant(-1) }); var squeeze_sizes = array_ops.squeeze(slice); - out_grads = array_ops.split(axis: grad, value: squeeze_sizes, num_split: (int)non_neg_concat_dim).ToList(); + out_grads = array_ops.split(axis: grad, value: squeeze_sizes, num_or_size_splits: (int)non_neg_concat_dim).ToList(); } else { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs index 76464147..db76fda0 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs @@ -4,8 +4,6 @@ { // TODO: maybe change the `RNNArgs` and implement this class. public bool UnitForgetBias { get; set; } - public float Dropout { get; set; } - public float RecurrentDropout { get; set; } public int Implementation { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs index 786236e4..1b26c05c 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs @@ -29,7 +29,7 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn [JsonProperty("unit_forget_bias")] public bool UnitForgetBias { get; set; } = true; [JsonProperty("implementation")] - public int Implementation { get; set; } = 2; + public int Implementation { get; set; } = 1; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index 116ff7a2..2d7fb001 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -7,12 +7,6 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn // TODO(Rinne): add regularizers. public class RNNArgs : AutoSerializeLayerArgs { - [JsonProperty("cell")] - // TODO: the cell should be serialized with `serialize_keras_object`. - public IRnnCell Cell { get; set; } = null; - [JsonProperty("cells")] - public IList Cells { get; set; } = null; - [JsonProperty("return_sequences")] public bool ReturnSequences { get; set; } = false; [JsonProperty("return_state")] @@ -25,8 +19,10 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn public bool Unroll { get; set; } = false; [JsonProperty("time_major")] public bool TimeMajor { get; set; } = false; + + public int? InputDim { get; set; } + public int? InputLength { get; set; } // TODO: Add `num_constants` and `zero_output_for_mask`. - public Dictionary Kwargs { get; set; } = null; public int Units { get; set; } public Activation Activation { get; set; } @@ -38,21 +34,5 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn public float Dropout { get; set; } = .0f; public bool ZeroOutputForMask { get; set; } = false; public float RecurrentDropout { get; set; } = .0f; - - // kernel_regularizer=None, - // recurrent_regularizer=None, - // bias_regularizer=None, - // activity_regularizer=None, - // kernel_constraint=None, - // recurrent_constraint=None, - // bias_constraint=None, - // dropout=0., - // recurrent_dropout=0., - // return_sequences=False, - // return_state=False, - // go_backwards=False, - // stateful=False, - // unroll=False, - // **kwargs): } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs index ea6f830b..50a6127d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs @@ -5,7 +5,6 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn { public class StackedRNNCellsArgs : LayerArgs { - public IList Cells { get; set; } - public Dictionary Kwargs { get; set; } = null; + public bool ReverseStateOrder = false; } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index a19508d4..1eb08e77 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -182,7 +182,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 2, + int implementation = 1, bool return_sequences = false, bool return_state = false, bool go_backwards = false, diff --git a/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs index b2cda952..16cbd001 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs @@ -89,7 +89,7 @@ namespace Tensorflow gate_inputs = nn_ops.bias_add(gate_inputs, _bias); // i = input_gate, j = new_input, f = forget_gate, o = output_gate - var tensors = array_ops.split(value: gate_inputs, num_split: 4, axis: one); + var tensors = array_ops.split(value: gate_inputs, num_or_size_splits: 4, axis: one); var (i, j, f, o) = (tensors[0], tensors[1], tensors[2], tensors[3]); var forget_bias_tensor = constant_op.constant(_forget_bias, dtype: f.dtype); diff --git a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs index 5ff5ccff..29e1f074 100644 --- a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs +++ b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs @@ -389,9 +389,13 @@ namespace Tensorflow case "list(type)": attr_value.List.Type.AddRange((value as IList).Select(x => _MakeType(x, attr_def))); break; + case "list(float)": + if (value != null) + attr_value.List.F.AddRange((value as IEnumerable).ToArray()); + break; case "list(int)": if (value != null) - attr_value.List.I.AddRange((value as int[]).Select(x => Convert.ToInt64(x))); + attr_value.List.I.AddRange((value as IEnumerable).Select(x => Convert.ToInt64(x))); break; case "bool": attr_value.B = (bool)value; @@ -428,6 +432,9 @@ namespace Tensorflow case "list(func)": attr_value.List.Func.AddRange(_MakeFuncList(value, attr_def.Name)); break; + case "list(string)": + attr_value.List.S.AddRange((value as IEnumerable).Select(x => ByteString.CopyFromUtf8(x))); + break; default: throw new TypeError($"SetAttrValue: can't not convert attr_def.Type '{attr_def.Type}' to protos."); } diff --git a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs index 4c3fde31..2384e814 100644 --- a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs +++ b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs @@ -390,7 +390,8 @@ namespace Tensorflow.Operations int ta_size; if(!_dynamic_size && (_size is not null)) { - ta_size = (int)tensor_util.constant_value(_size); + var size_tensor = tensor_util.constant_value(_size); + ta_size = size_tensor is null ? -1 : (int)size_tensor; } else { diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index c4ec974b..6b4fea63 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -1014,38 +1014,27 @@ namespace Tensorflow }); } - public static Tensor[] split(Tensor value, Tensor size_splits, int axis, int num = -1, + public static Tensor[] split(Tensor value, int num_or_size_splits, Tensor axis = null, string name = "split") { - if (num == -1) - num = (int)size_splits.shape[0]; - - return gen_array_ops.split_v(value, size_splits, tf.convert_to_tensor(axis), num, name: name); + return gen_array_ops.split(split_dim: axis, value: value, num_split: num_or_size_splits, name); } - public static Tensor[] split(Tensor value, int num_split, T axis, + public static Tensor[] split(Tensor value, int[] num_or_size_splits, Tensor axis = null, int num = -1, string name = "split") { - var size_splits = ops.convert_to_tensor(num_split); - - if (tf.Context.executing_eagerly()) + if(num_or_size_splits.Length == 0) { - return split_eager_fallback(axis, value, num_split: num_split, name: name, ctx: tf.Context); + throw new ValueError("Rank-0 tensors are not supported as the num_or_size_splits argument to split."); } + var size_splits = ops.convert_to_tensor(num_or_size_splits); - var _op = tf.OpDefLib._apply_op_helper("Split", name, new { split_dim = axis, value, num_split }); - return _op.outputs; - } - - private static Tensor[] split_eager_fallback(Ta axis, Tv value, int num_split, string name, Context ctx = null) - { - var (_attr_T, input) = tf.Runner.ArgsToMatchingEager(ctx, args: new object[] { value }); - var axis_tensor = ops.convert_to_tensor(axis, dtype: TF_DataType.TF_INT32); - var _inputs_flat = new List { axis_tensor }; - _inputs_flat.AddRange(input); - var _attrs = new object[] { "num_split", num_split, "T", _attr_T }; + if(num == -1) + { + num = (int)size_splits.shape[0]; + } - return tf.Runner.Execute(ctx, "Split", num_split, _inputs_flat.ToArray(), _attrs, name: name); + return gen_array_ops.split_v(value: value, size_splits: size_splits, split_dim: axis, num_split: num, name: name); } public static Tensor slice(Tensor input, Tensor[] begin, Tensor[] size, string name = null) diff --git a/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs b/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs index c4e8f8c4..db5f6813 100644 --- a/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs @@ -1,158 +1,1523 @@ -/***************************************************************************** - Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ +/*Wrappers around TensorFlow ops. This file is MACHINE GENERATED! Do not edit.*/ +using Tensorflow.Eager; +using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; -namespace Tensorflow +namespace Tensorflow; + +public static class gen_resource_variable_ops { - public static class gen_resource_variable_ops + /// + /// Adds a value to the current value of a variable. + /// + /// + /// + /// Any ReadVariableOp with a control dependency on this op is guaranteed to + /// see the incremented value or a subsequent newer one. + /// + /// + /// + /// + /// + public static Operation assign_add_variable_op(Tensor resource, Tensor value, string? name = null) { - public static Operation assign_sub_variable_op(Tensor resource, Tensor value, string name = null) + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) { - if (tf.Context.executing_eagerly()) + try { - tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo( - tf.Context, "AssignSubVariableOp", name, resource, value)); - + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AssignAddVariableOp", name) { args = new object[] { resource, value }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return assign_add_variable_op_eager_fallback(resource, value, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["value"] = value; + var _op = tf.OpDefLib._apply_op_helper("AssignAddVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype") }; + _execute.record_gradient("AssignAddVariableOp", _op.inputs, _attrs, _result); + } + return _op; + } - return null; + public static Operation assign_add_variable_op_eager_fallback(Tensor resource, Tensor value, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, value }; + object[] _attrs = new object[] { "dtype", value.dtype }; + var _result = _execute.execute("AssignAddVariableOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AssignAddVariableOp", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Subtracts a value from the current value of a variable. + /// + /// + /// + /// Any ReadVariableOp with a control dependency on this op is guaranteed to + /// see the decremented value or a subsequent newer one. + /// + /// + /// + /// + /// + public static Operation assign_sub_variable_op(Tensor resource, Tensor value, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AssignSubVariableOp", name) { args = new object[] { resource, value }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return assign_sub_variable_op_eager_fallback(resource, value, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["value"] = value; + var _op = tf.OpDefLib._apply_op_helper("AssignSubVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype") }; + _execute.record_gradient("AssignSubVariableOp", _op.inputs, _attrs, _result); + } + return _op; + } - /// - /// Adds a value to the current value of a variable. - /// - /// - /// - /// - /// - public static Operation assign_add_variable_op(Tensor resource, Tensor value, string name = null) + public static Operation assign_sub_variable_op_eager_fallback(Tensor resource, Tensor value, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, value }; + object[] _attrs = new object[] { "dtype", value.dtype }; + var _result = _execute.execute("AssignSubVariableOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AssignSubVariableOp", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Assigns a new value to a variable. + /// + /// + /// + /// Any ReadVariableOp with a control dependency on this op is guaranteed to return + /// this value or a subsequent newer value of the variable. + /// + /// + /// + /// + /// + /// + public static Operation assign_variable_op(Tensor resource, Tensor value, bool validate_shape = false, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) { - if (tf.Context.executing_eagerly()) + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AssignVariableOp", name) { args = new object[] { resource, value }, attrs = new Dictionary() { ["validate_shape"] = validate_shape } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) { - tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "AssignAddVariableOp", name, - resource, value)); + } + try + { + return assign_variable_op_eager_fallback(resource, value, validate_shape: validate_shape, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["value"] = value; + keywords["validate_shape"] = validate_shape; + var _op = tf.OpDefLib._apply_op_helper("AssignVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "validate_shape", _op._get_attr_bool("validate_shape") }; + _execute.record_gradient("AssignVariableOp", _op.inputs, _attrs, _result); + } + return _op; + } + public static Operation assign_variable_op_eager_fallback(Tensor resource, Tensor value, bool validate_shape, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, value }; + object[] _attrs = new object[] { "dtype", value.dtype, "validate_shape", validate_shape }; + var _result = _execute.execute("AssignVariableOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AssignVariableOp", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// This op consumes a lock created by `MutexLock`. + /// + /// + /// + /// This op exists to consume a tensor created by `MutexLock` (other than + /// direct control dependencies). It should be the only that consumes the tensor, + /// and will raise an error if it is not. Its only purpose is to keep the + /// mutex lock tensor alive until it is consumed by this op. + /// + /// **NOTE**: This operation must run on the same device as its input. This may + /// be enforced via the `colocate_with` mechanism. + /// + /// + /// + /// + public static Operation consume_mutex_lock(Tensor mutex_lock, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ConsumeMutexLock", name) { args = new object[] { mutex_lock }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return consume_mutex_lock_eager_fallback(mutex_lock, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["mutex_lock"] = mutex_lock; + var _op = tf.OpDefLib._apply_op_helper("ConsumeMutexLock", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("ConsumeMutexLock", _op.inputs, _attrs, _result); + } + return _op; + } - var _op = tf.OpDefLib._apply_op_helper("AssignAddVariableOp", name, new { resource, value }); + public static Operation consume_mutex_lock_eager_fallback(Tensor mutex_lock, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { mutex_lock }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("ConsumeMutexLock", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ConsumeMutexLock", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Deletes the resource specified by the handle. + /// + /// + /// + /// All subsequent operations using the resource will result in a NotFound + /// error status. + /// + /// + /// + /// + /// + /// whether to ignore the error when the resource + /// doesn't exist. + /// + /// + /// + public static Operation destroy_resource_op(Tensor resource, bool ignore_lookup_error = true, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DestroyResourceOp", name) { args = new object[] { resource }, attrs = new Dictionary() { ["ignore_lookup_error"] = ignore_lookup_error } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return destroy_resource_op_eager_fallback(resource, ignore_lookup_error: ignore_lookup_error, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["ignore_lookup_error"] = ignore_lookup_error; + var _op = tf.OpDefLib._apply_op_helper("DestroyResourceOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "ignore_lookup_error", _op._get_attr_bool("ignore_lookup_error") }; + _execute.record_gradient("DestroyResourceOp", _op.inputs, _attrs, _result); + } + return _op; + } - return _op; + public static Operation destroy_resource_op_eager_fallback(Tensor resource, bool ignore_lookup_error, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { "ignore_lookup_error", ignore_lookup_error }; + var _result = _execute.execute("DestroyResourceOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("DestroyResourceOp", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Turns off the copy-on-read mode. + /// + /// + /// + /// Turns off the copy-on-read mode of a resource variable. If the variable is not in copy-on-read mode, this op has no effect. + /// + /// + /// + /// + public static Operation disable_copy_on_read(Tensor resource, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DisableCopyOnRead", name) { args = new object[] { resource }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return disable_copy_on_read_eager_fallback(resource, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + var _op = tf.OpDefLib._apply_op_helper("DisableCopyOnRead", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("DisableCopyOnRead", _op.inputs, _attrs, _result); + } + return _op; + } - public static Operation assign_variable_op(Tensor resource, Tensor value, string name = null) + public static Operation disable_copy_on_read_eager_fallback(Tensor resource, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("DisableCopyOnRead", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - if (tf.Context.executing_eagerly()) + _execute.record_gradient("DisableCopyOnRead", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Locks a mutex resource. The output is the lock. So long as the lock tensor + /// + /// + /// + /// is alive, any other request to use `MutexLock` with this mutex will wait. + /// + /// This is particularly useful for creating a critical section when used in + /// conjunction with `MutexLockIdentity`: + /// + /// ```python + /// + /// mutex = mutex_v2( + /// shared_name=handle_name, container=container, name=name) + /// + /// def execute_in_critical_section(fn, *args, **kwargs): + /// lock = gen_resource_variable_ops.mutex_lock(mutex) + /// + /// with ops.control_dependencies([lock]): + /// r = fn(*args, **kwargs) + /// + /// with ops.control_dependencies(nest.flatten(r)): + /// with ops.colocate_with(mutex): + /// ensure_lock_exists = mutex_lock_identity(lock) + /// + /// # Make sure that if any element of r is accessed, all of + /// # them are executed together. + /// r = nest.map_structure(tf.identity, r) + /// + /// with ops.control_dependencies([ensure_lock_exists]): + /// return nest.map_structure(tf.identity, r) + /// ``` + /// + /// While `fn` is running in the critical section, no other functions which wish to + /// use this critical section may run. + /// + /// Often the use case is that two executions of the same graph, in parallel, + /// wish to run `fn`; and we wish to ensure that only one of them executes + /// at a time. This is especially important if `fn` modifies one or more + /// variables at a time. + /// + /// It is also useful if two separate functions must share a resource, but we + /// wish to ensure the usage is exclusive. + /// + /// + /// + /// + public static Tensor mutex_lock(Tensor mutex, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MutexLock", name) { args = new object[] { mutex }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try { - tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "AssignVariableOp", name, - resource, value)); + return mutex_lock_eager_fallback(mutex, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["mutex"] = mutex; + var _op = tf.OpDefLib._apply_op_helper("MutexLock", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("MutexLock", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return null; + public static Tensor mutex_lock_eager_fallback(Tensor mutex, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { mutex }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("MutexLock", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("MutexLock", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Creates a Mutex resource that can be locked by `MutexLock`. + /// + /// + /// + /// If non-empty, this variable is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this variable is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// + public static Tensor mutex_v2(string container = "", string shared_name = "", string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MutexV2", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { } + try + { + return mutex_v2_eager_fallback(container: container, shared_name: shared_name, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } + Dictionary keywords = new(); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("MutexV2", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "container", _op.get_attr("container"), "shared_name", _op.get_attr("shared_name") }; + _execute.record_gradient("MutexV2", _op.inputs, _attrs, _result); + } + return _result[0]; + } - var _op = tf.OpDefLib._apply_op_helper("AssignVariableOp", name, new { resource, value }); + public static Tensor mutex_v2_eager_fallback(string container, string shared_name, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { }; + object[] _attrs = new object[] { "container", container, "shared_name", shared_name }; + var _result = _execute.execute("MutexV2", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("MutexV2", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Reads the value of a variable. + /// + /// + /// + /// The tensor returned by this operation is immutable. + /// + /// The value returned by this operation is guaranteed to be influenced by all the + /// writes on which this operation depends directly or indirectly, and to not be + /// influenced by any of the writes which depend directly or indirectly on this + /// operation. + /// + /// + /// + /// + /// + /// the dtype of the value. + /// + /// + /// + public static Tensor read_variable_op(Tensor resource, TF_DataType dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReadVariableOp", name) { args = new object[] { resource }, attrs = new Dictionary() { ["dtype"] = dtype } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return read_variable_op_eager_fallback(resource, dtype: dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["dtype"] = dtype; + var _op = tf.OpDefLib._apply_op_helper("ReadVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype") }; + _execute.record_gradient("ReadVariableOp", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return _op; + public static Tensor read_variable_op_eager_fallback(Tensor resource, TF_DataType dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { "dtype", dtype }; + var _result = _execute.execute("ReadVariableOp", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ReadVariableOp", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Gather slices from the variable pointed to by `resource` according to `indices`. + /// + /// + /// + /// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D). + /// Produces an output tensor with shape `indices.shape + params.shape[1:]` where: + /// + /// ```python + /// # Scalar indices + /// output[:, ..., :] = params[indices, :, ... :] + /// + /// # Vector indices + /// output[i, :, ..., :] = params[indices[i], :, ... :] + /// + /// # Higher rank indices + /// output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :] + /// ``` + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor resource_gather(Tensor resource, Tensor indices, TF_DataType dtype, int batch_dims = 0, bool validate_indices = true, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceGather", name) { args = new object[] { resource, indices }, attrs = new Dictionary() { ["batch_dims"] = batch_dims, ["validate_indices"] = validate_indices, ["dtype"] = dtype } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_gather_eager_fallback(resource, indices, batch_dims: batch_dims, validate_indices: validate_indices, dtype: dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["batch_dims"] = batch_dims; + keywords["validate_indices"] = validate_indices; + keywords["dtype"] = dtype; + var _op = tf.OpDefLib._apply_op_helper("ResourceGather", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "batch_dims", _op._get_attr_int("batch_dims"), "validate_indices", _op._get_attr_bool("validate_indices"), "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceGather", _op.inputs, _attrs, _result); } + return _result[0]; + } - public static Tensor var_is_initialized_op(Tensor resource, string name = null) + public static Tensor resource_gather_eager_fallback(Tensor resource, Tensor indices, int batch_dims, bool validate_indices, TF_DataType dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices }; + object[] _attrs = new object[] { "batch_dims", batch_dims, "validate_indices", validate_indices, "dtype", dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceGather", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - if (tf.Context.executing_eagerly()) + _execute.record_gradient("ResourceGather", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// + /// + /// + /// + /// + /// + public static Tensor resource_gather_nd(Tensor resource, Tensor indices, TF_DataType dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try { - var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "VarIsInitializedOp", name, - resource)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceGatherNd", name) { args = new object[] { resource, indices }, attrs = new Dictionary() { ["dtype"] = dtype } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_gather_nd_eager_fallback(resource, indices, dtype: dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["dtype"] = dtype; + var _op = tf.OpDefLib._apply_op_helper("ResourceGatherNd", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceGatherNd", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return results[0]; + public static Tensor resource_gather_nd_eager_fallback(Tensor resource, Tensor indices, TF_DataType dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices }; + object[] _attrs = new object[] { "dtype", dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceGatherNd", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceGatherNd", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Adds sparse updates to the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] += updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] += updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] += updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions add. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_add(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterAdd", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; } + catch (Exception) + { + } + try + { + return resource_scatter_add_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterAdd", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterAdd", _op.inputs, _attrs, _result); + } + return _op; + } - var _op = tf.OpDefLib._apply_op_helper("VarIsInitializedOp", name, new { resource }); + public static Operation resource_scatter_add_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterAdd", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterAdd", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Divides sparse updates into the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] /= updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] /= updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] /= updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions multiply. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_div(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterDiv", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_div_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterDiv", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterDiv", _op.inputs, _attrs, _result); + } + return _op; + } - return _op.output; + public static Operation resource_scatter_div_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterDiv", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterDiv", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Reduces sparse updates into the variable referenced by `resource` using the `max` operation. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] = max(ref[indices, ...], updates[...]) + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] = max(ref[indices[i], ...], updates[i, ...]) + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] = max(ref[indices[i, ..., j], ...], updates[i, ..., j, ...]) + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions are combined. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_max(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterMax", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_max_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterMax", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterMax", _op.inputs, _attrs, _result); + } + return _op; + } - /// - /// Creates a handle to a Variable resource. - /// - /// - /// - /// - /// - /// - /// - public static Tensor var_handle_op(TF_DataType dtype, Shape shape, - string container = "", string shared_name = "", string name = null) + public static Operation resource_scatter_max_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterMax", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - if (tf.Context.executing_eagerly()) + _execute.record_gradient("ResourceScatterMax", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Reduces sparse updates into the variable referenced by `resource` using the `min` operation. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] = min(ref[indices, ...], updates[...]) + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] = min(ref[indices[i], ...], updates[i, ...]) + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] = min(ref[indices[i, ..., j], ...], updates[i, ..., j, ...]) + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions are combined. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_min(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterMin", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_min_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) { - var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "VarHandleOp", name) - { - attrs = ConvertToDict(new - { - dtype, - shape = shape.dims, - container, - shared_name, - allowed_devices = new string[0] - }) - }); + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterMin", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterMin", _op.inputs, _attrs, _result); + } + return _op; + } - return results[0]; + public static Operation resource_scatter_min_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterMin", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterMin", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Multiplies sparse updates into the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] *= updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] *= updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] *= updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions multiply. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_mul(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterMul", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_mul_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterMul", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterMul", _op.inputs, _attrs, _result); + } + return _op; + } - var _op = tf.OpDefLib._apply_op_helper("VarHandleOp", name, new + public static Operation resource_scatter_mul_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterMul", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterMul", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Subtracts sparse updates from the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] -= updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] -= updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions add. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_sub(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterSub", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_sub_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) { - dtype, - shape, - container, - shared_name - }); + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterSub", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterSub", _op.inputs, _attrs, _result); + } + return _op; + } - return _op.output; + public static Operation resource_scatter_sub_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterSub", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterSub", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Assigns sparse updates to the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] = updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] = updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] + /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_update(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterUpdate", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_update_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterUpdate", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterUpdate", _op.inputs, _attrs, _result); + } + return _op; + } - public static Tensor destroy_resource_op(Tensor resource, bool ignore_lookup_error = true, string name = null) - => tf.Context.ExecuteOp("DestroyResourceOp", name, - new ExecuteOpArgs(resource).SetAttributes(new { ignore_lookup_error })); + public static Operation resource_scatter_update_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterUpdate", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterUpdate", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Creates a handle to a Variable resource. + /// + /// + /// + /// the container this variable is placed in. + /// + /// + /// + /// + /// the name by which this variable is referred to. + /// + /// + /// + /// + /// the type of this variable. Must agree with the dtypes + /// of all ops using this variable. + /// + /// + /// + /// + /// The (possibly partially specified) shape of this variable. + /// + /// + /// + /// + /// DEPRECATED. The allowed devices containing the resource variable. Set when the + /// output ResourceHandle represents a per-replica/partitioned resource variable. + /// + /// + /// + public static Tensor var_handle_op(TF_DataType dtype, Shape shape, string container = "", string shared_name = "", string[] allowed_devices = null, string? name = null) + { + var _ctx = tf.Context; + if (allowed_devices is null) + { + allowed_devices = new string[] { }; + } + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "VarHandleOp", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name, ["dtype"] = dtype, ["shape"] = shape, ["allowed_devices"] = allowed_devices } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return var_handle_op_eager_fallback(container: container, shared_name: shared_name, dtype: dtype, shape: shape, allowed_devices: allowed_devices, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } + Dictionary keywords = new(); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + keywords["dtype"] = dtype; + keywords["shape"] = shape; + keywords["allowed_devices"] = allowed_devices; + var _op = tf.OpDefLib._apply_op_helper("VarHandleOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "container", _op.get_attr("container"), "shared_name", _op.get_attr("shared_name"), "dtype", _op._get_attr_type("dtype"), "shape", _op.get_attr("shape"), "allowed_devices", _op.get_attr("allowed_devices") }; + _execute.record_gradient("VarHandleOp", _op.inputs, _attrs, _result); + } + return _result[0]; + } - /// - /// Reads the value of a variable. - /// - /// - /// - /// - /// - public static Tensor read_variable_op(Tensor resource, TF_DataType dtype, string name = null) - => tf.Context.ExecuteOp("ReadVariableOp", name, new ExecuteOpArgs(resource) - .SetAttributes(new { dtype })); + public static Tensor var_handle_op_eager_fallback(string container, string shared_name, TF_DataType dtype, Shape shape, string[] allowed_devices, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { }; + object[] _attrs = new object[] { "container", container, "shared_name", shared_name, "dtype", dtype, "shape", shape, "allowed_devices", allowed_devices }; + var _result = _execute.execute("VarHandleOp", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("VarHandleOp", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Checks whether a resource handle-based variable has been initialized. + /// + /// + /// + public static Tensor var_is_initialized_op(Tensor resource, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "VarIsInitializedOp", name) { args = new object[] { resource }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return var_is_initialized_op_eager_fallback(resource, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + var _op = tf.OpDefLib._apply_op_helper("VarIsInitializedOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("VarIsInitializedOp", _op.inputs, _attrs, _result); + } + return _result[0]; + } - public static Tensor resource_gather(Tensor resource, Tensor indices, TF_DataType dtype, - int batch_dims = 0, bool validate_indices = true, string name = null) + public static Tensor var_is_initialized_op_eager_fallback(Tensor resource, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("VarIsInitializedOp", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - var _op = tf.OpDefLib._apply_op_helper("ResourceGather", name, new + _execute.record_gradient("VarIsInitializedOp", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Returns the shape of the variable pointed to by `resource`. + /// + /// + /// + /// This operation returns a 1-D integer tensor representing the shape of `input`. + /// + /// For example: + /// + /// ``` + /// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]] + /// shape(t) ==> [2, 2, 3] + /// ``` + /// + /// + /// + /// + /// + public static Tensor variable_shape(Tensor input, TF_DataType out_type = TF_DataType.TF_INT32, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "VariableShape", name) { args = new object[] { input }, attrs = new Dictionary() { ["out_type"] = out_type } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) { - resource, - indices, - dtype, - batch_dims, - validate_indices - }); + throw ex; + } + catch (Exception) + { + } + try + { + return variable_shape_eager_fallback(input, out_type: out_type, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input"] = input; + keywords["out_type"] = out_type; + var _op = tf.OpDefLib._apply_op_helper("VariableShape", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "out_type", _op._get_attr_type("out_type") }; + _execute.record_gradient("VariableShape", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return _op.output; + public static Tensor variable_shape_eager_fallback(Tensor input, TF_DataType out_type, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input }; + object[] _attrs = new object[] { "out_type", out_type }; + var _result = _execute.execute("VariableShape", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("VariableShape", _inputs_flat, _attrs, _result); } + return _result[0]; } } diff --git a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs index 9d52f516..126df9e4 100644 --- a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs +++ b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs @@ -1778,10 +1778,10 @@ new_height, new_width"); { // a_y_min: [0], a_x_min: [1], a_y_max: [2], a_x_max[3] var a_xy_minmax = array_ops.split( - value: boxes_a, num_split: 4, axis: 2); + value: boxes_a, num_or_size_splits: 4, axis: ops.convert_to_tensor(2)); // b_y_min: [0], b_x_min: [1], b_y_max: [2], b_x_max[3] var b_xy_minmax = array_ops.split( - value: boxes_b, num_split: 4, axis: 2); + value: boxes_b, num_or_size_splits: 4, axis: ops.convert_to_tensor(2)); var i_xmin = math_ops.maximum( a_xy_minmax[1], array_ops.transpose(b_xy_minmax[1], new[] { 0, 2, 1 })); @@ -1943,7 +1943,7 @@ new_height, new_width"); using (ops.name_scope("canonicalize_coordinates")) { // y_1 = [0], x_1 = [1], y_2 = [2], x_2 = [3] - var yx = array_ops.split(value: boxes, num_split: 4, axis: 2); + var yx = array_ops.split(value: boxes, num_or_size_splits: 4, axis: ops.convert_to_tensor(2)); var y_1_is_min = math_ops.reduce_all( gen_math_ops.less_equal(yx[0][0, 0, 0], yx[2][0, 0, 0])); var y_minmax = control_flow_ops.cond( diff --git a/src/TensorFlowNET.Core/Operations/while_v2.cs b/src/TensorFlowNET.Core/Operations/while_v2.cs index 7ee3e9e8..3f324f87 100644 --- a/src/TensorFlowNET.Core/Operations/while_v2.cs +++ b/src/TensorFlowNET.Core/Operations/while_v2.cs @@ -86,7 +86,7 @@ namespace Tensorflow.Operations } } - var cond_graph = FuncGraph.func_graph_from_func("cond", wrapped_cond, null, + var cond_graph = FuncGraph.func_graph_from_func(cond_name, wrapped_cond, null, null, signature: func_graph_signature, add_control_dependencies: add_control_dependencies); bool stateful_parallelism = false; @@ -111,7 +111,7 @@ namespace Tensorflow.Operations return new object[] { loop_counter + 1, maximum_iterations_arg }.Concat(outputs).ToArray(); } - var body_graph = FuncGraph.func_graph_from_func("body", wrapped_body, null, null, func_graph_signature, + var body_graph = FuncGraph.func_graph_from_func(body_name, wrapped_body, null, null, func_graph_signature, add_control_dependencies: add_control_dependencies, acd_record_initial_resource_uses: stateful_parallelism); // TODO(Rinne): possible wrong implementation here. diff --git a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs index b9a7022a..a54283bd 100644 --- a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs +++ b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs @@ -170,11 +170,28 @@ namespace Tensorflow public Tensor value() => GraphElement ?? _read_variable_op(); - protected Tensor _read_variable_op() + protected Tensor _read_variable_op(bool no_copy = false) { variable_accessed(this); - var result = gen_resource_variable_ops.read_variable_op(handle, _dtype); - resource_variable_ops._maybe_set_handle_data(_dtype, handle, result); + + Tensor read_and_set_handle(bool no_copy) + { + if (no_copy) + { + gen_resource_variable_ops.disable_copy_on_read(handle); + } + var result = gen_resource_variable_ops.read_variable_op(handle, _dtype); + resource_variable_ops._maybe_set_handle_data(_dtype, handle, result); + return result; + } + + // TODO(Rinne): deal with caching device. + var result = read_and_set_handle(no_copy); + if (!tf.Context.executing_eagerly()) + { + tf.Runner.TFE_TapeSetRecordOperation("ReadVariableOp", new Tensor[] { result }, new Tensor[] { handle }, + backward_function: (x, _) => x); + } // have to set shape when converting to substituent placeholder if (result.shape.ndim == -1) diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs index a0358f07..d52190fd 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs @@ -38,6 +38,8 @@ namespace Tensorflow.Keras.Engine _handle_activity_regularization(inputs, outputs); _set_mask_metadata(inputs, outputs, null); + // TODO(Rinne): set save spec if null + scope.__exit__(); return outputs; diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 66c3cdc1..efca9300 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -709,10 +709,7 @@ namespace Tensorflow.Keras.Layers public IRnnCell StackedRNNCells( IEnumerable cells) - => new StackedRNNCells(new StackedRNNCellsArgs - { - Cells = cells.ToList() - }); + => new StackedRNNCells(cells.ToList(), new StackedRNNCellsArgs()); /// /// @@ -757,9 +754,8 @@ namespace Tensorflow.Keras.Layers bool stateful = false, bool unroll = false, bool time_major = false) - => new RNN(new RNNArgs + => new RNN(cell, new RNNArgs { - Cell = cell, ReturnSequences = return_sequences, ReturnState = return_state, GoBackwards = go_backwards, @@ -776,9 +772,8 @@ namespace Tensorflow.Keras.Layers bool stateful = false, bool unroll = false, bool time_major = false) - => new RNN(new RNNArgs + => new RNN(cell, new RNNArgs { - Cells = cell.ToList(), ReturnSequences = return_sequences, ReturnState = return_state, GoBackwards = go_backwards, @@ -798,7 +793,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 2) + int implementation = 1) => new LSTMCell(new LSTMCellArgs { Units = uints, @@ -851,7 +846,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 2, + int implementation = 1, bool return_sequences = false, bool return_state = false, bool go_backwards = false, diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index 1449c908..025465fd 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -2,6 +2,7 @@ using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Common.Types; +using Tensorflow.Common.Extensions; namespace Tensorflow.Keras.Layers.Rnn { @@ -14,22 +15,105 @@ namespace Tensorflow.Keras.Layers.Rnn public class LSTM : RNN { LSTMArgs args; - InputSpec[] state_spec; - - int units => args.Units; + InputSpec[] _state_spec; + InputSpec _input_spec; + bool _could_use_gpu_kernel; public LSTM(LSTMArgs args) : - base(args) + base(CreateCell(args), args) { this.args = args; - state_spec = new[] { units, units } - .Select(dim => new InputSpec(shape: (-1, dim))) - .ToArray(); + _input_spec = new InputSpec(ndim: 3); + _state_spec = new[] { args.Units, args.Units }.Select(dim => new InputSpec(shape: (-1, dim))).ToArray(); + _could_use_gpu_kernel = args.Activation == keras.activations.Tanh + && args.RecurrentActivation == keras.activations.Sigmoid + && args.RecurrentDropout == 0 && !args.Unroll && args.UseBias + && ops.executing_eagerly_outside_functions(); + } + + private static IRnnCell CreateCell(LSTMArgs lstmArgs) + { + return new LSTMCell(new LSTMCellArgs() + { + Units = lstmArgs.Units, + Activation = lstmArgs.Activation, + RecurrentActivation = lstmArgs.RecurrentActivation, + UseBias = lstmArgs.UseBias, + KernelInitializer = lstmArgs.KernelInitializer, + RecurrentInitializer = lstmArgs.RecurrentInitializer, + UnitForgetBias = lstmArgs.UnitForgetBias, + BiasInitializer = lstmArgs.BiasInitializer, + // TODO(Rinne): kernel_regularizer + // TODO(Rinne): recurrent_regularizer + // TODO(Rinne): bias_regularizer + // TODO(Rinne): kernel_constriant + // TODO(Rinne): recurrent_constriant + // TODO(Rinne): bias_constriant + Dropout = lstmArgs.Dropout, + RecurrentDropout = lstmArgs.RecurrentDropout, + Implementation = lstmArgs.Implementation, + DType = lstmArgs.DType, + Trainable = lstmArgs.Trainable + }); } - protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) + protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bool? training = null, IOptionalArgs? optional_args = null) { - return base.Call(inputs, initial_state: state, training: training); + // skip the condition of ragged input + + (inputs, initial_state, _) = _process_inputs(inputs, initial_state, null); + + Tensor mask = null; + if(optional_args is RnnOptionalArgs rnnArgs) + { + mask = rnnArgs.Mask; + } + + var single_input = inputs.Single; + var input_shape = single_input.shape; + var timesteps = args.TimeMajor ? input_shape[0] : input_shape[1]; + + _maybe_reset_cell_dropout_mask(Cell); + + Func step = (inputs, states) => + { + var res = Cell.Apply(inputs, states, training is null ? true : training.Value); + var (output, state) = res; + return (output, state); + }; + + var (last_output, outputs, states) = keras.backend.rnn( + step, + inputs, + initial_state, + constants: null, + go_backwards: args.GoBackwards, + mask: mask, + unroll: args.Unroll, + input_length: ops.convert_to_tensor(timesteps), + time_major: args.TimeMajor, + zero_output_for_mask: args.ZeroOutputForMask, + return_all_outputs: args.ReturnSequences + ); + + Tensor output; + if (args.ReturnSequences) + { + output = keras.backend.maybe_convert_to_ragged(false, outputs, (int)timesteps, args.GoBackwards); + } + else + { + output = last_output; + } + + if (args.ReturnState) + { + return new Tensor[] { output }.Concat(states).ToArray().ToTensors(); + } + else + { + return output; + } } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index 17042767..bb71a914 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -1,5 +1,6 @@ using Serilog.Core; using System.Diagnostics; +using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; @@ -81,7 +82,7 @@ namespace Tensorflow.Keras.Layers.Rnn _bias_initializer = _args.BiasInitializer; } _bias = add_weight("bias", (_args.Units * 4), - initializer: _args.BiasInitializer); + initializer: _bias_initializer); } built = true; } @@ -94,7 +95,6 @@ namespace Tensorflow.Keras.Layers.Rnn var rec_dp_mask = get_recurrent_dropout_mask_for_cell( h_tm1, training.Value, count: 4); - Tensor c; Tensor o; if (_args.Implementation == 1) @@ -123,7 +123,7 @@ namespace Tensorflow.Keras.Layers.Rnn var x_f = math_ops.matmul(inputs_f, k_f); var x_c = math_ops.matmul(inputs_c, k_c); var x_o = math_ops.matmul(inputs_o, k_o); - if(_args.UseBias) + if (_args.UseBias) { var b = tf.split(_bias.AsTensor(), num_split: 4, axis: 0); Tensor b_i = b[0], b_f = b[1], b_c = b[2], b_o = b[3]; @@ -170,7 +170,7 @@ namespace Tensorflow.Keras.Layers.Rnn } var h = o * _args.Activation.Apply(c); // 这里是因为 Tensors 类初始化的时候会把第一个元素之后的元素打包成一个数组 - return new Tensors(h, h, c); + return new Nest(new INestStructure[] { new NestNode(h), new NestList(h, c) }).ToTensors(); } /// @@ -188,22 +188,21 @@ namespace Tensorflow.Keras.Layers.Rnn h_tm1_o = h_tm1[3]; var _recurrent_kernel_tensor = _recurrent_kernel.AsTensor(); - var startIndex = _recurrent_kernel_tensor.shape[0]; - var endIndex = _recurrent_kernel_tensor.shape[1]; + int startIndex = (int)_recurrent_kernel_tensor.shape[0]; var _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, new[] { 0, 0 }, new[] { startIndex, _args.Units }); var i = _args.RecurrentActivation.Apply( x_i + math_ops.matmul(h_tm1_i, _recurrent_kernel_slice)); _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, - new[] { 0, _args.Units }, new[] { startIndex, _args.Units * 2}); + new[] { 0, _args.Units }, new[] { startIndex, _args.Units}); var f = _args.RecurrentActivation.Apply( x_f + math_ops.matmul(h_tm1_f, _recurrent_kernel_slice)); _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, - new[] { 0, _args.Units * 2 }, new[] { startIndex, _args.Units * 3 }); + new[] { 0, _args.Units * 2 }, new[] { startIndex, _args.Units }); var c = f * c_tm1 + i * _args.Activation.Apply( x_c + math_ops.matmul(h_tm1_c, _recurrent_kernel_slice)); _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, - new[] { 0, _args.Units * 3 }, new[] { startIndex, endIndex }); + new[] { 0, _args.Units * 3 }, new[] { startIndex, _args.Units }); var o = _args.RecurrentActivation.Apply( x_o + math_ops.matmul(h_tm1_o, _recurrent_kernel_slice)); diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 0aeacc25..f86de8a8 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -45,23 +45,25 @@ namespace Tensorflow.Keras.Layers.Rnn } } - public RNN(RNNArgs args) : base(PreConstruct(args)) + public RNN(IRnnCell cell, RNNArgs args) : base(PreConstruct(args)) { _args = args; SupportsMasking = true; - // if is StackedRnncell - if (args.Cells != null) - { - Cell = new StackedRNNCells(new StackedRNNCellsArgs - { - Cells = args.Cells - }); - } - else - { - Cell = args.Cell; - } + Cell = cell; + + // get input_shape + _args = PreConstruct(args); + + _num_constants = 0; + } + + public RNN(IEnumerable cells, RNNArgs args) : base(PreConstruct(args)) + { + _args = args; + SupportsMasking = true; + + Cell = new StackedRNNCells(cells, new StackedRNNCellsArgs()); // get input_shape _args = PreConstruct(args); @@ -330,7 +332,7 @@ namespace Tensorflow.Keras.Layers.Rnn states = new Tensors(states.SkipLast(_num_constants).ToArray()); states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; var (output, new_states) = Cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); - return (output, new_states.Single); + return (output, new_states); }; } else @@ -382,6 +384,11 @@ namespace Tensorflow.Keras.Layers.Rnn } else { + //var tapeSet = tf.GetTapeSet(); + //foreach(var tape in tapeSet) + //{ + // tape.Watch(output); + //} return output; } } @@ -405,7 +412,7 @@ namespace Tensorflow.Keras.Layers.Rnn throw new NotImplementedException(); } - private (Tensors inputs, Tensors initial_state, Tensors constants) _process_inputs(Tensors inputs, Tensors initial_state, Tensors constants) + protected (Tensors inputs, Tensors initial_state, Tensors constants) _process_inputs(Tensors inputs, Tensors initial_state, Tensors constants) { if (inputs.Length > 1) { @@ -484,7 +491,7 @@ namespace Tensorflow.Keras.Layers.Rnn } - void _maybe_reset_cell_dropout_mask(ILayer cell) + protected void _maybe_reset_cell_dropout_mask(ILayer cell) { if (cell is DropoutRNNCellMixin CellDRCMixin) { @@ -495,26 +502,21 @@ namespace Tensorflow.Keras.Layers.Rnn private static RNNArgs PreConstruct(RNNArgs args) { - if (args.Kwargs == null) - { - args.Kwargs = new Dictionary(); - } - // If true, the output for masked timestep will be zeros, whereas in the // false case, output from previous timestep is returned for masked timestep. - var zeroOutputForMask = (bool)args.Kwargs.Get("zero_output_for_mask", false); + var zeroOutputForMask = args.ZeroOutputForMask; Shape input_shape; - var propIS = (Shape)args.Kwargs.Get("input_shape", null); - var propID = (int?)args.Kwargs.Get("input_dim", null); - var propIL = (int?)args.Kwargs.Get("input_length", null); + var propIS = args.InputShape; + var propID = args.InputDim; + var propIL = args.InputLength; if (propIS == null && (propID != null || propIL != null)) { input_shape = new Shape( propIL ?? -1, propID ?? -1); - args.Kwargs["input_shape"] = input_shape; + args.InputShape = input_shape; } return args; diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index 551c20cd..a22f31c7 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -10,14 +10,14 @@ namespace Tensorflow.Keras.Layers.Rnn public class SimpleRNN : RNN { SimpleRNNArgs args; - public SimpleRNN(SimpleRNNArgs args) : base(CreateCellForArgs(args)) + public SimpleRNN(SimpleRNNArgs args) : base(CreateCellForArgs(args), args) { this.args = args; } - private static SimpleRNNArgs CreateCellForArgs(SimpleRNNArgs args) + private static SimpleRNNCell CreateCellForArgs(SimpleRNNArgs args) { - args.Cell = new SimpleRNNCell(new SimpleRNNCellArgs() + return new SimpleRNNCell(new SimpleRNNCellArgs() { Units = args.Units, Activation = args.Activation, @@ -30,7 +30,6 @@ namespace Tensorflow.Keras.Layers.Rnn DType = args.DType, Trainable = args.Trainable, }); - return args; } } } \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 8fdc598e..c77f7779 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -115,10 +115,5 @@ namespace Tensorflow.Keras.Layers.Rnn return new Tensors(output, output); } } - - public Tensors get_initial_state(Tensors inputs = null, Tensor batch_size = null, TF_DataType dtype = TF_DataType.DtInvalid) - { - return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); - } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 3e7b227c..8799bfb2 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -15,15 +15,11 @@ namespace Tensorflow.Keras.Layers.Rnn public IList Cells { get; set; } public bool _reverse_state_order; - public StackedRNNCells(StackedRNNCellsArgs args) : base(args) + public StackedRNNCells(IEnumerable cells, StackedRNNCellsArgs args) : base(args) { - if (args.Kwargs == null) - { - args.Kwargs = new Dictionary(); - } - Cells = args.Cells; - - _reverse_state_order = (bool)args.Kwargs.Get("reverse_state_order", false); + Cells = cells.ToList(); + + _reverse_state_order = args.ReverseStateOrder; if (_reverse_state_order) { diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 54ea1565..ed9b6ae9 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -55,30 +55,56 @@ namespace Tensorflow.Keras.UnitTest.Layers Assert.AreEqual((2, 4), new_states[0].shape); } + [TestMethod] + public void TrainLSTMWithMnist() + { + var input = keras.Input((784)); + var x = keras.layers.Reshape((28, 28)).Apply(input); + //x = keras.layers.LSTM(50, return_sequences: true).Apply(x); + //x = keras.layers.LSTM(100, return_sequences: true).Apply(x); + //x = keras.layers.LSTM(150, return_sequences: true).Apply(x); + x = keras.layers.LSTM(4, implementation: 2).Apply(x); + //x = keras.layers.Dense(100).Apply(x); + var output = keras.layers.Dense(10, activation: "softmax").Apply(x); + + var model = keras.Model(input, output); + model.summary(); + model.compile(keras.optimizers.Adam(), keras.losses.SparseCategoricalCrossentropy(), new string[] { "accuracy" }); + + var data_loader = new MnistModelLoader(); + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 58000, + }).Result; + + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 30); + } + [TestMethod] public void SimpleRNN() { - //var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); - ///*var simple_rnn = keras.layers.SimpleRNN(4); - //var output = simple_rnn.Apply(inputs); - //Assert.AreEqual((32, 4), output.shape);*/ - - //var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); - //var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); - //Assert.AreEqual((6, 10, 4), whole_sequence_output.shape); - //Assert.AreEqual((6, 4), final_state.shape); + var input = keras.Input((784)); + var x = keras.layers.Reshape((28, 28)).Apply(input); + x = keras.layers.SimpleRNN(10).Apply(x); + var output = keras.layers.Dense(10, activation: "softmax").Apply(x); - var inputs = keras.Input(shape: (10, 8)); - var x = keras.layers.SimpleRNN(4).Apply(inputs); - var output = keras.layers.Dense(10).Apply(x); - var model = keras.Model(inputs, output); + var model = keras.Model(input, output); model.summary(); + model.compile(keras.optimizers.Adam(), keras.losses.CategoricalCrossentropy(), new string[] { "accuracy" }); - model.compile(keras.optimizers.Adam(), keras.losses.SparseCategoricalCrossentropy()); - var datax = np.ones((16, 10, 8), dtype: dtypes.float32); - var datay = np.ones((16)); - model.fit(datax, datay, epochs: 20); + var data_loader = new MnistModelLoader(); + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 58000, + }).Result; + + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 10); } + [TestMethod] public void RNNForSimpleRNNCell() { @@ -109,19 +135,5 @@ namespace Tensorflow.Keras.UnitTest.Layers Console.WriteLine($"output: {output}"); Assert.AreEqual((5, 4), output.shape); } - - [TestMethod] - public void MyTest() - { - var a = tf.zeros((2, 3)); - var b = tf.ones_like(a); - var c = tf.ones((3,4)); - - var d = new Tensors { a, b, c }; - var (A, BC) = d; - Console.WriteLine($"A:{A}"); - Console.WriteLine($"BC:{BC}"); - } - } } diff --git a/tools/Tensorflow.CodeGen/OpClassifier.cs b/tools/Tensorflow.CodeGen/OpClassifier.cs index eaad3fec..2d22c5d2 100644 --- a/tools/Tensorflow.CodeGen/OpClassifier.cs +++ b/tools/Tensorflow.CodeGen/OpClassifier.cs @@ -9,7 +9,7 @@ namespace Tensorflow.CodeGen { public class OpClassifier { - private static readonly string _filenamePattern = @"^gen_[a-z]*_ops.py$"; + private static readonly string _filenamePattern = @"^gen_[a-z_]*_ops.py$"; private static readonly string _pythonFunctionPattern = @"def\s+(\w+\d*\w*)\((?:\s*\w+\s*(?:=\s*[\S]*)*,\s*)*\s*name=None\):"; private Dictionary> _opSet = new(); public Dictionary> OpSet => _opSet; diff --git a/tools/Tensorflow.CodeGen/Utils.cs b/tools/Tensorflow.CodeGen/Utils.cs index 19de6c0e..6c69b7f9 100644 --- a/tools/Tensorflow.CodeGen/Utils.cs +++ b/tools/Tensorflow.CodeGen/Utils.cs @@ -178,10 +178,25 @@ namespace Tensorflow.CodeGen else if (attr.Type == "list(shape)") { res.Add((attr.Name, "Shape[]", "NOVALUE")); + if (attr.DefaultValue is not null && attr.DefaultValue.ValueCase == AttrValue.ValueOneofCase.List) + { + List exps = new(); + foreach (var value in attr.DefaultValue.List.Shape) + { + exps.Add($"new Shape({string.Join(", ", value.Dim.Select(x => x.Size))})"); + } + string expression = "new Shape[]{" + $"{string.Join(", ", exps)}" + "}"; + dynamicDefaultValues[attr.Name] = expression; + res.Add((attr.Name, "string[]", $"null")); + } + else + { + res.Add((attr.Name, "string[]", "NOVALUE")); + } } else if (attr.Type == "list(string)") { - if (attr.DefaultValue is not null && attr.DefaultValue.ValueCase == AttrValue.ValueOneofCase.S) + if (attr.DefaultValue is not null && attr.DefaultValue.ValueCase == AttrValue.ValueOneofCase.List) { List values = new(); foreach (var value in attr.DefaultValue.List.S)