public static Tensor conv2d_transpose(Tensor value = null,
IVariableV1 filter = null,
Tensor output_shape = null,
Shape strides = null,
string padding = "SAME",
string data_format = "NHWC",
string name = null,
Shape dilations = null)
{
if (dilations == null)
{
dilations = (1, 1, 1, 1);
}
return(tf_with(ops.name_scope(name, "conv2d_transpose", new { value, filter, output_shape }), scope =>
{
return gen_nn_ops.conv2d_backprop_input(
input_sizes: output_shape,
filter: filter.AsTensor(),
out_backprop: value,
strides: strides,
padding: padding,
data_format: data_format,
dilations: dilations,
name: name);
}));
}
protected override void build(TensorShape input_shape)
{
var last_dim = input_shape.dims.Last();
var axes = new Dictionary <int, int>();
axes[-1] = last_dim;
input_spec = new InputSpec(min_ndim: 2, axes: axes);
kernel = add_weight(
"kernel",
shape: new int[] { last_dim, units },
initializer: kernel_initializer,
dtype: _dtype,
trainable: true);
if (use_bias)
{
bias = add_weight(
"bias",
shape: new int[] { units },
initializer: bias_initializer,
dtype: _dtype,
trainable: true);
}
built = true;
}
public DataHandler(DataHandlerArgs args)
{
this.args = args;
if (args.StepsPerExecution == null)
{
_steps_per_execution = tf.Variable(1);
_steps_per_execution_value = 1;
}
else
{
_steps_per_execution = args.StepsPerExecution;
_steps_per_execution_value = args.StepsPerExecution.numpy();
}
_adapter = new TensorLikeDataAdapter(new TensorLikeDataAdapterArgs
{
X = args.X,
Y = args.Y,
BatchSize = args.BatchSize,
Steps = args.StepsPerEpoch,
Epochs = args.Epochs - args.InitialEpoch,
Shuffle = args.Shuffle,
MaxQueueSize = args.MaxQueueSize,
Worker = args.Workers,
UseMultiprocessing = args.UseMultiprocessing,
Model = args.Model
});
_dataset = _adapter.GetDataset();
_inferred_steps = _infer_steps(args.StepsPerEpoch, _dataset);
_current_step = 0;
_step_increment = args.StepsPerExecution.numpy() - 1;
_insufficient_data = false;
}
protected override void build(TensorShape input_shape)
{
embeddings = add_weight(shape: new int[] { input_dim, output_dim },
initializer: embeddings_initializer,
name: "embeddings");
built = true;
}
protected override Operation _resource_apply_dense(IVariableV1 var, Tensor grad, Dictionary <DeviceDType, Dictionary <string, Tensor> > _apply_state)
{
Dictionary <string, Tensor> coefficients = null;
foreach (var state in _apply_state)
{
if (state.Key.DType == var.dtype.as_base_dtype() &&
state.Key.Device == var.Device)
{
coefficients = state.Value;
break;
}
}
var rms = get_slot(var, "rms");
if (_momentum)
{
throw new NotImplementedException("");
}
else
{
var rms_t = coefficients["rho"] * rms.AsTensor() + coefficients["one_minus_rho"] * math_ops.square(grad);
rms_t = state_ops.assign(rms, rms_t, use_locking: _use_locking);
var denom_t = rms_t;
if (centered)
{
throw new NotImplementedException("");
}
var var_t = var.AsTensor() - coefficients["lr_t"] * grad / (math_ops.sqrt(denom_t) + coefficients["epsilon"]);
return(state_ops.assign(var, var_t, use_locking: _use_locking).op);
}
}
public static Tensor bias_add(Tensor value,
IVariableV1 bias,
string data_format = null,
string name = null)
{
if (tf.executing_eagerly())
{
var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
"BiasAdd", name,
null,
value, bias,
"data_format", data_format);
return(results[0]);
}
if (data_format == null)
{
data_format = "NHWC";
}
var _op = tf.OpDefLib._apply_op_helper("BiasAdd", name: name, args: new
{
value,
bias,
data_format
});
return(_op.outputs[0]);
}
/// <summary>
/// Adds a new softmax and fully-connected layer for training and eval.
///
/// We need to retrain the top layer to identify our new classes, so this function
/// adds the right operations to the graph, along with some variables to hold the
/// weights, and then sets up all the gradients for the backward pass.
///
/// The set up for the softmax and fully-connected layers is based on:
/// https://www.tensorflow.org/tutorials/mnist/beginners/index.html
/// </summary>
/// <param name="class_count"></param>
/// <param name="final_tensor_name"></param>
/// <param name="bottleneck_tensor"></param>
/// <param name="quantize_layer"></param>
/// <param name="is_training"></param>
/// <returns></returns>
private (Operation, Tensor, Tensor, Tensor, Tensor) add_final_retrain_ops(int class_count, string final_tensor_name,
Tensor bottleneck_tensor, bool quantize_layer, bool is_training)
{
var(batch_size, bottleneck_tensor_size) = (bottleneck_tensor.shape.dims[0], bottleneck_tensor.shape.dims[1]);
tf_with(tf.name_scope("input"), scope =>
{
bottleneck_input = tf.placeholder_with_default(
bottleneck_tensor,
shape: bottleneck_tensor.shape,
name: "BottleneckInputPlaceholder");
ground_truth_input = tf.placeholder(tf.int64, new Shape(batch_size), name: "GroundTruthInput");
});
// Organizing the following ops so they are easier to see in TensorBoard.
string layer_name = "final_retrain_ops";
Tensor logits = null;
tf_with(tf.name_scope(layer_name), scope =>
{
IVariableV1 layer_weights = null;
tf_with(tf.name_scope("weights"), delegate
{
var initial_value = tf.truncated_normal((bottleneck_tensor_size, (long)class_count), stddev: 0.001f);
layer_weights = tf.Variable(initial_value, name: "final_weights");
variable_summaries(layer_weights.AsTensor());
});
protected override void build(TensorShape input_shape)
{
var last_dim = input_shape.dims.Last();
var axes = new Dictionary <int, int>();
axes[-1] = last_dim;
inputSpec = new InputSpec(min_ndim: 2, axes: axes);
kernel = add_weight(
"kernel",
shape: new TensorShape(last_dim, args.Units),
initializer: args.KernelInitializer,
dtype: DType,
trainable: true);
if (args.UseBias)
{
bias = add_weight(
"bias",
shape: new TensorShape(args.Units),
initializer: args.BiasInitializer,
dtype: DType,
trainable: true);
}
built = true;
}
protected override Operation _resource_apply_dense(IVariableV1 var, Tensor grad, Dictionary <DeviceDType, Dictionary <string, Tensor> > apply_state)
{
var(var_device, var_dtype) = (var.Device, var.dtype.as_base_dtype());
var coefficients = apply_state.FirstOrDefault(x => x.Key.Device == var_device && x.Key.DType == var_dtype).Value ?? _fallback_apply_state(var_device, var_dtype);
var m = get_slot(var, "m");
var v = get_slot(var, "v");
if (!amsgrad)
{
return(gen_training_ops.resource_apply_adam(var.Handle,
m.Handle,
v.Handle,
coefficients["beta_1_power"],
coefficients["beta_2_power"],
coefficients["lr_t"],
coefficients["beta_1_t"],
coefficients["beta_2_t"],
coefficients["epsilon"],
grad,
use_locking: _use_locking));
}
else
{
throw new NotImplementedException("");
}
}
public static Tensor assign_sub(IVariableV1 @ref,
Tensor value,
bool use_locking = false,
string name = null) => gen_state_ops.assign_sub(@ref,
value,
use_locking: use_locking,
name: name);
private Operation _apply_sparse_shared(Tensor grad, IVariableV1 var, Tensor indices, Func <IVariableV1, Tensor, Tensor, Tensor> scatter_add)
{
var(beta1_power_v, beta2_power_v) = _get_beta_accumulators();
Tensor beta1_power = math_ops.cast(beta1_power_v, var.dtype.as_base_dtype());
Tensor beta2_power = math_ops.cast(beta2_power_v, var.dtype.as_base_dtype());
var lr_t = math_ops.cast(_lr_t, var.dtype.as_base_dtype());
var beta1_t = math_ops.cast(_beta1_t, var.dtype.as_base_dtype());
var beta2_t = math_ops.cast(_beta2_t, var.dtype.as_base_dtype());
var epsilon_t = math_ops.cast(_epsilon_t, var.dtype.as_base_dtype());
var lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power));
var m = get_slot(var, "m");
var m_scaled_g_values = grad * (1 - beta1_t);
var m_t = state_ops.assign(m.AsTensor(), m.AsTensor() * beta1_t, use_locking: _use_locking);
tf_with(ops.control_dependencies(new[] { m_t }), delegate
{
m_t = scatter_add(m, indices, m_scaled_g_values);
});
var v = get_slot(var, "v");
var v_scaled_g_values = (grad * grad) * (1 - beta2_t);
var v_t = state_ops.assign(v.AsTensor(), v.AsTensor() * beta2_t, use_locking: _use_locking);
tf_with(ops.control_dependencies(new[] { v_t }), delegate
{
v_t = scatter_add(v, indices, v_scaled_g_values);
});
var v_sqrt = math_ops.sqrt(v_t);
var var_update = state_ops.assign_sub(var, lr * m_t / (v_sqrt + epsilon_t), use_locking: _use_locking);
return(control_flow_ops.group(new[] { var_update, m_t, v_t }));
}
public Tensor __call__(IVariableV1 step)
{
return(tf_with(ops.name_scope(name ?? "PolynomialDecay"), scope =>
{
name = scope;
var initial_learning_rate_tensor = ops.convert_to_tensor(initial_learning_rate, name: "initial_learning_rate");
var dtype = initial_learning_rate_tensor.dtype;
var end_learning_rate_tensor = constant_op.constant(end_learning_rate, dtype);
var power_tensor = constant_op.constant(power, dtype);
var global_step_recomp = constant_op.constant(step, dtype);
var decay_steps_recomp = constant_op.constant(decay_steps, dtype);
if (cycle)
{
throw new NotImplementedException("PolynomialDecay cycle");
}
else
{
// Make sure that the global_step used is not bigger than decay_steps.
global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps);
}
var p = tf.divide(global_step_recomp, decay_steps_recomp);
var pow = tf.pow(1 - p, power_tensor);
var m = math_ops.multiply(initial_learning_rate_tensor - end_learning_rate_tensor, pow);
return math_ops.add(m,
end_learning_rate_tensor,
name: name);
}));
}
void _assign_new_value(IVariableV1 variable, Tensor value)
{
tf_with(ops.name_scope("AssignNewValue", null, new { variable, value, momentum }), scope =>
{
// var cm = ops.colocate_with(variable);
variable.assign_lazy_load(value, name: scope);
});
}
/// <summary>
/// Create lambdas which compute regularization losses.
/// </summary>
/// <param name="name"></param>
/// <param name="variable"></param>
/// <param name="regularizer"></param>
void _handle_weight_regularization(string name, IVariableV1 variable, IRegularizer regularizer)
{
add_loss(() => tf_with(ops.name_scope(name + "/Regularizer"), scope =>
regularizer.Apply(new RegularizerArgs(variable.AsTensor())
{
})
));
}
protected override void build(TensorShape input_shape)
{
tf.Context.eager_mode();
embeddings = add_weight(shape: (input_dim, output_dim),
initializer: embeddings_initializer,
name: "embeddings");
tf.Context.graph_mode();
built = true;
}
public static Tensor scatter_add(IVariableV1 @ref, Tensor indices, Tensor updates, bool use_locking = false, string name = null)
{
if (@ref.dtype.is_ref_dtype())
{
return(gen_state_ops.scatter_add(@ref, indices, updates, use_locking: use_locking, name: name));
}
throw new NotImplementedException("scatter_add");
}
public static Tensor assign_sub(IVariableV1 @ref,
Tensor value,
bool use_locking = false,
string name = null) => @ref.dtype.is_ref_dtype() ?
gen_state_ops.assign_sub(@ref,
value,
use_locking: use_locking,
name: name) :
@ref.assign_sub(value, name: name);
public static Tensor assign_sub(IVariableV1 @ref,
Tensor value,
bool use_locking = false,
string name = null)
{
var _op = tf.OpDefLib._apply_op_helper("AssignSub", name: name, args: new { @ref, value, use_locking });
return(_op.outputs[0]);
}
public static Tensor[] smart_cond <T>(IVariableV1 pred,
Func <T[]> true_fn = null,
Func <T[]> false_fn = null,
string name = null)
{
return(control_flow_ops.cond(pred.AsTensor(),
true_fn: true_fn,
false_fn: false_fn,
name: name));
}
void _assign_moving_average(IVariableV1 variable, Tensor value, Tensor momentum)
{
tf_with(ops.name_scope("AssignMovingAvg", null, new { variable, value, momentum }), scope =>
{
// var cm = ops.colocate_with(variable);
var decay = ops.convert_to_tensor(1.0f - momentum, name: "decay");
var update_delta = (variable.AsTensor() - math_ops.cast(value, variable.dtype)) * decay;
variable.assign_sub_lazy_load(update_delta, name: scope);
});
}
/// <summary>
/// Creates a slot initialized using an `Initializer`.
/// </summary>
/// <returns></returns>
public IVariableV1 create_slot_with_initializer(IVariableV1 primary, IInitializer initializer, Shape shape,
TF_DataType dtype, string name, bool colocate_with_primary = true)
{
var validate_shape = shape.IsFullyDefined;
var prefix = primary.Op.name;
return(tf_with(new variable_scope(string.Empty, prefix + "/" + name), delegate
{
return _create_slot_var(primary, initializer, "", validate_shape, shape, dtype);
}));
}
public bool Run()
{
tf.enable_eager_execution();
PrepareData();
// Store layers weight & bias
// A random value generator to initialize weights.
var random_normal = tf.initializers.random_normal_initializer();
// Conv Layer 1: 5x5 conv, 1 input, 32 filters (MNIST has 1 color channel only).
wc1 = tf.Variable(random_normal.Apply(new InitializerArgs((5, 5, 1, conv1_filters))));
// Conv Layer 2: 5x5 conv, 32 inputs, 64 filters.
wc2 = tf.Variable(random_normal.Apply(new InitializerArgs((5, 5, conv1_filters, conv2_filters))));
// FC Layer 1: 7*7*64 inputs, 1024 units.
wd1 = tf.Variable(random_normal.Apply(new InitializerArgs((7 * 7 * 64, fc1_units))));
// FC Out Layer: 1024 inputs, 10 units (total number of classes)
wout = tf.Variable(random_normal.Apply(new InitializerArgs((fc1_units, num_classes))));
bc1 = tf.Variable(tf.zeros(conv1_filters));
bc2 = tf.Variable(tf.zeros(conv2_filters));
bd1 = tf.Variable(tf.zeros(fc1_units));
bout = tf.Variable(tf.zeros(num_classes));
// ADAM optimizer.
var optimizer = keras.optimizers.Adam(learning_rate);
// Run training for the given number of steps.
foreach (var(step, (batch_x, batch_y)) in enumerate(train_data, 1))
{
// Run the optimization to update W and b values.
run_optimization(optimizer, batch_x, batch_y);
if (step % display_step == 0)
{
var pred = conv_net(batch_x);
var loss = cross_entropy(pred, batch_y);
var acc = accuracy(pred, batch_y);
print($"step: {step}, loss: {(float)loss}, accuracy: {(float)acc}");
}
}
// Test model on validation set.
{
x_test = x_test["::100"];
y_test = y_test["::100"];
var pred = conv_net(x_test);
accuracy_test = (float)accuracy(pred, y_test);
print($"Test Accuracy: {accuracy_test}");
}
return(accuracy_test >= 0.90);
}
public static Tensor FullyConnectedLinearLayer(Tensor input, int outputDim, NeuralNetworkInitializer initializer, int seed)
{
int inputDim = (int)input.shape[1];
IInitializer?init = GetInitializer(initializer, seed);
IVariableV1 W = tf.compat.v1.get_variable("W", new int[] { inputDim, outputDim }, tf.float32, init);
IVariableV1 b = tf.compat.v1.get_variable("b", new int[] { outputDim }, tf.float32, init);
return(tf.matmul(input, W.AsTensor()) + b.AsTensor());
}
/// <summary>
/// Adds `bias` to `value`.
/// </summary>
/// <param name="value"></param>
/// <param name="bias"></param>
/// <param name="data_format"></param>
/// <param name="name"></param>
/// <returns></returns>
public static Tensor bias_add(Tensor value,
IVariableV1 bias,
string data_format = null,
string name = null)
{
return tf_with(ops.name_scope(name, "BiasAdd", new { value, bias }), scope =>
{
name = scope;
return gen_nn_ops.bias_add(value, bias, data_format: data_format, name: name);
});
}
public static Tensor apply_gradient_descent(IVariableV1 var, Tensor alpha, Tensor delta, bool use_locking = false, string name = null)
{
var _op = tf.OpDefLib._apply_op_helper("ApplyGradientDescent", name, new
{
var,
alpha,
delta,
use_locking
});
return(_op.output);
}
public static Tensor embedding_lookup(IVariableV1 @params, Tensor ids,
string partition_strategy = "mod",
string name = null,
bool validate_indices = true,
string max_norm = null)
{
return(_embedding_lookup_and_transform(@params: @params,
ids: ids,
partition_strategy: partition_strategy,
name: name,
max_norm: max_norm));
}
public static Tensors fused_batch_norm_v3(Tensor x,
IVariableV1 scale,
IVariableV1 offset,
IVariableV1 mean,
IVariableV1 variance,
float epsilon = 0.0001f,
float exponential_avg_factor = 1.0f,
string data_format = "NHWC",
bool is_training = true,
string name = null)
=> tf.Context.ExecuteOp("FusedBatchNormV3", name, new ExecuteOpArgs(x, scale, offset, mean, variance)
.SetAttributes(new { epsilon, data_format, is_training }));
public Tensor __call__(Tensor inp, IVariableV1 filter)
{
return(conv_op(new Conv2dParams
{
Input = inp,
Filter = filter.AsTensor(),
Strides = strides,
Padding = padding,
DataFormat = data_format,
Name = name
}));
}
protected override void build(TensorShape input_shape)
{
var input_depth = input_shape.dims.Last();
var h_depth = _num_units;
_kernel = add_weight(_WEIGHTS_VARIABLE_NAME,
shape: new[] { input_depth + h_depth, 4 * _num_units });
_bias = add_weight(_BIAS_VARIABLE_NAME,
shape: new[] { 4 * _num_units },
initializer: tf.zeros_initializer);
built = true;
}
protected override void build(Tensors inputs)
{
Shape input_shape = inputs.shape;
int channel_axis = data_format == "channels_first" ? 1 : -1;
var input_channel = channel_axis < 0 ?
input_shape.dims[input_shape.ndim + channel_axis] :
input_shape.dims[channel_axis];
Shape kernel_shape = kernel_size.dims.concat(new long[] { input_channel / args.Groups, filters });
kernel = add_weight(name: "kernel",
shape: kernel_shape,
initializer: kernel_initializer,
regularizer: kernel_regularizer,
trainable: true,
dtype: DType);
if (use_bias)
{
bias = add_weight(name: "bias",
shape: new int[] { filters },
initializer: bias_initializer,
trainable: true,
dtype: DType);
}
var axes = new Dictionary <int, int>();
axes.Add(-1, (int)input_channel);
inputSpec = new InputSpec(min_ndim: rank + 2, axes: axes);
string tf_padding;
if (padding == "causal")
{
tf_padding = "VALID";
}
else
{
tf_padding = padding.ToUpper();
}
string tf_op_name = GetType().Name;
_convolution_op = nn_ops.convolution_internal(tf_padding,
strides,
dilation_rate,
rank,
data_format: _tf_data_format,
name: tf_op_name);
built = true;
}
