protected override Tensor call(Tensor inputs, bool training = false)
{
Tensor outputs = null;
var rank = inputs.rank;
if (rank > 2)
{
throw new NotImplementedException("call rank > 2");
}
else
{
outputs = gen_math_ops.mat_mul(inputs, kernel.AsTensor());
}
if (args.UseBias)
{
outputs = tf.nn.bias_add(outputs, bias);
}
if (args.Activation != null)
{
outputs = activation(outputs);
}
return(outputs);
}
/// <summary>
/// Create lambdas which compute regularization losses.
/// </summary>
/// <param name="name"></param>
/// <param name="variable"></param>
/// <param name="regularizer"></param>
void _handle_weight_regularization(string name, IVariableV1 variable, IRegularizer regularizer)
{
add_loss(() => tf_with(ops.name_scope(name + "/Regularizer"), scope =>
regularizer.Apply(new RegularizerArgs(variable.AsTensor())
{
})
));
}
public static Tensor[] smart_cond <T>(IVariableV1 pred,
Func <T[]> true_fn = null,
Func <T[]> false_fn = null,
string name = null)
{
return(control_flow_ops.cond(pred.AsTensor(),
true_fn: true_fn,
false_fn: false_fn,
name: name));
}
void _assign_moving_average(IVariableV1 variable, Tensor value, Tensor momentum)
{
tf_with(ops.name_scope("AssignMovingAvg", null, new { variable, value, momentum }), scope =>
{
// var cm = ops.colocate_with(variable);
var decay = ops.convert_to_tensor(1.0f - momentum, name: "decay");
var update_delta = (variable.AsTensor() - math_ops.cast(value, variable.dtype)) * decay;
variable.assign_sub_lazy_load(update_delta, name: scope);
});
}
protected Tensors Call(Tensors inputs, Tensor state = null, bool is_training = false)
{
// Most basic RNN: output = new_state = act(W * input + U * state + B).
var concat = array_ops.concat(new Tensor[] { inputs, state }, 1);
var gate_inputs = math_ops.matmul(concat, _kernel.AsTensor());
gate_inputs = nn_ops.bias_add(gate_inputs, _bias);
var output = _activation(gate_inputs, null);
return(new Tensors(output, output));
}
public static Tensor FullyConnectedLinearLayer(Tensor input, int outputDim, NeuralNetworkInitializer initializer, int seed)
{
int inputDim = (int)input.shape[1];
IInitializer?init = GetInitializer(initializer, seed);
IVariableV1 W = tf.compat.v1.get_variable("W", new int[] { inputDim, outputDim }, tf.float32, init);
IVariableV1 b = tf.compat.v1.get_variable("b", new int[] { outputDim }, tf.float32, init);
return(tf.matmul(input, W.AsTensor()) + b.AsTensor());
}
public static Tensor cast(IVariableV1 x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null)
{
var base_type = dtype.as_base_dtype();
if (base_type == x.dtype)
{
return(x.AsTensor());
}
return(tf_with(ops.name_scope(name, "Cast", new { x }), scope =>
{
name = scope;
var t_x = ops.convert_to_tensor(x, name: "x");
if (t_x.dtype.as_base_dtype() != base_type)
{
t_x = gen_math_ops.cast(t_x, base_type, name: name);
}
return x.AsTensor();
}));
}
public Tensor __call__(Tensor inp, IVariableV1 filter)
{
return(conv_op(new Conv2dParams
{
Input = inp,
Filter = filter.AsTensor(),
Strides = strides,
Padding = padding,
DataFormat = data_format,
Name = name
}));
}
public Tensor Apply(Tensors input, IVariableV1 filters)
{
var filters_rank = filters.shape.rank;
var inputs_rank = input.shape.rank;
var num_spatial_dims = args.NumSpatialDims;
if (num_spatial_dims == Unknown)
{
num_spatial_dims = filters_rank - 2;
}
// Channel dimension.
var num_batch_dims = inputs_rank - num_spatial_dims - 1;
if (!new[] { 1, 2, 3 }.Contains(num_spatial_dims))
{
throw new ValueError($"num_spatial_dims (input.shape.ndims - num_batch_dims - 1) must be one " +
$"of 1, 2 or 3 but saw {num_spatial_dims}. num_batch_dims: {num_batch_dims}.");
}
var channel_index = num_batch_dims + num_spatial_dims;
var dilations = _get_sequence(args.DilationRate, num_spatial_dims, channel_index);
var strides = _get_sequence(args.Strides, num_spatial_dims, channel_index);
Tensor result = null;
tf_with(ops.name_scope(name, default_name: null), scope =>
{
name = scope;
if (num_spatial_dims == 2)
{
var filters_tensor = filters.AsTensor();
result = gen_nn_ops.conv2d(new Conv2dParams
{
Input = input,
Filter = filters_tensor,
Strides = strides,
Padding = padding,
DataFormat = data_format,
Dilations = dilations,
Name = name
});
}
else
{
throw new NotImplementedException("");
}
});
return(result);
}
/// <summary>
/// Compute the moving average of a variable.
/// </summary>
/// <param name="variable"></param>
/// <param name="value"></param>
/// <param name="decay"></param>
/// <param name="zero_debias"></param>
/// <param name="name"></param>
/// <returns></returns>
public static Tensor assign_moving_average(IVariableV1 variable, IVariableV1 value, Tensor decay,
bool zero_debias = true, string name = null)
{
return(tf_with(ops.name_scope(name, "AssignMovingAvg", new { variable, value, decay }), scope =>
{
decay = ops.convert_to_tensor(1.0f - decay, name: "decay");
if (decay.dtype != variable.dtype.as_base_dtype())
{
decay = math_ops.cast(decay, variable.dtype.as_base_dtype());
}
return state_ops.assign_sub(variable, (variable.AsTensor() - value.AsTensor()) * decay, name: scope);
}));
}
/// <summary>
/// Long short-term memory cell (LSTM).
/// </summary>
/// <param name="inputs"></param>
/// <param name="training"></param>
/// <param name="state"></param>
/// <returns></returns>
protected override Tensors Call(Tensors inputs, Tensor state = null, bool is_training = false)
{
var one = constant_op.constant(1, dtype: dtypes.int32);
// Parameters of gates are concatenated into one multiply for efficiency.
Tensor c = null;
Tensor h = null;
if (_state_is_tuple)
{
(c, h) = ((Tensor)_state.c, (Tensor)_state.h);
}
else
{
// array_ops.split(value: state, num_or_size_splits: 2, axis: one);
throw new NotImplementedException("BasicLstmCell call");
}
var gate_inputs = math_ops.matmul(array_ops.concat(new[] { (Tensor)inputs, h }, 1), _kernel.AsTensor());
gate_inputs = nn_ops.bias_add(gate_inputs, _bias.AsTensor());
// i = input_gate, j = new_input, f = forget_gate, o = output_gate
var tensors = array_ops.split(value: gate_inputs, num_split: 4, axis: one);
var(i, j, f, o) = (tensors[0], tensors[1], tensors[2], tensors[3]);
var forget_bias_tensor = constant_op.constant(_forget_bias, dtype: f.dtype);
// Note that using `add` and `multiply` instead of `+` and `*` gives a
// performance improvement. So using those at the cost of readability.
var new_c = gen_math_ops.add(
math_ops.multiply(c, math_ops.sigmoid(gen_math_ops.add(f, forget_bias_tensor))),
math_ops.multiply(math_ops.sigmoid(i), _activation.Activate(j)));
var new_h = math_ops.multiply(_activation.Activate(new_c), math_ops.sigmoid(o));
if (_state_is_tuple)
{
return(new_c);
}
else
{
return(array_ops.concat(new[] { new_c, new_h }, 1));
}
}
public static Tensor[] fused_batch_norm_v3(Tensor x,
Tensor scale,
Tensor offset,
IVariableV1 mean,
IVariableV1 variance,
float epsilon = 0.0001f,
float exponential_avg_factor = 1.0f,
string data_format = "NHWC",
bool is_training = true,
string name = null)
{
if (tf.executing_eagerly())
{
var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
"FusedBatchNormV3", name,
null,
x,
scale,
offset,
mean.AsTensor(),
variance.AsTensor(),
"epsilon", epsilon,
"exponential_avg_factor", exponential_avg_factor,
"data_format", data_format,
"is_training", is_training);
return(results);
}
var _op = tf.OpDefLib._apply_op_helper("FusedBatchNormV3", name: name, args: new
{
x,
scale,
offset,
mean,
variance,
epsilon,
data_format,
is_training
});
return(_op.outputs);
}
/// <summary>
/// Create model
/// </summary>
/// <param name="x"></param>
/// <returns></returns>
Tensor neural_net(Tensor x)
{
// Hidden fully connected layer with 128 neurons.
var layer_1 = tf.add(tf.matmul(x, h1.AsTensor()), b1.AsTensor());
// Apply sigmoid to layer_1 output for non-linearity.
layer_1 = tf.nn.sigmoid(layer_1);
// Hidden fully connected layer with 256 neurons.
var layer_2 = tf.add(tf.matmul(layer_1, h2.AsTensor()), b2.AsTensor());
// Apply sigmoid to layer_2 output for non-linearity.
layer_2 = tf.nn.sigmoid(layer_2);
// Output fully connected layer with a neuron for each class.
var out_layer = tf.matmul(layer_2, wout.AsTensor()) + bout.AsTensor();
// Apply softmax to normalize the logits to a probability distribution.
return(tf.nn.softmax(out_layer));
}
// 搭建网络模型
Tensor neural_net(Tensor x)
{
// 第1层隐藏层采用128个神经元。
var layer_1 = tf.add(tf.matmul(x, h1.AsTensor()), b1.AsTensor());
// 使用 sigmoid 激活函数，增加层输出的非线性特征
layer_1 = tf.nn.sigmoid(layer_1);
// 第2层隐藏层采用256个神经元。
var layer_2 = tf.add(tf.matmul(layer_1, h2.AsTensor()), b2.AsTensor());
// 使用 sigmoid 激活函数，增加层输出的非线性特征
layer_2 = tf.nn.sigmoid(layer_2);
// 输出层的神经元数量和标签类型数量相同
var out_layer = tf.matmul(layer_2, wout.AsTensor()) + bout.AsTensor();
// 使用 Softmax 函数将输出类别转换为各类别的概率分布
return(tf.nn.softmax(out_layer));
}
public Tensor conv2d(Tensor input, IVariableV1 filter, int[] strides, string padding, bool use_cudnn_on_gpu = true,
string data_format = "NHWC", int[] dilations = null, string name = null)
{
var parameters = new Conv2dParams
{
Input = input,
Filter = filter.AsTensor(),
Strides = strides,
Padding = padding,
UseCudnnOnGpu = use_cudnn_on_gpu,
DataFormat = data_format,
Name = name
};
if (dilations != null)
{
parameters.Dilations = dilations;
}
return(gen_nn_ops.conv2d(parameters));
}
/// <summary>
/// Helper function for embedding_lookup and _compute_sampled_logits.
/// </summary>
/// <param name="params"></param>
/// <param name="ids"></param>
/// <param name="partition_strategy"></param>
/// <param name="name"></param>
/// <param name="max_norm"></param>
/// <returns></returns>
public static Tensor _embedding_lookup_and_transform(IVariableV1 @params,
Tensor ids,
string partition_strategy = "mod",
string name = null,
string max_norm = null)
{
return(tf_with(ops.name_scope(name, "embedding_lookup", new { @params, ids }), scope =>
{
name = scope;
int np = 1;
ids = ops.convert_to_tensor(ids, name: "ids");
if (np == 1)
{
var gather = array_ops.gather(@params.AsTensor(), ids, name: name);
var result = _clip(gather, ids, max_norm);
return array_ops.identity(result);
}
throw new NotImplementedException("_embedding_lookup_and_transform");
}));
}
protected override Tensor call(Tensor inputs, bool training = false)
{
var outputs = _convolution_op.__call__(inputs, kernel);
if (use_bias)
{
if (data_format == "channels_first")
{
throw new NotImplementedException("call channels_first");
}
else
{
outputs = nn_ops.bias_add(outputs, bias.AsTensor(), data_format: "NHWC");
}
}
if (activation != null)
{
outputs = activation(outputs);
}
return(outputs);
}
/// <summary>
/// Adds a new softmax and fully-connected layer for training and eval.
///
/// We need to retrain the top layer to identify our new classes, so this function
/// adds the right operations to the graph, along with some variables to hold the
/// weights, and then sets up all the gradients for the backward pass.
///
/// The set up for the softmax and fully-connected layers is based on:
/// https://www.tensorflow.org/tutorials/mnist/beginners/index.html
/// </summary>
/// <param name="class_count"></param>
/// <param name="final_tensor_name"></param>
/// <param name="bottleneck_tensor"></param>
/// <param name="quantize_layer"></param>
/// <param name="is_training"></param>
/// <returns></returns>
private (Operation, Tensor, Tensor, Tensor, Tensor) add_final_retrain_ops(int class_count, string final_tensor_name,
Tensor bottleneck_tensor, bool quantize_layer, bool is_training)
{
var(batch_size, bottleneck_tensor_size) = (bottleneck_tensor.TensorShape.dims[0], bottleneck_tensor.TensorShape.dims[1]);
tf_with(tf.name_scope("input"), scope =>
{
bottleneck_input = tf.placeholder_with_default(
bottleneck_tensor,
shape: bottleneck_tensor.TensorShape.dims,
name: "BottleneckInputPlaceholder");
ground_truth_input = tf.placeholder(tf.int64, new TensorShape(batch_size), name: "GroundTruthInput");
});
// Organizing the following ops so they are easier to see in TensorBoard.
string layer_name = "final_retrain_ops";
Tensor logits = null;
tf_with(tf.name_scope(layer_name), scope =>
{
IVariableV1 layer_weights = null;
tf_with(tf.name_scope("weights"), delegate
{
var initial_value = tf.truncated_normal(new int[] { bottleneck_tensor_size, class_count }, stddev: 0.001f);
layer_weights = tf.Variable(initial_value, name: "final_weights");
variable_summaries(layer_weights.AsTensor());
});
IVariableV1 layer_biases = null;
tf_with(tf.name_scope("biases"), delegate
{
layer_biases = tf.Variable(tf.zeros(new TensorShape(class_count)), name: "final_biases");
variable_summaries(layer_biases.AsTensor());
});
tf_with(tf.name_scope("Wx_plus_b"), delegate
{
logits = tf.matmul(bottleneck_input, layer_weights.AsTensor()) + layer_biases.AsTensor();
tf.summary.histogram("pre_activations", logits);
});
});
final_tensor = tf.nn.softmax(logits, name: final_tensor_name);
// The tf.contrib.quantize functions rewrite the graph in place for
// quantization. The imported model graph has already been rewritten, so upon
// calling these rewrites, only the newly added final layer will be
// transformed.
if (quantize_layer)
{
throw new NotImplementedException("quantize_layer");
/*if (is_training)
* tf.contrib.quantize.create_training_graph();
* else
* tf.contrib.quantize.create_eval_graph();*/
}
tf.summary.histogram("activations", final_tensor);
// If this is an eval graph, we don't need to add loss ops or an optimizer.
if (!is_training)
{
return(null, null, bottleneck_input, ground_truth_input, final_tensor);
}
Tensor cross_entropy_mean = null;
tf_with(tf.name_scope("cross_entropy"), delegate
{
cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
labels: ground_truth_input, logits: logits);
});
tf.summary.scalar("cross_entropy", cross_entropy_mean);
tf_with(tf.name_scope("train"), delegate
{
var optimizer = tf.train.GradientDescentOptimizer(learning_rate);
train_step = optimizer.minimize(cross_entropy_mean);
});
return(train_step, cross_entropy_mean, bottleneck_input, ground_truth_input,
final_tensor);
}
private Tensor _fused_batch_norm(Tensor inputs, Tensor training)
{
TensorShape input_batch_size = null;
var use_fused_avg_updates = true;
float exponential_avg_factor = 0;
if (use_fused_avg_updates)
{
exponential_avg_factor = 1.0f - momentum;
}
var beta = this.beta;
var gamma = this.gamma;
Func <Tensor[]> _fused_batch_norm_training = () =>
{
return(tf.nn.fused_batch_norm(
inputs,
gamma,
beta,
epsilon: epsilon,
data_format: _data_format));
};
Func <Tensor[]> _fused_batch_norm_inference = () =>
{
var moving_mean_tensor = moving_mean.AsTensor();
var moving_variance_tensor = moving_variance.AsTensor();
return(tf.nn.fused_batch_norm(
inputs,
gamma,
beta,
mean: moving_mean_tensor,
variance: moving_variance_tensor,
epsilon: epsilon,
is_training: false,
data_format: _data_format));
};
if (use_fused_avg_updates && input_batch_size != null)
{
throw new NotImplementedException("");
}
var results = tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference);
var(output, mean, variance) = (results[0], results[1], results[2]);
var training_value = tf_utils.constant_value(training);
Tensor momentum_tensor;
if (training_value == null)
{
momentum_tensor = tf_utils.smart_cond(training,
() => new float[] { momentum }, () => new float[] { 1.0f })[0];
}
else
{
momentum_tensor = ops.convert_to_tensor(momentum);
}
if (training_value == null)
{
var mean_update = _assign_moving_average(moving_mean.AsTensor(), mean, momentum_tensor);
var variance_update = _assign_moving_average(moving_variance.AsTensor(), variance, momentum_tensor);
add_update(new Tensor[] { mean_update }, inputs: true);
add_update(new Tensor[] { variance_update }, inputs: true);
}
return(output);
}
public static void colocate_with(IVariableV1 variable, bool ignore_existing = false)
{
_colocate_with_for_gradient(variable.AsTensor(), null, ignore_existing);
}
public Optimizer AdamOptimizer(IVariableV1 learning_rate, string name = "Adam") => new AdamOptimizer(learning_rate.AsTensor(), name: name);