protected override Tensor call(Tensor inputs, bool training = false) { Tensor outputs = null; var rank = inputs.rank; if (rank > 2) { throw new NotImplementedException("call rank > 2"); } else { outputs = gen_math_ops.mat_mul(inputs, kernel.AsTensor()); } if (args.UseBias) { outputs = tf.nn.bias_add(outputs, bias); } if (args.Activation != null) { outputs = activation(outputs); } return(outputs); }
/// <summary> /// Create lambdas which compute regularization losses. /// </summary> /// <param name="name"></param> /// <param name="variable"></param> /// <param name="regularizer"></param> void _handle_weight_regularization(string name, IVariableV1 variable, IRegularizer regularizer) { add_loss(() => tf_with(ops.name_scope(name + "/Regularizer"), scope => regularizer.Apply(new RegularizerArgs(variable.AsTensor()) { }) )); }
public static Tensor[] smart_cond <T>(IVariableV1 pred, Func <T[]> true_fn = null, Func <T[]> false_fn = null, string name = null) { return(control_flow_ops.cond(pred.AsTensor(), true_fn: true_fn, false_fn: false_fn, name: name)); }
void _assign_moving_average(IVariableV1 variable, Tensor value, Tensor momentum) { tf_with(ops.name_scope("AssignMovingAvg", null, new { variable, value, momentum }), scope => { // var cm = ops.colocate_with(variable); var decay = ops.convert_to_tensor(1.0f - momentum, name: "decay"); var update_delta = (variable.AsTensor() - math_ops.cast(value, variable.dtype)) * decay; variable.assign_sub_lazy_load(update_delta, name: scope); }); }
protected Tensors Call(Tensors inputs, Tensor state = null, bool is_training = false) { // Most basic RNN: output = new_state = act(W * input + U * state + B). var concat = array_ops.concat(new Tensor[] { inputs, state }, 1); var gate_inputs = math_ops.matmul(concat, _kernel.AsTensor()); gate_inputs = nn_ops.bias_add(gate_inputs, _bias); var output = _activation(gate_inputs, null); return(new Tensors(output, output)); }
public static Tensor FullyConnectedLinearLayer(Tensor input, int outputDim, NeuralNetworkInitializer initializer, int seed) { int inputDim = (int)input.shape[1]; IInitializer?init = GetInitializer(initializer, seed); IVariableV1 W = tf.compat.v1.get_variable("W", new int[] { inputDim, outputDim }, tf.float32, init); IVariableV1 b = tf.compat.v1.get_variable("b", new int[] { outputDim }, tf.float32, init); return(tf.matmul(input, W.AsTensor()) + b.AsTensor()); }
public static Tensor cast(IVariableV1 x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null) { var base_type = dtype.as_base_dtype(); if (base_type == x.dtype) { return(x.AsTensor()); } return(tf_with(ops.name_scope(name, "Cast", new { x }), scope => { name = scope; var t_x = ops.convert_to_tensor(x, name: "x"); if (t_x.dtype.as_base_dtype() != base_type) { t_x = gen_math_ops.cast(t_x, base_type, name: name); } return x.AsTensor(); })); }
public Tensor __call__(Tensor inp, IVariableV1 filter) { return(conv_op(new Conv2dParams { Input = inp, Filter = filter.AsTensor(), Strides = strides, Padding = padding, DataFormat = data_format, Name = name })); }
public Tensor Apply(Tensors input, IVariableV1 filters) { var filters_rank = filters.shape.rank; var inputs_rank = input.shape.rank; var num_spatial_dims = args.NumSpatialDims; if (num_spatial_dims == Unknown) { num_spatial_dims = filters_rank - 2; } // Channel dimension. var num_batch_dims = inputs_rank - num_spatial_dims - 1; if (!new[] { 1, 2, 3 }.Contains(num_spatial_dims)) { throw new ValueError($"num_spatial_dims (input.shape.ndims - num_batch_dims - 1) must be one " + $"of 1, 2 or 3 but saw {num_spatial_dims}. num_batch_dims: {num_batch_dims}."); } var channel_index = num_batch_dims + num_spatial_dims; var dilations = _get_sequence(args.DilationRate, num_spatial_dims, channel_index); var strides = _get_sequence(args.Strides, num_spatial_dims, channel_index); Tensor result = null; tf_with(ops.name_scope(name, default_name: null), scope => { name = scope; if (num_spatial_dims == 2) { var filters_tensor = filters.AsTensor(); result = gen_nn_ops.conv2d(new Conv2dParams { Input = input, Filter = filters_tensor, Strides = strides, Padding = padding, DataFormat = data_format, Dilations = dilations, Name = name }); } else { throw new NotImplementedException(""); } }); return(result); }
/// <summary> /// Compute the moving average of a variable. /// </summary> /// <param name="variable"></param> /// <param name="value"></param> /// <param name="decay"></param> /// <param name="zero_debias"></param> /// <param name="name"></param> /// <returns></returns> public static Tensor assign_moving_average(IVariableV1 variable, IVariableV1 value, Tensor decay, bool zero_debias = true, string name = null) { return(tf_with(ops.name_scope(name, "AssignMovingAvg", new { variable, value, decay }), scope => { decay = ops.convert_to_tensor(1.0f - decay, name: "decay"); if (decay.dtype != variable.dtype.as_base_dtype()) { decay = math_ops.cast(decay, variable.dtype.as_base_dtype()); } return state_ops.assign_sub(variable, (variable.AsTensor() - value.AsTensor()) * decay, name: scope); })); }
/// <summary> /// Long short-term memory cell (LSTM). /// </summary> /// <param name="inputs"></param> /// <param name="training"></param> /// <param name="state"></param> /// <returns></returns> protected override Tensors Call(Tensors inputs, Tensor state = null, bool is_training = false) { var one = constant_op.constant(1, dtype: dtypes.int32); // Parameters of gates are concatenated into one multiply for efficiency. Tensor c = null; Tensor h = null; if (_state_is_tuple) { (c, h) = ((Tensor)_state.c, (Tensor)_state.h); } else { // array_ops.split(value: state, num_or_size_splits: 2, axis: one); throw new NotImplementedException("BasicLstmCell call"); } var gate_inputs = math_ops.matmul(array_ops.concat(new[] { (Tensor)inputs, h }, 1), _kernel.AsTensor()); gate_inputs = nn_ops.bias_add(gate_inputs, _bias.AsTensor()); // i = input_gate, j = new_input, f = forget_gate, o = output_gate var tensors = array_ops.split(value: gate_inputs, num_split: 4, axis: one); var(i, j, f, o) = (tensors[0], tensors[1], tensors[2], tensors[3]); var forget_bias_tensor = constant_op.constant(_forget_bias, dtype: f.dtype); // Note that using `add` and `multiply` instead of `+` and `*` gives a // performance improvement. So using those at the cost of readability. var new_c = gen_math_ops.add( math_ops.multiply(c, math_ops.sigmoid(gen_math_ops.add(f, forget_bias_tensor))), math_ops.multiply(math_ops.sigmoid(i), _activation.Activate(j))); var new_h = math_ops.multiply(_activation.Activate(new_c), math_ops.sigmoid(o)); if (_state_is_tuple) { return(new_c); } else { return(array_ops.concat(new[] { new_c, new_h }, 1)); } }
public static Tensor[] fused_batch_norm_v3(Tensor x, Tensor scale, Tensor offset, IVariableV1 mean, IVariableV1 variance, float epsilon = 0.0001f, float exponential_avg_factor = 1.0f, string data_format = "NHWC", bool is_training = true, string name = null) { if (tf.executing_eagerly()) { var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName, "FusedBatchNormV3", name, null, x, scale, offset, mean.AsTensor(), variance.AsTensor(), "epsilon", epsilon, "exponential_avg_factor", exponential_avg_factor, "data_format", data_format, "is_training", is_training); return(results); } var _op = tf.OpDefLib._apply_op_helper("FusedBatchNormV3", name: name, args: new { x, scale, offset, mean, variance, epsilon, data_format, is_training }); return(_op.outputs); }
/// <summary> /// Create model /// </summary> /// <param name="x"></param> /// <returns></returns> Tensor neural_net(Tensor x) { // Hidden fully connected layer with 128 neurons. var layer_1 = tf.add(tf.matmul(x, h1.AsTensor()), b1.AsTensor()); // Apply sigmoid to layer_1 output for non-linearity. layer_1 = tf.nn.sigmoid(layer_1); // Hidden fully connected layer with 256 neurons. var layer_2 = tf.add(tf.matmul(layer_1, h2.AsTensor()), b2.AsTensor()); // Apply sigmoid to layer_2 output for non-linearity. layer_2 = tf.nn.sigmoid(layer_2); // Output fully connected layer with a neuron for each class. var out_layer = tf.matmul(layer_2, wout.AsTensor()) + bout.AsTensor(); // Apply softmax to normalize the logits to a probability distribution. return(tf.nn.softmax(out_layer)); }
// 搭建网络模型 Tensor neural_net(Tensor x) { // 第1层隐藏层采用128个神经元。 var layer_1 = tf.add(tf.matmul(x, h1.AsTensor()), b1.AsTensor()); // 使用 sigmoid 激活函数,增加层输出的非线性特征 layer_1 = tf.nn.sigmoid(layer_1); // 第2层隐藏层采用256个神经元。 var layer_2 = tf.add(tf.matmul(layer_1, h2.AsTensor()), b2.AsTensor()); // 使用 sigmoid 激活函数,增加层输出的非线性特征 layer_2 = tf.nn.sigmoid(layer_2); // 输出层的神经元数量和标签类型数量相同 var out_layer = tf.matmul(layer_2, wout.AsTensor()) + bout.AsTensor(); // 使用 Softmax 函数将输出类别转换为各类别的概率分布 return(tf.nn.softmax(out_layer)); }
public Tensor conv2d(Tensor input, IVariableV1 filter, int[] strides, string padding, bool use_cudnn_on_gpu = true, string data_format = "NHWC", int[] dilations = null, string name = null) { var parameters = new Conv2dParams { Input = input, Filter = filter.AsTensor(), Strides = strides, Padding = padding, UseCudnnOnGpu = use_cudnn_on_gpu, DataFormat = data_format, Name = name }; if (dilations != null) { parameters.Dilations = dilations; } return(gen_nn_ops.conv2d(parameters)); }
/// <summary> /// Helper function for embedding_lookup and _compute_sampled_logits. /// </summary> /// <param name="params"></param> /// <param name="ids"></param> /// <param name="partition_strategy"></param> /// <param name="name"></param> /// <param name="max_norm"></param> /// <returns></returns> public static Tensor _embedding_lookup_and_transform(IVariableV1 @params, Tensor ids, string partition_strategy = "mod", string name = null, string max_norm = null) { return(tf_with(ops.name_scope(name, "embedding_lookup", new { @params, ids }), scope => { name = scope; int np = 1; ids = ops.convert_to_tensor(ids, name: "ids"); if (np == 1) { var gather = array_ops.gather(@params.AsTensor(), ids, name: name); var result = _clip(gather, ids, max_norm); return array_ops.identity(result); } throw new NotImplementedException("_embedding_lookup_and_transform"); })); }
protected override Tensor call(Tensor inputs, bool training = false) { var outputs = _convolution_op.__call__(inputs, kernel); if (use_bias) { if (data_format == "channels_first") { throw new NotImplementedException("call channels_first"); } else { outputs = nn_ops.bias_add(outputs, bias.AsTensor(), data_format: "NHWC"); } } if (activation != null) { outputs = activation(outputs); } return(outputs); }
/// <summary> /// Adds a new softmax and fully-connected layer for training and eval. /// /// We need to retrain the top layer to identify our new classes, so this function /// adds the right operations to the graph, along with some variables to hold the /// weights, and then sets up all the gradients for the backward pass. /// /// The set up for the softmax and fully-connected layers is based on: /// https://www.tensorflow.org/tutorials/mnist/beginners/index.html /// </summary> /// <param name="class_count"></param> /// <param name="final_tensor_name"></param> /// <param name="bottleneck_tensor"></param> /// <param name="quantize_layer"></param> /// <param name="is_training"></param> /// <returns></returns> private (Operation, Tensor, Tensor, Tensor, Tensor) add_final_retrain_ops(int class_count, string final_tensor_name, Tensor bottleneck_tensor, bool quantize_layer, bool is_training) { var(batch_size, bottleneck_tensor_size) = (bottleneck_tensor.TensorShape.dims[0], bottleneck_tensor.TensorShape.dims[1]); tf_with(tf.name_scope("input"), scope => { bottleneck_input = tf.placeholder_with_default( bottleneck_tensor, shape: bottleneck_tensor.TensorShape.dims, name: "BottleneckInputPlaceholder"); ground_truth_input = tf.placeholder(tf.int64, new TensorShape(batch_size), name: "GroundTruthInput"); }); // Organizing the following ops so they are easier to see in TensorBoard. string layer_name = "final_retrain_ops"; Tensor logits = null; tf_with(tf.name_scope(layer_name), scope => { IVariableV1 layer_weights = null; tf_with(tf.name_scope("weights"), delegate { var initial_value = tf.truncated_normal(new int[] { bottleneck_tensor_size, class_count }, stddev: 0.001f); layer_weights = tf.Variable(initial_value, name: "final_weights"); variable_summaries(layer_weights.AsTensor()); }); IVariableV1 layer_biases = null; tf_with(tf.name_scope("biases"), delegate { layer_biases = tf.Variable(tf.zeros(new TensorShape(class_count)), name: "final_biases"); variable_summaries(layer_biases.AsTensor()); }); tf_with(tf.name_scope("Wx_plus_b"), delegate { logits = tf.matmul(bottleneck_input, layer_weights.AsTensor()) + layer_biases.AsTensor(); tf.summary.histogram("pre_activations", logits); }); }); final_tensor = tf.nn.softmax(logits, name: final_tensor_name); // The tf.contrib.quantize functions rewrite the graph in place for // quantization. The imported model graph has already been rewritten, so upon // calling these rewrites, only the newly added final layer will be // transformed. if (quantize_layer) { throw new NotImplementedException("quantize_layer"); /*if (is_training) * tf.contrib.quantize.create_training_graph(); * else * tf.contrib.quantize.create_eval_graph();*/ } tf.summary.histogram("activations", final_tensor); // If this is an eval graph, we don't need to add loss ops or an optimizer. if (!is_training) { return(null, null, bottleneck_input, ground_truth_input, final_tensor); } Tensor cross_entropy_mean = null; tf_with(tf.name_scope("cross_entropy"), delegate { cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy( labels: ground_truth_input, logits: logits); }); tf.summary.scalar("cross_entropy", cross_entropy_mean); tf_with(tf.name_scope("train"), delegate { var optimizer = tf.train.GradientDescentOptimizer(learning_rate); train_step = optimizer.minimize(cross_entropy_mean); }); return(train_step, cross_entropy_mean, bottleneck_input, ground_truth_input, final_tensor); }
private Tensor _fused_batch_norm(Tensor inputs, Tensor training) { TensorShape input_batch_size = null; var use_fused_avg_updates = true; float exponential_avg_factor = 0; if (use_fused_avg_updates) { exponential_avg_factor = 1.0f - momentum; } var beta = this.beta; var gamma = this.gamma; Func <Tensor[]> _fused_batch_norm_training = () => { return(tf.nn.fused_batch_norm( inputs, gamma, beta, epsilon: epsilon, data_format: _data_format)); }; Func <Tensor[]> _fused_batch_norm_inference = () => { var moving_mean_tensor = moving_mean.AsTensor(); var moving_variance_tensor = moving_variance.AsTensor(); return(tf.nn.fused_batch_norm( inputs, gamma, beta, mean: moving_mean_tensor, variance: moving_variance_tensor, epsilon: epsilon, is_training: false, data_format: _data_format)); }; if (use_fused_avg_updates && input_batch_size != null) { throw new NotImplementedException(""); } var results = tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference); var(output, mean, variance) = (results[0], results[1], results[2]); var training_value = tf_utils.constant_value(training); Tensor momentum_tensor; if (training_value == null) { momentum_tensor = tf_utils.smart_cond(training, () => new float[] { momentum }, () => new float[] { 1.0f })[0]; } else { momentum_tensor = ops.convert_to_tensor(momentum); } if (training_value == null) { var mean_update = _assign_moving_average(moving_mean.AsTensor(), mean, momentum_tensor); var variance_update = _assign_moving_average(moving_variance.AsTensor(), variance, momentum_tensor); add_update(new Tensor[] { mean_update }, inputs: true); add_update(new Tensor[] { variance_update }, inputs: true); } return(output); }
public static void colocate_with(IVariableV1 variable, bool ignore_existing = false) { _colocate_with_for_gradient(variable.AsTensor(), null, ignore_existing); }
public Optimizer AdamOptimizer(IVariableV1 learning_rate, string name = "Adam") => new AdamOptimizer(learning_rate.AsTensor(), name: name);