public Tensor categorical_crossentropy(Tensor target, Tensor output, bool from_logits = false)
{
// https://github.com/fchollet/keras/blob/f65a56fb65062c8d14d215c9f4b1015b97cc5bf3/keras/backend/cntk_backend.py#L1480
var _output = In(output);
var _target = In(target);
if (from_logits)
{
var result = C.CrossEntropyWithSoftmax(_output, _target);
// cntk's result shape is (batch, 1), while keras expect (batch, )
CNTK.Function r = C.Reshape(result, NDShape.CreateNDShape(new int[] { }));
return(Out(r));
}
else
{
// scale preds so that the class probas of each sample sum to 1
var o = C.ElementDivide(_output.function, C.ReduceSum(_output, Axis.EndStaticAxis()));
var eps = Constant.Scalar(epsilon(), DeviceDescriptor.CPUDevice);
var omeps = Constant.Scalar(1.0 - epsilon(), DeviceDescriptor.CPUDevice);
// avoid numerical instability with _EPSILON clipping
o = C.Clip(o, eps, omeps);
CNTK.Function r = C.Negate(C.ReduceSum(C.ElementTimes(_target, C.Log(_output)), Axis.EndStaticAxis()));
return(Out(r));
}
}
private CNTK.Function _reshape_dummy_dim(CNTK.Function x, params int[] axis)
{
// https://github.com/fchollet/keras/blob/f65a56fb65062c8d14d215c9f4b1015b97cc5bf3/keras/backend/cntk_backend.py#L680
List <int> shape = In(x.Output.Shape).ToList();
var _axis = axis.Select(i => i < 0 ? (i + shape.Count) : i).ToArray();
if (shape.Count(s => s == NDShape.InferredDimension) > 1)
{
var result = x;
foreach (int index in _axis.Sorted().Reverse())
{
result = C.Reshape(result, replacementShape: NDShape.CreateNDShape(new int[] { }),
beginAxis: new Axis(index), endAxis: new Axis(index + 1));
}
return(result);
}
else
{
foreach (int index in _axis.Sorted().Reverse())
{
shape.RemoveAt(index);
}
return(C.Reshape(x, NDShape.CreateNDShape(shape)));
}
}
C.Function create_decoder(int[] digits_capsules_output_shape)
{
var decoder_input = Util.inputVariable(digits_capsules_output_shape);
var decoder = Layers.Dense(decoder_input, 512, computeDevice, activation: CC.ReLU);
decoder = Layers.Dense(decoder, 1024, computeDevice, activation: CC.ReLU);
decoder = Layers.Dense(decoder, Util.np_prod(input_shape), computeDevice, activation: CC.Sigmoid);
decoder = CC.Reshape(decoder, input_shape, name: "out_recon");
return(decoder);
}
/// <summary>
/// Turn a nD tensor into a 2D tensor with same 0th dimension. In other words, it flattens each data samples of a batch.
/// </summary>
///
public Tensor batch_flatten(Tensor x)
{
// https://github.com/fchollet/keras/blob/f65a56fb65062c8d14d215c9f4b1015b97cc5bf3/keras/backend/cntk_backend.py#L1460
// cntk's batch axis is not in shape,
// so just flatten all the dim in x.shape
int dim = Matrix.Product(x.shape.Select(s => s.Value).ToArray());
x = Out(C.Reshape(In(x), NDShape.CreateNDShape(new[] { -1 })));
x._keras_shape = new int?[] { null, dim };
return(x);
}
public Tensor bias_add(Tensor output, Tensor bias)
{
CNTKTensor _x = In(output);
CNTKTensor _b = In(bias);
var _shape = In(_x.CNTK_Shape).Select(x => x < 0 ? 1 : x).ToArray();
var shape = NDShape.CreateNDShape(_shape);
var b = C.Reshape(_b, shape);
return(Out(new Variable(_x.function) + b));
}
public static Function Dense(Variable operand, int outputDim, DataType dataType, CNTKDictionary weightInitializer, DeviceDescriptor device, string name)
{
if (operand.Shape.Rank == 1)
{
return(FullyConnectedLinearLayer(operand, outputDim, dataType, device, weightInitializer, name));
}
//flatten input layer
var newDim = operand.Shape.Dimensions.Aggregate((d1, d2) => d1 * d2);
operand = CNTKLib.Reshape(operand, new[] { newDim });
return(FullyConnectedLinearLayer(operand, outputDim, dataType, device, weightInitializer, name));
}
C.Function create_capsule_layer(C.Function inputs, int num_capsule, int dim_capsule, int routings, string name)
{
var inputs_shape = inputs.Output.Shape.Dimensions;
var input_num_capsule = inputs_shape[0];
var input_dim_capsule = inputs_shape[1];
var W = new C.Parameter(
new int[] { num_capsule, dim_capsule, input_num_capsule, input_dim_capsule },
C.DataType.Float,
CC.GlorotUniformInitializer(),
computeDevice,
name: "W");
inputs = CC.Reshape(inputs, new int[] { 1, 1, input_num_capsule, input_dim_capsule }); // [1, 1, 1152, 8])
var inputs_hat = CC.ElementTimes(W, inputs);
inputs_hat = CC.ReduceSum(inputs_hat, new C.Axis(3));
inputs_hat = CC.Squeeze(inputs_hat);
C.Function outputs = null;
var zeros = new C.Constant(new int[] { num_capsule, 1, input_num_capsule }, C.DataType.Float, 0, computeDevice);
var b = CC.Combine(new C.VariableVector()
{
zeros
});
for (int i = 0; i < routings; i++)
{
var c = CC.Softmax(b, new C.Axis(0));
var batch_dot_result = CC.ElementTimes(c, inputs_hat);
batch_dot_result = CC.ReduceSum(batch_dot_result, new C.Axis(2));
batch_dot_result = CC.Squeeze(batch_dot_result);
outputs = squash(batch_dot_result, name: $"squashed_{i}", axis: 1);
if (i < (routings - 1))
{
outputs = CC.Reshape(outputs, new int[] { num_capsule, dim_capsule, 1 });
batch_dot_result = CC.ElementTimes(outputs, inputs_hat);
batch_dot_result = CC.ReduceSum(batch_dot_result, new C.Axis(1));
b = CC.Plus(b, batch_dot_result);
}
}
outputs = CC.Combine(new C.VariableVector()
{
outputs
}, name);
return(outputs);
}
C.Function get_mask_and_infer_from_last_dimension(C.Function inputs, C.Function mask)
{
if (mask == null)
{
var inputs_shape = inputs.Output.Shape.Dimensions.ToArray();
var ndims = inputs_shape.Length - 1;
var x = CC.Sqrt(CC.ReduceSum(CC.Square(inputs), new C.Axis(ndims - 1)));
x = CC.Squeeze(x);
System.Diagnostics.Debug.Assert(x.Output.Shape.Dimensions.Count == 1);
x = CC.Argmax(x, new C.Axis(0));
mask = CC.OneHotOp(x, numClass: (uint)inputs_shape[0], outputSparse: false, axis: new C.Axis(0));
}
mask = CC.Reshape(mask, mask.Output.Shape.AppendShape(new int[] { 1 }));
var masked = CC.ElementTimes(inputs, mask);
masked = CC.Flatten(masked);
masked = CC.Squeeze(masked);
return(masked);
}
C.Function create_capsule(string name, int[] input_shape, int[] extra_input_shape, int recognizer_dim, int generator_dim)
{
var input_dim = Util.np_prod(input_shape);
var x = Util.placeholderVariable(input_shape, "x");
var extra_input = Util.placeholderVariable(extra_input_shape, "extra_input");
var x_flat = CC.Flatten(x);
var recognition = Layers.Dense(x_flat, recognizer_dim, computeDevice, "recognition_layer", CC.Sigmoid);
var probability = Layers.Dense(recognition, 1, computeDevice, "probability", CC.Sigmoid);
var learnt_transformation = Layers.Dense(recognition, 2, computeDevice, "xy_prediction");
var learnt_transformation_extended = CC.Plus(learnt_transformation, extra_input, "learnt_transformation_extended");
var generation = Layers.Dense(learnt_transformation_extended, generator_dim, computeDevice, "generator_layer", CC.Sigmoid);
var out_flat = Layers.Dense(generation, input_dim, computeDevice, "output");
out_flat = CC.ElementTimes(out_flat, probability);
var output = CC.Reshape(out_flat, input_shape);
return(output);
}
public Tensor bias_add(Tensor output, Tensor bias, DataFormatType?data_format = null, string name = null)
{
if (data_format != null)
{
throw new NotImplementedException();
}
using (this.name_scope("bias_add"))
{
CNTKTensor _x = In(output);
CNTKTensor _b = In(bias);
var _shape = In(_x.CNTK_Shape).Select(x => x < 0 ? 1 : x).ToArray();
var shape = NDShape.CreateNDShape(_shape);
var b = C.Reshape(_b, shape);
return(Out(new Variable(_x.function) + b));
}
}
C.Function create_primary_cap(C.Function inputs, int dim_capsule, int n_channels, int[] kernel_size, int[] strides, bool pad)
{
var output = Layers.Convolution2D(
inputs,
dim_capsule * n_channels,
kernel_size,
computeDevice,
strides: strides,
use_padding: pad,
name: "primarycap_conv2d");
var outputShape = output.Output.Shape.Dimensions;
System.Diagnostics.Debug.Assert((outputShape[2] == 256) && (outputShape[1] == 6) && (outputShape[0] == 6));
var num_rows = (int)(Util.np_prod(outputShape.ToArray()) / dim_capsule);
var target_shape = new int[] { num_rows, dim_capsule };
var outputs = CC.Reshape(output, target_shape, name: "primarycap_reshape");
var rtrn = squash(outputs, name: "primarycap_squash", axis: 1);
return(rtrn);
}
public Tensor Reshape(Tensor x, params long[] shape)
{
return(Out(C.Reshape(In(x), BackendUtil.CastShapeInt(shape))));
}