/// <summary>
///
/// </summary>
/// <param name="input"></param>
/// <param name="convolution_map_size"></param>
/// <param name="device"></param>
/// <param name="use_padding"></param>
/// <param name="use_bias"></param>
/// <param name="strides"></param>
/// <param name="activation"></param>
/// <param name="outputName"></param>
/// <returns></returns>
public static Function Convolution(
Variable input,
int[] convolution_map_size,
DeviceDescriptor device,
bool use_padding,
bool use_bias,
int[] strides,
Func <Variable, string, Function> activation = null,
string outputName = "")
{
var W = new Parameter(
NDShape.CreateNDShape(convolution_map_size),
DataType.Float,
CNTKLib.GlorotUniformInitializer(CNTKLib.DefaultParamInitScale, CNTKLib.SentinelValueForInferParamInitRank, CNTKLib.SentinelValueForInferParamInitRank, 1),
device,
outputName + "_W");
//y = Wx+b
Variable y = CNTKLib.Convolution(W, input, strides, new BoolVector(new bool[] { true }) /* sharing */, new BoolVector(new bool[] { use_padding }));
//apply bias
if (use_bias)
{
var num_output_channels = convolution_map_size[convolution_map_size.Length - 1];
var b_shape = Concat(Enumerable.Repeat(1, convolution_map_size.Length - 2).ToArray(), new int[] { num_output_channels });
var b = new Parameter(b_shape, 0.0f, device, outputName + "_b");
y = CNTKLib.Plus(y, b);
}
//apply activation
if (activation != null)
{
y = activation(y, outputName);
}
return(y);
}
/// <summary>
/// Implementation of : http://colah.github.io/posts/2015-08-Understanding-LSTMs/
/// </summary>
public LSTM LSTMNode(Variable input, int cellDims)
{
int outputDim = cellDims;
Variable previousC = CNTKLib.PlaceholderVariable(NDShape.CreateNDShape(new int[] { cellDims }), DataType.Double, "previousC", new AxisVector(input.DynamicAxes.ToList())); // placeholder for graph building
Variable previousH = CNTKLib.PlaceholderVariable(NDShape.CreateNDShape(new int[] { outputDim }), DataType.Double, "previousH", new AxisVector(input.DynamicAxes.ToList())); // placeholder for graph building
Variable forgetGate = DenseNode(CNTKLib.Plus(LinearNode(input, cellDims), LinearNode(previousH, cellDims)), cellDims, CNTKLib.Sigmoid);
Variable inputGate = DenseNode(CNTKLib.Plus(LinearNode(input, cellDims), LinearNode(previousH, cellDims)), cellDims, CNTKLib.Sigmoid);
Variable partialC = DenseNode(CNTKLib.Plus(LinearNode(input, cellDims), LinearNode(previousH, cellDims)), cellDims, CNTKLib.Sigmoid);
Function filteredOldState = CNTKLib.ElementTimes(forgetGate, previousC); // element wise multiplication https://www.mathworks.com/help/matlab/ref/times.html
Function filteredNewState = CNTKLib.ElementTimes(inputGate, partialC);
Function c = CNTKLib.Plus(filteredOldState, filteredNewState);
Function outputGate = DenseNode(CNTKLib.Plus(LinearNode(input, cellDims), LinearNode(previousH, cellDims)), outputDim, CNTKLib.Sigmoid);
Function h = CNTKLib.ElementTimes(outputGate, c);
// replace placeholders by recursive variables
h.ReplacePlaceholders(new Dictionary <Variable, Variable>
{
{ previousH, CNTKLib.PastValue(h) },
{ previousC, CNTKLib.PastValue(c) },
});
return(new LSTM
{
H_output = h,
C_cellstate = c
});
}
private Function Conv(int[] convMapSize, Variable x, DataType dType, DeviceDescriptor device, bool usePadding, bool useBias, string name, uint seed)
{
var shape = CNTK.NDShape.CreateNDShape(convMapSize);
var W = Weights(shape, dType, device, seed);
//
var strides = new int[] { 1 };
var sharing = new bool[] { true };
var aPadding = new bool[] { usePadding };
//
var result = CNTKLib.Convolution(W, x, strides, sharing, aPadding);
if (useBias)
{
var intArray = convMapSize.Length == 4 ?
new int[] { 1, 1, NDShape.InferredDimension } :
new int[] { 1, };
var bShape = NDShape.CreateNDShape(intArray);
var b = Bias(bShape, dType, device);
result = CNTK.CNTKLib.Plus(result, b);
}
result = CNTK.CNTKLib.ReLU(result, name);
return(result);
}
// Todo: move it to separate unit tests.
public void NDArrayViewTest(DeviceDescriptor device)
{
Console.WriteLine("Test creating NDArrayView on devices.");
var data = new float[10];
var shape = new NDShape(1, 10);
var n1 = new NDArrayView(shape, data, device);
var n1Clone = n1.DeepClone(device);
var n1CloneCPU = n1.DeepClone(DeviceDescriptor.CPUDevice);
Console.WriteLine("n1: on " + n1.Device.AsString() + ", Storage:" + n1.StorageFormat + ", Shape:" + n1.Shape.AsString());
Console.WriteLine("n1Clone: on " + n1Clone.Device.AsString() + ", Storage:" + n1Clone.StorageFormat + ", Shape:" + n1Clone.Shape.AsString());
Console.WriteLine("n1CloneCPU: on " + n1CloneCPU.Device.AsString() + ", Storage:" + n1CloneCPU.StorageFormat + ", Shape:" + n1CloneCPU.Shape.AsString());
int[] dimensions = { 4, 5 };
var shape2 = NDShape.CreateNDShape(dimensions);
float[] nonZeroValues = { 1, 5, 4, 2, 3, 9, 7, 8, 6 };
int[] rowIndices = { 0, 2, 0, 1, 1, 3, 2, 2, 3 };
int[] colStarts = { 0, 2, 4, 6, 7, 9 };
var s1 = new NDArrayView(shape2, colStarts, rowIndices, nonZeroValues, device, true);
var s1Clone = s1.DeepClone(device);
var s1DenseCPU = new NDArrayView(DataType.Float, StorageFormat.Dense, shape2, DeviceDescriptor.CPUDevice);
s1DenseCPU.CopyFrom(s1);
Console.WriteLine("s1: on " + s1.Device.AsString() + ", Storage:" + s1.StorageFormat + ", Shape:" + s1.Shape.AsString());
Console.WriteLine("s1Clone: on " + s1Clone.Device.AsString() + ", Storage:" + s1Clone.StorageFormat + ", Shape:" + s1Clone.Shape.AsString());
Console.WriteLine("s1DenseCPU: on " + s1DenseCPU.Device.AsString() + ", Storage:" + s1DenseCPU.StorageFormat + ", Shape:" + s1DenseCPU.Shape.AsString());
}
/// <summary>
/// create model by w,h,c,outputClassNum
/// </summary>
/// <param name="w"></param>
/// <param name="h"></param>
/// <param name="c"></param>
/// <param name="outputClassNum"></param>
/// <param name="deviceName"></param>
public FCN7(int w, int h, int c, int outputClassNum, string deviceName)
{
device = NP.CNTKHelper.GetDeviceByName(deviceName);
//input output variable
int[] inputDim = new int[] { w, h, c };
int[] outputDim = new int[] { outputClassNum };
inputVariable = Variable.InputVariable(NDShape.CreateNDShape(inputDim), DataType.Float, "inputVariable");
outputVariable = Variable.InputVariable(NDShape.CreateNDShape(outputDim), DataType.Float, "labelVariable");
//build model
classifierOutput = CreateFullyChannelNetwork(inputVariable, c, outputClassNum);
Function loss = CNTKLib.SquaredError(classifierOutput, outputVariable);
Function pred = CNTKLib.ClassificationError(classifierOutput, outputVariable);
//adam leaner
ParameterVector parameterVector = new ParameterVector(classifierOutput.Parameters().ToList());
TrainingParameterScheduleDouble learningRateSchedule = new TrainingParameterScheduleDouble(0.00178125, BatchSize);
TrainingParameterScheduleDouble momentumRateSchedule = new TrainingParameterScheduleDouble(0.9, BatchSize);
Learner leaner = CNTKLib.AdamLearner(parameterVector, learningRateSchedule, momentumRateSchedule, true);
//æž„é€ leaner方法
trainer = Trainer.CreateTrainer(classifierOutput, loss, pred, new List <Learner>()
{
leaner
});
//TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.00178125, BatchSize); //0.00178125
//TrainingParameterScheduleDouble momentumTimeConstant = CNTKLib.MomentumAsTimeConstantSchedule(256);
//IList<Learner> parameterLearners = new List<Learner>() { Learner.MomentumSGDLearner(classifierOutput.Parameters(), learningRatePerSample, momentumTimeConstant, true) };
//trainer = Trainer.CreateTrainer(classifierOutput, loss, pred, parameterLearners);
}
Function Discriminator(Function input, Func <CNTKDictionary> weightInit, CNTKDictionary biasInit,
DeviceDescriptor device, DataType dataType)
{
var discriminatorNetwork = input
.Reshape(NDShape.CreateNDShape(new int[] { 28, 28, 1 }))
.Conv2D((5, 5), 1, (2, 2), Padding.None, weightInit(), biasInit, device, dataType)
.BatchNorm(BatchNorm.Spatial, device, dataType)
.LeakyReLU(0.2)
.Conv2D((5, 5), 64, (2, 2), Padding.None, weightInit(), biasInit, device, dataType)
.BatchNorm(BatchNorm.Spatial, device, dataType)
.LeakyReLU(0.2)
.Dense(1024, weightInit(), biasInit, device, dataType)
.BatchNorm(BatchNorm.Regular, device, dataType)
.LeakyReLU(0.2)
.Dense(1, weightInit(), biasInit, device, dataType)
.Sigmoid();
Trace.Write(Model.Summary(discriminatorNetwork));
return(discriminatorNetwork);
}
public Tensor categorical_crossentropy(Tensor target, Tensor output, bool from_logits = false)
{
// https://github.com/fchollet/keras/blob/f65a56fb65062c8d14d215c9f4b1015b97cc5bf3/keras/backend/cntk_backend.py#L1480
var _output = In(output);
var _target = In(target);
if (from_logits)
{
var result = C.CrossEntropyWithSoftmax(_output, _target);
// cntk's result shape is (batch, 1), while keras expect (batch, )
CNTK.Function r = C.Reshape(result, NDShape.CreateNDShape(new int[] { }));
return(Out(r));
}
else
{
// scale preds so that the class probas of each sample sum to 1
var o = C.ElementDivide(_output.function, C.ReduceSum(_output, Axis.EndStaticAxis()));
var eps = Constant.Scalar(epsilon(), DeviceDescriptor.CPUDevice);
var omeps = Constant.Scalar(1.0 - epsilon(), DeviceDescriptor.CPUDevice);
// avoid numerical instability with _EPSILON clipping
o = C.Clip(o, eps, omeps);
CNTK.Function r = C.Negate(C.ReduceSum(C.ElementTimes(_target, C.Log(_output)), Axis.EndStaticAxis()));
return(Out(r));
}
}
private CNTK.Function _reshape_dummy_dim(CNTK.Function x, params int[] axis)
{
// https://github.com/fchollet/keras/blob/f65a56fb65062c8d14d215c9f4b1015b97cc5bf3/keras/backend/cntk_backend.py#L680
List <int> shape = In(x.Output.Shape).ToList();
var _axis = axis.Select(i => i < 0 ? (i + shape.Count) : i).ToArray();
if (shape.Count(s => s == NDShape.InferredDimension) > 1)
{
var result = x;
foreach (int index in _axis.Sorted().Reverse())
{
result = C.Reshape(result, replacementShape: NDShape.CreateNDShape(new int[] { }),
beginAxis: new Axis(index), endAxis: new Axis(index + 1));
}
return(result);
}
else
{
foreach (int index in _axis.Sorted().Reverse())
{
shape.RemoveAt(index);
}
return(C.Reshape(x, NDShape.CreateNDShape(shape)));
}
}
public Function LinearNode(Variable input, int outputDim)
{
if (input.Shape.Rank != 1)
{
// un dense layer prend un vecteur seulement en entrée => on transforme l'entrée en vecteur
int nbDims = 1;
for (int i = 0; i < input.Shape.Dimensions.Count; i++)
{
nbDims = nbDims * input.Shape.Dimensions[i];
}
var vector = CNTKLib.Reshape(input, NDShape.CreateNDShape(new[] { nbDims }));
return(LinearNode(vector, outputDim));
}
List <int> weightDimensions = new List <int>();
weightDimensions.Add(outputDim);
weightDimensions.Add(input.Shape.Dimensions.Single());
Parameter weights = new Parameter(NDShape.CreateNDShape(weightDimensions), DataType.Double,
CNTKLib.GlorotUniformInitializer( // valeur par défaut aléatoire
CNTKLib.DefaultParamInitScale,
CNTKLib.SentinelValueForInferParamInitRank,
CNTKLib.SentinelValueForInferParamInitRank, 1)); // les coefficients à trouver
Parameter bias = new Parameter(NDShape.CreateNDShape(new[] { outputDim }), DataType.Double, 0); // une abscisse pour chaque dimension
Function ax = CNTKLib.Times(weights, input);
Function ax_plus_b = CNTKLib.Plus(bias, ax);
return(ax_plus_b);
}
Parameter Parameter(IEnumerable <int> dims)
{
return(new Parameter(NDShape.CreateNDShape(dims), DataType.Double,
CNTKLib.GlorotUniformInitializer( // valeur par défaut aléatoire
CNTKLib.DefaultParamInitScale,
CNTKLib.SentinelValueForInferParamInitRank,
CNTKLib.SentinelValueForInferParamInitRank, 1)));
}
/// <summary>
/// Based on Dense from: https://github.com/Microsoft/CNTK/blob/master/bindings/python/cntk/layers/layers.py
/// </summary>
public static Function Dense(this Function input,
int units,
CNTKDictionary weightInitializer,
CNTKDictionary biasInitializer,
DeviceDescriptor device,
DataType dataType,
int inputRank = 0,
int mapRank = 0)
{
if (inputRank != 0 && mapRank != 0)
{
throw new ArgumentException("Dense: inputRank and mapRank cannot be specified at the same time.");
}
var outputShape = NDShape.CreateNDShape(new int[] { units });
var outputRank = outputShape.Dimensions.Count;
var inputRanks = (inputRank != 0) ? inputRank : 1;
var dimensions = Enumerable.Range(0, inputRanks)
.Select(v => NDShape.InferredDimension).ToArray(); // infer all dimensions.
var inputShape = NDShape.CreateNDShape(dimensions);
int inferInputRankToMap;
if (inputRank != 0)
{
inferInputRankToMap = -1; // means map_rank is not specified; input_rank rules.
}
else if (mapRank == 0)
{
inferInputRankToMap = 0; // neither given: default to 'infer W to use all input dims'.
}
else
{
inferInputRankToMap = mapRank; // infer W to use all input dims except the first static 'map_rank' ones.
}
var weightsDimensions = outputShape.Dimensions.ToList();
weightsDimensions.AddRange(inputShape.Dimensions);
var weightsShape = NDShape.CreateNDShape(weightsDimensions);
var weights = new Parameter(weightsShape, dataType, weightInitializer, device, "w");
// Weights and input is in reversed order compared to the original python code.
// Same goes for the dimensions. This is because the python api reverses the dimensions internally.
// The python API was made in this way to be similar to other deep learning toolkits.
// The C# and the C++ share the same column major layout.
var r = CNTKLib.Times(weights, input, (uint)outputRank, inferInputRankToMap);
if (biasInitializer != null)
{
var biasParameter = new Parameter(outputShape, dataType, biasInitializer, device, "b");
r = r + biasParameter;
}
return(r);
}
public void cntk_partial_shape_test()
{
// Note: Keras/TensorFlow represent unknown dimensions
// as None, whereas TensorFlowSharp represents as -1:
/*
* import keras
*
* from keras.models import Sequential
* from keras.layers import Dense
* from keras import backend as K
* import numpy as np
*
* a = K.placeholder(shape = (None, 2))
* b = K.variable(np.matrix([[1, 2, 3], [4, 5, 6]]))
* ab = K.dot(a, b)
*
* shape_a = K.int_shape(a)
* shape_b = K.int_shape(b)
* shape_ab = K.int_shape(ab)
*
* print(shape_a)
* print(shape_b)
* print(shape_ab)
*
* >>> Using TensorFlow backend.
* (None, 2)
* (2, 3)
* (None, 3)
*/
using (var K = new CNTKBackend())
{
Tensor a = K.placeholder(shape: new int?[] { null, 2 });
Tensor b = K.variable(array: new float[, ] {
{ 1, 2, 3 },
{ 4, 5, 6 }
});
var ab = K.dot(a, b);
int?[] shape_a = K.int_shape(a);
int?[] shape_b = K.int_shape(b);
int?[] shape_ab = K.int_shape(ab);
NDShape tf_shape_a = K.In(a).CNTK_Shape;
NDShape tf_shape_b = K.In(b).CNTK_Shape;
NDShape tf_shape_ab = K.In(ab).CNTK_Shape;
AssertEx.AreEqual(new int?[] { null, 2 }, shape_a);
AssertEx.AreEqual(new int?[] { 2, 3 }, shape_b);
AssertEx.AreEqual(new int?[] { null, 3 }, shape_ab);
Assert.AreEqual(NDShape.CreateNDShape(new[] { NDShape.FreeDimension, 2 }), tf_shape_a);
Assert.AreEqual(NDShape.CreateNDShape(new[] { 2, 3 }), tf_shape_b);
Assert.AreEqual(NDShape.CreateNDShape(new[] { NDShape.FreeDimension, 3 }), tf_shape_ab);
}
}
internal static Function Conv(this Function input,
int[] filterShape,
int filterCount,
int[] filterStride,
Padding padding, // TODO: Consider if padding should be decided pr. dimension.
CNTKDictionary weightInitializer,
CNTKDictionary biasInitializer,
DeviceDescriptor device,
DataType dataType)
{
var weights = new Parameter(NDShape.CreateNDShape(filterShape), dataType,
weightInitializer, device);
var strideShape = NDShape.CreateNDShape(filterStride);
// Currently, only sharing=true is supported by CNTK. So these are hardcoded.
// sharing dimensions follows stride dimensions. 1D, 2D, 3D, etc.
var sharing = CntkUtilities.CreateFilledBoolVector(filterStride.Length, true);
// Padding dimensions follows stride dimensions. 1D, 2D, 3D, etc.
var usePadding = padding.ToBoolean();
var autoPadding = CntkUtilities.CreateFilledBoolVector(filterStride.Length, usePadding);
// TODO: Consider if we want to surface the additional options for Convolution:
// - dilation
// - reductionRank
// - groups
// - maxTempMemSizeInSamples
// Default for dilation seems to be a shape of size (1) with value 1
var dilation = NDShape.CreateNDShape(new[] { 1 });
// Following are defaults extrapolated from CNTK code
var reductionRank = 1u;
var groups = 1u;
var maxTempMemSizeInSamples = 0u;
var sequential = false;
var result = CNTKLib.Convolution(
weights, input, strideShape, sharing, autoPadding,
dilation, reductionRank, groups, maxTempMemSizeInSamples, sequential);
if (biasInitializer != null)
{
// Bias dimensions should be defined for filter dimensions.
// For instance for 2D case: (1, 1, filterChannels).
var biasShape = filterStride.Select(s => 1).ToList();
biasShape.Add(filterCount);
var bias = new Parameter(NDShape.CreateNDShape(biasShape.ToArray()),
dataType, biasInitializer, device);
result = CNTKLib.Plus(result, bias);
}
return(result);
}
/// <summary>
/// Turn a nD tensor into a 2D tensor with same 0th dimension. In other words, it flattens each data samples of a batch.
/// </summary>
///
public Tensor batch_flatten(Tensor x)
{
// https://github.com/fchollet/keras/blob/f65a56fb65062c8d14d215c9f4b1015b97cc5bf3/keras/backend/cntk_backend.py#L1460
// cntk's batch axis is not in shape,
// so just flatten all the dim in x.shape
int dim = Matrix.Product(x.shape.Select(s => s.Value).ToArray());
x = Out(C.Reshape(In(x), NDShape.CreateNDShape(new[] { -1 })));
x._keras_shape = new int?[] { null, dim };
return(x);
}
private static Function LinearLayer(Variable input, int ouputDims)
{
int nbDimensionsInput = input.Shape[0];
Parameter bias = new Parameter(NDShape.CreateNDShape(new[] { ouputDims }), DataType.Double, 0); // une abscisse pour chaque dimension
Parameter weights = new Parameter(NDShape.CreateNDShape(new[] { nbDimensionsInput, ouputDims }), DataType.Double, 0); // les coefficients à trouver
// 2 variable d'input, 2 estimations en sortie (proba vrai et proba faux)
Function linearLayer = CNTKLib.Plus(CNTKLib.Times(weights, input), bias);
return(linearLayer);
}
public Tensor bias_add(Tensor output, Tensor bias)
{
CNTKTensor _x = In(output);
CNTKTensor _b = In(bias);
var _shape = In(_x.CNTK_Shape).Select(x => x < 0 ? 1 : x).ToArray();
var shape = NDShape.CreateNDShape(_shape);
var b = C.Reshape(_b, shape);
return(Out(new Variable(_x.function) + b));
}
public void TestShapeSpecialDimensions()
{
NDShape shapeWithInferredDimension = NDShape.CreateNDShape(new int[] { 3, 7, NDShape.InferredDimension });
Assert.IsTrue(shapeWithInferredDimension.HasInferredDimension);
Assert.AreEqual(shapeWithInferredDimension[2], NDShape.InferredDimension);
NDShape shapeWithFreeDimension = NDShape.CreateNDShape(new int[] { 3, 7, NDShape.FreeDimension });
Assert.IsTrue(shapeWithFreeDimension.HasFreeDimension);
Assert.AreEqual(shapeWithFreeDimension[2], NDShape.FreeDimension);
}
public static Function BatchNorm(this Function input,
BatchNorm batchNorm,
DeviceDescriptor device,
DataType dataType,
double initialScaleValue = 1,
double initialBiasValue = 0,
int normalizationTimeConstant = 5000)
{
var inferredDimension1D = NDShape.CreateNDShape(new int[] { NDShape.InferredDimension });
var scaleInitializer = CNTKLib.ConstantInitializer(initialScaleValue);
var scaleParams = new Parameter(inferredDimension1D,
dataType, scaleInitializer, device);
var biasInitializer = CNTKLib.ConstantInitializer(initialBiasValue);
var biasParams = new Parameter(inferredDimension1D,
dataType, biasInitializer, device);
const double zeroInit = 1.0;
// Batch normalization initial state are constants.
var runningMean = new Constant(inferredDimension1D, dataType, zeroInit, device);
var runningInvStd = new Constant(inferredDimension1D, dataType, zeroInit, device);
var runningCount = new Constant(NDShape.CreateNDShape(new[] { 1 }), dataType, zeroInit, device);
bool spatial = batchNorm == LayerFunctions.BatchNorm.Spatial;
// Allows to smooth batch estimates with the running statistics.
// However, this has not been found useful so far in our experiments (from CNTK team).
const double blendTimeConstant = 0.0;
// Epsilon is added to the variance to avoid division by 0.
const double epsilon = 0.00001;
bool useCudnn = device.Type == DeviceKind.GPU;
const bool disableRegularization = false;
// TODO: Consider if we want to surface the additional options for BatchNorm:
// - blendTimeConstant
// - epsilon
// - useCudnn
// - disableRegularization
// - name
return(CNTKLib.BatchNormalization(input,
scaleParams, biasParams,
runningMean, runningInvStd, runningCount,
spatial,
normalizationTimeConstant, blendTimeConstant,
epsilon,
useCudnn,
disableRegularization));
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
int nbObservationsInItem = 5;
List <Example_106_Data> allData = GenerateExampleData(10000, nbObservationsInItem, 5);
List <Example_106_Data> trainingData = allData.Take(8000).ToList();
List <Example_106_Data> evalData = allData.Skip(8000).ToList();
int[] inputDim = new int[] { 1 };
Variable input = Variable.InputVariable(NDShape.CreateNDShape(inputDim), DataType.Double, "input", dynamicAxes: new List <Axis>()
{
Axis.DefaultDynamicAxis(), Axis.DefaultBatchAxis()
}); // default dynamic axis ???
int outputDim = 1;
Function model = DefineModel_LSTM(input, nbObservationsInItem, outputDim);
Function output = model.Output;
//Function output = Variable.InputVariable(NDShape.CreateNDShape(new[] { 1 }), DataType.Float, "output", model.Output.DynamicAxes);
//IEnumerable<int> inputSequence = ...; // pas clair
//Value sequence = Value.CreateSequence(NDShape.CreateNDShape(new int[] { 1 }), inputSequence, device);
uint minibatchSize = 100;
Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new int[] { outputDim }), DataType.Double, "expectedOutput", dynamicAxes: new List <Axis>()
{
Axis.DefaultBatchAxis()
}); // default dynamic axis ???
Trainer trainer = MakeTrainer(expectedOutput, output, model, minibatchSize);
{ // train
int nbSamplesToUseForTraining = trainingData.Count;
int numSweepsToTrainWith = 20;
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var trainingInput = trainingData.Select(x => x.Observations.Select(y => y)).ToList();
var trainingOutput = trainingData.Select(x => new[] { x.ExpectedPrediction }).ToList();
var trainingMinibatchSource = new GenericMinibatchSequenceSource(input, trainingInput, expectedOutput, trainingOutput, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
RunTraining(trainer, trainingMinibatchSource, numMinibatchesToTrain, device);
}
// evaluate
Evaluate(model, evalData, input, device);
}
public static Value ToValue(this IEnumerable <Vector3> vectors, DeviceDescriptor device)
{
var count = vectors.Count();
Assert.AreNotEqual(count, 0, "Empty list");
float[] floatArray = new float[count * 3];
NDShape shape = NDShape.CreateNDShape(new int[] { 3 });
Parallel.For(0, count, (int batchNum) =>
{
var vector = vectors.ElementAt(batchNum);
floatArray[batchNum * 3] = vector.x;
floatArray[batchNum * 3 + 1] = vector.y;
floatArray[batchNum * 3 + 2] = vector.z;
});
return(Value.CreateBatch(shape, floatArray, device, false));
}
public static Value ToValue(this IEnumerable <Quaternion> quats, DeviceDescriptor device)
{
Assert.AreNotEqual(quats.Count(), 0);
float[] floatArray = new float[quats.Count()];
NDShape shape = NDShape.CreateNDShape(new int[] { 4 });
int count = quats.Count();
Parallel.For(0, count, (int batchNum) =>
{
var quat = quats.ElementAt(batchNum);
floatArray[batchNum * 4] = quat.w;
floatArray[batchNum * 4 + 1] = quat.x;
floatArray[batchNum * 4 + 2] = quat.y;
floatArray[batchNum * 4 + 3] = quat.z;
});
return(Value.CreateBatch(shape, floatArray, device, false));
}
public Tensor placeholder(int?[] shape = null, int?ndim = null, DataType?dtype = null, bool sparse = false, string name = null)
{
log(new { shape, ndim, dtype, sparse, name });
// https://github.com/fchollet/keras/blob/f65a56fb65062c8d14d215c9f4b1015b97cc5bf3/keras/backend/cntk_backend.py
if (shape == null)
{
if (ndim != null)
{
shape = new int?[ndim.Value];
}
}
var cntk_shape = shape.Select(s => s == null ? NDShape.FreeDimension : s.Value);
//if (dynamic_axis_num > len(cntk_shape)
//{
// raise ValueError('CNTK backend: creating placeholder with '
// '%d dimension is not supported, at least '
// '%d dimensions are needed.'
// % (len(cntk_shape, dynamic_axis_num)))
//}
if (dtype == null)
{
dtype = floatx();
}
if (name is null)
{
name = String.Empty;
}
// cntk_shape = cntk_shape[dynamic_axis_num:]
NDShape _shape = NDShape.CreateNDShape(cntk_shape);
CNTKDataType _dtype = In(dtype.Value);
Variable v = Variable.InputVariable(shape: _shape, dataType: _dtype, isSparse: sparse, name: name);
var x = Out(v);
x._keras_shape = shape;
x._uses_learning_phase = false;
return(x);
}
public Tensor bias_add(Tensor output, Tensor bias, DataFormatType?data_format = null, string name = null)
{
if (data_format != null)
{
throw new NotImplementedException();
}
using (this.name_scope("bias_add"))
{
CNTKTensor _x = In(output);
CNTKTensor _b = In(bias);
var _shape = In(_x.CNTK_Shape).Select(x => x < 0 ? 1 : x).ToArray();
var shape = NDShape.CreateNDShape(_shape);
var b = C.Reshape(_b, shape);
return(Out(new Variable(_x.function) + b));
}
}
/// <summary>
/// Create embedding sequence
/// </summary>
/// <param name="input"></param>
/// <param name="embeddingDim"></param>
/// <param name="device"></param>
/// <returns></returns>
public Function Embedding(Variable input, int embeddingDim)
{
//checking the dimension of the input variable
//it has to be always a vector which represents the tensor with rank=1
System.Diagnostics.Debug.Assert(input.Shape.Rank == 1);
//extract input dimensions from the input variable
int inputDim = input.Shape[0];
//initialization of the default parameters
var glorotInit = CNTKLib.GlorotUniformInitializer();
//create ND shape and parameter
var dims = new int[] { embeddingDim, inputDim };
var embededParam = new Parameter(NDShape.CreateNDShape(dims), DataType.Float, glorotInit, m_device);
Function embededL = CNTKLib.Times(embededParam, input, "embedded_" + input.Name);
return(embededL);
}
private static double[][] sum(params int[] axes)
{
var arr = new[]
{
/* total:
* /* */1.0, 2.0, 3.0, /* 6.0 */
/* */ 4.0, 5.0, 6.0, /* 15.0 */
/* total: 5.0, 7.0, 9.0 21.0 */
};
var shape = NDShape.CreateNDShape(new[] { 2, 3 });
Value vx = Value.CreateBatch(shape, arr, DeviceDescriptor.CPUDevice, readOnly: true);
Variable x = Variable.InputVariable(shape, CNTK.DataType.Double, name: "input");
CNTK.Function f;
if (axes == null)
{
f = CNTKLib.Alias(x);
}
else
{
var axisVector = new AxisVector(axes.Select(ax => new Axis(ax)).ToArray());
f = CNTKLib.ReduceSum(x, axis: axisVector);
}
var inputs = new Dictionary <Variable, Value>()
{
{ x, vx }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.CPUDevice);
var r = outputs[f].GetDenseData <double>((Variable)f);
return(r.Select(ri => ri.ToArray()).ToArray());
}
internal Function Convolution(Variable input, int[] filterSize, int nbFilter, int[] strides, Func <Variable, Function> activation)
{
int numInputChannels = input.Shape.Dimensions.Last();
List <int> kernelDims = new List <int>();
kernelDims.AddRange(filterSize);
kernelDims.Add(numInputChannels);
kernelDims.Add(nbFilter);
List <int> strideDims = new List <int>();
strideDims.AddRange(strides);
strideDims.Add(numInputChannels);
var convParams = Parameter(kernelDims);
var conv = CNTKLib.Convolution(convParams, input, NDShape.CreateNDShape(strideDims));
if (activation == null)
{
return(conv);
}
return(activation(conv));
}
Function Generator(Function input, Func <CNTKDictionary> weightInit, CNTKDictionary biasInit,
DeviceDescriptor device, DataType dataType)
{
var generatorNetwork = input
.Dense(1024, weightInit(), biasInit, device, dataType)
.BatchNorm(BatchNorm.Regular, device, dataType)
.ReLU()
.Dense(7 * 7 * 128, weightInit(), biasInit, device, dataType)
.BatchNorm(BatchNorm.Regular, device, dataType)
.ReLU()
.Reshape(NDShape.CreateNDShape(new int[] { 7, 7, 128 }))
.ConvTranspose2D((5, 5), 128, (2, 2), Padding.Zeros, (14, 14), weightInit(), biasInit, device, dataType)
.BatchNorm(BatchNorm.Spatial, device, dataType)
.ReLU()
.ConvTranspose2D((5, 5), 1, (2, 2), Padding.Zeros, (28, 28), weightInit(), biasInit, device, dataType)
.Tanh();
Trace.Write(Model.Summary(generatorNetwork));
return(generatorNetwork.Reshape(NDShape.CreateNDShape(new int[] { 784 })));
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
var util = new Example_103_Util();
Example_201_Data datasource = new Example_201_Data();
IEnumerable <Example_201_Item> trainingImages = datasource.LoadTrainingImages().ToList();
IEnumerable <Example_201_Item> testImages = datasource.LoadTestImages().ToList();
IDictionary <double, string> labelIndex = datasource.LoadLabelIndex().ToDictionary(x => (double)x.Key, x => x.Value);
int image_height = 32, image_width = 32, num_channels = 3, num_classes = 10;
Variable input = Variable.InputVariable(NDShape.CreateNDShape(new[] { image_height, image_width, num_channels }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(new int[] { num_classes }, DataType.Double, "expectedOutput");
Function normalizedInput = CNTKLib.ElementTimes(Constant.Scalar(1.0 / 255.0, device), input);
Function model = DefineModel_C(normalizedInput, num_classes, util);
Variable output = model.Output;
uint minibatchSize = 64;
Trainer trainer = MakeTrainer(expectedOutput, output, model, minibatchSize);
{ // train
int nbSamplesToUseForTraining = trainingImages.Count();
int numSweepsToTrainWith = 5;
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var trainingInput = trainingImages.Select(x => x.Image.Select(y => (double)y).ToArray()).ToList();
var trainingOutput = trainingImages.Select(x => ToOneHotVector(x.Label, labelIndex.Count)).ToList();
var trainingMinibatchSource = new GenericMinibatchSource(input, trainingInput, expectedOutput, trainingOutput, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
RunTraining(trainer, trainingMinibatchSource, numMinibatchesToTrain, device);
}
// evaluate
Evaluate(model, testImages, input, device, labelIndex);
}
public void Run()
{
// Prepare data
var baseDataDirectoryPath = @"E:\DataSets\Mnist";
var trainFilePath = Path.Combine(baseDataDirectoryPath, "Train-28x28_cntk_text.txt");
// Define data type and device for the model.
var dataType = DataType.Float;
var device = DeviceDescriptor.UseDefaultDevice();
// Setup initializers
var random = new Random(232);
Func <CNTKDictionary> weightInit = () => Initializers.Xavier(random.Next(), scale: 0.02);
var biasInit = Initializers.Zero();
// Ensure reproducible results with CNTK.
CNTKLib.SetFixedRandomSeed((uint)random.Next());
CNTKLib.ForceDeterministicAlgorithms();
// Setup generator
var ganeratorInputShape = NDShape.CreateNDShape(new int[] { 100 });
var generatorInput = Variable.InputVariable(ganeratorInputShape, dataType);
var generatorNetwork = Generator(generatorInput, weightInit, biasInit, device, dataType);
// Setup discriminator
var discriminatorInputShape = NDShape.CreateNDShape(new int[] { 784 }); // 28 * 28 * 1.
var discriminatorInput = Variable.InputVariable(discriminatorInputShape, dataType);
// scale image input between -1.0 and 1.0.
var discriminatorInputScaled = CNTKLib.Minus(
CNTKLib.ElementTimes(Constant.Scalar(2 * 0.00390625f, device), discriminatorInput),
Constant.Scalar(1.0f, device));
var discriminatorNetwork = Discriminator(discriminatorInputScaled, weightInit, biasInit, device, dataType);
// The discriminator must be used on both the real MNIST images and fake images generated by the generator function.
// One way to represent this in the computational graph is to create a clone of the output of the discriminator function,
// but with substituted inputs. Setting method = share in the clone function ensures that both paths through the discriminator model
// use the same set of parameters.
var discriminatorNetworkFake = discriminatorNetwork
.Clone(ParameterCloningMethod.Share, replacements:
new Dictionary <Variable, Variable>
{
{ discriminatorInputScaled.Output, generatorNetwork.Output },
});
// Create minibatch source for providing the real images.
var imageNameToVariable = new Dictionary <string, Variable> {
{ "features", discriminatorInput }
};
var imageMinibatchSource = CreateMinibatchSource(trainFilePath, imageNameToVariable, randomize: true);
// Create minibatch source for providing the noise.
var noiseNameToVariable = new Dictionary <string, Variable> {
{ "noise", generatorInput }
};
var noiseMinibatchSource = new UniformNoiseMinibatchSource(noiseNameToVariable, min: -1.0f, max: 1.0f, seed: random.Next());
// Combine both sources in the composite minibatch source.
var compositeMinibatchSource = new CompositeMinibatchSource(imageMinibatchSource, noiseMinibatchSource);
// Setup generator loss: 1.0 - C.log(D_fake)
var generatorLossFunc = CNTKLib.Minus(Constant.Scalar(1.0f, device),
CNTKLib.Log(discriminatorNetworkFake));
// Setup discriminator loss: -(C.log(D_real) + C.log(1.0 - D_fake))
var discriminatorLossFunc = CNTKLib.Negate(CNTKLib.Plus(CNTKLib.Log(discriminatorNetwork),
CNTKLib.Log(CNTKLib.Minus(Constant.Scalar(1.0f, device), discriminatorNetworkFake))));
// Create fitters for the training loop.
// Generator uses Adam and discriminator SGD.
// Advice from: https://github.com/soumith/ganhacks
var generatorLearner = Learners.Adam(generatorNetwork.Parameters(),
learningRate: 0.0002, momentum: 0.5, gradientClippingThresholdPerSample: 1.0);
var generatorFitter = CreateFitter(generatorLearner, generatorNetwork, generatorLossFunc, device);
var discriminatorLearner = Learners.SGD(discriminatorNetwork.Parameters(),
learningRate: 0.0002, gradientClippingThresholdPerSample: 1.0);
var discriminatorFitter = CreateFitter(discriminatorLearner, discriminatorNetwork, discriminatorLossFunc, device);
int epochs = 30;
int batchSize = 128;
// Controls how many steps the discriminator takes,
// each time the generator takes 1 step.
// Default from the original paper is 1.
int discriminatorSteps = 1;
var isSweepEnd = false;
for (int epoch = 0; epoch < epochs;)
{
for (int step = 0; step < discriminatorSteps; step++)
{
// Discriminator needs both real images and noise,
// so uses the composite minibatch source.
var minibatchItems = compositeMinibatchSource.GetNextMinibatch(batchSize, device);
isSweepEnd = minibatchItems.isSweepEnd;
discriminatorFitter.FitNextStep(minibatchItems.minibatch, batchSize);
DisposeValues(minibatchItems.minibatch);
}
// Generator only needs noise images,
// so uses the noise minibatch source separately.
var noiseMinibatchItems = noiseMinibatchSource.GetNextMinibatch(batchSize, device);
generatorFitter.FitNextStep(noiseMinibatchItems.minibatch, batchSize);
DisposeValues(noiseMinibatchItems.minibatch);
if (isSweepEnd)
{
var generatorCurrentLoss = generatorFitter.CurrentLoss;
generatorFitter.ResetLossAndMetricAccumulators();
var discriminatorCurrentLoss = discriminatorFitter.CurrentLoss;
discriminatorFitter.ResetLossAndMetricAccumulators();
var traceOutput = $"Epoch: {epoch + 1:000} Generator Loss = {generatorCurrentLoss:F8}, Discriminator Loss = {discriminatorCurrentLoss:F8}";
Trace.WriteLine(traceOutput);
++epoch;
}
}
// Sample 6x6 images from generator.
var samples = 6 * 6;
var batch = noiseMinibatchSource.GetNextMinibatch(samples, device);
var noise = batch.minibatch;
var predictor = new Predictor(generatorNetwork, device);
var images = predictor.PredictNextStep(noise);
var imagesData = images.SelectMany(t => t).ToArray();
// Show examples
var app = new Application();
var window = new PlotWindowBitMap("Generated Images", imagesData, 28, 28, 1, true);
window.Show();
app.Run(window);
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
var util = new Example_103_Util();
// data
string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz";
//string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz";
List <double[]> trainImages = util.LoadImages(trainImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> trainLabels = util.LoadLabels(trainLabelsPath);
//List<double[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).Select(x => x.Cast<double>().ToArray()).ToList();
string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz";
//string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz";
List <double[]> evalImages = util.LoadImages(evelImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> evalLabels = util.LoadLabels(evalLabelsPath);
//List<int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList();
// model
int sampleSize = trainImages.Count;
int nbDimensionsInput = 28 * 28;
Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "output");
Function encodeDecode = DefineModel_104B(util, inputVariables, device);
//var scaledModelOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), encodeDecode);
//var scaledExpectedOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), expectedOutput);
//{
// Function test = CNTKLib.ElementTimes(
// Constant.Scalar(-1.0d, device),
// inputVariables);
//}
//Function lossFunction = -scaledExpectedOutput * CNTKLib.Log(scaledModelOutput) - (Constant.Scalar(-1.0d, device) - scaledExpectedOutput) * CNTKLib.Log(1 - scaledModelOutput);
var scaledExpectedOutput = CNTKLib.ElementTimes(expectedOutput, Constant.Scalar(1 / 255.0, device));
//Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(encodeDecode, expectedOutput);
// Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(scaledModelOutput, scaledExpectedOutput);
Function lossFunction = CNTKLib.Square(CNTKLib.Minus(scaledExpectedOutput, encodeDecode));
Function evalErrorFunction = CNTKLib.ClassificationError(encodeDecode, scaledExpectedOutput);
// training
Trainer trainer;
{
// define training
//int epochSize = 30000;
uint minibatchSize = 64;
//double learningRate = 0.8;
int numSweepsToTrainWith = 2; // traduction de sweep ?
int nbSamplesToUseForTraining = 60000; // trainImages.Count;
double lr_per_sample = 0.2;
//double lr_per_sample = 0.2; // 0.00003;
//double lr_per_sample = 0.00003; // 0.00003;
uint epoch_size = 30000; // # 30000 samples is half the dataset size
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(lr_per_sample, epoch_size);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize);
var parameters = new ParameterVector();
foreach (var p in encodeDecode.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
//IList<Learner> parameterLearners = new List<Learner>() { Learner.SGDLearner(encodeDecode.Parameters(), learningRatePerSample) };
trainer = Trainer.CreateTrainer(encodeDecode, lossFunction, evalErrorFunction, parameterLearners);
// run training
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var minibatchSource = new GenericMinibatchSource(inputVariables, trainImages, expectedOutput, trainImages, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
double aggregate_metric = 0;
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
trainer.TrainMinibatch(data, device);
double samples = trainer.PreviousMinibatchSampleCount();
double avg = trainer.PreviousMinibatchEvaluationAverage();
aggregate_metric += avg * samples;
double nbSampleSeen = trainer.TotalNumberOfSamplesSeen();
double train_error = aggregate_metric / nbSampleSeen;
Debug.WriteLine($"{minibatchCount} Average training error: {train_error:p2}");
}
}
// evaluate
{
uint testMinibatchSize = 32;
int nbSamplesToTest = 32;// evalImages.Count;
int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize;
double metric_numer = 0;
double metric_denom = 0;
var minibatchSource = new GenericMinibatchSource(inputVariables, evalImages, expectedOutput, evalImages, nbSamplesToTest, 1, testMinibatchSize, device);
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
////UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData();
////foreach (var row in data)
//// evalInput[row.Key] = row.Value;
////double error = trainer.TestMinibatch(evalInput, device);
////metric_numer += Math.Abs(error * testMinibatchSize);
////metric_denom += testMinibatchSize;
////MinibatchData outputValue = evalInput[expectedOutput];
//IList<IList<double>> inputPixels = outputValue.data.GetDenseData<double>(inputVariables);
//IList<IList<double>> actualLabelSoftMax = outputValue.data.GetDenseData<double>(encodeDecode.Output);
//for (int i = 0; i < actualLabelSoftMax.Count; i++)
// PrintBitmap(inputPixels[i], actualLabelSoftMax[i], i);
// var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), inputVariables);
Dictionary <Variable, Value> input = new Dictionary <Variable, Value>()
{
{ inputVariables, data[inputVariables].data }
};
Dictionary <Variable, Value> output = new Dictionary <Variable, Value>()
{
// { normalizedInput.Output, null }
{ encodeDecode.Output, null }
};
encodeDecode.Evaluate(input, output, device);
IList <IList <double> > inputPixels = input[inputVariables].GetDenseData <double>(inputVariables);
IList <IList <double> > outputPixels = output[encodeDecode.Output].GetDenseData <double>(encodeDecode.Output);
for (int i = 0; i < inputPixels.Count; i++)
{
PrintBitmap(inputPixels[i], outputPixels[i], i);
}
}
double test_error = (metric_numer * 100.0) / (metric_denom);
Debug.WriteLine($"Average test error: {test_error:p2}");
}
}
