public Trainer createTrainer(Function network, Variable target, double lrvalue)
{
//learning rate
// var lrate = 0.05;
var momentum = 0.9;
var lr = new TrainingParameterScheduleDouble(lrvalue);
var mm = CNTKLib.MomentumAsTimeConstantSchedule(momentum);
var l = new AdditionalLearningOptions()
{
l1RegularizationWeight = 0.001, l2RegularizationWeight = 0.1
};
//network parameters
var zParams = new ParameterVector(network.Parameters().ToList());
//create loss and eval
Function loss = CNTKLib.SquaredError(network, target);
Function eval = StatMetrics.RMSError(network, target);
//learners
//
var llr = new List <Learner>();
var msgd = Learner.SGDLearner(network.Parameters(), lr, l);
llr.Add(msgd);
//trainer
var trainer = Trainer.CreateTrainer(network, loss, eval, llr);
//
return(trainer);
}
public ParameterVectorEnumerator(ParameterVector collection)
{
collectionRef = collection;
currentIndex = -1;
currentObject = null;
currentSize = collectionRef.Count;
}
private Trainer MakeTrainer(Variable expectedOutput, Variable output, Function model, uint minibatchSize)
{
double learningRate = 0.01;
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9983550962823424, minibatchSize);
//Function lossFunction = CrossEntropyWithSoftmax(output, expectedOutput);
Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(output, expectedOutput);
Function evalErrorFunction = CNTKLib.ClassificationError(output, expectedOutput);
var parameters = new ParameterVector();
foreach (var p in model.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.MomentumSGDLearner(parameters, learningRatePerSample, momentumSchedule, true, new AdditionalLearningOptions()
{
l2RegularizationWeight = 0.001
})
};
Trainer trainer = Trainer.CreateTrainer(model, lossFunction, evalErrorFunction, parameterLearners);
return(trainer);
}
public Trainer createTrainer(Function network, Variable target)
{
//learning rate
var lrate = 0.082;
var lr = new TrainingParameterScheduleDouble(lrate);
//network parameters
var zParams = new ParameterVector(network.Parameters().ToList());
//create loss and eval
Function loss = CNTKLib.SquaredError(network, target);
Function eval = CNTKLib.SquaredError(network, target);
//learners
//
var llr = new List <Learner>();
var msgd = Learner.SGDLearner(network.Parameters(), lr);
llr.Add(msgd);
//trainer
var trainer = Trainer.CreateTrainer(network, loss, eval, llr);
//
return(trainer);
}
private Trainer MakeTrainer(Variable expectedOutput, Variable output, Function model, uint minibatchSize)
{
double learningRate = 0.02;
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9, minibatchSize);
Function lossFunction = CNTKLib.SquaredError(expectedOutput, output);
Function evalErrorFunction = CNTKLib.SquaredError(expectedOutput, output);
var parameters = new ParameterVector();
foreach (var p in model.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
Trainer trainer = Trainer.CreateTrainer(model, lossFunction, evalErrorFunction, parameterLearners);
return(trainer);
}
public ParameterVector(ParameterVector other) : this(CNTKLibPINVOKE.new_ParameterVector__SWIG_1(ParameterVector.getCPtr(other)), true)
{
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
}
/// <summary>
/// create model by w,h,c,outputClassNum
/// </summary>
/// <param name="w"></param>
/// <param name="h"></param>
/// <param name="c"></param>
/// <param name="outputClassNum"></param>
/// <param name="deviceName"></param>
public FCN7(int w, int h, int c, int outputClassNum, string deviceName)
{
device = NP.CNTKHelper.GetDeviceByName(deviceName);
//input output variable
int[] inputDim = new int[] { w, h, c };
int[] outputDim = new int[] { outputClassNum };
inputVariable = Variable.InputVariable(NDShape.CreateNDShape(inputDim), DataType.Float, "inputVariable");
outputVariable = Variable.InputVariable(NDShape.CreateNDShape(outputDim), DataType.Float, "labelVariable");
//build model
classifierOutput = CreateFullyChannelNetwork(inputVariable, c, outputClassNum);
Function loss = CNTKLib.SquaredError(classifierOutput, outputVariable);
Function pred = CNTKLib.ClassificationError(classifierOutput, outputVariable);
//adam leaner
ParameterVector parameterVector = new ParameterVector(classifierOutput.Parameters().ToList());
TrainingParameterScheduleDouble learningRateSchedule = new TrainingParameterScheduleDouble(0.00178125, BatchSize);
TrainingParameterScheduleDouble momentumRateSchedule = new TrainingParameterScheduleDouble(0.9, BatchSize);
Learner leaner = CNTKLib.AdamLearner(parameterVector, learningRateSchedule, momentumRateSchedule, true);
//构造leaner方法
trainer = Trainer.CreateTrainer(classifierOutput, loss, pred, new List <Learner>()
{
leaner
});
//TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.00178125, BatchSize); //0.00178125
//TrainingParameterScheduleDouble momentumTimeConstant = CNTKLib.MomentumAsTimeConstantSchedule(256);
//IList<Learner> parameterLearners = new List<Learner>() { Learner.MomentumSGDLearner(classifierOutput.Parameters(), learningRatePerSample, momentumTimeConstant, true) };
//trainer = Trainer.CreateTrainer(classifierOutput, loss, pred, parameterLearners);
}
public void SetRange(int index, ParameterVector values)
{
CNTKLibPINVOKE.ParameterVector_SetRange(swigCPtr, index, ParameterVector.getCPtr(values));
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
}
internal static ParameterVector CreateParameterVector(IList <Parameter> input)
{
ParameterVector inputVector = new ParameterVector();
foreach (var element in input)
{
inputVector.Add(element);
}
return(inputVector);
}
public virtual ParameterVector Parameters()
{
ParameterVector ret = new ParameterVector(CNTKLibPINVOKE.Learner_Parameters(swigCPtr), false);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
private ParameterVector _Parameters()
{
ParameterVector ret = new ParameterVector(CNTKLibPINVOKE.Function__Parameters(swigCPtr), true);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
public ParameterVector GetRange(int index, int count)
{
global::System.IntPtr cPtr = CNTKLibPINVOKE.ParameterVector_GetRange(swigCPtr, index, count);
ParameterVector ret = (cPtr == global::System.IntPtr.Zero) ? null : new ParameterVector(cPtr, true);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
public static ParameterVector Repeat(Parameter value, int count)
{
global::System.IntPtr cPtr = CNTKLibPINVOKE.ParameterVector_Repeat(Parameter.getCPtr(value), count);
ParameterVector ret = (cPtr == global::System.IntPtr.Zero) ? null : new ParameterVector(cPtr, true);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
internal static Learner GetInitializer(IList <Parameter> parameters, NeuralNetworkSettingsEntity s)
{
var vector = new ParameterVector((ICollection)parameters);
switch (s.Learner)
{
case NeuralNetworkLearner.Adam: return(CNTKLib.AdamLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false,
s.LearningVarianceMomentum?.ToTrainParam()));
case NeuralNetworkLearner.AdaDelta:
return(CNTKLib.AdaDeltaLearner(vector,
s.LearningRate.ToTrainParam()));
case NeuralNetworkLearner.AdaGrad:
return(CNTKLib.AdaGradLearner(vector,
s.LearningRate.ToTrainParam()));
case NeuralNetworkLearner.FSAdaGrad:
return(CNTKLib.FSAdaGradLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false,
s.LearningVarianceMomentum?.ToTrainParam()));
case NeuralNetworkLearner.RMSProp:
return(CNTKLib.FSAdaGradLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false,
s.LearningVarianceMomentum?.ToTrainParam()));
case NeuralNetworkLearner.MomentumSGD:
return(CNTKLib.MomentumSGDLearner(vector,
s.LearningRate.ToTrainParam(),
s.LearningMomentum?.ToTrainParam(),
s.LearningUnitGain ?? false));
case NeuralNetworkLearner.SGD:
return(CNTKLib.SGDLearner(vector,
s.LearningRate.ToTrainParam()));
default:
throw new InvalidOperationException("Unexpected Learner");
}
}
/// <summary>
/// Creates trained based on training and learning parameters
/// </summary>
/// <param name="network">Network model being trained</param>
/// <param name="lrParams">Learning parameters</param>
/// <param name="trParams">Training parameters</param>
/// <returns></returns>
private Trainer createTrainer(Function network, LearningParameters lrParams, TrainingParameters trParams)
{
//network parameters
var zParams = new ParameterVector(network.Parameters().ToList());
//create loss and eval
(Function loss, Function eval) = createLossEvalFunction(lrParams, network);
//learners
var learners = createLearners(network, lrParams);
//trainer
var trainer = Trainer.CreateTrainer(network, loss, eval, learners);
//
return(trainer);
}
/// <summary>
/// create from saved model
/// </summary>
/// <param name="model"></param>
/// <param name="deviceName"></param>
public FCN7(byte[] modelBuffer, string deviceName)
{
device = NP.CNTKHelper.GetDeviceByName(deviceName);
Function model = Function.Load(modelBuffer, device);
inputVariable = model.Inputs.First(v => v.Name == "inputVariable");
outputVariable = Variable.InputVariable(model.Output.Shape, DataType.Float, "labelVariable");
classifierOutput = model;
Function loss = CNTKLib.SquaredError(classifierOutput, outputVariable);
Function pred = CNTKLib.ClassificationError(classifierOutput, outputVariable);
//adam leaner
ParameterVector parameterVector = new ParameterVector(classifierOutput.Parameters().ToList());
TrainingParameterScheduleDouble learningRateSchedule = new TrainingParameterScheduleDouble(0.00178125, BatchSize);
TrainingParameterScheduleDouble momentumRateSchedule = new TrainingParameterScheduleDouble(0.9, BatchSize);
Learner leaner = CNTKLib.AdamLearner(parameterVector, learningRateSchedule, momentumRateSchedule, true);
//构造leaner方法
trainer = Trainer.CreateTrainer(classifierOutput, loss, pred, new List <Learner>()
{
leaner
});
}
static void Main(string[] args)
{
const bool useSparse = false;
const int vectorSize = 300;
const int vocabularySize = 6000;
const float xMax = 100f;
const float alphaOrder = 0.75f;
const int imagimableBatchSize = 1;
var device = DeviceDescriptor.GPUDevice(0);
var scalarDimension = new[] { imagimableBatchSize, 1 };
var matrixSize = new[] { vectorSize, vocabularySize };
var vectorDimension = new[] { vocabularySize, 1 };
var iterationScalarShape = NDShape.CreateNDShape(scalarDimension);
var iterationMatrixShape = NDShape.CreateNDShape(matrixSize);
var iterationVectorShape = NDShape.CreateNDShape(vectorDimension);
var coOccurrences = Variable.InputVariable(iterationScalarShape, DataType.Float, "coOccurrences - " + vocabularySize, null, false);
var columns = Variable.InputVariable(iterationScalarShape, DataType.Float, "columns - " + vocabularySize, null, false);
var rows = Variable.InputVariable(iterationScalarShape, DataType.Float, "rows - " + vocabularySize, null, false);
var mainVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device);
PrintDim(mainVectors, nameof(mainVectors));
var contextVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device);
PrintDim(contextVectors, nameof(contextVectors));
var mainBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device);
PrintDim(mainBiases, nameof(mainBiases));
var contextBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device);
PrintDim(contextBiases, nameof(contextBiases));
var one = new Constant(iterationScalarShape, DataType.Float, 1d, device);
PrintDim(one, nameof(one));
var xmax = new Constant(iterationScalarShape, DataType.Float, xMax, device);
PrintDim(xmax, nameof(xmax));
var alpha = new Constant(iterationScalarShape, DataType.Float, alphaOrder, device);
PrintDim(alpha, nameof(alpha));
var divide = CNTKLib.ElementDivide(coOccurrences, xmax);
PrintDim(divide, nameof(divide));
var pow = CNTKLib.Pow(divide, alpha);
PrintDim(pow, nameof(pow));
var weight = CNTKLib.ElementMin(one, pow, "min");
PrintDim(weight, nameof(weight));
var oneHotRow = CNTKLib.OneHotOp(rows, vocabularySize, useSparse, new Axis(0));
PrintDim(oneHotRow, nameof(oneHotRow));
var oneHotColumn = CNTKLib.OneHotOp(columns, vocabularySize, useSparse, new Axis(0));
PrintDim(oneHotColumn, nameof(oneHotColumn));
var mainVector = CNTKLib.Alias(CNTKLib.Times(mainVectors, oneHotColumn));
PrintDim(mainVector, nameof(mainVector));
var contextVector = CNTKLib.Alias(CNTKLib.Times(contextVectors, oneHotRow));
PrintDim(contextVector, nameof(contextVector));
var mainBias = CNTKLib.Alias(CNTKLib.TransposeTimes(mainBiases, oneHotColumn));
PrintDim(mainBias, nameof(mainBias));
var contextBias = CNTKLib.Alias(CNTKLib.TransposeTimes(contextBiases, oneHotRow));
PrintDim(contextBias, nameof(contextBias));
var model = CNTKLib.ElementTimes(mainVector, contextVector);
PrintDim(model, "CNTKLib.ElementTimes(mainVector, contextVector)");
model = CNTKLib.ReduceSum(model, new Axis(0));
PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(0))");
model = CNTKLib.Plus(model, mainBias);
PrintDim(model, "CNTKLib.Plus(model, mainBias)");
model = CNTKLib.Plus(model, contextBias);
PrintDim(model, "CNTKLib.Plus(model, contextBias)");
model = CNTKLib.Minus(model, CNTKLib.Log(coOccurrences));
PrintDim(model, "CNTKLib.Minus(model, CNTKLib.Log(coOccurrences))");
model = CNTKLib.Square(model);
PrintDim(model, "CNTKLib.Square(model)");
model = CNTKLib.ElementTimes(model, weight);
PrintDim(model, "CNTKLib.ElementTimes(model, weight)");
model = CNTKLib.ReduceSum(model, new Axis(1));
PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(1))");
var thisBatchShape = NDShape.CreateNDShape(new[] { imagimableBatchSize });
var parameterVector = new ParameterVector(model.Parameters().ToList());
var learner = CNTKLib.SGDLearner(
parameterVector,
new TrainingParameterScheduleDouble(0.1, (uint)(vocabularySize * vocabularySize)));
var learners = new LearnerVector()
{
learner
};
var trainer = CNTKLib.CreateTrainer(model, model, model, learners);
var count = (int)(vocabularySize * vocabularySize * 0.2d * 0.2d);
var floats = GetRandomFloats(count).ToArray();
var fColumns = GetRandomInts(count, 0, vocabularySize).ToArray();
var fRows = GetRandomInts(count, 0, vocabularySize).ToArray();
const int batchSize = 10000;
var all = floats.Zip(fColumns, (f, c) => (f: f, c: c)).Zip(fRows, (tuple, r) => (tuple.c, tuple.c, r))
.ToObservable()
.Buffer(batchSize)
.Select(x => (f: x.Select(y => y.Item1).ToArray(), c: x.Select(y => y.Item2).ToArray(), r: x.Select(y => y.Item3).ToArray()))
.ToArray()
.Wait();
Console.WriteLine($"count: {count}");
var stopwatch = new Stopwatch();
stopwatch.Start();
for (var e = 0; e < 3; e++)
{
for (var i = 0; i < all.Length; i++)
{
var valueTuple = all[i];
var cooccurenceValue = Value.CreateBatch(thisBatchShape, valueTuple.f, device);
var columnsValue = Value.CreateBatch(thisBatchShape, valueTuple.c, device);
var rowsValue = Value.CreateBatch(thisBatchShape, valueTuple.r, device);
var trainDictionary = new Dictionary <Variable, Value>
{
{ coOccurrences, cooccurenceValue },
{ columns, columnsValue },
{ rows, rowsValue }
};
trainer.TrainMinibatch(trainDictionary, false, device);
if (i % 100 == 0)
{
Console.WriteLine($"e: {e}\ti: {stopwatch.Elapsed:g}");
}
}
}
stopwatch.Stop();
Console.WriteLine($"success: {stopwatch.Elapsed:g}");
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
var util = new Example_103_Util();
// data
string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz";
//string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz";
List <double[]> trainImages = util.LoadImages(trainImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> trainLabels = util.LoadLabels(trainLabelsPath);
//List<double[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).Select(x => x.Cast<double>().ToArray()).ToList();
string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz";
//string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz";
List <double[]> evalImages = util.LoadImages(evelImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> evalLabels = util.LoadLabels(evalLabelsPath);
//List<int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList();
// model
int sampleSize = trainImages.Count;
int nbDimensionsInput = 28 * 28;
Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "output");
Function encodeDecode = DefineModel_104B(util, inputVariables, device);
//var scaledModelOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), encodeDecode);
//var scaledExpectedOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), expectedOutput);
//{
// Function test = CNTKLib.ElementTimes(
// Constant.Scalar(-1.0d, device),
// inputVariables);
//}
//Function lossFunction = -scaledExpectedOutput * CNTKLib.Log(scaledModelOutput) - (Constant.Scalar(-1.0d, device) - scaledExpectedOutput) * CNTKLib.Log(1 - scaledModelOutput);
var scaledExpectedOutput = CNTKLib.ElementTimes(expectedOutput, Constant.Scalar(1 / 255.0, device));
//Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(encodeDecode, expectedOutput);
// Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(scaledModelOutput, scaledExpectedOutput);
Function lossFunction = CNTKLib.Square(CNTKLib.Minus(scaledExpectedOutput, encodeDecode));
Function evalErrorFunction = CNTKLib.ClassificationError(encodeDecode, scaledExpectedOutput);
// training
Trainer trainer;
{
// define training
//int epochSize = 30000;
uint minibatchSize = 64;
//double learningRate = 0.8;
int numSweepsToTrainWith = 2; // traduction de sweep ?
int nbSamplesToUseForTraining = 60000; // trainImages.Count;
double lr_per_sample = 0.2;
//double lr_per_sample = 0.2; // 0.00003;
//double lr_per_sample = 0.00003; // 0.00003;
uint epoch_size = 30000; // # 30000 samples is half the dataset size
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(lr_per_sample, epoch_size);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize);
var parameters = new ParameterVector();
foreach (var p in encodeDecode.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
//IList<Learner> parameterLearners = new List<Learner>() { Learner.SGDLearner(encodeDecode.Parameters(), learningRatePerSample) };
trainer = Trainer.CreateTrainer(encodeDecode, lossFunction, evalErrorFunction, parameterLearners);
// run training
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var minibatchSource = new GenericMinibatchSource(inputVariables, trainImages, expectedOutput, trainImages, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
double aggregate_metric = 0;
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
trainer.TrainMinibatch(data, device);
double samples = trainer.PreviousMinibatchSampleCount();
double avg = trainer.PreviousMinibatchEvaluationAverage();
aggregate_metric += avg * samples;
double nbSampleSeen = trainer.TotalNumberOfSamplesSeen();
double train_error = aggregate_metric / nbSampleSeen;
Debug.WriteLine($"{minibatchCount} Average training error: {train_error:p2}");
}
}
// evaluate
{
uint testMinibatchSize = 32;
int nbSamplesToTest = 32;// evalImages.Count;
int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize;
double metric_numer = 0;
double metric_denom = 0;
var minibatchSource = new GenericMinibatchSource(inputVariables, evalImages, expectedOutput, evalImages, nbSamplesToTest, 1, testMinibatchSize, device);
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
////UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData();
////foreach (var row in data)
//// evalInput[row.Key] = row.Value;
////double error = trainer.TestMinibatch(evalInput, device);
////metric_numer += Math.Abs(error * testMinibatchSize);
////metric_denom += testMinibatchSize;
////MinibatchData outputValue = evalInput[expectedOutput];
//IList<IList<double>> inputPixels = outputValue.data.GetDenseData<double>(inputVariables);
//IList<IList<double>> actualLabelSoftMax = outputValue.data.GetDenseData<double>(encodeDecode.Output);
//for (int i = 0; i < actualLabelSoftMax.Count; i++)
// PrintBitmap(inputPixels[i], actualLabelSoftMax[i], i);
// var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), inputVariables);
Dictionary <Variable, Value> input = new Dictionary <Variable, Value>()
{
{ inputVariables, data[inputVariables].data }
};
Dictionary <Variable, Value> output = new Dictionary <Variable, Value>()
{
// { normalizedInput.Output, null }
{ encodeDecode.Output, null }
};
encodeDecode.Evaluate(input, output, device);
IList <IList <double> > inputPixels = input[inputVariables].GetDenseData <double>(inputVariables);
IList <IList <double> > outputPixels = output[encodeDecode.Output].GetDenseData <double>(encodeDecode.Output);
for (int i = 0; i < inputPixels.Count; i++)
{
PrintBitmap(inputPixels[i], outputPixels[i], i);
}
}
double test_error = (metric_numer * 100.0) / (metric_denom);
Debug.WriteLine($"Average test error: {test_error:p2}");
}
}
/// <summary>
/// Train and evaluate an image classifier with CIFAR-10 data.
/// The classification model is saved after training.
/// For repeated runs, the caller may choose whether to retrain a model or
/// just validate an existing one.
/// </summary>
/// <param name="device">CPU or GPU device to run</param>
/// <param name="forceRetrain">whether to override an existing model.
/// if true, any existing model will be overridden and the new one evaluated.
/// if false and there is an existing model, the existing model is evaluated.</param>
public static void TrainAndEvaluate(DeviceDescriptor device, bool forceRetrain)
{
string modelFile = Path.Combine(CifarDataFolder, "CNTK-CSharp.model");
// If a model already exists and not set to force retrain, validate the model and return.
if (File.Exists(modelFile) && !forceRetrain)
{
ValidateModel(device, modelFile);
return;
}
// prepare training data
var minibatchSource = CreateMinibatchSource(Path.Combine(CifarDataFolder, "train_map.txt"),
Path.Combine(CifarDataFolder, "CIFAR-10_mean.xml"), imageDim, numClasses, MaxEpochs);
var imageStreamInfo = minibatchSource.StreamInfo("features");
var labelStreamInfo = minibatchSource.StreamInfo("labels");
// build a model
var imageInput = CNTKLib.InputVariable(imageDim, imageStreamInfo.m_elementType, "Images");
var labelsVar = CNTKLib.InputVariable(new int[] { numClasses }, labelStreamInfo.m_elementType, "Labels");
var classifierOutput = ResNetClassifier(imageInput, numClasses, device, "classifierOutput");
// prepare for training
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(classifierOutput, labelsVar, "lossFunction");
var prediction = CNTKLib.ClassificationError(classifierOutput, labelsVar, 3, "predictionError");
//学习率策略
double[] lrs = { 3e-2, 3e-3, 3e-4, 3e-4, 5e-5 }; //学习率
int[] check_point = { 80, 120, 160, 180 }; //学习率在epoch到达多少时更新
uint minibatchSize = 32;
PairSizeTDouble p1 = new PairSizeTDouble(80, lrs[0]);
PairSizeTDouble p2 = new PairSizeTDouble(40, lrs[1]);
PairSizeTDouble p3 = new PairSizeTDouble(40, lrs[2]);
PairSizeTDouble p4 = new PairSizeTDouble(20, lrs[3]);
PairSizeTDouble p5 = new PairSizeTDouble(20, lrs[4]);
VectorPairSizeTDouble vp = new VectorPairSizeTDouble()
{
p1, p2, p3, p4, p5
};
int sample_num_in_a_epoch = 50000;
TrainingParameterScheduleDouble learningRateSchedule = new TrainingParameterScheduleDouble(vp, (uint)sample_num_in_a_epoch);
//动量
var momentum = new TrainingParameterScheduleDouble(0.9, 1);
//SGD Learner
//var sgdLearner = Learner.SGDLearner(classifierOutput.Parameters(), learningRateSchedule);
//Adam Learner
ParameterVector parameterVector = new ParameterVector();
foreach (var parameter in classifierOutput.Parameters())
{
parameterVector.Add(parameter);
}
var adamLearner = CNTKLib.AdamLearner(parameterVector, learningRateSchedule, momentum);
//Trainer
var trainer = Trainer.CreateTrainer(classifierOutput, trainingLoss, prediction, new List <Learner> {
adamLearner
});
int outputFrequencyInMinibatches = 20, miniBatchCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
// Feed data to the trainer for number of epochs.
Console.WriteLine("*****************Train Start*****************");
while (true)
{
var minibatchData = minibatchSource.GetNextMinibatch(minibatchSize, device);
// Stop training once max epochs is reached.
if (minibatchData.empty())
{
break;
}
trainer.TrainMinibatch(new Dictionary <Variable, MinibatchData>()
{
{ imageInput, minibatchData[imageStreamInfo] },
{ labelsVar, minibatchData[labelStreamInfo] }
}, device);
TestHelper.PrintTrainingProgress(trainer, adamLearner, miniBatchCount++, outputFrequencyInMinibatches);
}
// save the model
var imageClassifier = Function.Combine(new List <Variable>()
{
trainingLoss, prediction, classifierOutput
}, "ImageClassifier");
imageClassifier.Save(modelFile);
Console.WriteLine("*****************Train Stop*****************");
// validate the model
float acc = ValidateModel(device, modelFile);
sw.Stop();
TimeSpan ts2 = sw.Elapsed;
Console.WriteLine("*****************Validate Stop*****************");
string logstr = "Total time :" + ts2.TotalSeconds + "s. acc:" + acc;
Console.WriteLine(logstr);
int i = 1;
while (System.IO.File.Exists("../../../../log_" + i.ToString() + ".txt"))
{
i++;
}
var file = System.IO.File.Create("../../../../log_" + i.ToString() + ".txt");
byte[] data = System.Text.Encoding.Default.GetBytes(logstr);
file.Write(data, 0, data.Length);
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
// 1. Generate Data
int sampleSize = 32;
int nbDimensionsInput = 2; // 2 dimensions (age&tumorsize)
int nbLabels = 2; // l'output est un vecteur de probabilités qui doit sommer à 1. Si on ne met qu'une seule dimension de sortie, l'output sera toujours de 1.
// on met donc deux dimension, une dimension 'vrai' et une dimension 'faux'. L'output sera du genre 0.25 vrai et 0.75 faux => total des poids = 1;
// premier label = faux, second = vrai
IEnumerable <DataPoint> data = GenerateData(sampleSize);
//foreach (var pt in data)
// Debug.WriteLine($"{pt.Age};{pt.TumorSize};{(pt.HasCancer ? 1 : 0)}");
Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new[] { nbDimensionsInput }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(new int[] { nbLabels }, DataType.Double, "output");
Parameter bias = new Parameter(NDShape.CreateNDShape(new[] { nbLabels }), DataType.Double, 0); // une abscisse pour chaque dimension
Parameter weights = new Parameter(NDShape.CreateNDShape(new[] { nbDimensionsInput, nbLabels }), DataType.Double, 0); // les coefficients à trouver
// 2 variable d'input, 2 estimations en sortie (proba vrai et proba faux)
Function predictionFunction = CNTKLib.Plus(CNTKLib.Times(weights, inputVariables), bias);
Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(predictionFunction, expectedOutput);
Function evalErrorFunction = CNTKLib.ClassificationError(predictionFunction, expectedOutput);
//Function logisticClassifier = CNTKLib.Sigmoid(evaluationFunction, "LogisticClassifier");
uint minibatchSize = 25;
//double learningRate = 0.5;
//TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, minibatchSize);
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.3, (uint)(data.Count() / 1.0));
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize);
var parameters = new ParameterVector();
foreach (var p in predictionFunction.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
Trainer trainer = Trainer.CreateTrainer(predictionFunction, lossFunction, evalErrorFunction, parameterLearners);
double nbSamplesToUseForTraining = 20000;
int numMinibatchesToTrain = (int)(nbSamplesToUseForTraining / (int)minibatchSize);
// train the model
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IEnumerable <DataPoint> trainingData = GenerateData((int)minibatchSize);
List <double> minibatchInput = new List <double>();
List <double> minibatchOutput = new List <double>();
foreach (DataPoint row in trainingData)
{
minibatchInput.Add(row.Age);
minibatchInput.Add(row.TumorSize);
minibatchOutput.Add(row.HasCancer ? 0d : 1d);
minibatchOutput.Add(row.HasCancer ? 1d : 0d);
}
Value inputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbDimensionsInput }), minibatchInput, device);
Value outputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbLabels }), minibatchOutput, device);
trainer.TrainMinibatch(new Dictionary <Variable, Value>()
{
{ inputVariables, inputData }, { expectedOutput, outputData }
}, device);
PrintTrainingProgress(trainer, minibatchCount);
}
// test
{
int testSize = 100;
IEnumerable <DataPoint> trainingData = GenerateData(testSize);
List <double> minibatchInput = new List <double>();
List <double> minibatchOutput = new List <double>();
foreach (DataPoint row in trainingData)
{
minibatchInput.Add(row.Age);
minibatchInput.Add(row.TumorSize);
minibatchOutput.Add(row.HasCancer ? 0d : 1d);
minibatchOutput.Add(row.HasCancer ? 1d : 0d);
}
Value inputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbDimensionsInput }), minibatchInput, device);
Value outputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbLabels }), minibatchOutput, device);
IList <IList <double> > expectedOneHot = outputData.GetDenseData <double>(predictionFunction.Output);
IList <int> expectedLabels = expectedOneHot.Select(l => l.IndexOf(1.0d)).ToList();
var outputDataMap = new Dictionary <Variable, Value>()
{
{ predictionFunction.Output, null }
};
predictionFunction.Evaluate(
new Dictionary <Variable, Value>()
{
{ inputVariables, inputData }
},
outputDataMap,
device);
Value outputValue = outputDataMap[predictionFunction.Output];
IList <IList <double> > actualLabelSoftMax = outputValue.GetDenseData <double>(predictionFunction.Output);
var actualLabels = actualLabelSoftMax.Select((IList <double> l) => l.IndexOf(l.Max())).ToList();
int misMatches = actualLabels.Zip(expectedLabels, (a, b) => a.Equals(b) ? 0 : 1).Sum();
Debug.WriteLine($"Validating Model: Total Samples = {testSize}, Misclassify Count = {misMatches}");
}
}
internal static global::System.Runtime.InteropServices.HandleRef getCPtr(ParameterVector obj)
{
return((obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr);
}
