private Trainer MakeTrainer(Variable expectedOutput, Variable output, Function model, uint minibatchSize)
{
double learningRate = 0.01;
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9983550962823424, minibatchSize);
//Function lossFunction = CrossEntropyWithSoftmax(output, expectedOutput);
Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(output, expectedOutput);
Function evalErrorFunction = CNTKLib.ClassificationError(output, expectedOutput);
var parameters = new ParameterVector();
foreach (var p in model.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.MomentumSGDLearner(parameters, learningRatePerSample, momentumSchedule, true, new AdditionalLearningOptions()
{
l2RegularizationWeight = 0.001
})
};
Trainer trainer = Trainer.CreateTrainer(model, lossFunction, evalErrorFunction, parameterLearners);
return(trainer);
}
private Trainer MakeTrainer(Variable expectedOutput, Variable output, Function model, uint minibatchSize)
{
double learningRate = 0.02;
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9, minibatchSize);
Function lossFunction = CNTKLib.SquaredError(expectedOutput, output);
Function evalErrorFunction = CNTKLib.SquaredError(expectedOutput, output);
var parameters = new ParameterVector();
foreach (var p in model.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
Trainer trainer = Trainer.CreateTrainer(model, lossFunction, evalErrorFunction, parameterLearners);
return(trainer);
}
internal static ParameterVector CreateParameterVector(IList <Parameter> input)
{
ParameterVector inputVector = new ParameterVector();
foreach (var element in input)
{
inputVector.Add(element);
}
return(inputVector);
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
var util = new Example_103_Util();
// data
string trainImagesPath = "./Example_103/train-images-idx3-ubyte.gz";
//string trainLabelsPath = "./Example_103/train-labels-idx1-ubyte.gz";
List <double[]> trainImages = util.LoadImages(trainImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> trainLabels = util.LoadLabels(trainLabelsPath);
//List<double[]> trainLabels1Hot = trainLabels.Select(x => util.ConvertTo1Hot(x)).Select(x => x.Cast<double>().ToArray()).ToList();
string evelImagesPath = "./Example_103/t10k-images-idx3-ubyte.gz";
//string evalLabelsPath = "./Example_103/t10k-labels-idx1-ubyte.gz";
List <double[]> evalImages = util.LoadImages(evelImagesPath).Select(x => x.Select(y => (double)y).ToArray()).ToList();
//List<int> evalLabels = util.LoadLabels(evalLabelsPath);
//List<int[]> evalLabels1Hot = evalLabels.Select(x => util.ConvertTo1Hot(x)).ToList();
// model
int sampleSize = trainImages.Count;
int nbDimensionsInput = 28 * 28;
Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(NDShape.CreateNDShape(new [] { nbDimensionsInput }), DataType.Double, "output");
Function encodeDecode = DefineModel_104B(util, inputVariables, device);
//var scaledModelOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), encodeDecode);
//var scaledExpectedOutput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), expectedOutput);
//{
// Function test = CNTKLib.ElementTimes(
// Constant.Scalar(-1.0d, device),
// inputVariables);
//}
//Function lossFunction = -scaledExpectedOutput * CNTKLib.Log(scaledModelOutput) - (Constant.Scalar(-1.0d, device) - scaledExpectedOutput) * CNTKLib.Log(1 - scaledModelOutput);
var scaledExpectedOutput = CNTKLib.ElementTimes(expectedOutput, Constant.Scalar(1 / 255.0, device));
//Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(encodeDecode, expectedOutput);
// Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(scaledModelOutput, scaledExpectedOutput);
Function lossFunction = CNTKLib.Square(CNTKLib.Minus(scaledExpectedOutput, encodeDecode));
Function evalErrorFunction = CNTKLib.ClassificationError(encodeDecode, scaledExpectedOutput);
// training
Trainer trainer;
{
// define training
//int epochSize = 30000;
uint minibatchSize = 64;
//double learningRate = 0.8;
int numSweepsToTrainWith = 2; // traduction de sweep ?
int nbSamplesToUseForTraining = 60000; // trainImages.Count;
double lr_per_sample = 0.2;
//double lr_per_sample = 0.2; // 0.00003;
//double lr_per_sample = 0.00003; // 0.00003;
uint epoch_size = 30000; // # 30000 samples is half the dataset size
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(lr_per_sample, epoch_size);
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize);
var parameters = new ParameterVector();
foreach (var p in encodeDecode.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
//IList<Learner> parameterLearners = new List<Learner>() { Learner.SGDLearner(encodeDecode.Parameters(), learningRatePerSample) };
trainer = Trainer.CreateTrainer(encodeDecode, lossFunction, evalErrorFunction, parameterLearners);
// run training
int numMinibatchesToTrain = nbSamplesToUseForTraining * numSweepsToTrainWith / (int)minibatchSize;
var minibatchSource = new GenericMinibatchSource(inputVariables, trainImages, expectedOutput, trainImages, nbSamplesToUseForTraining, numSweepsToTrainWith, minibatchSize, device);
double aggregate_metric = 0;
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
trainer.TrainMinibatch(data, device);
double samples = trainer.PreviousMinibatchSampleCount();
double avg = trainer.PreviousMinibatchEvaluationAverage();
aggregate_metric += avg * samples;
double nbSampleSeen = trainer.TotalNumberOfSamplesSeen();
double train_error = aggregate_metric / nbSampleSeen;
Debug.WriteLine($"{minibatchCount} Average training error: {train_error:p2}");
}
}
// evaluate
{
uint testMinibatchSize = 32;
int nbSamplesToTest = 32;// evalImages.Count;
int numMinibatchesToTrain = nbSamplesToTest / (int)testMinibatchSize;
double metric_numer = 0;
double metric_denom = 0;
var minibatchSource = new GenericMinibatchSource(inputVariables, evalImages, expectedOutput, evalImages, nbSamplesToTest, 1, testMinibatchSize, device);
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IDictionary <Variable, MinibatchData> data = minibatchSource.GetNextRandomMinibatch();
////UnorderedMapVariableMinibatchData evalInput = new UnorderedMapVariableMinibatchData();
////foreach (var row in data)
//// evalInput[row.Key] = row.Value;
////double error = trainer.TestMinibatch(evalInput, device);
////metric_numer += Math.Abs(error * testMinibatchSize);
////metric_denom += testMinibatchSize;
////MinibatchData outputValue = evalInput[expectedOutput];
//IList<IList<double>> inputPixels = outputValue.data.GetDenseData<double>(inputVariables);
//IList<IList<double>> actualLabelSoftMax = outputValue.data.GetDenseData<double>(encodeDecode.Output);
//for (int i = 0; i < actualLabelSoftMax.Count; i++)
// PrintBitmap(inputPixels[i], actualLabelSoftMax[i], i);
// var normalizedInput = CNTKLib.ElementTimes(Constant.Scalar<double>(1.0 / 255.0, device), inputVariables);
Dictionary <Variable, Value> input = new Dictionary <Variable, Value>()
{
{ inputVariables, data[inputVariables].data }
};
Dictionary <Variable, Value> output = new Dictionary <Variable, Value>()
{
// { normalizedInput.Output, null }
{ encodeDecode.Output, null }
};
encodeDecode.Evaluate(input, output, device);
IList <IList <double> > inputPixels = input[inputVariables].GetDenseData <double>(inputVariables);
IList <IList <double> > outputPixels = output[encodeDecode.Output].GetDenseData <double>(encodeDecode.Output);
for (int i = 0; i < inputPixels.Count; i++)
{
PrintBitmap(inputPixels[i], outputPixels[i], i);
}
}
double test_error = (metric_numer * 100.0) / (metric_denom);
Debug.WriteLine($"Average test error: {test_error:p2}");
}
}
public void Run()
{
var device = DeviceDescriptor.UseDefaultDevice();
// 1. Generate Data
int sampleSize = 32;
int nbDimensionsInput = 2; // 2 dimensions (age&tumorsize)
int nbLabels = 2; // l'output est un vecteur de probabilités qui doit sommer à 1. Si on ne met qu'une seule dimension de sortie, l'output sera toujours de 1.
// on met donc deux dimension, une dimension 'vrai' et une dimension 'faux'. L'output sera du genre 0.25 vrai et 0.75 faux => total des poids = 1;
// premier label = faux, second = vrai
IEnumerable <DataPoint> data = GenerateData(sampleSize);
//foreach (var pt in data)
// Debug.WriteLine($"{pt.Age};{pt.TumorSize};{(pt.HasCancer ? 1 : 0)}");
Variable inputVariables = Variable.InputVariable(NDShape.CreateNDShape(new[] { nbDimensionsInput }), DataType.Double, "input");
Variable expectedOutput = Variable.InputVariable(new int[] { nbLabels }, DataType.Double, "output");
Parameter bias = new Parameter(NDShape.CreateNDShape(new[] { nbLabels }), DataType.Double, 0); // une abscisse pour chaque dimension
Parameter weights = new Parameter(NDShape.CreateNDShape(new[] { nbDimensionsInput, nbLabels }), DataType.Double, 0); // les coefficients à trouver
// 2 variable d'input, 2 estimations en sortie (proba vrai et proba faux)
Function predictionFunction = CNTKLib.Plus(CNTKLib.Times(weights, inputVariables), bias);
Function lossFunction = CNTKLib.CrossEntropyWithSoftmax(predictionFunction, expectedOutput);
Function evalErrorFunction = CNTKLib.ClassificationError(predictionFunction, expectedOutput);
//Function logisticClassifier = CNTKLib.Sigmoid(evaluationFunction, "LogisticClassifier");
uint minibatchSize = 25;
//double learningRate = 0.5;
//TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(learningRate, minibatchSize);
TrainingParameterScheduleDouble learningRatePerSample = new TrainingParameterScheduleDouble(0.3, (uint)(data.Count() / 1.0));
TrainingParameterScheduleDouble momentumSchedule = new TrainingParameterScheduleDouble(0.9126265014311797, minibatchSize);
var parameters = new ParameterVector();
foreach (var p in predictionFunction.Parameters())
{
parameters.Add(p);
}
List <Learner> parameterLearners = new List <Learner>()
{
CNTKLib.FSAdaGradLearner(parameters, learningRatePerSample, momentumSchedule, true)
};
Trainer trainer = Trainer.CreateTrainer(predictionFunction, lossFunction, evalErrorFunction, parameterLearners);
double nbSamplesToUseForTraining = 20000;
int numMinibatchesToTrain = (int)(nbSamplesToUseForTraining / (int)minibatchSize);
// train the model
for (int minibatchCount = 0; minibatchCount < numMinibatchesToTrain; minibatchCount++)
{
IEnumerable <DataPoint> trainingData = GenerateData((int)minibatchSize);
List <double> minibatchInput = new List <double>();
List <double> minibatchOutput = new List <double>();
foreach (DataPoint row in trainingData)
{
minibatchInput.Add(row.Age);
minibatchInput.Add(row.TumorSize);
minibatchOutput.Add(row.HasCancer ? 0d : 1d);
minibatchOutput.Add(row.HasCancer ? 1d : 0d);
}
Value inputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbDimensionsInput }), minibatchInput, device);
Value outputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbLabels }), minibatchOutput, device);
trainer.TrainMinibatch(new Dictionary <Variable, Value>()
{
{ inputVariables, inputData }, { expectedOutput, outputData }
}, device);
PrintTrainingProgress(trainer, minibatchCount);
}
// test
{
int testSize = 100;
IEnumerable <DataPoint> trainingData = GenerateData(testSize);
List <double> minibatchInput = new List <double>();
List <double> minibatchOutput = new List <double>();
foreach (DataPoint row in trainingData)
{
minibatchInput.Add(row.Age);
minibatchInput.Add(row.TumorSize);
minibatchOutput.Add(row.HasCancer ? 0d : 1d);
minibatchOutput.Add(row.HasCancer ? 1d : 0d);
}
Value inputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbDimensionsInput }), minibatchInput, device);
Value outputData = Value.CreateBatch <double>(NDShape.CreateNDShape(new int[] { nbLabels }), minibatchOutput, device);
IList <IList <double> > expectedOneHot = outputData.GetDenseData <double>(predictionFunction.Output);
IList <int> expectedLabels = expectedOneHot.Select(l => l.IndexOf(1.0d)).ToList();
var outputDataMap = new Dictionary <Variable, Value>()
{
{ predictionFunction.Output, null }
};
predictionFunction.Evaluate(
new Dictionary <Variable, Value>()
{
{ inputVariables, inputData }
},
outputDataMap,
device);
Value outputValue = outputDataMap[predictionFunction.Output];
IList <IList <double> > actualLabelSoftMax = outputValue.GetDenseData <double>(predictionFunction.Output);
var actualLabels = actualLabelSoftMax.Select((IList <double> l) => l.IndexOf(l.Max())).ToList();
int misMatches = actualLabels.Zip(expectedLabels, (a, b) => a.Equals(b) ? 0 : 1).Sum();
Debug.WriteLine($"Validating Model: Total Samples = {testSize}, Misclassify Count = {misMatches}");
}
}
/// <summary>
/// Train and evaluate an image classifier with CIFAR-10 data.
/// The classification model is saved after training.
/// For repeated runs, the caller may choose whether to retrain a model or
/// just validate an existing one.
/// </summary>
/// <param name="device">CPU or GPU device to run</param>
/// <param name="forceRetrain">whether to override an existing model.
/// if true, any existing model will be overridden and the new one evaluated.
/// if false and there is an existing model, the existing model is evaluated.</param>
public static void TrainAndEvaluate(DeviceDescriptor device, bool forceRetrain)
{
string modelFile = Path.Combine(CifarDataFolder, "CNTK-CSharp.model");
// If a model already exists and not set to force retrain, validate the model and return.
if (File.Exists(modelFile) && !forceRetrain)
{
ValidateModel(device, modelFile);
return;
}
// prepare training data
var minibatchSource = CreateMinibatchSource(Path.Combine(CifarDataFolder, "train_map.txt"),
Path.Combine(CifarDataFolder, "CIFAR-10_mean.xml"), imageDim, numClasses, MaxEpochs);
var imageStreamInfo = minibatchSource.StreamInfo("features");
var labelStreamInfo = minibatchSource.StreamInfo("labels");
// build a model
var imageInput = CNTKLib.InputVariable(imageDim, imageStreamInfo.m_elementType, "Images");
var labelsVar = CNTKLib.InputVariable(new int[] { numClasses }, labelStreamInfo.m_elementType, "Labels");
var classifierOutput = ResNetClassifier(imageInput, numClasses, device, "classifierOutput");
// prepare for training
var trainingLoss = CNTKLib.CrossEntropyWithSoftmax(classifierOutput, labelsVar, "lossFunction");
var prediction = CNTKLib.ClassificationError(classifierOutput, labelsVar, 3, "predictionError");
//学习率策略
double[] lrs = { 3e-2, 3e-3, 3e-4, 3e-4, 5e-5 }; //学习率
int[] check_point = { 80, 120, 160, 180 }; //学习率在epoch到达多少时更新
uint minibatchSize = 32;
PairSizeTDouble p1 = new PairSizeTDouble(80, lrs[0]);
PairSizeTDouble p2 = new PairSizeTDouble(40, lrs[1]);
PairSizeTDouble p3 = new PairSizeTDouble(40, lrs[2]);
PairSizeTDouble p4 = new PairSizeTDouble(20, lrs[3]);
PairSizeTDouble p5 = new PairSizeTDouble(20, lrs[4]);
VectorPairSizeTDouble vp = new VectorPairSizeTDouble()
{
p1, p2, p3, p4, p5
};
int sample_num_in_a_epoch = 50000;
TrainingParameterScheduleDouble learningRateSchedule = new TrainingParameterScheduleDouble(vp, (uint)sample_num_in_a_epoch);
//动量
var momentum = new TrainingParameterScheduleDouble(0.9, 1);
//SGD Learner
//var sgdLearner = Learner.SGDLearner(classifierOutput.Parameters(), learningRateSchedule);
//Adam Learner
ParameterVector parameterVector = new ParameterVector();
foreach (var parameter in classifierOutput.Parameters())
{
parameterVector.Add(parameter);
}
var adamLearner = CNTKLib.AdamLearner(parameterVector, learningRateSchedule, momentum);
//Trainer
var trainer = Trainer.CreateTrainer(classifierOutput, trainingLoss, prediction, new List <Learner> {
adamLearner
});
int outputFrequencyInMinibatches = 20, miniBatchCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
// Feed data to the trainer for number of epochs.
Console.WriteLine("*****************Train Start*****************");
while (true)
{
var minibatchData = minibatchSource.GetNextMinibatch(minibatchSize, device);
// Stop training once max epochs is reached.
if (minibatchData.empty())
{
break;
}
trainer.TrainMinibatch(new Dictionary <Variable, MinibatchData>()
{
{ imageInput, minibatchData[imageStreamInfo] },
{ labelsVar, minibatchData[labelStreamInfo] }
}, device);
TestHelper.PrintTrainingProgress(trainer, adamLearner, miniBatchCount++, outputFrequencyInMinibatches);
}
// save the model
var imageClassifier = Function.Combine(new List <Variable>()
{
trainingLoss, prediction, classifierOutput
}, "ImageClassifier");
imageClassifier.Save(modelFile);
Console.WriteLine("*****************Train Stop*****************");
// validate the model
float acc = ValidateModel(device, modelFile);
sw.Stop();
TimeSpan ts2 = sw.Elapsed;
Console.WriteLine("*****************Validate Stop*****************");
string logstr = "Total time :" + ts2.TotalSeconds + "s. acc:" + acc;
Console.WriteLine(logstr);
int i = 1;
while (System.IO.File.Exists("../../../../log_" + i.ToString() + ".txt"))
{
i++;
}
var file = System.IO.File.Create("../../../../log_" + i.ToString() + ".txt");
byte[] data = System.Text.Encoding.Default.GetBytes(logstr);
file.Write(data, 0, data.Length);
}