/// <summary>
/// Convert the data to suitable type trainer supports
/// </summary>
/// <param name="args"> data being converted</param>
/// <returns></returns>
private UnorderedMapVariableMinibatchData prepareForTrainerEx(UnorderedMapStreamInformationMinibatchData args, bool cloneData)
{
// var d = new Dictionary<Variable, MinibatchData>();
var d = new UnorderedMapVariableMinibatchData();
for (int i = 0; i < args.Count; i++)
{
var k = args.ElementAt(i);
var var = InputVariables.Union(OutputVariables).Where(x => x.Name.Equals(k.Key.m_name)).FirstOrDefault();
if (var == null)
{
throw new Exception("Variable doesn't exist!");
}
//clone data first
var key = k.Key;
MinibatchData mbData = null;
if (cloneData)
{
mbData = new MinibatchData(k.Value.data.DeepClone(), k.Value.numberOfSequences, k.Value.numberOfSamples, k.Value.sweepEnd);
}
else
{
mbData = k.Value;
}
d.Add(var, mbData);
}
return(d);
}
/// <summary>
/// The method is called during training process. The method returns the chunk of data specified by Batch size.
/// </summary>
/// <param name="minibatchSizeInSamples"></param>
/// <param name="device"></param>
/// <returns></returns>
public UnorderedMapStreamInformationMinibatchData GetNextMinibatch(uint minibatchSizeInSamples, DeviceDescriptor device)
{
if (Type == MinibatchType.Default)
{
var retVal = defaultmb.GetNextMinibatch(minibatchSizeInSamples, device);
return(retVal);
}
else if (Type == MinibatchType.Custom)
{
var retVal = nextBatch(custommb, StreamConfigurations, (int)minibatchSizeInSamples, 1, device);
var mb = new UnorderedMapStreamInformationMinibatchData();
for (int i = 0; i < retVal.Count; i++)
{
var k = retVal.ElementAt(i);
//this is fix for 2.6 version since the Value data is not valid, so it must be clone in order to create MiniBatchData
// var mbData = new MinibatchData(k.Value.data.DeepClone(), k.Value.numberOfSequences, k.Value.numberOfSamples, k.Value.sweepEnd);
var si = new StreamInformation();
si.m_definesMbSize = StreamConfigurations[i].m_definesMbSize;
si.m_storageFormat = k.Value.data.StorageFormat;
si.m_name = StreamConfigurations[i].m_streamName;
//
//mb.Add(si, mbData);
mb.Add(si, k.Value);
}
return(mb);
}
else
{
throw new Exception("Unsupported Mini-batch-source type!");
}
}
/// <summary>
/// The method is called during training process. The method returns the chunk of data specified by Batch size.
/// </summary>
/// <param name="minibatchSizeInSamples"></param>
/// <param name="device"></param>
/// <returns></returns>
public UnorderedMapStreamInformationMinibatchData GetNextMinibatch(uint minibatchSizeInSamples, DeviceDescriptor device)
{
if (Type == MinibatchType.Default || Type == MinibatchType.Image)
{
var retVal = defaultmb.GetNextMinibatch(minibatchSizeInSamples, device);
return(retVal);
}
else if (Type == MinibatchType.Custom)
{
var retVal = nextBatch(custommb, StreamConfigurations, (int)minibatchSizeInSamples);
var mb = new UnorderedMapStreamInformationMinibatchData();
var eofs = custommb.EndOfStream;
//create minibatch
foreach (var d in retVal)
{
var v = Value.CreateBatchOfSequences <float>(new NDShape(1, d.Key.m_dim), d.Value, device);
var mbd = new MinibatchData(v, (uint)d.Value.Count(), (uint)d.Value.Sum(x => x.Count), eofs);
var si = new StreamInformation();
si.m_definesMbSize = d.Key.m_definesMbSize;
si.m_storageFormat = StorageFormat.Dense;
si.m_name = d.Key.m_streamName;
mb.Add(si, mbd);
}
return(mb);
}
else
{
throw new Exception("Unsupported Mini-batch-source type!");
}
}
public UnorderedMapStreamInformationMinibatchData(UnorderedMapStreamInformationMinibatchData other) : this(CNTKLibPINVOKE.new_UnorderedMapStreamInformationMinibatchData__SWIG_1(UnorderedMapStreamInformationMinibatchData.getCPtr(other)), true)
{
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
}
public UnorderedMapStreamInformationMinibatchDataEnumerator(UnorderedMapStreamInformationMinibatchData collection)
{
collectionRef = collection;
keyCollection = new global::System.Collections.Generic.List <StreamInformation>(collection.Keys);
currentIndex = -1;
currentObject = null;
currentSize = collectionRef.Count;
}
public static float ValidateModelWithMinibatchSource(string modelFile, MinibatchSource testMinibatchSource, int[] imageDim, int numClasses, string featureInputName, string labelInputName, string outputName, DeviceDescriptor device, int maxCount = 1000)
{
Function model = Function.Load(modelFile, device);
Variable imageInput = model.Arguments[0];
Variable labelOutput = model.Outputs.Single(o => o.Name == outputName);
StreamInformation featureStreamInfo = testMinibatchSource.StreamInfo(featureInputName);
StreamInformation labelStreamInfo = testMinibatchSource.StreamInfo(labelInputName);
int batchSize = 50;
int miscountTotal = 0, totalCount = 0;
while (true)
{
UnorderedMapStreamInformationMinibatchData minibatchData = testMinibatchSource.GetNextMinibatch((uint)batchSize, device);
if (minibatchData == null || minibatchData.Count == 0)
{
break;
}
totalCount += (int)minibatchData[featureStreamInfo].numberOfSamples;
// Expected labels are in the minibatch data.
IList <IList <float> > labelData = minibatchData[labelStreamInfo].data.GetDenseData <float>(labelOutput);
List <int> expectedLabels = labelData.Select(l => l.IndexOf(l.Max())).ToList();
Dictionary <Variable, Value> inputDataMap = new Dictionary <Variable, Value>()
{
{ imageInput, minibatchData[featureStreamInfo].data }
};
Dictionary <Variable, Value> outputDataMap = new Dictionary <Variable, Value>()
{
{ labelOutput, null }
};
model.Evaluate(inputDataMap, outputDataMap, device);
IList <IList <float> > outputData = outputDataMap[labelOutput].GetDenseData <float>(labelOutput);
List <int> actualLabels = outputData.Select(l => l.IndexOf(l.Max())).ToList();
int misMatches = actualLabels.Zip(expectedLabels, (a, b) => a.Equals(b) ? 0 : 1).Sum();
miscountTotal += misMatches;
Console.WriteLine($"Validating Model: Total Samples = {totalCount}, Misclassify Count = {miscountTotal}");
if (totalCount > maxCount)
{
break;
}
}
float errorRate = 1.0F * miscountTotal / totalCount;
Console.WriteLine($"Model Validation Error = {errorRate}");
return(errorRate);
}
public virtual UnorderedMapStreamInformationMinibatchData GetNextMinibatch(uint minibatchSizeInSequences, uint minibatchSizeInSamples, uint numberOfWorkers, uint workerRank)
{
UnorderedMapStreamInformationMinibatchData ret = new UnorderedMapStreamInformationMinibatchData(CNTKLibPINVOKE.MinibatchSource_GetNextMinibatch__SWIG_3(swigCPtr, minibatchSizeInSequences, minibatchSizeInSamples, numberOfWorkers, workerRank), false);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
public UnorderedMapStreamInformationMinibatchData GetNextMinibatch(uint minibatchSizeInSequences, uint minibatchSizeInSamples, DeviceDescriptor device)
{
UnorderedMapStreamInformationMinibatchData ret = new UnorderedMapStreamInformationMinibatchData(CNTKLibPINVOKE.MinibatchSource_GetNextMinibatch__SWIG_0(swigCPtr, minibatchSizeInSequences, minibatchSizeInSamples, DeviceDescriptor.getCPtr(device)), false);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
public UnorderedMapStreamInformationMinibatchData GetNextMinibatch(uint minibatchSizeInSamples)
{
UnorderedMapStreamInformationMinibatchData ret = new UnorderedMapStreamInformationMinibatchData(CNTKLibPINVOKE.MinibatchSource_GetNextMinibatch__SWIG_5(swigCPtr, minibatchSizeInSamples), false);
if (CNTKLibPINVOKE.SWIGPendingException.Pending)
{
throw CNTKLibPINVOKE.SWIGPendingException.Retrieve();
}
return(ret);
}
/// <summary>
/// Convert minibatch data to Value
/// </summary>
/// <param name="args"> data being converted</param>
/// <returns></returns>
public static Dictionary <Variable, Value> ToMinibatchValueData(UnorderedMapStreamInformationMinibatchData args, List <Variable> vars)
{
var arguments = new Dictionary <Variable, Value>();
foreach (var mbd in args)
{
var v = vars.Where(x => x.Name == mbd.Key.m_name).FirstOrDefault();
if (v == null)
{
throw new Exception("Stream is invalid!");
}
arguments.Add(v, mbd.Value.data.DeepClone());
}
return(arguments);
}
/// <summary>
/// The method is called during Evaluation of the model for specific data set which is specified as an argument
/// </summary>
/// <param name="type"></param>
/// <param name="strFilePath">dataset file path</param>
/// <param name="streamConfigurations">stream configuration which provides meta-data information</param>
/// <param name="device"></param>
/// <returns></returns>
public static UnorderedMapStreamInformationMinibatchData GetFullBatch(MinibatchType type, string strFilePath, StreamConfiguration[] streamConfigurations, DeviceDescriptor device)
{
if (type == MinibatchType.Default)
{
var mbs = MinibatchSource.TextFormatMinibatchSource(strFilePath, streamConfigurations, MinibatchSource.FullDataSweep, false);
//
var minibatchData = mbs.GetNextMinibatch(int.MaxValue, device);
//
return(minibatchData);
}
else if (type == MinibatchType.Custom)
{
using (var mbreader = new StreamReader(strFilePath))
{
var retVal = nextBatch(mbreader, streamConfigurations, -1, 1, device);
var mb = new UnorderedMapStreamInformationMinibatchData();
for (int i = 0; i < retVal.Count; i++)
{
var k = retVal.ElementAt(i);
var key = k.Key;
var si = new StreamInformation();
si.m_definesMbSize = streamConfigurations[i].m_definesMbSize;
si.m_storageFormat = k.Value.data.StorageFormat;
si.m_name = streamConfigurations[i].m_streamName;
var stream = streamConfigurations[i];
mb.Add(si, k.Value);
}
return(mb);
}
}
else
{
throw new Exception("Minibatch is not supported.");
}
}
/// <summary>
/// Convert minibatch data
/// </summary>
/// <param name="args"> data being converted</param>
/// <returns></returns>
public static UnorderedMapVariableMinibatchData ToMinibatchData(UnorderedMapStreamInformationMinibatchData args, List <Variable> vars, MinibatchType type)
{
var arguments = new UnorderedMapVariableMinibatchData();
foreach (var mbd in args)
{
var v = vars.Where(x => x.Name == mbd.Key.m_name).FirstOrDefault();
if (v == null)
{
throw new Exception("Stream is invalid!");
}
if (type == MinibatchType.Custom)
{
var mbd1 = new MinibatchData(mbd.Value.data.DeepClone(), mbd.Value.numberOfSamples, mbd.Value.sweepEnd);
arguments.Add(v, mbd1);
}
else
{
arguments.Add(v, mbd.Value);
}
}
return(arguments);
}
internal static global::System.Runtime.InteropServices.HandleRef getCPtr(UnorderedMapStreamInformationMinibatchData obj)
{
return((obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr);
}
private static void Main(string[] args)
{
DeviceDescriptor device = DeviceDescriptor.CPUDevice;
Console.WriteLine("Welcome to CNTK machine learning world");
Console.WriteLine($"Device: {device.AsString()}");
// Queue: Convert and create the CSV file for ML.net?
Console.WriteLine("Do you want to convert and create the text minibatch file for CNTK? If the file is already exist, type 'N' to skip, else type 'Y' to avoid errors.");
if (Console.ReadKey().Key == ConsoleKey.Y)
{
MNISTDataConvertor convertor = new MNISTDataConvertor();
convertor.ConvertAndSave();
}
Console.WriteLine();
string featureStreamName = "features";
string labelsStreamName = "labels";
string classifierName = "classifierOutput";
int[] image_dimension = new int[] { 28, 28, 1 };
int[] result_dimension = new int[] { 10 };
int image_flat_size = 28 * 28;
int number_class = 10;
// Input and output
Variable features = InputVariable(shape: image_dimension, dataType: DataType.Float, name: "features");
Variable labels = InputVariable(shape: result_dimension, dataType: DataType.Float, name: "labels");
// Scaled and CNN
Function scaled_input = ElementTimes(Constant.Scalar(1.0f / 256.0f, device), features);
MNISTConvolutionNN convolutionNN = new MNISTConvolutionNN(scaled_input, device, classifierName: classifierName);
Function classifier_output = convolutionNN.CNN_Function;
// Loss functions
Function loss_function = CrossEntropyWithSoftmax(new Variable(classifier_output), labels, "lossFunction");
Function perdiction = ClassificationError(new Variable(classifier_output), labels, "classificationError");
// Learning rate
TrainingParameterScheduleDouble trainingParameter = new TrainingParameterScheduleDouble(0.01125, 1);
// Learners
IList<Learner> learners = new List<Learner>()
{
Learner.SGDLearner(classifier_output.Parameters(), trainingParameter),
};
// Trainer
Trainer trainer = Trainer.CreateTrainer(classifier_output, loss_function, perdiction, learners);
// Minibatch(Training data)
const uint minibatchSize = 35;
int outputFrequencyMinibatch = 35;
int Epchos = 15;
IList<StreamConfiguration> streamConfigurations = new StreamConfiguration[]
{
new StreamConfiguration(featureStreamName, image_flat_size),
new StreamConfiguration(labelsStreamName, number_class),
};
MinibatchSource minibatchSource = MinibatchSource.TextFormatMinibatchSource("mnist-train.txt", streamConfigurations, MinibatchSource.InfinitelyRepeat);
StreamInformation featureStreamInfo = minibatchSource.StreamInfo(featureStreamName);
StreamInformation labelStreamInfo = minibatchSource.StreamInfo(labelsStreamName);
// Training
int iteration = 0;
while (Epchos > 0)
{
UnorderedMapStreamInformationMinibatchData minibatchData = minibatchSource.GetNextMinibatch(minibatchSize, device);
Dictionary<Variable, MinibatchData> arguments = new Dictionary<Variable, MinibatchData>
{
{ features, minibatchData[featureStreamInfo] },
{ labels, minibatchData[labelStreamInfo] }
};
trainer.TrainMinibatch(arguments, device);
PrintTrainingProgress(trainer, iteration++, outputFrequencyMinibatch);
if (minibatchData.Values.Any(a => a.sweepEnd))
{
Epchos--;
}
}
Console.WriteLine("*****Training done*****");
// Save the trained model
classifier_output.Save("cnn-model.mld");
// Validate the model
MinibatchSource minibatchSourceNewModel = MinibatchSource.TextFormatMinibatchSource("mnist-train.txt", streamConfigurations, MinibatchSource.FullDataSweep);
MNISTConvolutionNN.ValidateModelWithMinibatchSource("cnn-model.mld", minibatchSourceNewModel,
image_dimension, number_class, featureStreamName, labelsStreamName, classifierName, device, maxCount: 10000);
Console.ReadKey(true);
}
