/// <summary>
/// Evaluates multiple instances of a model in the same process.
/// </summary>
/// <remarks>
/// Although all models execute concurrently (multiple tasks), each model is evaluated with a single task at a time.
/// </remarks>
private static void EvaluateMultipleModels()
{
// Specifies the number of models in memory as well as the number of parallel tasks feeding these models (1 to 1)
int numConcurrentModels = 4;
// Specifies the number of times to iterate through the test file (epochs)
int numRounds = 1;
// Counts the number of evaluations accross all models
int count = 0;
// Counts the number of failed evaluations (output != expected) accross all models
int errorCount = 0;
// The examples assume the executable is running from the data folder
// We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder
Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\MNIST\Data\");
// Load model
string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\02_Convolution");
if (!File.Exists(modelFilePath))
{
Console.WriteLine("Error: The model {0} does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/MNIST to create the model.", modelFilePath);
throw new FileNotFoundException(string.Format("File {0} not found.", modelFilePath));
}
// Initializes the model instances
ModelEvaluator.Initialize(numConcurrentModels, modelFilePath);
string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28_cntk_text.txt");
if (!File.Exists(testfile))
{
Console.WriteLine("Error: The test file {0} does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/MNIST to download the data.", testfile);
throw new FileNotFoundException(string.Format("File {0} not found.", testfile));
}
Stopwatch sw = new Stopwatch();
sw.Start();
try
{
for (int i = 0; i < numRounds; i++)
{
// Feed each line to a single model in parallel
Parallel.ForEach(File.ReadLines(testfile), new ParallelOptions()
{
MaxDegreeOfParallelism = numConcurrentModels
}, (line) =>
{
Interlocked.Increment(ref count);
// The file format correspond to the CNTK Text Format Reader format (https://github.com/Microsoft/CNTK/wiki/CNTKTextFormat-Reader)
var sets = line.Split('|');
var labels = sets[1].Trim().Split(' ').Skip(1);
var features = sets[2].Trim().Split(' ').Skip(1);
// Retrieve the 1-hot vector with the label index
var expected = labels.Select(float.Parse).Select((v, index) => new { Value = v, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
// Retrieve the features
var inputs = features.Select(float.Parse).ToList();
// We can call the evaluate method and get back the results (single layer)...
var outputs = ModelEvaluator.Evaluate(inputs);
// Retrieve the outcome index (so we can compare it with the expected index)
var max = outputs.Select((v, index) => new { Value = v, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
// Count the errors
if (expected != max)
{
Interlocked.Increment(ref errorCount);
}
});
}
}
catch (CNTKException ex)
{
Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
}
catch (Exception ex)
{
Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
}
sw.Stop();
ModelEvaluator.DisposeAll();
Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28_cntk_text.txt",
numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds);
}
/// <summary>
/// Evaluates multiple instances of a model in the same process.
/// </summary>
/// <remarks>
/// Although all models execute concurrently (multiple tasks), each model is evaluated with a single task at a time.
/// </remarks>
private static void EvaluateMultipleModels()
{
// Specifies the number of models in memory as well as the number of parallel tasks feeding these models (1 to 1)
int numConcurrentModels = 4;
// Specifies the number of times to iterate through the test file (epochs)
int numRounds = 1;
// Counts the number of evaluations accross all models
int count = 0;
// Counts the number of failed evaluations (output != expected) accross all models
int errorCount = 0;
// The examples assume the executable is running from the data folder
// We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder
Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\MNIST\Data\");
// Load model
string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\02_Convolution");
// Initializes the model instances
ModelEvaluator.Initialize(numConcurrentModels, modelFilePath);
string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28.txt");
Stopwatch sw = new Stopwatch();
sw.Start();
try
{
for (int i = 0; i < numRounds; i++)
{
// Feed each line to a single model in parallel
Parallel.ForEach(File.ReadLines(testfile), new ParallelOptions()
{
MaxDegreeOfParallelism = numConcurrentModels
}, (line) =>
{
Interlocked.Increment(ref count);
// The first value in the line is the expected label index for the record's outcome
int expected = int.Parse(line.Substring(0, line.IndexOf('\t')));
var inputs = line.Substring(line.IndexOf('\t') + 1).Split('\t').Select(float.Parse).ToList();
// We can call the evaluate method and get back the results (single layer)...
var outputs = ModelEvaluator.Evaluate(inputs);
// Retrieve the outcome index (so we can compare it with the expected index)
int index = 0;
var max = outputs.Select(v => new { Value = v, Index = index++ })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
// Count the errors
if (expected != max)
{
Interlocked.Increment(ref errorCount);
}
});
}
}
catch (CNTKException ex)
{
Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
}
catch (Exception ex)
{
Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
}
sw.Stop();
ModelEvaluator.DisposeAll();
Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28.txt",
numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds);
}
