public static Initialize ( int numConcurrentModels, string modelFilePath, int numThreads = 1 ) : void | ||
numConcurrentModels | int | The number of concurrent models |
modelFilePath | string | The model file path to load the model from |
numThreads | int | |
return | void |
/// <summary> /// Evaluates multiple instances of a model in the same process. /// </summary> /// <remarks> /// Although all models execute concurrently (multiple tasks), each model is evaluated with a single task at a time. /// </remarks> private static void EvaluateMultipleModels() { // Specifies the number of models in memory as well as the number of parallel tasks feeding these models (1 to 1) int numConcurrentModels = 4; // Specifies the number of times to iterate through the test file (epochs) int numRounds = 1; // Counts the number of evaluations accross all models int count = 0; // Counts the number of failed evaluations (output != expected) accross all models int errorCount = 0; // The examples assume the executable is running from the data folder // We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\MNIST\Data\"); // Load model string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\02_Convolution"); if (!File.Exists(modelFilePath)) { Console.WriteLine("Error: The model {0} does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/MNIST to create the model.", modelFilePath); throw new FileNotFoundException(string.Format("File {0} not found.", modelFilePath)); } // Initializes the model instances ModelEvaluator.Initialize(numConcurrentModels, modelFilePath); string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28_cntk_text.txt"); if (!File.Exists(testfile)) { Console.WriteLine("Error: The test file {0} does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/MNIST to download the data.", testfile); throw new FileNotFoundException(string.Format("File {0} not found.", testfile)); } Stopwatch sw = new Stopwatch(); sw.Start(); try { for (int i = 0; i < numRounds; i++) { // Feed each line to a single model in parallel Parallel.ForEach(File.ReadLines(testfile), new ParallelOptions() { MaxDegreeOfParallelism = numConcurrentModels }, (line) => { Interlocked.Increment(ref count); // The file format correspond to the CNTK Text Format Reader format (https://github.com/Microsoft/CNTK/wiki/CNTKTextFormat-Reader) var sets = line.Split('|'); var labels = sets[1].Trim().Split(' ').Skip(1); var features = sets[2].Trim().Split(' ').Skip(1); // Retrieve the 1-hot vector with the label index var expected = labels.Select(float.Parse).Select((v, index) => new { Value = v, Index = index }) .Aggregate((a, b) => (a.Value > b.Value) ? a : b) .Index; // Retrieve the features var inputs = features.Select(float.Parse).ToList(); // We can call the evaluate method and get back the results (single layer)... var outputs = ModelEvaluator.Evaluate(inputs); // Retrieve the outcome index (so we can compare it with the expected index) var max = outputs.Select((v, index) => new { Value = v, Index = index }) .Aggregate((a, b) => (a.Value > b.Value) ? a : b) .Index; // Count the errors if (expected != max) { Interlocked.Increment(ref errorCount); } }); } } catch (CNTKException ex) { Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception"); } catch (Exception ex) { Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception"); } sw.Stop(); ModelEvaluator.DisposeAll(); Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28_cntk_text.txt", numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds); }
/// <summary> /// Evaluates multiple instances of a model in the same process. /// </summary> /// <remarks> /// Although all models execute concurrently (multiple tasks), each model is evaluated with a single task at a time. /// </remarks> private static void EvaluateMultipleModels() { // Specifies the number of models in memory as well as the number of parallel tasks feeding these models (1 to 1) int numConcurrentModels = 4; // Specifies the number of times to iterate through the test file (epochs) int numRounds = 1; // Counts the number of evaluations accross all models int count = 0; // Counts the number of failed evaluations (output != expected) accross all models int errorCount = 0; // The examples assume the executable is running from the data folder // We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\MNIST\Data\"); // Load model string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\02_Convolution"); // Initializes the model instances ModelEvaluator.Initialize(numConcurrentModels, modelFilePath); string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28.txt"); Stopwatch sw = new Stopwatch(); sw.Start(); try { for (int i = 0; i < numRounds; i++) { // Feed each line to a single model in parallel Parallel.ForEach(File.ReadLines(testfile), new ParallelOptions() { MaxDegreeOfParallelism = numConcurrentModels }, (line) => { Interlocked.Increment(ref count); // The first value in the line is the expected label index for the record's outcome int expected = int.Parse(line.Substring(0, line.IndexOf('\t'))); var inputs = line.Substring(line.IndexOf('\t') + 1).Split('\t').Select(float.Parse).ToList(); // We can call the evaluate method and get back the results (single layer)... var outputs = ModelEvaluator.Evaluate(inputs); // Retrieve the outcome index (so we can compare it with the expected index) int index = 0; var max = outputs.Select(v => new { Value = v, Index = index++ }) .Aggregate((a, b) => (a.Value > b.Value) ? a : b) .Index; // Count the errors if (expected != max) { Interlocked.Increment(ref errorCount); } }); } } catch (CNTKException ex) { Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception"); } catch (Exception ex) { Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception"); } sw.Stop(); ModelEvaluator.DisposeAll(); Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28.txt", numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds); }