static void Main(string[] args)
{
#if CPUONLY
Console.WriteLine("======== Evaluate model on CPU using CPUOnly build ========");
#else
Console.WriteLine("======== Evaluate model on CPU using GPU build ========");
#endif
if (ShouldRunOnCpu())
{
var device = DeviceDescriptor.CPUDevice;
CNTKLibraryManagedExamples.EvaluationSingleImage(device);
// Run memory tests.
MemoryTests.ValidateObjectReferences(device);
CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);
CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallel(device);
// Run memory tests again.
MemoryTests.ValidateObjectReferences(device);
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);
// It is sufficient to test loading model from memory buffer only on CPU.
CNTKLibraryManagedExamples.LoadModelFromMemory(device);
MemoryTests.WriteOutputs();
}
if (ShouldRunOnGpu())
{
Console.WriteLine(" ====== Evaluate model on GPU =====");
var device = DeviceDescriptor.GPUDevice(0);
// Run memory tests.
MemoryTests.ValidateObjectReferences(device);
CNTKLibraryManagedExamples.EvaluationSingleImage(device);
CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);
CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallel(device);
// Run memory tests.
MemoryTests.ValidateObjectReferences(device);
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);
// Run memory tests again.
MemoryTests.WriteOutputs();
}
Console.WriteLine("======== Evaluation completes. ========");
}
static void Main(string[] args)
{
// Todo: move to a separate unit test.
Console.WriteLine("Test Utils");
int maxThreads = Utils.GetMaxNumCPUThreads();
Utils.SetMaxNumCPUThreads(2);
Console.WriteLine("MaxNumCPUThreads: before: " + maxThreads + ", after " + Utils.GetMaxNumCPUThreads());
Utils.SetMaxNumCPUThreads(maxThreads);
Console.WriteLine("reset MaxNumCPuThreads to " + Utils.GetMaxNumCPUThreads());
var level = Utils.GetTraceLevel();
Utils.SetTraceLevel(TraceLevel.Info);
Console.WriteLine("TraceLevel: before: " + level + ", after " + Utils.GetTraceLevel());
Utils.SetTraceLevel(level);
Console.WriteLine("reset TraceLevel to " + Utils.GetTraceLevel());
Console.WriteLine(Utils.DataTypeName(DataType.Float));
Console.WriteLine(Utils.DataTypeSize(DataType.Double));
Console.WriteLine(Utils.DeviceKindName(DeviceDescriptor.CPUDevice.Type));
Console.WriteLine(Utils.DeviceKindName(DeviceKind.GPU));
Console.WriteLine(Utils.IsSparseStorageFormat(StorageFormat.Dense));
Console.WriteLine(Utils.IsSparseStorageFormat(StorageFormat.SparseCSC));
Console.WriteLine(Utils.IsSparseStorageFormat(StorageFormat.SparseBlockCol));
Console.WriteLine(Utils.VariableKindName(VariableKind.Constant));
Console.WriteLine(Utils.VariableKindName(VariableKind.Placeholder));
Console.WriteLine(Utils.VariableKindName(VariableKind.Input));
Console.WriteLine(Utils.VariableKindName(VariableKind.Output));
Console.WriteLine(Utils.VariableKindName(VariableKind.Parameter));
#if CPUONLY
Console.WriteLine("======== Evaluate model on CPU using CPUOnly build ========");
#else
Console.WriteLine("======== Evaluate model on CPU using GPU build ========");
#endif
if (args.Length >= 1)
{
Console.WriteLine($"-------- running with test data in {args[0]} --------");
CNTKLibraryManagedExamples.ExampleTestDataDir = args[0];
}
CNTKLibraryManagedExamples.Setup();
if (ShouldRunOnCpu())
{
var device = DeviceDescriptor.CPUDevice;
CNTKLibraryManagedExamples.EvaluationSingleImage(device);
// Run memory tests.
MemoryTests.ValidateObjectReferences(device);
CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);
MemoryTests.WriteOutputs();
CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallelAsync(device).Wait();
// Run memory tests again.
MemoryTests.ValidateObjectReferences(device);
Task evalTask = CNTKLibraryManagedExamples.EvaluationSingleImageAsync(device);
evalTask.Wait();
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);
// It is sufficient to test loading model from memory buffer only on CPU.
CNTKLibraryManagedExamples.LoadModelFromMemory(device);
MemoryTests.WriteOutputs();
MemoryTests.ValueCopyToSparseCSCTest <float>(device);
MemoryTests.ValueCopyToSparseCSCTest <double>(device);
CNTKLibraryManagedExamples.EvaluateIntermediateLayer(device);
CNTKLibraryManagedExamples.EvaluateCombinedOutputs(device);
}
if (ShouldRunOnGpu())
{
Console.WriteLine(" ====== Evaluate model on GPU =====");
var device = DeviceDescriptor.GPUDevice(0);
// Run memory tests.
MemoryTests.ValidateObjectReferences(device);
CNTKLibraryManagedExamples.EvaluationSingleImage(device);
CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);
CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallelAsync(device).Wait();
// Run memory tests.
MemoryTests.ValidateObjectReferences(device);
Task evalTask = CNTKLibraryManagedExamples.EvaluationSingleImageAsync(device);
evalTask.Wait();
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);
// Run memory tests again.
MemoryTests.WriteOutputs();
MemoryTests.ValueCopyToSparseCSCTest <float>(device);
MemoryTests.ValueCopyToSparseCSCTest <double>(device);
CNTKLibraryManagedExamples.EvaluateIntermediateLayer(device);
CNTKLibraryManagedExamples.EvaluateCombinedOutputs(device);
}
Console.WriteLine("======== Evaluation completes. ========");
}
public void SetupUsingResetModel(DeviceDescriptor device)
{
try
{
Console.WriteLine("\n===== Setup memory tests using Resnet Model =====");
var deviceList = DeviceDescriptor.AllDevices();
MemoryTests.Device0 = deviceList[0];
// Load the model.
string modelFilePath = "resnet20.dnn";
CNTKLibraryManagedExamples.ThrowIfFileNotExist(modelFilePath, string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/Classification/ResNet to create the model.", modelFilePath));
Function modelFunc = Function.LoadModel(modelFilePath, device);
Variable inputVar = modelFunc.Arguments.Single();
MemoryTests.ArgumentVar0 = inputVar;
MemoryTests.InputVar0 = modelFunc.Inputs.First();
MemoryTests.OutputVar0 = modelFunc.Outputs[0];
MemoryTests.OutputVar = modelFunc.Output;
Variable outputVar = MemoryTests.OutputVar;
MemoryTests.Axis0 = outputVar.DynamicAxes.FirstOrDefault();
// Get shape data for the input variable
NDShape inputShape = inputVar.Shape;
int imageWidth = inputShape[0];
int imageHeight = inputShape[1];
int imageChannels = inputShape[2];
int imageSize = inputShape.TotalSize;
var inputDataMap = new Dictionary <Variable, Value>();
var outputDataMap = new Dictionary <Variable, Value>();
var imageList = new List <string>()
{
"00000.png", "00001.png", "00002.png"
};
foreach (var image in imageList)
{
CNTKLibraryManagedExamples.ThrowIfFileNotExist(image, string.Format("Error: The sample image '{0}' does not exist. Please see README.md in <CNTK>/Examples/Image/DataSets/CIFAR-10 about how to download the CIFAR-10 dataset.", image));
}
Bitmap bmp, resized;
List <float> resizedCHW;
var seqData = new List <float>();
for (int sampleIndex = 0; sampleIndex < imageList.Count; sampleIndex++)
{
bmp = new Bitmap(Bitmap.FromFile(imageList[sampleIndex]));
resized = bmp.Resize((int)imageWidth, (int)imageHeight, true);
resizedCHW = resized.ParallelExtractCHW();
seqData.AddRange(resizedCHW);
}
var inputVal = Value.CreateBatch(inputVar.Shape, seqData, device);
inputDataMap.Add(inputVar, inputVal);
outputDataMap.Add(outputVar, null);
modelFunc.Evaluate(inputDataMap, outputDataMap, device);
var outputVal = outputDataMap[outputVar];
var outputData = outputVal.GetDenseData <float>(outputVar);
MemoryTests.OutputVal = outputVal;
Console.WriteLine("\nTest object reference inside SetupUsingResetModel.\n");
MemoryTests.WriteOutputs();
}
catch (Exception ex)
{
Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
throw ex;
}
}
public void SetupUsingResetModel(DeviceDescriptor device)
{
try
{
Console.WriteLine("\n===== Setup memory tests using Resnet Model =====");
var deviceList = DeviceDescriptor.AllDevices();
MemoryTests.Device0 = deviceList[0];
// Load the model.
string modelFilePath = "resnet20.dnn";
CNTKLibraryManagedExamples.ThrowIfFileNotExist(modelFilePath, string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/Classification/ResNet to create the model.", modelFilePath));
Function modelFunc = Function.Load(modelFilePath, device);
Variable inputVar = modelFunc.Arguments.Single();
MemoryTests.ArgumentVar0 = inputVar;
MemoryTests.InputVar0 = modelFunc.Inputs.First();
MemoryTests.OutputVar0 = modelFunc.Outputs[0];
MemoryTests.OutputVar = modelFunc.Output;
Variable outputVar = MemoryTests.OutputVar;
MemoryTests.Axis0 = outputVar.DynamicAxes.FirstOrDefault();
// Get shape data for the input variable
NDShape inputShape = inputVar.Shape;
int imageWidth = inputShape[0];
int imageHeight = inputShape[1];
int imageChannels = inputShape[2];
int imageSize = inputShape.TotalSize;
var imageList = new List <string>()
{
"00000.png", "00001.png", "00002.png"
};
foreach (var image in imageList)
{
CNTKLibraryManagedExamples.ThrowIfFileNotExist(image, string.Format("Error: The sample image '{0}' does not exist. Please see README.md in <CNTK>/Examples/Image/DataSets/CIFAR-10 about how to download the CIFAR-10 dataset.", image));
}
Bitmap bmp, resized;
List <float> resizedCHW;
var seqData1 = new List <float>();
var seqData2 = new List <float>();
for (int sampleIndex = 0; sampleIndex < imageList.Count; sampleIndex++)
{
bmp = new Bitmap(Bitmap.FromFile(imageList[sampleIndex]));
resized = bmp.Resize((int)imageWidth, (int)imageHeight, true);
resizedCHW = resized.ParallelExtractCHW();
if (sampleIndex < imageList.Count - 1)
{
seqData1.AddRange(resizedCHW);
}
seqData2.AddRange(resizedCHW);
}
var inputDataMap1 = new Dictionary <Variable, Value>();
var outputDataMap1 = new Dictionary <Variable, Value>();
var inputVal1 = Value.CreateBatch(inputVar.Shape, seqData1, device);
inputDataMap1.Add(inputVar, inputVal1);
outputDataMap1.Add(outputVar, null);
// Using temprary Value object returned by Evaluate().
modelFunc.Evaluate(inputDataMap1, outputDataMap1, device);
var outputVal1 = outputDataMap1[outputVar];
var outputData1 = outputVal1.GetDenseData <float>(outputVar);
// Using cloned persistent Value object returned by Evaluate().
var outputDataMap1WithClone = new Dictionary <Variable, Value>();
outputDataMap1WithClone.Add(outputVar, null);
modelFunc.Evaluate(inputDataMap1, outputDataMap1WithClone, true, device);
// Using temprary Value object which overwrites the one returned by the previous Evaluate().
var inputDataMap2 = new Dictionary <Variable, Value>();
var outputDataMap2 = new Dictionary <Variable, Value>();
var inputVal2 = Value.CreateBatch(inputVar.Shape, seqData2, device);
inputDataMap2.Add(inputVar, inputVal2);
outputDataMap2.Add(outputVar, null);
modelFunc.Evaluate(inputDataMap2, outputDataMap2, device);
// Test access to the persistent Value object, which should be still valid.
var outputVal1WithClone = outputDataMap1WithClone[outputVar];
var outputData1WithClone = outputVal1WithClone.GetDenseData <float>(outputVar);
// Test access to the temprary Value object returned by the latest Evaluate().
var outputVal2 = outputDataMap2[outputVar];
var outputData2 = outputVal2.GetDenseData <float>(outputVar);
// Test access to the temprary Value object returned by the previous Evaluate(), which is not valid any more.
bool exceptionCaught = false;
try
{
var data = outputVal1.GetDenseData <float>(outputVar);
}
catch (Exception ex)
{
if (ex is ApplicationException && ex.Message.StartsWith("This Value object is invalid and can no longer be accessed."))
{
exceptionCaught = true;
}
}
if (exceptionCaught == false)
{
throw new ApplicationException("The expected exception has not been caught.");
}
MemoryTests.OutputVal = outputVal1WithClone;
Console.WriteLine("\nTest object reference inside SetupUsingResetModel.\n");
MemoryTests.WriteOutputs();
}
catch (Exception ex)
{
Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
throw ex;
}
}