Beispiel #1
0
        static void Main(string[] args)
        {
#if CPUONLY
            Console.WriteLine("======== Evaluate model on CPU using CPUOnly build ========");
#else
            Console.WriteLine("======== Evaluate model on CPU using GPU build ========");
#endif

            if (ShouldRunOnCpu())
            {
                var device = DeviceDescriptor.CPUDevice;

                CNTKLibraryManagedExamples.EvaluationSingleImage(device);
                // Run memory tests.
                MemoryTests.ValidateObjectReferences(device);
                CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);
                CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallel(device);
                // Run memory tests again.
                MemoryTests.ValidateObjectReferences(device);

                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);
                // It is sufficient to test loading model from memory buffer only on CPU.
                CNTKLibraryManagedExamples.LoadModelFromMemory(device);

                MemoryTests.WriteOutputs();
            }

            if (ShouldRunOnGpu())
            {
                Console.WriteLine(" ====== Evaluate model on GPU =====");
                var device = DeviceDescriptor.GPUDevice(0);
                // Run memory tests.
                MemoryTests.ValidateObjectReferences(device);
                CNTKLibraryManagedExamples.EvaluationSingleImage(device);
                CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);
                CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallel(device);
                // Run memory tests.
                MemoryTests.ValidateObjectReferences(device);

                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);

                // Run memory tests again.
                MemoryTests.WriteOutputs();
            }

            Console.WriteLine("======== Evaluation completes. ========");
        }
        static void Main(string[] args)
        {
            // Todo: move to a separate unit test.
            Console.WriteLine("Test Utils");

            int maxThreads = Utils.GetMaxNumCPUThreads();

            Utils.SetMaxNumCPUThreads(2);
            Console.WriteLine("MaxNumCPUThreads: before: " + maxThreads + ", after " + Utils.GetMaxNumCPUThreads());
            Utils.SetMaxNumCPUThreads(maxThreads);
            Console.WriteLine("reset MaxNumCPuThreads to " + Utils.GetMaxNumCPUThreads());

            var level = Utils.GetTraceLevel();

            Utils.SetTraceLevel(TraceLevel.Info);
            Console.WriteLine("TraceLevel: before: " + level + ", after " + Utils.GetTraceLevel());
            Utils.SetTraceLevel(level);
            Console.WriteLine("reset TraceLevel to " + Utils.GetTraceLevel());

            Console.WriteLine(Utils.DataTypeName(DataType.Float));
            Console.WriteLine(Utils.DataTypeSize(DataType.Double));
            Console.WriteLine(Utils.DeviceKindName(DeviceDescriptor.CPUDevice.Type));
            Console.WriteLine(Utils.DeviceKindName(DeviceKind.GPU));
            Console.WriteLine(Utils.IsSparseStorageFormat(StorageFormat.Dense));
            Console.WriteLine(Utils.IsSparseStorageFormat(StorageFormat.SparseCSC));
            Console.WriteLine(Utils.IsSparseStorageFormat(StorageFormat.SparseBlockCol));
            Console.WriteLine(Utils.VariableKindName(VariableKind.Constant));
            Console.WriteLine(Utils.VariableKindName(VariableKind.Placeholder));
            Console.WriteLine(Utils.VariableKindName(VariableKind.Input));
            Console.WriteLine(Utils.VariableKindName(VariableKind.Output));
            Console.WriteLine(Utils.VariableKindName(VariableKind.Parameter));

#if CPUONLY
            Console.WriteLine("======== Evaluate model on CPU using CPUOnly build ========");
#else
            Console.WriteLine("======== Evaluate model on CPU using GPU build ========");
#endif
            if (args.Length >= 1)
            {
                Console.WriteLine($"-------- running with test data in {args[0]} --------");
                CNTKLibraryManagedExamples.ExampleTestDataDir = args[0];
            }
            CNTKLibraryManagedExamples.Setup();

            if (ShouldRunOnCpu())
            {
                var device = DeviceDescriptor.CPUDevice;

                CNTKLibraryManagedExamples.EvaluationSingleImage(device);

                // Run memory tests.
                MemoryTests.ValidateObjectReferences(device);

                CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);

                MemoryTests.WriteOutputs();
                CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallelAsync(device).Wait();

                // Run memory tests again.
                MemoryTests.ValidateObjectReferences(device);

                Task evalTask = CNTKLibraryManagedExamples.EvaluationSingleImageAsync(device);
                evalTask.Wait();

                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);
                // It is sufficient to test loading model from memory buffer only on CPU.
                CNTKLibraryManagedExamples.LoadModelFromMemory(device);

                MemoryTests.WriteOutputs();

                MemoryTests.ValueCopyToSparseCSCTest <float>(device);
                MemoryTests.ValueCopyToSparseCSCTest <double>(device);

                CNTKLibraryManagedExamples.EvaluateIntermediateLayer(device);
                CNTKLibraryManagedExamples.EvaluateCombinedOutputs(device);
            }

            if (ShouldRunOnGpu())
            {
                Console.WriteLine(" ====== Evaluate model on GPU =====");
                var device = DeviceDescriptor.GPUDevice(0);
                // Run memory tests.
                MemoryTests.ValidateObjectReferences(device);
                CNTKLibraryManagedExamples.EvaluationSingleImage(device);
                CNTKLibraryManagedExamples.EvaluationBatchOfImages(device);
                CNTKLibraryManagedExamples.EvaluateMultipleImagesInParallelAsync(device).Wait();
                // Run memory tests.
                MemoryTests.ValidateObjectReferences(device);

                Task evalTask = CNTKLibraryManagedExamples.EvaluationSingleImageAsync(device);
                evalTask.Wait();

                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationBatchOfSequencesUsingOneHot(device);
                CNTKLibraryManagedExamples.EvaluationSingleSequenceUsingSparse(device);

                // Run memory tests again.
                MemoryTests.WriteOutputs();

                MemoryTests.ValueCopyToSparseCSCTest <float>(device);
                MemoryTests.ValueCopyToSparseCSCTest <double>(device);

                CNTKLibraryManagedExamples.EvaluateIntermediateLayer(device);
                CNTKLibraryManagedExamples.EvaluateCombinedOutputs(device);
            }

            Console.WriteLine("======== Evaluation completes. ========");
        }
Beispiel #3
0
        public void SetupUsingResetModel(DeviceDescriptor device)
        {
            try
            {
                Console.WriteLine("\n===== Setup memory tests using Resnet Model =====");

                var deviceList = DeviceDescriptor.AllDevices();
                MemoryTests.Device0 = deviceList[0];

                // Load the model.
                string modelFilePath = "resnet20.dnn";
                CNTKLibraryManagedExamples.ThrowIfFileNotExist(modelFilePath, string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/Classification/ResNet to create the model.", modelFilePath));
                Function modelFunc = Function.LoadModel(modelFilePath, device);

                Variable inputVar = modelFunc.Arguments.Single();
                MemoryTests.ArgumentVar0 = inputVar;
                MemoryTests.InputVar0    = modelFunc.Inputs.First();

                MemoryTests.OutputVar0 = modelFunc.Outputs[0];
                MemoryTests.OutputVar  = modelFunc.Output;
                Variable outputVar = MemoryTests.OutputVar;

                MemoryTests.Axis0 = outputVar.DynamicAxes.FirstOrDefault();

                // Get shape data for the input variable
                NDShape inputShape    = inputVar.Shape;
                int     imageWidth    = inputShape[0];
                int     imageHeight   = inputShape[1];
                int     imageChannels = inputShape[2];
                int     imageSize     = inputShape.TotalSize;
                var     inputDataMap  = new Dictionary <Variable, Value>();
                var     outputDataMap = new Dictionary <Variable, Value>();

                var imageList = new List <string>()
                {
                    "00000.png", "00001.png", "00002.png"
                };
                foreach (var image in imageList)
                {
                    CNTKLibraryManagedExamples.ThrowIfFileNotExist(image, string.Format("Error: The sample image '{0}' does not exist. Please see README.md in <CNTK>/Examples/Image/DataSets/CIFAR-10 about how to download the CIFAR-10 dataset.", image));
                }
                Bitmap       bmp, resized;
                List <float> resizedCHW;
                var          seqData = new List <float>();
                for (int sampleIndex = 0; sampleIndex < imageList.Count; sampleIndex++)
                {
                    bmp        = new Bitmap(Bitmap.FromFile(imageList[sampleIndex]));
                    resized    = bmp.Resize((int)imageWidth, (int)imageHeight, true);
                    resizedCHW = resized.ParallelExtractCHW();
                    seqData.AddRange(resizedCHW);
                }

                var inputVal = Value.CreateBatch(inputVar.Shape, seqData, device);
                inputDataMap.Add(inputVar, inputVal);
                outputDataMap.Add(outputVar, null);
                modelFunc.Evaluate(inputDataMap, outputDataMap, device);

                var outputVal  = outputDataMap[outputVar];
                var outputData = outputVal.GetDenseData <float>(outputVar);

                MemoryTests.OutputVal = outputVal;

                Console.WriteLine("\nTest object reference inside SetupUsingResetModel.\n");
                MemoryTests.WriteOutputs();
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
                throw ex;
            }
        }
        public void SetupUsingResetModel(DeviceDescriptor device)
        {
            try
            {
                Console.WriteLine("\n===== Setup memory tests using Resnet Model =====");

                var deviceList = DeviceDescriptor.AllDevices();
                MemoryTests.Device0 = deviceList[0];

                // Load the model.
                string modelFilePath = "resnet20.dnn";
                CNTKLibraryManagedExamples.ThrowIfFileNotExist(modelFilePath, string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/Classification/ResNet to create the model.", modelFilePath));
                Function modelFunc = Function.Load(modelFilePath, device);

                Variable inputVar = modelFunc.Arguments.Single();
                MemoryTests.ArgumentVar0 = inputVar;
                MemoryTests.InputVar0    = modelFunc.Inputs.First();

                MemoryTests.OutputVar0 = modelFunc.Outputs[0];
                MemoryTests.OutputVar  = modelFunc.Output;
                Variable outputVar = MemoryTests.OutputVar;

                MemoryTests.Axis0 = outputVar.DynamicAxes.FirstOrDefault();

                // Get shape data for the input variable
                NDShape inputShape    = inputVar.Shape;
                int     imageWidth    = inputShape[0];
                int     imageHeight   = inputShape[1];
                int     imageChannels = inputShape[2];
                int     imageSize     = inputShape.TotalSize;

                var imageList = new List <string>()
                {
                    "00000.png", "00001.png", "00002.png"
                };
                foreach (var image in imageList)
                {
                    CNTKLibraryManagedExamples.ThrowIfFileNotExist(image, string.Format("Error: The sample image '{0}' does not exist. Please see README.md in <CNTK>/Examples/Image/DataSets/CIFAR-10 about how to download the CIFAR-10 dataset.", image));
                }
                Bitmap       bmp, resized;
                List <float> resizedCHW;
                var          seqData1 = new List <float>();
                var          seqData2 = new List <float>();
                for (int sampleIndex = 0; sampleIndex < imageList.Count; sampleIndex++)
                {
                    bmp        = new Bitmap(Bitmap.FromFile(imageList[sampleIndex]));
                    resized    = bmp.Resize((int)imageWidth, (int)imageHeight, true);
                    resizedCHW = resized.ParallelExtractCHW();
                    if (sampleIndex < imageList.Count - 1)
                    {
                        seqData1.AddRange(resizedCHW);
                    }
                    seqData2.AddRange(resizedCHW);
                }

                var inputDataMap1  = new Dictionary <Variable, Value>();
                var outputDataMap1 = new Dictionary <Variable, Value>();
                var inputVal1      = Value.CreateBatch(inputVar.Shape, seqData1, device);
                inputDataMap1.Add(inputVar, inputVal1);
                outputDataMap1.Add(outputVar, null);

                // Using temprary Value object returned by Evaluate().
                modelFunc.Evaluate(inputDataMap1, outputDataMap1, device);
                var outputVal1  = outputDataMap1[outputVar];
                var outputData1 = outputVal1.GetDenseData <float>(outputVar);

                // Using cloned persistent Value object returned by Evaluate().
                var outputDataMap1WithClone = new Dictionary <Variable, Value>();
                outputDataMap1WithClone.Add(outputVar, null);
                modelFunc.Evaluate(inputDataMap1, outputDataMap1WithClone, true, device);

                // Using temprary Value object which overwrites the one returned by the previous Evaluate().
                var inputDataMap2  = new Dictionary <Variable, Value>();
                var outputDataMap2 = new Dictionary <Variable, Value>();
                var inputVal2      = Value.CreateBatch(inputVar.Shape, seqData2, device);
                inputDataMap2.Add(inputVar, inputVal2);
                outputDataMap2.Add(outputVar, null);
                modelFunc.Evaluate(inputDataMap2, outputDataMap2, device);

                // Test access to the persistent Value object, which should be still valid.
                var outputVal1WithClone  = outputDataMap1WithClone[outputVar];
                var outputData1WithClone = outputVal1WithClone.GetDenseData <float>(outputVar);

                // Test access to the temprary Value object returned by the latest Evaluate().
                var outputVal2  = outputDataMap2[outputVar];
                var outputData2 = outputVal2.GetDenseData <float>(outputVar);

                // Test access to the temprary Value object returned by the previous Evaluate(), which is not valid any more.
                bool exceptionCaught = false;
                try
                {
                    var data = outputVal1.GetDenseData <float>(outputVar);
                }
                catch (Exception ex)
                {
                    if (ex is ApplicationException && ex.Message.StartsWith("This Value object is invalid and can no longer be accessed."))
                    {
                        exceptionCaught = true;
                    }
                }
                if (exceptionCaught == false)
                {
                    throw new ApplicationException("The expected exception has not been caught.");
                }

                MemoryTests.OutputVal = outputVal1WithClone;

                Console.WriteLine("\nTest object reference inside SetupUsingResetModel.\n");
                MemoryTests.WriteOutputs();
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
                throw ex;
            }
        }