Esempio n. 1
0
        // Run benchmark - actually run benchmark the specified number of times and using the specified
        // controls and return results.

        public Results RunBenchmark(Benchmark benchmark)
        {
            bool doRun = Controls.DoRun;
            int numberOfRuns = Controls.NumberOfRunsPerBenchmark;
            bool doWarmUpRun = Controls.DoWarmUpRun;
            string runner = Controls.Runner;
            bool doDebugBenchmark = Controls.DoDebugBenchmark;
            bool doVerbose = Controls.DoVerbose;
            string complusVersion = Controls.ComplusVersion;
            string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
            string benchmarkDirectory = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory);
            bool doRunInShell = benchmark.DoRunInShell;
            bool useSSE = benchmark.UseSSE;
            bool useAVX = benchmark.UseAVX;
            int expectedResults = benchmark.ExpectedResults;
            int failureResults = ~expectedResults;
            int actualResults;
            int failures = 0;

            Results results = new Results(benchmark, numberOfRuns);

            if (!doRun)
            {
                return results;
            }

            string workingDirectory = benchmarkDirectory;
            string fileName = System.IO.Path.Combine(workingDirectory, benchmark.ExeName);
            string args = benchmark.ExeArgs;

            if (runner != "")
            {
                args = fileName + " " + args;
                fileName = runner;
            }

            if (doDebugBenchmark)
            {
                args = "/debugexe " + fileName + " " + args;
                fileName = "devenv.exe";
            }

            if (doRunInShell)
            {
                args = "/C " + fileName + " " + args;
                fileName = "cmd.exe";
            }

            if (doVerbose)
            {
                Console.WriteLine("Running benchmark {0} ...", benchmark.Name);
                Console.WriteLine("Invoking: {0} {1}", fileName, args);
            }

            for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++)
            {
                ProcessStartInfo startInfo = new ProcessStartInfo()
                {
                    FileName = fileName,
                    Arguments = args,
                    WorkingDirectory = workingDirectory,
                    UseShellExecute = false
                };

                if (complusVersion != "")
                {
                    startInfo.Environment["COMPlus_Version"] = complusVersion;
                    startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion;
                }
                if (useSSE)
                {
                    startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
                    startInfo.Environment["COMPlus_EnableAVX"] = "0";
                }
                if (useAVX)
                {
                    startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
                    startInfo.Environment["COMPlus_EnableAVX"] = "1";
                }
                startInfo.Environment["COMPlus_gcConcurrent"] = "0";
                startInfo.Environment["COMPlus_gcServer"] = "0";
                startInfo.Environment["COMPlus_NoGuiOnAssert"] = "1";
                startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0";

                var clockTime = Stopwatch.StartNew();
                int exitCode = 0;
                try
                {
                    using (var proc = Process.Start(startInfo))
                    {
                        proc.EnableRaisingEvents = true;
                        proc.WaitForExit();
                        exitCode = proc.ExitCode;
                    }
                    this.NumberOfBenchmarksRun++;
                }
                catch (Exception exception)
                {
                    Console.WriteLine("Could not launch test {0} exception: {1}",
                        startInfo.FileName, exception);
                    exitCode = failureResults;
                }
                clockTime.Stop();
                actualResults = exitCode;

                long time = clockTime.ElapsedMilliseconds;
                if (actualResults != expectedResults)
                {
                    failures++;
                    time = 0;
                }
                results.Times[run] = time;

                if (doVerbose)
                {
                    Console.Write("Iteration benchmark {0} ", benchmark.Name);
                    if (actualResults == expectedResults)
                    {
                        Console.Write("elapsed time {0}ms", time);
                    }
                    else
                    {
                        Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults);
                    }
                    if (run == 0)
                    {
                        Console.Write(" (warmup)");
                    }
                    Console.WriteLine("");
                }
            }

            // Calculate min, max, avg, and std devation.

            long sum = 0;
            long minimum = results.Times[1];
            long maximum = minimum;

            for (int run = 1; run <= numberOfRuns; run++)
            {
                long time = results.Times[run];

                sum += time;
                minimum = Math.Min(minimum, time);
                maximum = Math.Max(maximum, time);
            }

            long average = sum / (long)numberOfRuns;
            double standardDeviation = 0.0;

            if (numberOfRuns > 1)
            {
                double s = 0.0;
                double a = (double)average;
                double n = (double)numberOfRuns;
                for (int run = 1; run <= numberOfRuns; run++)
                {
                    double time = (double)results.Times[run];
                    double t = (time - a);
                    s += (t * t);
                }
                double variance = s / n;

                if (a == 0.0)
                {
                    standardDeviation = 0.0;
                }
                else
                {
                    standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage
                    standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero);
                }
            }

            // Record results and return.

            results.Average = average;
            results.Minimum = minimum;
            results.Maximum = maximum;
            results.StandardDeviation = standardDeviation;
            results.Failures = failures;

            return results;
        }
Esempio n. 2
0
 public Results
 (
     Benchmark benchmark,
     int numberOfRuns
 )
 {
     Benchmark = benchmark;
     Times = new long[numberOfRuns + 1]; // leave empty slot at index 0, not used.
     Minimum = 0;
     Maximum = 0;
     Average = 0;
     StandardDeviation = 0.0;
     Failures = 0;
 }
Esempio n. 3
0
        // Add a benchmark to the list of benchmarks read in from the .XML file.

        public void AddBenchmark
        (
            string name,
            string suiteName,
            string tags,
            string workingDirectory,
            string exeName,
            string exeArgs,
            bool doRunInShell,
            bool useSSE,
            bool useAVX,
            int expectedResults
        )
        {
            BenchmarkSuite benchmarkSuite;
            BenchmarkTagSet benchmarkTagSet;
            Benchmark benchmark;

            benchmark = new Benchmark(name, suiteName, tags,
                workingDirectory, exeName, exeArgs, doRunInShell, useSSE, useAVX, expectedResults);
            BenchmarkList.Add(benchmark);

            if (!BenchmarkSuiteTable.TryGetValue(suiteName, out benchmarkSuite))
            {
                benchmarkSuite = new BenchmarkSuite(suiteName);
                BenchmarkSuiteTable.Add(suiteName, benchmarkSuite);
            }
            benchmarkSuite.BenchmarkList.Add(benchmark);

            string[] tagList = tags.Split(ListSeparatorCharSet, StringSplitOptions.RemoveEmptyEntries);
            foreach (string tag in tagList)
            {
                if (!BenchmarkTagSetTable.TryGetValue(tag, out benchmarkTagSet))
                {
                    benchmarkTagSet = new BenchmarkTagSet(tag);
                    BenchmarkTagSetTable.Add(tag, benchmarkTagSet);
                }
                benchmarkTagSet.BenchmarkList.Add(benchmark);
            }
        }
Esempio n. 4
0
        // Run benchmark - actually run benchmark the specified number of times and using the specified
        // controls and return results.

        public Results RunBenchmark(Benchmark benchmark)
        {
            bool   doRun                   = Controls.DoRun;
            int    numberOfRuns            = Controls.NumberOfRunsPerBenchmark;
            bool   doWarmUpRun             = Controls.DoWarmUpRun;
            string runner                  = Controls.Runner;
            bool   doDebugBenchmark        = Controls.DoDebugBenchmark;
            bool   doVerbose               = Controls.DoVerbose;
            string complusVersion          = Controls.ComplusVersion;
            string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
            string benchmarkDirectory      = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory);
            bool   doRunInShell            = benchmark.DoRunInShell;
            bool   useSSE                  = benchmark.UseSSE;
            bool   useAVX                  = benchmark.UseAVX;
            int    expectedResults         = benchmark.ExpectedResults;
            int    failureResults          = ~expectedResults;
            int    actualResults;
            int    failures = 0;

            Results results = new Results(benchmark, numberOfRuns);

            if (!doRun)
            {
                return(results);
            }

            string workingDirectory = benchmarkDirectory;
            string fileName         = System.IO.Path.Combine(workingDirectory, benchmark.ExeName);
            string args             = benchmark.ExeArgs;

            if (runner != "")
            {
                args     = fileName + " " + args;
                fileName = runner;
            }

            if (doDebugBenchmark)
            {
                args     = "/debugexe " + fileName + " " + args;
                fileName = "devenv.exe";
            }

            if (doRunInShell)
            {
                args     = "/C " + fileName + " " + args;
                fileName = "cmd.exe";
            }

            if (doVerbose)
            {
                Console.WriteLine("Running benchmark {0} ...", benchmark.Name);
                Console.WriteLine("Invoking: {0} {1}", fileName, args);
            }

            for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++)
            {
                ProcessStartInfo startInfo = new ProcessStartInfo()
                {
                    FileName         = fileName,
                    Arguments        = args,
                    WorkingDirectory = workingDirectory,
                    UseShellExecute  = false
                };

                if (complusVersion != "")
                {
                    startInfo.Environment["COMPlus_Version"]        = complusVersion;
                    startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion;
                }
                if (useSSE)
                {
                    startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
                    startInfo.Environment["COMPlus_EnableAVX"]   = "0";
                }
                if (useAVX)
                {
                    startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
                    startInfo.Environment["COMPlus_EnableAVX"]   = "1";
                }
                startInfo.Environment["COMPlus_gcConcurrent"]             = "0";
                startInfo.Environment["COMPlus_gcServer"]                 = "0";
                startInfo.Environment["COMPlus_NoGuiOnAssert"]            = "1";
                startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0";

                var clockTime = Stopwatch.StartNew();
                int exitCode  = 0;
                try
                {
                    using (var proc = Process.Start(startInfo))
                    {
                        proc.EnableRaisingEvents = true;
                        proc.WaitForExit();
                        exitCode = proc.ExitCode;
                    }
                    this.NumberOfBenchmarksRun++;
                }
                catch (Exception exception)
                {
                    Console.WriteLine("Could not launch test {0} exception: {1}",
                                      startInfo.FileName, exception);
                    exitCode = failureResults;
                }
                clockTime.Stop();
                actualResults = exitCode;

                long time = clockTime.ElapsedMilliseconds;
                if (actualResults != expectedResults)
                {
                    failures++;
                    time = 0;
                }
                results.Times[run] = time;

                if (doVerbose)
                {
                    Console.Write("Iteration benchmark {0} ", benchmark.Name);
                    if (actualResults == expectedResults)
                    {
                        Console.Write("elapsed time {0}ms", time);
                    }
                    else
                    {
                        Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults);
                    }
                    if (run == 0)
                    {
                        Console.Write(" (warmup)");
                    }
                    Console.WriteLine("");
                }
            }

            // Calculate min, max, avg, and std devation.

            long sum     = 0;
            long minimum = results.Times[1];
            long maximum = minimum;

            for (int run = 1; run <= numberOfRuns; run++)
            {
                long time = results.Times[run];

                sum    += time;
                minimum = Math.Min(minimum, time);
                maximum = Math.Max(maximum, time);
            }

            long   average           = sum / (long)numberOfRuns;
            double standardDeviation = 0.0;

            if (numberOfRuns > 1)
            {
                double s = 0.0;
                double a = (double)average;
                double n = (double)numberOfRuns;
                for (int run = 1; run <= numberOfRuns; run++)
                {
                    double time = (double)results.Times[run];
                    double t    = (time - a);
                    s += (t * t);
                }
                double variance = s / n;

                if (a == 0.0)
                {
                    standardDeviation = 0.0;
                }
                else
                {
                    standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage
                    standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero);
                }
            }

            // Record results and return.

            results.Average           = average;
            results.Minimum           = minimum;
            results.Maximum           = maximum;
            results.StandardDeviation = standardDeviation;
            results.Failures          = failures;

            return(results);
        }