Esempio n. 1
0
        private void WriteBenchmarkIntro(StreamWriter textFile, Summary summary, BenchmarkInfo info)
        {
            var benchmark = summary.Benchmarks.First(b => b.Target.Type.Name == info.ClassName);

            var property = benchmark.Target.Type.GetTypeInfo().DeclaredProperties.FirstOrDefault(p => p.Name == "Description");

            var description = (string)property?.GetValue(null) ?? "";

            if (info.Parameters != null && info.Parameters.Count > 0)
            {
                foreach (var parameter in info.Parameters)
                {
                    var replaceName = "{" + parameter.Name + "}";

                    description = description.Replace(replaceName, parameter.Value?.ToString());
                }
            }

            var link = benchmark.Target.Type.FullName.Replace(nameof(DotNet) + "." + nameof(DependencyInjectionBenchmarks), "");

            link = link.Replace('.', '/');

            link = CodePath + link + ".cs";

            textFile.WriteLine("<div class=\"jumbotron\">");
            textFile.WriteLine($"<h1>{info.DisplayName}</h1>");
            textFile.WriteLine($"<p>{description} The code can be found <a href=\"{link}\">here</a>.</p>");
            textFile.WriteLine("</div>");
        }
Esempio n. 2
0
        public void setResult(PredictorInfo predictor, BenchmarkInfo benchmark, BenchmarkStatisticsResult value)
        {
            int predictorIndex;
            int benchmarkIndex;

            if (!predictorIndexMap.TryGetValue(predictor, out predictorIndex))
            {
                throw new Exception("Invalid predictor provided.");
            }
            if (!benchmarkIndexMap.TryGetValue(benchmark, out benchmarkIndex))
            {
                throw new Exception("Invalid simulator provided.");
            }
            if ((value != null) && (resultMatrix[predictorIndex, benchmarkIndex] == null))
            {
                valuesEntered++;
                observableCollections[predictor].addSorted(value);
                if (newValueReceived != null)
                {
                    newValueReceived(this, new SimulationResultsDictionaryNewValueReceivedEventArgs(predictorIndex, benchmarkIndex, value, observableCollections[predictor]));
                }
            }
            else
            {
                throw new NotSupportedException();
            }

            resultMatrix[predictorIndex, benchmarkIndex] = value;

            if ((valuesEntered == numberOfPredictors * numberOfBenchmarks) && (filled != null))
            {
                filled(this, EventArgs.Empty);
            }
        }
Esempio n. 3
0
 public void addBenchmark(BenchmarkInfo benchmark)
 {
     try
     {
         benchmarkIndexMap.Add(benchmark, numberOfBenchmarks++);
     }
     catch (Exception e)
     {
     }
 }
        public IEnumerable <BenchmarkInfo> GetBenchmarksByIsoCountryCode(String isoCode)
        {
            IEnumerable <BenchmarkInfo> found;

            if (this.benchmarkByCountry.Contains(isoCode))
            {
                found = this.benchmarkByCountry[isoCode];
            }
            else
            {
                found = new BenchmarkInfo[] { };
            }
            return(found);
        }
        private void Run(BenchmarkInfo benchmarkInfo)
        {
            switch (Stage)
            {
            case StageEnum.Build:
                benchmarkInfo.BuildSchema();
                break;

            case StageEnum.TypicalExecution:
                _documentExecuter.ExecuteAsync(o =>
                {
                    o.Schema = benchmarkInfo.Schema;
                    o.Query  = benchmarkInfo.Query;
                    o.Inputs = benchmarkInfo.InputsString?.ToInputs();
                }).GetAwaiter().GetResult();
                break;

            case StageEnum.Parse:
                benchmarkInfo.Parse();
                break;

            case StageEnum.Convert:
                benchmarkInfo.Convert();
                break;

            case StageEnum.Validate:
                benchmarkInfo.Validate();
                break;

            case StageEnum.DeserializeVars:
                benchmarkInfo.DeserializeInputs();
                break;

            case StageEnum.ParseVariables:
                benchmarkInfo.ParseVariables();
                break;

            case StageEnum.Execute:
                benchmarkInfo.Execute();
                break;

            case StageEnum.Serialize:
                benchmarkInfo.Serialize();
                break;

            default:
                throw new InvalidOperationException();
            }
        }
Esempio n. 6
0
        public BenchmarkStatisticsResult getResult(PredictorInfo predictor, BenchmarkInfo benchmark)
        {
            int predictorIndex;
            int benchmarkIndex;

            if (!predictorIndexMap.TryGetValue(predictor, out predictorIndex))
            {
                throw new Exception("Invalid predictor provided.");
            }
            if (!benchmarkIndexMap.TryGetValue(benchmark, out benchmarkIndex))
            {
                throw new Exception("Invalid simulator provided.");
            }
            return(resultMatrix[predictorIndex, benchmarkIndex]);
        }
        public void GlobalSetup()
        {
            Func <ISchema> starWarsSchemaBuilder = () =>
            {
                var services = new ServiceCollection();

                services.AddSingleton <StarWarsData>();
                services.AddSingleton <StarWarsQuery>();
                services.AddSingleton <StarWarsMutation>();
                services.AddSingleton <HumanType>();
                services.AddSingleton <HumanInputType>();
                services.AddSingleton <DroidType>();
                services.AddSingleton <CharacterInterface>();
                services.AddSingleton <EpisodeEnum>();
                services.AddSingleton <ISchema, StarWarsSchema>();

                var provider = services.BuildServiceProvider();
                var schema   = provider.GetRequiredService <ISchema>();
                schema.Initialize();
                return(schema);
            };

            _bIntrospection = new BenchmarkInfo(Queries.Introspection, null, starWarsSchemaBuilder);
            _bHero          = new BenchmarkInfo(Queries.Hero, null, starWarsSchemaBuilder);

            Func <ISchema> variableSchemaBuilder = () =>
            {
                var services = new ServiceCollection();

                services.AddSingleton <VariableBenchmark.MyQueryGraphType>();
                services.AddSingleton <VariableBenchmark.MyInputObjectGraphType>();
                services.AddSingleton <VariableBenchmark.MySubInputObjectGraphType>();
                services.AddSingleton <ISchema, VariableBenchmark.MySchema>();

                var provider = services.BuildServiceProvider();
                var schema   = provider.GetRequiredService <ISchema>();
                schema.Initialize();
                return(schema);
            };

            _bVariable = new BenchmarkInfo(Queries.VariablesVariable, Benchmarks.Variables.VariablesVariable, variableSchemaBuilder);
            _bLiteral  = new BenchmarkInfo(Queries.VariablesLiteral, null, variableSchemaBuilder);
        }
Esempio n. 8
0
        private void Run(BenchmarkInfo benchmarkInfo)
        {
            switch (Stage)
            {
            case StageEnum.Build:
                benchmarkInfo.BuildSchema();
                break;

            case StageEnum.Parse:
                benchmarkInfo.Parse();
                break;

            case StageEnum.Convert:
                benchmarkInfo.Convert();
                break;

            case StageEnum.Validate:
                benchmarkInfo.Validate();
                break;

            case StageEnum.DeserializeVars:
                benchmarkInfo.DeserializeInputs();
                break;

            case StageEnum.ParseVariables:
                benchmarkInfo.ParseVariables();
                break;

            case StageEnum.Execute:
                benchmarkInfo.Execute();
                break;

            case StageEnum.Serialize:
                benchmarkInfo.Serialize();
                break;

            default:
                throw new InvalidOperationException();
            }
        }
Esempio n. 9
0
 public SimulationInfo(PredictorInfo predictorInfo, string benchmarkName, BenchmarkType benchmarkType)
 {
     this.predictorInfo = predictorInfo;
     this.benchmarkInfo = new BenchmarkInfo(benchmarkName, benchmarkType);
 }
Esempio n. 10
0
 public SimulationInfo(PredictorInfo predictorInfo, BenchmarkInfo benchmarkInfo)
 {
     this.predictorInfo = predictorInfo;
     this.benchmarkInfo = benchmarkInfo;
 }
Esempio n. 11
0
        private void ExportBenchmarkPage(string path, Summary summary, SummaryInfo calculatedHeaders, BenchmarkInfo info)
        {
            using (var file = File.Create(Path.Combine(path, info.BenchmarkName.Replace('|', '_') + ".html")))
            {
                using (var textFile = new StreamWriter(file))
                {
                    textFile.WriteLine("<html>");
                    textFile.Indent(1);
                    textFile.WriteLine("<head>");
                    textFile.Indent(2);
                    textFile.Write(_headString);
                    textFile.Indent(2);
                    textFile.WriteLine("</head>");
                    textFile.Indent(1);
                    textFile.WriteLine("<body>");

                    WriteMenu(summary, calculatedHeaders, textFile);

                    textFile.Indent(2);
                    textFile.WriteLine("<div class=\"container-fluid theme-showcase\" role=\"main\">");

                    WriteBenchmarkIntro(textFile, summary, info);

                    WriteBenchmarkTableData(textFile, summary, info);

                    OutputSummaryDetails(summary, textFile);

                    var environmentSwitches = BuildEnvironmentSwitches(summary);

                    textFile.Write(_scriptString.Replace("{AppendEnvironments}", environmentSwitches));

                    textFile.Indent(1);
                    textFile.WriteLine("</div></body>");
                    textFile.WriteLine("</html>");
                }
            }
        }
Esempio n. 12
0
        private void WriteBenchmarkTableData(StreamWriter textFile, Summary summary, BenchmarkInfo info)
        {
            var reports = summary.Reports.Where(r => r.Benchmark.Target.Type.Name == info.ClassName);

            if (info.Parameters.Count > 0)
            {
                reports = reports.Where(r =>
                {
                    foreach (var parameter in info.Parameters)
                    {
                        if (!r.Benchmark.Parameters.Items.Any(
                                p => p.Name == parameter.Name && p.Value == parameter.Value))
                        {
                            return(false);
                        }
                    }

                    return(true);
                });
            }

            textFile.Indent(4);
            textFile.WriteLine("<table class=\"table table-striped table-bordered responsive nowrap\" id=\"benchmarkDataTable\" width=\"100%\" cellspacing=\"0\">");

            textFile.WriteLine("<thead>");
            textFile.WriteLine("<tr>");
            textFile.WriteLine("<th>Container</th>");
            textFile.WriteLine("<th>Env</th>");
            textFile.WriteLine("<th>Mean (ns)</th>");
            textFile.WriteLine("<th>Median (ns)</th>");
            textFile.WriteLine("<th>Max (ns)</th>");
            textFile.WriteLine("<th>Outliers</th>");
            textFile.WriteLine("<th>Std Dev (ns)</th>");
            textFile.WriteLine("<th>Std Err (ns)</th>");
            textFile.WriteLine("<th>Gen 1</th>");
            textFile.WriteLine("<th>Gen 2</th>");
            textFile.WriteLine("<th>Bytes Alloc</th>");
            textFile.WriteLine("</tr>");
            textFile.WriteLine("</thead>");

            textFile.WriteLine("<tbody>");
            foreach (var report in reports)
            {
                if (report.ResultStatistics != null)
                {
                    textFile.Write("<tr>");
                    textFile.Write($"<td>{CreateContainerHtml(report.Benchmark.Target.Method.Name)}</td>");
                    textFile.Write($"<td>{report.Benchmark.Job.ResolvedId}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Mean:F1}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Median:F1}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Max:F1}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Outliers?.Length ?? 0}</td>");
                    textFile.Write(
                        $"<td style= \"text-align: right\">{report.ResultStatistics.StandardDeviation:F3}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.StandardError:F3}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.GcStats.Gen1Collections}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.GcStats.Gen2Collections}</td>");
                    textFile.Write($"<td style=\"text-align: right\">{report.GcStats.BytesAllocatedPerOperation}</td>");
                    textFile.WriteLine("</tr>");
                }
            }

            textFile.WriteLine("</tbody>");

            textFile.Indent(4);
            textFile.WriteLine("</table>");
        }
Esempio n. 13
0
        public static BenchmarkStatisticsResult simulate(PredictorInfo predictorInfo, BenchmarkInfo benchmarkInfo, SimulationOptions simulationParameters, ApplicationOptions applicationOptions)
        {
            if (simulationParameters == null)
            {
                simulationParameters = SimulationOptions.defaultOptions;
            }

            string       folder      = "";
            ITraceReader traceReader = null;
            BenchmarkStatisticsResult currentResult = null;

            switch (benchmarkInfo.benchmarkType)
            {
            case BenchmarkType.Stanford:
                folder      = applicationOptions.TracePathStanford;
                traceReader = new StanfordReader();
                break;

            case BenchmarkType.SPEC2000:
                folder      = applicationOptions.TracePathSpec2000;
                traceReader = new Spec2000Reader();
                break;

            case BenchmarkType.CBP2:
                folder      = applicationOptions.TracePathCBP2;
                traceReader = new CBP2Reader();
                break;
            }

            // obtain predictor
            IPredictor predictor = predictorInfo.getPredictor();

            if (predictor == null)
            {
                return(new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName + " - Could not be performed: The predictor could not be instantiated on the client."));
            }

            // open the trace file
            if (!traceReader.openTrace(folder, benchmarkInfo.benchmarkName))
            {
                return(new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName + " - Could not be performed: The trace file could not be opened on the client."));
            }

            // create new entry
            currentResult = new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName);
            currentResult.NumberOfCorrectPredictions   = 0;
            currentResult.NumberOfIncorrectPredictions = 0;


            // check if we should skip N branches when counting the performance (to avoid accounting for the predictor warmup)
            int branchIndex            = 0;
            int numberOfBranchesToSkip = simulationParameters.NumberOfBranchesToSkip;

            while (true)
            {
                // get the next branch
                IBranch branch = traceReader.getNextBranch();

                // null means trace end
                if (branch == null)
                {
                    break;
                }

                BranchInfo currentJumpInfo = branch.getBranchInfo();
                if (!simulationParameters.ConditionalOnly || ((branch.getBranchInfo().branchFlags & BranchInfo.BR_CONDITIONAL) > 0))
                {
                    // predict
                    bool prediction = predictor.predictBranch(branch.getBranchInfo());

                    if (branchIndex >= numberOfBranchesToSkip)
                    {
                        if (branch.taken() == prediction)
                        {
                            currentResult.NumberOfCorrectPredictions++;
                        }
                        else
                        {
                            currentResult.NumberOfIncorrectPredictions++;
                        }
                    }

                    // update the predictor
                    predictor.update(branch);

                    branchIndex++;
                }
            }

            traceReader.closeTrace();

            currentResult.NumberOfBranches = currentResult.NumberOfCorrectPredictions + currentResult.NumberOfIncorrectPredictions;
            currentResult.Accuracy         = (double)currentResult.NumberOfCorrectPredictions / currentResult.NumberOfBranches;

            // free the predictor instance for future reusability
            predictorInfo.freePredictor(predictor);

            return(currentResult);
        }