public static RawResults Load(Context context, string path) { var ret = new RawResults(); var doc = new XmlDocument(); doc.Load(path); XmlElement root = doc.DocumentElement; ret.DateUTC = ReadTimeStamp(path, root); ret.Configuration = ReadConfiguration(path, root); ret.AndroidDevice = ReadDeviceInfo(path, root.SelectSingleNode("//device")); ReadGitInfo(path, ret, root.SelectSingleNode("//projectGitInfo")); ret.XAVersion = ReadXamarinAndroidVersion(path, root.SelectSingleNode("//xamarinAndroidVersion")); ret.SessionLogPath = ReadSessionLogPath(path, root.SelectSingleNode("//sessionLog")); XmlNode?runs = root.SelectSingleNode("//runs"); ret.RepetitionCount = ReadRepetitions(path, runs); foreach (XmlNode?runNode in runs.SelectNodes("./run")) { if (runNode == null) { continue; } var run = new RunDefinition(context); run.LoadRaw(runNode); ret.Runs.Add(run); } return(ret); }
public ReportAverages(RunDefinition run, uint repetitionCount, Action <List <RunResults> > sorter) { All = new RunResults(run); if (repetitionCount >= 2) { NoSlowest = new RunResults(run); if (repetitionCount >= 3) { NoOutliers = new RunResults(run); } } Calculate(run, sorter); }
public ComparisonData(RunDefinition before, RunDefinition after, ReportAverages beforeDisplayed, ReportAverages afterDisplayed, ReportAverages beforeNativeToManaged, ReportAverages afterNativeToManaged, ReportAverages beforeTotalInit, ReportAverages afterTotalInit) { Before = before; After = after; BeforeDisplayed = beforeDisplayed; AfterDisplayed = afterDisplayed; BeforeNativeToManaged = beforeNativeToManaged; AfterNativeToManaged = afterNativeToManaged; BeforeTotalInit = beforeTotalInit; AfterTotalInit = afterTotalInit; }
void Calculate(RunDefinition run, Action <List <RunResults> >?sorter) { bool haveOutliers = NoOutliers != null; bool haveNoSlowest = NoSlowest != null; var runResults = new List <RunResults> (run.Results); decimal count = runResults.Count; if ((haveOutliers || haveNoSlowest) && sorter != null) { sorter(runResults); } for (int i = 0; i < count; i++) { RunResults results = runResults [i]; if ((haveOutliers || haveNoSlowest) && i < runResults.Count - 1) { if (i > 0) { NoOutliers !.NativeToManaged += results.NativeToManaged / (count - 2); NoOutliers.TotalInit += results.TotalInit / (count - 2); NoOutliers.Displayed += results.Displayed / (count - 2); } NoSlowest !.NativeToManaged += results.NativeToManaged / (count - 1); NoSlowest.TotalInit += results.TotalInit / (count - 1); NoSlowest.Displayed += results.Displayed / (count - 1); } All.NativeToManaged += results.NativeToManaged / count; All.TotalInit += results.TotalInit / count; All.Displayed += results.Displayed / count; } }
public string Compare(Context context, string resultsDir, string reportOnePath, string reportTwoPath) { var reportOne = RawResults.Load(context, reportOnePath); var reportTwo = RawResults.Load(context, reportTwoPath); var warnings = new List <string> (); if (reportOne.RepetitionCount != reportTwo.RepetitionCount) { warnings.Add($"Reports were created based on different number of repetitions"); } if (reportOne.Runs.Count != reportTwo.Runs.Count) { warnings.Add($"Reports have a different number of runs ({reportOne.Runs.Count} and {reportTwo.Runs.Count}). Only the first {Math.Min (reportOne.Runs.Count, reportTwo.Runs.Count)} runs will be compared."); } var comparisons = new List <ComparisonData> (); for (int i = 0; i < Math.Min(reportOne.Runs.Count, reportTwo.Runs.Count); i++) { RunDefinition one = reportOne.Runs[i]; RunDefinition two = reportTwo.Runs[i]; bool argsDiffer = false; if (one.Args.Count != two.Args.Count) { argsDiffer = true; } else { var argsOne = new List <string> (one.Args); argsOne.Sort(); var argsTwo = new List <string> (two.Args); argsTwo.Sort(); for (int j = 0; j < argsOne.Count; j++) { if (String.Compare(argsOne[j].Trim(), argsTwo[j].Trim(), StringComparison.Ordinal) != 0) { argsDiffer = true; break; } } } if (argsDiffer) { warnings.Add($"Arguments for run {i} differ between the two reports"); } comparisons.Add( new ComparisonData( before: one, after: two, beforeDisplayed: new ReportAverages(one, reportOne.RepetitionCount, SortDisplayed), afterDisplayed: new ReportAverages(two, reportTwo.RepetitionCount, SortDisplayed), beforeNativeToManaged: new ReportAverages(one, reportOne.RepetitionCount, SortNativeToManaged), afterNativeToManaged: new ReportAverages(two, reportTwo.RepetitionCount, SortNativeToManaged), beforeTotalInit: new ReportAverages(one, reportOne.RepetitionCount, SortTotalInit), afterTotalInit: new ReportAverages(two, reportTwo.RepetitionCount, SortTotalInit) ) ); } var displayedLinesAll = new List <ReportLineComparison> (); var displayedLinesNoOutliers = new List <ReportLineComparison> (); var displayedLinesNoSlowest = new List <ReportLineComparison> (); var nativeToManagedLinesAll = new List <ReportLineComparison> (); var nativeToManagedLinesNoOutliers = new List <ReportLineComparison> (); var nativeToManagedLinesNoSlowest = new List <ReportLineComparison> (); var totalInitLinesAll = new List <ReportLineComparison> (); var totalInitLinesNoOutliers = new List <ReportLineComparison> (); var totalInitLinesNoSlowest = new List <ReportLineComparison> (); foreach (ComparisonData cdata in comparisons) { string notes; if (String.Compare(cdata.Before.Summary, cdata.After.Summary, StringComparison.OrdinalIgnoreCase) == 0) { notes = cdata.Before.Summary; } else { notes = $"{cdata.Before.Summary} / {cdata.After.Summary}"; } displayedLinesAll.Add(CreateComparisonLine(cdata.BeforeDisplayed.All.Displayed, cdata.AfterDisplayed.All.Displayed, notes)); if (cdata.BeforeDisplayed.NoOutliers != null) { displayedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoOutliers.Displayed, cdata.AfterDisplayed.NoOutliers !.Displayed, notes)); } if (cdata.BeforeDisplayed.NoSlowest != null) { displayedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoSlowest.Displayed, cdata.AfterDisplayed.NoSlowest !.Displayed, notes)); } nativeToManagedLinesAll.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.All.NativeToManaged, cdata.AfterNativeToManaged.All.NativeToManaged, notes)); if (cdata.BeforeNativeToManaged.NoOutliers != null) { nativeToManagedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoOutliers.NativeToManaged, cdata.AfterNativeToManaged.NoOutliers !.NativeToManaged, notes)); } if (cdata.BeforeNativeToManaged.NoSlowest != null) { nativeToManagedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoSlowest.NativeToManaged, cdata.AfterNativeToManaged.NoSlowest !.NativeToManaged, notes)); } totalInitLinesAll.Add(CreateComparisonLine(cdata.BeforeTotalInit.All.TotalInit, cdata.AfterTotalInit.All.TotalInit, notes)); if (cdata.BeforeTotalInit.NoOutliers != null) { totalInitLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoOutliers.TotalInit, cdata.AfterTotalInit.NoOutliers !.TotalInit, notes)); } if (cdata.BeforeTotalInit.NoSlowest != null) { totalInitLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoSlowest.TotalInit, cdata.AfterTotalInit.NoSlowest !.TotalInit, notes)); } } string reportFile = Path.Combine(resultsDir, Constants.ComparisonFileName); using (var sw = new StreamWriter(reportFile, false, Utilities.UTF8NoBOM)) { sw.WriteLine("# Reports"); sw.WriteLine("## Before"); WriteDescription(sw, reportOne.DateUTC, reportOne.Configuration, reportOne.XAVersion, reportOne.AndroidDevice); sw.WriteLine("## After"); WriteDescription(sw, reportTwo.DateUTC, reportTwo.Configuration, reportTwo.XAVersion, reportTwo.AndroidDevice); sw.WriteLine("# Comparison"); if (warnings.Count > 0) { sw.WriteLine(); sw.WriteLine("**Warnings**: "); foreach (string w in warnings) { sw.WriteLine($" * {w}"); } sw.WriteLine(); } sw.WriteLine(); sw.WriteLine("## Displayed"); WriteComparison(sw, "All runs", displayedLinesAll); WriteComparison(sw, "Without slowest and fastest runs", displayedLinesNoOutliers); WriteComparison(sw, "Without the slowest runs", displayedLinesNoSlowest); sw.WriteLine(); sw.WriteLine("## Native to managed"); WriteComparison(sw, "All runs", nativeToManagedLinesAll); WriteComparison(sw, "Without slowest and fastest runs", nativeToManagedLinesNoOutliers); WriteComparison(sw, "Without the slowest runs", nativeToManagedLinesNoSlowest); sw.WriteLine(); sw.WriteLine("## Total init"); WriteComparison(sw, "All runs", totalInitLinesAll); WriteComparison(sw, "Without slowest and fastest runs", totalInitLinesNoOutliers); WriteComparison(sw, "Without the slowest runs", totalInitLinesNoSlowest); sw.Flush(); } return(reportFile); }
async Task <(bool, BuildInfo?)> BuildAndInstall(RunDefinition run) { if (projectUsesGit && projectGitCommit == null) { (projectGitCommit, projectGitBranch) = await GetCommitHashAndBranch(FullProjectDirPath); } string buildCommand = run.BuildCommand; bool usesDotnet = String.Compare("dotnet", Path.GetFileName(buildCommand), StringComparison.OrdinalIgnoreCase) == 0; var args = new List <string> { $"-v:quiet" }; args.AddRange(run.Args); string projectPath = Path.GetRelativePath(FullProjectDirPath, FullProjectFilePath); string binlogBasePath = String.Empty; MSBuildCommon builder; BuildInfo buildInfo; if (!usesDotnet) { var msbuild = ConfigureRunner(new MSBuildRunner(context, buildCommand)); binlogBasePath = GetBinlogBasePath("restore"); if (!await msbuild.Run(projectPath, binlogBasePath, "Restore", configuration, args)) { return(false, null); } binlogBasePath = GetBinlogBasePath("build"); run.BinlogPath = GetRelativeBinlogPath(binlogBasePath); if (!await msbuild.Run(projectPath, binlogBasePath, "SignAndroidPackage", configuration, args)) { return(false, null); } buildInfo = FindFirstAndroidBuildInfo(await msbuild.GetBuildInfo(binlogBasePath)); await Uninstall(buildInfo); binlogBasePath = GetBinlogBasePath("install"); if (!await msbuild.Run(projectPath, binlogBasePath, "Install", configuration, args)) { return(false, null); } builder = msbuild; } else { var dotnet = ConfigureRunner(new DotnetRunner(context, buildCommand)); binlogBasePath = GetBinlogBasePath("build"); run.BinlogPath = GetRelativeBinlogPath(binlogBasePath); if (!await dotnet.Build(projectPath, binlogBasePath, configuration, args)) { return(false, null); } buildInfo = FindFirstAndroidBuildInfo(await dotnet.GetBuildInfo(binlogBasePath)); await Uninstall(buildInfo); binlogBasePath = GetBinlogBasePath("install"); if (!await dotnet.Install(projectPath, binlogBasePath, buildInfo.TargetFramework, configuration, args)) { return(false, null); } builder = dotnet; } if (xaVersionNotDetectedYet) { const string NotGit = "not a git build"; Log.InfoLine("Retrieving Xamarin.Android version information"); xaVersion = await GetXAVersion(builder, binlogBasePath); Log.InfoLine($" Location: {xaVersion.RootDir}"); Log.InfoLine($" Version: {xaVersion.Version}"); string hash = String.IsNullOrEmpty(xaVersion.Commit) ? NotGit : xaVersion.Commit; string branch = String.IsNullOrEmpty(xaVersion.Commit) ? NotGit : xaVersion.Branch; Log.InfoLine($" Git branch: {branch}"); Log.InfoLine($" Git commit: {hash}"); xaVersionNotDetectedYet = false; } return(true, buildInfo); async Task <bool> Uninstall(BuildInfo buildInfo) { (string packageName, _) = GetPackageAndMainActivityNames(buildInfo, run); var adb = new AdbRunner(context); return(await adb.Uninstall(packageName)); } T ConfigureRunner <T> (T runner) where T : MSBuildCommon { runner.WorkingDirectory = FullProjectDirPath; runner.EchoStandardOutput = true; runner.EchoStandardError = true; return(runner); } string GetBinlogBasePath(string phase) { return(GetLogBasePath(Constants.MSBuildLogDir, phase, run.LogTag, projectGitCommit, projectGitBranch)); } string GetRelativeBinlogPath(string binlogBasePath) { return(Path.GetRelativePath(FullDataDirectoryPath, $"{binlogBasePath}.binlog")); } }