public static RawResults Load(Context context, string path)
{
var ret = new RawResults();
var doc = new XmlDocument();
doc.Load(path);
XmlElement root = doc.DocumentElement;
ret.DateUTC = ReadTimeStamp(path, root);
ret.Configuration = ReadConfiguration(path, root);
ret.AndroidDevice = ReadDeviceInfo(path, root.SelectSingleNode("//device"));
ReadGitInfo(path, ret, root.SelectSingleNode("//projectGitInfo"));
ret.XAVersion = ReadXamarinAndroidVersion(path, root.SelectSingleNode("//xamarinAndroidVersion"));
ret.SessionLogPath = ReadSessionLogPath(path, root.SelectSingleNode("//sessionLog"));
XmlNode?runs = root.SelectSingleNode("//runs");
ret.RepetitionCount = ReadRepetitions(path, runs);
foreach (XmlNode?runNode in runs.SelectNodes("./run"))
{
if (runNode == null)
{
continue;
}
var run = new RunDefinition(context);
run.LoadRaw(runNode);
ret.Runs.Add(run);
}
return(ret);
}
public ReportAverages(RunDefinition run, uint repetitionCount, Action <List <RunResults> > sorter)
{
All = new RunResults(run);
if (repetitionCount >= 2)
{
NoSlowest = new RunResults(run);
if (repetitionCount >= 3)
{
NoOutliers = new RunResults(run);
}
}
Calculate(run, sorter);
}
public ComparisonData(RunDefinition before, RunDefinition after,
ReportAverages beforeDisplayed, ReportAverages afterDisplayed,
ReportAverages beforeNativeToManaged, ReportAverages afterNativeToManaged,
ReportAverages beforeTotalInit, ReportAverages afterTotalInit)
{
Before = before;
After = after;
BeforeDisplayed = beforeDisplayed;
AfterDisplayed = afterDisplayed;
BeforeNativeToManaged = beforeNativeToManaged;
AfterNativeToManaged = afterNativeToManaged;
BeforeTotalInit = beforeTotalInit;
AfterTotalInit = afterTotalInit;
}
void Calculate(RunDefinition run, Action <List <RunResults> >?sorter)
{
bool haveOutliers = NoOutliers != null;
bool haveNoSlowest = NoSlowest != null;
var runResults = new List <RunResults> (run.Results);
decimal count = runResults.Count;
if ((haveOutliers || haveNoSlowest) && sorter != null)
{
sorter(runResults);
}
for (int i = 0; i < count; i++)
{
RunResults results = runResults [i];
if ((haveOutliers || haveNoSlowest) && i < runResults.Count - 1)
{
if (i > 0)
{
NoOutliers !.NativeToManaged += results.NativeToManaged / (count - 2);
NoOutliers.TotalInit += results.TotalInit / (count - 2);
NoOutliers.Displayed += results.Displayed / (count - 2);
}
NoSlowest !.NativeToManaged += results.NativeToManaged / (count - 1);
NoSlowest.TotalInit += results.TotalInit / (count - 1);
NoSlowest.Displayed += results.Displayed / (count - 1);
}
All.NativeToManaged += results.NativeToManaged / count;
All.TotalInit += results.TotalInit / count;
All.Displayed += results.Displayed / count;
}
}
public string Compare(Context context, string resultsDir, string reportOnePath, string reportTwoPath)
{
var reportOne = RawResults.Load(context, reportOnePath);
var reportTwo = RawResults.Load(context, reportTwoPath);
var warnings = new List <string> ();
if (reportOne.RepetitionCount != reportTwo.RepetitionCount)
{
warnings.Add($"Reports were created based on different number of repetitions");
}
if (reportOne.Runs.Count != reportTwo.Runs.Count)
{
warnings.Add($"Reports have a different number of runs ({reportOne.Runs.Count} and {reportTwo.Runs.Count}). Only the first {Math.Min (reportOne.Runs.Count, reportTwo.Runs.Count)} runs will be compared.");
}
var comparisons = new List <ComparisonData> ();
for (int i = 0; i < Math.Min(reportOne.Runs.Count, reportTwo.Runs.Count); i++)
{
RunDefinition one = reportOne.Runs[i];
RunDefinition two = reportTwo.Runs[i];
bool argsDiffer = false;
if (one.Args.Count != two.Args.Count)
{
argsDiffer = true;
}
else
{
var argsOne = new List <string> (one.Args);
argsOne.Sort();
var argsTwo = new List <string> (two.Args);
argsTwo.Sort();
for (int j = 0; j < argsOne.Count; j++)
{
if (String.Compare(argsOne[j].Trim(), argsTwo[j].Trim(), StringComparison.Ordinal) != 0)
{
argsDiffer = true;
break;
}
}
}
if (argsDiffer)
{
warnings.Add($"Arguments for run {i} differ between the two reports");
}
comparisons.Add(
new ComparisonData(
before: one,
after: two,
beforeDisplayed: new ReportAverages(one, reportOne.RepetitionCount, SortDisplayed),
afterDisplayed: new ReportAverages(two, reportTwo.RepetitionCount, SortDisplayed),
beforeNativeToManaged: new ReportAverages(one, reportOne.RepetitionCount, SortNativeToManaged),
afterNativeToManaged: new ReportAverages(two, reportTwo.RepetitionCount, SortNativeToManaged),
beforeTotalInit: new ReportAverages(one, reportOne.RepetitionCount, SortTotalInit),
afterTotalInit: new ReportAverages(two, reportTwo.RepetitionCount, SortTotalInit)
)
);
}
var displayedLinesAll = new List <ReportLineComparison> ();
var displayedLinesNoOutliers = new List <ReportLineComparison> ();
var displayedLinesNoSlowest = new List <ReportLineComparison> ();
var nativeToManagedLinesAll = new List <ReportLineComparison> ();
var nativeToManagedLinesNoOutliers = new List <ReportLineComparison> ();
var nativeToManagedLinesNoSlowest = new List <ReportLineComparison> ();
var totalInitLinesAll = new List <ReportLineComparison> ();
var totalInitLinesNoOutliers = new List <ReportLineComparison> ();
var totalInitLinesNoSlowest = new List <ReportLineComparison> ();
foreach (ComparisonData cdata in comparisons)
{
string notes;
if (String.Compare(cdata.Before.Summary, cdata.After.Summary, StringComparison.OrdinalIgnoreCase) == 0)
{
notes = cdata.Before.Summary;
}
else
{
notes = $"{cdata.Before.Summary} / {cdata.After.Summary}";
}
displayedLinesAll.Add(CreateComparisonLine(cdata.BeforeDisplayed.All.Displayed, cdata.AfterDisplayed.All.Displayed, notes));
if (cdata.BeforeDisplayed.NoOutliers != null)
{
displayedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoOutliers.Displayed, cdata.AfterDisplayed.NoOutliers !.Displayed, notes));
}
if (cdata.BeforeDisplayed.NoSlowest != null)
{
displayedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoSlowest.Displayed, cdata.AfterDisplayed.NoSlowest !.Displayed, notes));
}
nativeToManagedLinesAll.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.All.NativeToManaged, cdata.AfterNativeToManaged.All.NativeToManaged, notes));
if (cdata.BeforeNativeToManaged.NoOutliers != null)
{
nativeToManagedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoOutliers.NativeToManaged, cdata.AfterNativeToManaged.NoOutliers !.NativeToManaged, notes));
}
if (cdata.BeforeNativeToManaged.NoSlowest != null)
{
nativeToManagedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoSlowest.NativeToManaged, cdata.AfterNativeToManaged.NoSlowest !.NativeToManaged, notes));
}
totalInitLinesAll.Add(CreateComparisonLine(cdata.BeforeTotalInit.All.TotalInit, cdata.AfterTotalInit.All.TotalInit, notes));
if (cdata.BeforeTotalInit.NoOutliers != null)
{
totalInitLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoOutliers.TotalInit, cdata.AfterTotalInit.NoOutliers !.TotalInit, notes));
}
if (cdata.BeforeTotalInit.NoSlowest != null)
{
totalInitLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoSlowest.TotalInit, cdata.AfterTotalInit.NoSlowest !.TotalInit, notes));
}
}
string reportFile = Path.Combine(resultsDir, Constants.ComparisonFileName);
using (var sw = new StreamWriter(reportFile, false, Utilities.UTF8NoBOM)) {
sw.WriteLine("# Reports");
sw.WriteLine("## Before");
WriteDescription(sw, reportOne.DateUTC, reportOne.Configuration, reportOne.XAVersion, reportOne.AndroidDevice);
sw.WriteLine("## After");
WriteDescription(sw, reportTwo.DateUTC, reportTwo.Configuration, reportTwo.XAVersion, reportTwo.AndroidDevice);
sw.WriteLine("# Comparison");
if (warnings.Count > 0)
{
sw.WriteLine();
sw.WriteLine("**Warnings**: ");
foreach (string w in warnings)
{
sw.WriteLine($" * {w}");
}
sw.WriteLine();
}
sw.WriteLine();
sw.WriteLine("## Displayed");
WriteComparison(sw, "All runs", displayedLinesAll);
WriteComparison(sw, "Without slowest and fastest runs", displayedLinesNoOutliers);
WriteComparison(sw, "Without the slowest runs", displayedLinesNoSlowest);
sw.WriteLine();
sw.WriteLine("## Native to managed");
WriteComparison(sw, "All runs", nativeToManagedLinesAll);
WriteComparison(sw, "Without slowest and fastest runs", nativeToManagedLinesNoOutliers);
WriteComparison(sw, "Without the slowest runs", nativeToManagedLinesNoSlowest);
sw.WriteLine();
sw.WriteLine("## Total init");
WriteComparison(sw, "All runs", totalInitLinesAll);
WriteComparison(sw, "Without slowest and fastest runs", totalInitLinesNoOutliers);
WriteComparison(sw, "Without the slowest runs", totalInitLinesNoSlowest);
sw.Flush();
}
return(reportFile);
}
async Task <(bool, BuildInfo?)> BuildAndInstall(RunDefinition run)
{
if (projectUsesGit && projectGitCommit == null)
{
(projectGitCommit, projectGitBranch) = await GetCommitHashAndBranch(FullProjectDirPath);
}
string buildCommand = run.BuildCommand;
bool usesDotnet = String.Compare("dotnet", Path.GetFileName(buildCommand), StringComparison.OrdinalIgnoreCase) == 0;
var args = new List <string> {
$"-v:quiet"
};
args.AddRange(run.Args);
string projectPath = Path.GetRelativePath(FullProjectDirPath, FullProjectFilePath);
string binlogBasePath = String.Empty;
MSBuildCommon builder;
BuildInfo buildInfo;
if (!usesDotnet)
{
var msbuild = ConfigureRunner(new MSBuildRunner(context, buildCommand));
binlogBasePath = GetBinlogBasePath("restore");
if (!await msbuild.Run(projectPath, binlogBasePath, "Restore", configuration, args))
{
return(false, null);
}
binlogBasePath = GetBinlogBasePath("build");
run.BinlogPath = GetRelativeBinlogPath(binlogBasePath);
if (!await msbuild.Run(projectPath, binlogBasePath, "SignAndroidPackage", configuration, args))
{
return(false, null);
}
buildInfo = FindFirstAndroidBuildInfo(await msbuild.GetBuildInfo(binlogBasePath));
await Uninstall(buildInfo);
binlogBasePath = GetBinlogBasePath("install");
if (!await msbuild.Run(projectPath, binlogBasePath, "Install", configuration, args))
{
return(false, null);
}
builder = msbuild;
}
else
{
var dotnet = ConfigureRunner(new DotnetRunner(context, buildCommand));
binlogBasePath = GetBinlogBasePath("build");
run.BinlogPath = GetRelativeBinlogPath(binlogBasePath);
if (!await dotnet.Build(projectPath, binlogBasePath, configuration, args))
{
return(false, null);
}
buildInfo = FindFirstAndroidBuildInfo(await dotnet.GetBuildInfo(binlogBasePath));
await Uninstall(buildInfo);
binlogBasePath = GetBinlogBasePath("install");
if (!await dotnet.Install(projectPath, binlogBasePath, buildInfo.TargetFramework, configuration, args))
{
return(false, null);
}
builder = dotnet;
}
if (xaVersionNotDetectedYet)
{
const string NotGit = "not a git build";
Log.InfoLine("Retrieving Xamarin.Android version information");
xaVersion = await GetXAVersion(builder, binlogBasePath);
Log.InfoLine($" Location: {xaVersion.RootDir}");
Log.InfoLine($" Version: {xaVersion.Version}");
string hash = String.IsNullOrEmpty(xaVersion.Commit) ? NotGit : xaVersion.Commit;
string branch = String.IsNullOrEmpty(xaVersion.Commit) ? NotGit : xaVersion.Branch;
Log.InfoLine($" Git branch: {branch}");
Log.InfoLine($" Git commit: {hash}");
xaVersionNotDetectedYet = false;
}
return(true, buildInfo);
async Task <bool> Uninstall(BuildInfo buildInfo)
{
(string packageName, _) = GetPackageAndMainActivityNames(buildInfo, run);
var adb = new AdbRunner(context);
return(await adb.Uninstall(packageName));
}
T ConfigureRunner <T> (T runner) where T : MSBuildCommon
{
runner.WorkingDirectory = FullProjectDirPath;
runner.EchoStandardOutput = true;
runner.EchoStandardError = true;
return(runner);
}
string GetBinlogBasePath(string phase)
{
return(GetLogBasePath(Constants.MSBuildLogDir, phase, run.LogTag, projectGitCommit, projectGitBranch));
}
string GetRelativeBinlogPath(string binlogBasePath)
{
return(Path.GetRelativePath(FullDataDirectoryPath, $"{binlogBasePath}.binlog"));
}
}