public static RawResults Load(Context context, string path)
{
var ret = new RawResults();
var doc = new XmlDocument();
doc.Load(path);
XmlElement root = doc.DocumentElement;
ret.DateUTC = ReadTimeStamp(path, root);
ret.Configuration = ReadConfiguration(path, root);
ret.AndroidDevice = ReadDeviceInfo(path, root.SelectSingleNode("//device"));
ReadGitInfo(path, ret, root.SelectSingleNode("//projectGitInfo"));
ret.XAVersion = ReadXamarinAndroidVersion(path, root.SelectSingleNode("//xamarinAndroidVersion"));
ret.SessionLogPath = ReadSessionLogPath(path, root.SelectSingleNode("//sessionLog"));
XmlNode?runs = root.SelectSingleNode("//runs");
ret.RepetitionCount = ReadRepetitions(path, runs);
foreach (XmlNode?runNode in runs.SelectNodes("./run"))
{
if (runNode == null)
{
continue;
}
var run = new RunDefinition(context);
run.LoadRaw(runNode);
ret.Runs.Add(run);
}
return(ret);
}
static void ReadGitInfo(string path, RawResults ret, XmlNode?node)
{
if (node == null)
{
ret.GitCommit = Constants.Unknown;
ret.GitBranch = Constants.Unknown;
Log.WarningLine($"Raw results '{path}' are missing Git info.");
return;
}
ret.GitCommit = Utilities.GetAttributeValue(node, "commit");
ret.GitBranch = Utilities.GetAttributeValue(node, "branch");
}
public string Compare(Context context, string resultsDir, string reportOnePath, string reportTwoPath)
{
var reportOne = RawResults.Load(context, reportOnePath);
var reportTwo = RawResults.Load(context, reportTwoPath);
var warnings = new List <string> ();
if (reportOne.RepetitionCount != reportTwo.RepetitionCount)
{
warnings.Add($"Reports were created based on different number of repetitions");
}
if (reportOne.Runs.Count != reportTwo.Runs.Count)
{
warnings.Add($"Reports have a different number of runs ({reportOne.Runs.Count} and {reportTwo.Runs.Count}). Only the first {Math.Min (reportOne.Runs.Count, reportTwo.Runs.Count)} runs will be compared.");
}
var comparisons = new List <ComparisonData> ();
for (int i = 0; i < Math.Min(reportOne.Runs.Count, reportTwo.Runs.Count); i++)
{
RunDefinition one = reportOne.Runs[i];
RunDefinition two = reportTwo.Runs[i];
bool argsDiffer = false;
if (one.Args.Count != two.Args.Count)
{
argsDiffer = true;
}
else
{
var argsOne = new List <string> (one.Args);
argsOne.Sort();
var argsTwo = new List <string> (two.Args);
argsTwo.Sort();
for (int j = 0; j < argsOne.Count; j++)
{
if (String.Compare(argsOne[j].Trim(), argsTwo[j].Trim(), StringComparison.Ordinal) != 0)
{
argsDiffer = true;
break;
}
}
}
if (argsDiffer)
{
warnings.Add($"Arguments for run {i} differ between the two reports");
}
comparisons.Add(
new ComparisonData(
before: one,
after: two,
beforeDisplayed: new ReportAverages(one, reportOne.RepetitionCount, SortDisplayed),
afterDisplayed: new ReportAverages(two, reportTwo.RepetitionCount, SortDisplayed),
beforeNativeToManaged: new ReportAverages(one, reportOne.RepetitionCount, SortNativeToManaged),
afterNativeToManaged: new ReportAverages(two, reportTwo.RepetitionCount, SortNativeToManaged),
beforeTotalInit: new ReportAverages(one, reportOne.RepetitionCount, SortTotalInit),
afterTotalInit: new ReportAverages(two, reportTwo.RepetitionCount, SortTotalInit)
)
);
}
var displayedLinesAll = new List <ReportLineComparison> ();
var displayedLinesNoOutliers = new List <ReportLineComparison> ();
var displayedLinesNoSlowest = new List <ReportLineComparison> ();
var nativeToManagedLinesAll = new List <ReportLineComparison> ();
var nativeToManagedLinesNoOutliers = new List <ReportLineComparison> ();
var nativeToManagedLinesNoSlowest = new List <ReportLineComparison> ();
var totalInitLinesAll = new List <ReportLineComparison> ();
var totalInitLinesNoOutliers = new List <ReportLineComparison> ();
var totalInitLinesNoSlowest = new List <ReportLineComparison> ();
foreach (ComparisonData cdata in comparisons)
{
string notes;
if (String.Compare(cdata.Before.Summary, cdata.After.Summary, StringComparison.OrdinalIgnoreCase) == 0)
{
notes = cdata.Before.Summary;
}
else
{
notes = $"{cdata.Before.Summary} / {cdata.After.Summary}";
}
displayedLinesAll.Add(CreateComparisonLine(cdata.BeforeDisplayed.All.Displayed, cdata.AfterDisplayed.All.Displayed, notes));
if (cdata.BeforeDisplayed.NoOutliers != null)
{
displayedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoOutliers.Displayed, cdata.AfterDisplayed.NoOutliers !.Displayed, notes));
}
if (cdata.BeforeDisplayed.NoSlowest != null)
{
displayedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoSlowest.Displayed, cdata.AfterDisplayed.NoSlowest !.Displayed, notes));
}
nativeToManagedLinesAll.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.All.NativeToManaged, cdata.AfterNativeToManaged.All.NativeToManaged, notes));
if (cdata.BeforeNativeToManaged.NoOutliers != null)
{
nativeToManagedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoOutliers.NativeToManaged, cdata.AfterNativeToManaged.NoOutliers !.NativeToManaged, notes));
}
if (cdata.BeforeNativeToManaged.NoSlowest != null)
{
nativeToManagedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoSlowest.NativeToManaged, cdata.AfterNativeToManaged.NoSlowest !.NativeToManaged, notes));
}
totalInitLinesAll.Add(CreateComparisonLine(cdata.BeforeTotalInit.All.TotalInit, cdata.AfterTotalInit.All.TotalInit, notes));
if (cdata.BeforeTotalInit.NoOutliers != null)
{
totalInitLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoOutliers.TotalInit, cdata.AfterTotalInit.NoOutliers !.TotalInit, notes));
}
if (cdata.BeforeTotalInit.NoSlowest != null)
{
totalInitLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoSlowest.TotalInit, cdata.AfterTotalInit.NoSlowest !.TotalInit, notes));
}
}
string reportFile = Path.Combine(resultsDir, Constants.ComparisonFileName);
using (var sw = new StreamWriter(reportFile, false, Utilities.UTF8NoBOM)) {
sw.WriteLine("# Reports");
sw.WriteLine("## Before");
WriteDescription(sw, reportOne.DateUTC, reportOne.Configuration, reportOne.XAVersion, reportOne.AndroidDevice);
sw.WriteLine("## After");
WriteDescription(sw, reportTwo.DateUTC, reportTwo.Configuration, reportTwo.XAVersion, reportTwo.AndroidDevice);
sw.WriteLine("# Comparison");
if (warnings.Count > 0)
{
sw.WriteLine();
sw.WriteLine("**Warnings**: ");
foreach (string w in warnings)
{
sw.WriteLine($" * {w}");
}
sw.WriteLine();
}
sw.WriteLine();
sw.WriteLine("## Displayed");
WriteComparison(sw, "All runs", displayedLinesAll);
WriteComparison(sw, "Without slowest and fastest runs", displayedLinesNoOutliers);
WriteComparison(sw, "Without the slowest runs", displayedLinesNoSlowest);
sw.WriteLine();
sw.WriteLine("## Native to managed");
WriteComparison(sw, "All runs", nativeToManagedLinesAll);
WriteComparison(sw, "Without slowest and fastest runs", nativeToManagedLinesNoOutliers);
WriteComparison(sw, "Without the slowest runs", nativeToManagedLinesNoSlowest);
sw.WriteLine();
sw.WriteLine("## Total init");
WriteComparison(sw, "All runs", totalInitLinesAll);
WriteComparison(sw, "Without slowest and fastest runs", totalInitLinesNoOutliers);
WriteComparison(sw, "Without the slowest runs", totalInitLinesNoSlowest);
sw.Flush();
}
return(reportFile);
}
public async Task <bool> Run()
{
var adb = new AdbRunner(context);
AndroidDeviceInfo?info = await adb.GetDeviceInfo();
if (info == null)
{
Log.FatalLine("Failed to obtain Android device info");
return(false);
}
adi = info;
Log.InfoLine($"Device: {adi.Model}");
Log.InfoLine($"Device architecture: {adi.Architecture}");
Log.InfoLine($"Device SDK: {adi.SdkVersion}");
if (!await adb.SetPropertyValue("debug.mono.log", "default,timing=bare"))
{
Log.FatalLine("Failed to set Mono debugging properties");
return(false);
}
if (!await adb.SetLogcatBufferSize("16M"))
{
Log.WarningLine("Failed to set logcat buffer size");
}
Utilities.CreateDirectory(FullDataDirectoryPath);
foreach (RunDefinition run in runs)
{
if (run.RunPerformanceTest)
{
if (!await RunPerformanceTest(run, adb))
{
return(false);
}
}
}
foreach (RunDefinition run in runs)
{
if (run.RunManagedProfiler)
{
if (!await RunManagedProfiler(run))
{
return(false);
}
}
}
foreach (RunDefinition run in runs)
{
if (run.RunNativeProfiler)
{
if (!await RunNativeProfiler(run))
{
return(false);
}
}
}
RawResults.Save(adi, this);
return(true);
}