예제 #1
0
        public static RawResults Load(Context context, string path)
        {
            var ret = new RawResults();

            var doc = new XmlDocument();

            doc.Load(path);

            XmlElement root = doc.DocumentElement;

            ret.DateUTC       = ReadTimeStamp(path, root);
            ret.Configuration = ReadConfiguration(path, root);
            ret.AndroidDevice = ReadDeviceInfo(path, root.SelectSingleNode("//device"));
            ReadGitInfo(path, ret, root.SelectSingleNode("//projectGitInfo"));
            ret.XAVersion      = ReadXamarinAndroidVersion(path, root.SelectSingleNode("//xamarinAndroidVersion"));
            ret.SessionLogPath = ReadSessionLogPath(path, root.SelectSingleNode("//sessionLog"));

            XmlNode?runs = root.SelectSingleNode("//runs");

            ret.RepetitionCount = ReadRepetitions(path, runs);

            foreach (XmlNode?runNode in runs.SelectNodes("./run"))
            {
                if (runNode == null)
                {
                    continue;
                }

                var run = new RunDefinition(context);
                run.LoadRaw(runNode);
                ret.Runs.Add(run);
            }

            return(ret);
        }
예제 #2
0
        static void ReadGitInfo(string path, RawResults ret, XmlNode?node)
        {
            if (node == null)
            {
                ret.GitCommit = Constants.Unknown;
                ret.GitBranch = Constants.Unknown;
                Log.WarningLine($"Raw results '{path}' are missing Git info.");
                return;
            }

            ret.GitCommit = Utilities.GetAttributeValue(node, "commit");
            ret.GitBranch = Utilities.GetAttributeValue(node, "branch");
        }
예제 #3
0
        public string Compare(Context context, string resultsDir, string reportOnePath, string reportTwoPath)
        {
            var reportOne = RawResults.Load(context, reportOnePath);
            var reportTwo = RawResults.Load(context, reportTwoPath);
            var warnings  = new List <string> ();

            if (reportOne.RepetitionCount != reportTwo.RepetitionCount)
            {
                warnings.Add($"Reports were created based on different number of repetitions");
            }

            if (reportOne.Runs.Count != reportTwo.Runs.Count)
            {
                warnings.Add($"Reports have a different number of runs ({reportOne.Runs.Count} and {reportTwo.Runs.Count}). Only the first {Math.Min (reportOne.Runs.Count, reportTwo.Runs.Count)} runs will be compared.");
            }

            var comparisons = new List <ComparisonData> ();

            for (int i = 0; i < Math.Min(reportOne.Runs.Count, reportTwo.Runs.Count); i++)
            {
                RunDefinition one = reportOne.Runs[i];
                RunDefinition two = reportTwo.Runs[i];

                bool argsDiffer = false;
                if (one.Args.Count != two.Args.Count)
                {
                    argsDiffer = true;
                }
                else
                {
                    var argsOne = new List <string> (one.Args);
                    argsOne.Sort();

                    var argsTwo = new List <string> (two.Args);
                    argsTwo.Sort();

                    for (int j = 0; j < argsOne.Count; j++)
                    {
                        if (String.Compare(argsOne[j].Trim(), argsTwo[j].Trim(), StringComparison.Ordinal) != 0)
                        {
                            argsDiffer = true;
                            break;
                        }
                    }
                }

                if (argsDiffer)
                {
                    warnings.Add($"Arguments for run {i} differ between the two reports");
                }

                comparisons.Add(
                    new ComparisonData(
                        before: one,
                        after: two,

                        beforeDisplayed: new ReportAverages(one, reportOne.RepetitionCount, SortDisplayed),
                        afterDisplayed: new ReportAverages(two, reportTwo.RepetitionCount, SortDisplayed),

                        beforeNativeToManaged: new ReportAverages(one, reportOne.RepetitionCount, SortNativeToManaged),
                        afterNativeToManaged: new ReportAverages(two, reportTwo.RepetitionCount, SortNativeToManaged),

                        beforeTotalInit: new ReportAverages(one, reportOne.RepetitionCount, SortTotalInit),
                        afterTotalInit: new ReportAverages(two, reportTwo.RepetitionCount, SortTotalInit)
                        )
                    );
            }

            var displayedLinesAll              = new List <ReportLineComparison> ();
            var displayedLinesNoOutliers       = new List <ReportLineComparison> ();
            var displayedLinesNoSlowest        = new List <ReportLineComparison> ();
            var nativeToManagedLinesAll        = new List <ReportLineComparison> ();
            var nativeToManagedLinesNoOutliers = new List <ReportLineComparison> ();
            var nativeToManagedLinesNoSlowest  = new List <ReportLineComparison> ();
            var totalInitLinesAll              = new List <ReportLineComparison> ();
            var totalInitLinesNoOutliers       = new List <ReportLineComparison> ();
            var totalInitLinesNoSlowest        = new List <ReportLineComparison> ();

            foreach (ComparisonData cdata in comparisons)
            {
                string notes;

                if (String.Compare(cdata.Before.Summary, cdata.After.Summary, StringComparison.OrdinalIgnoreCase) == 0)
                {
                    notes = cdata.Before.Summary;
                }
                else
                {
                    notes = $"{cdata.Before.Summary} / {cdata.After.Summary}";
                }

                displayedLinesAll.Add(CreateComparisonLine(cdata.BeforeDisplayed.All.Displayed, cdata.AfterDisplayed.All.Displayed, notes));

                if (cdata.BeforeDisplayed.NoOutliers != null)
                {
                    displayedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoOutliers.Displayed, cdata.AfterDisplayed.NoOutliers !.Displayed, notes));
                }

                if (cdata.BeforeDisplayed.NoSlowest != null)
                {
                    displayedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoSlowest.Displayed, cdata.AfterDisplayed.NoSlowest !.Displayed, notes));
                }

                nativeToManagedLinesAll.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.All.NativeToManaged, cdata.AfterNativeToManaged.All.NativeToManaged, notes));

                if (cdata.BeforeNativeToManaged.NoOutliers != null)
                {
                    nativeToManagedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoOutliers.NativeToManaged, cdata.AfterNativeToManaged.NoOutliers !.NativeToManaged, notes));
                }

                if (cdata.BeforeNativeToManaged.NoSlowest != null)
                {
                    nativeToManagedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoSlowest.NativeToManaged, cdata.AfterNativeToManaged.NoSlowest !.NativeToManaged, notes));
                }

                totalInitLinesAll.Add(CreateComparisonLine(cdata.BeforeTotalInit.All.TotalInit, cdata.AfterTotalInit.All.TotalInit, notes));

                if (cdata.BeforeTotalInit.NoOutliers != null)
                {
                    totalInitLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoOutliers.TotalInit, cdata.AfterTotalInit.NoOutliers !.TotalInit, notes));
                }

                if (cdata.BeforeTotalInit.NoSlowest != null)
                {
                    totalInitLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoSlowest.TotalInit, cdata.AfterTotalInit.NoSlowest !.TotalInit, notes));
                }
            }

            string reportFile = Path.Combine(resultsDir, Constants.ComparisonFileName);

            using (var sw = new StreamWriter(reportFile, false, Utilities.UTF8NoBOM)) {
                sw.WriteLine("# Reports");

                sw.WriteLine("## Before");
                WriteDescription(sw, reportOne.DateUTC, reportOne.Configuration, reportOne.XAVersion, reportOne.AndroidDevice);

                sw.WriteLine("## After");
                WriteDescription(sw, reportTwo.DateUTC, reportTwo.Configuration, reportTwo.XAVersion, reportTwo.AndroidDevice);

                sw.WriteLine("# Comparison");
                if (warnings.Count > 0)
                {
                    sw.WriteLine();
                    sw.WriteLine("**Warnings**:  ");

                    foreach (string w in warnings)
                    {
                        sw.WriteLine($"  * {w}");
                    }
                    sw.WriteLine();
                }

                sw.WriteLine();
                sw.WriteLine("## Displayed");
                WriteComparison(sw, "All runs", displayedLinesAll);
                WriteComparison(sw, "Without slowest and fastest runs", displayedLinesNoOutliers);
                WriteComparison(sw, "Without the slowest runs", displayedLinesNoSlowest);

                sw.WriteLine();
                sw.WriteLine("## Native to managed");
                WriteComparison(sw, "All runs", nativeToManagedLinesAll);
                WriteComparison(sw, "Without slowest and fastest runs", nativeToManagedLinesNoOutliers);
                WriteComparison(sw, "Without the slowest runs", nativeToManagedLinesNoSlowest);

                sw.WriteLine();
                sw.WriteLine("## Total init");
                WriteComparison(sw, "All runs", totalInitLinesAll);
                WriteComparison(sw, "Without slowest and fastest runs", totalInitLinesNoOutliers);
                WriteComparison(sw, "Without the slowest runs", totalInitLinesNoSlowest);

                sw.Flush();
            }

            return(reportFile);
        }
예제 #4
0
        public async Task <bool> Run()
        {
            var adb = new AdbRunner(context);
            AndroidDeviceInfo?info = await adb.GetDeviceInfo();

            if (info == null)
            {
                Log.FatalLine("Failed to obtain Android device info");
                return(false);
            }
            adi = info;
            Log.InfoLine($"Device: {adi.Model}");
            Log.InfoLine($"Device architecture: {adi.Architecture}");
            Log.InfoLine($"Device SDK: {adi.SdkVersion}");

            if (!await adb.SetPropertyValue("debug.mono.log", "default,timing=bare"))
            {
                Log.FatalLine("Failed to set Mono debugging properties");
                return(false);
            }

            if (!await adb.SetLogcatBufferSize("16M"))
            {
                Log.WarningLine("Failed to set logcat buffer size");
            }

            Utilities.CreateDirectory(FullDataDirectoryPath);

            foreach (RunDefinition run in runs)
            {
                if (run.RunPerformanceTest)
                {
                    if (!await RunPerformanceTest(run, adb))
                    {
                        return(false);
                    }
                }
            }

            foreach (RunDefinition run in runs)
            {
                if (run.RunManagedProfiler)
                {
                    if (!await RunManagedProfiler(run))
                    {
                        return(false);
                    }
                }
            }

            foreach (RunDefinition run in runs)
            {
                if (run.RunNativeProfiler)
                {
                    if (!await RunNativeProfiler(run))
                    {
                        return(false);
                    }
                }
            }

            RawResults.Save(adi, this);

            return(true);
        }