Пример #1
0
        public static RawResults Load(Context context, string path)
        {
            var ret = new RawResults();

            var doc = new XmlDocument();

            doc.Load(path);

            XmlElement root = doc.DocumentElement;

            ret.DateUTC       = ReadTimeStamp(path, root);
            ret.Configuration = ReadConfiguration(path, root);
            ret.AndroidDevice = ReadDeviceInfo(path, root.SelectSingleNode("//device"));
            ReadGitInfo(path, ret, root.SelectSingleNode("//projectGitInfo"));
            ret.XAVersion      = ReadXamarinAndroidVersion(path, root.SelectSingleNode("//xamarinAndroidVersion"));
            ret.SessionLogPath = ReadSessionLogPath(path, root.SelectSingleNode("//sessionLog"));

            XmlNode?runs = root.SelectSingleNode("//runs");

            ret.RepetitionCount = ReadRepetitions(path, runs);

            foreach (XmlNode?runNode in runs.SelectNodes("./run"))
            {
                if (runNode == null)
                {
                    continue;
                }

                var run = new RunDefinition(context);
                run.LoadRaw(runNode);
                ret.Runs.Add(run);
            }

            return(ret);
        }
Пример #2
0
        public ReportAverages(RunDefinition run, uint repetitionCount, Action <List <RunResults> > sorter)
        {
            All = new RunResults(run);

            if (repetitionCount >= 2)
            {
                NoSlowest = new RunResults(run);
                if (repetitionCount >= 3)
                {
                    NoOutliers = new RunResults(run);
                }
            }

            Calculate(run, sorter);
        }
Пример #3
0
            public ComparisonData(RunDefinition before, RunDefinition after,
                                  ReportAverages beforeDisplayed, ReportAverages afterDisplayed,
                                  ReportAverages beforeNativeToManaged, ReportAverages afterNativeToManaged,
                                  ReportAverages beforeTotalInit, ReportAverages afterTotalInit)
            {
                Before = before;
                After  = after;

                BeforeDisplayed = beforeDisplayed;
                AfterDisplayed  = afterDisplayed;

                BeforeNativeToManaged = beforeNativeToManaged;
                AfterNativeToManaged  = afterNativeToManaged;

                BeforeTotalInit = beforeTotalInit;
                AfterTotalInit  = afterTotalInit;
            }
Пример #4
0
        void Calculate(RunDefinition run, Action <List <RunResults> >?sorter)
        {
            bool haveOutliers  = NoOutliers != null;
            bool haveNoSlowest = NoSlowest != null;

            var     runResults = new List <RunResults> (run.Results);
            decimal count      = runResults.Count;

            if ((haveOutliers || haveNoSlowest) && sorter != null)
            {
                sorter(runResults);
            }

            for (int i = 0; i < count; i++)
            {
                RunResults results = runResults [i];

                if ((haveOutliers || haveNoSlowest) && i < runResults.Count - 1)
                {
                    if (i > 0)
                    {
                        NoOutliers !.NativeToManaged += results.NativeToManaged / (count - 2);
                        NoOutliers.TotalInit         += results.TotalInit / (count - 2);
                        NoOutliers.Displayed         += results.Displayed / (count - 2);
                    }

                    NoSlowest !.NativeToManaged += results.NativeToManaged / (count - 1);
                    NoSlowest.TotalInit         += results.TotalInit / (count - 1);
                    NoSlowest.Displayed         += results.Displayed / (count - 1);
                }

                All.NativeToManaged += results.NativeToManaged / count;
                All.TotalInit       += results.TotalInit / count;
                All.Displayed       += results.Displayed / count;
            }
        }
Пример #5
0
        public string Compare(Context context, string resultsDir, string reportOnePath, string reportTwoPath)
        {
            var reportOne = RawResults.Load(context, reportOnePath);
            var reportTwo = RawResults.Load(context, reportTwoPath);
            var warnings  = new List <string> ();

            if (reportOne.RepetitionCount != reportTwo.RepetitionCount)
            {
                warnings.Add($"Reports were created based on different number of repetitions");
            }

            if (reportOne.Runs.Count != reportTwo.Runs.Count)
            {
                warnings.Add($"Reports have a different number of runs ({reportOne.Runs.Count} and {reportTwo.Runs.Count}). Only the first {Math.Min (reportOne.Runs.Count, reportTwo.Runs.Count)} runs will be compared.");
            }

            var comparisons = new List <ComparisonData> ();

            for (int i = 0; i < Math.Min(reportOne.Runs.Count, reportTwo.Runs.Count); i++)
            {
                RunDefinition one = reportOne.Runs[i];
                RunDefinition two = reportTwo.Runs[i];

                bool argsDiffer = false;
                if (one.Args.Count != two.Args.Count)
                {
                    argsDiffer = true;
                }
                else
                {
                    var argsOne = new List <string> (one.Args);
                    argsOne.Sort();

                    var argsTwo = new List <string> (two.Args);
                    argsTwo.Sort();

                    for (int j = 0; j < argsOne.Count; j++)
                    {
                        if (String.Compare(argsOne[j].Trim(), argsTwo[j].Trim(), StringComparison.Ordinal) != 0)
                        {
                            argsDiffer = true;
                            break;
                        }
                    }
                }

                if (argsDiffer)
                {
                    warnings.Add($"Arguments for run {i} differ between the two reports");
                }

                comparisons.Add(
                    new ComparisonData(
                        before: one,
                        after: two,

                        beforeDisplayed: new ReportAverages(one, reportOne.RepetitionCount, SortDisplayed),
                        afterDisplayed: new ReportAverages(two, reportTwo.RepetitionCount, SortDisplayed),

                        beforeNativeToManaged: new ReportAverages(one, reportOne.RepetitionCount, SortNativeToManaged),
                        afterNativeToManaged: new ReportAverages(two, reportTwo.RepetitionCount, SortNativeToManaged),

                        beforeTotalInit: new ReportAverages(one, reportOne.RepetitionCount, SortTotalInit),
                        afterTotalInit: new ReportAverages(two, reportTwo.RepetitionCount, SortTotalInit)
                        )
                    );
            }

            var displayedLinesAll              = new List <ReportLineComparison> ();
            var displayedLinesNoOutliers       = new List <ReportLineComparison> ();
            var displayedLinesNoSlowest        = new List <ReportLineComparison> ();
            var nativeToManagedLinesAll        = new List <ReportLineComparison> ();
            var nativeToManagedLinesNoOutliers = new List <ReportLineComparison> ();
            var nativeToManagedLinesNoSlowest  = new List <ReportLineComparison> ();
            var totalInitLinesAll              = new List <ReportLineComparison> ();
            var totalInitLinesNoOutliers       = new List <ReportLineComparison> ();
            var totalInitLinesNoSlowest        = new List <ReportLineComparison> ();

            foreach (ComparisonData cdata in comparisons)
            {
                string notes;

                if (String.Compare(cdata.Before.Summary, cdata.After.Summary, StringComparison.OrdinalIgnoreCase) == 0)
                {
                    notes = cdata.Before.Summary;
                }
                else
                {
                    notes = $"{cdata.Before.Summary} / {cdata.After.Summary}";
                }

                displayedLinesAll.Add(CreateComparisonLine(cdata.BeforeDisplayed.All.Displayed, cdata.AfterDisplayed.All.Displayed, notes));

                if (cdata.BeforeDisplayed.NoOutliers != null)
                {
                    displayedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoOutliers.Displayed, cdata.AfterDisplayed.NoOutliers !.Displayed, notes));
                }

                if (cdata.BeforeDisplayed.NoSlowest != null)
                {
                    displayedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeDisplayed.NoSlowest.Displayed, cdata.AfterDisplayed.NoSlowest !.Displayed, notes));
                }

                nativeToManagedLinesAll.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.All.NativeToManaged, cdata.AfterNativeToManaged.All.NativeToManaged, notes));

                if (cdata.BeforeNativeToManaged.NoOutliers != null)
                {
                    nativeToManagedLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoOutliers.NativeToManaged, cdata.AfterNativeToManaged.NoOutliers !.NativeToManaged, notes));
                }

                if (cdata.BeforeNativeToManaged.NoSlowest != null)
                {
                    nativeToManagedLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeNativeToManaged.NoSlowest.NativeToManaged, cdata.AfterNativeToManaged.NoSlowest !.NativeToManaged, notes));
                }

                totalInitLinesAll.Add(CreateComparisonLine(cdata.BeforeTotalInit.All.TotalInit, cdata.AfterTotalInit.All.TotalInit, notes));

                if (cdata.BeforeTotalInit.NoOutliers != null)
                {
                    totalInitLinesNoOutliers.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoOutliers.TotalInit, cdata.AfterTotalInit.NoOutliers !.TotalInit, notes));
                }

                if (cdata.BeforeTotalInit.NoSlowest != null)
                {
                    totalInitLinesNoSlowest.Add(CreateComparisonLine(cdata.BeforeTotalInit.NoSlowest.TotalInit, cdata.AfterTotalInit.NoSlowest !.TotalInit, notes));
                }
            }

            string reportFile = Path.Combine(resultsDir, Constants.ComparisonFileName);

            using (var sw = new StreamWriter(reportFile, false, Utilities.UTF8NoBOM)) {
                sw.WriteLine("# Reports");

                sw.WriteLine("## Before");
                WriteDescription(sw, reportOne.DateUTC, reportOne.Configuration, reportOne.XAVersion, reportOne.AndroidDevice);

                sw.WriteLine("## After");
                WriteDescription(sw, reportTwo.DateUTC, reportTwo.Configuration, reportTwo.XAVersion, reportTwo.AndroidDevice);

                sw.WriteLine("# Comparison");
                if (warnings.Count > 0)
                {
                    sw.WriteLine();
                    sw.WriteLine("**Warnings**:  ");

                    foreach (string w in warnings)
                    {
                        sw.WriteLine($"  * {w}");
                    }
                    sw.WriteLine();
                }

                sw.WriteLine();
                sw.WriteLine("## Displayed");
                WriteComparison(sw, "All runs", displayedLinesAll);
                WriteComparison(sw, "Without slowest and fastest runs", displayedLinesNoOutliers);
                WriteComparison(sw, "Without the slowest runs", displayedLinesNoSlowest);

                sw.WriteLine();
                sw.WriteLine("## Native to managed");
                WriteComparison(sw, "All runs", nativeToManagedLinesAll);
                WriteComparison(sw, "Without slowest and fastest runs", nativeToManagedLinesNoOutliers);
                WriteComparison(sw, "Without the slowest runs", nativeToManagedLinesNoSlowest);

                sw.WriteLine();
                sw.WriteLine("## Total init");
                WriteComparison(sw, "All runs", totalInitLinesAll);
                WriteComparison(sw, "Without slowest and fastest runs", totalInitLinesNoOutliers);
                WriteComparison(sw, "Without the slowest runs", totalInitLinesNoSlowest);

                sw.Flush();
            }

            return(reportFile);
        }
Пример #6
0
        async Task <(bool, BuildInfo?)> BuildAndInstall(RunDefinition run)
        {
            if (projectUsesGit && projectGitCommit == null)
            {
                (projectGitCommit, projectGitBranch) = await GetCommitHashAndBranch(FullProjectDirPath);
            }

            string buildCommand = run.BuildCommand;
            bool   usesDotnet   = String.Compare("dotnet", Path.GetFileName(buildCommand), StringComparison.OrdinalIgnoreCase) == 0;

            var args = new List <string> {
                $"-v:quiet"
            };

            args.AddRange(run.Args);

            string        projectPath    = Path.GetRelativePath(FullProjectDirPath, FullProjectFilePath);
            string        binlogBasePath = String.Empty;
            MSBuildCommon builder;
            BuildInfo     buildInfo;

            if (!usesDotnet)
            {
                var msbuild = ConfigureRunner(new MSBuildRunner(context, buildCommand));
                binlogBasePath = GetBinlogBasePath("restore");
                if (!await msbuild.Run(projectPath, binlogBasePath, "Restore", configuration, args))
                {
                    return(false, null);
                }

                binlogBasePath = GetBinlogBasePath("build");
                run.BinlogPath = GetRelativeBinlogPath(binlogBasePath);

                if (!await msbuild.Run(projectPath, binlogBasePath, "SignAndroidPackage", configuration, args))
                {
                    return(false, null);
                }

                buildInfo = FindFirstAndroidBuildInfo(await msbuild.GetBuildInfo(binlogBasePath));
                await Uninstall(buildInfo);

                binlogBasePath = GetBinlogBasePath("install");
                if (!await msbuild.Run(projectPath, binlogBasePath, "Install", configuration, args))
                {
                    return(false, null);
                }

                builder = msbuild;
            }
            else
            {
                var dotnet = ConfigureRunner(new DotnetRunner(context, buildCommand));
                binlogBasePath = GetBinlogBasePath("build");
                run.BinlogPath = GetRelativeBinlogPath(binlogBasePath);
                if (!await dotnet.Build(projectPath, binlogBasePath, configuration, args))
                {
                    return(false, null);
                }

                buildInfo = FindFirstAndroidBuildInfo(await dotnet.GetBuildInfo(binlogBasePath));
                await Uninstall(buildInfo);

                binlogBasePath = GetBinlogBasePath("install");
                if (!await dotnet.Install(projectPath, binlogBasePath, buildInfo.TargetFramework, configuration, args))
                {
                    return(false, null);
                }

                builder = dotnet;
            }

            if (xaVersionNotDetectedYet)
            {
                const string NotGit = "not a git build";

                Log.InfoLine("Retrieving Xamarin.Android version information");
                xaVersion = await GetXAVersion(builder, binlogBasePath);

                Log.InfoLine($"    Location: {xaVersion.RootDir}");
                Log.InfoLine($"     Version: {xaVersion.Version}");

                string hash   = String.IsNullOrEmpty(xaVersion.Commit) ? NotGit : xaVersion.Commit;
                string branch = String.IsNullOrEmpty(xaVersion.Commit) ? NotGit : xaVersion.Branch;
                Log.InfoLine($"  Git branch: {branch}");
                Log.InfoLine($"  Git commit: {hash}");
                xaVersionNotDetectedYet = false;
            }

            return(true, buildInfo);

            async Task <bool> Uninstall(BuildInfo buildInfo)
            {
                (string packageName, _) = GetPackageAndMainActivityNames(buildInfo, run);

                var adb = new AdbRunner(context);

                return(await adb.Uninstall(packageName));
            }

            T ConfigureRunner <T> (T runner) where T : MSBuildCommon
            {
                runner.WorkingDirectory   = FullProjectDirPath;
                runner.EchoStandardOutput = true;
                runner.EchoStandardError  = true;
                return(runner);
            }

            string GetBinlogBasePath(string phase)
            {
                return(GetLogBasePath(Constants.MSBuildLogDir, phase, run.LogTag, projectGitCommit, projectGitBranch));
            }

            string GetRelativeBinlogPath(string binlogBasePath)
            {
                return(Path.GetRelativePath(FullDataDirectoryPath, $"{binlogBasePath}.binlog"));
            }
        }