private string CompareEnvironmentSummary(ConcurrentDictionary <string, DependencySummary <string> > otherEnvironmentSummary)
        {
            var environmentCompare = SummaryAnalyzer.GenerateEnvironmentDifference(m_analyzer.Summary.EnvironmentSummary, otherEnvironmentSummary);

            if (environmentCompare.enviromentChanges.Count == 0 && environmentCompare.enviromentMissing.Count == 0)
            {
                return("Environments are identical.");
            }

            using (var output = new StringWriter(CultureInfo.InvariantCulture))
            {
                // Now compare
                output.WriteLine("List differences in distinct EnvironmentVariableName = value; pairs");
                output.WriteLine("defined in process pips between execution log current and previous.");
                output.WriteLine("=====================================================================");
                foreach (var values in environmentCompare.enviromentChanges)
                {
                    output.WriteLine("Environment variable ({0} pips): {1} = {2} <> {3}", values[1], values[0], values[2], values[3]);
                }

                if (environmentCompare.Item2.Count > 0)
                {
                    output.WriteLine("Current log missing following environmet variables:");
                    foreach (var values in environmentCompare.Item2)
                    {
                        output.WriteLine("Variable ({0} pips): {1} = {2}", values[1], values[0], values[3]);
                    }
                }

                return(output.ToString());
            }
        }
        internal static string GetPipEnvironmentMissReport(
            ConcurrentDictionary <string, DependencySummary <string> > environmentSummary,
            ConcurrentDictionary <string, DependencySummary <string> > otherEnvironmentSummary)
        {
            var environmentCompare = SummaryAnalyzer.GenerateEnvironmentDifference(environmentSummary, otherEnvironmentSummary);

            if (environmentCompare.enviromentChanges.Count == 0 && environmentCompare.enviromentMissing.Count == 0)
            {
                return(string.Empty);
            }

            using (var output = new StringWriter(CultureInfo.InvariantCulture))
            {
                output.WriteLine("\n   Environment Changes:");
                foreach (var values in environmentCompare.enviromentChanges)
                {
                    output.WriteLine("Environment variable ({0} pips): {1} = {2} <> {3}", values[1], values[0], values[2], values[3]);
                }

                if (environmentCompare.Item2.Count > 0)
                {
                    output.WriteLine("Current log missing following environmet variables:");
                    foreach (var values in environmentCompare.Item2)
                    {
                        output.WriteLine("Variable ({0} pips): {1} = {2}", values[1], values[0], values[3]);
                    }
                }

                return(output.ToString());
            }
        }
        internal string CompareDirectoryDependencySummary(ConcurrentDictionary <string, DependencySummary <string> > otherDirectorySummary, string otherLogPath)
        {
            var directoriesNotInTwo = SummaryAnalyzer.GetDependencyDiff(m_analyzer.Summary.DirectorySummary, otherDirectorySummary).ToList();
            var directoriesNotInOne = SummaryAnalyzer.GetDependencyDiff(otherDirectorySummary, m_analyzer.Summary.DirectorySummary).ToList();

            // Directory dependencies in some PIPs are distinct.
            using (var output = new StringWriter(CultureInfo.InvariantCulture))
            {
                var hadOutput = false;
                int longest   = directoriesNotInOne.Select(s => s.Name.Length).Concat(new[] { 50 }).Max();
                longest = directoriesNotInTwo.Select(s => s.Name.Length).Concat(new[] { longest }).Max();

                // List the differences
                output.WriteLine("List differences between a summary of distinct directory dependencies");
                output.WriteLine("defined in process pips between execution log current and previous.");
                output.WriteLine("=====================================================================");
                string formatString = "{0,-" + longest + "} | {1,19} | {2, 15} | {3, 6} | {4, 6}";
                output.WriteLine("Top Directory Dependency Differences:");
                output.WriteLine("Current Execution Log = {0}", m_analyzer.ExecutionLogPath);
                output.WriteLine("Previous Execution Log = {0}", otherLogPath);
                output.WriteLine(formatString, "Directory Dependency", "Pip Reference count", "% Pip Reference", "Current", "Previous");

                foreach (var s in directoriesNotInTwo)
                {
                    var referencePercentage = m_analyzer.GetProcessPipCount() > 0
                        ? (int)((s.Count * 100) / m_analyzer.GetProcessPipCount())
                        : 0;
                    output.WriteLine(formatString, s.Name.ToLowerInvariant(), s.Count, referencePercentage, "yes", "no");
                    hadOutput = true;
                }

                foreach (var s in directoriesNotInOne)
                {
                    var referencePercentage = m_analyzer.GetProcessPipCount() > 0
                        ? (int)((s.Count * 100) / m_analyzer.GetProcessPipCount())
                        : 0;
                    output.WriteLine(formatString, s.Name.ToLowerInvariant(), s.Count, referencePercentage, "no", "yes");
                    hadOutput = true;
                }

                return(hadOutput ? output.ToString() : "Top Directory Dependency Differences: none");
            }
        }
        public void PrintHygienatorTwoLogsCsvOutput(StreamWriter writer, SummaryAnalyzer analyzer)
        {
            // TODO: allo adding prefix : SourceNamespace,Island,BuildName,Branch,BuildDate,ComputerName,Architecture,
            writer.WriteLine(
                "timeStart,Name,directCount,rootPipTransitiveCount,pipDuration,pipKernelTime,pipUserTime,pipDurationCriticalPath,pipKernelTimeCriticalPath,pipUserTimeCriticalPath,dependencyType,FileHash");
            if (!m_analyzer.CompareSaltsEquals(analyzer))
            {
                // Add the flag changes
                var saltDiffs = m_analyzer.GetSaltsDifference(analyzer);
                foreach (var diff in saltDiffs)
                {
                    writer.WriteLine(
                        "0,{0},{1},{2},0,0,0,0,0,0,GlobalFlagsChange,",
                        diff,
                        m_analyzer.GetProcessPipCount(),
                        m_analyzer.GetProcessPipCount());
                }
            }

            PrintTransitiveDependenciesImpact(writer);
        }
        private void WriteSummary(SummaryAnalyzer otherAnalyzer, StreamWriter writer)
        {
            writer.WriteLine("Current Execution Log = {0}", m_analyzer.ExecutionLogPath);
            writer.WriteLine("Previous Execution Log: {0}", otherAnalyzer.ExecutionLogPath);
            writer.WriteLine("Number of Process Pips :  {0}", m_analyzer.GetProcessPipCount());
            writer.WriteLine("Number of executed Process Pips :  {0}", m_analyzer.GetExecutedProcessPipCount());

            var fileDependencyDiff = SummaryAnalyzer.GetFileDependencyDiff(m_analyzer.Summary.FileArtifactSummary, otherAnalyzer.Summary.FileArtifactSummary);

            writer.WriteLine("Number of dependency files with mismatch hash : {0}", fileDependencyDiff.Count());

            var observedDiff = SummaryAnalyzer.GetFileDependencyDiff(m_analyzer.Summary.ObservedSummary, otherAnalyzer.Summary.ObservedSummary);

            writer.WriteLine("Number of observed inputs with mismatch hash : {0}", observedDiff.Count());

            if (m_analyzer.Summary.DirectoryMembership.Count == 0 || otherAnalyzer.Summary.DirectoryMembership.Count == 0)
            {
                var fileName = m_analyzer.Summary.DirectoryMembership.Count == 0 ? m_analyzer.ExecutionLogPath : otherAnalyzer.ExecutionLogPath;
                writer.WriteLine("No directory membership event in execution log : {0}", fileName);
            }
            else
            {
                var directoryMembershipNotInTwo = SummaryAnalyzer.GetDirectoryMembershipDiff(m_analyzer.Summary.DirectoryMembership, otherAnalyzer.Summary.DirectoryMembership);
                var directoryMembershipNotInOne = SummaryAnalyzer.GetDirectoryMembershipDiff(otherAnalyzer.Summary.DirectoryMembership, m_analyzer.Summary.DirectoryMembership);
                writer.WriteLine("Number of directory membership differences : {0}", directoryMembershipNotInTwo.Count() + directoryMembershipNotInOne.Count);
            }

            int produced = m_analyzer.GetProducedFileCount();
            var cached   = m_analyzer.GetCachedFileCount();
            var upToDate = m_analyzer.GetUpToDateFileCount();

            writer.WriteLine("Current XLG Output Files : {0}/{1}/{2}(Produced/FromCache/UpToDate)", produced, cached, upToDate);

            produced = otherAnalyzer.GetProducedFileCount();
            cached   = otherAnalyzer.GetCachedFileCount();
            upToDate = otherAnalyzer.GetUpToDateFileCount();
            writer.WriteLine("Previous XLG Output Files : {0}/{1}/{2}(Produced/FromCache/UpToDate)", produced, cached, upToDate);
            writer.WriteLine();
        }
Esempio n. 6
0
        public int Analyze()
        {
            if (m_analyzer == null)
            {
                return(0);
            }

            m_analyzer.Prepare();
            if (m_analysisInput.ExecutionLogPath != null)
            {
                // NOTE: We call Prepare above so we don't need to prepare as a part of reading the execution log
                var reader = Task.Run(() => m_analyzer.ReadExecutionLog(prepare: false));
                if (m_mode == AnalysisMode.LogCompare)
                {
                    m_analyzerOther.Prepare();
                    var otherReader = Task.Run(() => m_analyzerOther.ReadExecutionLog());
                    otherReader.Wait();
                }

                if (m_mode == AnalysisMode.FailedPipsDump && m_analyzerOther != null)
                {
                    var start = DateTime.Now;
                    Console.WriteLine($"[{start}] Reading compare to Log");
                    var otherReader = Task.Run(() => m_analyzerOther.ReadExecutionLog());
                    otherReader.Wait();
                    var duration = DateTime.Now - start;
                    Console.WriteLine($"Done reading compare to log : duration = [{duration}]");
                }

                reader.Wait();

                if (m_mode == AnalysisMode.CacheMissLegacy)
                {
                    // First pass just to read in PipCacheMissType data
                    var otherReader = Task.Run(() => m_analyzerOther.ReadExecutionLog());
                    otherReader.Wait();

                    // Second pass to do fingerprint differences analysis
                    otherReader = Task.Run(() => m_analyzerOther.ReadExecutionLog());
                    otherReader.Wait();
                }
            }

            var exitCode = m_analyzer.Analyze();

            if (m_mode == AnalysisMode.FailedPipsDump && m_analyzerOther != null)
            {
                var failedPipsDump = (FailedPipsDumpAnalyzer)m_analyzer;
                exitCode = failedPipsDump.Compare(m_analyzerOther);
            }

            if (m_mode == AnalysisMode.LogCompare)
            {
                m_analyzerOther.Analyze();
                SummaryAnalyzer summary = (SummaryAnalyzer)m_analyzer;
                exitCode = summary.Compare((SummaryAnalyzer)m_analyzerOther);
            }

            if (m_mode == AnalysisMode.CacheMissLegacy)
            {
                exitCode = m_analyzerOther.Analyze();
            }

            m_analyzer?.Dispose();
            m_analyzerOther?.Dispose();

            TelemetryShutdown();
            return(exitCode);
        }
        /// <summary>
        /// Writes the artifact reason for execution when comparing the two pips into the csv file.
        /// The pips comparared are the roots in order of longest critical path and only its difference is written. No transitive down pips are compared.
        /// </summary>
        private void PrintTransitiveDependenciesImpact(StreamWriter writer)
        {
            var summariesToReport = m_analyzer.GetDifferecesToReport();

            // keep track of distinct dependencies
            var fileArtifacts        = new HashSet <FileArtifactSummary>();
            var environmentVariables = new HashSet <DependencySummary <string> >();
            var observedInputs       = new HashSet <ObservedInputSummary>();

            foreach (var pipDiff in summariesToReport.OrderByDescending(a => a.pipSummary1.CriticalPath.Time))
            {
                var pipSummary      = pipDiff.pipSummary1;
                var otherPipSummary = pipDiff.pipSummary2;

                if (pipSummary.NewPip)
                {
                    continue;
                }

                // TODO: add removed dependencies in output
                if (!pipSummary.Fingerprint.Equals(otherPipSummary.Fingerprint))
                {
                    var filesDiff = SummaryAnalyzer.GetFileDependencyDiff(pipSummary.DependencySummary, otherPipSummary.DependencySummary).ToList();
                    foreach (var item in filesDiff)
                    {
                        if (item.CachedCount > 0)
                        {
                            // This was reported cached in other pips so it has not changed
                            continue;
                        }

                        if (item.OutputOrigin == PipOutputOrigin.Produced || fileArtifacts.Contains(item))
                        {
                            // Output file so not interesting or already written
                            continue;
                        }

                        PrintCsvArtifact(writer, pipSummary, item, "Input", item.GetHashName());
                        fileArtifacts.Add(item);
                    }

                    var environmentDiff = SummaryAnalyzer.GetDependencyDiff(pipSummary.EnvironmentSummary, otherPipSummary.EnvironmentSummary).ToList();
                    foreach (var item in environmentDiff)
                    {
                        if (!environmentVariables.Contains(item))
                        {
                            PrintCsvArtifact(writer, pipSummary, item, "EnvironmentVariable", string.Empty);
                            environmentVariables.Add(item);
                        }
                    }
                }

                var observedDiff = SummaryAnalyzer.GetFileDependencyDiff(pipSummary.ObservedInputSummary, otherPipSummary.ObservedInputSummary).ToList();
                foreach (var item in observedDiff)
                {
                    if (item.CachedCount > 0)
                    {
                        // This was reported cached in other pips so it has not changed
                        continue;
                    }

                    if (!observedInputs.Contains(item))
                    {
                        PrintCsvArtifact(writer, pipSummary, item, "ObservedInput", item.GetHashName());
                        observedInputs.Add(item);
                    }
                }
            }
        }
        /// <summary>
        /// Summary of directory membership differences
        /// </summary>
        internal string CompareDirectoryMembership(SummaryAnalyzer otherSummaryAnalyzer)
        {
            var directoryMembership = otherSummaryAnalyzer.Summary.DirectoryMembership;
            var otherLogPath        = otherSummaryAnalyzer.ExecutionLogPath;

            if (m_analyzer.Summary.DirectoryMembership.Count == 0 || otherSummaryAnalyzer.Summary.DirectoryMembership.Count == 0)
            {
                var fileName = m_analyzer.Summary.DirectoryMembership.Count == 0 ? m_analyzer.ExecutionLogPath : otherLogPath;
                return(string.Format(CultureInfo.InvariantCulture, "No directory membership events in execution log : {0}", fileName));
            }

            // Find the top observed imput directories that have changed and make sure the enumeration is listed
            var observedDiff           = SummaryAnalyzer.GetFileDependencyDiff(m_analyzer.Summary.ObservedSummary, otherSummaryAnalyzer.Summary.ObservedSummary).ToList();
            var count                  = 0;
            var directoriesToEnumerate = new HashSet <string>();

            foreach (var entry in observedDiff)
            {
                if (m_analyzer.Summary.DirectoryMembership.ContainsKey(entry.Name))
                {
                    directoriesToEnumerate.Add(entry.Name);
                }

                if (count++ > m_analyzer.MaxDifferenceReportCount)
                {
                    break;
                }
            }

            var directoryMembershipNotInTwo = SummaryAnalyzer.GetDirectoryMembershipDiff(m_analyzer.Summary.DirectoryMembership, directoryMembership);
            var directoryMembershipNotInOne = SummaryAnalyzer.GetDirectoryMembershipDiff(directoryMembership, m_analyzer.Summary.DirectoryMembership);

            using (var output = new StringWriter(CultureInfo.InvariantCulture))
            {
                var hadOutput = false;
                output.WriteLine("Top Directory Membership Differences:");
                output.WriteLine("Current Execution Log = {0}", m_analyzer.ExecutionLogPath);
                output.WriteLine("Previous Execution Log = {0}", otherLogPath);
                output.WriteLine("Directory  |  Current Execution Log   | Previous Execution Log");

                int outReportCount = 0;

                // Enumerate changes in observed inputs first to show those relevant changes
                foreach (var directory in directoriesToEnumerate)
                {
                    List <string> entry;
                    if (directoryMembershipNotInTwo.TryGetValue(directory, out entry))
                    {
                        hadOutput = PrintDirectoryEnumeration(output, directory, entry);
                        if (outReportCount++ > m_analyzer.MaxDifferenceReportCount)
                        {
                            break;
                        }
                    }
                }

                foreach (var s in directoryMembershipNotInTwo)
                {
                    if (directoriesToEnumerate.Contains(s.Key))
                    {
                        continue;
                    }

                    hadOutput = PrintDirectoryEnumeration(output, s.Key, s.Value);
                    if (outReportCount++ > m_analyzer.MaxDifferenceReportCount)
                    {
                        break;
                    }
                }

                foreach (var s in directoryMembershipNotInOne)
                {
                    hadOutput = PrintDirectoryEnumeration(output, s.Key, s.Value);
                    if (outReportCount++ > m_analyzer.MaxDifferenceReportCount)
                    {
                        break;
                    }
                }

                return(hadOutput ? output.ToString() : "Directory Membership Differences: none");
            }
        }
        public void PrintTextReport(SummaryAnalyzer analyzer, StreamWriter writer)
        {
            WriteSummary(analyzer, writer);
            PrintTransitiveImpact(writer);
            writer.WriteLine(" Summary Section");
            writer.WriteLine("=====================================================================");
            writer.WriteLine("Environment variables Analysis");
            writer.WriteLine("=====================================================================");
            writer.WriteLine(CompareEnvironmentSummary(analyzer.Summary.EnvironmentSummary));
            writer.WriteLine();
            writer.WriteLine("=====================================================================");
            writer.WriteLine("Directory dependency summary analysis");
            writer.WriteLine("=====================================================================");
            writer.WriteLine(CompareDirectoryDependencySummary(analyzer.Summary.DirectorySummary, analyzer.ExecutionLogPath));
            writer.WriteLine();

            writer.WriteLine("=====================================================================");
            writer.WriteLine("File dependency summary analysis");
            writer.WriteLine("=====================================================================");
            var compareHeader = "Summary of file dependencies with mismatch hash" + Environment.NewLine +
                                "defined in process pips between execution log current and previous." +
                                Environment.NewLine +
                                "=====================================================================" +
                                Environment.NewLine +
                                "Top File Dependency Differences: orderd by Pip reference count" +
                                Environment.NewLine +
                                "File Name, source/output, origin, % direct PIP referenced, direct PIP reference Count, File Hash - Current, File Hash - Previous";
            var fileArtifactDiff = CompareFileDependencySummary(
                compareHeader,
                m_analyzer.Summary.FileArtifactSummary,
                analyzer.Summary.FileArtifactSummary);

            if (string.IsNullOrEmpty(fileArtifactDiff))
            {
                fileArtifactDiff = "Top File Dependency Differences: None";
            }

            writer.WriteLine(fileArtifactDiff);
            writer.WriteLine();

            writer.WriteLine("=====================================================================");
            writer.WriteLine("Observed inputs dependency summary analysis");
            writer.WriteLine("=====================================================================");
            compareHeader = "List differences between a summary of observed inputs" + Environment.NewLine +
                            "between execution log current and previous." + Environment.NewLine +
                            "=====================================================================" +
                            Environment.NewLine +
                            "Top Observed File Dependency Differences: orderd by Pip reference count" +
                            Environment.NewLine +
                            "File Name, Observed Input Type, % PIP referenced, PIP dependent Count , File Hash - Current, File Hash - Previous";
            var observedInputDiff = CompareObservedDependencySummary(
                compareHeader,
                m_analyzer.Summary.ObservedSummary,
                analyzer.Summary.ObservedSummary);

            if (string.IsNullOrEmpty(observedInputDiff))
            {
                observedInputDiff = "Top Observed File Dependency Differences: None";
            }

            writer.WriteLine(observedInputDiff);
            writer.WriteLine();
            writer.WriteLine("=====================================================================");
            writer.WriteLine("Directory membership difference");
            writer.WriteLine("=====================================================================");
            writer.WriteLine(CompareDirectoryMembership(analyzer));
            writer.WriteLine();
        }
 public SummaryAnalyzerTextWriter(SummaryAnalyzer analyzer)
 {
     m_analyzer = analyzer;
 }
        private void PrintTransitiveImpact(StreamWriter writer)
        {
            writer.WriteLine("=====================================================================");
            writer.WriteLine("Executed Process Analysis Current Execution Log : {0}", m_analyzer.ExecutionLogPath);
            writer.WriteLine("=====================================================================");
            writer.WriteLine("This section lists executed process pips in longest pole order in File A.");
            writer.WriteLine("Reason for execution by comparing to matching pip in File B.");
            writer.WriteLine("Dependent process pips executed");
            writer.WriteLine("Critical path of transitive down dependent pips.");
            writer.WriteLine("Dependent list of process Pips (optional)");
            writer.WriteLine("Reporting Top level executed pips");
            writer.WriteLine("=====================================================================");

            var summariesToReport  = m_analyzer.GetDifferecesToReport();
            var invalidationReason = new HashSet <string>();

            foreach (var pipDiff in summariesToReport.OrderByDescending(a => a.Item1.CriticalPath.Time))
            {
                var pipSummary      = pipDiff.pipSummary1;
                var otherPipSummary = pipDiff.pipSummary2;
                var pip             = pipSummary.Pip;

                bool weakFingerprintCacheMiss;
                m_analyzer.TryGetWeakFingerprintCacheMiss(pipSummary.Pip.PipId, out weakFingerprintCacheMiss);
                var missReason = pipSummary.UncacheablePip
                    ? "Un-cacheable Pip"
                    : weakFingerprintCacheMiss ? "Weak Fingerprint Cache-Miss" : "Strong Fingerprint Cache-Miss";
                if (!pipSummary.NewPip)
                {
                    if (!pipSummary.Fingerprint.Equals(otherPipSummary.Fingerprint))
                    {
                        missReason = CompareFileDependencySummary(
                            "   File Dependency Changes:",
                            pipSummary.DependencySummary,
                            otherPipSummary.DependencySummary);
                        missReason += GetPipEnvironmentMissReport(
                            pipSummary.EnvironmentSummary,
                            otherPipSummary.EnvironmentSummary);
                    }

                    missReason += CompareObservedDependencySummary(
                        "   Observed File Dependency Changes:",
                        pipSummary.ObservedInputSummary, otherPipSummary.ObservedInputSummary);
                }

                if (invalidationReason.Contains(missReason))
                {
                    // This invalidation is known, so cut down the noise on the report
                    continue;
                }

                writer.WriteLine(m_analyzer.GetPipDescription(pip));
                writer.WriteLine(" Transitive Process invalidated count: {0}", pipSummary.ExecutedDependentProcessCount);
                writer.WriteLine(" Cache-Miss reason analysis:");
                writer.WriteLine(SummaryAnalyzer.GetFingerprintMismatch(pipSummary, otherPipSummary));
                writer.WriteLine(missReason);
                if (!string.IsNullOrEmpty(missReason))
                {
                    invalidationReason.Add(missReason);
                }

                var executionStart     = m_analyzer.GetPipStartTime(pip);
                var executionStartText = executionStart.Equals(DateTime.MinValue) ? "-" : executionStart.ToString("h:mm:ss.ff", CultureInfo.InvariantCulture);

                writer.WriteLine(" Start time: {0}", executionStartText);
                writer.WriteLine(
                    " Critical path (seconds): Duration: ( {0} )  Kernel Time: ( {1} )  User Time: ( {2} )",
                    ToSeconds(pipSummary.CriticalPath.Time),
                    ToSeconds(pipSummary.CriticalPath.KernelTime),
                    ToSeconds(pipSummary.CriticalPath.UserTime));

                if (pipSummary.ExecutedDependentProcessCount > 0)
                {
                    var criticalPathReport = m_analyzer.GetCriticalPathText(pipSummary.CriticalPath);
                    writer.WriteLine();
                    writer.WriteLine(criticalPathReport);
                }

                if (m_analyzer.WriteTransitiveDownPips && pipSummary.ExecutedDependentProcessCount > 0)
                {
                    writer.WriteLine(WriteDependentProcess(pipSummary.DependentNodes));
                }

                writer.WriteLine("=====================================================================");
            }
        }