private void WriteActualAndSimulationResults(PipExecutionData data, SimulationResult actualSimulation) { // write results actual File.WriteAllLines(GetResultsPath("actual.durations.csv"), data.Spans.Where(ps => data.GetPipType(ps.Id) == PipType.Process).Select(ps => string.Join(",", data.GetName(ps.Id), ps.Duration.ToMinutes().ToString()))); File.WriteAllLines(GetResultsPath("actual.durations.txt"), data.Spans.Select(ps => ps.Duration.ToMinutes().ToString())); File.WriteAllLines(GetResultsPath("actual.starts.txt"), data.Spans.Select(ps => ps.StartTime.ToMinutes().ToString())); File.WriteAllLines(GetResultsPath("actual.ends.txt"), data.Spans.Select(ps => ps.EndTime.ToMinutes().ToString())); // write simulation actual File.WriteAllLines(GetResultsPath("actualSimulation.durations.txt"), actualSimulation.GetSpans().Select(ps => ps.Duration.ToMinutes().ToString())); File.WriteAllLines(GetResultsPath("actualSimulation.starts.txt"), actualSimulation.GetSpans().Select(ps => ps.StartTime.ToMinutes().ToString())); File.WriteAllLines(GetResultsPath("actualSimulation.ends.txt"), actualSimulation.GetSpans().Select(ps => ps.EndTime.ToMinutes().ToString())); File.WriteAllLines(GetResultsPath("heights.txt"), data.Spans.Where(ps => data.GetPipType(ps.Id) == PipType.Process) .Select(ps => data.DirectedGraph.GetNodeHeight(ps.Id)) .GroupBy(i => i) .OrderBy(g => g.Key) .Select(g => $"Height: {g.Key}, Count: {g.Count()}")); // information on each process during simulation DisplayTable <SimColumns> table = new DisplayTable <SimColumns>(" , "); using (var streamWriter = new StreamWriter(GetResultsPath("actualSimulation.txt"))) { foreach (var ps in actualSimulation.GetSpans()) { table.NextRow(); table.Set(SimColumns.Id, data.FormattedSemistableHashes[ps.Id]); table.Set(SimColumns.Thread, ps.Thread); table.Set(SimColumns.MinimumStartTime, actualSimulation.MinimumStartTimes[ps.Id].ToMinutes()); table.Set(SimColumns.StartTime, ps.StartTime.ToMinutes()); table.Set(SimColumns.EndTime, ps.EndTime.ToMinutes()); table.Set(SimColumns.Duration, ps.Duration.ToMinutes()); table.Set(SimColumns.Incoming, data.DirectedGraph.GetIncomingEdgesCount(ps.Id)); table.Set(SimColumns.Outgoing, data.DirectedGraph.GetIncomingEdgesCount(ps.Id)); table.Set(SimColumns.LongestRunningDependency, data.FormattedSemistableHashes.GetOrDefault(actualSimulation.LongestRunningDependency[ps.Id])); table.Set(SimColumns.LastRunningDependency, data.FormattedSemistableHashes.GetOrDefault(actualSimulation.LastRunningDependency[ps.Id])); } table.Write(streamWriter); } }
public void ProcessFingerprintComputed(ProcessFingerprintComputationEventData data) { if (data.Kind != FingerprintComputationKind.Execution) { return; } m_consumer = m_analyzer.GetEntry(data.PipId); m_consumedFilesByPath.Clear(); m_dependencyConsumedFileIndex.Clear(); m_dependencyConsumedFileEndIndex.Clear(); m_dependencies.Clear(); m_builder.Clear(); m_directoryDependenciesFilterMap.Clear(); m_directoryHasSources.Clear(); var computation = data.StrongFingerprintComputations[0]; var pip = (Process)m_analyzer.GetPip(data.PipId); PipArtifacts.ForEachInput(pip, input => { if (input.IsFile) { AddConsumedFile(input.FileArtifact, DirectoryArtifact.Invalid, ContentFlag.Static | ContentFlag.Consumed); } else { foreach (var file in m_analyzer.GetContents(input.DirectoryArtifact)) { if (file.IsSourceFile) { m_directoryHasSources.Add(input.DirectoryArtifact); } AddConsumedFile(file, input.DirectoryArtifact, ContentFlag.Dynamic); } } return(true); }, includeLazyInputs: false); foreach (var input in computation.ObservedInputs) { var flag = (ContentFlag)((int)ContentFlag.AbsentPathProbe << (int)input.Type) | ContentFlag.Consumed; if (input.Type == ObservedInputType.FileContentRead || input.Type == ObservedInputType.ExistingFileProbe) { if (m_consumedFilesByPath.TryGetValue(input.Path, out var file)) { file.AddFlag(ContentFlag.Consumed); if (file.SourceFile != null) { file.SourceFile.AddFlag(ContentFlag.Consumed); } if (file.FinalFile != null) { file.FinalFile.AddFlag(ContentFlag.Consumed); } } else { AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag | ContentFlag.Unknown); } } else if (m_analyzer.AllAccesses) { AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag); } } var entry = m_consumer; // Sort file dependencies for consistent output entry.FileDependencies.Sort(s_fileReferenceComparer); foreach (var fileDependency in entry.FileDependencies) { if (fileDependency.Producer != null) { var reference = entry.PipDependencies.GetOrAdd(fileDependency.Producer.PipId, p => new PipReference()); if (reference.Pip == null) { reference.Pip = m_analyzer.GetEntry(fileDependency.Producer.PipId); } reference.Flags |= fileDependency.ConsumedFile.Flags; } } string describe(PipEntry pe) { return($"{pe.SpecFileName}-{m_analyzer.GetDescription(m_analyzer.GetPip(pe.PipId))}"); } m_builder.AppendLine(describe(entry)); foreach (var fileDependency in entry.FileDependencies) { if (fileDependency.Producer != null && fileDependency.ConsumedFile.File.Artifact.IsOutputFile) { var pipId = fileDependency.Producer.PipId; var pipReference = entry.PipDependencies[pipId]; var directory = fileDependency.Directory?.Directory ?? DirectoryArtifact.Invalid; if (m_dependencies.Add((pipId, directory))) { if (pipReference.HasFlag(ContentFlag.Consumed)) { m_directoryDependenciesFilterMap[directory] = true; m_builder.AppendLine($"{entry.Identifier} -> Retaining pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}') (consumes '{ToString(fileDependency.ConsumedFile.File.Artifact)}')"); } else { m_directoryDependenciesFilterMap.TryAdd(directory, false); m_builder.AppendLine($"{entry.Identifier} -> Removing pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}')"); } } } } var trimmedDirectoryDependencies = new List <DirectoryArtifact>(); foreach (var d in entry.Process.DirectoryDependencies) { if (m_directoryDependenciesFilterMap.TryGetValue(d, out var shouldInclude)) { if (shouldInclude) { m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (used)"); } else if (m_directoryHasSources.Contains(d)) { m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (has sources)"); } else { m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}'"); continue; } } else { var sealId = m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId(); if (!m_directoryHasSources.Contains(d) && !m_analyzer.PipTable.GetSealDirectoryKind(sealId).IsSourceSeal()) { m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}' (unused output directory)"); continue; } } entry.PipDependencies.TryAdd(m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId(), default); trimmedDirectoryDependencies.Add(d); } // Update directory dependencies which trimmed directory dependencies to allow writing // a pip into the serialized pip table that can run without the unnecessary dependencies entry.Process.UnsafeUpdateDirectoryDependencies(trimmedDirectoryDependencies.ToReadOnlyArray()); m_builder.AppendLine(); // Update the graph var modifiedGraph = m_analyzer.m_mutableGraph; using (var scope = modifiedGraph.AcquireExclusiveIncomingEdgeScope(entry.PipId.ToNodeId())) { foreach (var dependency in entry.PipDependencies) { if (dependency.Value == null || dependency.Value.HasFlag(ContentFlag.Consumed)) { scope.AddEdge(dependency.Key.ToNodeId()); } } entry.AddedEdges = true; } if (m_analyzer.SemiStableHashes.Contains(entry.SemistableHash)) { using (var writer = new StreamWriter(Path.Combine(m_analyzer.OutputFilePath, $"{GetFileName(entry.SpecFile)}_Pip{pip.FormattedSemiStableHash}.csv"))) { var table = new DisplayTable <Columns>(" , "); foreach (var dependency in entry.FileDependencies) { table.NextRow(); table.Set(Columns.Path, ToString(dependency.ConsumedFile.File.Artifact.Path)); table.Set(Columns.RwCount, dependency.ConsumedFile.File.Artifact.RewriteCount.ToString()); table.Set(Columns.Flags, dependency.ConsumedFile.Flags.ToString()); table.Set(Columns.Producer, dependency.Producer?.Identifier); table.Set(Columns.ProducerSpec, GetFileName(dependency.Producer?.SpecFile ?? AbsolutePath.Invalid)); table.Set(Columns.Dir, ToString(dependency.Directory)); table.Set(Columns.DirId, dependency.Directory?.Id); table.Set(Columns.DirSsh, dependency.Directory?.SemistableHash); } table.Write(writer); } } if (m_builder.Length != 0) { m_analyzer.Write(m_builder); } }
public override int Analyze() { Console.WriteLine($"Analyzing"); m_block.Complete(); m_block.CompletionAsync().GetAwaiter().GetResult(); Console.WriteLine($"Writing Graph"); foreach (var pip in PipGraph.RetrieveAllPips()) { var serializedPip = pip; var nodeId = pip.PipId.ToNodeId(); bool addEdges = true; if (pip.PipType == PipType.Process) { var entry = GetEntry(pip.PipId); serializedPip = entry.Process; addEdges = !entry.AddedEdges; } if (addEdges && AddEdgesForPips) { using (var scope = m_mutableGraph.AcquireExclusiveIncomingEdgeScope(nodeId)) { foreach (var edge in DataflowGraph.GetIncomingEdges(nodeId)) { scope.AddEdge(edge.OtherNode, edge.IsLight); } } } serializedPip.ResetPipIdForTesting(); m_pipTable.Add(nodeId.Value, serializedPip); } m_mutableGraph.Seal(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.DirectedGraph, m_mutableGraph.Serialize, Path.Combine(OutputFilePath, nameof(GraphCacheFile.DirectedGraph))) .GetAwaiter().GetResult(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.PipTable, w => m_pipTable.Serialize(w, Environment.ProcessorCount), Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipTable))) .GetAwaiter().GetResult(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.PipGraphId, PipGraph.SerializeGraphId, Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipGraphId))) .GetAwaiter().GetResult(); Console.WriteLine($"Simulating [Reading]"); var simulator = new BuildSimulatorAnalyzer(Input); simulator.Increment = SimulatorIncrement ?? simulator.Increment; simulator.ExecutionData.DataflowGraph = m_mutableGraph; simulator.OutputDirectory = OutputFilePath; simulator.ReadExecutionLog(); Console.WriteLine($"Simulating [Analyzing]"); simulator.Analyze(); Console.WriteLine($"Blocking Dependency Analysis"); DisplayTable <DepColumn> depTable = new DisplayTable <DepColumn>(" , "); foreach (var pipId in PipTable.Keys) { var pipType = PipTable.GetPipType(pipId); if (pipType == PipType.Process) { var entry = GetEntry(pipId); (PipId node, ulong cost)maxConsumedDependency = default; (PipId node, ulong cost)maxDependency = default; foreach (var dep in entry.PipDependencies) { var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.Key.ToNodeId()]; if (!maxDependency.node.IsValid || cost > maxDependency.cost) { maxDependency = (dep.Key, cost); } if (dep.Value != null && dep.Value.HasFlag(ContentFlag.Consumed)) { if (!maxConsumedDependency.node.IsValid || cost > maxConsumedDependency.cost) { maxConsumedDependency = (dep.Key, cost); } } } depTable.NextRow(); depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier}"); depTable.Set(DepColumn.MaxConsumedDependency, ToString(maxConsumedDependency.node)); depTable.Set(DepColumn.MaxConsumedDependencyChainCost, maxConsumedDependency.cost.ToMinutes()); depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node)); depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes()); } else if (pipType == PipType.SealDirectory && !PipTable.GetSealDirectoryKind(pipId).IsSourceSeal() && !IsSourceOnlySeal(pipId)) { var seal = (SealDirectory)GetPip(pipId); var entry = GetEntry(seal.Directory); (PipId node, ulong cost)maxDependency = default; foreach (var dep in DataflowGraph.GetIncomingEdges(pipId.ToNodeId())) { var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.OtherNode]; if (!maxDependency.node.IsValid || cost > maxDependency.cost) { maxDependency = (dep.OtherNode.ToPipId(), cost); } } depTable.NextRow(); depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier} ({entry.FileCount} files)"); depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node)); depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes()); depTable.Set(DepColumn.Directory, seal.DirectoryRoot.ToString(PathTable)); } } using (var blockAnalysisWriter = new StreamWriter(Path.Combine(OutputFilePath, "blockAnalysis.txt"))) { depTable.Write(blockAnalysisWriter); } m_writer.Dispose(); Console.WriteLine($"Analyzing complete"); return(0); }