private void WriteActualAndSimulationResults(PipExecutionData data, SimulationResult actualSimulation)
{
// write results actual
File.WriteAllLines(GetResultsPath("actual.durations.csv"), data.Spans.Where(ps => data.GetPipType(ps.Id) == PipType.Process).Select(ps => string.Join(",", data.GetName(ps.Id), ps.Duration.ToMinutes().ToString())));
File.WriteAllLines(GetResultsPath("actual.durations.txt"), data.Spans.Select(ps => ps.Duration.ToMinutes().ToString()));
File.WriteAllLines(GetResultsPath("actual.starts.txt"), data.Spans.Select(ps => ps.StartTime.ToMinutes().ToString()));
File.WriteAllLines(GetResultsPath("actual.ends.txt"), data.Spans.Select(ps => ps.EndTime.ToMinutes().ToString()));
// write simulation actual
File.WriteAllLines(GetResultsPath("actualSimulation.durations.txt"), actualSimulation.GetSpans().Select(ps => ps.Duration.ToMinutes().ToString()));
File.WriteAllLines(GetResultsPath("actualSimulation.starts.txt"), actualSimulation.GetSpans().Select(ps => ps.StartTime.ToMinutes().ToString()));
File.WriteAllLines(GetResultsPath("actualSimulation.ends.txt"), actualSimulation.GetSpans().Select(ps => ps.EndTime.ToMinutes().ToString()));
File.WriteAllLines(GetResultsPath("heights.txt"), data.Spans.Where(ps => data.GetPipType(ps.Id) == PipType.Process)
.Select(ps => data.DirectedGraph.GetNodeHeight(ps.Id))
.GroupBy(i => i)
.OrderBy(g => g.Key)
.Select(g => $"Height: {g.Key}, Count: {g.Count()}"));
// information on each process during simulation
DisplayTable <SimColumns> table = new DisplayTable <SimColumns>(" , ");
using (var streamWriter = new StreamWriter(GetResultsPath("actualSimulation.txt")))
{
foreach (var ps in actualSimulation.GetSpans())
{
table.NextRow();
table.Set(SimColumns.Id, data.FormattedSemistableHashes[ps.Id]);
table.Set(SimColumns.Thread, ps.Thread);
table.Set(SimColumns.MinimumStartTime, actualSimulation.MinimumStartTimes[ps.Id].ToMinutes());
table.Set(SimColumns.StartTime, ps.StartTime.ToMinutes());
table.Set(SimColumns.EndTime, ps.EndTime.ToMinutes());
table.Set(SimColumns.Duration, ps.Duration.ToMinutes());
table.Set(SimColumns.Incoming, data.DirectedGraph.GetIncomingEdgesCount(ps.Id));
table.Set(SimColumns.Outgoing, data.DirectedGraph.GetIncomingEdgesCount(ps.Id));
table.Set(SimColumns.LongestRunningDependency, data.FormattedSemistableHashes.GetOrDefault(actualSimulation.LongestRunningDependency[ps.Id]));
table.Set(SimColumns.LastRunningDependency, data.FormattedSemistableHashes.GetOrDefault(actualSimulation.LastRunningDependency[ps.Id]));
}
table.Write(streamWriter);
}
}
public void ProcessFingerprintComputed(ProcessFingerprintComputationEventData data)
{
if (data.Kind != FingerprintComputationKind.Execution)
{
return;
}
m_consumer = m_analyzer.GetEntry(data.PipId);
m_consumedFilesByPath.Clear();
m_dependencyConsumedFileIndex.Clear();
m_dependencyConsumedFileEndIndex.Clear();
m_dependencies.Clear();
m_builder.Clear();
m_directoryDependenciesFilterMap.Clear();
m_directoryHasSources.Clear();
var computation = data.StrongFingerprintComputations[0];
var pip = (Process)m_analyzer.GetPip(data.PipId);
PipArtifacts.ForEachInput(pip, input =>
{
if (input.IsFile)
{
AddConsumedFile(input.FileArtifact, DirectoryArtifact.Invalid, ContentFlag.Static | ContentFlag.Consumed);
}
else
{
foreach (var file in m_analyzer.GetContents(input.DirectoryArtifact))
{
if (file.IsSourceFile)
{
m_directoryHasSources.Add(input.DirectoryArtifact);
}
AddConsumedFile(file, input.DirectoryArtifact, ContentFlag.Dynamic);
}
}
return(true);
}, includeLazyInputs: false);
foreach (var input in computation.ObservedInputs)
{
var flag = (ContentFlag)((int)ContentFlag.AbsentPathProbe << (int)input.Type) | ContentFlag.Consumed;
if (input.Type == ObservedInputType.FileContentRead || input.Type == ObservedInputType.ExistingFileProbe)
{
if (m_consumedFilesByPath.TryGetValue(input.Path, out var file))
{
file.AddFlag(ContentFlag.Consumed);
if (file.SourceFile != null)
{
file.SourceFile.AddFlag(ContentFlag.Consumed);
}
if (file.FinalFile != null)
{
file.FinalFile.AddFlag(ContentFlag.Consumed);
}
}
else
{
AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag | ContentFlag.Unknown);
}
}
else if (m_analyzer.AllAccesses)
{
AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag);
}
}
var entry = m_consumer;
// Sort file dependencies for consistent output
entry.FileDependencies.Sort(s_fileReferenceComparer);
foreach (var fileDependency in entry.FileDependencies)
{
if (fileDependency.Producer != null)
{
var reference = entry.PipDependencies.GetOrAdd(fileDependency.Producer.PipId, p => new PipReference());
if (reference.Pip == null)
{
reference.Pip = m_analyzer.GetEntry(fileDependency.Producer.PipId);
}
reference.Flags |= fileDependency.ConsumedFile.Flags;
}
}
string describe(PipEntry pe)
{
return($"{pe.SpecFileName}-{m_analyzer.GetDescription(m_analyzer.GetPip(pe.PipId))}");
}
m_builder.AppendLine(describe(entry));
foreach (var fileDependency in entry.FileDependencies)
{
if (fileDependency.Producer != null &&
fileDependency.ConsumedFile.File.Artifact.IsOutputFile)
{
var pipId = fileDependency.Producer.PipId;
var pipReference = entry.PipDependencies[pipId];
var directory = fileDependency.Directory?.Directory ?? DirectoryArtifact.Invalid;
if (m_dependencies.Add((pipId, directory)))
{
if (pipReference.HasFlag(ContentFlag.Consumed))
{
m_directoryDependenciesFilterMap[directory] = true;
m_builder.AppendLine($"{entry.Identifier} -> Retaining pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}') (consumes '{ToString(fileDependency.ConsumedFile.File.Artifact)}')");
}
else
{
m_directoryDependenciesFilterMap.TryAdd(directory, false);
m_builder.AppendLine($"{entry.Identifier} -> Removing pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}')");
}
}
}
}
var trimmedDirectoryDependencies = new List <DirectoryArtifact>();
foreach (var d in entry.Process.DirectoryDependencies)
{
if (m_directoryDependenciesFilterMap.TryGetValue(d, out var shouldInclude))
{
if (shouldInclude)
{
m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (used)");
}
else if (m_directoryHasSources.Contains(d))
{
m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (has sources)");
}
else
{
m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}'");
continue;
}
}
else
{
var sealId = m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId();
if (!m_directoryHasSources.Contains(d) && !m_analyzer.PipTable.GetSealDirectoryKind(sealId).IsSourceSeal())
{
m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}' (unused output directory)");
continue;
}
}
entry.PipDependencies.TryAdd(m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId(), default);
trimmedDirectoryDependencies.Add(d);
}
// Update directory dependencies which trimmed directory dependencies to allow writing
// a pip into the serialized pip table that can run without the unnecessary dependencies
entry.Process.UnsafeUpdateDirectoryDependencies(trimmedDirectoryDependencies.ToReadOnlyArray());
m_builder.AppendLine();
// Update the graph
var modifiedGraph = m_analyzer.m_mutableGraph;
using (var scope = modifiedGraph.AcquireExclusiveIncomingEdgeScope(entry.PipId.ToNodeId()))
{
foreach (var dependency in entry.PipDependencies)
{
if (dependency.Value == null || dependency.Value.HasFlag(ContentFlag.Consumed))
{
scope.AddEdge(dependency.Key.ToNodeId());
}
}
entry.AddedEdges = true;
}
if (m_analyzer.SemiStableHashes.Contains(entry.SemistableHash))
{
using (var writer = new StreamWriter(Path.Combine(m_analyzer.OutputFilePath,
$"{GetFileName(entry.SpecFile)}_Pip{pip.FormattedSemiStableHash}.csv")))
{
var table = new DisplayTable <Columns>(" , ");
foreach (var dependency in entry.FileDependencies)
{
table.NextRow();
table.Set(Columns.Path, ToString(dependency.ConsumedFile.File.Artifact.Path));
table.Set(Columns.RwCount, dependency.ConsumedFile.File.Artifact.RewriteCount.ToString());
table.Set(Columns.Flags, dependency.ConsumedFile.Flags.ToString());
table.Set(Columns.Producer, dependency.Producer?.Identifier);
table.Set(Columns.ProducerSpec, GetFileName(dependency.Producer?.SpecFile ?? AbsolutePath.Invalid));
table.Set(Columns.Dir, ToString(dependency.Directory));
table.Set(Columns.DirId, dependency.Directory?.Id);
table.Set(Columns.DirSsh, dependency.Directory?.SemistableHash);
}
table.Write(writer);
}
}
if (m_builder.Length != 0)
{
m_analyzer.Write(m_builder);
}
}
public override int Analyze()
{
Console.WriteLine($"Analyzing");
m_block.Complete();
m_block.CompletionAsync().GetAwaiter().GetResult();
Console.WriteLine($"Writing Graph");
foreach (var pip in PipGraph.RetrieveAllPips())
{
var serializedPip = pip;
var nodeId = pip.PipId.ToNodeId();
bool addEdges = true;
if (pip.PipType == PipType.Process)
{
var entry = GetEntry(pip.PipId);
serializedPip = entry.Process;
addEdges = !entry.AddedEdges;
}
if (addEdges && AddEdgesForPips)
{
using (var scope = m_mutableGraph.AcquireExclusiveIncomingEdgeScope(nodeId))
{
foreach (var edge in DataflowGraph.GetIncomingEdges(nodeId))
{
scope.AddEdge(edge.OtherNode, edge.IsLight);
}
}
}
serializedPip.ResetPipIdForTesting();
m_pipTable.Add(nodeId.Value, serializedPip);
}
m_mutableGraph.Seal();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.DirectedGraph,
m_mutableGraph.Serialize,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.DirectedGraph)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipTable,
w => m_pipTable.Serialize(w, Environment.ProcessorCount),
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipTable)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipGraphId,
PipGraph.SerializeGraphId,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipGraphId)))
.GetAwaiter().GetResult();
Console.WriteLine($"Simulating [Reading]");
var simulator = new BuildSimulatorAnalyzer(Input);
simulator.Increment = SimulatorIncrement ?? simulator.Increment;
simulator.ExecutionData.DataflowGraph = m_mutableGraph;
simulator.OutputDirectory = OutputFilePath;
simulator.ReadExecutionLog();
Console.WriteLine($"Simulating [Analyzing]");
simulator.Analyze();
Console.WriteLine($"Blocking Dependency Analysis");
DisplayTable <DepColumn> depTable = new DisplayTable <DepColumn>(" , ");
foreach (var pipId in PipTable.Keys)
{
var pipType = PipTable.GetPipType(pipId);
if (pipType == PipType.Process)
{
var entry = GetEntry(pipId);
(PipId node, ulong cost)maxConsumedDependency = default;
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in entry.PipDependencies)
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.Key.ToNodeId()];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.Key, cost);
}
if (dep.Value != null && dep.Value.HasFlag(ContentFlag.Consumed))
{
if (!maxConsumedDependency.node.IsValid || cost > maxConsumedDependency.cost)
{
maxConsumedDependency = (dep.Key, cost);
}
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier}");
depTable.Set(DepColumn.MaxConsumedDependency, ToString(maxConsumedDependency.node));
depTable.Set(DepColumn.MaxConsumedDependencyChainCost, maxConsumedDependency.cost.ToMinutes());
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
}
else if (pipType == PipType.SealDirectory &&
!PipTable.GetSealDirectoryKind(pipId).IsSourceSeal() &&
!IsSourceOnlySeal(pipId))
{
var seal = (SealDirectory)GetPip(pipId);
var entry = GetEntry(seal.Directory);
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in DataflowGraph.GetIncomingEdges(pipId.ToNodeId()))
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.OtherNode];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.OtherNode.ToPipId(), cost);
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier} ({entry.FileCount} files)");
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
depTable.Set(DepColumn.Directory, seal.DirectoryRoot.ToString(PathTable));
}
}
using (var blockAnalysisWriter = new StreamWriter(Path.Combine(OutputFilePath, "blockAnalysis.txt")))
{
depTable.Write(blockAnalysisWriter);
}
m_writer.Dispose();
Console.WriteLine($"Analyzing complete");
return(0);
}
