private DirectoryEntry CreateEntry(DirectoryArtifact dir)
{
var sealId = PipGraph.GetSealedDirectoryNode(dir).ToPipId();
var sealKind = PipTable.GetSealDirectoryKind(sealId);
var pipId = PipGraph.TryGetProducer(dir);
var producer = GetEntry(pipId);
var pip = (SealDirectory)GetPip(sealId);
var kind = pip.Kind.ToString();
if (pip.IsComposite)
{
kind = "Composite" + kind;
}
var entry = new DirectoryEntry()
{
Directory = dir,
Producer = GetEntry(pipId),
FileCount = GetContents(dir).Length,
Kind = sealKind,
SemistableHash = producer?.Identifier ?? string.Empty,
Identifier = $"{pip.FormattedSemiStableHash} [{kind}]",
Id = dir.PartialSealId.ToString(),
};
if (pip.Provenance.Token.Path.IsValid)
{
entry.SpecFileName = pip.Provenance.Token.Path.GetName(PathTable).ToString(StringTable);
}
return(entry);
}
public bool IsSourceOnlySeal(PipId pipId)
{
return(m_isSourceOnlySeal.GetOrAdd(pipId, true, (k, v) =>
{
var sealKind = PipTable.GetSealDirectoryKind(k);
if (sealKind == SealDirectoryKind.SourceAllDirectories || sealKind == SealDirectoryKind.SourceTopDirectoryOnly)
{
return true;
}
if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial)
{
var isSourceOnly = true;
var sd = (SealDirectory)GetPip(k);
foreach (var file in sd.Contents)
{
if (file.IsOutputFile)
{
isSourceOnly = false;
}
}
return isSourceOnly;
}
return false;
}).Item.Value);
}
public override void Prepare()
{
Directory.CreateDirectory(OutputFilePath);
m_writer = new StreamWriter(Path.Combine(OutputFilePath, "results.txt"));
Console.WriteLine("Creating nodes");
foreach (var node in DataflowGraph.Nodes)
{
m_mutableGraph.CreateNode();
}
Console.WriteLine("Created nodes");
foreach (var entry in PipGraph.AllOutputDirectoriesAndProducers)
{
var sealId = PipGraph.GetSealedDirectoryNode(entry.Key).ToPipId();
m_dynamicDirectoryProducers[sealId] = entry.Value;
}
foreach (CopyFile copyFile in PipGraph.RetrievePipsOfType(PipType.CopyFile))
{
m_copiedFilesByTarget[copyFile.Destination] = copyFile.Source;
}
foreach (var directory in PipGraph.AllSealDirectories)
{
var sealId = PipGraph.GetSealedDirectoryNode(directory).ToPipId();
var sealKind = PipTable.GetSealDirectoryKind(sealId);
// Populate map of whether this is a source only seal
IsSourceOnlySeal(sealId);
if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial)
{
PipId?singleProducer = null;
foreach (var file in PipGraph.ListSealedDirectoryContents(directory))
{
if (file.IsOutputFile)
{
var producer = PipGraph.TryGetProducer(file);
if (singleProducer == null)
{
singleProducer = producer;
}
else if (singleProducer != producer)
{
singleProducer = PipId.Invalid;
}
}
}
if (singleProducer.HasValue && singleProducer.Value.IsValid)
{
m_dynamicDirectoryProducers[sealId] = singleProducer.Value;
}
}
}
}
private string ToString(PipId pipId)
{
if (!pipId.IsValid)
{
return(string.Empty);
}
var pipType = PipTable.GetPipType(pipId);
var type = pipType == PipType.SealDirectory ? PipTable.GetSealDirectoryKind(pipId).ToString() : pipType.ToString();
return($"{PipTable.GetFormattedSemiStableHash(pipId)} [{type}]");
}
private ReadOnlyArray <FileArtifact> GetContents(DirectoryArtifact directoryArtifact)
{
var sealId = PipGraph.GetSealedDirectoryNode(directoryArtifact).ToPipId();
var sealKind = PipTable.GetSealDirectoryKind(sealId);
if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial)
{
return(PipGraph.ListSealedDirectoryContents(directoryArtifact));
}
else if (sealKind.IsOpaqueOutput())
{
return(m_dynamicContents[directoryArtifact]);
}
return(ReadOnlyArray <FileArtifact> .Empty);
}
public override int Analyze()
{
var rootExpander = new RootExpander(PathTable);
HashSet <ContentHash> hashes = new HashSet <ContentHash>();
hashes.Add(ContentHashingUtilities.ZeroHash);
HashSet <FileArtifact> files = new HashSet <FileArtifact>();
foreach (var root in Roots)
{
rootExpander.Add(AbsolutePath.Create(PathTable, root.Key), root.Value);
}
Func <AbsolutePath, string> expandRoot = absPath => absPath.ToString(PathTable, rootExpander);
var orderedPips = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process)
.Where(lazyPip => TargetSemiStableHash == null || TargetSemiStableHash == lazyPip.SemiStableHash)
.Select(lazyPip => (Process)lazyPip.HydratePip())
.ToLookup(process => process.Provenance.Token.Path)
.OrderBy(grouping => grouping.Key.ToString(PathTable, rootExpander))
.ToList();
using (var fingerprintStream = File.Create(DumpFilePath, bufferSize: 64 << 10 /* 64 KB */))
using (var hashWriter = new StreamWriter(DumpFilePath + ".hashes.txt"))
{
using (
var fingerprintArchive = CompressFile
? new ZipArchive(fingerprintStream, ZipArchiveMode.Create)
: null)
{
using (
var writer =
XmlWriter.Create(
CompressFile
? fingerprintArchive.CreateEntry("dump.xml", CompressionLevel.Fastest).Open()
: fingerprintStream, new XmlWriterSettings()
{
Indent = true
}))
{
int doneProcesses = 0;
var t = new Timer(
o =>
{
var done = doneProcesses;
Console.WriteLine("Processes Done: {0} of {1}", done, orderedPips.Count);
},
null,
5000,
5000);
try
{
writer.WriteStartElement("ProcessDump");
writer.WriteAttributeString("Count", orderedPips.Count.ToString(CultureInfo.InvariantCulture));
foreach (var specPipGroup in orderedPips)
{
writer.WriteStartElement("SpecFile");
writer.WriteAttributeString("Path", specPipGroup.Key.ToString(PathTable, rootExpander));
foreach (var pip in specPipGroup)
{
doneProcesses++;
writer.WriteStartElement("Process");
writer.WriteAttributeString("Name", pip.Executable.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("CMD", RenderProcessArguments(pip));
writer.WriteElementString("Description", pip.GetDescription(PipGraph.Context));
writer.WriteStartElement("EnvironmentVariables");
foreach (var environmentVariable in pip.EnvironmentVariables)
{
writer.WriteStartElement("Environment");
writer.WriteAttributeString("Name", environmentVariable.Name.ToString(PathTable.StringTable));
if (environmentVariable.Value.IsValid)
{
writer.WriteAttributeString("Value", environmentVariable.Value.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer));
}
else
{
writer.WriteAttributeString("Value", "Unset");
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Dependencies");
foreach (var input in pip.Dependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(input, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryDependencies");
foreach (var input in pip.DirectoryDependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
var kind = PipTable.GetSealDirectoryKind(PipGraph.GetSealedDirectoryNode(input).ToPipId());
writer.WriteAttributeString("Kind", kind.ToString());
// Print directory dependency file details when dumping a specific process
if (TargetSemiStableHash != null && (kind == SealDirectoryKind.Full || kind == SealDirectoryKind.Partial))
{
foreach (var file in PipGraph.ListSealedDirectoryContents(input))
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
else if (m_contents.TryGetValue(input, out var contents))
{
m_observedInputs.TryGetValue(pip.PipId.Value, out var observedInputs);
foreach (var file in contents)
{
// skip the files that were not accessed
if (observedInputs != null && !observedInputs.Contains(file.Path))
{
continue;
}
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Outputs");
foreach (var input in pip.FileOutputs)
{
writer.WriteStartElement("Item");
if (input.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
}
writer.WriteString(input.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryOutputs");
foreach (var output in pip.DirectoryOutputs)
{
writer.WriteStartElement("Directory");
{
writer.WriteAttributeString("Path", output.Path.ToString(PathTable, rootExpander));
if (m_contents.TryGetValue(output, out var contents))
{
writer.WriteStartElement("Contents");
{
foreach (var file in contents)
{
writer.WriteStartElement("Item");
if (file.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
}
writer.WriteString(file.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
if (pip.TempDirectory.IsValid)
{
writer.WriteElementString("TempDirectory", pip.TempDirectory.ToString(PathTable, rootExpander));
}
writer.WriteStartElement("AdditionalTempDirectories");
foreach (var item in pip.AdditionalTempDirectories)
{
writer.WriteElementString("Item", item.ToString(PathTable, rootExpander));
}
writer.WriteEndElement();
writer.WriteEndElement(); // Process
}
writer.WriteEndElement(); // SpecFile
}
writer.WriteEndElement(); // ProcessDump
}
finally
{
// kill and wait for the status timer to die...
using (var e = new AutoResetEvent(false))
{
t.Dispose(e);
e.WaitOne();
}
}
}
}
}
return(0);
}
public override int Analyze()
{
Console.WriteLine($"Analyzing");
m_block.Complete();
m_block.CompletionAsync().GetAwaiter().GetResult();
Console.WriteLine($"Writing Graph");
foreach (var pip in PipGraph.RetrieveAllPips())
{
var serializedPip = pip;
var nodeId = pip.PipId.ToNodeId();
bool addEdges = true;
if (pip.PipType == PipType.Process)
{
var entry = GetEntry(pip.PipId);
serializedPip = entry.Process;
addEdges = !entry.AddedEdges;
}
if (addEdges && AddEdgesForPips)
{
using (var scope = m_mutableGraph.AcquireExclusiveIncomingEdgeScope(nodeId))
{
foreach (var edge in DataflowGraph.GetIncomingEdges(nodeId))
{
scope.AddEdge(edge.OtherNode, edge.IsLight);
}
}
}
serializedPip.ResetPipIdForTesting();
m_pipTable.Add(nodeId.Value, serializedPip);
}
m_mutableGraph.Seal();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.DirectedGraph,
m_mutableGraph.Serialize,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.DirectedGraph)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipTable,
w => m_pipTable.Serialize(w, Environment.ProcessorCount),
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipTable)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipGraphId,
PipGraph.SerializeGraphId,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipGraphId)))
.GetAwaiter().GetResult();
Console.WriteLine($"Simulating [Reading]");
var simulator = new BuildSimulatorAnalyzer(Input);
simulator.Increment = SimulatorIncrement ?? simulator.Increment;
simulator.ExecutionData.DataflowGraph = m_mutableGraph;
simulator.OutputDirectory = OutputFilePath;
simulator.ReadExecutionLog();
Console.WriteLine($"Simulating [Analyzing]");
simulator.Analyze();
Console.WriteLine($"Blocking Dependency Analysis");
DisplayTable <DepColumn> depTable = new DisplayTable <DepColumn>(" , ");
foreach (var pipId in PipTable.Keys)
{
var pipType = PipTable.GetPipType(pipId);
if (pipType == PipType.Process)
{
var entry = GetEntry(pipId);
(PipId node, ulong cost)maxConsumedDependency = default;
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in entry.PipDependencies)
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.Key.ToNodeId()];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.Key, cost);
}
if (dep.Value != null && dep.Value.HasFlag(ContentFlag.Consumed))
{
if (!maxConsumedDependency.node.IsValid || cost > maxConsumedDependency.cost)
{
maxConsumedDependency = (dep.Key, cost);
}
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier}");
depTable.Set(DepColumn.MaxConsumedDependency, ToString(maxConsumedDependency.node));
depTable.Set(DepColumn.MaxConsumedDependencyChainCost, maxConsumedDependency.cost.ToMinutes());
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
}
else if (pipType == PipType.SealDirectory &&
!PipTable.GetSealDirectoryKind(pipId).IsSourceSeal() &&
!IsSourceOnlySeal(pipId))
{
var seal = (SealDirectory)GetPip(pipId);
var entry = GetEntry(seal.Directory);
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in DataflowGraph.GetIncomingEdges(pipId.ToNodeId()))
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.OtherNode];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.OtherNode.ToPipId(), cost);
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier} ({entry.FileCount} files)");
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
depTable.Set(DepColumn.Directory, seal.DirectoryRoot.ToString(PathTable));
}
}
using (var blockAnalysisWriter = new StreamWriter(Path.Combine(OutputFilePath, "blockAnalysis.txt")))
{
depTable.Write(blockAnalysisWriter);
}
m_writer.Dispose();
Console.WriteLine($"Analyzing complete");
return(0);
}