public void Serialize(string path)
{
Console.WriteLine("Serializing data...");
using (var fs = File.Open(path, FileMode.OpenOrCreate))
using (var writer = new BuildXLWriter(false, fs, false, false))
{
writer.Write(VersionGuid);
var maxNode = (uint)DataflowGraph.NodeCount;
DataflowGraph.Serialize(writer);
//writer.Write(maxNode, AggregateCosts, (w, i) => w.Write(i));
writer.Write(maxNode, PipIds, (w, i) => w.Write(i));
writer.Write(maxNode, SemiStableHashes, (w, i) => w.Write(i));
writer.Write(maxNode, StartTimes, (w, i) => w.Write(i));
writer.Write(maxNode, Durations, (w, i) => w.Write(i));
writer.Write(maxNode, PipTypes, (w, i) => w.Write((byte)i));
//writer.Write(maxNode, CriticalChain, WriteNode);
writer.Write(MinStartTime);
writer.Write(MaxEndTime);
writer.Write(TotalDuration);
//writer.Write(MaxAggregateCost);
//WriteNode(writer, CriticalPathHeadNode);
//writer.Write(CriticalPath, WriteNode);
SymbolTable.StringTable.Serialize(writer);
SymbolTable.Serialize(writer);
writer.Write(maxNode, OutputValues, (w, i) => w.Write(i));
}
Console.WriteLine("End serializing data");
}
public ConcurrentNodeDictionary <ulong> ComputeAggregateCosts(ConcurrentNodeDictionary <ulong> durations)
{
ConcurrentNodeDictionary <ulong> aggregateCosts = new ConcurrentNodeDictionary <ulong>(true);
List <NodeId> sortedNodes = new List <NodeId>();
sortedNodes.AddRange(DataflowGraph.Nodes);
sortedNodes.Sort((n1, n2) => - DataflowGraph.GetNodeHeight(n1).CompareTo(DataflowGraph.GetNodeHeight(n2)));
foreach (var node in sortedNodes)
{
// int maxConeConcurrency = 0;
ulong aggregateCost = 0;
NodeId maxChild = NodeId.Invalid;
foreach (var outgoing in DataflowGraph.GetOutgoingEdges(node))
{
if (aggregateCosts[outgoing.OtherNode].Max(ref aggregateCost) || !maxChild.IsValid)
{
maxChild = outgoing.OtherNode;
}
}
aggregateCost += durations[node];
aggregateCosts[node] = aggregateCost;
}
return(aggregateCosts);
}
public void ComputeAggregateCosts()
{
PipAndPriority maxPipAndPriority = default(PipAndPriority);
List <NodeId> sortedNodes = new List <NodeId>();
sortedNodes.AddRange(DataflowGraph.Nodes);
sortedNodes.Sort((n1, n2) => DataflowGraph.GetNodeHeight(n1).CompareTo(DataflowGraph.GetNodeHeight(n2)));
foreach (var node in sortedNodes)
{
ulong aggregateCost = 0;
foreach (var incoming in DataflowGraph.GetIncomingEdges(node))
{
BottomUpAggregateCosts[incoming.OtherNode].Max(ref aggregateCost);
}
aggregateCost += Durations[node];
BottomUpAggregateCosts[node] = aggregateCost;
}
sortedNodes.Sort((n1, n2) => - DataflowGraph.GetNodeHeight(n1).CompareTo(DataflowGraph.GetNodeHeight(n2)));
foreach (var node in sortedNodes)
{
// int maxConeConcurrency = 0;
ulong aggregateCost = 0, constrainedAggregateCost = 0;
NodeId maxChild = NodeId.Invalid;
foreach (var outgoing in DataflowGraph.GetOutgoingEdges(node))
{
ConstrainedAggregateCosts[outgoing.OtherNode].Max(ref constrainedAggregateCost);
if (AggregateCosts[outgoing.OtherNode].Max(ref aggregateCost) || !maxChild.IsValid)
{
maxChild = outgoing.OtherNode;
}
}
FormattedSemistableHashes[node] = CachedGraph.PipTable.GetFormattedSemiStableHash(node.ToPipId());
aggregateCost += Durations[node];
AggregateCosts[node] = aggregateCost;
aggregateCost.Max(ref MaxAggregateCost);
CriticalChain[node] = maxChild;
new PipAndPriority
{
Node = node,
Priority = aggregateCost,
}.Max(ref maxPipAndPriority);
}
CriticalPathHeadNode = maxPipAndPriority.Node;
NodeId criticalChainNode = CriticalPathHeadNode;
while (criticalChainNode.IsValid)
{
CriticalPath.Add(criticalChainNode);
criticalChainNode = CriticalChain[criticalChainNode];
}
}
/// <summary>
/// Exports the contents of the given pip graph. It is expected to no longer change as of the start of this call.
/// </summary>
public override int Analyze()
{
DgmlWriter writer = new DgmlWriter();
Dictionary <PipId, PipReference> concretePipValues = new Dictionary <PipId, PipReference>();
HashSet <PipReference> allValuePips = new HashSet <PipReference>(PipGraph.RetrievePipReferencesOfType(PipType.Value));
foreach (var valuePip in allValuePips)
{
foreach (var incoming in DataflowGraph.GetIncomingEdges(valuePip.PipId.ToNodeId()))
{
if (!PipTable.GetPipType(incoming.OtherNode.ToPipId()).IsMetaPip())
{
concretePipValues[valuePip.PipId] = valuePip;
}
}
}
foreach (var concretePipValue in concretePipValues.Values)
{
var value = ((ValuePip)concretePipValue.HydratePip()).Symbol.ToString(SymbolTable);
writer.AddNode(new DgmlWriter.Node(concretePipValue.PipId.ToString(), value));
foreach (var incoming in DataflowGraph.GetIncomingEdges(concretePipValue.PipId.ToNodeId()))
{
var incomingId = incoming.OtherNode.ToPipId();
if (concretePipValues.ContainsKey(incomingId))
{
writer.AddLink(new DgmlWriter.Link(
incomingId.ToString(),
concretePipValue.PipId.ToString(),
label: null));
}
}
}
writer.Serialize(m_dgmlFilePath);
return(0);
}
public override int Analyze()
{
Console.WriteLine($"Analyzing");
m_block.Complete();
m_block.CompletionAsync().GetAwaiter().GetResult();
Console.WriteLine($"Writing Graph");
foreach (var pip in PipGraph.RetrieveAllPips())
{
var serializedPip = pip;
var nodeId = pip.PipId.ToNodeId();
bool addEdges = true;
if (pip.PipType == PipType.Process)
{
var entry = GetEntry(pip.PipId);
serializedPip = entry.Process;
addEdges = !entry.AddedEdges;
}
if (addEdges && AddEdgesForPips)
{
using (var scope = m_mutableGraph.AcquireExclusiveIncomingEdgeScope(nodeId))
{
foreach (var edge in DataflowGraph.GetIncomingEdges(nodeId))
{
scope.AddEdge(edge.OtherNode, edge.IsLight);
}
}
}
serializedPip.ResetPipIdForTesting();
m_pipTable.Add(nodeId.Value, serializedPip);
}
m_mutableGraph.Seal();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.DirectedGraph,
m_mutableGraph.Serialize,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.DirectedGraph)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipTable,
w => m_pipTable.Serialize(w, Environment.ProcessorCount),
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipTable)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipGraphId,
PipGraph.SerializeGraphId,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipGraphId)))
.GetAwaiter().GetResult();
Console.WriteLine($"Simulating [Reading]");
var simulator = new BuildSimulatorAnalyzer(Input);
simulator.Increment = SimulatorIncrement ?? simulator.Increment;
simulator.ExecutionData.DataflowGraph = m_mutableGraph;
simulator.OutputDirectory = OutputFilePath;
simulator.ReadExecutionLog();
Console.WriteLine($"Simulating [Analyzing]");
simulator.Analyze();
Console.WriteLine($"Blocking Dependency Analysis");
DisplayTable <DepColumn> depTable = new DisplayTable <DepColumn>(" , ");
foreach (var pipId in PipTable.Keys)
{
var pipType = PipTable.GetPipType(pipId);
if (pipType == PipType.Process)
{
var entry = GetEntry(pipId);
(PipId node, ulong cost)maxConsumedDependency = default;
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in entry.PipDependencies)
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.Key.ToNodeId()];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.Key, cost);
}
if (dep.Value != null && dep.Value.HasFlag(ContentFlag.Consumed))
{
if (!maxConsumedDependency.node.IsValid || cost > maxConsumedDependency.cost)
{
maxConsumedDependency = (dep.Key, cost);
}
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier}");
depTable.Set(DepColumn.MaxConsumedDependency, ToString(maxConsumedDependency.node));
depTable.Set(DepColumn.MaxConsumedDependencyChainCost, maxConsumedDependency.cost.ToMinutes());
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
}
else if (pipType == PipType.SealDirectory &&
!PipTable.GetSealDirectoryKind(pipId).IsSourceSeal() &&
!IsSourceOnlySeal(pipId))
{
var seal = (SealDirectory)GetPip(pipId);
var entry = GetEntry(seal.Directory);
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in DataflowGraph.GetIncomingEdges(pipId.ToNodeId()))
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.OtherNode];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.OtherNode.ToPipId(), cost);
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier} ({entry.FileCount} files)");
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
depTable.Set(DepColumn.Directory, seal.DirectoryRoot.ToString(PathTable));
}
}
using (var blockAnalysisWriter = new StreamWriter(Path.Combine(OutputFilePath, "blockAnalysis.txt")))
{
depTable.Write(blockAnalysisWriter);
}
m_writer.Dispose();
Console.WriteLine($"Analyzing complete");
return(0);
}
public override int Analyze()
{
Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath);
using (var streamWriter = new StreamWriter(OutputFilePath))
using (JsonWriter writer = new JsonTextWriter(streamWriter))
{
writer.WriteStartObject();
writer.WritePropertyName("Mounts");
{
writer.WriteStartObject();
var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander;
foreach (var mountRoot in mountPathExpander.GetAllRoots())
{
var mount = mountPathExpander.GetSemanticPathInfo(mountRoot);
writer.WritePropertyName(mount.RootName.ToString(StringTable));
writer.WriteValue(mountRoot.ToString(PathTable));
}
writer.WriteEndObject();
}
writer.WritePropertyName("FileInfo");
{
writer.WriteStartObject();
foreach (var fileEntry in m_fileToConsumerMap)
{
var file = fileEntry.Key;
var consumers = fileEntry.Value;
var path = ToDisplayFilePath(file);
if (path != null)
{
writer.WritePropertyName(path, true);
{
writer.WriteStartArray();
foreach (var consumer in consumers)
{
writer.WriteValue(ToDisplayString(consumer));
}
writer.WriteEndArray();
}
}
}
writer.WriteEndObject();
}
writer.WritePropertyName("PipGraph");
{
writer.WriteStartObject();
m_failedPipsClosure.UnsafeReset();
writer.WritePropertyName("root");
{
writer.WriteStartArray();
foreach (var failedPip in m_failedPips)
{
writer.WriteValue(ToDisplayString(failedPip));
}
writer.WriteEndArray();
}
List <NodeId> dependencyBuffer = new List <NodeId>();
nodeVisitor.VisitTransitiveDependencies(m_failedPips.Select(p => p.ToNodeId()), m_failedPipsClosure, visitNode: node =>
{
dependencyBuffer.Clear();
foreach (var dependencyEdge in DataflowGraph.GetIncomingEdges(node))
{
if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile)
{
dependencyBuffer.Add(dependencyEdge.OtherNode);
}
}
if (dependencyBuffer.Count != 0)
{
writer.WritePropertyName(ToDisplayString(node.ToPipId()));
{
writer.WriteStartArray();
foreach (var dependencyNode in dependencyBuffer)
{
writer.WriteValue(ToDisplayString(dependencyNode.ToPipId()));
}
writer.WriteEndArray();
}
}
return(true);
});
writer.WriteEndObject();
}
writer.WriteEndObject();
}
return(0);
}
/// <summary>
/// Generate the output of failure data
/// Failed pips + process chain : This points to the tool that failed.
/// Set of dependencies diff contains: source and output changes. Which of those are directe referenced
///
/// </summary>
/// <param name="isDiff">There is diff data from another log</param>
private void GenerateOutput(bool isDiff)
{
Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath);
using (var streamWriter = new StreamWriter(OutputFilePath))
using (JsonWriter writer = new JsonTextWriter(streamWriter))
{
writer.WriteStartObject();
writer.WritePropertyName("Mounts");
{
writer.WriteStartObject();
var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander;
foreach (var mountRoot in mountPathExpander.GetAllRoots())
{
var mount = mountPathExpander.GetSemanticPathInfo(mountRoot);
writer.WritePropertyName(mount.RootName.ToString(StringTable));
writer.WriteValue(mountRoot.ToString(PathTable));
}
writer.WriteEndObject();
}
writer.WriteWhitespace(Environment.NewLine);
writer.WritePropertyName("Failures");
{
writer.WriteStartArray();
foreach (var failedPip in m_failedPips)
{
var pip = (Process)m_pipTable.HydratePip(failedPip, PipQueryContext.ViewerAnalyzer);
writer.WriteWhitespace(Environment.NewLine);
writer.WriteStartObject();
// prints the semistable hash.
WritePropertyAndValue(writer, "PipId", ToDisplayString(failedPip));
WritePropertyAndValue(writer, "Working Directory", ToDisplayFilePath(pip.WorkingDirectory));
var provenance = pip.Provenance;
WritePropertyAndValue(
writer,
"Qualifier",
provenance != null ? PipGraph.Context.QualifierTable.GetCanonicalDisplayString(provenance.QualifierId) : string.Empty);
WritePropertyAndValue(
writer,
"OutputValueSymbol",
provenance != null ? provenance.OutputValueSymbol.ToString(SymbolTable) : string.Empty);
var pipPerformance = m_pipPerformance[pip.PipId.Value];
WritePropertyAndValue(writer, "PeakMemoryUsageMb", pipPerformance.PeakMemoryUsageMb.ToString());
WritePropertyAndValue(writer, "NumberOfProcesses", pipPerformance.NumberOfProcesses.ToString());
WritePropertyAndValue(
writer,
"FileMonitoringViolationsNotWhitelisted",
pipPerformance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted.ToString());
if (isDiff)
{
WritePropertyAndValue(writer, "NumberOfImputChanges", GetDependencyChangesForPip(pip).ToString());
}
if (m_pipDependencyViolationEventData.TryGetValue(pip.PipId, out var pipDependencyViolationEvent))
{
foreach (var data in pipDependencyViolationEvent)
{
WritePropertyAndValue(writer, "DependencyViolationType", data.ViolationType.ToString());
WritePropertyAndValue(writer, "Path causing violation", ToDisplayFilePath(data.Path));
}
}
WriteProcessChain(writer, pip);
// TODO : Are environment variables usefull for analysis
// : Are are all the failures required or choose count as cmd line arg to take top n
writer.WriteWhitespace(Environment.NewLine);
writer.WriteEndObject();
}
writer.WriteEndArray();
}
writer.WriteWhitespace(Environment.NewLine);
var fileToConsumerMap = m_fileToConsumerMap;
var propertyName = "FileInfo";
if (isDiff)
{
propertyName = "FileInfoDiff";
fileToConsumerMap = m_fileToConsumerMapDiff;
// If any added dependencies add them to the output
if (m_fileToConsumerMapAdded.Count > 0)
{
WriteFileDependencies("FileInfoAdded", m_fileToConsumerMapAdded, writer);
}
// TODO:Add removed dependencies when compared to other log
}
WriteFileDependencies(propertyName, fileToConsumerMap, writer);
writer.WriteWhitespace(Environment.NewLine);
writer.WritePropertyName("PipGraph");
{
writer.WriteStartObject();
m_failedPipsClosure.UnsafeReset();
writer.WritePropertyName("root");
{
writer.WriteStartArray();
foreach (var failedPip in m_failedPips)
{
writer.WriteValue(ToDisplayString(failedPip));
}
writer.WriteEndArray();
}
List <NodeId> dependencyBuffer = new List <NodeId>();
nodeVisitor.VisitTransitiveDependencies(
m_failedPips.Select(p => p.ToNodeId()),
m_failedPipsClosure,
visitNode: node =>
{
dependencyBuffer.Clear();
foreach (var dependencyEdge in DataflowGraph.GetIncomingEdges(node))
{
if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile)
{
dependencyBuffer.Add(dependencyEdge.OtherNode);
}
}
if (dependencyBuffer.Count != 0)
{
writer.WritePropertyName(ToDisplayString(node.ToPipId()));
{
writer.WriteStartArray();
foreach (var dependencyNode in dependencyBuffer)
{
writer.WriteValue(ToDisplayString(dependencyNode.ToPipId()));
}
writer.WriteEndArray();
}
}
return(true);
});
writer.WriteEndObject();
}
writer.WriteEndObject();
}
}
