/// <summary>
/// Adds non materialized outputs.
/// </summary>
public void AddNonMaterializedPip(Pip pip)
{
Contract.Requires(pip != null);
int numberOfOutputs = 0;
PipArtifacts.ForEachOutput(pip, artifact => { ++numberOfOutputs; return(true); }, true);
m_nonMaterializedPips.TryAdd(pip.PipId, numberOfOutputs);
}
private int ProcessPips()
{
var pipCount = 0;
Console.WriteLine($"Processing pip graph (mapping input/outputs)");
Stopwatch stopWatch = null;
try
{
stopWatch = Stopwatch.StartNew();
foreach (var pip in CachedGraph.PipGraph.RetrieveScheduledPips())
{
if (IncludePip(pip))
{
// these are for the pip inputs. We want to identify who is the input of who,
// since we need to know their relationship...
PipArtifacts.ForEachInput(pip, input =>
{
if (input.IsFile && input.FileArtifact.IsSourceFile)
{
m_artifactInputPips.Add(input.FileArtifact, pip);
m_scheduledPipInputCount.Add(pip.PipId.Value, 1);
}
return(true);
},
includeLazyInputs: true);
// these are for the pip outputs.
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
m_artifactOutputPips.Add(output.FileArtifact, pip);
m_scheduledPipOutputCount.Add(pip.PipId.Value, 1);
}
return(true);
},
includeUncacheable: true);
// the dependencies, cause we want to count them...
foreach (var dependency in CachedGraph.PipGraph.RetrievePipImmediateDependencies(pip))
{
m_scheduledPipDependencyCount.Add(pip.PipId.Value, 1);
}
++pipCount;
}
}
return(pipCount);
}
catch (Exception)
{
Console.WriteLine("Propagating exception when processing pips...");
throw;
}
finally
{
if (stopWatch != null)
{
stopWatch.Stop();
Console.WriteLine($"Done, processed {pipCount} pips in {stopWatch.ElapsedMilliseconds}ms");
}
else
{
Console.WriteLine("There is an error when creating a stopwatch...");
}
}
}
private void WriteGraphNodeEntry(
Pip pip,
IPipScheduleTraversal graph,
PipExecutionContext context,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(pip != null);
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteStartObject();
{
WriteStaticPipDetails(pip, context);
{
// We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips).
bool writtenHeader = false;
foreach (var dependency in graph.RetrievePipImmediateDependencies(pip))
{
if (!writtenHeader)
{
m_writer.WritePropertyName("dependsOn");
m_writer.WriteStartArray();
writtenHeader = true;
}
if (IncludePip(dependency.PipType))
{
m_writer.WriteValue(dependency.PipId.Value);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
writtenHeader = false;
PipArtifacts.ForEachInput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
if (!writtenHeader)
{
m_writer.WritePropertyName("consumes");
m_writer.WriteStartArray();
writtenHeader = true;
}
m_writer.WriteValue(dependencyId);
}
return(true);
}, includeLazyInputs: true);
// Write reads from shared opaque directories
if (directoryInputContent.TryGetValue(pip.PipId, out var pipInput))
{
foreach (var input in pipInput.ReportedFileAccesses)
{
int dependencyId;
if (!m_fileIds.TryGetValue(input.ManifestPath.ToString(context.PathTable), out dependencyId))
{
// Ignore unrecognized reads (these are usually directories / files that aren't produced, so we don't have their ID)
continue;
}
m_writer.WriteValue(dependencyId);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
}
{
m_writer.WritePropertyName("produces");
m_writer.WriteStartArray();
SortedSet <int> writes = new SortedSet <int>();
PipArtifacts.ForEachOutput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
writes.Add(dependencyId);
}
return(true);
}, includeUncacheable: true);
// Write outputs into shared opaque directories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
int dependencyId;
if (!m_fileIds.TryGetValue(file.Path.ToString(context.PathTable), out dependencyId))
{
Contract.Assume(false, "Expected file artifact not found in fileId table " + file.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
writes.Add(dependencyId);
}
}
}
foreach (int dependencyId in writes)
{
m_writer.WriteValue(dependencyId);
}
m_writer.WriteEndArray();
}
}
m_writer.WriteEndObject();
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
m_nextId = graph.PipCount + 1;
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
// Add all of the inputs to the file list
PipArtifacts.ForEachInput(pip, dependency =>
{
if (dependency.IsFile)
{
// Use producerId=0, if first referenced will indicate consumed source file
AssignFileIdAndWriteFileEntry(dependency.FileArtifact.Path.ToString(context.PathTable), null);
}
return(true);
}, includeLazyInputs: true);
// Now add all of the outputs
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact.Path.ToString(context.PathTable), pip.PipId);
}
return(true);
},
includeUncacheable: true);
// Record files that are written into SharedOpaqueDirectories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
AssignFileIdAndWriteFileEntry(file.Path.ToString(context.PathTable), pip.PipId);
}
}
}
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
directoryPips.Add(directoryPip);
// Add any file artifacts from the sealed directory, many will be
// outputs of process pips but some will be source files
foreach (FileArtifact directoryMember in directoryPip.Contents)
{
AssignFileIdAndWriteFileEntry(directoryMember.Path.ToString(context.PathTable), pip.PipId);
}
}
}
//// Now write out the directory entries
//foreach (SealDirectory directoryPip in directoryPips)
//{
// AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents);
//}
}
m_writer.WriteEndArray();
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryInputContent, directoryOutputContent);
}
}
m_writer.WriteEndArray();
if (executionData != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
ExportWorkerDetails(workers);
}
if (fileData != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
ExportFileDetails(fileData, context);
}
// End the outermost object
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteEndObject();
m_writer.Flush();
}
private void WriteGraphNodeEntry(
Pip pip,
IPipScheduleTraversal graph,
PipExecutionContext context,
Dictionary <DirectoryArtifact, int> directoryIds)
{
Contract.Requires(pip != null);
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteStartObject();
{
WriteStaticPipDetails(pip, context);
{
// We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips).
bool writtenHeader = false;
foreach (var dependency in graph.RetrievePipImmediateDependencies(pip))
{
if (!writtenHeader)
{
m_writer.WritePropertyName("dependsOn");
m_writer.WriteStartArray();
writtenHeader = true;
}
if (IncludePip(dependency.PipType))
{
m_writer.WriteValue(dependency.PipId.Value);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
writtenHeader = false;
PipArtifacts.ForEachInput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (input of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
if (!writtenHeader)
{
m_writer.WritePropertyName("consumes");
m_writer.WriteStartArray();
writtenHeader = true;
}
m_writer.WriteValue(dependencyId);
return(true);
}, includeLazyInputs: true);
if (writtenHeader)
{
m_writer.WriteEndArray();
}
}
{
m_writer.WritePropertyName("produces");
m_writer.WriteStartArray();
PipArtifacts.ForEachOutput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
m_writer.WriteValue(dependencyId);
return(true);
}, includeUncacheable: true);
m_writer.WriteEndArray();
}
}
m_writer.WriteEndObject();
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
int nextId = graph.PipCount + 1;
var directoryIds = new Dictionary <DirectoryArtifact, int>(capacity: 1000);
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
PipArtifacts.ForEachInput(pip, input =>
{
if (input.IsFile && input.FileArtifact.IsSourceFile)
{
AssignFileIdAndWriteFileEntry(input.FileArtifact, context, ref nextId);
}
return(true);
},
includeLazyInputs: true
);
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact, context, ref nextId);
}
return(true);
},
includeUncacheable: true);
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
foreach (FileArtifact f in directoryPip.Contents)
{
AssignFileIdAndWriteFileEntry(f, context, ref nextId);
}
directoryPips.Add(directoryPip);
}
}
foreach (SealDirectory directoryPip in directoryPips)
{
AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents, directoryIds, ref nextId);
}
}
m_writer.WriteEndArray();
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryIds);
}
}
m_writer.WriteEndArray();
// Avoid holding a reference to this map if it isn't later needed by the special file details exporter
if (fileData == null)
{
m_fileIds = null;
}
if (executionData != null)
{
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
ExportWorkerDetails(workers);
}
if (fileData != null)
{
ExportFileDetails(fileData);
}
// End the outermost object
m_writer.WriteEndObject();
m_writer.Flush();
}