/// <summary>
/// Serializes pip graph fragment.
/// </summary>
public void Serialize(AbsolutePath filePath, IPipScheduleTraversal pipGraph, string fragmentDescription = null, bool useTopSortSerialization = false)
{
if (useTopSortSerialization)
{
var topSorter = new PipGraphFragmentTopSort(pipGraph);
var sortedPips = topSorter.Sort();
SerializeTopSort(filePath, sortedPips, fragmentDescription);
}
else
{
SerializeSerially(filePath, pipGraph.RetrieveScheduledPips().ToList(), fragmentDescription);
}
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
m_nextId = graph.PipCount + 1;
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
// Add all of the inputs to the file list
PipArtifacts.ForEachInput(pip, dependency =>
{
if (dependency.IsFile)
{
// Use producerId=0, if first referenced will indicate consumed source file
AssignFileIdAndWriteFileEntry(dependency.FileArtifact.Path.ToString(context.PathTable), null);
}
return(true);
}, includeLazyInputs: true);
// Now add all of the outputs
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact.Path.ToString(context.PathTable), pip.PipId);
}
return(true);
},
includeUncacheable: true);
// Record files that are written into SharedOpaqueDirectories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
AssignFileIdAndWriteFileEntry(file.Path.ToString(context.PathTable), pip.PipId);
}
}
}
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
directoryPips.Add(directoryPip);
// Add any file artifacts from the sealed directory, many will be
// outputs of process pips but some will be source files
foreach (FileArtifact directoryMember in directoryPip.Contents)
{
AssignFileIdAndWriteFileEntry(directoryMember.Path.ToString(context.PathTable), pip.PipId);
}
}
}
//// Now write out the directory entries
//foreach (SealDirectory directoryPip in directoryPips)
//{
// AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents);
//}
}
m_writer.WriteEndArray();
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryInputContent, directoryOutputContent);
}
}
m_writer.WriteEndArray();
if (executionData != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
ExportWorkerDetails(workers);
}
if (fileData != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
ExportFileDetails(fileData, context);
}
// End the outermost object
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteEndObject();
m_writer.Flush();
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
int nextId = graph.PipCount + 1;
var directoryIds = new Dictionary <DirectoryArtifact, int>(capacity: 1000);
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
PipArtifacts.ForEachInput(pip, input =>
{
if (input.IsFile && input.FileArtifact.IsSourceFile)
{
AssignFileIdAndWriteFileEntry(input.FileArtifact, context, ref nextId);
}
return(true);
},
includeLazyInputs: true
);
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact, context, ref nextId);
}
return(true);
},
includeUncacheable: true);
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
foreach (FileArtifact f in directoryPip.Contents)
{
AssignFileIdAndWriteFileEntry(f, context, ref nextId);
}
directoryPips.Add(directoryPip);
}
}
foreach (SealDirectory directoryPip in directoryPips)
{
AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents, directoryIds, ref nextId);
}
}
m_writer.WriteEndArray();
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryIds);
}
}
m_writer.WriteEndArray();
// Avoid holding a reference to this map if it isn't later needed by the special file details exporter
if (fileData == null)
{
m_fileIds = null;
}
if (executionData != null)
{
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
ExportWorkerDetails(workers);
}
if (fileData != null)
{
ExportFileDetails(fileData);
}
// End the outermost object
m_writer.WriteEndObject();
m_writer.Flush();
}
/// <summary> /// Topologically sort pips in the pip graph. /// </summary> public List <List <Pip> > Sort() => TopSort(m_pipGraph.RetrieveScheduledPips().ToList());
