/// <summary>
/// Serializes pip graph fragment.
/// </summary>
public void Serialize(AbsolutePath filePath, IPipScheduleTraversal pipGraph, string fragmentDescription = null, bool useTopSortSerialization = false)
{
if (useTopSortSerialization)
{
var topSorter = new PipGraphFragmentTopSort(pipGraph);
var sortedPips = topSorter.Sort();
SerializeTopSort(filePath, sortedPips, fragmentDescription);
}
else
{
SerializeSerially(filePath, pipGraph.RetrieveScheduledPips().ToList(), fragmentDescription);
}
}
private static bool SerializeFragmentIfRequested(
PipGraphFragmentGeneratorConfiguration pipGraphFragmentGeneratorConfig,
FrontEndContext context,
IPipScheduleTraversal pipGraph)
{
Contract.Requires(context != null);
Contract.Requires(pipGraph != null);
if (!pipGraphFragmentGeneratorConfig.OutputFile.IsValid)
{
return(true);
}
try
{
var serializer = new PipGraphFragmentSerializer(context, new PipGraphFragmentContext())
{
AlternateSymbolSeparator = pipGraphFragmentGeneratorConfig.AlternateSymbolSeparator
};
serializer.Serialize(
pipGraphFragmentGeneratorConfig.OutputFile,
pipGraph,
pipGraphFragmentGeneratorConfig.Description,
pipGraphFragmentGeneratorConfig.TopSort);
Logger.Log.GraphFragmentSerializationStats(context.LoggingContext, serializer.FragmentDescription, serializer.Stats.ToString());
return(true);
}
catch (Exception e)
{
Logger.Log.GraphFragmentExceptionOnSerializingFragment(
context.LoggingContext,
pipGraphFragmentGeneratorConfig.OutputFile.ToString(context.PathTable),
e.ToString());
return(false);
}
}
private void WriteGraphNodeEntry(
Pip pip,
IPipScheduleTraversal graph,
PipExecutionContext context,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(pip != null);
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteStartObject();
{
WriteStaticPipDetails(pip, context);
{
// We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips).
bool writtenHeader = false;
foreach (var dependency in graph.RetrievePipImmediateDependencies(pip))
{
if (!writtenHeader)
{
m_writer.WritePropertyName("dependsOn");
m_writer.WriteStartArray();
writtenHeader = true;
}
if (IncludePip(dependency.PipType))
{
m_writer.WriteValue(dependency.PipId.Value);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
writtenHeader = false;
PipArtifacts.ForEachInput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
if (!writtenHeader)
{
m_writer.WritePropertyName("consumes");
m_writer.WriteStartArray();
writtenHeader = true;
}
m_writer.WriteValue(dependencyId);
}
return(true);
}, includeLazyInputs: true);
// Write reads from shared opaque directories
if (directoryInputContent.TryGetValue(pip.PipId, out var pipInput))
{
foreach (var input in pipInput.ReportedFileAccesses)
{
int dependencyId;
if (!m_fileIds.TryGetValue(input.ManifestPath.ToString(context.PathTable), out dependencyId))
{
// Ignore unrecognized reads (these are usually directories / files that aren't produced, so we don't have their ID)
continue;
}
m_writer.WriteValue(dependencyId);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
}
{
m_writer.WritePropertyName("produces");
m_writer.WriteStartArray();
SortedSet <int> writes = new SortedSet <int>();
PipArtifacts.ForEachOutput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
writes.Add(dependencyId);
}
return(true);
}, includeUncacheable: true);
// Write outputs into shared opaque directories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
int dependencyId;
if (!m_fileIds.TryGetValue(file.Path.ToString(context.PathTable), out dependencyId))
{
Contract.Assume(false, "Expected file artifact not found in fileId table " + file.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
writes.Add(dependencyId);
}
}
}
foreach (int dependencyId in writes)
{
m_writer.WriteValue(dependencyId);
}
m_writer.WriteEndArray();
}
}
m_writer.WriteEndObject();
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
m_nextId = graph.PipCount + 1;
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
// Add all of the inputs to the file list
PipArtifacts.ForEachInput(pip, dependency =>
{
if (dependency.IsFile)
{
// Use producerId=0, if first referenced will indicate consumed source file
AssignFileIdAndWriteFileEntry(dependency.FileArtifact.Path.ToString(context.PathTable), null);
}
return(true);
}, includeLazyInputs: true);
// Now add all of the outputs
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact.Path.ToString(context.PathTable), pip.PipId);
}
return(true);
},
includeUncacheable: true);
// Record files that are written into SharedOpaqueDirectories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
AssignFileIdAndWriteFileEntry(file.Path.ToString(context.PathTable), pip.PipId);
}
}
}
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
directoryPips.Add(directoryPip);
// Add any file artifacts from the sealed directory, many will be
// outputs of process pips but some will be source files
foreach (FileArtifact directoryMember in directoryPip.Contents)
{
AssignFileIdAndWriteFileEntry(directoryMember.Path.ToString(context.PathTable), pip.PipId);
}
}
}
//// Now write out the directory entries
//foreach (SealDirectory directoryPip in directoryPips)
//{
// AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents);
//}
}
m_writer.WriteEndArray();
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryInputContent, directoryOutputContent);
}
}
m_writer.WriteEndArray();
if (executionData != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
ExportWorkerDetails(workers);
}
if (fileData != null)
{
m_writer.WriteWhitespace(Environment.NewLine);
ExportFileDetails(fileData, context);
}
// End the outermost object
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteEndObject();
m_writer.Flush();
}
private void WriteGraphNodeEntry(
Pip pip,
IPipScheduleTraversal graph,
PipExecutionContext context,
Dictionary <DirectoryArtifact, int> directoryIds)
{
Contract.Requires(pip != null);
m_writer.WriteWhitespace(Environment.NewLine);
m_writer.WriteStartObject();
{
WriteStaticPipDetails(pip, context);
{
// We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips).
bool writtenHeader = false;
foreach (var dependency in graph.RetrievePipImmediateDependencies(pip))
{
if (!writtenHeader)
{
m_writer.WritePropertyName("dependsOn");
m_writer.WriteStartArray();
writtenHeader = true;
}
if (IncludePip(dependency.PipType))
{
m_writer.WriteValue(dependency.PipId.Value);
}
}
if (writtenHeader)
{
m_writer.WriteEndArray();
}
writtenHeader = false;
PipArtifacts.ForEachInput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (input of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
if (!writtenHeader)
{
m_writer.WritePropertyName("consumes");
m_writer.WriteStartArray();
writtenHeader = true;
}
m_writer.WriteValue(dependencyId);
return(true);
}, includeLazyInputs: true);
if (writtenHeader)
{
m_writer.WriteEndArray();
}
}
{
m_writer.WritePropertyName("produces");
m_writer.WriteStartArray();
PipArtifacts.ForEachOutput(pip, dependency =>
{
int dependencyId;
if (dependency.IsFile)
{
if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId))
{
Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
else
{
if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId))
{
Contract.Assume(false, "Expected directory artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable));
throw new InvalidOperationException("Unreachable");
}
}
m_writer.WriteValue(dependencyId);
return(true);
}, includeUncacheable: true);
m_writer.WriteEndArray();
}
}
m_writer.WriteEndObject();
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
int nextId = graph.PipCount + 1;
var directoryIds = new Dictionary <DirectoryArtifact, int>(capacity: 1000);
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
PipArtifacts.ForEachInput(pip, input =>
{
if (input.IsFile && input.FileArtifact.IsSourceFile)
{
AssignFileIdAndWriteFileEntry(input.FileArtifact, context, ref nextId);
}
return(true);
},
includeLazyInputs: true
);
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact, context, ref nextId);
}
return(true);
},
includeUncacheable: true);
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
foreach (FileArtifact f in directoryPip.Contents)
{
AssignFileIdAndWriteFileEntry(f, context, ref nextId);
}
directoryPips.Add(directoryPip);
}
}
foreach (SealDirectory directoryPip in directoryPips)
{
AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents, directoryIds, ref nextId);
}
}
m_writer.WriteEndArray();
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryIds);
}
}
m_writer.WriteEndArray();
// Avoid holding a reference to this map if it isn't later needed by the special file details exporter
if (fileData == null)
{
m_fileIds = null;
}
if (executionData != null)
{
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
ExportWorkerDetails(workers);
}
if (fileData != null)
{
ExportFileDetails(fileData);
}
// End the outermost object
m_writer.WriteEndObject();
m_writer.Flush();
}
/// <summary>
/// Create an instance of <see cref="PipGraphFragmentTopSort"/>/
/// </summary>
/// <param name="pipGraph">Pip graph.</param>
public PipGraphFragmentTopSort(IPipScheduleTraversal pipGraph)
{
Contract.Requires(pipGraph != null);
m_pipGraph = pipGraph;
}
/// <nodoc />
public ScopePipGraph(IPipScheduleTraversal graph)
{
Graph = graph;
}
public ObjectInfo GetObjectInfo(object context, object obj)
{
obj = obj is EvaluationResult evalResult
? evalResult.Value
: obj;
if (obj == null || IsInvalid(obj))
{
return(s_nullObj);
}
if (obj.GetType().IsArray)
{
return(ArrayObjInfo(((IEnumerable)obj).Cast <object>().ToArray()));
}
var customResult = m_customRenderer?.Invoke(this, context, obj);
if (customResult != null)
{
return(customResult);
}
return(obj switch
{
ScopeLocals scope => new ObjectInfo(LocalsScopeName, null, Lazy.Create(() => GetLocalsForStackEntry(scope.EvalState, scope.FrameIndex))),
ScopePipGraph scope => PipGraphInfo(scope.Graph).WithPreview(PipGraphScopeName),
ScopeAllModules scope => ArrayObjInfo(scope.EvaluatedModules.ToArray()).WithPreview(EvaluatedModulesScopeName),
IModuleAndContext mc => GetObjectInfo(mc.Tree.RootContext, mc.Module),
ObjectInfo objInf => objInf,
IPipScheduleTraversal graph => PipGraphInfo(graph),
Pip pip => GenericObjectInfo(pip, $"<{pip.PipType}>").Build(),
PipProvenance prov => ProvenanceInfo(prov),
EnvironmentVariable envVar => EnvironmentVariableInfo(envVar),
PipFragment pipFrag => PipFragmentInfo(context, pipFrag),
Thunk thunk => thunk.Value != null?GetObjectInfo(context, thunk.Value) : new ObjectInfo("<not evaluated>"),
FunctionLikeExpression lambda => LambdaInfo(lambda),
Closure cls => LambdaInfo(cls.Function),
SymbolAtom sym => new ObjectInfo(sym.ToString(StringTable)),
StringId id => new ObjectInfo(id.ToString(StringTable)),
PipId id => new ObjectInfo($"{id.Value}"),
UndefinedLiteral _ => new ObjectInfo("undefined", UndefinedLiteral.Instance),
UndefinedValue _ => new ObjectInfo("undefined", UndefinedValue.Instance),
AbsolutePath path => new ObjectInfo($"p`{path.ToString(PathTable)}`", path),
RelativePath path => new ObjectInfo($"r`{path.ToString(StringTable)}`", path),
PathAtom atom => new ObjectInfo($"a`{atom.ToString(StringTable)}`", atom),
FileArtifact file => new ObjectInfo($"f`{file.Path.ToString(PathTable)}`", file),
DirectoryArtifact dir => new ObjectInfo($"d`{dir.Path.ToString(PathTable)}`", dir),
int num => new ObjectInfo($"{num}"),
uint num => new ObjectInfo($"{num}"),
short num => new ObjectInfo($"{num}", (int)num),
long num => new ObjectInfo($"{num}"),
char ch => new ObjectInfo($"'{ch}'", ch.ToString()),
string str => new ObjectInfo($"\"{str}\"", str),
Enum e => new ObjectInfo($"{e.GetType().Name}.{e}", e),
NumberLiteral numLit => new ObjectInfo(numLit.UnboxedValue.ToString(), numLit),
Func <object> func => FuncObjInfo(func),
ArraySegment <object> arrSeg => ArrayObjInfo(arrSeg),
IEnumerable enu => new ObjectInfoBuilder().Preview("IEnumerable").Prop("Result", Lazy.Create <object>(() => enu.Cast <object>().ToArray())).Build(),
ArrayLiteral arrLit => ArrayObjInfo(arrLit.Values.Select(v => v.Value).ToArray()).WithOriginal(arrLit),
ModuleBinding binding => GetObjectInfo(context, binding.Body),
ErrorValue error => ErrorValueInfo(),
object o => GenericObjectInfo(o).Build(),
_ => s_nullObj
});
