/// <summary> /// Adds non materialized outputs. /// </summary> public void AddNonMaterializedPip(Pip pip) { Contract.Requires(pip != null); int numberOfOutputs = 0; PipArtifacts.ForEachOutput(pip, artifact => { ++numberOfOutputs; return(true); }, true); m_nonMaterializedPips.TryAdd(pip.PipId, numberOfOutputs); }
private int ProcessPips() { var pipCount = 0; Console.WriteLine($"Processing pip graph (mapping input/outputs)"); Stopwatch stopWatch = null; try { stopWatch = Stopwatch.StartNew(); foreach (var pip in CachedGraph.PipGraph.RetrieveScheduledPips()) { if (IncludePip(pip)) { // these are for the pip inputs. We want to identify who is the input of who, // since we need to know their relationship... PipArtifacts.ForEachInput(pip, input => { if (input.IsFile && input.FileArtifact.IsSourceFile) { m_artifactInputPips.Add(input.FileArtifact, pip); m_scheduledPipInputCount.Add(pip.PipId.Value, 1); } return(true); }, includeLazyInputs: true); // these are for the pip outputs. PipArtifacts.ForEachOutput(pip, output => { if (output.IsFile) { m_artifactOutputPips.Add(output.FileArtifact, pip); m_scheduledPipOutputCount.Add(pip.PipId.Value, 1); } return(true); }, includeUncacheable: true); // the dependencies, cause we want to count them... foreach (var dependency in CachedGraph.PipGraph.RetrievePipImmediateDependencies(pip)) { m_scheduledPipDependencyCount.Add(pip.PipId.Value, 1); } ++pipCount; } } return(pipCount); } catch (Exception) { Console.WriteLine("Propagating exception when processing pips..."); throw; } finally { if (stopWatch != null) { stopWatch.Stop(); Console.WriteLine($"Done, processed {pipCount} pips in {stopWatch.ElapsedMilliseconds}ms"); } else { Console.WriteLine("There is an error when creating a stopwatch..."); } } }
private void WriteGraphNodeEntry( Pip pip, IPipScheduleTraversal graph, PipExecutionContext context, Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent, Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent) { Contract.Requires(pip != null); m_writer.WriteWhitespace(Environment.NewLine); m_writer.WriteStartObject(); { WriteStaticPipDetails(pip, context); { // We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips). bool writtenHeader = false; foreach (var dependency in graph.RetrievePipImmediateDependencies(pip)) { if (!writtenHeader) { m_writer.WritePropertyName("dependsOn"); m_writer.WriteStartArray(); writtenHeader = true; } if (IncludePip(dependency.PipType)) { m_writer.WriteValue(dependency.PipId.Value); } } if (writtenHeader) { m_writer.WriteEndArray(); } writtenHeader = false; PipArtifacts.ForEachInput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } if (!writtenHeader) { m_writer.WritePropertyName("consumes"); m_writer.WriteStartArray(); writtenHeader = true; } m_writer.WriteValue(dependencyId); } return(true); }, includeLazyInputs: true); // Write reads from shared opaque directories if (directoryInputContent.TryGetValue(pip.PipId, out var pipInput)) { foreach (var input in pipInput.ReportedFileAccesses) { int dependencyId; if (!m_fileIds.TryGetValue(input.ManifestPath.ToString(context.PathTable), out dependencyId)) { // Ignore unrecognized reads (these are usually directories / files that aren't produced, so we don't have their ID) continue; } m_writer.WriteValue(dependencyId); } } if (writtenHeader) { m_writer.WriteEndArray(); } } { m_writer.WritePropertyName("produces"); m_writer.WriteStartArray(); SortedSet <int> writes = new SortedSet <int>(); PipArtifacts.ForEachOutput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } writes.Add(dependencyId); } return(true); }, includeUncacheable: true); // Write outputs into shared opaque directories if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput)) { foreach (var output in pipOutput.DirectoryOutputs) { DirectoryArtifact dir = output.directoryArtifact; var content = output.fileArtifactArray; foreach (var file in content) { int dependencyId; if (!m_fileIds.TryGetValue(file.Path.ToString(context.PathTable), out dependencyId)) { Contract.Assume(false, "Expected file artifact not found in fileId table " + file.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } writes.Add(dependencyId); } } } foreach (int dependencyId in writes) { m_writer.WriteValue(dependencyId); } m_writer.WriteEndArray(); } } m_writer.WriteEndObject(); }
/// <summary> /// Save the graph. This method produces the whole file content. It must be called only once. /// </summary> public void SaveGraphData( IPipScheduleTraversal graph, PipExecutionContext context, IEnumerable <PipExecutionPerformanceEventData> executionData, IEnumerable <string> workers, IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData, Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent, Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent) { Contract.Requires(graph != null); Contract.Requires(context != null); Contract.Assume(m_fileIds.Count == 0); // We have transitioned to the Exporting state. // Observed execution info may get queued up for when we begin draining the queue (below). // Don't overlap pip ids with file/directory ids m_nextId = graph.PipCount + 1; m_writer.WriteStartObject(); // Outermost object WritePreamble(); m_writer.WritePropertyName("artifacts"); m_writer.WriteStartArray(); // To save some work on pip deserialization (we are pathologically visiting every pip in the pip table), // we hold pips alive between the various passes. var pips = new List <Pip>(); { var directoryPips = new List <SealDirectory>(); // TODO: This is pretty expensive, as it loads all pips in memory foreach (Pip pip in graph.RetrieveScheduledPips()) { pips.Add(pip); // Add all of the inputs to the file list PipArtifacts.ForEachInput(pip, dependency => { if (dependency.IsFile) { // Use producerId=0, if first referenced will indicate consumed source file AssignFileIdAndWriteFileEntry(dependency.FileArtifact.Path.ToString(context.PathTable), null); } return(true); }, includeLazyInputs: true); // Now add all of the outputs PipArtifacts.ForEachOutput(pip, output => { if (output.IsFile) { AssignFileIdAndWriteFileEntry(output.FileArtifact.Path.ToString(context.PathTable), pip.PipId); } return(true); }, includeUncacheable: true); // Record files that are written into SharedOpaqueDirectories if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput)) { foreach (var output in pipOutput.DirectoryOutputs) { DirectoryArtifact dir = output.directoryArtifact; var content = output.fileArtifactArray; foreach (var file in content) { AssignFileIdAndWriteFileEntry(file.Path.ToString(context.PathTable), pip.PipId); } } } // SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output, // we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before // writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited. if (pip.PipType == PipType.SealDirectory) { var directoryPip = (SealDirectory)pip; Contract.Assume(directoryPip.IsInitialized); directoryPips.Add(directoryPip); // Add any file artifacts from the sealed directory, many will be // outputs of process pips but some will be source files foreach (FileArtifact directoryMember in directoryPip.Contents) { AssignFileIdAndWriteFileEntry(directoryMember.Path.ToString(context.PathTable), pip.PipId); } } } //// Now write out the directory entries //foreach (SealDirectory directoryPip in directoryPips) //{ // AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents); //} } m_writer.WriteEndArray(); m_writer.WriteWhitespace(Environment.NewLine); m_writer.WritePropertyName("graph"); m_writer.WriteStartArray(); { // Note we are using the 'pips' list captured above rather than possibly deserializing pips again. foreach (Pip pip in pips) { if (!IncludePip(pip.PipType)) { continue; } WriteGraphNodeEntry(pip, graph, context, directoryInputContent, directoryOutputContent); } } m_writer.WriteEndArray(); if (executionData != null) { m_writer.WriteWhitespace(Environment.NewLine); m_writer.WritePropertyName("execution"); m_writer.WriteStartArray(); // Begin draining the execution entry queue, now that we are in an allowed state for that. // Iteration will complete only after Finish() is called, which marks the queue as complete (no new items). foreach (var pipWithPerf in executionData) { WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance); } // End the execution: array m_writer.WriteEndArray(); } if (workers != null) { m_writer.WriteWhitespace(Environment.NewLine); ExportWorkerDetails(workers); } if (fileData != null) { m_writer.WriteWhitespace(Environment.NewLine); ExportFileDetails(fileData, context); } // End the outermost object m_writer.WriteWhitespace(Environment.NewLine); m_writer.WriteEndObject(); m_writer.Flush(); }
private void WriteGraphNodeEntry( Pip pip, IPipScheduleTraversal graph, PipExecutionContext context, Dictionary <DirectoryArtifact, int> directoryIds) { Contract.Requires(pip != null); m_writer.WriteWhitespace(Environment.NewLine); m_writer.WriteStartObject(); { WriteStaticPipDetails(pip, context); { // We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips). bool writtenHeader = false; foreach (var dependency in graph.RetrievePipImmediateDependencies(pip)) { if (!writtenHeader) { m_writer.WritePropertyName("dependsOn"); m_writer.WriteStartArray(); writtenHeader = true; } if (IncludePip(dependency.PipType)) { m_writer.WriteValue(dependency.PipId.Value); } } if (writtenHeader) { m_writer.WriteEndArray(); } writtenHeader = false; PipArtifacts.ForEachInput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } else { if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId)) { Contract.Assume(false, "Expected directory artifact already written (input of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } if (!writtenHeader) { m_writer.WritePropertyName("consumes"); m_writer.WriteStartArray(); writtenHeader = true; } m_writer.WriteValue(dependencyId); return(true); }, includeLazyInputs: true); if (writtenHeader) { m_writer.WriteEndArray(); } } { m_writer.WritePropertyName("produces"); m_writer.WriteStartArray(); PipArtifacts.ForEachOutput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } else { if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId)) { Contract.Assume(false, "Expected directory artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } m_writer.WriteValue(dependencyId); return(true); }, includeUncacheable: true); m_writer.WriteEndArray(); } } m_writer.WriteEndObject(); }
/// <summary> /// Save the graph. This method produces the whole file content. It must be called only once. /// </summary> public void SaveGraphData( IPipScheduleTraversal graph, PipExecutionContext context, IEnumerable <PipExecutionPerformanceEventData> executionData, IEnumerable <string> workers, IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData) { Contract.Requires(graph != null); Contract.Requires(context != null); Contract.Assume(m_fileIds.Count == 0); // We have transitioned to the Exporting state. // Observed execution info may get queued up for when we begin draining the queue (below). // Don't overlap pip ids with file/directory ids int nextId = graph.PipCount + 1; var directoryIds = new Dictionary <DirectoryArtifact, int>(capacity: 1000); m_writer.WriteStartObject(); // Outermost object WritePreamble(); m_writer.WritePropertyName("artifacts"); m_writer.WriteStartArray(); // To save some work on pip deserialization (we are pathologically visiting every pip in the pip table), // we hold pips alive between the various passes. var pips = new List <Pip>(); { var directoryPips = new List <SealDirectory>(); // TODO: This is pretty expensive, as it loads all pips in memory foreach (Pip pip in graph.RetrieveScheduledPips()) { pips.Add(pip); PipArtifacts.ForEachInput(pip, input => { if (input.IsFile && input.FileArtifact.IsSourceFile) { AssignFileIdAndWriteFileEntry(input.FileArtifact, context, ref nextId); } return(true); }, includeLazyInputs: true ); PipArtifacts.ForEachOutput(pip, output => { if (output.IsFile) { AssignFileIdAndWriteFileEntry(output.FileArtifact, context, ref nextId); } return(true); }, includeUncacheable: true); // SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output, // we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before // writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited. if (pip.PipType == PipType.SealDirectory) { var directoryPip = (SealDirectory)pip; Contract.Assume(directoryPip.IsInitialized); foreach (FileArtifact f in directoryPip.Contents) { AssignFileIdAndWriteFileEntry(f, context, ref nextId); } directoryPips.Add(directoryPip); } } foreach (SealDirectory directoryPip in directoryPips) { AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents, directoryIds, ref nextId); } } m_writer.WriteEndArray(); m_writer.WritePropertyName("graph"); m_writer.WriteStartArray(); { // Note we are using the 'pips' list captured above rather than possibly deserializing pips again. foreach (Pip pip in pips) { if (!IncludePip(pip.PipType)) { continue; } WriteGraphNodeEntry(pip, graph, context, directoryIds); } } m_writer.WriteEndArray(); // Avoid holding a reference to this map if it isn't later needed by the special file details exporter if (fileData == null) { m_fileIds = null; } if (executionData != null) { m_writer.WritePropertyName("execution"); m_writer.WriteStartArray(); // Begin draining the execution entry queue, now that we are in an allowed state for that. // Iteration will complete only after Finish() is called, which marks the queue as complete (no new items). foreach (var pipWithPerf in executionData) { WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance); } // End the execution: array m_writer.WriteEndArray(); } if (workers != null) { ExportWorkerDetails(workers); } if (fileData != null) { ExportFileDetails(fileData); } // End the outermost object m_writer.WriteEndObject(); m_writer.Flush(); }