private IEnumerable <AbsolutePath> GetInputs(ProcessFingerprintComputationEventData data) { inputBuffer.Clear(); var pip = GetPip(data.PipId); PipArtifacts.ForEachInput(pip, input => { if (input.IsFile) { inputBuffer.Add(input.Path); } return(true); }, includeLazyInputs: true); foreach (var input in inputBuffer) { yield return(input); } if (CacheMissHelpers.TryGetUsedStrongFingerprintComputation(data, out var usedComputation)) { foreach (var observedInput in usedComputation.ObservedInputs) { if (observedInput.Type == ObservedInputType.FileContentRead || observedInput.Type == ObservedInputType.ExistingFileProbe) { yield return(observedInput.Path); } } } }
public void ProcessFingerprintComputedCore(ProcessFingerprintComputationEventData data) { var computation = data.StrongFingerprintComputations[0]; var pip = m_analyzer.GetPip(data.PipId); PipArtifacts.ForEachInput(pip, f => { if (f.IsFile && f.FileArtifact.IsOutputFile) { if (m_deployedFiles.TryGetValue(f.Path, out var size)) { AddFlag(f.Path, ContentFlag.Static | ContentFlag.Consumed); } } return(true); }, includeLazyInputs: false); foreach (var input in computation.ObservedInputs) { if (input.Type == ObservedInputType.FileContentRead || input.Type == ObservedInputType.ExistingFileProbe) { if (m_deployedFiles.TryGetValue(input.Path, out var size)) { var flag = input.Type == ObservedInputType.FileContentRead ? ContentFlag.DynamicContent : ContentFlag.DynamicProbe; AddFlag(input.Path, flag | ContentFlag.Consumed); } } } }
private int ProcessPips() { var pipCount = 0; Console.WriteLine($"Processing pip graph (mapping input/outputs)"); Stopwatch stopWatch = null; try { stopWatch = Stopwatch.StartNew(); foreach (var pip in CachedGraph.PipGraph.RetrieveScheduledPips()) { if (IncludePip(pip)) { // these are for the pip inputs. We want to identify who is the input of who, // since we need to know their relationship... PipArtifacts.ForEachInput(pip, input => { if (input.IsFile && input.FileArtifact.IsSourceFile) { m_artifactInputPips.Add(input.FileArtifact, pip); m_scheduledPipInputCount.Add(pip.PipId.Value, 1); } return(true); }, includeLazyInputs: true); // these are for the pip outputs. PipArtifacts.ForEachOutput(pip, output => { if (output.IsFile) { m_artifactOutputPips.Add(output.FileArtifact, pip); m_scheduledPipOutputCount.Add(pip.PipId.Value, 1); } return(true); }, includeUncacheable: true); // the dependencies, cause we want to count them... foreach (var dependency in CachedGraph.PipGraph.RetrievePipImmediateDependencies(pip)) { m_scheduledPipDependencyCount.Add(pip.PipId.Value, 1); } ++pipCount; } } return(pipCount); } catch (Exception) { Console.WriteLine("Propagating exception when processing pips..."); throw; } finally { if (stopWatch != null) { stopWatch.Stop(); Console.WriteLine($"Done, processed {pipCount} pips in {stopWatch.ElapsedMilliseconds}ms"); } else { Console.WriteLine("There is an error when creating a stopwatch..."); } } }
private void WriteGraphNodeEntry( Pip pip, IPipScheduleTraversal graph, PipExecutionContext context, Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent, Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent) { Contract.Requires(pip != null); m_writer.WriteWhitespace(Environment.NewLine); m_writer.WriteStartObject(); { WriteStaticPipDetails(pip, context); { // We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips). bool writtenHeader = false; foreach (var dependency in graph.RetrievePipImmediateDependencies(pip)) { if (!writtenHeader) { m_writer.WritePropertyName("dependsOn"); m_writer.WriteStartArray(); writtenHeader = true; } if (IncludePip(dependency.PipType)) { m_writer.WriteValue(dependency.PipId.Value); } } if (writtenHeader) { m_writer.WriteEndArray(); } writtenHeader = false; PipArtifacts.ForEachInput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } if (!writtenHeader) { m_writer.WritePropertyName("consumes"); m_writer.WriteStartArray(); writtenHeader = true; } m_writer.WriteValue(dependencyId); } return(true); }, includeLazyInputs: true); // Write reads from shared opaque directories if (directoryInputContent.TryGetValue(pip.PipId, out var pipInput)) { foreach (var input in pipInput.ReportedFileAccesses) { int dependencyId; if (!m_fileIds.TryGetValue(input.ManifestPath.ToString(context.PathTable), out dependencyId)) { // Ignore unrecognized reads (these are usually directories / files that aren't produced, so we don't have their ID) continue; } m_writer.WriteValue(dependencyId); } } if (writtenHeader) { m_writer.WriteEndArray(); } } { m_writer.WritePropertyName("produces"); m_writer.WriteStartArray(); SortedSet <int> writes = new SortedSet <int>(); PipArtifacts.ForEachOutput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact.Path.ToString(context.PathTable), out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } writes.Add(dependencyId); } return(true); }, includeUncacheable: true); // Write outputs into shared opaque directories if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput)) { foreach (var output in pipOutput.DirectoryOutputs) { DirectoryArtifact dir = output.directoryArtifact; var content = output.fileArtifactArray; foreach (var file in content) { int dependencyId; if (!m_fileIds.TryGetValue(file.Path.ToString(context.PathTable), out dependencyId)) { Contract.Assume(false, "Expected file artifact not found in fileId table " + file.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } writes.Add(dependencyId); } } } foreach (int dependencyId in writes) { m_writer.WriteValue(dependencyId); } m_writer.WriteEndArray(); } } m_writer.WriteEndObject(); }
/// <summary> /// Save the graph. This method produces the whole file content. It must be called only once. /// </summary> public void SaveGraphData( IPipScheduleTraversal graph, PipExecutionContext context, IEnumerable <PipExecutionPerformanceEventData> executionData, IEnumerable <string> workers, IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData, Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent, Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent) { Contract.Requires(graph != null); Contract.Requires(context != null); Contract.Assume(m_fileIds.Count == 0); // We have transitioned to the Exporting state. // Observed execution info may get queued up for when we begin draining the queue (below). // Don't overlap pip ids with file/directory ids m_nextId = graph.PipCount + 1; m_writer.WriteStartObject(); // Outermost object WritePreamble(); m_writer.WritePropertyName("artifacts"); m_writer.WriteStartArray(); // To save some work on pip deserialization (we are pathologically visiting every pip in the pip table), // we hold pips alive between the various passes. var pips = new List <Pip>(); { var directoryPips = new List <SealDirectory>(); // TODO: This is pretty expensive, as it loads all pips in memory foreach (Pip pip in graph.RetrieveScheduledPips()) { pips.Add(pip); // Add all of the inputs to the file list PipArtifacts.ForEachInput(pip, dependency => { if (dependency.IsFile) { // Use producerId=0, if first referenced will indicate consumed source file AssignFileIdAndWriteFileEntry(dependency.FileArtifact.Path.ToString(context.PathTable), null); } return(true); }, includeLazyInputs: true); // Now add all of the outputs PipArtifacts.ForEachOutput(pip, output => { if (output.IsFile) { AssignFileIdAndWriteFileEntry(output.FileArtifact.Path.ToString(context.PathTable), pip.PipId); } return(true); }, includeUncacheable: true); // Record files that are written into SharedOpaqueDirectories if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput)) { foreach (var output in pipOutput.DirectoryOutputs) { DirectoryArtifact dir = output.directoryArtifact; var content = output.fileArtifactArray; foreach (var file in content) { AssignFileIdAndWriteFileEntry(file.Path.ToString(context.PathTable), pip.PipId); } } } // SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output, // we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before // writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited. if (pip.PipType == PipType.SealDirectory) { var directoryPip = (SealDirectory)pip; Contract.Assume(directoryPip.IsInitialized); directoryPips.Add(directoryPip); // Add any file artifacts from the sealed directory, many will be // outputs of process pips but some will be source files foreach (FileArtifact directoryMember in directoryPip.Contents) { AssignFileIdAndWriteFileEntry(directoryMember.Path.ToString(context.PathTable), pip.PipId); } } } //// Now write out the directory entries //foreach (SealDirectory directoryPip in directoryPips) //{ // AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents); //} } m_writer.WriteEndArray(); m_writer.WriteWhitespace(Environment.NewLine); m_writer.WritePropertyName("graph"); m_writer.WriteStartArray(); { // Note we are using the 'pips' list captured above rather than possibly deserializing pips again. foreach (Pip pip in pips) { if (!IncludePip(pip.PipType)) { continue; } WriteGraphNodeEntry(pip, graph, context, directoryInputContent, directoryOutputContent); } } m_writer.WriteEndArray(); if (executionData != null) { m_writer.WriteWhitespace(Environment.NewLine); m_writer.WritePropertyName("execution"); m_writer.WriteStartArray(); // Begin draining the execution entry queue, now that we are in an allowed state for that. // Iteration will complete only after Finish() is called, which marks the queue as complete (no new items). foreach (var pipWithPerf in executionData) { WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance); } // End the execution: array m_writer.WriteEndArray(); } if (workers != null) { m_writer.WriteWhitespace(Environment.NewLine); ExportWorkerDetails(workers); } if (fileData != null) { m_writer.WriteWhitespace(Environment.NewLine); ExportFileDetails(fileData, context); } // End the outermost object m_writer.WriteWhitespace(Environment.NewLine); m_writer.WriteEndObject(); m_writer.Flush(); }
public void ProcessFingerprintComputed(ProcessFingerprintComputationEventData data) { if (data.Kind != FingerprintComputationKind.Execution) { return; } m_consumer = m_analyzer.GetEntry(data.PipId); m_consumedFilesByPath.Clear(); m_dependencyConsumedFileIndex.Clear(); m_dependencyConsumedFileEndIndex.Clear(); m_dependencies.Clear(); m_builder.Clear(); m_directoryDependenciesFilterMap.Clear(); m_directoryHasSources.Clear(); var computation = data.StrongFingerprintComputations[0]; var pip = (Process)m_analyzer.GetPip(data.PipId); PipArtifacts.ForEachInput(pip, input => { if (input.IsFile) { AddConsumedFile(input.FileArtifact, DirectoryArtifact.Invalid, ContentFlag.Static | ContentFlag.Consumed); } else { foreach (var file in m_analyzer.GetContents(input.DirectoryArtifact)) { if (file.IsSourceFile) { m_directoryHasSources.Add(input.DirectoryArtifact); } AddConsumedFile(file, input.DirectoryArtifact, ContentFlag.Dynamic); } } return(true); }, includeLazyInputs: false); foreach (var input in computation.ObservedInputs) { var flag = (ContentFlag)((int)ContentFlag.AbsentPathProbe << (int)input.Type) | ContentFlag.Consumed; if (input.Type == ObservedInputType.FileContentRead || input.Type == ObservedInputType.ExistingFileProbe) { if (m_consumedFilesByPath.TryGetValue(input.Path, out var file)) { file.AddFlag(ContentFlag.Consumed); if (file.SourceFile != null) { file.SourceFile.AddFlag(ContentFlag.Consumed); } if (file.FinalFile != null) { file.FinalFile.AddFlag(ContentFlag.Consumed); } } else { AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag | ContentFlag.Unknown); } } else if (m_analyzer.AllAccesses) { AddConsumedFile(FileArtifact.CreateSourceFile(input.Path), m_analyzer.PipGraph.TryGetSealSourceAncestor(input.Path), flag); } } var entry = m_consumer; // Sort file dependencies for consistent output entry.FileDependencies.Sort(s_fileReferenceComparer); foreach (var fileDependency in entry.FileDependencies) { if (fileDependency.Producer != null) { var reference = entry.PipDependencies.GetOrAdd(fileDependency.Producer.PipId, p => new PipReference()); if (reference.Pip == null) { reference.Pip = m_analyzer.GetEntry(fileDependency.Producer.PipId); } reference.Flags |= fileDependency.ConsumedFile.Flags; } } string describe(PipEntry pe) { return($"{pe.SpecFileName}-{m_analyzer.GetDescription(m_analyzer.GetPip(pe.PipId))}"); } m_builder.AppendLine(describe(entry)); foreach (var fileDependency in entry.FileDependencies) { if (fileDependency.Producer != null && fileDependency.ConsumedFile.File.Artifact.IsOutputFile) { var pipId = fileDependency.Producer.PipId; var pipReference = entry.PipDependencies[pipId]; var directory = fileDependency.Directory?.Directory ?? DirectoryArtifact.Invalid; if (m_dependencies.Add((pipId, directory))) { if (pipReference.HasFlag(ContentFlag.Consumed)) { m_directoryDependenciesFilterMap[directory] = true; m_builder.AppendLine($"{entry.Identifier} -> Retaining pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}') (consumes '{ToString(fileDependency.ConsumedFile.File.Artifact)}')"); } else { m_directoryDependenciesFilterMap.TryAdd(directory, false); m_builder.AppendLine($"{entry.Identifier} -> Removing pip dependency on '{describe(pipReference.Pip)}' (declared via directory '{ToString(fileDependency.Directory)}')"); } } } } var trimmedDirectoryDependencies = new List <DirectoryArtifact>(); foreach (var d in entry.Process.DirectoryDependencies) { if (m_directoryDependenciesFilterMap.TryGetValue(d, out var shouldInclude)) { if (shouldInclude) { m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (used)"); } else if (m_directoryHasSources.Contains(d)) { m_builder.AppendLine($"{entry.Identifier} -> Retaining directory dependency on '{ToString(d)}' (has sources)"); } else { m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}'"); continue; } } else { var sealId = m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId(); if (!m_directoryHasSources.Contains(d) && !m_analyzer.PipTable.GetSealDirectoryKind(sealId).IsSourceSeal()) { m_builder.AppendLine($"{entry.Identifier} -> Removing directory dependency on '{ToString(d)}' (unused output directory)"); continue; } } entry.PipDependencies.TryAdd(m_analyzer.PipGraph.GetSealedDirectoryNode(d).ToPipId(), default); trimmedDirectoryDependencies.Add(d); } // Update directory dependencies which trimmed directory dependencies to allow writing // a pip into the serialized pip table that can run without the unnecessary dependencies entry.Process.UnsafeUpdateDirectoryDependencies(trimmedDirectoryDependencies.ToReadOnlyArray()); m_builder.AppendLine(); // Update the graph var modifiedGraph = m_analyzer.m_mutableGraph; using (var scope = modifiedGraph.AcquireExclusiveIncomingEdgeScope(entry.PipId.ToNodeId())) { foreach (var dependency in entry.PipDependencies) { if (dependency.Value == null || dependency.Value.HasFlag(ContentFlag.Consumed)) { scope.AddEdge(dependency.Key.ToNodeId()); } } entry.AddedEdges = true; } if (m_analyzer.SemiStableHashes.Contains(entry.SemistableHash)) { using (var writer = new StreamWriter(Path.Combine(m_analyzer.OutputFilePath, $"{GetFileName(entry.SpecFile)}_Pip{pip.FormattedSemiStableHash}.csv"))) { var table = new DisplayTable <Columns>(" , "); foreach (var dependency in entry.FileDependencies) { table.NextRow(); table.Set(Columns.Path, ToString(dependency.ConsumedFile.File.Artifact.Path)); table.Set(Columns.RwCount, dependency.ConsumedFile.File.Artifact.RewriteCount.ToString()); table.Set(Columns.Flags, dependency.ConsumedFile.Flags.ToString()); table.Set(Columns.Producer, dependency.Producer?.Identifier); table.Set(Columns.ProducerSpec, GetFileName(dependency.Producer?.SpecFile ?? AbsolutePath.Invalid)); table.Set(Columns.Dir, ToString(dependency.Directory)); table.Set(Columns.DirId, dependency.Directory?.Id); table.Set(Columns.DirSsh, dependency.Directory?.SemistableHash); } table.Write(writer); } } if (m_builder.Length != 0) { m_analyzer.Write(m_builder); } }
private void WriteGraphNodeEntry( Pip pip, IPipScheduleTraversal graph, PipExecutionContext context, Dictionary <DirectoryArtifact, int> directoryIds) { Contract.Requires(pip != null); m_writer.WriteWhitespace(Environment.NewLine); m_writer.WriteStartObject(); { WriteStaticPipDetails(pip, context); { // We get nice space savings by omitting dependsOn when it is empty (think HashSourceFile and WriteFile pips). bool writtenHeader = false; foreach (var dependency in graph.RetrievePipImmediateDependencies(pip)) { if (!writtenHeader) { m_writer.WritePropertyName("dependsOn"); m_writer.WriteStartArray(); writtenHeader = true; } if (IncludePip(dependency.PipType)) { m_writer.WriteValue(dependency.PipId.Value); } } if (writtenHeader) { m_writer.WriteEndArray(); } writtenHeader = false; PipArtifacts.ForEachInput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (dependency of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } else { if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId)) { Contract.Assume(false, "Expected directory artifact already written (input of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } if (!writtenHeader) { m_writer.WritePropertyName("consumes"); m_writer.WriteStartArray(); writtenHeader = true; } m_writer.WriteValue(dependencyId); return(true); }, includeLazyInputs: true); if (writtenHeader) { m_writer.WriteEndArray(); } } { m_writer.WritePropertyName("produces"); m_writer.WriteStartArray(); PipArtifacts.ForEachOutput(pip, dependency => { int dependencyId; if (dependency.IsFile) { if (!m_fileIds.TryGetValue(dependency.FileArtifact, out dependencyId)) { Contract.Assume(false, "Expected file artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } else { if (!directoryIds.TryGetValue(dependency.DirectoryArtifact, out dependencyId)) { Contract.Assume(false, "Expected directory artifact already written (output of pip) " + dependency.Path.ToString(context.PathTable)); throw new InvalidOperationException("Unreachable"); } } m_writer.WriteValue(dependencyId); return(true); }, includeUncacheable: true); m_writer.WriteEndArray(); } } m_writer.WriteEndObject(); }
/// <summary> /// Save the graph. This method produces the whole file content. It must be called only once. /// </summary> public void SaveGraphData( IPipScheduleTraversal graph, PipExecutionContext context, IEnumerable <PipExecutionPerformanceEventData> executionData, IEnumerable <string> workers, IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData) { Contract.Requires(graph != null); Contract.Requires(context != null); Contract.Assume(m_fileIds.Count == 0); // We have transitioned to the Exporting state. // Observed execution info may get queued up for when we begin draining the queue (below). // Don't overlap pip ids with file/directory ids int nextId = graph.PipCount + 1; var directoryIds = new Dictionary <DirectoryArtifact, int>(capacity: 1000); m_writer.WriteStartObject(); // Outermost object WritePreamble(); m_writer.WritePropertyName("artifacts"); m_writer.WriteStartArray(); // To save some work on pip deserialization (we are pathologically visiting every pip in the pip table), // we hold pips alive between the various passes. var pips = new List <Pip>(); { var directoryPips = new List <SealDirectory>(); // TODO: This is pretty expensive, as it loads all pips in memory foreach (Pip pip in graph.RetrieveScheduledPips()) { pips.Add(pip); PipArtifacts.ForEachInput(pip, input => { if (input.IsFile && input.FileArtifact.IsSourceFile) { AssignFileIdAndWriteFileEntry(input.FileArtifact, context, ref nextId); } return(true); }, includeLazyInputs: true ); PipArtifacts.ForEachOutput(pip, output => { if (output.IsFile) { AssignFileIdAndWriteFileEntry(output.FileArtifact, context, ref nextId); } return(true); }, includeUncacheable: true); // SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output, // we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before // writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited. if (pip.PipType == PipType.SealDirectory) { var directoryPip = (SealDirectory)pip; Contract.Assume(directoryPip.IsInitialized); foreach (FileArtifact f in directoryPip.Contents) { AssignFileIdAndWriteFileEntry(f, context, ref nextId); } directoryPips.Add(directoryPip); } } foreach (SealDirectory directoryPip in directoryPips) { AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents, directoryIds, ref nextId); } } m_writer.WriteEndArray(); m_writer.WritePropertyName("graph"); m_writer.WriteStartArray(); { // Note we are using the 'pips' list captured above rather than possibly deserializing pips again. foreach (Pip pip in pips) { if (!IncludePip(pip.PipType)) { continue; } WriteGraphNodeEntry(pip, graph, context, directoryIds); } } m_writer.WriteEndArray(); // Avoid holding a reference to this map if it isn't later needed by the special file details exporter if (fileData == null) { m_fileIds = null; } if (executionData != null) { m_writer.WritePropertyName("execution"); m_writer.WriteStartArray(); // Begin draining the execution entry queue, now that we are in an allowed state for that. // Iteration will complete only after Finish() is called, which marks the queue as complete (no new items). foreach (var pipWithPerf in executionData) { WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance); } // End the execution: array m_writer.WriteEndArray(); } if (workers != null) { ExportWorkerDetails(workers); } if (fileData != null) { ExportFileDetails(fileData); } // End the outermost object m_writer.WriteEndObject(); m_writer.Flush(); }