public override int Analyze() { Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath); using (var streamWriter = new StreamWriter(OutputFilePath)) using (JsonWriter writer = new JsonTextWriter(streamWriter)) { writer.WriteStartObject(); writer.WritePropertyName("Mounts"); { writer.WriteStartObject(); var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander; foreach (var mountRoot in mountPathExpander.GetAllRoots()) { var mount = mountPathExpander.GetSemanticPathInfo(mountRoot); writer.WritePropertyName(mount.RootName.ToString(StringTable)); writer.WriteValue(mountRoot.ToString(PathTable)); } writer.WriteEndObject(); } writer.WritePropertyName("FileInfo"); { writer.WriteStartObject(); foreach (var fileEntry in m_fileToConsumerMap) { var file = fileEntry.Key; var consumers = fileEntry.Value; var path = ToDisplayFilePath(file); if (path != null) { writer.WritePropertyName(path, true); { writer.WriteStartArray(); foreach (var consumer in consumers) { writer.WriteValue(ToDisplayString(consumer)); } writer.WriteEndArray(); } } } writer.WriteEndObject(); } writer.WritePropertyName("PipGraph"); { writer.WriteStartObject(); m_failedPipsClosure.UnsafeReset(); writer.WritePropertyName("root"); { writer.WriteStartArray(); foreach (var failedPip in m_failedPips) { writer.WriteValue(ToDisplayString(failedPip)); } writer.WriteEndArray(); } List <NodeId> dependencyBuffer = new List <NodeId>(); nodeVisitor.VisitTransitiveDependencies(m_failedPips.Select(p => p.ToNodeId()), m_failedPipsClosure, visitNode: node => { dependencyBuffer.Clear(); foreach (var dependencyEdge in DirectedGraph.GetIncomingEdges(node)) { if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile) { dependencyBuffer.Add(dependencyEdge.OtherNode); } } if (dependencyBuffer.Count != 0) { writer.WritePropertyName(ToDisplayString(node.ToPipId())); { writer.WriteStartArray(); foreach (var dependencyNode in dependencyBuffer) { writer.WriteValue(ToDisplayString(dependencyNode.ToPipId())); } writer.WriteEndArray(); } } return(true); }); writer.WriteEndObject(); } writer.WritePropertyName("TopFilesConsumedByFailedPips"); { writer.WriteStartObject(); writer.WritePropertyName("FailedPipsConsumedDirectly"); WriteConsumerMap(writer, failedPipDirectDepsOnly: true); writer.WritePropertyName("ConsumedByFailedAndFailedDependencies"); WriteConsumerMap(writer, failedPipDirectDepsOnly: false); writer.WriteEndObject(); } writer.WriteEndObject(); } return(0); }
/// <summary> /// Generate the output of failure data /// Failed pips + process chain : This points to the tool that failed. /// Set of dependencies diff contains: source and output changes. Which of those are directe referenced /// /// </summary> /// <param name="isDiff">There is diff data from another log</param> private void GenerateOutput(bool isDiff) { Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath); using (var streamWriter = new StreamWriter(OutputFilePath)) using (JsonWriter writer = new JsonTextWriter(streamWriter)) { writer.WriteStartObject(); writer.WritePropertyName("Mounts"); { writer.WriteStartObject(); var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander; foreach (var mountRoot in mountPathExpander.GetAllRoots()) { var mount = mountPathExpander.GetSemanticPathInfo(mountRoot); writer.WritePropertyName(mount.RootName.ToString(StringTable)); writer.WriteValue(mountRoot.ToString(PathTable)); } writer.WriteEndObject(); } writer.WriteWhitespace(Environment.NewLine); writer.WritePropertyName("Failures"); { writer.WriteStartArray(); foreach (var failedPip in m_failedPips) { var pip = (Process)m_pipTable.HydratePip(failedPip, PipQueryContext.ViewerAnalyzer); writer.WriteWhitespace(Environment.NewLine); writer.WriteStartObject(); // prints the semistable hash. WritePropertyAndValue(writer, "PipId", ToDisplayString(failedPip)); WritePropertyAndValue(writer, "Working Directory", ToDisplayFilePath(pip.WorkingDirectory)); var provenance = pip.Provenance; WritePropertyAndValue( writer, "Qualifier", provenance != null ? PipGraph.Context.QualifierTable.GetCanonicalDisplayString(provenance.QualifierId) : string.Empty); WritePropertyAndValue( writer, "OutputValueSymbol", provenance != null ? provenance.OutputValueSymbol.ToString(SymbolTable) : string.Empty); var pipPerformance = m_pipPerformance[pip.PipId.Value]; WritePropertyAndValue(writer, "PeakMemoryUsageMb", pipPerformance.PeakMemoryUsageMb.ToString()); WritePropertyAndValue(writer, "NumberOfProcesses", pipPerformance.NumberOfProcesses.ToString()); WritePropertyAndValue( writer, "FileMonitoringViolationsNotWhitelisted", pipPerformance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted.ToString()); if (isDiff) { WritePropertyAndValue(writer, "NumberOfImputChanges", GetDependencyChangesForPip(pip).ToString()); } if (m_pipDependencyViolationEventData.TryGetValue(pip.PipId, out var pipDependencyViolationEvent)) { foreach (var data in pipDependencyViolationEvent) { WritePropertyAndValue(writer, "DependencyViolationType", data.ViolationType.ToString()); WritePropertyAndValue(writer, "Path causing violation", ToDisplayFilePath(data.Path)); } } WriteProcessChain(writer, pip); // TODO : Are environment variables usefull for analysis // : Are are all the failures required or choose count as cmd line arg to take top n writer.WriteWhitespace(Environment.NewLine); writer.WriteEndObject(); } writer.WriteEndArray(); } writer.WriteWhitespace(Environment.NewLine); var fileToConsumerMap = m_fileToConsumerMap; var propertyName = "FileInfo"; if (isDiff) { propertyName = "FileInfoDiff"; fileToConsumerMap = m_fileToConsumerMapDiff; // If any added dependencies add them to the output if (m_fileToConsumerMapAdded.Count > 0) { WriteFileDependencies("FileInfoAdded", m_fileToConsumerMapAdded, writer); } // TODO:Add removed dependencies when compared to other log } WriteFileDependencies(propertyName, fileToConsumerMap, writer); writer.WriteWhitespace(Environment.NewLine); writer.WritePropertyName("PipGraph"); { writer.WriteStartObject(); m_failedPipsClosure.UnsafeReset(); writer.WritePropertyName("root"); { writer.WriteStartArray(); foreach (var failedPip in m_failedPips) { writer.WriteValue(ToDisplayString(failedPip)); } writer.WriteEndArray(); } List <NodeId> dependencyBuffer = new List <NodeId>(); nodeVisitor.VisitTransitiveDependencies( m_failedPips.Select(p => p.ToNodeId()), m_failedPipsClosure, visitNode: node => { dependencyBuffer.Clear(); foreach (var dependencyEdge in DataflowGraph.GetIncomingEdges(node)) { if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile) { dependencyBuffer.Add(dependencyEdge.OtherNode); } } if (dependencyBuffer.Count != 0) { writer.WritePropertyName(ToDisplayString(node.ToPipId())); { writer.WriteStartArray(); foreach (var dependencyNode in dependencyBuffer) { writer.WriteValue(ToDisplayString(dependencyNode.ToPipId())); } writer.WriteEndArray(); } } return(true); }); writer.WriteEndObject(); } writer.WriteEndObject(); } }
public async Task Stress() { const int N = 5; const int M = N * N; var context = BuildXLContext.CreateInstanceForTesting(); var loggingContext = CreateLoggingContextForTest(); var pathTable = context.PathTable; using (var tempFiles = new TempFileStorage(canGetFileNames: true)) { var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles); using (var pipTable = new PipTable( context.PathTable, context.SymbolTable, initialBufferSize: 1024, maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3, debug: false)) { var executionEnvironment = new PipQueueTestExecutionEnvironment(context, config, pipTable, GetSandboxedKextConnection()); Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) => { PipResult result; var operationTracker = new OperationTracker(runnablePip.LoggingContext); var pip = runnablePip.Pip; using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext)) { result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip); } executionEnvironment.MarkExecuted(pip); return(result); }; string executable = CmdHelper.OsShellExe; FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable)); // This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed). executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile); using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule)) { // phase 1: create some files var baseFileArtifacts = new List <FileArtifact>(); for (int i = 0; i < N; i++) { string destination = tempFiles.GetUniqueFileName(); AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination); FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion(); baseFileArtifacts.Add(destinationArtifact); PipData contents = PipDataBuilder.CreatePipData( context.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, i.ToString(CultureInfo.InvariantCulture)); var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context)); var pipId = pipTable.Add((uint)(i + 1), writeFile); var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable)); executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash); var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0); runnable.Start(new OperationTracker(loggingContext), loggingContext); runnable.SetDispatcherKind(DispatcherKind.IO); phase1PipQueue.Enqueue(runnable); } phase1PipQueue.SetAsFinalized(); phase1PipQueue.DrainQueues(); await Task.WhenAll( Enumerable.Range(0, 2).Select( async range => { using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule)) { // phase 2: do some more with those files var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >(); var checkerTasks = new ConcurrentQueue <Task>(); Action <PipId, Task <PipResult> > callback = (id, task) => { XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion); XAssert.IsFalse(task.Result.Status.IndicatesFailure()); Tuple <string, int> t; if (!pips.TryRemove(id, out t)) { XAssert.Fail(); } checkerTasks.Enqueue( Task.Run( () => { string actual = File.ReadAllText(t.Item1).Trim(); // TODO: Make this async XAssert.AreEqual(actual, t.Item2.ToString()); })); }; var r = new Random(0); for (int i = 0; i < M; i++) { int sourceIndex = r.Next(baseFileArtifacts.Count); FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex]; string destination = tempFiles.GetUniqueFileName(); AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination); FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion(); Pip pip; DispatcherKind queueKind; switch (r.Next(2)) { case 0: pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context)); queueKind = DispatcherKind.IO; executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact)); break; case 1: string workingDirectory = OperatingSystemHelper.IsUnixOS ? "/tmp" : Environment.GetFolderPath(Environment.SpecialFolder.Windows); AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory); var pipData = OperatingSystemHelper.IsUnixOS ? PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") : PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact); queueKind = DispatcherKind.CPU; pip = new Process( executableArtifact, workingDirectoryAbsolutePath, pipData, FileArtifact.Invalid, PipData.Invalid, ReadOnlyArray <EnvironmentVariable> .Empty, FileArtifact.Invalid, FileArtifact.Invalid, FileArtifact.Invalid, tempFiles.GetUniqueDirectory(pathTable), null, null, ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact), ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()), ReadOnlyArray <DirectoryArtifact> .Empty, ReadOnlyArray <DirectoryArtifact> .Empty, ReadOnlyArray <PipId> .Empty, ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)), ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)), ReadOnlyArray <StringId> .Empty, ReadOnlyArray <int> .Empty, ReadOnlyArray <ProcessSemaphoreInfo> .Empty, provenance: PipProvenance.CreateDummy(context), toolDescription: StringId.Invalid, additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty); executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact)); break; default: Contract.Assert(false); continue; } var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip); Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) => { var task = taskFactory(runnablePip); var pipResult = await task; callback(pipId, task); }; var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0); runnable.Start(new OperationTracker(loggingContext), loggingContext); runnable.SetDispatcherKind(queueKind); phase2PipQueue.Enqueue(runnable); if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex))) { Contract.Assert(false); } } phase2PipQueue.SetAsFinalized(); phase2PipQueue.DrainQueues(); XAssert.AreEqual(0, pips.Count); await Task.WhenAll(checkerTasks); } })); } } } }
/// <inheritdoc /> public GraphPatchingStatistics PartiallyReloadGraph(HashSet <AbsolutePath> affectedSpecs) { Contract.Requires(affectedSpecs != null); var startTime = DateTime.UtcNow; var mustSkipDueToTransitivity = new VisitationTracker(m_oldPipGraph); var mustPostponeDueToServicePips = new VisitationTracker(m_oldPipGraph); var toPostpone = new SortedList <uint, Pip>(); MultiValueDictionary <int, NodeId> nodesByHeight = m_oldPipGraph.TopSort(); int numReloadedPips = 0; int numNotReloadablePips = 0; m_builder.SealDirectoryTable.StartPatching(); // go one level at a time: // - process nodes at the same height in parallel // - postpone service-related pips because they must be added in the order of creation // (because, for example, pip builder adds forward edges from service client to service finalization pips) for (int height = 0; height < nodesByHeight.Count; height++) { Parallel.ForEach( nodesByHeight[height], new ParallelOptions { MaxDegreeOfParallelism = m_maxDegreeOfParallelism }, body: (node) => { var pipId = node.ToPipId(); var pipType = m_oldPipTable.GetPipType(pipId); // skip non-reloadable pips if (!s_reloadablePipTypes.Contains(pipType)) { Interlocked.Increment(ref numNotReloadablePips); return; } var pip = HydratePip(pipId); AbsolutePath?producingSpec = GetProducerSpecFile(pip); // check if this node must be skipped due to its spec file being affected if (producingSpec == null || affectedSpecs.Contains(producingSpec.Value)) { mustSkipDueToTransitivity.MarkVisited(node); MarkAllDependentsVisited(mustSkipDueToTransitivity, node); return; } // check if this pip is a service-related pip which must be postponed if (ServicePipKindUtil.IsServiceRelatedPip(pip)) { SynchronizedAddToSortedList(toPostpone, pip); MarkAllDependentsVisited(mustPostponeDueToServicePips, node); return; } // check if this node must be postponed because it depends on a node that was already postponed if (mustPostponeDueToServicePips.WasVisited(node)) { SynchronizedAddToSortedList(toPostpone, pip); MarkAllDependentsVisited(mustPostponeDueToServicePips, node); return; } // check if this node must be skipped due to transitivity ThrowIfVisited(mustSkipDueToTransitivity, pip); // everything passed: reload this node ReloadPip(pip, ref numReloadedPips); }); } // add postponed nodes sequentially in the order of creation foreach (var pip in toPostpone.Values) { var serviceKind = ServicePipKindUtil.ServiceKind(pip); if (serviceKind != ServicePipKind.ServiceShutdown && serviceKind != ServicePipKind.ServiceFinalization) { // 'shutdown' and 'finalization' are exception because of forward edges that are added // during construction from service client pips to service shutdown/finalization pips. ThrowIfVisited(mustSkipDueToTransitivity, pip); } ReloadPip(pip, ref numReloadedPips); } m_builder.SealDirectoryTable.FinishPatching(); return(new GraphPatchingStatistics { ElapsedMilliseconds = (int)DateTime.UtcNow.Subtract(startTime).TotalMilliseconds, NumPipsReloaded = numReloadedPips, NumPipsAutomaticallyAdded = m_oldPipGraph.NodeCount - numReloadedPips, NumPipsNotReloadable = numNotReloadablePips, NumPipsSkipped = mustSkipDueToTransitivity.VisitedCount, AffectedSpecs = affectedSpecs, }); }
public override int Analyze() { Console.WriteLine($"Analyzing"); m_block.Complete(); m_block.CompletionAsync().GetAwaiter().GetResult(); Console.WriteLine($"Writing Graph"); foreach (var pip in PipGraph.RetrieveAllPips()) { var serializedPip = pip; var nodeId = pip.PipId.ToNodeId(); bool addEdges = true; if (pip.PipType == PipType.Process) { var entry = GetEntry(pip.PipId); serializedPip = entry.Process; addEdges = !entry.AddedEdges; } if (addEdges && AddEdgesForPips) { using (var scope = m_mutableGraph.AcquireExclusiveIncomingEdgeScope(nodeId)) { foreach (var edge in DirectedGraph.GetIncomingEdges(nodeId)) { scope.AddEdge(edge.OtherNode, edge.IsLight); } } } serializedPip.ResetPipId(); m_pipTable.Add(nodeId.Value, serializedPip); } m_mutableGraph.Seal(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.DirectedGraph, m_mutableGraph.Serialize, Path.Combine(OutputFilePath, nameof(GraphCacheFile.DirectedGraph))) .GetAwaiter().GetResult(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.PipTable, w => m_pipTable.Serialize(w, Environment.ProcessorCount), Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipTable))) .GetAwaiter().GetResult(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.PipGraphId, PipGraph.SerializeGraphId, Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipGraphId))) .GetAwaiter().GetResult(); Console.WriteLine($"Simulating [Reading]"); var simulator = new BuildSimulatorAnalyzer(Input); simulator.Increment = SimulatorIncrement ?? simulator.Increment; simulator.ExecutionData.DirectedGraph = m_mutableGraph; simulator.OutputDirectory = OutputFilePath; if (!simulator.ReadExecutionLog()) { Args.TruncatedXlgWarning(); } Console.WriteLine($"Simulating [Analyzing]"); simulator.Analyze(); Console.WriteLine($"Blocking Dependency Analysis"); DisplayTable <DepColumn> depTable = new DisplayTable <DepColumn>(" , "); foreach (var pipId in PipTable.Keys) { var pipType = PipTable.GetPipType(pipId); if (pipType == PipType.Process) { var entry = GetEntry(pipId); (PipId node, ulong cost)maxConsumedDependency = default; (PipId node, ulong cost)maxDependency = default; foreach (var dep in entry.PipDependencies) { var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.Key.ToNodeId()]; if (!maxDependency.node.IsValid || cost > maxDependency.cost) { maxDependency = (dep.Key, cost); } if (dep.Value != null && dep.Value.HasFlag(ContentFlag.Consumed)) { if (!maxConsumedDependency.node.IsValid || cost > maxConsumedDependency.cost) { maxConsumedDependency = (dep.Key, cost); } } } depTable.NextRow(); depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier}"); depTable.Set(DepColumn.MaxConsumedDependency, ToString(maxConsumedDependency.node)); depTable.Set(DepColumn.MaxConsumedDependencyChainCost, maxConsumedDependency.cost.ToMinutes()); depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node)); depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes()); } else if (pipType == PipType.SealDirectory && !PipTable.GetSealDirectoryKind(pipId).IsSourceSeal() && !IsSourceOnlySeal(pipId)) { var seal = (SealDirectory)GetPip(pipId); var entry = GetEntry(seal.Directory); (PipId node, ulong cost)maxDependency = default; foreach (var dep in DirectedGraph.GetIncomingEdges(pipId.ToNodeId())) { var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.OtherNode]; if (!maxDependency.node.IsValid || cost > maxDependency.cost) { maxDependency = (dep.OtherNode.ToPipId(), cost); } } depTable.NextRow(); depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier} ({entry.FileCount} files)"); depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node)); depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes()); depTable.Set(DepColumn.Directory, seal.DirectoryRoot.ToString(PathTable)); } } using (var blockAnalysisWriter = new StreamWriter(Path.Combine(OutputFilePath, "blockAnalysis.txt"))) { depTable.Write(blockAnalysisWriter); } m_writer.Dispose(); Console.WriteLine($"Analyzing complete"); return(0); }
private IEnumerable <PipReference> AllPipIds(PipTable pipTable) { return(pipTable.StableKeys .Where(pipId => pipTable.GetPipType(pipId) != PipType.HashSourceFile) .Select(pipId => new PipReference(pipTable, pipId, PipQueryContext.Explorer))); }