private static void AddIfNotVisited(Queue <NodeId> queue, NodeId node, VisitationTracker visitedNodes) { if (visitedNodes.MarkVisited(node)) { queue.Enqueue(node); } }
public override int Analyze() { Console.WriteLine("Starting analysis"); HashSet <NodeId> sourceNodes = new HashSet <NodeId>(); foreach (NodeId nodeId in CachedGraph.DirectedGraph.GetSourceNodes()) { if (PipTable.GetPipType(nodeId.ToPipId()) == PipType.HashSourceFile) { foreach (var edge in CachedGraph.DirectedGraph.GetOutgoingEdges(nodeId)) { sourceNodes.Add(edge.OtherNode); } } else { sourceNodes.Add(nodeId); } } NodeAndCriticalPath[] criticalPaths = new NodeAndCriticalPath[sourceNodes.Count]; Console.WriteLine("Computing critical paths"); VisitationTracker visitedNodes = new VisitationTracker(CachedGraph.DirectedGraph); int i = 0; foreach (var sourceNode in sourceNodes) { criticalPaths[i] = ComputeCriticalPath(sourceNode, visitedNodes); i++; } Console.WriteLine("Sorting critical paths"); Array.Sort(criticalPaths, (c1, c2) => - c1.Time.CompareTo(c2.Time)); Console.WriteLine("Writing critical paths"); int count = Math.Min(m_criticalPathCount, criticalPaths.Length); m_writer.WriteLine("Top {0} critical paths (dependencies first)", count); for (int j = 0; j < count; j++) { var nodeAndCriticalPath = criticalPaths[j]; m_writer.WriteLine("CRITICAL PATH {0}", j + 1); m_writer.WriteLine("TIME: {0} seconds", ToSeconds(nodeAndCriticalPath.Time)); m_writer.WriteLine(); m_writer.WriteLine(PrintCriticalPath(nodeAndCriticalPath, GetElapsed, GetKernelTime, GetUserTime, GetCriticalPath, out var criticalpath)); m_writer.WriteLine(); criticalPathData.criticalPaths.Add(criticalpath); } criticalPathData.wallClockTopPips = PrintQueue("Wall Clock", m_topWallClockPriorityQueue); criticalPathData.userTimeTopPips = PrintQueue("User Time", m_topUserTimePriorityQueue); criticalPathData.kernelTimeTopPips = PrintQueue("Kernel Time", m_topKernelTimePriorityQueue); return(0); }
private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, IConfiguration configuration, string downLoadedPreviousFingerprintStoreSavedPath, FingerprintStoreTestHooks testHooks = null) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; m_batchLoggingQueue = configuration.Logging.CacheMissBatch ? NagleQueue <JProperty> .Create( BatchLogging, maxDegreeOfParallelism : 1, interval : TimeSpan.FromMinutes(5), batchSize : 100) : null; m_testHooks = testHooks; m_testHooks?.InitRuntimeCacheMisses(); m_configuration = configuration; m_downLoadedPreviousFingerprintStoreSavedPath = downLoadedPreviousFingerprintStoreSavedPath; }
private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, CacheMissDiffFormat cacheMissDiffFormat, bool cacheMissBatch, FingerprintStoreTestHooks testHooks = null) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; m_cacheMissDiffFormat = cacheMissDiffFormat; m_maxCacheMissCanPerform = cacheMissBatch ? EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value * EngineEnvironmentSettings.MaxMessagesPerBatch : EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value; m_batchLoggingQueue = cacheMissBatch ? NagleQueue <JProperty> .Create( BatchLogging, maxDegreeOfParallelism : 1, interval : TimeSpan.FromMinutes(1), batchSize : EngineEnvironmentSettings.MaxMessagesPerBatch) : null; m_testHooks = testHooks; m_testHooks?.InitRuntimeCacheMisses(); }
public AnalysisModel(CachedGraph graph) { CachedGraph = graph; ChangedPips = new VisitationTracker(CachedGraph.DataflowGraph); visitor = new NodeVisitor(graph.DataflowGraph); LookupHashFunction = LookupHash; }
public FailedPipInputAnalyzer(AnalysisInput input) : base(input) { nodeVisitor = new NodeVisitor(DirectedGraph); m_failedPipsClosure = new VisitationTracker(DirectedGraph); m_cachedPips = new VisitationTracker(DirectedGraph); }
public FailedPipsDumpAnalyzer(AnalysisInput input) : base(input) { nodeVisitor = new NodeVisitor(DirectedGraph); m_failedPipsClosure = new VisitationTracker(DirectedGraph); m_cachedPips = new VisitationTracker(DirectedGraph); m_pipTable = input.CachedGraph.PipTable; }
public DiffAnalyzer(AnalysisInput input, FailedPipsDumpAnalyzer analyzer) : base(input) { m_failedPipsDumpAnalyzer = analyzer; m_failedPipsClosure = new VisitationTracker(DirectedGraph); nodeVisitor = new NodeVisitor(DirectedGraph); m_pipTable = input.CachedGraph.PipTable; m_tokenizeByMounts = analyzer.TokenizeByMounts; InclusionMountNames = analyzer.InclusionMountNames; }
private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; }
private void VisitTransitiveClosure( Queue <NodeId> queue, VisitationTracker visitedNodes, VisitNode visitNodeDependencies, VisitNode visitNodeDependents, bool dependencies, bool dependents) { Contract.Assert(visitNodeDependencies == null || dependencies); Contract.Assert(visitNodeDependents == null || dependents); while (queue.Count != 0) { var node = queue.Dequeue(); if (dependencies) { if (visitNodeDependencies == null || visitNodeDependencies(node)) { foreach (Edge inEdge in m_dataflowGraph.GetIncomingEdges(node)) { if (visitedNodes.MarkVisited(inEdge.OtherNode)) { queue.Enqueue(inEdge.OtherNode); } } } } if (dependents) { if (visitNodeDependents == null || visitNodeDependents(node)) { foreach (Edge outEdge in m_dataflowGraph.GetOutgoingEdges(node)) { if (visitedNodes.MarkVisited(outEdge.OtherNode)) { queue.Enqueue(outEdge.OtherNode); } } } } } }
/// <summary> /// Visits the transitive dependencies of the node /// </summary> public void VisitTransitiveDependencies( NodeId startNode, VisitationTracker visitedNodes, VisitNode visitNode) { using (var queueWrapper = m_nodeQueuePool.GetInstance()) { var queue = queueWrapper.Instance; AddIfNotVisited(queue, startNode, visitedNodes); VisitTransitiveClosure( queue, visitedNodes, visitNodeDependents: null, visitNodeDependencies: visitNode, dependencies: true, dependents: false); } }
/// <summary> /// Visits the transitive reachable nodes of the given nodes /// </summary> public void VisitTransitiveReachableNodes( IEnumerable <NodeId> startNodes, VisitationTracker visitedNodes, VisitNode visitNodeDependencies, VisitNode visitNodeDependents) { using (var queueWrapper = m_nodeQueuePool.GetInstance()) { var queue = queueWrapper.Instance; AddIfNotVisited(queue, startNodes, visitedNodes); VisitTransitiveClosure( queue, visitedNodes, visitNodeDependencies: visitNodeDependencies, visitNodeDependents: visitNodeDependents, dependencies: true, dependents: true); } }
private NodeAndCriticalPath ComputeCriticalPath(NodeId node, VisitationTracker visitedNodes) { var criticalPath = GetCriticalPath(node); if (criticalPath.Node.IsValid) { return(criticalPath); } criticalPath = new NodeAndCriticalPath() { Node = node, Time = GetElapsed(node), }; if (!visitedNodes.MarkVisited(node)) { return(criticalPath); } NodeAndCriticalPath maxDependencyCriticalPath = default(NodeAndCriticalPath); foreach (var dependency in CachedGraph.DirectedGraph.GetOutgoingEdges(node)) { var dependencyCriticalPath = ComputeCriticalPath(dependency.OtherNode, visitedNodes); if (dependencyCriticalPath.Time > maxDependencyCriticalPath.Time) { maxDependencyCriticalPath = dependencyCriticalPath; } } criticalPath.Next = maxDependencyCriticalPath.Node; criticalPath.Time += maxDependencyCriticalPath.Time; SetCriticalPath(node, criticalPath); return(criticalPath); }
public ComputationContext(MutableDirectedGraph graph) { Dependencies = new VisitationTracker(graph); Dependents = new VisitationTracker(graph); Visitor = new NodeVisitor(graph); }
private IEnumerable<NodeId> GetNodesToSchedule( HashSet<NodeId> nodesToSchedule, VisitationTracker transitiveDependencyNodeFilter, ForceSkipDependenciesMode forceSkipDepsMode, bool scheduleMetaPips, HashSet<NodeId> mustExecute, BuildSetCalculatorStats stats, ref int metaPipCount) { if (forceSkipDepsMode == ForceSkipDependenciesMode.Disabled) { ScheduleDependenciesUntilCleanAndMaterialized(nodesToSchedule, transitiveDependencyNodeFilter, stats); } else { int numExplicitlySelectedProcesses, numScheduledProcesses, numExecutedProcesses, numExecutedProcessesWithoutDirty = 0; Func<NodeId, bool> isProcess = (node) => GetPipType(node) == PipType.Process; using (m_counters.StartStopwatch(PipExecutorCounter.ForceSkipDependenciesScheduleDependenciesUntilInputsPresentDuration)) { numExplicitlySelectedProcesses = nodesToSchedule.Count(isProcess); // Calculate how many process pips are in the transitive dependency closure of the filtered pips foreach (var node in m_graph.Nodes) { if (transitiveDependencyNodeFilter.WasVisited(node) && isProcess(node)) { numExecutedProcessesWithoutDirty++; } } ScheduleDependenciesUntilRequiredInputsPresent(nodesToSchedule, transitiveDependencyNodeFilter, mustExecute, forceSkipDepsMode); numScheduledProcesses = nodesToSchedule.Where(isProcess).Count(); numExecutedProcesses = mustExecute.Where(isProcess).Count(); } Logger.Log.DirtyBuildStats( m_loggingContext, (long)m_counters.GetElapsedTime(PipExecutorCounter.ForceSkipDependenciesScheduleDependenciesUntilInputsPresentDuration).TotalMilliseconds, forceSkipDepsMode == ForceSkipDependenciesMode.Module, numExplicitlySelectedProcesses, numScheduledProcesses, numExecutedProcesses, numExecutedProcessesWithoutDirty - numExecutedProcesses); } int scheduledMetaPipCount = 0; if (scheduleMetaPips) { using (m_counters.StartStopwatch(PipExecutorCounter.BuildSetCalculatorComputeAffectedMetaPips)) { // We compute the affected meta pips from the scheduled nodes. Simply traversing the graph from // the scheduled nodes is expensive, so we split the process by first computing the meta-pip frontiers, // i.e., the meta-pips that directly depend on a scheduled node. This computation can be done in parallel. // Assumption: the dependents of meta pips are always meta pips. // (1) Compute the meta-pips frontiers from the scheduled nodes. VisitationTracker visitedNodes = new VisitationTracker(m_graph); ConcurrentDictionary<NodeId, Unit> metaPipFrontier = new ConcurrentDictionary<NodeId, Unit>(); Parallel.ForEach( nodesToSchedule, node => { foreach (var oe in m_graph.GetOutgoingEdges(node)) { if (GetPipType(oe.OtherNode).IsMetaPip()) { metaPipFrontier.TryAdd(oe.OtherNode, Unit.Void); } } }); // (2) Traverse the graph from the frontiers. m_visitor.VisitTransitiveDependents( metaPipFrontier.Keys, visitedNodes, node => { nodesToSchedule.Add(node); ++scheduledMetaPipCount; return true; }); } metaPipCount = scheduledMetaPipCount; } return nodesToSchedule; }
private void ScheduleDependenciesUntilCleanAndMaterialized( HashSet<NodeId> nodesToSchedule, VisitationTracker buildCone, BuildSetCalculatorStats stats) { int initialNodesToScheduleCount = nodesToSchedule.Count; int initialProcessesToScheduleCount = nodesToSchedule.Count(IsProcess); int nodesAddedDueToNotCleanMaterializedCount = 0; int processesAddedDueToNotCleanMaterializedCount = 0; int nodesAddedDueToCollateralDirtyCount = 0; int processesAddedDueToCollateralDirtyCount = 0; // TODO: If this method turns out to be the bottleneck, we can make it parallel later. var cleanMaterializedNodes = new HashSet<NodeId>(); using (m_counters.StartStopwatch(PipExecutorCounter.BuildSetCalculatorScheduleDependenciesUntilCleanAndMaterialized)) { // Add all dirty nodes from nodesToSchedule. From those dirty nodes, we try to add to nodesToSchedule // their dependencies that need to be built. // Nodes that are not materialized should have been marked dirty by CalculateDirtyNodes. nodesToSchedule.RemoveWhere(node => !IsNodeDirty(node)); var nodeQueue = new Queue<NodeId>(nodesToSchedule); Action<NodeId> addNode = markedNode => { if (buildCone.WasVisited(markedNode) && !GetPipType(markedNode).IsMetaPip() && nodesToSchedule.Add(markedNode)) { // Node is in the build cone, and has not been scheduled yet. nodeQueue.Enqueue(markedNode); ++nodesAddedDueToCollateralDirtyCount; if (IsProcess(markedNode)) { ++processesAddedDueToCollateralDirtyCount; } } }; Action<NodeId> scheduleDependencyNode = node => { if (buildCone.WasVisited(node)) { // The node is in the build cone. var pipType = GetPipType(node); // The node is clean if it's (1) marked as clean and materialized and (2) none of its outputs are rewritten. // Condition (2) is conservative, and is needed for correctness in the presence of rewritten files. bool isCleanMaterialized = IsNodeCleanAndMaterialized(node) && !IsRewrittenPip(node); if (!isCleanMaterialized && nodesToSchedule.Add(node)) { // (1) Node is dirty or has not materialized its outputs. // (2) Node has not been scheduled yet. // Mark process node dirty, and add its dependents so that the dependencies of its dependents can be added later. MarkProcessNodeDirtyAndAddItsDependents(node, addNode); ++nodesAddedDueToNotCleanMaterializedCount; if (pipType != PipType.HashSourceFile) { nodeQueue.Enqueue(node); } if (IsProcess(node)) { ++processesAddedDueToNotCleanMaterializedCount; } } if (isCleanMaterialized) { cleanMaterializedNodes.Add(node); } } }; while (nodeQueue.Count > 0) { NodeId node = nodeQueue.Dequeue(); foreach (Edge inEdge in m_graph.GetIncomingEdges(node)) { scheduleDependencyNode(inEdge.OtherNode); } } nodesToSchedule.UnionWith(cleanMaterializedNodes); } stats.CleanMaterializedNodeFrontierCount = cleanMaterializedNodes.Count; stats.CleanMaterializedProcessFrontierCount = cleanMaterializedNodes.Count(IsProcess); Logger.Log.BuildSetCalculatorScheduleDependenciesUntilCleanAndMaterializedStats( m_loggingContext, initialNodesToScheduleCount, initialProcessesToScheduleCount, nodesAddedDueToNotCleanMaterializedCount, processesAddedDueToNotCleanMaterializedCount, nodesAddedDueToCollateralDirtyCount, processesAddedDueToCollateralDirtyCount, stats.CleanMaterializedNodeFrontierCount, stats.CleanMaterializedProcessFrontierCount, (int)m_counters.GetElapsedTime(PipExecutorCounter.BuildSetCalculatorScheduleDependenciesUntilCleanAndMaterialized).TotalMilliseconds); }
private static void AddIfNotVisited(Queue <NodeId> queue, IEnumerable <NodeId> nodes, VisitationTracker visitedNodes) { foreach (var node in nodes) { if (visitedNodes.MarkVisited(node)) { queue.Enqueue(node); } } }
/// <summary> /// Gets nodes to schedule. /// </summary> /// <param name="scheduleDependents">If true, then include all transitive dependents of the explicitly scheduled nodes.</param> /// <param name="explicitlyScheduledNodes">Explicitly scheduled nodes.</param> /// <param name="forceSkipDepsMode">If not disabled, then skip dependencies. This corresponds to "dirty" build.</param> /// <param name="scheduleMetaPips">If true, metapips will be scheduled</param> /// <returns>Nodes to schedule.</returns> public GetScheduledNodesResult GetNodesToSchedule( bool scheduleDependents, IEnumerable<NodeId> explicitlyScheduledNodes, ForceSkipDependenciesMode forceSkipDepsMode, bool scheduleMetaPips) { int explicitlySelectedNodeCount; int explicitlySelectedProcessCount; int dirtyNodeCount; int dirtyProcessCount; int nonMaterializedNodeCount; int nonMaterializedProcessCount; int processesInBuildCone = 0; HashSet<NodeId> nodesToSchedule; VisitationTracker transitiveDependencyNodeFilter; using (m_counters.StartStopwatch(PipExecutorCounter.BuildSetCalculatorComputeBuildCone)) { var visitedNodes = new VisitationTracker(m_graph); nodesToSchedule = new HashSet<NodeId>(explicitlyScheduledNodes); explicitlySelectedNodeCount = nodesToSchedule.Count; explicitlySelectedProcessCount = nodesToSchedule.Count(IsProcess); // 1. Calculate dirty nodes. // The filter-passing set may include nodes which are dirty/clean and schedulable/not-schedulable (w.r.t. state). // We want stats on dirty vs. not-dirty, and want to drop anything not schedulable. // This step also marks dirty non-materialized nodes. CalculateDirtyNodes( nodesToSchedule, out dirtyNodeCount, out dirtyProcessCount, out nonMaterializedNodeCount, out nonMaterializedProcessCount); if (dirtyNodeCount == 0) { int duration = (int) m_counters.GetElapsedTime(PipExecutorCounter.BuildSetCalculatorComputeBuildCone).TotalMilliseconds; // Build cone is the same as the explicitly selected processes. Logger.Log.BuildSetCalculatorProcessStats( m_loggingContext, m_graph.Nodes.Count(IsProcess), explicitlySelectedProcessCount, explicitlySelectedProcessCount, explicitlySelectedProcessCount, 0, duration); Logger.Log.BuildSetCalculatorStats( m_loggingContext, 0, 0, explicitlySelectedNodeCount, explicitlySelectedProcessCount, nonMaterializedNodeCount, nonMaterializedProcessCount, duration, 0, 0, 0, 0); return GetScheduledNodesResult.CreateForNoOperationBuild(explicitlySelectedProcessCount); } // 2. Add transitive dependents of explicitly scheduled nodes (if requested). if (scheduleDependents) { m_visitor.VisitTransitiveDependents( nodesToSchedule, visitedNodes, node => { // Don't schedule dependents that are meta pips. These may artificially connect unrequested // pips since we will later schedule their dependencies. For example, this would cause // everything referenced by a spec file pip to be scheduled as a single unit. PipType pipType = GetPipType(node); if (!pipType.IsMetaPip()) { nodesToSchedule.Add(node); if (pipType == PipType.Process) { ++processesInBuildCone; } return true; } return false; }); } // At this point nodesToSchedule contains // (1) all nodes that are explicitly scheduled (explicitlyScheduledNodes), and // (2) if scheduleDependents is true, all dependents of (1) transitively. transitiveDependencyNodeFilter = visitedNodes; // 3. Collect/visit transitive dependencies, but don't put it in nodesToSchedule. transitiveDependencyNodeFilter.UnsafeReset(); // The code below essentially does m_visitor.VisitTransitiveDependencies(nodesToSchedule, transitiveDependencyNodeFilter, node => true), but in parallel. foreach (var nodeId in nodesToSchedule) { if (transitiveDependencyNodeFilter.MarkVisited(nodeId)) { if (IsProcess(nodeId)) { ++processesInBuildCone; } } } ParallelAlgorithms.WhileNotEmpty( nodesToSchedule, (node, add) => { foreach (Edge inEdge in m_graph.GetIncomingEdges(node)) { if (visitedNodes.MarkVisited(inEdge.OtherNode)) { add(inEdge.OtherNode); if (IsProcess(inEdge.OtherNode)) { Interlocked.Increment(ref processesInBuildCone); } } } }); // At this point nodesToSchedule hasn't change from step 2. // But now, transitiveDependencyNodeFilter have already marked all nodes in nodesToSchedule, plus // their dependencies transitively. } IEnumerable<NodeId> scheduledNodes; var mustExecute = new HashSet<NodeId>(); var stats = new BuildSetCalculatorStats(); var metaPipCount = 0; using (m_counters.StartStopwatch(PipExecutorCounter.BuildSetCalculatorGetNodesToSchedule)) { scheduledNodes = GetNodesToSchedule( nodesToSchedule, transitiveDependencyNodeFilter, forceSkipDepsMode, scheduleMetaPips, mustExecute, stats, ref metaPipCount); } int buildConeDuration = (int) m_counters.GetElapsedTime(PipExecutorCounter.BuildSetCalculatorComputeBuildCone).TotalMilliseconds; int getScheduledNodesDuration = (int) m_counters.GetElapsedTime(PipExecutorCounter.BuildSetCalculatorGetNodesToSchedule).TotalMilliseconds; int scheduledProcessCount = scheduledNodes.Count(IsProcess); Logger.Log.BuildSetCalculatorProcessStats( m_loggingContext, m_graph.Nodes.Count(IsProcess), explicitlySelectedProcessCount, processesInBuildCone, (processesInBuildCone - scheduledProcessCount) + stats.CleanMaterializedProcessFrontierCount, scheduledProcessCount, buildConeDuration + getScheduledNodesDuration); Logger.Log.BuildSetCalculatorStats( m_loggingContext, dirtyNodeCount, dirtyProcessCount, explicitlySelectedNodeCount, explicitlySelectedProcessCount, nonMaterializedNodeCount, nonMaterializedProcessCount, buildConeDuration, scheduledNodes.Count(), scheduledProcessCount, metaPipCount, getScheduledNodesDuration); int incrementalSchedulingCacheHits = forceSkipDepsMode == ForceSkipDependenciesMode.Disabled ? (processesInBuildCone - scheduledProcessCount + stats.CleanMaterializedProcessFrontierCount) : 0; return new GetScheduledNodesResult( scheduledNodes: scheduledNodes, mustExecuteNodes: mustExecute, incrementalSchedulingCacheHitProcesses: incrementalSchedulingCacheHits, cleanMaterializedProcessFrontierCount: forceSkipDepsMode == ForceSkipDependenciesMode.Disabled ? stats.CleanMaterializedProcessFrontierCount : 0); }
public CodexAnalyzer(AnalysisInput input, string outputDirectory) : base(input) { m_executedPipsTracker = new VisitationTracker(input.CachedGraph.DirectedGraph); OutputDirectory = outputDirectory; }