private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, CacheMissDiffFormat cacheMissDiffFormat, bool cacheMissBatch, FingerprintStoreTestHooks testHooks = null) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; m_cacheMissDiffFormat = cacheMissDiffFormat; m_maxCacheMissCanPerform = cacheMissBatch ? EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value * EngineEnvironmentSettings.MaxMessagesPerBatch : EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value; m_batchLoggingQueue = cacheMissBatch ? NagleQueue <JProperty> .Create( BatchLogging, maxDegreeOfParallelism : 1, interval : TimeSpan.FromMinutes(1), batchSize : EngineEnvironmentSettings.MaxMessagesPerBatch) : null; m_testHooks = testHooks; m_testHooks?.InitRuntimeCacheMisses(); }
private static void AddIncidentNodes( IReadonlyDirectedGraph graph, RangedNodeSet walkFromNodes, Func <IReadonlyDirectedGraph, NodeId, IEnumerable <Edge> > getEdges, Func <NodeId, NodeId, bool> validateEdgeTopoProperty, NodeRange incidentNodeFilter, bool skipOutOfOrderNodes, RangedNodeSet addTo) { Contract.Requires(!incidentNodeFilter.IsEmpty); foreach (NodeId existingNode in walkFromNodes) { IEnumerable <Edge> edges = getEdges(graph, existingNode); foreach (Edge edge in edges) { NodeId other = edge.OtherNode; if (skipOutOfOrderNodes && !validateEdgeTopoProperty(existingNode, other)) { continue; } if (incidentNodeFilter.Contains(other)) { addTo.Add(edge.OtherNode); } } } }
/// <summary> /// Constructor. /// </summary> private FingerprintStoreExecutionLogTarget( LoggingContext loggingContext, PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, FingerprintStore fingerprintStore, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { m_context = context; m_pipTable = pipTable; PipContentFingerprinter = pipContentFingerprinter; FingerprintStore = fingerprintStore; m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>(); m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync( this, loggingContext, context, configuration, cache, graph, runnablePipPerformance); }
private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, IConfiguration configuration, string downLoadedPreviousFingerprintStoreSavedPath, FingerprintStoreTestHooks testHooks = null) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; m_batchLoggingQueue = configuration.Logging.CacheMissBatch ? NagleQueue <JProperty> .Create( BatchLogging, maxDegreeOfParallelism : 1, interval : TimeSpan.FromMinutes(5), batchSize : 100) : null; m_testHooks = testHooks; m_testHooks?.InitRuntimeCacheMisses(); m_configuration = configuration; m_downLoadedPreviousFingerprintStoreSavedPath = downLoadedPreviousFingerprintStoreSavedPath; }
/// <summary> /// Constructor. /// </summary> private FingerprintStoreExecutionLogTarget( LoggingContext loggingContext, PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, FingerprintStore fingerprintStore, FingerprintStore cacheLookupFingerprintStore, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, CounterCollection <FingerprintStoreCounters> counters, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { m_context = context; m_pipTable = pipTable; LoggingContext = loggingContext; PipContentFingerprinter = pipContentFingerprinter; ExecutionFingerprintStore = fingerprintStore; CacheLookupFingerprintStore = cacheLookupFingerprintStore; // Cache lookup store is per-build state and doesn't need to be garbage collected (vs. execution fignerprint store which is persisted build-over-build) CacheLookupFingerprintStore?.GarbageCollectCancellationToken.Cancel(); m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>(); Counters = counters; FingerprintStoreMode = configuration.Logging.FingerprintStoreMode; m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync( this, loggingContext, context, configuration, cache, graph, runnablePipPerformance); Contract.Assume(FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly || CacheLookupFingerprintStore != null, "Unless /storeFingerprints flag is set to /storeFingerprints:ExecutionFingerprintsOnly, the cache lookup FingerprintStore must exist."); }
public FilteredDirectedGraph(IReadonlyDirectedGraph graph, VisitationTracker nodeFilter) { m_graph = graph; m_nodeFilter = nodeFilter; m_nodePredicate = node => nodeFilter.WasVisited(node); m_edgePredicate = edge => nodeFilter.WasVisited(edge.OtherNode); m_nodeHeights = Lazy.Create(ComputeHeights); }
public Context( PipExecutionContext pipContext, PipGraph pipGraph, IReadonlyDirectedGraph scheduledGraph, AbsolutePath configFilePath, IIdeConfiguration ideConfig) { Contract.Requires(pipGraph != null); Contract.Requires(scheduledGraph != null); Contract.Requires(configFilePath.IsValid); Contract.Requires(ideConfig.SolutionRoot.IsValid); Contract.Requires(ideConfig.SolutionName.IsValid); Contract.Requires(ideConfig.IsEnabled); Contract.Requires(ideConfig.IsNewEnabled); PipGraph = pipGraph; ScheduledGraph = scheduledGraph; StringTable = pipContext.StringTable; PathTable = pipContext.PathTable; SymbolTable = pipContext.SymbolTable; QualifierTable = pipContext.QualifierTable; IdeConfig = ideConfig; DotSettingsPathStr = ideConfig.DotSettingsFile.IsValid ? ideConfig.DotSettingsFile.ToString(PathTable) : null; ConfigFilePathStr = configFilePath.ToString(PathTable); EnlistmentRoot = configFilePath.GetParent(PathTable); EnlistmentRootStr = EnlistmentRoot.ToString(PathTable); SolutionRoot = ideConfig.SolutionRoot; SolutionRootStr = SolutionRoot.ToString(PathTable); SolutionFilePathStr = IdeGenerator.GetSolutionPath(ideConfig, PathTable).ToString(PathTable); CanWriteToSrc = ideConfig.CanWriteToSrc ?? false; ProjectsRoot = CanWriteToSrc ? EnlistmentRoot : SolutionRoot.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "Projects")); ResxExtensionName = PathAtom.Create(StringTable, ".resx"); CscExeName = PathAtom.Create(StringTable, "csc.exe"); CscDllName = PathAtom.Create(StringTable, "csc.dll"); XunitConsoleDllName = PathAtom.Create(StringTable, "xunit.console.dll"); XunitConsoleExeName = PathAtom.Create(StringTable, "xunit.console.exe"); QtestExeName = PathAtom.Create(StringTable, "DBS.QTest.exe"); DotnetName = PathAtom.Create(StringTable, "dotnet"); DotnetExeName = PathAtom.Create(StringTable, "dotnet.exe"); ResgenExeName = PathAtom.Create(StringTable, "ResGen.exe"); ResourcesExtensionName = PathAtom.Create(StringTable, ".resources"); CsExtensionName = PathAtom.Create(StringTable, ".cs"); DllExtensionName = PathAtom.Create(StringTable, ".dll"); ClExeName = PathAtom.Create(StringTable, "cl.exe"); LinkExeName = PathAtom.Create(StringTable, "Link.exe"); VsTestExeName = PathAtom.Create(StringTable, "vstest.console.exe"); AssemblyDeploymentTag = StringId.Create(StringTable, "assemblyDeployment"); TestDeploymentTag = StringId.Create(StringTable, "testDeployment"); }
/// <summary> /// Topologically sorts nodes in a given graph. If no <paramref name="nodes"/> are specified all nodes in the graph /// are used (<see cref="IReadonlyDirectedGraph.Nodes"/>); if <paramref name="nodes"/> are specified, they must all /// belong to the given <paramref name="graph"/>. /// /// Returns a multi-value dictionary with keys ranging from 0 to max node height. /// </summary> public static MultiValueDictionary <int, NodeId> TopSort(this IReadonlyDirectedGraph graph, IEnumerable <NodeId> nodes = null) { nodes = nodes ?? graph.Nodes; MultiValueDictionary <int, NodeId> nodesByHeight = new MultiValueDictionary <int, NodeId>(); foreach (var node in nodes) { var height = graph.GetNodeHeight(node); nodesByHeight.Add(height, node); } return(nodesByHeight); }
/// <summary> /// Creates an instence of <see cref="DirtyNodeTracker"/>. /// </summary> /// <param name="graph">the node dataflow graph.</param> /// <param name="dirtyNodes">the set of dirty nodes.</param> /// <param name="perpetualDirtyNodes">the set of nodes that stay dirty even after execution or running from cache.</param> /// <param name="dirtyNodesChanged">flag indicating whether dirty nodes have changed</param> /// <param name="materializedNodes">the set of nodes that have materialized their outputs</param> public DirtyNodeTracker(IReadonlyDirectedGraph graph, RangedNodeSet dirtyNodes, RangedNodeSet perpetualDirtyNodes, bool dirtyNodesChanged, RangedNodeSet materializedNodes) { Contract.Requires(graph != null); Contract.Requires(dirtyNodes != null); Contract.Requires(materializedNodes != null); Contract.Requires(perpetualDirtyNodes != null); Graph = graph; DirtyNodes = dirtyNodes; PerpetualDirtyNodes = perpetualDirtyNodes; DirtyNodesChanged = dirtyNodesChanged; MaterializedNodes = materializedNodes; }
/// <summary> /// Class constructor /// </summary> public CachedGraph(PipGraph pipGraph, IReadonlyDirectedGraph directedGraph, PipExecutionContext context, MountPathExpander mountPathExpander, EngineSerializer serializer = null) { Contract.Requires(pipGraph != null); Contract.Requires(directedGraph != null); Contract.Requires(context != null); Contract.Requires(mountPathExpander != null); DirectedGraph = directedGraph; PipTable = pipGraph.PipTable; MountPathExpander = mountPathExpander; Context = context; PipGraph = pipGraph; Serializer = serializer; }
/// <summary> /// Constructor. /// </summary> /// <param name="oldPipGraph">Old pip graph.</param> /// <param name="oldPipTable">Old pip table.</param> /// <param name="graphBuilder">Pip graph builder to which to delegate all "add pip" operations.</param> /// <param name="maxDegreeOfParallelism">Max concurrency for graph reloading (<see cref="PartiallyReloadGraph"/>).</param> public PatchablePipGraph( IReadonlyDirectedGraph oldPipGraph, PipTable oldPipTable, PipGraph.Builder graphBuilder, int maxDegreeOfParallelism) { m_oldPipGraph = oldPipGraph; m_oldPipTable = oldPipTable; m_builder = graphBuilder; m_maxDegreeOfParallelism = maxDegreeOfParallelism; m_reloadedSealDirectories = new ConcurrentBigMap <long, DirectoryArtifact>(); m_reloadedServicePips = new ConcurrentBigMap <long, PipId>(); m_pipIdMap = new ConcurrentBigMap <PipId, PipId>(); }
/// <summary> /// Constructor for build set calculator. /// </summary> /// <param name="loggingContext">Logging context.</param> /// <param name="graph">Pip graph.</param> /// <param name="dirtyNodeTracker">Dirty node tracker.</param> /// <param name="counters">Counter collection.</param> protected BuildSetCalculator( LoggingContext loggingContext, IReadonlyDirectedGraph graph, DirtyNodeTracker dirtyNodeTracker, CounterCollection <PipExecutorCounter> counters) { Contract.Requires(loggingContext != null); Contract.Requires(graph != null); Contract.Requires(counters != null); m_loggingContext = loggingContext; m_graph = graph; m_visitor = new NodeVisitor(graph); m_dirtyNodeTracker = dirtyNodeTracker; m_counters = counters; }
/// <summary> /// Simple reachability query reference implementation. /// </summary> private static bool IsReachableTrivial(IReadonlyDirectedGraph graph, NodeId from, NodeId to) { if (from == to) { return(true); } foreach (Edge edge in graph.GetOutgoingEdges(from)) { if (IsReachableTrivial(graph, edge.OtherNode, to)) { return(true); } } return(false); }
private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; }
private static void VerifyReachability(IReadonlyDirectedGraph graph) { foreach (NodeId node in graph.Nodes) { foreach (NodeId otherNode in graph.Nodes) { if (otherNode.Value < node.Value) { continue; } XAssert.AreEqual( IsReachableTrivial(graph, node, otherNode), graph.IsReachableFrom(node, otherNode), "Incorrect reachability between {0} and {1}", node, otherNode); } } }
private void VerifyNodeHeights(IReadonlyDirectedGraph graph, NodeId[] nodes) { XAssert.AreEqual(0, graph.GetNodeHeight(nodes[7])); XAssert.AreEqual(0, graph.GetNodeHeight(nodes[10])); XAssert.AreEqual(0, graph.GetNodeHeight(nodes[11])); XAssert.AreEqual(1, graph.GetNodeHeight(nodes[8])); XAssert.AreEqual(1, graph.GetNodeHeight(nodes[9])); XAssert.AreEqual(2, graph.GetNodeHeight(nodes[3])); XAssert.AreEqual(2, graph.GetNodeHeight(nodes[4])); XAssert.AreEqual(2, graph.GetNodeHeight(nodes[5])); XAssert.AreEqual(2, graph.GetNodeHeight(nodes[6])); XAssert.AreEqual(3, graph.GetNodeHeight(nodes[0])); XAssert.AreEqual(3, graph.GetNodeHeight(nodes[1])); XAssert.AreEqual(3, graph.GetNodeHeight(nodes[2])); }
/// <summary> /// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>. /// </summary> /// <returns> /// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to /// a <see cref="Tracing.FingerprintStore"/> at the provided directory; /// otherwise, null. /// </returns> public static FingerprintStoreExecutionLogTarget Create( PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, string fingerprintStoreDirectory, LoggingContext loggingContext, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null, FingerprintStoreTestHooks testHooks = null) { var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0); var possibleStore = FingerprintStore.Open( fingerprintStoreDirectory, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, testHooks: testHooks); if (possibleStore.Succeeded) { return(new FingerprintStoreExecutionLogTarget( loggingContext, context, pipTable, pipContentFingerprinter, possibleStore.Result, configuration, cache, graph, runnablePipPerformance)); } else { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleStore.Failure.DescribeIncludingInnerFailures()); } return(null); }
/// <summary> /// Checks if there exists a path between <paramref name="from"/> and <paramref name="to"/> (following directed edges 'outward'). /// </summary> /// <remarks> /// Since the underlying <see cref="IReadonlyDirectedGraph"/> is not thread-safe, the caller is responsible for synchronizing access to it. /// This algorithm requires a particular graph structure: /// - The <see cref="IReadonlyDirectedGraph"/> must not contain cycles. /// - The <see cref="NodeId"/>s of each node visited must form a topological labelling. /// Precisely, for any edge N -> M (outgoing from N, incoming to M), the node ID M must have a value strictly greater than N. /// (note that traversing a cycle fails the second condition, so no separate cycle validation is needed). /// A <see cref="BuildXLException"/> is thrown if these conditions are found to be violated (in a very limited set of cases, depending on the part of the graph actually visited); /// this check can instead be suppressed if <paramref name="skipOutOfOrderNodes"/> is set, but one must then be very careful to know which nodes may be skipped as a result of possible misordering. /// </remarks> public static bool IsReachableFrom(this IReadonlyDirectedGraph graph, NodeId from, NodeId to, bool skipOutOfOrderNodes = false) { Contract.Requires(graph != null); Contract.Requires(from.IsValid && to.IsValid); Contract.Requires(graph.ContainsNode(from) && graph.ContainsNode(to)); // First, some fast paths that don't need to grab RangedNodeSets. if (from == to) { return(true); } if (from.Value > to.Value) { return(false); } using (PooledObjectWrapper <RangedNodeSet> pooledSetA = RangedNodeSetPool.GetInstance()) using (PooledObjectWrapper <RangedNodeSet> pooledSetB = RangedNodeSetPool.GetInstance()) using (PooledObjectWrapper <RangedNodeSet> pooledSetC = RangedNodeSetPool.GetInstance()) { return(IsReachableFromInternal(graph, from, to, pooledSetA.Instance, pooledSetB.Instance, pooledSetC.Instance, skipOutOfOrderNodes)); } }
/// <summary> /// Creates a new node visitor /// </summary> /// <param name="dataflowGraph">the dataflow graph</param> public NodeVisitor(IReadonlyDirectedGraph dataflowGraph) { m_dataflowGraph = dataflowGraph; }
/// <summary> /// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>. /// </summary> /// <returns> /// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to /// a <see cref="Tracing.FingerprintStore"/> at the provided directory; /// otherwise, null. /// </returns> public static FingerprintStoreExecutionLogTarget Create( PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, LoggingContext loggingContext, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, CounterCollection <FingerprintStoreCounters> counters, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null, FingerprintStoreTestHooks testHooks = null) { var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable); var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable); try { FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString); } catch (BuildXLException ex) { Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message); throw new BuildXLException("Unable to create fingerprint store directory: ", ex); } var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0); var possibleExecutionStore = FingerprintStore.Open( fingerprintStorePathString, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, counters: counters, testHooks: testHooks); Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet."); if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly) { try { FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString); } catch (BuildXLException ex) { Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message); throw new BuildXLException("Unable to create fingerprint store directory: ", ex); } possibleCacheLookupStore = FingerprintStore.Open( cacheLookupFingerprintStorePathString, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, counters: counters, testHooks: testHooks); } if (possibleExecutionStore.Succeeded && (possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly)) { return(new FingerprintStoreExecutionLogTarget( loggingContext, context, pipTable, pipContentFingerprinter, possibleExecutionStore.Result, possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null, configuration, cache, graph, counters, runnablePipPerformance)); } else { if (!possibleExecutionStore.Succeeded) { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures()); } if (!possibleCacheLookupStore.Succeeded) { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures()); } } return(null); }
/// <summary> /// Constructs an Ide Generator from the BuildXL.Execution.Analyzer /// </summary> public IdeGenerator(PipExecutionContext pipContext, PipGraph pipGraph, IReadonlyDirectedGraph scheduledGraph, AbsolutePath configFilePath, IIdeConfiguration ideConfig) { m_context = new Context(pipContext, pipGraph, scheduledGraph, configFilePath, ideConfig); }
/// <summary> /// Constructs an Ide Generator and generates the files /// </summary> public static bool Generate(PipExecutionContext pipContext, PipGraph pipGraph, IReadonlyDirectedGraph scheduledGraph, AbsolutePath configFilePath, IIdeConfiguration ideConfig) { var generator = new IdeGenerator(pipContext, pipGraph, scheduledGraph, configFilePath, ideConfig); return(generator.Generate()); }
/// <summary> /// Creates an instence of <see cref="DirtyNodeTracker"/>. /// </summary> public DirtyNodeTracker(IReadonlyDirectedGraph graph, DirtyNodeTrackerSerializedState dirtyNodeTrackerSerializedState) : this(graph, dirtyNodeTrackerSerializedState.DirtyNodes, dirtyNodeTrackerSerializedState.PerpetualDirtyNodes, false, dirtyNodeTrackerSerializedState.MaterializedNodes) { Contract.Requires(graph != null); Contract.Requires(dirtyNodeTrackerSerializedState != null); }
private void TestGraphSerializationPerformCommonValidations(DirectedGraph graph, NodeId[] nodes, IReadonlyDirectedGraph sourceGraph) { XAssert.AreEqual(graph.NodeCount, nodes.Length); XAssert.AreEqual(graph.EdgeCount, 12); XAssert.IsTrue(graph.IsSinkNode(nodes[0])); XAssert.IsTrue(graph.IsSinkNode(nodes[1])); XAssert.IsTrue(graph.IsSinkNode(nodes[2])); XAssert.IsTrue(graph.IsSinkNode(nodes[3])); XAssert.IsTrue(graph.IsSinkNode(nodes[4])); XAssert.IsTrue(graph.IsSourceNode(nodes[7])); XAssert.IsTrue(graph.IsSourceNode(nodes[10])); XAssert.IsTrue(graph.IsSourceNode(nodes[11])); XAssert.IsFalse(graph.IsSourceNode(nodes[6])); XAssert.IsFalse(graph.IsSinkNode(nodes[6])); foreach (var node in sourceGraph.Nodes) { XAssert.IsTrue(EdgeSetEqual(new HashSet <Edge>(sourceGraph.GetOutgoingEdges(node)), new HashSet <Edge>(graph.GetOutgoingEdges(node)))); XAssert.IsTrue(EdgeSetEqual(new HashSet <Edge>(sourceGraph.GetIncomingEdges(node)), new HashSet <Edge>(graph.GetIncomingEdges(node)))); } VerifyNodeHeights(graph, nodes); var succOfNode9 = new HashSet <Edge>(graph.GetOutgoingEdges(nodes[9])); XAssert.IsTrue( EdgeSetEqual( succOfNode9, new HashSet <Edge> { new Edge(nodes[6]), new Edge(nodes[3]), new Edge(nodes[4]) })); var predOfNode8 = new HashSet <Edge>(graph.GetIncomingEdges(nodes[8])); XAssert.IsTrue( EdgeSetEqual( predOfNode8, new HashSet <Edge> { new Edge(nodes[10]), new Edge(nodes[11]) })); }
/// <summary> /// Initiates the task to load the fingerprint store that will be used for cache miss analysis /// </summary> public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration)) { var option = configuration.Logging.CacheMissAnalysisOption; if (option.Mode == CacheMissMode.Disabled) { return(null); } Possible <FingerprintStore> possibleStore; if (option.Mode == CacheMissMode.Local) { possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext); } else { string path = null; if (option.Mode == CacheMissMode.CustomPath) { path = option.CustomPath.ToString(context.PathTable); } else { Contract.Assert(option.Mode == CacheMissMode.Remote); foreach (var key in option.Keys) { var cacheSavePath = configuration.Logging.FingerprintsLogDirectory .Combine(context.PathTable, Scheduler.FingerprintStoreDirectory + "." + key); #pragma warning disable AsyncFixer02 // This should explicitly happen synchronously since it interacts with the PathTable and StringTable var result = cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, context.PathTable, key, configuration.Schedule.EnvironmentFingerprint).Result; #pragma warning restore AsyncFixer02 if (result.Succeeded && result.Result) { path = cacheSavePath.ToString(context.PathTable); break; } } if (string.IsNullOrEmpty(path)) { Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}")); return(null); } } // Unblock caller // WARNING: The rest can simultenously happen with saving the graph files to disk. // We should not create any paths or strings by using PathTable and StringTable. await Task.Yield(); possibleStore = FingerprintStore.Open(path, readOnly: true); } if (possibleStore.Succeeded) { Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory); return(new RuntimeCacheMissAnalyzer(logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance)); } Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}")); return(null); } }
private static bool RangeIncidentNodesAndIntersect( IReadonlyDirectedGraph graph, RangedNodeSet walkFromNodes, Func <IReadonlyDirectedGraph, NodeId, IEnumerable <Edge> > getEdges, Func <NodeId, NodeId, bool> validateEdgeTopoProperty, NodeRange incidentNodeFilter, RangedNodeSet intersectWith, bool skipOutOfOrderNodes, out NodeRange range, out NodeId intersection) { // Note that initially, currentMin > currentMax so NodeRange.CreatePossiblyEmpty // would return an empty range. We return an empty range iff no nodes pass incidentNodeFilter below. uint currentMin = NodeId.MaxValue; uint currentMax = NodeId.MinValue; foreach (NodeId existingNode in walkFromNodes) { IEnumerable <Edge> edges = getEdges(graph, existingNode); foreach (Edge edge in edges) { NodeId other = edge.OtherNode; if (!validateEdgeTopoProperty(existingNode, other)) { if (skipOutOfOrderNodes) { continue; } throw new BuildXLException(I($"Topological order violated due to an edge between nodes {existingNode} and {other}")); } if (!incidentNodeFilter.Contains(other)) { continue; } if (other.Value > currentMax) { currentMax = edge.OtherNode.Value; Contract.AssertDebug(currentMax <= NodeId.MaxValue && currentMax >= NodeId.MinValue); } if (other.Value < currentMin) { currentMin = edge.OtherNode.Value; Contract.AssertDebug(currentMin <= NodeId.MaxValue && currentMin >= NodeId.MinValue); } if (intersectWith.Contains(other)) { intersection = other; Contract.AssertDebug(currentMin <= NodeId.MaxValue && currentMin >= NodeId.MinValue); Contract.AssertDebug(currentMax <= NodeId.MaxValue && currentMax >= NodeId.MinValue); range = NodeRange.CreatePossiblyEmpty(new NodeId(currentMin), new NodeId(currentMax)); return(true); } } } intersection = NodeId.Invalid; Contract.AssertDebug(currentMin <= NodeId.MaxValue && currentMin >= NodeId.MinValue); Contract.AssertDebug(currentMax <= NodeId.MaxValue && currentMax >= NodeId.MinValue); range = NodeRange.CreatePossiblyEmpty(new NodeId(currentMin), new NodeId(currentMax)); return(false); }
public void RandomGraph30() { IReadonlyDirectedGraph graph = CreateRandomAcyclicGraph(new Random(42), nodeCount: 30); VerifyReachability(graph); }
/// <summary> /// Creates a VisitationTracker for a specific graph. Only valid while that graph remains unchanged. /// </summary> public VisitationTracker(IReadonlyDirectedGraph graph) { Contract.Requires(graph != null); m_visited = new ConcurrentBitArray(graph.NodeCount); }
/// <summary> /// Initiates the task to load the fingerprint store that will be used for cache miss analysis /// </summary> public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, FingerprintStoreTestHooks testHooks = null) { // Unblock caller await Task.Yield(); using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration)) { var option = configuration.Logging.CacheMissAnalysisOption; string downLoadedPriviousFingerprintStoreSavedPath = null; if (option.Mode == CacheMissMode.Disabled) { return(null); } Possible <FingerprintStore> possibleStore; PathTable newPathTable = new PathTable(); if (option.Mode == CacheMissMode.Local) { possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext); } else { string path = null; if (option.Mode == CacheMissMode.CustomPath) { path = option.CustomPath.ToString(context.PathTable); } else { Contract.Assert(option.Mode == CacheMissMode.Remote); foreach (var key in option.Keys) { var fingerprintsLogDirectoryStr = configuration.Logging.FingerprintsLogDirectory.ToString(context.PathTable); var fingerprintsLogDirectory = AbsolutePath.Create(newPathTable, fingerprintsLogDirectoryStr); var cacheSavePath = fingerprintsLogDirectory.Combine(newPathTable, Scheduler.FingerprintStoreDirectory + "." + key); var result = await cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, newPathTable, key, configuration, context.CancellationToken); if (result.Succeeded && result.Result) { path = cacheSavePath.ToString(newPathTable); downLoadedPriviousFingerprintStoreSavedPath = path; break; } } if (string.IsNullOrEmpty(path)) { Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}")); return(null); } } possibleStore = FingerprintStore.Open(path, readOnly: true); } if (possibleStore.Succeeded) { Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory); return(new RuntimeCacheMissAnalyzer( logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance, configuration, downLoadedPriviousFingerprintStoreSavedPath, testHooks: testHooks)); } Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}")); return(null); } }
private static bool IsReachableFromInternal( IReadonlyDirectedGraph graph, NodeId from, NodeId to, RangedNodeSet pooledSetA, RangedNodeSet pooledSetB, RangedNodeSet pooledSetC, bool skipOutOfOrderNodes) { // This implementation attempts to efficiently traverse a graph without any precomputed indices or labeling beyond topologically ordered node values. // Index-based approaches are tricky for the expected usage (the BuildXL scheduler) in which the underlying graph is dynamic. // Instead, we traverse the graph with no prior information in hand, with a careful traversal order and some pruning. // The thinking on pruning / use of topological labels is not new; for some more robust examples see e.g. FELINE: // Veloso, Renê Rodrigues, et al. "Reachability Queries in Very Large Graphs: A Fast Refined Online Search Approach." EDBT. 2014. // Figure 6 in particular gives some geometric insight, though the pruning here is less effective. // Now, let's build some intuition about this implementation. We begin from a naive approach and will refine to what's actually implemented. // First, consider the problem of traversing an _undirected_ graph to determine reachability from some point M to N. // o\ /o o\ /o // M -- o -- o -- N // o/ \o o/ \o // We can imagine the graph in some two-dimensional layout. To perform well with M and N fairly close, it would be wise to proceed in a breadth-first fashion: // ●\ /● o\ /o // M -- ● -- o -- N // ●/ \● o/ \o // On the first iteration, we have all nodes reachable in one hop from M. On the i'th iteration, we have all nodes reachable in 'i' hops. In the example above, // N would be found on iteration 3. Geometrically, think of the reached set as a circle expanding outward from M. Note that on each iteration i, we only need to // hold *the nodes i hopes away* (not i - 1 hops etc.) so in fact we are tracking the outer circumference of this circle. This works since there exists some single // integer i by which N is at least i hops away (if reachable). // We can leverage the geometric intuition of a circle to improve that approach a bit. Assume that on some iteration i, we have visited all nodes interior to the circle, // and the discrete nodes are so numerous and dense as to approximate a circle's area. We can then think of i as a radius and the area as a count of nodes visited - // on the order of i^2. If we instead traverse the same radius via two circles - each of radius (i/2) then the visited area is (i/2)^2 * 2 = i^2 / 2. Intersection of // the circles implies a path between the two nodes. // ●\ /● ●\ /● // M -- ● -- ● -- N // ●/ \● ●/ \● // (above: the prior example on iteration 1 when expanding outward from both endpoints; a path will be found on iteration 2 instead of 3). // We can still track only the outer circumference of the circles, so long as we alternately expand each circle (rather than expanding both instantaneously; each expansion // increases the effective search radius by one and so there's no way for the circles to skip past each other). // Now we first leverage the assumption of a *directed* graph. Simply, we can follow edges in opposing directions from each node (now labelling specifically as 'to' {T} and from'{F}), // which geometrically looks like expanding semi-circles rather than circles (the nodes labeled X were skipped based on direction): // X\ />● ●>\ />X // >F --> ● --> ● --> T // X/ \>● ●>/ \>X // Finally we consider the usefulness of a topological labeling of the nodes. This is in fact a generalization of following edges in only one direction: // - Outgoing-incident nodes must have higher labels than the one current. // - Incoming-incident nodes must by symmetry have lower labels than the one currnet. // This means that traversing outgoing edges result in a node-set (for the semi-circle's edge) that increases monotonically over iterations. Symmetrically // the set for incoming edge traversal decreases monotonically. The example below adds bracketed topological labels; note that for outgoing edges we initially // have [4, 4] and then [5, 7], and for incoming we first have [11, 11] and then [8, 10]. With this in mind we can see it is futile to traverse from node 10 to node 1, // when expanding the incoming range since that node is on the 'wrong side' of the outgoing range already. // X[3]\ />●[7] ●[9]>\ />X[12] // >F[4] --> ●[6] --> ●[8] --> T[11] // /->X[2]/ \>●[5] ->●[10]>/ \>X[13] // X[1] -----------------------/ RangedNodeSet incomingRangeSet = pooledSetA; incomingRangeSet.SetSingular(to); var outgoingRangeSet = pooledSetB; outgoingRangeSet.SetSingular(from); var swap = pooledSetC; swap.Clear(); bool toggle = false; // Loop condition is effectively !incomingRangeSet.IsEmpty && !outgoingRangeSet.IsEmpty, but checked as one of the ranges changes. while (true) { Func <IReadonlyDirectedGraph, NodeId, IEnumerable <Edge> > getEdges; Func <NodeId, NodeId, bool> validateEdgeTopoProperty; RangedNodeSet walkFromNodes; RangedNodeSet intersectWith; NodeRange incidentNodeFilter; if (toggle) { // Decreasing from 'to' to NodeId.Min getEdges = (g, n) => g.GetIncomingEdges(n); validateEdgeTopoProperty = (node, other) => node.Value > other.Value; incidentNodeFilter = NodeRange.CreateLowerBound(outgoingRangeSet.Range.FromInclusive); walkFromNodes = incomingRangeSet; intersectWith = outgoingRangeSet; } else { // Increasing from 'from' to NodeId.Max getEdges = (g, n) => g.GetOutgoingEdges(n); validateEdgeTopoProperty = (node, other) => node.Value < other.Value; incidentNodeFilter = NodeRange.CreateUpperBound(incomingRangeSet.Range.ToInclusive); walkFromNodes = outgoingRangeSet; intersectWith = incomingRangeSet; } NodeId intersection; NodeRange range; if (RangeIncidentNodesAndIntersect( graph, walkFromNodes, getEdges, validateEdgeTopoProperty, incidentNodeFilter, intersectWith, skipOutOfOrderNodes, range: out range, intersection: out intersection)) { return(true); } if (range.IsEmpty) { break; } swap.ClearAndSetRange(range); AddIncidentNodes(graph, walkFromNodes, getEdges, validateEdgeTopoProperty, incidentNodeFilter, skipOutOfOrderNodes, swap); if (toggle) { RangedNodeSet temp = incomingRangeSet; incomingRangeSet = swap; swap = temp; } else { RangedNodeSet temp = outgoingRangeSet; outgoingRangeSet = swap; swap = temp; } toggle = !toggle; } return(false); }