private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
CacheMissDiffFormat cacheMissDiffFormat,
bool cacheMissBatch,
FingerprintStoreTestHooks testHooks = null)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_cacheMissDiffFormat = cacheMissDiffFormat;
m_maxCacheMissCanPerform = cacheMissBatch ? EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value * EngineEnvironmentSettings.MaxMessagesPerBatch : EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value;
m_batchLoggingQueue = cacheMissBatch ? NagleQueue <JProperty> .Create(
BatchLogging,
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : EngineEnvironmentSettings.MaxMessagesPerBatch) : null;
m_testHooks = testHooks;
m_testHooks?.InitRuntimeCacheMisses();
}
private static void AddIncidentNodes(
IReadonlyDirectedGraph graph,
RangedNodeSet walkFromNodes,
Func <IReadonlyDirectedGraph, NodeId, IEnumerable <Edge> > getEdges,
Func <NodeId, NodeId, bool> validateEdgeTopoProperty,
NodeRange incidentNodeFilter,
bool skipOutOfOrderNodes,
RangedNodeSet addTo)
{
Contract.Requires(!incidentNodeFilter.IsEmpty);
foreach (NodeId existingNode in walkFromNodes)
{
IEnumerable <Edge> edges = getEdges(graph, existingNode);
foreach (Edge edge in edges)
{
NodeId other = edge.OtherNode;
if (skipOutOfOrderNodes && !validateEdgeTopoProperty(existingNode, other))
{
continue;
}
if (incidentNodeFilter.Contains(other))
{
addTo.Add(edge.OtherNode);
}
}
}
}
/// <summary>
/// Constructor.
/// </summary>
private FingerprintStoreExecutionLogTarget(
LoggingContext loggingContext,
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
FingerprintStore fingerprintStore,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_context = context;
m_pipTable = pipTable;
PipContentFingerprinter = pipContentFingerprinter;
FingerprintStore = fingerprintStore;
m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>();
m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync(
this,
loggingContext,
context,
configuration,
cache,
graph,
runnablePipPerformance);
}
private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
IConfiguration configuration,
string downLoadedPreviousFingerprintStoreSavedPath,
FingerprintStoreTestHooks testHooks = null)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_batchLoggingQueue = configuration.Logging.CacheMissBatch ? NagleQueue <JProperty> .Create(
BatchLogging,
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(5),
batchSize : 100) : null;
m_testHooks = testHooks;
m_testHooks?.InitRuntimeCacheMisses();
m_configuration = configuration;
m_downLoadedPreviousFingerprintStoreSavedPath = downLoadedPreviousFingerprintStoreSavedPath;
}
/// <summary>
/// Constructor.
/// </summary>
private FingerprintStoreExecutionLogTarget(
LoggingContext loggingContext,
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
FingerprintStore fingerprintStore,
FingerprintStore cacheLookupFingerprintStore,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_context = context;
m_pipTable = pipTable;
LoggingContext = loggingContext;
PipContentFingerprinter = pipContentFingerprinter;
ExecutionFingerprintStore = fingerprintStore;
CacheLookupFingerprintStore = cacheLookupFingerprintStore;
// Cache lookup store is per-build state and doesn't need to be garbage collected (vs. execution fignerprint store which is persisted build-over-build)
CacheLookupFingerprintStore?.GarbageCollectCancellationToken.Cancel();
m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>();
Counters = counters;
FingerprintStoreMode = configuration.Logging.FingerprintStoreMode;
m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync(
this,
loggingContext,
context,
configuration,
cache,
graph,
runnablePipPerformance);
Contract.Assume(FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly || CacheLookupFingerprintStore != null, "Unless /storeFingerprints flag is set to /storeFingerprints:ExecutionFingerprintsOnly, the cache lookup FingerprintStore must exist.");
}
public FilteredDirectedGraph(IReadonlyDirectedGraph graph, VisitationTracker nodeFilter)
{
m_graph = graph;
m_nodeFilter = nodeFilter;
m_nodePredicate = node => nodeFilter.WasVisited(node);
m_edgePredicate = edge => nodeFilter.WasVisited(edge.OtherNode);
m_nodeHeights = Lazy.Create(ComputeHeights);
}
public Context(
PipExecutionContext pipContext,
PipGraph pipGraph,
IReadonlyDirectedGraph scheduledGraph,
AbsolutePath configFilePath,
IIdeConfiguration ideConfig)
{
Contract.Requires(pipGraph != null);
Contract.Requires(scheduledGraph != null);
Contract.Requires(configFilePath.IsValid);
Contract.Requires(ideConfig.SolutionRoot.IsValid);
Contract.Requires(ideConfig.SolutionName.IsValid);
Contract.Requires(ideConfig.IsEnabled);
Contract.Requires(ideConfig.IsNewEnabled);
PipGraph = pipGraph;
ScheduledGraph = scheduledGraph;
StringTable = pipContext.StringTable;
PathTable = pipContext.PathTable;
SymbolTable = pipContext.SymbolTable;
QualifierTable = pipContext.QualifierTable;
IdeConfig = ideConfig;
DotSettingsPathStr = ideConfig.DotSettingsFile.IsValid ? ideConfig.DotSettingsFile.ToString(PathTable) : null;
ConfigFilePathStr = configFilePath.ToString(PathTable);
EnlistmentRoot = configFilePath.GetParent(PathTable);
EnlistmentRootStr = EnlistmentRoot.ToString(PathTable);
SolutionRoot = ideConfig.SolutionRoot;
SolutionRootStr = SolutionRoot.ToString(PathTable);
SolutionFilePathStr = IdeGenerator.GetSolutionPath(ideConfig, PathTable).ToString(PathTable);
CanWriteToSrc = ideConfig.CanWriteToSrc ?? false;
ProjectsRoot = CanWriteToSrc
? EnlistmentRoot
: SolutionRoot.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "Projects"));
ResxExtensionName = PathAtom.Create(StringTable, ".resx");
CscExeName = PathAtom.Create(StringTable, "csc.exe");
CscDllName = PathAtom.Create(StringTable, "csc.dll");
XunitConsoleDllName = PathAtom.Create(StringTable, "xunit.console.dll");
XunitConsoleExeName = PathAtom.Create(StringTable, "xunit.console.exe");
QtestExeName = PathAtom.Create(StringTable, "DBS.QTest.exe");
DotnetName = PathAtom.Create(StringTable, "dotnet");
DotnetExeName = PathAtom.Create(StringTable, "dotnet.exe");
ResgenExeName = PathAtom.Create(StringTable, "ResGen.exe");
ResourcesExtensionName = PathAtom.Create(StringTable, ".resources");
CsExtensionName = PathAtom.Create(StringTable, ".cs");
DllExtensionName = PathAtom.Create(StringTable, ".dll");
ClExeName = PathAtom.Create(StringTable, "cl.exe");
LinkExeName = PathAtom.Create(StringTable, "Link.exe");
VsTestExeName = PathAtom.Create(StringTable, "vstest.console.exe");
AssemblyDeploymentTag = StringId.Create(StringTable, "assemblyDeployment");
TestDeploymentTag = StringId.Create(StringTable, "testDeployment");
}
/// <summary>
/// Topologically sorts nodes in a given graph. If no <paramref name="nodes"/> are specified all nodes in the graph
/// are used (<see cref="IReadonlyDirectedGraph.Nodes"/>); if <paramref name="nodes"/> are specified, they must all
/// belong to the given <paramref name="graph"/>.
///
/// Returns a multi-value dictionary with keys ranging from 0 to max node height.
/// </summary>
public static MultiValueDictionary <int, NodeId> TopSort(this IReadonlyDirectedGraph graph, IEnumerable <NodeId> nodes = null)
{
nodes = nodes ?? graph.Nodes;
MultiValueDictionary <int, NodeId> nodesByHeight = new MultiValueDictionary <int, NodeId>();
foreach (var node in nodes)
{
var height = graph.GetNodeHeight(node);
nodesByHeight.Add(height, node);
}
return(nodesByHeight);
}
/// <summary>
/// Creates an instence of <see cref="DirtyNodeTracker"/>.
/// </summary>
/// <param name="graph">the node dataflow graph.</param>
/// <param name="dirtyNodes">the set of dirty nodes.</param>
/// <param name="perpetualDirtyNodes">the set of nodes that stay dirty even after execution or running from cache.</param>
/// <param name="dirtyNodesChanged">flag indicating whether dirty nodes have changed</param>
/// <param name="materializedNodes">the set of nodes that have materialized their outputs</param>
public DirtyNodeTracker(IReadonlyDirectedGraph graph, RangedNodeSet dirtyNodes, RangedNodeSet perpetualDirtyNodes, bool dirtyNodesChanged, RangedNodeSet materializedNodes)
{
Contract.Requires(graph != null);
Contract.Requires(dirtyNodes != null);
Contract.Requires(materializedNodes != null);
Contract.Requires(perpetualDirtyNodes != null);
Graph = graph;
DirtyNodes = dirtyNodes;
PerpetualDirtyNodes = perpetualDirtyNodes;
DirtyNodesChanged = dirtyNodesChanged;
MaterializedNodes = materializedNodes;
}
/// <summary>
/// Class constructor
/// </summary>
public CachedGraph(PipGraph pipGraph, IReadonlyDirectedGraph directedGraph, PipExecutionContext context, MountPathExpander mountPathExpander, EngineSerializer serializer = null)
{
Contract.Requires(pipGraph != null);
Contract.Requires(directedGraph != null);
Contract.Requires(context != null);
Contract.Requires(mountPathExpander != null);
DirectedGraph = directedGraph;
PipTable = pipGraph.PipTable;
MountPathExpander = mountPathExpander;
Context = context;
PipGraph = pipGraph;
Serializer = serializer;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="oldPipGraph">Old pip graph.</param>
/// <param name="oldPipTable">Old pip table.</param>
/// <param name="graphBuilder">Pip graph builder to which to delegate all "add pip" operations.</param>
/// <param name="maxDegreeOfParallelism">Max concurrency for graph reloading (<see cref="PartiallyReloadGraph"/>).</param>
public PatchablePipGraph(
IReadonlyDirectedGraph oldPipGraph,
PipTable oldPipTable,
PipGraph.Builder graphBuilder,
int maxDegreeOfParallelism)
{
m_oldPipGraph = oldPipGraph;
m_oldPipTable = oldPipTable;
m_builder = graphBuilder;
m_maxDegreeOfParallelism = maxDegreeOfParallelism;
m_reloadedSealDirectories = new ConcurrentBigMap <long, DirectoryArtifact>();
m_reloadedServicePips = new ConcurrentBigMap <long, PipId>();
m_pipIdMap = new ConcurrentBigMap <PipId, PipId>();
}
/// <summary>
/// Constructor for build set calculator.
/// </summary>
/// <param name="loggingContext">Logging context.</param>
/// <param name="graph">Pip graph.</param>
/// <param name="dirtyNodeTracker">Dirty node tracker.</param>
/// <param name="counters">Counter collection.</param>
protected BuildSetCalculator(
LoggingContext loggingContext,
IReadonlyDirectedGraph graph,
DirtyNodeTracker dirtyNodeTracker,
CounterCollection <PipExecutorCounter> counters)
{
Contract.Requires(loggingContext != null);
Contract.Requires(graph != null);
Contract.Requires(counters != null);
m_loggingContext = loggingContext;
m_graph = graph;
m_visitor = new NodeVisitor(graph);
m_dirtyNodeTracker = dirtyNodeTracker;
m_counters = counters;
}
/// <summary>
/// Simple reachability query reference implementation.
/// </summary>
private static bool IsReachableTrivial(IReadonlyDirectedGraph graph, NodeId from, NodeId to)
{
if (from == to)
{
return(true);
}
foreach (Edge edge in graph.GetOutgoingEdges(from))
{
if (IsReachableTrivial(graph, edge.OtherNode, to))
{
return(true);
}
}
return(false);
}
private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
}
private static void VerifyReachability(IReadonlyDirectedGraph graph)
{
foreach (NodeId node in graph.Nodes)
{
foreach (NodeId otherNode in graph.Nodes)
{
if (otherNode.Value < node.Value)
{
continue;
}
XAssert.AreEqual(
IsReachableTrivial(graph, node, otherNode),
graph.IsReachableFrom(node, otherNode),
"Incorrect reachability between {0} and {1}", node, otherNode);
}
}
}
private void VerifyNodeHeights(IReadonlyDirectedGraph graph, NodeId[] nodes)
{
XAssert.AreEqual(0, graph.GetNodeHeight(nodes[7]));
XAssert.AreEqual(0, graph.GetNodeHeight(nodes[10]));
XAssert.AreEqual(0, graph.GetNodeHeight(nodes[11]));
XAssert.AreEqual(1, graph.GetNodeHeight(nodes[8]));
XAssert.AreEqual(1, graph.GetNodeHeight(nodes[9]));
XAssert.AreEqual(2, graph.GetNodeHeight(nodes[3]));
XAssert.AreEqual(2, graph.GetNodeHeight(nodes[4]));
XAssert.AreEqual(2, graph.GetNodeHeight(nodes[5]));
XAssert.AreEqual(2, graph.GetNodeHeight(nodes[6]));
XAssert.AreEqual(3, graph.GetNodeHeight(nodes[0]));
XAssert.AreEqual(3, graph.GetNodeHeight(nodes[1]));
XAssert.AreEqual(3, graph.GetNodeHeight(nodes[2]));
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
string fingerprintStoreDirectory,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleStore = FingerprintStore.Open(
fingerprintStoreDirectory,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
testHooks: testHooks);
if (possibleStore.Succeeded)
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleStore.Result,
configuration,
cache,
graph,
runnablePipPerformance));
}
else
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleStore.Failure.DescribeIncludingInnerFailures());
}
return(null);
}
/// <summary>
/// Checks if there exists a path between <paramref name="from"/> and <paramref name="to"/> (following directed edges 'outward').
/// </summary>
/// <remarks>
/// Since the underlying <see cref="IReadonlyDirectedGraph"/> is not thread-safe, the caller is responsible for synchronizing access to it.
/// This algorithm requires a particular graph structure:
/// - The <see cref="IReadonlyDirectedGraph"/> must not contain cycles.
/// - The <see cref="NodeId"/>s of each node visited must form a topological labelling.
/// Precisely, for any edge N -> M (outgoing from N, incoming to M), the node ID M must have a value strictly greater than N.
/// (note that traversing a cycle fails the second condition, so no separate cycle validation is needed).
/// A <see cref="BuildXLException"/> is thrown if these conditions are found to be violated (in a very limited set of cases, depending on the part of the graph actually visited);
/// this check can instead be suppressed if <paramref name="skipOutOfOrderNodes"/> is set, but one must then be very careful to know which nodes may be skipped as a result of possible misordering.
/// </remarks>
public static bool IsReachableFrom(this IReadonlyDirectedGraph graph, NodeId from, NodeId to, bool skipOutOfOrderNodes = false)
{
Contract.Requires(graph != null);
Contract.Requires(from.IsValid && to.IsValid);
Contract.Requires(graph.ContainsNode(from) && graph.ContainsNode(to));
// First, some fast paths that don't need to grab RangedNodeSets.
if (from == to)
{
return(true);
}
if (from.Value > to.Value)
{
return(false);
}
using (PooledObjectWrapper <RangedNodeSet> pooledSetA = RangedNodeSetPool.GetInstance())
using (PooledObjectWrapper <RangedNodeSet> pooledSetB = RangedNodeSetPool.GetInstance())
using (PooledObjectWrapper <RangedNodeSet> pooledSetC = RangedNodeSetPool.GetInstance())
{
return(IsReachableFromInternal(graph, from, to, pooledSetA.Instance, pooledSetB.Instance, pooledSetC.Instance, skipOutOfOrderNodes));
}
}
/// <summary>
/// Creates a new node visitor
/// </summary>
/// <param name="dataflowGraph">the dataflow graph</param>
public NodeVisitor(IReadonlyDirectedGraph dataflowGraph)
{
m_dataflowGraph = dataflowGraph;
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable);
var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable);
try
{
FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleExecutionStore = FingerprintStore.Open(
fingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet.");
if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly)
{
try
{
FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
possibleCacheLookupStore = FingerprintStore.Open(
cacheLookupFingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
}
if (possibleExecutionStore.Succeeded &&
(possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly))
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleExecutionStore.Result,
possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null,
configuration,
cache,
graph,
counters,
runnablePipPerformance));
}
else
{
if (!possibleExecutionStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures());
}
if (!possibleCacheLookupStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures());
}
}
return(null);
}
/// <summary>
/// Constructs an Ide Generator from the BuildXL.Execution.Analyzer
/// </summary>
public IdeGenerator(PipExecutionContext pipContext, PipGraph pipGraph, IReadonlyDirectedGraph scheduledGraph, AbsolutePath configFilePath, IIdeConfiguration ideConfig)
{
m_context = new Context(pipContext, pipGraph, scheduledGraph, configFilePath, ideConfig);
}
/// <summary>
/// Constructs an Ide Generator and generates the files
/// </summary>
public static bool Generate(PipExecutionContext pipContext, PipGraph pipGraph, IReadonlyDirectedGraph scheduledGraph, AbsolutePath configFilePath, IIdeConfiguration ideConfig)
{
var generator = new IdeGenerator(pipContext, pipGraph, scheduledGraph, configFilePath, ideConfig);
return(generator.Generate());
}
/// <summary>
/// Creates an instence of <see cref="DirtyNodeTracker"/>.
/// </summary>
public DirtyNodeTracker(IReadonlyDirectedGraph graph, DirtyNodeTrackerSerializedState dirtyNodeTrackerSerializedState)
: this(graph, dirtyNodeTrackerSerializedState.DirtyNodes, dirtyNodeTrackerSerializedState.PerpetualDirtyNodes, false, dirtyNodeTrackerSerializedState.MaterializedNodes)
{
Contract.Requires(graph != null);
Contract.Requires(dirtyNodeTrackerSerializedState != null);
}
private void TestGraphSerializationPerformCommonValidations(DirectedGraph graph, NodeId[] nodes, IReadonlyDirectedGraph sourceGraph)
{
XAssert.AreEqual(graph.NodeCount, nodes.Length);
XAssert.AreEqual(graph.EdgeCount, 12);
XAssert.IsTrue(graph.IsSinkNode(nodes[0]));
XAssert.IsTrue(graph.IsSinkNode(nodes[1]));
XAssert.IsTrue(graph.IsSinkNode(nodes[2]));
XAssert.IsTrue(graph.IsSinkNode(nodes[3]));
XAssert.IsTrue(graph.IsSinkNode(nodes[4]));
XAssert.IsTrue(graph.IsSourceNode(nodes[7]));
XAssert.IsTrue(graph.IsSourceNode(nodes[10]));
XAssert.IsTrue(graph.IsSourceNode(nodes[11]));
XAssert.IsFalse(graph.IsSourceNode(nodes[6]));
XAssert.IsFalse(graph.IsSinkNode(nodes[6]));
foreach (var node in sourceGraph.Nodes)
{
XAssert.IsTrue(EdgeSetEqual(new HashSet <Edge>(sourceGraph.GetOutgoingEdges(node)), new HashSet <Edge>(graph.GetOutgoingEdges(node))));
XAssert.IsTrue(EdgeSetEqual(new HashSet <Edge>(sourceGraph.GetIncomingEdges(node)), new HashSet <Edge>(graph.GetIncomingEdges(node))));
}
VerifyNodeHeights(graph, nodes);
var succOfNode9 = new HashSet <Edge>(graph.GetOutgoingEdges(nodes[9]));
XAssert.IsTrue(
EdgeSetEqual(
succOfNode9,
new HashSet <Edge>
{
new Edge(nodes[6]),
new Edge(nodes[3]),
new Edge(nodes[4])
}));
var predOfNode8 = new HashSet <Edge>(graph.GetIncomingEdges(nodes[8]));
XAssert.IsTrue(
EdgeSetEqual(
predOfNode8,
new HashSet <Edge>
{
new Edge(nodes[10]),
new Edge(nodes[11])
}));
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var cacheSavePath = configuration.Logging.FingerprintsLogDirectory
.Combine(context.PathTable, Scheduler.FingerprintStoreDirectory + "." + key);
#pragma warning disable AsyncFixer02 // This should explicitly happen synchronously since it interacts with the PathTable and StringTable
var result = cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, context.PathTable, key, configuration.Schedule.EnvironmentFingerprint).Result;
#pragma warning restore AsyncFixer02
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(context.PathTable);
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
// Unblock caller
// WARNING: The rest can simultenously happen with saving the graph files to disk.
// We should not create any paths or strings by using PathTable and StringTable.
await Task.Yield();
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}
private static bool RangeIncidentNodesAndIntersect(
IReadonlyDirectedGraph graph,
RangedNodeSet walkFromNodes,
Func <IReadonlyDirectedGraph, NodeId, IEnumerable <Edge> > getEdges,
Func <NodeId, NodeId, bool> validateEdgeTopoProperty,
NodeRange incidentNodeFilter,
RangedNodeSet intersectWith,
bool skipOutOfOrderNodes,
out NodeRange range,
out NodeId intersection)
{
// Note that initially, currentMin > currentMax so NodeRange.CreatePossiblyEmpty
// would return an empty range. We return an empty range iff no nodes pass incidentNodeFilter below.
uint currentMin = NodeId.MaxValue;
uint currentMax = NodeId.MinValue;
foreach (NodeId existingNode in walkFromNodes)
{
IEnumerable <Edge> edges = getEdges(graph, existingNode);
foreach (Edge edge in edges)
{
NodeId other = edge.OtherNode;
if (!validateEdgeTopoProperty(existingNode, other))
{
if (skipOutOfOrderNodes)
{
continue;
}
throw new BuildXLException(I($"Topological order violated due to an edge between nodes {existingNode} and {other}"));
}
if (!incidentNodeFilter.Contains(other))
{
continue;
}
if (other.Value > currentMax)
{
currentMax = edge.OtherNode.Value;
Contract.AssertDebug(currentMax <= NodeId.MaxValue && currentMax >= NodeId.MinValue);
}
if (other.Value < currentMin)
{
currentMin = edge.OtherNode.Value;
Contract.AssertDebug(currentMin <= NodeId.MaxValue && currentMin >= NodeId.MinValue);
}
if (intersectWith.Contains(other))
{
intersection = other;
Contract.AssertDebug(currentMin <= NodeId.MaxValue && currentMin >= NodeId.MinValue);
Contract.AssertDebug(currentMax <= NodeId.MaxValue && currentMax >= NodeId.MinValue);
range = NodeRange.CreatePossiblyEmpty(new NodeId(currentMin), new NodeId(currentMax));
return(true);
}
}
}
intersection = NodeId.Invalid;
Contract.AssertDebug(currentMin <= NodeId.MaxValue && currentMin >= NodeId.MinValue);
Contract.AssertDebug(currentMax <= NodeId.MaxValue && currentMax >= NodeId.MinValue);
range = NodeRange.CreatePossiblyEmpty(new NodeId(currentMin), new NodeId(currentMax));
return(false);
}
public void RandomGraph30()
{
IReadonlyDirectedGraph graph = CreateRandomAcyclicGraph(new Random(42), nodeCount: 30);
VerifyReachability(graph);
}
/// <summary>
/// Creates a VisitationTracker for a specific graph. Only valid while that graph remains unchanged.
/// </summary>
public VisitationTracker(IReadonlyDirectedGraph graph)
{
Contract.Requires(graph != null);
m_visited = new ConcurrentBitArray(graph.NodeCount);
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
FingerprintStoreTestHooks testHooks = null)
{
// Unblock caller
await Task.Yield();
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
string downLoadedPriviousFingerprintStoreSavedPath = null;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
PathTable newPathTable = new PathTable();
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var fingerprintsLogDirectoryStr = configuration.Logging.FingerprintsLogDirectory.ToString(context.PathTable);
var fingerprintsLogDirectory = AbsolutePath.Create(newPathTable, fingerprintsLogDirectoryStr);
var cacheSavePath = fingerprintsLogDirectory.Combine(newPathTable, Scheduler.FingerprintStoreDirectory + "." + key);
var result = await cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, newPathTable, key, configuration, context.CancellationToken);
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(newPathTable);
downLoadedPriviousFingerprintStoreSavedPath = path;
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(
logTarget,
loggingContext,
context,
possibleStore.Result,
graph,
runnablePipPerformance,
configuration,
downLoadedPriviousFingerprintStoreSavedPath,
testHooks: testHooks));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}
private static bool IsReachableFromInternal(
IReadonlyDirectedGraph graph,
NodeId from,
NodeId to,
RangedNodeSet pooledSetA,
RangedNodeSet pooledSetB,
RangedNodeSet pooledSetC,
bool skipOutOfOrderNodes)
{
// This implementation attempts to efficiently traverse a graph without any precomputed indices or labeling beyond topologically ordered node values.
// Index-based approaches are tricky for the expected usage (the BuildXL scheduler) in which the underlying graph is dynamic.
// Instead, we traverse the graph with no prior information in hand, with a careful traversal order and some pruning.
// The thinking on pruning / use of topological labels is not new; for some more robust examples see e.g. FELINE:
// Veloso, Renê Rodrigues, et al. "Reachability Queries in Very Large Graphs: A Fast Refined Online Search Approach." EDBT. 2014.
// Figure 6 in particular gives some geometric insight, though the pruning here is less effective.
// Now, let's build some intuition about this implementation. We begin from a naive approach and will refine to what's actually implemented.
// First, consider the problem of traversing an _undirected_ graph to determine reachability from some point M to N.
// o\ /o o\ /o
// M -- o -- o -- N
// o/ \o o/ \o
// We can imagine the graph in some two-dimensional layout. To perform well with M and N fairly close, it would be wise to proceed in a breadth-first fashion:
// ●\ /● o\ /o
// M -- ● -- o -- N
// ●/ \● o/ \o
// On the first iteration, we have all nodes reachable in one hop from M. On the i'th iteration, we have all nodes reachable in 'i' hops. In the example above,
// N would be found on iteration 3. Geometrically, think of the reached set as a circle expanding outward from M. Note that on each iteration i, we only need to
// hold *the nodes i hopes away* (not i - 1 hops etc.) so in fact we are tracking the outer circumference of this circle. This works since there exists some single
// integer i by which N is at least i hops away (if reachable).
// We can leverage the geometric intuition of a circle to improve that approach a bit. Assume that on some iteration i, we have visited all nodes interior to the circle,
// and the discrete nodes are so numerous and dense as to approximate a circle's area. We can then think of i as a radius and the area as a count of nodes visited -
// on the order of i^2. If we instead traverse the same radius via two circles - each of radius (i/2) then the visited area is (i/2)^2 * 2 = i^2 / 2. Intersection of
// the circles implies a path between the two nodes.
// ●\ /● ●\ /●
// M -- ● -- ● -- N
// ●/ \● ●/ \●
// (above: the prior example on iteration 1 when expanding outward from both endpoints; a path will be found on iteration 2 instead of 3).
// We can still track only the outer circumference of the circles, so long as we alternately expand each circle (rather than expanding both instantaneously; each expansion
// increases the effective search radius by one and so there's no way for the circles to skip past each other).
// Now we first leverage the assumption of a *directed* graph. Simply, we can follow edges in opposing directions from each node (now labelling specifically as 'to' {T} and from'{F}),
// which geometrically looks like expanding semi-circles rather than circles (the nodes labeled X were skipped based on direction):
// X\ />● ●>\ />X
// >F --> ● --> ● --> T
// X/ \>● ●>/ \>X
// Finally we consider the usefulness of a topological labeling of the nodes. This is in fact a generalization of following edges in only one direction:
// - Outgoing-incident nodes must have higher labels than the one current.
// - Incoming-incident nodes must by symmetry have lower labels than the one currnet.
// This means that traversing outgoing edges result in a node-set (for the semi-circle's edge) that increases monotonically over iterations. Symmetrically
// the set for incoming edge traversal decreases monotonically. The example below adds bracketed topological labels; note that for outgoing edges we initially
// have [4, 4] and then [5, 7], and for incoming we first have [11, 11] and then [8, 10]. With this in mind we can see it is futile to traverse from node 10 to node 1,
// when expanding the incoming range since that node is on the 'wrong side' of the outgoing range already.
// X[3]\ />●[7] ●[9]>\ />X[12]
// >F[4] --> ●[6] --> ●[8] --> T[11]
// /->X[2]/ \>●[5] ->●[10]>/ \>X[13]
// X[1] -----------------------/
RangedNodeSet incomingRangeSet = pooledSetA;
incomingRangeSet.SetSingular(to);
var outgoingRangeSet = pooledSetB;
outgoingRangeSet.SetSingular(from);
var swap = pooledSetC;
swap.Clear();
bool toggle = false;
// Loop condition is effectively !incomingRangeSet.IsEmpty && !outgoingRangeSet.IsEmpty, but checked as one of the ranges changes.
while (true)
{
Func <IReadonlyDirectedGraph, NodeId, IEnumerable <Edge> > getEdges;
Func <NodeId, NodeId, bool> validateEdgeTopoProperty;
RangedNodeSet walkFromNodes;
RangedNodeSet intersectWith;
NodeRange incidentNodeFilter;
if (toggle)
{
// Decreasing from 'to' to NodeId.Min
getEdges = (g, n) => g.GetIncomingEdges(n);
validateEdgeTopoProperty = (node, other) => node.Value > other.Value;
incidentNodeFilter = NodeRange.CreateLowerBound(outgoingRangeSet.Range.FromInclusive);
walkFromNodes = incomingRangeSet;
intersectWith = outgoingRangeSet;
}
else
{
// Increasing from 'from' to NodeId.Max
getEdges = (g, n) => g.GetOutgoingEdges(n);
validateEdgeTopoProperty = (node, other) => node.Value < other.Value;
incidentNodeFilter = NodeRange.CreateUpperBound(incomingRangeSet.Range.ToInclusive);
walkFromNodes = outgoingRangeSet;
intersectWith = incomingRangeSet;
}
NodeId intersection;
NodeRange range;
if (RangeIncidentNodesAndIntersect(
graph,
walkFromNodes,
getEdges,
validateEdgeTopoProperty,
incidentNodeFilter,
intersectWith,
skipOutOfOrderNodes,
range: out range,
intersection: out intersection))
{
return(true);
}
if (range.IsEmpty)
{
break;
}
swap.ClearAndSetRange(range);
AddIncidentNodes(graph, walkFromNodes, getEdges, validateEdgeTopoProperty, incidentNodeFilter, skipOutOfOrderNodes, swap);
if (toggle)
{
RangedNodeSet temp = incomingRangeSet;
incomingRangeSet = swap;
swap = temp;
}
else
{
RangedNodeSet temp = outgoingRangeSet;
outgoingRangeSet = swap;
swap = temp;
}
toggle = !toggle;
}
return(false);
}
