private GraphReuseResult ReloadEngineSchedule( EngineSerializer serializer, CacheInitializationTask cacheInitializationTask, JournalState journalState, LoggingContext loggingContext, EngineState engineState, InputTracker.InputChanges inputChanges, string buildEngineFingerprint) { Tuple <EngineSchedule, EngineContext, IConfiguration> t = null; try { t = EngineSchedule.LoadAsync( Context, serializer, cacheInitializationTask, FileContentTable, journalState, Configuration, loggingContext, m_collector, m_directoryTranslator, engineState, tempCleaner: m_tempCleaner, buildEngineFingerprint).GetAwaiter().GetResult(); } catch (BuildXLException e) { Logger.Log.FailedReloadPipGraph(loggingContext, e.ToString()); } if (t == null) { return(GraphReuseResult.CreateForNoReuse(inputChanges)); } // Invalidate the old context to ensure nothing uses it anymore // Update engine state to deserialized state Context.Invalidate(); Context = t.Item2; // Update configuration state to deserialized state Configuration = t.Item3; // Copy the graph files to the session output if (Configuration.Distribution.BuildRole != DistributedBuildRoles.Worker) { // No need to link these files to the logs directory on workers since they are redundant with what's on the orchestrator m_executionLogGraphCopy = TryCreateHardlinksToScheduleFilesInSessionFolder(loggingContext, serializer); m_previousInputFilesCopy = TryCreateHardlinksToPreviousInputFilesInSessionFolder(loggingContext, serializer); } return(GraphReuseResult.CreateForFullReuse(t.Item1, inputChanges)); }
private GraphReuseResult ReloadEngineSchedule( EngineSerializer serializer, CacheInitializationTask cacheInitializationTask, JournalState journalState, LoggingContext loggingContext, EngineState engineState, InputTracker.InputChanges inputChanges, string buildEngineFingerprint) { Tuple <EngineSchedule, EngineContext, IConfiguration> t = EngineSchedule.LoadAsync( Context, serializer, cacheInitializationTask, FileContentTable, journalState, Configuration, loggingContext, m_collector, m_directoryTranslator, engineState, symlinkDefinitionFile: IsDistributedWorker ? m_workerSymlinkDefinitionFile.Value : Configuration.Layout.SymlinkDefinitionFile, tempCleaner: m_tempCleaner, buildEngineFingerprint).GetAwaiter().GetResult(); if (t == null) { return(GraphReuseResult.CreateForNoReuse(inputChanges)); } // Invalidate the old context to ensure nothing uses it anymore // Update engine state to deserialized state Context.Invalidate(); Context = t.Item2; // Update configuration state to deserialized state Configuration = t.Item3; // Copy the graph files to the session output m_executionLogGraphCopy = TryCreateHardlinksToScheduleFilesInSessionFolder(loggingContext, serializer); m_previousInputFilesCopy = TryCreateHardlinksToPreviousInputFilesInSessionFolder(loggingContext, serializer); return(GraphReuseResult.CreateForFullReuse(t.Item1, inputChanges)); }
/// <summary> /// Loads configured symlink definitions (if not already loaded) /// Stores to cache for use by workers in distributed build /// Eagerly creates symlinks if lazy symlink creation is disabled /// </summary> public static async Task <Possible <SymlinkDefinitions> > TryPrepareSymlinkDefinitionsAsync( LoggingContext loggingContext, GraphReuseResult reuseResult, IConfiguration configuration, MasterService masterService, CacheInitializationTask cacheInitializerTask, PipExecutionContext context, ITempDirectoryCleaner tempDirectoryCleaner = null) { var pathTable = context.PathTable; bool isDistributedMaster = configuration.Distribution.BuildRole == DistributedBuildRoles.Master; Possible <SymlinkDefinitions> maybeSymlinkDefinitions = new Possible <SymlinkDefinitions>((SymlinkDefinitions)null); if (reuseResult?.IsFullReuse == true) { maybeSymlinkDefinitions = reuseResult.EngineSchedule.Scheduler.SymlinkDefinitions; } else if (configuration.Layout.SymlinkDefinitionFile.IsValid) { var symlinkFilePath = configuration.Layout.SymlinkDefinitionFile.ToString(pathTable); Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Loading symlink file from location '{symlinkFilePath}'.")); maybeSymlinkDefinitions = await SymlinkDefinitions.TryLoadAsync( loggingContext, pathTable, symlinkFilePath, symlinksDebugPath : configuration.Logging.LogsDirectory.Combine(pathTable, "DebugSymlinksDefinitions.log").ToString(pathTable), tempDirectoryCleaner : tempDirectoryCleaner); } if (!maybeSymlinkDefinitions.Succeeded || maybeSymlinkDefinitions.Result == null) { return(maybeSymlinkDefinitions); } // Need to store symlinks to cache for workers if (configuration.Distribution.BuildRole == DistributedBuildRoles.Master) { var possibleCacheInitializer = await cacheInitializerTask; if (!possibleCacheInitializer.Succeeded) { return(possibleCacheInitializer.Failure); } Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Storing symlink file for use by workers.")); var symlinkFile = configuration.Layout.SymlinkDefinitionFile.Expand(pathTable); var possibleStore = await TryStoreToCacheAsync( loggingContext, cache : possibleCacheInitializer.Result.CreateCacheForContext(context).ArtifactContentCache, symlinkFile : symlinkFile); if (!possibleStore.Succeeded) { return(possibleStore.Failure); } masterService.SymlinkFileContentHash = possibleStore.Result; Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Stored symlink file for use by workers.")); } if (!configuration.Schedule.UnsafeLazySymlinkCreation || configuration.Engine.PopulateSymlinkDirectories.Count != 0) { // Symlink definition file is defined, and BuildXL intends to create it eagerly. // At this point master and worker should have had its symlink definition file, if specified. if (!FileContentManager.CreateSymlinkEagerly(loggingContext, configuration, pathTable, maybeSymlinkDefinitions.Result, context.CancellationToken)) { return(new Failure <string>("Failed eagerly creating symlinks")); } } return(maybeSymlinkDefinitions); }
/// <summary> /// Attempt to reuse the pip graph from a previous run /// </summary> private GraphReuseResult AttemptToReuseGraph( LoggingContext outerLoggingContext, int maxDegreeOfParallelism, GraphFingerprint graphFingerprint, IReadOnlyDictionary <string, string> properties, CacheInitializationTask cacheInitializationTask, JournalState journalState, EngineState engineState) { Contract.Ensures(Contract.Result <GraphReuseResult>() != null); GraphCacheCheckStatistics cacheGraphStats = CheckGraphCacheReuse( outerLoggingContext, maxDegreeOfParallelism, graphFingerprint, properties, cacheInitializationTask, journalState, out var serializer, out var inputChanges); // There are 3 cases in which we should reload the graph // - we have a graph cache hit // - the build is configured to reload the graph no matter what // - graph patching is enabled and the reason for cache miss was 'SpecFileChanges' var shouldReload = cacheGraphStats.WasHit || Configuration.Cache.CachedGraphPathToLoad.IsValid || PartialReloadCondition(Configuration.FrontEnd, cacheGraphStats); if (!shouldReload) { return(GraphReuseResult.CreateForNoReuse(inputChanges)); } bool fullReload = !PartialReloadCondition(Configuration.FrontEnd, cacheGraphStats); // Now we actually reload the graph var reloadStats = default(GraphCacheReloadStatistics); using (var tb = TimedBlock <EmptyStruct, GraphCacheReloadStatistics> .Start( outerLoggingContext, Statistics.GraphCacheReload, (context, emptyStruct) => { if (fullReload) { Logger.Log.ReloadingPipGraphStart(context); } else { Logger.Log.PartiallyReloadingEngineState(context); } }, default(EmptyStruct), (context, graphCacheCheckStatistics) => { Logger.Log.PartiallyReloadingEngineStateComplete(context, graphCacheCheckStatistics); m_enginePerformanceInfo.GraphReloadDurationMs = graphCacheCheckStatistics.ElapsedMilliseconds; }, () => reloadStats)) { var reuseResult = fullReload ? ReloadEngineSchedule( serializer, cacheInitializationTask, journalState, tb.LoggingContext, engineState, inputChanges, graphFingerprint?.ExactFingerprint.BuildEngineHash.ToString()) : ReloadPipGraphOnly(serializer, tb.LoggingContext, engineState, inputChanges); // Set telemetry statistics reloadStats.SerializedFileSizeBytes = serializer.BytesDeserialized; reloadStats.Success = !reuseResult.IsNoReuse; return(reuseResult); } }
private bool ConstructAndEvaluateGraph( LoggingContext loggingContext, FrontEndEngineAbstraction frontEndEngineAbstration, CacheInitializationTask engineCacheTask, MountsTable mountsTable, EvaluationFilter evaluationFilter, [CanBeNull] GraphReuseResult reuseResult, out PipGraph pipGraph) { Contract.Requires(frontEndEngineAbstration != null); Contract.Requires(engineCacheTask != null); Contract.Requires(mountsTable != null); pipGraph = null; IPipGraphBuilder pipGraphBuilder = null; if (!AddConfigurationMountsAndCompleteInitialization(loggingContext, mountsTable)) { return(false); } IDictionary <ModuleId, MountsTable> moduleMountsTableMap; if (!mountsTable.PopulateModuleMounts(Configuration.ModulePolicies.Values, out moduleMountsTableMap)) { Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()"); return(false); } m_visualization?.MountsTable.MakeAvailable(mountsTable); if ((Configuration.Engine.Phase & EnginePhases.Schedule) != 0) { pipGraphBuilder = CreatePipGraphBuilder(loggingContext, mountsTable, reuseResult); } // Have to do some horrible magic here to get to a proper Task<T> with the BuildXL cache since // someone updated the engine cache to be an await style pattern, and there is no way to get to the EngineCache // If the cache was fast to startup, but perhaps blocked itself on first access we wouldn't have to do all these hoops. Func <Task <Possible <EngineCache> > > getBuildCacheTask = async() => { return((await engineCacheTask).Then(engineCache => engineCache.CreateCacheForContext())); }; if (!FrontEndController.PopulateGraph( getBuildCacheTask(), pipGraphBuilder, frontEndEngineAbstration, evaluationFilter, Configuration, m_initialCommandLineConfiguration.Startup)) { LogFrontEndStats(loggingContext); Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after FrontEndController.PopulateGraph()"); return(false); } LogFrontEndStats(loggingContext); // Pip graph must become immutable now that evaluation is done (required to construct a scheduler). return(pipGraphBuilder == null || (pipGraph = pipGraphBuilder.Build()) != null); }
/// <summary> /// Check if pip graph can be reused. /// /// There are 3 opportunities to determine a graph match. The applicability of each depends on the distributed build roles. /// (1) from engine cache, /// (2) from content cache, and /// (3) from master node (if running on a worker node in a distributed build) /// </summary> private GraphCacheCheckStatistics CheckGraphCacheReuse( LoggingContext outerLoggingContext, int maxDegreeOfParallelism, GraphFingerprint graphFingerprint, IReadOnlyDictionary <string, string> properties, CacheInitializationTask cacheInitializationTask, JournalState journalState, out EngineSerializer serializer, out InputTracker.InputChanges inputChanges) { serializer = CreateEngineSerializer(outerLoggingContext); inputChanges = null; var cacheGraphStats = default(GraphCacheCheckStatistics); using (var timeBlock = TimedBlock <EmptyStruct, GraphCacheCheckStatistics> .Start( outerLoggingContext, Statistics.GraphCacheReuseCheck, (context, emptyStruct) => Logger.Log.CheckingForPipGraphReuseStart(context), default(EmptyStruct), (loggingContext, stats) => { Logger.Log.CheckingForPipGraphReuseComplete(loggingContext, stats); // On misses we want to give the user a message for why there was a miss if (!stats.WasHit) { Contract.Assume(stats.MissReason != GraphCacheMissReason.NoMiss); Logger.Log.GraphNotReusedDueToChangedInput(loggingContext, stats.MissMessageForConsole, stats.MissDescription); } m_enginePerformanceInfo.GraphCacheCheckDurationMs = stats.ElapsedMilliseconds; m_enginePerformanceInfo.GraphCacheCheckJournalEnabled = stats.JournalEnabled; }, () => cacheGraphStats)) { var loggingContext = timeBlock.LoggingContext; var effectiveEnvironmentVariables = FrontEndEngineImplementation.PopulateFromEnvironmentAndApplyOverrides(properties); var availableMounts = MountsTable.CreateAndRegister(loggingContext, Context, Configuration, m_initialCommandLineConfiguration.Startup.Properties); if (!AddConfigurationMountsAndCompleteInitialization(loggingContext, availableMounts)) { return(cacheGraphStats); } cacheGraphStats.JournalEnabled = journalState.IsEnabled; // ************************************************************ // 1. Engine cache check: // ************************************************************ // * Single machine builds // Distributed builds rely on the graph being available via the cache for it to be shared between master // and workers. So even if the master could have had a hit from the engine cache, it must be ignored // since the workers would not be able to retrieve it. if (!HasExplicitlyLoadedGraph(Configuration.Cache) && !Configuration.Schedule.ForceUseEngineInfoFromCache && Configuration.Distribution.BuildRole == DistributedBuildRoles.None) { Contract.Assume( graphFingerprint != null, "When looking up a cached graph on a distributed master or single-machine build, a graph fingerprint must be computed"); InputTracker.MatchResult engineCacheMatchResult = CheckIfAvailableInputsToGraphMatchPreviousRun( loggingContext, serializer, graphFingerprint: graphFingerprint, availableEnvironmentVariables: effectiveEnvironmentVariables, availableMounts: availableMounts, journalState: journalState, maxDegreeOfParallelism: maxDegreeOfParallelism); cacheGraphStats.ObjectDirectoryHit = engineCacheMatchResult.Matches; cacheGraphStats.ObjectDirectoryMissReason = engineCacheMatchResult.MissType; cacheGraphStats.MissReason = engineCacheMatchResult.MissType; cacheGraphStats.MissDescription = engineCacheMatchResult.FirstMissIdentifier; cacheGraphStats.WasHit = engineCacheMatchResult.Matches; cacheGraphStats.InputFilesChecked = engineCacheMatchResult.FilesChecked; // Checking the engine cache may have used a FileChangeTracker and provided information about // files/ContentHash pairs that were unchanged from the previous run. Hold onto this information as it may // be useful when eventually parsing files. // The FileChangeTracker is now up to date. All changed files have been removed from it. It can be reused // for a future build with the same fingerprint, though it may be tracking extra files, for example if // a spec was removed since the previous build. inputChanges = engineCacheMatchResult.InputChanges; } var shouldTryContentCache = !cacheGraphStats.WasHit && Configuration.Distribution.BuildRole != DistributedBuildRoles.Worker && Configuration.Cache.AllowFetchingCachedGraphFromContentCache && !HasExplicitlyLoadedGraph(Configuration.Cache) && (!Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.HasValue || !Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.Value); // ************************************************************ // 2. Content cache check: // ************************************************************ // * Single machine builds that missed earlier // * Distributed masters // This is the only valid place for the master to get a hit since it must be in the cache for the // workers to get it. if (shouldTryContentCache) { // Since an in-place match did not succeed, we need a readied cache to try again. // TODO: This logs an error if it fails. We are assuming some later thing will ensure that, if failed, // the engine fails overall. Possible <CacheInitializer> possibleCacheInitializerForFallback = cacheInitializationTask.GetAwaiter().GetResult(); if (possibleCacheInitializerForFallback.Succeeded) { CacheInitializer cacheInitializerForFallback = possibleCacheInitializerForFallback.Result; using (EngineCache cacheForFallback = cacheInitializerForFallback.CreateCacheForContext()) { var cacheGraphProvider = new CachedGraphProvider( loggingContext, Context, cacheForFallback, FileContentTable, maxDegreeOfParallelism); var cachedGraphDescriptor = cacheGraphProvider.TryGetPipGraphCacheDescriptorAsync(graphFingerprint, effectiveEnvironmentVariables, availableMounts.MountsByName).Result; if (cachedGraphDescriptor == null) { // There was no matching fingerprint in the cache. Record the status for logging before returning. cacheGraphStats.CacheMissReason = GraphCacheMissReason.FingerprintChanged; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } var fetchEngineScheduleContent = EngineSchedule.TryFetchFromCacheAsync( loggingContext, Context, cacheForFallback, cachedGraphDescriptor, serializer, FileContentTable, m_tempCleaner).Result; if (!fetchEngineScheduleContent) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoPreviousRunToCheck; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } // If a distributed master, take note of the graph fingerprint if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Master) { Contract.Assert(cachedGraphDescriptor != null); m_masterService.CachedGraphDescriptor = cachedGraphDescriptor; } Logger.Log.FetchedSerializedGraphFromCache(outerLoggingContext); cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; cacheGraphStats.WasHit = true; } } else { cacheGraphStats.CacheMissReason = GraphCacheMissReason.CacheFailure; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } } // ************************************************************ // 3. Query distributed master // ************************************************************ // * Distributed workers only if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Worker) { Contract.Assume( graphFingerprint == null, "Distributed workers should request a graph fingerprint from the master (not compute one locally)"); Possible <CacheInitializer> possibleCacheInitializerForWorker = cacheInitializationTask.GetAwaiter().GetResult(); Contract.Assume(possibleCacheInitializerForWorker.Succeeded, "Workers must have a valid cache"); CacheInitializer cacheInitializerForWorker = possibleCacheInitializerForWorker.Result; using (EngineCache cacheForWorker = cacheInitializerForWorker.CreateCacheForContext()) { PipGraphCacheDescriptor schedulerStateDescriptor; if (!m_workerService.TryGetBuildScheduleDescriptor(out schedulerStateDescriptor) || !EngineSchedule.TryFetchFromCacheAsync( outerLoggingContext, Context, cacheForWorker, schedulerStateDescriptor, serializer, FileContentTable, m_tempCleaner).Result) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; return(cacheGraphStats); } AsyncOut <AbsolutePath> symlinkFileLocation = new AsyncOut <AbsolutePath>(); if (!SymlinkDefinitionFileProvider.TryFetchWorkerSymlinkFileAsync( outerLoggingContext, Context.PathTable, cacheForWorker, Configuration.Layout, m_workerService, symlinkFileLocation).Result) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; return(cacheGraphStats); } m_workerSymlinkDefinitionFile = symlinkFileLocation.Value; // Success. Populate the stats cacheGraphStats.WasHit = true; cacheGraphStats.WorkerHit = true; cacheGraphStats.MissReason = GraphCacheMissReason.NoMiss; cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss; } } } return(cacheGraphStats); }