private RootFilter ParseFilter(IEnumerable <string> values, string commandLineFilter, string defaultFilter, out SymbolTable symbolTable, out PathTable pathTable) { var loggingContext = new LoggingContext("EngineScheduleTests.ParseFilter"); pathTable = new PathTable(); // EngineSchedulTest operate on the real filesystem for now var fileSystem = new PassThroughFileSystem(pathTable); BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem); symbolTable = context.SymbolTable; var config = new CommandLineConfiguration { Filter = commandLineFilter, Startup = { ImplicitFilters = new List <string>(values ?? Enumerable.Empty <string>()), }, Engine = { DefaultFilter = defaultFilter, } }; RootFilter rootFilter; XAssert.IsTrue(EngineSchedule.TryGetPipFilter(loggingContext, context, config, config, TryGetPathByMountName, rootFilter: out rootFilter)); return(rootFilter); }
/// <summary> /// Connects to the master and enables this node to receive build requests /// </summary> /// <param name="schedule">the engine schedule</param> /// <returns>true if the operation was successful</returns> internal async Task <bool> ConnectToMasterAsync(EngineSchedule schedule) { Contract.Requires(schedule != null); Contract.Requires(schedule.Scheduler != null); Contract.Requires(schedule.SchedulingQueue != null); Contract.Assert(AttachCompletion.IsCompleted && AttachCompletion.GetAwaiter().GetResult(), "ProcessBuildRequests called before finishing attach on worker"); m_workerPipStateManager = new WorkerServicePipStateManager(this); m_pipTable = schedule.PipTable; m_pipQueue = schedule.SchedulingQueue; m_scheduler = schedule.Scheduler; m_operationTracker = m_scheduler.OperationTracker; m_environment = m_scheduler; m_environment.ContentFingerprinter.FingerprintSalt = BuildStartData.FingerprintSalt; m_environment.State.PipEnvironment.MasterEnvironmentVariables = BuildStartData.EnvironmentVariables; m_resultSerializer = new ExecutionResultSerializer(maxSerializableAbsolutePathIndex: schedule.MaxSerializedAbsolutePath, executionContext: m_scheduler.Context); m_forwardingEventListener = new ForwardingEventListener(this); bool success = await SendAttachCompletedAfterProcessBuildRequestStartedAsync(); // Wait until the build finishes or we discovered that the master is dead success &= await ExitCompletion; success &= !m_hasFailures; m_pipQueue.SetAsFinalized(); return(success); }
private void RunGraphFingerprintValueTest(string[] expectedValues, string commandLineFilter, string[] commandLineValues, bool expectSuccess = true) { var pathTable = new PathTable(); // EngineSchedulTest operate on the real filesystem for now var fileSystem = new PassThroughFileSystem(pathTable); BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem); var config = new CommandLineConfiguration { Filter = commandLineFilter, Startup = { ImplicitFilters = new List <string>(commandLineValues) }, Engine = { DefaultFilter = null } }; EvaluationFilter evaluationFilter; if (!expectSuccess) { XAssert.IsFalse(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter)); } else { XAssert.IsTrue(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter)); foreach (FullSymbol symbol in evaluationFilter.ValueNamesToResolve) { string value = symbol.ToString(context.SymbolTable); XAssert.IsTrue(expectedValues.Contains(value)); } XAssert.AreEqual(expectedValues.Length, evaluationFilter.ValueNamesToResolve.Count()); } }
public PipResultListener(WorkerNotificationManager notificationManager, EngineSchedule schedule, IPipExecutionEnvironment environment) { m_notificationManager = notificationManager; m_resultSerializer = new ExecutionResultSerializer(maxSerializableAbsolutePathIndex: schedule.MaxSerializedAbsolutePath, executionContext: schedule.Scheduler.Context); m_pipTable = schedule.PipTable; m_environment = environment; }
private IPipGraphBuilder CreatePipGraphBuilder( LoggingContext loggingContext, MountsTable mountsTable, [CanBeNull] GraphReuseResult reuseResult) { var searchPathToolsHash = new Scheduler.DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable).ComputeSearchPathToolsHash(); var builder = new PipGraph.Builder( EngineSchedule.CreateEmptyPipTable(Context), Context, BuildXL.Pips.Tracing.Logger.Log, loggingContext, Configuration, mountsTable.MountPathExpander, fingerprintSalt: Configuration.Cache.CacheSalt, searchPathToolsHash: searchPathToolsHash); PatchablePipGraph patchableGraph = null; if (Configuration.FrontEnd.UseGraphPatching() && reuseResult?.IsPartialReuse == true) { Logger.Log.UsingPatchableGraphBuilder(loggingContext); patchableGraph = new PatchablePipGraph( oldPipGraph: reuseResult.PipGraph.DirectedGraph, oldPipTable: reuseResult.PipGraph.PipTable, graphBuilder: builder, maxDegreeOfParallelism: Configuration.FrontEnd.MaxFrontEndConcurrency()); } return((IPipGraphBuilder)patchableGraph ?? builder); }
private GraphReuseResult(PipGraph pipGraph, EngineSchedule engineSchedule, InputTracker.InputChanges inputChanges) { m_pipGraph = pipGraph; m_engineSchedule = engineSchedule; InputChanges = inputChanges; // Calling invariant method explicitely because this is the only way to check it at least once. Invariant(); }
/// <summary> /// Prepares the master for pips execution /// </summary> public void EnableDistribution(EngineSchedule schedule) { Contract.Requires(schedule != null); m_environment = schedule.Scheduler; schedule.Scheduler.EnableDistribution(m_remoteWorkers); m_resultSerializer = new ExecutionResultSerializer(schedule.MaxSerializedAbsolutePath, m_environment.Context); }
/// <summary> /// Factory method for the case when everything can be reused. /// A non-null engine schedule must be provided; input changes may optionally be provided too. /// </summary> internal static GraphReuseResult CreateForFullReuse(EngineSchedule engineSchedule, [CanBeNull] InputTracker.InputChanges inputChanges) { Contract.Requires(engineSchedule != null); return(new GraphReuseResult( pipGraph: null, engineSchedule: engineSchedule, inputChanges: inputChanges)); }
private GraphReuseResult ReloadEngineSchedule( EngineSerializer serializer, CacheInitializationTask cacheInitializationTask, JournalState journalState, LoggingContext loggingContext, EngineState engineState, InputTracker.InputChanges inputChanges, string buildEngineFingerprint) { Tuple <EngineSchedule, EngineContext, IConfiguration> t = null; try { t = EngineSchedule.LoadAsync( Context, serializer, cacheInitializationTask, FileContentTable, journalState, Configuration, loggingContext, m_collector, m_directoryTranslator, engineState, tempCleaner: m_tempCleaner, buildEngineFingerprint).GetAwaiter().GetResult(); } catch (BuildXLException e) { Logger.Log.FailedReloadPipGraph(loggingContext, e.ToString()); } if (t == null) { return(GraphReuseResult.CreateForNoReuse(inputChanges)); } // Invalidate the old context to ensure nothing uses it anymore // Update engine state to deserialized state Context.Invalidate(); Context = t.Item2; // Update configuration state to deserialized state Configuration = t.Item3; // Copy the graph files to the session output if (Configuration.Distribution.BuildRole != DistributedBuildRoles.Worker) { // No need to link these files to the logs directory on workers since they are redundant with what's on the orchestrator m_executionLogGraphCopy = TryCreateHardlinksToScheduleFilesInSessionFolder(loggingContext, serializer); m_previousInputFilesCopy = TryCreateHardlinksToPreviousInputFilesInSessionFolder(loggingContext, serializer); } return(GraphReuseResult.CreateForFullReuse(t.Item1, inputChanges)); }
private GraphReuseResult ReloadPipGraphOnly( EngineSerializer serializer, LoggingContext loggingContext, EngineState engineState, InputTracker.InputChanges inputChanges) { Tuple <PipGraph, EngineContext> t = null; try { t = EngineSchedule.LoadPipGraphAsync( Context, serializer, Configuration, loggingContext, engineState).GetAwaiter().GetResult(); } catch (BuildXLException e) { Logger.Log.FailedReloadPipGraph(loggingContext, e.ToString()); } if (t == null) { return(GraphReuseResult.CreateForNoReuse(inputChanges)); } var newContext = t.Item2; if (!ShouldReuseReloadedEngineContextGivenHistoricData(loggingContext, newContext.NextHistoricTableSizes)) { return(GraphReuseResult.CreateForNoReuse(inputChanges)); } var newPathTable = newContext.PathTable; var pathRemapper = new PathRemapper(Context.PathTable, newPathTable); Configuration = new ConfigurationImpl(Configuration, pathRemapper); m_initialCommandLineConfiguration = new CommandLineConfiguration(m_initialCommandLineConfiguration, pathRemapper); // Invalidate the old context to ensure nothing uses it anymore // Update engine state to deserialized state Context.Invalidate(); Context = newContext; // Additionally recreate front end controller, because the old one uses the invalidated context. // - to fully initialize the front end, we have to go through all the steps that have already been // executed on the old controller; those steps are (1) InitializeHost, and (2) ParseConfig FrontEndController = m_frontEndControllerFactory.Create(Context.PathTable, Context.SymbolTable); FrontEndController.InitializeHost(Context.ToFrontEndContext(loggingContext), m_initialCommandLineConfiguration); FrontEndController.ParseConfig(m_initialCommandLineConfiguration); return(GraphReuseResult.CreateForPartialReuse(t.Item1, inputChanges)); }
/// <nodoc/> public WorkerNotificationManager(WorkerService workerService, EngineSchedule schedule, IPipExecutionEnvironment environment, DistributionServices services) { WorkerService = workerService; DistributionServices = services; m_scheduler = schedule.Scheduler; m_environment = environment; m_forwardingEventListener = new ForwardingEventListener(this); m_sendCancellationSource = new CancellationTokenSource(); m_pipResultListener = new PipResultListener(this, schedule, environment); m_sendThread = new Thread(() => SendNotifications(m_sendCancellationSource.Token)); }
public void Start(IOrchestratorClient orchestratorClient, EngineSchedule schedule, IPipResultSerializer serializer) { Contract.AssertNotNull(orchestratorClient); m_orchestratorClient = orchestratorClient; m_executionLogTarget = new NotifyOrchestratorExecutionLogTarget(this, schedule); m_forwardingEventListener = new ForwardingEventListener(this); m_pipResultListener = new PipResultListener(this, serializer); m_sendCancellationSource = new CancellationTokenSource(); m_sendThread = new Thread(() => SendNotifications(m_sendCancellationSource.Token)); m_sendThread.Start(); m_started = true; }
private GraphReuseResult ReloadEngineSchedule( EngineSerializer serializer, CacheInitializationTask cacheInitializationTask, JournalState journalState, LoggingContext loggingContext, EngineState engineState, InputTracker.InputChanges inputChanges, string buildEngineFingerprint) { Tuple <EngineSchedule, EngineContext, IConfiguration> t = EngineSchedule.LoadAsync( Context, serializer, cacheInitializationTask, FileContentTable, journalState, Configuration, loggingContext, m_collector, m_directoryTranslator, engineState, symlinkDefinitionFile: IsDistributedWorker ? m_workerSymlinkDefinitionFile.Value : Configuration.Layout.SymlinkDefinitionFile, tempCleaner: m_tempCleaner, buildEngineFingerprint).GetAwaiter().GetResult(); if (t == null) { return(GraphReuseResult.CreateForNoReuse(inputChanges)); } // Invalidate the old context to ensure nothing uses it anymore // Update engine state to deserialized state Context.Invalidate(); Context = t.Item2; // Update configuration state to deserialized state Configuration = t.Item3; // Copy the graph files to the session output m_executionLogGraphCopy = TryCreateHardlinksToScheduleFilesInSessionFolder(loggingContext, serializer); m_previousInputFilesCopy = TryCreateHardlinksToPreviousInputFilesInSessionFolder(loggingContext, serializer); return(GraphReuseResult.CreateForFullReuse(t.Item1, inputChanges)); }
public static EngineState CreateNew(EngineSchedule engineSchedule) { Contract.Requires(engineSchedule != null); var schedulerState = new SchedulerState(engineSchedule.Scheduler); return(new EngineState( engineSchedule.Scheduler.PipGraph.GraphId, engineSchedule.Context.StringTable, engineSchedule.Context.PathTable, engineSchedule.Context.SymbolTable, engineSchedule.Context.QualifierTable, engineSchedule.PipTable, engineSchedule.Scheduler.PipGraph, engineSchedule.MountPathExpander, schedulerState, engineSchedule.Context.HistoricTableSizes)); }
internal void Start(EngineSchedule schedule) { Contract.Requires(schedule != null); Contract.Requires(schedule.Scheduler != null); Contract.Requires(schedule.SchedulingQueue != null); Contract.Assert(AttachCompletion.IsCompleted && AttachCompletion.GetAwaiter().GetResult(), "ProcessBuildRequests called before finishing attach on worker"); m_workerPipStateManager = new WorkerServicePipStateManager(this); m_pipTable = schedule.PipTable; m_pipQueue = schedule.SchedulingQueue; m_scheduler = schedule.Scheduler; m_operationTracker = m_scheduler.OperationTracker; m_environment = m_scheduler; m_environment.ContentFingerprinter.FingerprintSalt = BuildStartData.FingerprintSalt; m_environment.State.PipEnvironment.MasterEnvironmentVariables = BuildStartData.EnvironmentVariables; m_resultSerializer = new ExecutionResultSerializer(maxSerializableAbsolutePathIndex: schedule.MaxSerializedAbsolutePath, executionContext: m_scheduler.Context); m_forwardingEventListener = new ForwardingEventListener(this); }
/// <summary> /// Serializes and saves state needed for execution analyzers. /// </summary> private Task <bool> SaveExecutionStateToDiskAsync(ScheduleRunResult result) { // Every scheduler run has a unique log directory based off timestamp of run string logDirectory = Path.Combine( result.Config.Logging.LogsDirectory.ToString(Context.PathTable), result.Config.Logging.LogPrefix); var serializer = new EngineSerializer( LoggingContext, logDirectory, correlationId: FileEnvelopeId.Create()); var dummyHistoricData = (IReadOnlyList <HistoricDataPoint>)CollectionUtilities.EmptyArray <HistoricDataPoint>(); var dummyHistoricTableSizes = new HistoricTableSizes(dummyHistoricData); return(EngineSchedule.SaveExecutionStateToDiskAsync( serializer, Context, PipTable, result.Graph, Expander, dummyHistoricTableSizes)); }
public RequiredDependencyAnalyzer(AnalysisInput input) : base(input) { m_pool = new ObjectPool <WorkerAnalyzer>(() => CreateAnalyzer(), w => { }); m_pipTable = EngineSchedule.CreateEmptyPipTable(input.CachedGraph.Context); }
void IWorkerNotificationManager.Start(IOrchestratorClient orchestratorClient, EngineSchedule schedule, IPipResultSerializer serializer) { Interlocked.Increment(ref StartCalls); }
internal NotifyOrchestratorExecutionLogTarget(WorkerNotificationManager notificationManager, EngineSchedule engineSchedule) : this(new NotifyStream(notificationManager), engineSchedule.Context, engineSchedule.Scheduler.PipGraph.GraphId, engineSchedule.Scheduler.PipGraph.MaxAbsolutePathIndex) { m_scheduler = engineSchedule?.Scheduler; m_scheduler?.AddExecutionLogTarget(this); }
public static bool TryBuildWorkspace( ICommandLineConfiguration commandLineConfig, FrontEndContext frontEndContext, EngineContext engineContext, EvaluationFilter evaluationFilter, EventHandler <WorkspaceProgressEventArgs> progressHandler, out Workspace workspace, out FrontEndHostController frontEndHostController, out IPipGraph pipGraph, WorkspaceBuilderConfiguration configuration, FrontEndEngineAbstraction frontEndEngineAbstraction = null, bool collectMemoryAsSoonAsPossible = true) { Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None); Contract.Requires(frontEndContext != null); Contract.Requires(engineContext != null); Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid); Contract.Requires(evaluationFilter != null); workspace = null; frontEndHostController = null; pipGraph = null; var pathTable = engineContext.PathTable; var loggingContext = frontEndContext.LoggingContext; var mutableCommandlineConfig = GetCommandLineConfiguration( commandLineConfig, configuration); BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext); BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null); var statistics = new FrontEndStatistics(progressHandler); var frontEndControllerFactory = FrontEndControllerFactory.Create( mode: FrontEndMode.NormalMode, loggingContext: loggingContext, configuration: mutableCommandlineConfig, collector: null, statistics: statistics, collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible); var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable); controller.InitializeHost(frontEndContext, mutableCommandlineConfig); frontEndHostController = (FrontEndHostController)controller; // If there is an explicit engine abstraction, we set it. This is used by IDE test. if (frontEndEngineAbstraction != null) { frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig); } var config = controller.ParseConfig(mutableCommandlineConfig); if (config == null) { return(false); } IPipGraph pipGraphBuilder = null; using (var cache = Task.FromResult <Possible <EngineCache> >( new EngineCache( new InMemoryArtifactContentCache(), // Note that we have an 'empty' store (no hits ever) rather than a normal in memory one. new EmptyTwoPhaseFingerprintStore()))) { if (frontEndEngineAbstraction == null) { if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule)) { var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties); frontEndEngineAbstraction = new FrontEndEngineImplementation( loggingContext, frontEndContext.PathTable, config, mutableCommandlineConfig.Startup, mountsTable, InputTracker.CreateDisabledTracker(loggingContext), null, null, () => FileContentTable.CreateStub(), 5000, false); pipGraphBuilder = new PipGraph.Builder( EngineSchedule.CreateEmptyPipTable(engineContext), engineContext, Scheduler.Tracing.Logger.Log, loggingContext, config, mountsTable.MountPathExpander, fingerprintSalt: config.Cache.CacheSalt, directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable)); if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable)) { return(false); } IDictionary <ModuleId, MountsTable> moduleMountsTableMap; if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap)) { Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()"); return(false); } } else { frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config); } } using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null) { // Attempt to build and/or analyze the workspace if (!controller.PopulateGraph( cache: cache, graph: pipGraphBuilder, engineAbstraction: frontEndEngineAbstraction, evaluationFilter: evaluationFilter, configuration: config, startupConfiguration: mutableCommandlineConfig.Startup)) { workspace = frontEndHostController.GetWorkspace(); // Error has been reported already return(false); } pipGraph = pipGraphBuilder; } } Contract.Assert(frontEndHostController != null); workspace = frontEndHostController.GetWorkspace(); if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace) { // If workspace construction is successful, we run the linter on all specs. // This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path workspace = CreateLintedWorkspace( workspace, frontEndContext.LoggingContext, config.FrontEnd, pathTable); } return(true); }
/// <summary> /// Runs the scheduler allowing various options to be specifically set /// </summary> public ScheduleRunResult RunSchedulerSpecific( PipGraph graph, SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, TempCleaner tempCleaner = null) { var config = new CommandLineConfiguration(Configuration); // Populating the configuration may modify the configuration, so it should occur first. BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, Context.PathTable, bxlExeLocation: null, inTestMode: true); BuildXLEngine.PopulateAndValidateConfiguration(config, config, Context.PathTable, LoggingContext); FileAccessWhitelist whitelist = new FileAccessWhitelist(Context); whitelist.Initialize(config); IReadOnlyList <string> junctionRoots = Configuration.Engine.DirectoriesToTranslate?.Select(a => a.ToPath.ToString(Context.PathTable)).ToList(); var map = VolumeMap.TryCreateMapOfAllLocalVolumes(LoggingContext, junctionRoots); var optionalAccessor = TryGetJournalAccessor(map); // Although scan change journal is enabled, but if we cannot create an enabled journal accessor, then create a disabled one. m_journalState = map == null || !optionalAccessor.IsValid ? JournalState.DisabledJournal : JournalState.CreateEnabledJournal(map, optionalAccessor.Value); if (config.Schedule.IncrementalScheduling) { // Ensure that we can scan the journal when incremental scheduling is enabled. XAssert.IsTrue(m_journalState.IsEnabled, "Incremental scheduling requires that journal is enabled"); } // Seal the translator if not sealed DirectoryTranslator.Seal(); // ..................................................................................... // some dummy setup in order to get a PreserveOutputsSalt.txt file and an actual salt // ..................................................................................... string dummyCacheDir = Path.Combine(TemporaryDirectory, "Out", "Cache"); Directory.CreateDirectory(dummyCacheDir); // EngineSchedule tries to put the PreserveOutputsSalt.txt here ContentHash?previousOutputsSalt = EngineSchedule.PreparePreviousOutputsSalt(LoggingContext, Context.PathTable, config); Contract.Assert(previousOutputsSalt.HasValue); // ..................................................................................... testHooks = testHooks ?? new SchedulerTestHooks(); Contract.Assert(!(config.Engine.CleanTempDirectories && tempCleaner == null)); using (var queue = new PipQueue(config.Schedule)) using (var testQueue = new TestPipQueue(queue, LoggingContext, initiallyPaused: false)) using (var testScheduler = new TestScheduler( graph: graph, pipQueue: testQueue, context: Context, fileContentTable: FileContentTable, loggingContext: LoggingContext, cache: Cache, configuration: config, journalState: m_journalState, fileAccessWhitelist: whitelist, fingerprintSalt: Configuration.Cache.CacheSalt, directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable), tempCleaner: tempCleaner, previousInputsSalt: previousOutputsSalt.Value, successfulPips: null, failedPips: null, ipcProvider: null, directoryTranslator: DirectoryTranslator, testHooks: testHooks)) { if (filter == null) { EngineSchedule.TryGetPipFilter(LoggingContext, Context, config, config, Expander.TryGetRootByMountName, out filter); } XAssert.IsTrue(testScheduler.InitForMaster(LoggingContext, filter, schedulerState), "Failed to initialized test scheduler"); testScheduler.Start(LoggingContext); bool success = testScheduler.WhenDone().GetAwaiter().GetResult(); testScheduler.SaveFileChangeTrackerAsync(LoggingContext).Wait(); return(new ScheduleRunResult { Graph = graph, Config = config, Success = success, PipResults = testScheduler.PipResults, PipExecutorCounters = testScheduler.PipExecutionCounters, PathSets = testScheduler.PathSets, ProcessPipCountersByFilter = testScheduler.ProcessPipCountersByFilter, ProcessPipCountersByTelemetryTag = testScheduler.ProcessPipCountersByTelemetryTag, SchedulerState = new SchedulerState(testScheduler) }); } }
void IWorkerNotificationManager.Start(IOrchestratorClient orchestratorClient, EngineSchedule schedule, IPipResultSerializer serializer) { StartCalls++; }
/// <summary> /// Check if pip graph can be reused. /// /// There are 3 opportunities to determine a graph match. The applicability of each depends on the distributed build roles. /// (1) from engine cache, /// (2) from content cache, and /// (3) from master node (if running on a worker node in a distributed build) /// </summary> private GraphCacheCheckStatistics CheckGraphCacheReuse( LoggingContext outerLoggingContext, int maxDegreeOfParallelism, GraphFingerprint graphFingerprint, IReadOnlyDictionary <string, string> properties, CacheInitializationTask cacheInitializationTask, JournalState journalState, out EngineSerializer serializer, out InputTracker.InputChanges inputChanges) { serializer = CreateEngineSerializer(outerLoggingContext); inputChanges = null; var cacheGraphStats = default(GraphCacheCheckStatistics); using (var timeBlock = TimedBlock <EmptyStruct, GraphCacheCheckStatistics> .Start( outerLoggingContext, Statistics.GraphCacheReuseCheck, (context, emptyStruct) => Logger.Log.CheckingForPipGraphReuseStart(context), default(EmptyStruct), (loggingContext, stats) => { Logger.Log.CheckingForPipGraphReuseComplete(loggingContext, stats); // On misses we want to give the user a message for why there was a miss if (!stats.WasHit) { Contract.Assume(stats.MissReason != GraphCacheMissReason.NoMiss); Logger.Log.GraphNotReusedDueToChangedInput(loggingContext, stats.MissMessageForConsole, stats.MissDescription); } m_enginePerformanceInfo.GraphCacheCheckDurationMs = stats.ElapsedMilliseconds; m_enginePerformanceInfo.GraphCacheCheckJournalEnabled = stats.JournalEnabled; }, () => cacheGraphStats)) { var loggingContext = timeBlock.LoggingContext; var effectiveEnvironmentVariables = FrontEndEngineImplementation.PopulateFromEnvironmentAndApplyOverrides(properties); var availableMounts = MountsTable.CreateAndRegister(loggingContext, Context, Configuration, m_initialCommandLineConfiguration.Startup.Properties); if (!AddConfigurationMountsAndCompleteInitialization(loggingContext, availableMounts)) { return(cacheGraphStats); } cacheGraphStats.JournalEnabled = journalState.IsEnabled; // ************************************************************ // 1. Engine cache check: // ************************************************************ // * Single machine builds // Distributed builds rely on the graph being available via the cache for it to be shared between master // and workers. So even if the master could have had a hit from the engine cache, it must be ignored // since the workers would not be able to retrieve it. if (!HasExplicitlyLoadedGraph(Configuration.Cache) && !Configuration.Schedule.ForceUseEngineInfoFromCache && Configuration.Distribution.BuildRole == DistributedBuildRoles.None) { Contract.Assume( graphFingerprint != null, "When looking up a cached graph on a distributed master or single-machine build, a graph fingerprint must be computed"); InputTracker.MatchResult engineCacheMatchResult = CheckIfAvailableInputsToGraphMatchPreviousRun( loggingContext, serializer, graphFingerprint: graphFingerprint, availableEnvironmentVariables: effectiveEnvironmentVariables, availableMounts: availableMounts, journalState: journalState, maxDegreeOfParallelism: maxDegreeOfParallelism); cacheGraphStats.ObjectDirectoryHit = engineCacheMatchResult.Matches; cacheGraphStats.ObjectDirectoryMissReason = engineCacheMatchResult.MissType; cacheGraphStats.MissReason = engineCacheMatchResult.MissType; cacheGraphStats.MissDescription = engineCacheMatchResult.FirstMissIdentifier; cacheGraphStats.WasHit = engineCacheMatchResult.Matches; cacheGraphStats.InputFilesChecked = engineCacheMatchResult.FilesChecked; // Checking the engine cache may have used a FileChangeTracker and provided information about // files/ContentHash pairs that were unchanged from the previous run. Hold onto this information as it may // be useful when eventually parsing files. // The FileChangeTracker is now up to date. All changed files have been removed from it. It can be reused // for a future build with the same fingerprint, though it may be tracking extra files, for example if // a spec was removed since the previous build. inputChanges = engineCacheMatchResult.InputChanges; } var shouldTryContentCache = !cacheGraphStats.WasHit && Configuration.Distribution.BuildRole != DistributedBuildRoles.Worker && Configuration.Cache.AllowFetchingCachedGraphFromContentCache && !HasExplicitlyLoadedGraph(Configuration.Cache) && (!Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.HasValue || !Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.Value); // ************************************************************ // 2. Content cache check: // ************************************************************ // * Single machine builds that missed earlier // * Distributed masters // This is the only valid place for the master to get a hit since it must be in the cache for the // workers to get it. if (shouldTryContentCache) { // Since an in-place match did not succeed, we need a readied cache to try again. // TODO: This logs an error if it fails. We are assuming some later thing will ensure that, if failed, // the engine fails overall. Possible <CacheInitializer> possibleCacheInitializerForFallback = cacheInitializationTask.GetAwaiter().GetResult(); if (possibleCacheInitializerForFallback.Succeeded) { CacheInitializer cacheInitializerForFallback = possibleCacheInitializerForFallback.Result; using (EngineCache cacheForFallback = cacheInitializerForFallback.CreateCacheForContext()) { var cacheGraphProvider = new CachedGraphProvider( loggingContext, Context, cacheForFallback, FileContentTable, maxDegreeOfParallelism); var cachedGraphDescriptor = cacheGraphProvider.TryGetPipGraphCacheDescriptorAsync(graphFingerprint, effectiveEnvironmentVariables, availableMounts.MountsByName).Result; if (cachedGraphDescriptor == null) { // There was no matching fingerprint in the cache. Record the status for logging before returning. cacheGraphStats.CacheMissReason = GraphCacheMissReason.FingerprintChanged; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } var fetchEngineScheduleContent = EngineSchedule.TryFetchFromCacheAsync( loggingContext, Context, cacheForFallback, cachedGraphDescriptor, serializer, FileContentTable, m_tempCleaner).Result; if (!fetchEngineScheduleContent) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoPreviousRunToCheck; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } // If a distributed master, take note of the graph fingerprint if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Master) { Contract.Assert(cachedGraphDescriptor != null); m_masterService.CachedGraphDescriptor = cachedGraphDescriptor; } Logger.Log.FetchedSerializedGraphFromCache(outerLoggingContext); cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; cacheGraphStats.WasHit = true; } } else { cacheGraphStats.CacheMissReason = GraphCacheMissReason.CacheFailure; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } } // ************************************************************ // 3. Query distributed master // ************************************************************ // * Distributed workers only if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Worker) { Contract.Assume( graphFingerprint == null, "Distributed workers should request a graph fingerprint from the master (not compute one locally)"); Possible <CacheInitializer> possibleCacheInitializerForWorker = cacheInitializationTask.GetAwaiter().GetResult(); Contract.Assume(possibleCacheInitializerForWorker.Succeeded, "Workers must have a valid cache"); CacheInitializer cacheInitializerForWorker = possibleCacheInitializerForWorker.Result; using (EngineCache cacheForWorker = cacheInitializerForWorker.CreateCacheForContext()) { PipGraphCacheDescriptor schedulerStateDescriptor; if (!m_workerService.TryGetBuildScheduleDescriptor(out schedulerStateDescriptor) || !EngineSchedule.TryFetchFromCacheAsync( outerLoggingContext, Context, cacheForWorker, schedulerStateDescriptor, serializer, FileContentTable, m_tempCleaner).Result) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; return(cacheGraphStats); } AsyncOut <AbsolutePath> symlinkFileLocation = new AsyncOut <AbsolutePath>(); if (!SymlinkDefinitionFileProvider.TryFetchWorkerSymlinkFileAsync( outerLoggingContext, Context.PathTable, cacheForWorker, Configuration.Layout, m_workerService, symlinkFileLocation).Result) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; return(cacheGraphStats); } m_workerSymlinkDefinitionFile = symlinkFileLocation.Value; // Success. Populate the stats cacheGraphStats.WasHit = true; cacheGraphStats.WorkerHit = true; cacheGraphStats.MissReason = GraphCacheMissReason.NoMiss; cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss; } } } return(cacheGraphStats); }
private static bool TryBuildWorkspaceInternal( ICommandLineConfiguration commandLineConfig, FrontEndContext frontEndContext, EngineContext engineContext, EvaluationFilter evaluationFilter, EventHandler <WorkspaceProgressEventArgs> progressHandler, out Workspace workspace, out FrontEndHostController frontEndHostController, out IMutablePipGraph pipGraph, WorkspaceBuilderConfiguration configuration, bool forIDE, FrontEndEngineAbstraction frontEndEngineAbstraction = null, bool collectMemoryAsSoonAsPossible = true) { Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None); Contract.Requires(frontEndContext != null); Contract.Requires(engineContext != null); Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid); Contract.Requires(evaluationFilter != null); workspace = null; frontEndHostController = null; pipGraph = null; var pathTable = engineContext.PathTable; var loggingContext = frontEndContext.LoggingContext; var mutableCommandlineConfig = GetCommandLineConfiguration( commandLineConfig, configuration); BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext); BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null); var statistics = new FrontEndStatistics(progressHandler); var frontEndControllerFactory = FrontEndControllerFactory.Create( mode: FrontEndMode.NormalMode, loggingContext: loggingContext, configuration: mutableCommandlineConfig, collector: null, statistics: statistics, collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible); var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable); controller.InitializeHost(frontEndContext, mutableCommandlineConfig); frontEndHostController = (FrontEndHostController)controller; // If there is an explicit engine abstraction, we set it. This is used by the IDE. if (frontEndEngineAbstraction != null) { // The IDE engine typically doesn't have mounts configured. We do it here if they haven't been configured yet. // Observe these are just the default mounts used for config evaluation. if (frontEndEngineAbstraction is BasicFrontEndEngineAbstraction basicEngine && !frontEndEngineAbstraction.GetMountNames("Script", BuildXL.Utilities.ModuleId.Invalid).Any()) { // If this fails we just ignore the failure. Mounts not being properly configured doesn't prevent the IDE plugin from working. basicEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties); } frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig); } else { // Otherwise we construct one with all mounts populated for config evaluation var configurationEngine = new BasicFrontEndEngineAbstraction(engineContext.PathTable, engineContext.FileSystem, mutableCommandlineConfig); if (!configurationEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties)) { // Errors are logged already return(false); } frontEndEngineAbstraction = configurationEngine; } var config = controller.ParseConfig(frontEndEngineAbstraction, mutableCommandlineConfig); if (config == null) { return(false); } IMutablePipGraph pipGraphBuilder = null; using (var cache = Task.FromResult <Possible <EngineCache> >( new EngineCache( new InMemoryArtifactContentCache(), // Note that we have an 'empty' store (no hits ever) rather than a normal in memory one. new EmptyTwoPhaseFingerprintStore()))) { var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties); // For the IDE case, we want to make sure all config-specific mounts are properly populated if (forIDE && frontEndEngineAbstraction is BasicFrontEndEngineAbstraction languageServiceEngine) { Contract.AssertNotNull(frontEndEngineAbstraction); AddConfigurationMounts(config, mountsTable); languageServiceEngine.SetMountsTable(mountsTable); } if (frontEndEngineAbstraction == null) { if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule)) { frontEndEngineAbstraction = new FrontEndEngineImplementation( loggingContext, frontEndContext.PathTable, config, mutableCommandlineConfig.Startup, mountsTable, InputTracker.CreateDisabledTracker(loggingContext), null, null, () => FileContentTable.CreateStub(loggingContext), 5000, false, controller.RegisteredFrontEnds); var searchPathToolsHash = new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable).ComputeSearchPathToolsHash(); pipGraphBuilder = new PipGraph.Builder( EngineSchedule.CreateEmptyPipTable(engineContext), engineContext, Pips.Tracing.Logger.Log, loggingContext, config, mountsTable.MountPathExpander, fingerprintSalt: config.Cache.CacheSalt, searchPathToolsHash: searchPathToolsHash); // Observe mount table is completed during workspace construction AddConfigurationMounts(config, mountsTable); IDictionary <ModuleId, MountsTable> moduleMountsTableMap; if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap)) { Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()"); return(false); } } else { frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config); } } using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null) { // Attempt to build and/or analyze the workspace if (!controller.PopulateGraph( cache: cache, graph: pipGraphBuilder, engineAbstraction: frontEndEngineAbstraction, evaluationFilter: evaluationFilter, configuration: config, startupConfiguration: mutableCommandlineConfig.Startup)) { workspace = frontEndHostController.GetWorkspace(); // Error has been reported already return(false); } pipGraph = pipGraphBuilder; } } Contract.Assert(frontEndHostController != null); workspace = frontEndHostController.GetWorkspace(); if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace) { // If workspace construction is successful, we run the linter on all specs. // This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path workspace = CreateLintedWorkspace( workspace, frontEndContext.LoggingContext, config.FrontEnd, pathTable); } return(true); }