/// <summary> /// Constructor. /// </summary> private FingerprintStoreExecutionLogTarget( LoggingContext loggingContext, PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, FingerprintStore fingerprintStore, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { m_context = context; m_pipTable = pipTable; PipContentFingerprinter = pipContentFingerprinter; FingerprintStore = fingerprintStore; m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>(); m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync( this, loggingContext, context, configuration, cache, graph, runnablePipPerformance); }
private WeakContentFingerprint GenerateSaltedWeakFingerprint(ContentHash hash) => new WeakContentFingerprint(FingerprintUtilities.Hash($"Hash: '{hash.ToHex()}' Salt: '{m_buildManifestHashCacheSalt}'")); // Changes to this string will invalidate all existing cache entries /// <nodoc /> public ApiServer( IIpcProvider ipcProvider, string ipcMonikerId, FileContentManager fileContentManager, PipExecutionContext context, IServerConfig config, EngineCache engineCache, Tracing.IExecutionLogTarget executionLog, Tracing.BuildManifestGenerator buildManifestGenerator) { Contract.Requires(ipcMonikerId != null); Contract.Requires(fileContentManager != null); Contract.Requires(context != null); Contract.Requires(config != null); Contract.Requires(engineCache != null); Contract.Requires(executionLog != null); m_fileContentManager = fileContentManager; m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config); m_context = context; m_engineCache = engineCache; m_executionLog = executionLog; m_buildManifestGenerator = buildManifestGenerator; m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, ContentHash>(); m_buildManifestHashCacheSalt = string.IsNullOrEmpty(Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt) ? string.Empty : Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt; }
/// <summary> /// Constructor. /// </summary> private FingerprintStoreExecutionLogTarget( LoggingContext loggingContext, PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, FingerprintStore fingerprintStore, FingerprintStore cacheLookupFingerprintStore, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, CounterCollection <FingerprintStoreCounters> counters, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { m_context = context; m_pipTable = pipTable; LoggingContext = loggingContext; PipContentFingerprinter = pipContentFingerprinter; ExecutionFingerprintStore = fingerprintStore; CacheLookupFingerprintStore = cacheLookupFingerprintStore; // Cache lookup store is per-build state and doesn't need to be garbage collected (vs. execution fignerprint store which is persisted build-over-build) CacheLookupFingerprintStore?.GarbageCollectCancellationToken.Cancel(); m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>(); Counters = counters; FingerprintStoreMode = configuration.Logging.FingerprintStoreMode; m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync( this, loggingContext, context, configuration, cache, graph, runnablePipPerformance); Contract.Assume(FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly || CacheLookupFingerprintStore != null, "Unless /storeFingerprints flag is set to /storeFingerprints:ExecutionFingerprintsOnly, the cache lookup FingerprintStore must exist."); }
/// <summary> /// Retrieve the running time table from the cache /// </summary> public static async Task <Possible <bool> > TryRetrieveRunningTimeTableAsync( this EngineCache cache, LoggingContext loggingContext, string path, PathTable pathTable, ContentFingerprint graphSemistableFingerprint, DateTime?time = null) { var possibleCacheEntry = await cache.TwoPhaseFingerprintStore.TryGetLatestCacheEntryAsync(loggingContext, graphSemistableFingerprint, time); if (!possibleCacheEntry.Succeeded) { Logger.Log.PerformanceDataCacheTrace( loggingContext, I($"Failed loading running time table entry from cache: Failure:{possibleCacheEntry.Failure.DescribeIncludingInnerFailures()}")); return(possibleCacheEntry.Failure); } Logger.Log.PerformanceDataCacheTrace( loggingContext, I($"Loaded running time table entry from cache: Fingerprint='{graphSemistableFingerprint}' MetadataHash={possibleCacheEntry.Result?.MetadataHash ?? ContentHashingUtilities.ZeroHash}")); if (!possibleCacheEntry.Result.HasValue) { return(false); } var runningTimeTableHash = possibleCacheEntry.Result.Value.MetadataHash; var absolutePath = AbsolutePath.Create(pathTable, path); var maybePinned = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { runningTimeTableHash }); var result = await maybePinned.ThenAsync( async pinResult => { if (!pinResult.AllContentAvailable) { return(new Failure <string>(I($"Could not pin content for running time table '{runningTimeTableHash}'"))); } return(await cache.ArtifactContentCache.TryMaterializeAsync( FileRealizationMode.Copy, absolutePath.Expand(pathTable), runningTimeTableHash)); }); if (!result.Succeeded) { Logger.Log.PerformanceDataCacheTrace( loggingContext, I($"Failed loading running time table from cache: Failure:{result.Failure.DescribeIncludingInnerFailures()}")); return(result.Failure); } Logger.Log.PerformanceDataCacheTrace(loggingContext, I($"Loaded running time table from cache: Path='{path}'")); return(true); }
private static Scheduler CreateInternal( PipExecutionContext context, PipGraph pipGraph, IPipQueue queue, EngineCache cache, IConfiguration configuration) { Contract.Requires(context != null); Contract.Requires(queue != null); Contract.Requires(cache != null); Contract.Requires(configuration != null); var fileContentTable = FileContentTable.CreateNew(); var fileAccessWhiteList = new FileAccessWhitelist(context); var testHooks = new SchedulerTestHooks(); return(new Scheduler( pipGraph, queue, context, fileContentTable, cache: cache, loggingContext: Events.StaticContext, configuration: configuration, fileAccessWhitelist: fileAccessWhiteList, testHooks: testHooks, buildEngineFingerprint: null)); }
/// <summary> /// Store the running time table to the cache /// </summary> public static async Task <Possible <Unit> > TryStoreRunningTimeTableAsync( this EngineCache cache, LoggingContext loggingContext, string path, PathTable pathTable, ContentFingerprint performanceDataFingerprint, DateTime?storeTime = null) { var absolutePath = AbsolutePath.Create(pathTable, path); var storeResult = await cache.ArtifactContentCache.TryStoreAsync( FileRealizationMode.Copy, absolutePath.Expand(pathTable)); Logger.Log.HistoricPerfDataCacheTrace(loggingContext, I($"Storing running time table to cache: Success='{storeResult.Succeeded}'")); if (!storeResult.Succeeded) { return(storeResult.Failure); } ContentHash hash = storeResult.Result; var cacheEntry = new CacheEntry(hash, null, ArrayView <ContentHash> .Empty); var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, performanceDataFingerprint, cacheEntry, storeTime); Logger.Log.HistoricPerfDataCacheTrace(loggingContext, I($"Publishing running time table from cache: Fingerprint='{performanceDataFingerprint}' Hash={hash}")); return(publishResult); }
public static Tuple <Scheduler, EngineCache> CreateWithCaching( PipExecutionContext context, LoggingContext loggingContext, IConfiguration configuration, PipGraph.Builder graphBuilder, IPipQueue queue) { Contract.Requires(graphBuilder != null); Contract.Requires(context != null); Contract.Requires(queue != null); var cacheLayer = new EngineCache( new InMemoryArtifactContentCache(), new InMemoryTwoPhaseFingerprintStore()); Scheduler scheduler = CreateInternal( context, loggingContext, graphBuilder.Build(), queue, cacheLayer, configuration); return(Tuple.Create(scheduler, cacheLayer)); }
private static Scheduler CreateInternal( PipExecutionContext context, PipGraph pipGraph, IPipQueue queue, EngineCache cache, IConfiguration configuration) { Contract.Requires(context != null); Contract.Requires(queue != null); Contract.Requires(cache != null); Contract.Requires(configuration != null); var fileContentTable = FileContentTable.CreateNew(); var fileAccessWhiteList = new FileAccessWhitelist(context); var testHooks = new SchedulerTestHooks(); return(new Scheduler( pipGraph, queue, context, fileContentTable, cache: cache, loggingContext: Events.StaticContext, configuration: configuration, fileAccessWhitelist: fileAccessWhiteList, testHooks: testHooks, buildEngineFingerprint: null, tempCleaner: new TestMoveDeleteCleaner(Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "MoveDeletionTemp")))); }
/// <nodoc /> public ApiServer( IIpcProvider ipcProvider, string ipcMonikerId, FileContentManager fileContentManager, PipExecutionContext context, IServerConfig config, EngineCache engineCache, Tracing.IExecutionLogTarget executionLog, Tracing.BuildManifestGenerator buildManifestGenerator) { Contract.Requires(ipcMonikerId != null); Contract.Requires(fileContentManager != null); Contract.Requires(context != null); Contract.Requires(config != null); Contract.Requires(engineCache != null); Contract.Requires(executionLog != null); m_fileContentManager = fileContentManager; m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config); m_context = context; m_engineCache = engineCache; m_executionLog = executionLog; m_buildManifestGenerator = buildManifestGenerator; m_inMemoryBuildManifestStore = new ConcurrentDictionary <ContentHash, ContentHash>(); }
private FrontEndHostController CreateHost() { var factory = new FrontEndFactory(); factory.AddFrontEnd(new DummyFrontEnd1()); factory.TrySeal(new LoggingContext("UnitTest")); var controller = new FrontEndHostController(factory, new DScriptWorkspaceResolverFactory(), new EvaluationScheduler(degreeOfParallelism: 8), collectMemoryAsSoonAsPossible: false); var context = BuildXLContext.CreateInstanceForTesting(); var fileSystem = new InMemoryFileSystem(context.PathTable); ((IFrontEndController)controller).InitializeHost( new FrontEndContext(context, new LoggingContext("UnitTest"), fileSystem), new ConfigurationImpl() { FrontEnd = new FrontEndConfiguration() { MaxFrontEndConcurrency = 1, } }); var inMemoryCache = new EngineCache( new InMemoryArtifactContentCache(context), new InMemoryTwoPhaseFingerprintStore()); controller.InitializeInternalForTesting( Task.FromResult(new Possible <EngineCache>(inMemoryCache)), AbsolutePath.Create(context.PathTable, TestOutputDirectory)); return(controller); }
EngineContainer LoadFromCache(string key) { if (EngineCache == null || !EngineCache.ContainsKey(key)) { return(null); } return(EngineCache[key]); }
public void GetEngine_PYandRBEngine_CachedEnginesCountEquals2() { LanguageSettings ruby = Helper.CreateIronRubySettings(); LanguageSettings python = Helper.CreateIronPythonSettings(); EngineCache cache = new EngineCache(); cache.GetEngine(ruby, null); cache.GetEngine(python, null); Assert.AreEqual(2, cache.CachedEngines); }
/// <summary> /// Tries to fetch symlink file from the cache. /// </summary> public static async Task <bool> TryFetchWorkerSymlinkFileAsync( LoggingContext loggingContext, PathTable pathTable, EngineCache cache, ILayoutConfiguration layoutConfiguration, WorkerService workerService, AsyncOut <AbsolutePath> symlinkPathAsyncOut) { Contract.Requires(loggingContext != null); Contract.Requires(pathTable != null); Contract.Requires(cache != null); Contract.Requires(workerService != null); Contract.Requires(symlinkPathAsyncOut != null); symlinkPathAsyncOut.Value = AbsolutePath.Invalid; var symlinkFileContentHash = workerService.BuildStartData.SymlinkFileContentHash.ToContentHash(); if (symlinkFileContentHash == WellKnownContentHashes.AbsentFile) { // Absent file meaning there is no symlink file to use return(true); } Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Attempting to retrieve symlink file with hash '{symlinkFileContentHash}'.")); var maybeLoaded = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { symlinkFileContentHash }); if (!maybeLoaded.Succeeded) { Tracing.Logger.Log.FailedLoadSymlinkFileFromCache(loggingContext, maybeLoaded.Failure.DescribeIncludingInnerFailures()); return(false); } var destinationPath = layoutConfiguration.EngineCacheDirectory.Combine(pathTable, SymlinkFileName).Expand(pathTable); var materializedFile = await cache.ArtifactContentCache.TryMaterializeAsync( FileRealizationMode.HardLinkOrCopy, destinationPath, symlinkFileContentHash); if (!materializedFile.Succeeded) { Tracing.Logger.Log.FailedMaterializeSymlinkFileFromCache( loggingContext, destinationPath.ExpandedPath, materializedFile.Failure.DescribeIncludingInnerFailures()); return(false); } Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Symlink file with hash '{symlinkFileContentHash}' materialized to location '{destinationPath}'.")); symlinkPathAsyncOut.Value = destinationPath.Path; return(true); }
public void AppendPathToEngines_ValidPath_EngineHavePathAdded() { string path = Environment.CurrentDirectory; LanguageSettings ruby = Helper.CreateIronRubySettings(); EngineCache cache = new EngineCache(); cache.AppendPathToEngines(path); IEngine engine = cache.GetEngine(ruby, null); bool found = EngineHasPath(path, engine); Assert.IsTrue(found); }
public TestScheduler( PipGraph graph, TestPipQueue pipQueue, PipExecutionContext context, FileContentTable fileContentTable, EngineCache cache, IConfiguration configuration, FileAccessWhitelist fileAccessWhitelist, DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null, ITempCleaner tempCleaner = null, PipRuntimeTimeTable runningTimeTable = null, JournalState journalState = null, PerformanceCollector performanceCollector = null, string fingerprintSalt = null, PreserveOutputsInfo?previousInputsSalt = null, IEnumerable <Pip> successfulPips = null, IEnumerable <Pip> failedPips = null, LoggingContext loggingContext = null, IIpcProvider ipcProvider = null, DirectoryTranslator directoryTranslator = null, VmInitializer vmInitializer = null, SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache, configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules, tempCleaner, AsyncLazy <PipRuntimeTimeTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt, ipcProvider: ipcProvider, directoryTranslator: directoryTranslator, journalState: journalState, vmInitializer: vmInitializer, testHooks: testHooks) { m_testPipQueue = pipQueue; if (successfulPips != null) { foreach (var pip in successfulPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded); } } if (failedPips != null) { foreach (var pip in failedPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed); } } m_loggingContext = loggingContext; }
/// <nodoc /> public PipTwoPhaseCache(LoggingContext loggingContext, EngineCache cache, PipExecutionContext context, PathExpander pathExpander) { Contract.Requires(loggingContext != null); Contract.Requires(cache != null); Contract.Requires(context != null); LoggingContext = loggingContext; ArtifactContentCache = cache.ArtifactContentCache; TwoPhaseFingerprintStore = cache.TwoPhaseFingerprintStore; Context = context; Counters = new CounterCollection <PipCachingCounter>(); m_pathExpander = pathExpander; }
/// <summary> /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified. /// </summary> public DummyPipExecutionEnvironment( LoggingContext loggingContext, PipExecutionContext context, IConfiguration config, FileContentTable fileContentTable = null, EngineCache pipCache = null, SemanticPathExpander semanticPathExpander = null, PipContentFingerprinter.PipDataLookup pipDataLookup = null, FileAccessWhitelist fileAccessWhitelist = null, bool allowUnspecifiedSealedDirectories = false, PipTable pipTable = null, IIpcProvider ipcProvider = null, (string substSource, string substTarget)?subst = default,
/// <nodoc/> public SchedulerIntegrationTestBase(ITestOutputHelper output) : base(output) { m_testOutputHelper = output; CaptureAllDiagnosticMessages = false; // Each event listener that we want to capture events from must be listed here foreach (var eventSource in BuildXLApp.GeneratedEventSources) { RegisterEventSource(eventSource); } // Go through the command line configuration handling used by the real app to get the appropriate defaults ICommandLineConfiguration config; XAssert.IsTrue(Args.TryParseArguments(new[] { "/c:" + Path.Combine(TemporaryDirectory, "config.dc") }, Context.PathTable, null, out config), "Failed to construct arguments"); Configuration = new CommandLineConfiguration(config); Cache = InMemoryCacheFactory.Create(); FileContentTable = FileContentTable.CreateNew(LoggingContext); // Disable defaults that write disk IO but are not critical to correctness or performance Configuration.Logging.StoreFingerprints = false; Configuration.Logging.LogExecution = false; Configuration.Engine.TrackBuildsInUserFolder = false; Configuration.Engine.CleanTempDirectories = false; Configuration.Sandbox.EnsureTempDirectoriesExistenceBeforePipExecution = true; // Skip hash source file pips // Some error becomes a contract exception when SkipHashSourceFile is enabled. // Mostly on methods that traverse graph, e.g., DirectedGraph.GetOutgoingEdges(); // BUG 1472567 // Configuration.Schedule.SkipHashSourceFile = true; // Compute static pip fingerprints for incremental scheduling tests. Configuration.Schedule.ComputePipStaticFingerprints = true; // Disable currently enabled unsafe option. Configuration.Sandbox.UnsafeSandboxConfigurationMutable.IgnoreCreateProcessReport = false; // Populate file system capabilities. // Here, for example, we use copy-on-write instead of hardlinks when Unix file system supports copy-on-write. // Particular tests can override this by setting Configuration.Engine.UseHardlinks. BuildXLEngine.PopulateFileSystemCapabilities(Configuration, Configuration, Context.PathTable, LoggingContext); // Reset pip graph builder to use the populated configuration. ResetPipGraphBuilder(); }
public void AppendPathToEngines_ValidPath_2EnginesHavePathAdded() { string path = Environment.CurrentDirectory; LanguageSettings ruby = Helper.CreateIronRubySettings(); LanguageSettings python = Helper.CreateIronPythonSettings(); EngineCache cache = new EngineCache(); cache.AppendPathToEngines(path); IEngine rubyEngine = cache.GetEngine(ruby, null); IEngine pythonEngine = cache.GetEngine(python, null); bool rubyFound = EngineHasPath(path, rubyEngine); bool pythonFound = EngineHasPath(path, pythonEngine); Assert.IsTrue(rubyFound); Assert.IsTrue(pythonFound); }
/// <summary> /// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>. /// </summary> /// <returns> /// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to /// a <see cref="Tracing.FingerprintStore"/> at the provided directory; /// otherwise, null. /// </returns> public static FingerprintStoreExecutionLogTarget Create( PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, string fingerprintStoreDirectory, LoggingContext loggingContext, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null, FingerprintStoreTestHooks testHooks = null) { var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0); var possibleStore = FingerprintStore.Open( fingerprintStoreDirectory, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, testHooks: testHooks); if (possibleStore.Succeeded) { return(new FingerprintStoreExecutionLogTarget( loggingContext, context, pipTable, pipContentFingerprinter, possibleStore.Result, configuration, cache, graph, runnablePipPerformance)); } else { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleStore.Failure.DescribeIncludingInnerFailures()); } return(null); }
/// <summary> /// Run the test /// </summary> public bool Run(string testFolder, string specFile, string fullIdentifier, string shortName, string lkgFile, params string[] sdksToResolve) { Contract.Requires(!string.IsNullOrEmpty(testFolder)); Contract.Requires(!string.IsNullOrEmpty(specFile)); Contract.Requires(sdksToResolve != null); // Sadly the frontend doesn't use the engine abstractions file api's so we have to materialize stuff on disk for now... // TODO: Fix this code once the frontend supports a proper virtual FileSystem. // TODO: Change the package semantics to implicit when we expose a way to evaluate a single value var testFileName = Path.GetFileName(specFile); var mainFileName = "testMain.bp"; var testMainFile = Path.Combine(testFolder, mainFileName); Directory.CreateDirectory(testFolder); File.WriteAllText(Path.Combine(testFolder, Names.ModuleConfigBm), I($@"module( {{ name: 'TestPackage', nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences, projects: [ f`{mainFileName}`, f`{testFileName}`, ], }});")); File.WriteAllText(testMainFile, I($@" export const testFolder = d`{Path.GetDirectoryName(specFile).Replace('\\', '/')}`; @@public export const main = {fullIdentifier}();")); File.Copy(specFile, Path.Combine(testFolder, testFileName)); // Create a fake package for Sdk.TestRunner so that you can safely test packages that have the tests embedded in them. var testRunnerFolder = Path.Combine(testFolder, "Sdk.TestRunner"); Directory.CreateDirectory(testRunnerFolder); File.WriteAllText(Path.Combine(testRunnerFolder, Names.ModuleConfigBm), I($"module({{\n\tname: 'Sdk.TestRunner',\n}});")); File.WriteAllText(Path.Combine(testRunnerFolder, "package" + Names.DotDscExtension), I($@" export interface TestArguments {{ testFiles: File[]; sdkFolders?: (Directory|StaticDirectory)[]; autoFixLkgs?: boolean; }} export interface TestResult {{ xmlResults: File; }} export function test(args: TestArguments): TestResult {{ Contract.fail(""Can't run a DScript UnitTest inside of a DScript UnitTest""); }}")); // Setup Context and configuration var frontEndContext = FrontEndContext.CreateInstanceForTesting(); var pipContext = new SchedulerContext(CancellationToken.None, frontEndContext.StringTable, frontEndContext.PathTable, frontEndContext.SymbolTable, frontEndContext.QualifierTable); var pathTable = frontEndContext.PathTable; var testFolderPath = AbsolutePath.Create(pathTable, testFolder); var configuration = CreateConfiguration(sdksToResolve.Union(new[] { testRunnerFolder }), pathTable, testFolderPath); var engineAbstraction = new TestEngineAbstraction(pathTable, frontEndContext.StringTable, testFolderPath, new PassThroughFileSystem(pathTable)); var frontEndStatistics = new FrontEndStatistics(); if (!CreateFactories( frontEndContext, engineAbstraction, frontEndStatistics, configuration, out var ambientTesting, out var moduleRegistry, out var frontEndFactory)) { return(false); } // Set the timeout to a large number to avoid useless performance collections in tests. using (var performanceCollector = new PerformanceCollector(TimeSpan.FromHours(1))) using (var frontEndHostController = new FrontEndHostController( frontEndFactory, new EvaluationScheduler(1), moduleRegistry, frontEndStatistics, m_tracingLogger, performanceCollector, collectMemoryAsSoonAsPossible: true)) { var frontEndController = (IFrontEndController)frontEndHostController; frontEndController.InitializeHost(frontEndContext, configuration); frontEndController.ParseConfig(configuration); // Populate the graph using (var pipTable = new PipTable( pipContext.PathTable, pipContext.SymbolTable, initialBufferSize: 16384, maxDegreeOfParallelism: 1, debug: true)) { var mountPathExpander = new MountPathExpander(pathTable); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "testFolder"), testFolderPath, allowHashing: true, readable: true, writable: false)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "src"), testFolderPath.Combine(pathTable, "src"), allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "out"), testFolderPath.Combine(pathTable, "out"), allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "noRead"), testFolderPath.Combine(pathTable, "noRead"), allowHashing: true, readable: false, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "temp"), engineAbstraction.Layout.TempDirectory, allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "obj"), engineAbstraction.Layout.ObjectDirectory, allowHashing: true, readable: true, writable: true)); var graph = new PipGraph.Builder( pipTable, pipContext, m_pipLogger, frontEndContext.LoggingContext, configuration, mountPathExpander); using (var cacheLayer = new EngineCache( new InMemoryArtifactContentCache(), new InMemoryTwoPhaseFingerprintStore())) { var cache = Task.FromResult(Possible.Create(cacheLayer)); try { var evaluationFilter = new EvaluationFilter( pipContext.SymbolTable, pipContext.PathTable, new FullSymbol[0], new[] { AbsolutePath.Create(frontEndContext.PathTable, testMainFile), }, CollectionUtilities.EmptyArray <StringId>()); if (!frontEndController.PopulateGraph(cache, graph, engineAbstraction, evaluationFilter, configuration, configuration.Startup)) { HandleDiagnostics(); return(false); } } catch (AggregateException e) { var baseException = e.GetBaseException(); if (baseException is XunitException) { // If it is an XUnit assert, then unwrap the exception and throw that because XUnit other doesn't display the error nicely. ExceptionDispatchInfo.Capture(baseException).Throw(); } throw; } } if (!ValidatePips(frontEndContext, graph, testFolderPath, specFile, shortName, lkgFile, ambientTesting.DontValidatePipsEnabled)) { return(false); } } } HandleDiagnostics(); return(true); }
/// <summary> /// Initiates the task to load the fingerprint store that will be used for cache miss analysis /// </summary> public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, FingerprintStoreTestHooks testHooks = null) { // Unblock caller await Task.Yield(); using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration)) { var option = configuration.Logging.CacheMissAnalysisOption; string downLoadedPriviousFingerprintStoreSavedPath = null; if (option.Mode == CacheMissMode.Disabled) { return(null); } Possible <FingerprintStore> possibleStore; PathTable newPathTable = new PathTable(); if (option.Mode == CacheMissMode.Local) { possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext); } else { string path = null; if (option.Mode == CacheMissMode.CustomPath) { path = option.CustomPath.ToString(context.PathTable); } else { Contract.Assert(option.Mode == CacheMissMode.Remote); foreach (var key in option.Keys) { var fingerprintsLogDirectoryStr = configuration.Logging.FingerprintsLogDirectory.ToString(context.PathTable); var fingerprintsLogDirectory = AbsolutePath.Create(newPathTable, fingerprintsLogDirectoryStr); var cacheSavePath = fingerprintsLogDirectory.Combine(newPathTable, Scheduler.FingerprintStoreDirectory + "." + key); var result = await cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, newPathTable, key, configuration, context.CancellationToken); if (result.Succeeded && result.Result) { path = cacheSavePath.ToString(newPathTable); downLoadedPriviousFingerprintStoreSavedPath = path; break; } } if (string.IsNullOrEmpty(path)) { Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}")); return(null); } } possibleStore = FingerprintStore.Open(path, readOnly: true); } if (possibleStore.Succeeded) { Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory); return(new RuntimeCacheMissAnalyzer( logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance, configuration, downLoadedPriviousFingerprintStoreSavedPath, testHooks: testHooks)); } Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}")); return(null); } }
public void GetEngine_PYEngine_TwoCalls_SameObjectReturned() { LanguageSettings python = Helper.CreateIronPythonSettings(); EngineCache cache = new EngineCache(); IEngine engine1 = cache.GetEngine(python, null); IEngine engine2 = cache.GetEngine(python, null); Assert.AreEqual(engine1, engine2); Assert.AreEqual(1, cache.CachedEngines); }
public void GetEngine_PYEngine_CachedEnginesCountEquals1() { LanguageSettings python = Helper.CreateIronPythonSettings(); EngineCache cache = new EngineCache(); cache.GetEngine(python, null); Assert.AreEqual(1, cache.CachedEngines); }
/// <summary> /// Retrieve the fingerprint store from the cache /// </summary> public static async Task <Possible <bool> > TryRetrieveFingerprintStoreAsync( this EngineCache cache, LoggingContext loggingContext, AbsolutePath path, PathTable pathTable, string key, string environment) { var fingerprint = ComputeFingerprint(pathTable, key, environment); var possibleCacheEntry = await cache.TwoPhaseFingerprintStore.TryGetLatestCacheEntryAsync(loggingContext, fingerprint); if (!possibleCacheEntry.Succeeded) { Logger.Log.GettingFingerprintStoreTrace( loggingContext, I($"Failed loading fingerprint store cache entry from cache: Key='{key}' Failure: {possibleCacheEntry.Failure.DescribeIncludingInnerFailures()}")); return(possibleCacheEntry.Failure); } Logger.Log.GettingFingerprintStoreTrace( loggingContext, I($"Loaded fingerprint store entry from cache: Key='{key}' Fingerprint='{fingerprint}' MetadataHash='{possibleCacheEntry.Result?.MetadataHash ?? ContentHashingUtilities.ZeroHash}'")); if (!possibleCacheEntry.Result.HasValue) { return(false); } var fingerprintStoreDescriptorHash = possibleCacheEntry.Result.Value.MetadataHash; var maybePinned = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(possibleCacheEntry.Result.Value.ToArray()); var result = await maybePinned.ThenAsync <Unit>( async pinResult => { if (!pinResult.AllContentAvailable) { return(new Failure <string>(I($"Could not pin content for fingerprint store '{string.Join(", ", pinResult.Results.Where(r => !r.IsAvailable).Select(r => r.Hash))}'"))); } var maybeLoadedDescriptor = await cache.ArtifactContentCache.TryLoadAndDeserializeContent <PackageDownloadDescriptor>(fingerprintStoreDescriptorHash); if (!maybeLoadedDescriptor.Succeeded) { return(maybeLoadedDescriptor.Failure); } Logger.Log.GettingFingerprintStoreTrace( loggingContext, I($"Loaded fingerprint store cache descriptor from cache: Key='{key}' Hash='{fingerprintStoreDescriptorHash}'")); PackageDownloadDescriptor descriptor = maybeLoadedDescriptor.Result; SemaphoreSlim concurrencyLimiter = new SemaphoreSlim(8); var materializedFiles = await Task.WhenAll(descriptor.Contents.Select(async subPathKeyedHash => { await Task.Yield(); using (await concurrencyLimiter.AcquireAsync()) { var filePath = path.Combine(pathTable, subPathKeyedHash.Key); var maybeMaterialized = await cache.ArtifactContentCache.TryMaterializeAsync( FileRealizationMode.Copy, filePath.Expand(pathTable), subPathKeyedHash.ContentHash.ToContentHash()); return(maybeMaterialized); } }).ToList()); var failure = materializedFiles.Where(p => !p.Succeeded).Select(p => p.Failure).FirstOrDefault(); if (failure != null) { return(failure); } return(Unit.Void); }); if (!result.Succeeded) { Logger.Log.GettingFingerprintStoreTrace( loggingContext, I($"Failed loading fingerprint store from cache: Key='{key}' Failure: {result.Failure.DescribeIncludingInnerFailures()}")); return(result.Failure); } Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Loaded fingerprint store from cache: Key='{key}' Path='{path.ToString(pathTable)}'")); return(true); }
/// <summary> /// Store the fingerprint store directory to the cache /// </summary> public static async Task <Possible <long> > TrySaveFingerprintStoreAsync( this EngineCache cache, LoggingContext loggingContext, AbsolutePath path, PathTable pathTable, string key, string environment) { var fingerprint = ComputeFingerprint(pathTable, key, environment); var pathStr = path.ToString(pathTable); BoxRef <long> size = 0; SemaphoreSlim concurrencyLimiter = new SemaphoreSlim(8); var tasks = new List <Task <Possible <StringKeyedHash, Failure> > >(); FileUtilities.EnumerateDirectoryEntries(pathStr, (name, attr) => { var task = Task.Run(async() => { using (await concurrencyLimiter.AcquireAsync()) { var filePath = path.Combine(pathTable, name); var storeResult = await cache.ArtifactContentCache.TryStoreAsync( FileRealizationMode.Copy, filePath.Expand(pathTable)); if (storeResult.Succeeded) { Interlocked.Add(ref size.Value, new FileInfo(filePath.ToString(pathTable)).Length); } return(storeResult.Then(result => new StringKeyedHash() { Key = path.ExpandRelative(pathTable, filePath), ContentHash = result.ToBondContentHash() })); } }); tasks.Add(task); }); var storedFiles = await Task.WhenAll(tasks); var failure = storedFiles.Where(p => !p.Succeeded).Select(p => p.Failure).FirstOrDefault(); Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store to cache: Success='{failure == null}', FileCount={storedFiles.Length} Size={size.Value}")); if (failure != null) { return(failure); } PackageDownloadDescriptor descriptor = new PackageDownloadDescriptor() { TraceInfo = loggingContext.Session.Environment, FriendlyName = nameof(FingerprintStore), Contents = storedFiles.Select(p => p.Result).ToList() }; var storeDescriptorResult = await cache.ArtifactContentCache.TrySerializeAndStoreContent(descriptor); Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store descriptor to cache: Success='{storeDescriptorResult.Succeeded}'")); if (!storeDescriptorResult.Succeeded) { return(storeDescriptorResult.Failure); } var associatedFileHashes = descriptor.Contents.Select(s => s.ContentHash.ToContentHash()).ToArray().ToReadOnlyArray().GetSubView(0); var cacheEntry = new CacheEntry(storeDescriptorResult.Result, null, associatedFileHashes); var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, fingerprint, cacheEntry); Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Publishing fingerprint store to cache: Fingerprint='{fingerprint}' Hash={storeDescriptorResult.Result}")); return(size.Value); }
public void GetEngine_RBEngine_CreatesEngine_ReturnsObject() { LanguageSettings ruby = Helper.CreateIronRubySettings(); EngineCache cache = new EngineCache(); IEngine engine = cache.GetEngine(ruby, null); Assert.AreEqual("IronRuby", engine.LanguageName); }
public void GetEngine_PYandRBEngine_TwoDifferentObjectsReturned() { LanguageSettings ruby = Helper.CreateIronRubySettings(); LanguageSettings python = Helper.CreateIronPythonSettings(); EngineCache cache = new EngineCache(); IEngine rubyEngine = cache.GetEngine(ruby, null); IEngine pythonEngine = cache.GetEngine(python, null); Assert.AreNotEqual(rubyEngine, pythonEngine); }
/// <summary> /// Initiates the task to load the fingerprint store that will be used for cache miss analysis /// </summary> public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration)) { var option = configuration.Logging.CacheMissAnalysisOption; if (option.Mode == CacheMissMode.Disabled) { return(null); } Possible <FingerprintStore> possibleStore; if (option.Mode == CacheMissMode.Local) { possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext); } else { string path = null; if (option.Mode == CacheMissMode.CustomPath) { path = option.CustomPath.ToString(context.PathTable); } else { Contract.Assert(option.Mode == CacheMissMode.Remote); foreach (var key in option.Keys) { var cacheSavePath = configuration.Logging.FingerprintsLogDirectory .Combine(context.PathTable, Scheduler.FingerprintStoreDirectory + "." + key); #pragma warning disable AsyncFixer02 // This should explicitly happen synchronously since it interacts with the PathTable and StringTable var result = cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, context.PathTable, key, configuration.Schedule.EnvironmentFingerprint).Result; #pragma warning restore AsyncFixer02 if (result.Succeeded && result.Result) { path = cacheSavePath.ToString(context.PathTable); break; } } if (string.IsNullOrEmpty(path)) { Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}")); return(null); } } // Unblock caller // WARNING: The rest can simultenously happen with saving the graph files to disk. // We should not create any paths or strings by using PathTable and StringTable. await Task.Yield(); possibleStore = FingerprintStore.Open(path, readOnly: true); } if (possibleStore.Succeeded) { Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory); return(new RuntimeCacheMissAnalyzer(logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance)); } Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}")); return(null); } }
public void GetEngine_PYEngine_ByLanguageSettingsObject_ReturnsDLREngine() { LanguageSettings python = Helper.CreateIronPythonSettings(); EngineCache cache = new EngineCache(); IEngine engine = cache.GetEngine(python, null); Assert.AreEqual("IronPython 2.0 Beta", engine.LanguageName); }
/// <summary> /// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>. /// </summary> /// <returns> /// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to /// a <see cref="Tracing.FingerprintStore"/> at the provided directory; /// otherwise, null. /// </returns> public static FingerprintStoreExecutionLogTarget Create( PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, LoggingContext loggingContext, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, CounterCollection <FingerprintStoreCounters> counters, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null, FingerprintStoreTestHooks testHooks = null) { var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable); var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable); try { FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString); } catch (BuildXLException ex) { Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message); throw new BuildXLException("Unable to create fingerprint store directory: ", ex); } var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0); var possibleExecutionStore = FingerprintStore.Open( fingerprintStorePathString, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, counters: counters, testHooks: testHooks); Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet."); if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly) { try { FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString); } catch (BuildXLException ex) { Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message); throw new BuildXLException("Unable to create fingerprint store directory: ", ex); } possibleCacheLookupStore = FingerprintStore.Open( cacheLookupFingerprintStorePathString, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, counters: counters, testHooks: testHooks); } if (possibleExecutionStore.Succeeded && (possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly)) { return(new FingerprintStoreExecutionLogTarget( loggingContext, context, pipTable, pipContentFingerprinter, possibleExecutionStore.Result, possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null, configuration, cache, graph, counters, runnablePipPerformance)); } else { if (!possibleExecutionStore.Succeeded) { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures()); } if (!possibleCacheLookupStore.Succeeded) { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures()); } } return(null); }
/// <summary> /// Check if pip graph can be reused. /// /// There are 3 opportunities to determine a graph match. The applicability of each depends on the distributed build roles. /// (1) from engine cache, /// (2) from content cache, and /// (3) from master node (if running on a worker node in a distributed build) /// </summary> private GraphCacheCheckStatistics CheckGraphCacheReuse( LoggingContext outerLoggingContext, int maxDegreeOfParallelism, GraphFingerprint graphFingerprint, IReadOnlyDictionary <string, string> properties, CacheInitializationTask cacheInitializationTask, JournalState journalState, out EngineSerializer serializer, out InputTracker.InputChanges inputChanges) { serializer = CreateEngineSerializer(outerLoggingContext); inputChanges = null; var cacheGraphStats = default(GraphCacheCheckStatistics); using (var timeBlock = TimedBlock <EmptyStruct, GraphCacheCheckStatistics> .Start( outerLoggingContext, Statistics.GraphCacheReuseCheck, (context, emptyStruct) => Logger.Log.CheckingForPipGraphReuseStart(context), default(EmptyStruct), (loggingContext, stats) => { Logger.Log.CheckingForPipGraphReuseComplete(loggingContext, stats); // On misses we want to give the user a message for why there was a miss if (!stats.WasHit) { Contract.Assume(stats.MissReason != GraphCacheMissReason.NoMiss); Logger.Log.GraphNotReusedDueToChangedInput(loggingContext, stats.MissMessageForConsole, stats.MissDescription); } m_enginePerformanceInfo.GraphCacheCheckDurationMs = stats.ElapsedMilliseconds; m_enginePerformanceInfo.GraphCacheCheckJournalEnabled = stats.JournalEnabled; }, () => cacheGraphStats)) { var loggingContext = timeBlock.LoggingContext; var effectiveEnvironmentVariables = FrontEndEngineImplementation.PopulateFromEnvironmentAndApplyOverrides(properties); var availableMounts = MountsTable.CreateAndRegister(loggingContext, Context, Configuration, m_initialCommandLineConfiguration.Startup.Properties); if (!AddConfigurationMountsAndCompleteInitialization(loggingContext, availableMounts)) { return(cacheGraphStats); } cacheGraphStats.JournalEnabled = journalState.IsEnabled; // ************************************************************ // 1. Engine cache check: // ************************************************************ // * Single machine builds // Distributed builds rely on the graph being available via the cache for it to be shared between master // and workers. So even if the master could have had a hit from the engine cache, it must be ignored // since the workers would not be able to retrieve it. if (!HasExplicitlyLoadedGraph(Configuration.Cache) && !Configuration.Schedule.ForceUseEngineInfoFromCache && Configuration.Distribution.BuildRole == DistributedBuildRoles.None) { Contract.Assume( graphFingerprint != null, "When looking up a cached graph on a distributed master or single-machine build, a graph fingerprint must be computed"); InputTracker.MatchResult engineCacheMatchResult = CheckIfAvailableInputsToGraphMatchPreviousRun( loggingContext, serializer, graphFingerprint: graphFingerprint, availableEnvironmentVariables: effectiveEnvironmentVariables, availableMounts: availableMounts, journalState: journalState, maxDegreeOfParallelism: maxDegreeOfParallelism); cacheGraphStats.ObjectDirectoryHit = engineCacheMatchResult.Matches; cacheGraphStats.ObjectDirectoryMissReason = engineCacheMatchResult.MissType; cacheGraphStats.MissReason = engineCacheMatchResult.MissType; cacheGraphStats.MissDescription = engineCacheMatchResult.FirstMissIdentifier; cacheGraphStats.WasHit = engineCacheMatchResult.Matches; cacheGraphStats.InputFilesChecked = engineCacheMatchResult.FilesChecked; // Checking the engine cache may have used a FileChangeTracker and provided information about // files/ContentHash pairs that were unchanged from the previous run. Hold onto this information as it may // be useful when eventually parsing files. // The FileChangeTracker is now up to date. All changed files have been removed from it. It can be reused // for a future build with the same fingerprint, though it may be tracking extra files, for example if // a spec was removed since the previous build. inputChanges = engineCacheMatchResult.InputChanges; } var shouldTryContentCache = !cacheGraphStats.WasHit && Configuration.Distribution.BuildRole != DistributedBuildRoles.Worker && Configuration.Cache.AllowFetchingCachedGraphFromContentCache && !HasExplicitlyLoadedGraph(Configuration.Cache) && (!Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.HasValue || !Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.Value); // ************************************************************ // 2. Content cache check: // ************************************************************ // * Single machine builds that missed earlier // * Distributed masters // This is the only valid place for the master to get a hit since it must be in the cache for the // workers to get it. if (shouldTryContentCache) { // Since an in-place match did not succeed, we need a readied cache to try again. // TODO: This logs an error if it fails. We are assuming some later thing will ensure that, if failed, // the engine fails overall. Possible <CacheInitializer> possibleCacheInitializerForFallback = cacheInitializationTask.GetAwaiter().GetResult(); if (possibleCacheInitializerForFallback.Succeeded) { CacheInitializer cacheInitializerForFallback = possibleCacheInitializerForFallback.Result; using (EngineCache cacheForFallback = cacheInitializerForFallback.CreateCacheForContext()) { var cacheGraphProvider = new CachedGraphProvider( loggingContext, Context, cacheForFallback, FileContentTable, maxDegreeOfParallelism); var cachedGraphDescriptor = cacheGraphProvider.TryGetPipGraphCacheDescriptorAsync(graphFingerprint, effectiveEnvironmentVariables, availableMounts.MountsByName).Result; if (cachedGraphDescriptor == null) { // There was no matching fingerprint in the cache. Record the status for logging before returning. cacheGraphStats.CacheMissReason = GraphCacheMissReason.FingerprintChanged; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } var fetchEngineScheduleContent = EngineSchedule.TryFetchFromCacheAsync( loggingContext, Context, cacheForFallback, cachedGraphDescriptor, serializer, FileContentTable, m_tempCleaner).Result; if (!fetchEngineScheduleContent) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoPreviousRunToCheck; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } // If a distributed master, take note of the graph fingerprint if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Master) { Contract.Assert(cachedGraphDescriptor != null); m_masterService.CachedGraphDescriptor = cachedGraphDescriptor; } Logger.Log.FetchedSerializedGraphFromCache(outerLoggingContext); cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; cacheGraphStats.WasHit = true; } } else { cacheGraphStats.CacheMissReason = GraphCacheMissReason.CacheFailure; SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason); return(cacheGraphStats); } } // ************************************************************ // 3. Query distributed master // ************************************************************ // * Distributed workers only if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Worker) { Contract.Assume( graphFingerprint == null, "Distributed workers should request a graph fingerprint from the master (not compute one locally)"); Possible <CacheInitializer> possibleCacheInitializerForWorker = cacheInitializationTask.GetAwaiter().GetResult(); Contract.Assume(possibleCacheInitializerForWorker.Succeeded, "Workers must have a valid cache"); CacheInitializer cacheInitializerForWorker = possibleCacheInitializerForWorker.Result; using (EngineCache cacheForWorker = cacheInitializerForWorker.CreateCacheForContext()) { PipGraphCacheDescriptor schedulerStateDescriptor; if (!m_workerService.TryGetBuildScheduleDescriptor(out schedulerStateDescriptor) || !EngineSchedule.TryFetchFromCacheAsync( outerLoggingContext, Context, cacheForWorker, schedulerStateDescriptor, serializer, FileContentTable, m_tempCleaner).Result) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; return(cacheGraphStats); } AsyncOut <AbsolutePath> symlinkFileLocation = new AsyncOut <AbsolutePath>(); if (!SymlinkDefinitionFileProvider.TryFetchWorkerSymlinkFileAsync( outerLoggingContext, Context.PathTable, cacheForWorker, Configuration.Layout, m_workerService, symlinkFileLocation).Result) { cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster; cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason; return(cacheGraphStats); } m_workerSymlinkDefinitionFile = symlinkFileLocation.Value; // Success. Populate the stats cacheGraphStats.WasHit = true; cacheGraphStats.WorkerHit = true; cacheGraphStats.MissReason = GraphCacheMissReason.NoMiss; cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss; } } } return(cacheGraphStats); }
/// <summary> /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified. /// </summary> public DummyPipExecutionEnvironment( LoggingContext loggingContext, PipExecutionContext context, IConfiguration config, FileContentTable fileContentTable = null, EngineCache pipCache = null, SemanticPathExpander semanticPathExpander = null, PipContentFingerprinter.PipDataLookup pipDataLookup = null, FileAccessWhitelist fileAccessWhitelist = null, bool allowUnspecifiedSealedDirectories = false, PipTable pipTable = null, IIpcProvider ipcProvider = null, IKextConnection sandboxedKextConnection = null) { Contract.Requires(context != null); Contract.Requires(config != null); LoggingContext = loggingContext; Context = context; // Ensure paths visible when debugging PathTable.DebugPathTable = Context.PathTable; Configuration = config; PipTable = pipTable; PathExpander = semanticPathExpander ?? SemanticPathExpander.Default; ContentFingerprinter = new PipContentFingerprinter( Context.PathTable, artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo, new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null), pathExpander: PathExpander, pipDataLookup: pipDataLookup); PipFragmentRenderer = this.CreatePipFragmentRenderer(); IpcProvider = ipcProvider ?? IpcFactory.GetProvider(); FileContentTable = fileContentTable ?? FileContentTable.CreateNew(); Cache = pipCache; FileAccessWhitelist = fileAccessWhitelist; m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories; m_sandboxedKextConnection = sandboxedKextConnection; if (Cache == null) { Cache = InMemoryCacheFactory.Create(context); } var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext); LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker); PipGraphView = new TestPipGraphFilesystemView(Context.PathTable); m_operationTracker = new OperationTracker(loggingContext); var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore); var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed; if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled) { preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString()); } State = new PipExecutionState( config, cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander), fileAccessWhitelist: FileAccessWhitelist, directoryMembershipFingerprinter: this, pathExpander: PathExpander, executionLog: ExecutionLogRecorder, fileSystemView: fileSystemView, fileContentManager: GetFileContentManager(), directoryMembershipFinterprinterRuleSet: null, unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration, preserveOutputsSalt: preserveOutputsSalt, serviceManager: new DummyServiceManager()); m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>(); ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable); }