/// <summary>
/// Constructor.
/// </summary>
private FingerprintStoreExecutionLogTarget(
LoggingContext loggingContext,
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
FingerprintStore fingerprintStore,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_context = context;
m_pipTable = pipTable;
PipContentFingerprinter = pipContentFingerprinter;
FingerprintStore = fingerprintStore;
m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>();
m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync(
this,
loggingContext,
context,
configuration,
cache,
graph,
runnablePipPerformance);
}
private WeakContentFingerprint GenerateSaltedWeakFingerprint(ContentHash hash) => new WeakContentFingerprint(FingerprintUtilities.Hash($"Hash: '{hash.ToHex()}' Salt: '{m_buildManifestHashCacheSalt}'")); // Changes to this string will invalidate all existing cache entries
/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config,
EngineCache engineCache,
Tracing.IExecutionLogTarget executionLog,
Tracing.BuildManifestGenerator buildManifestGenerator)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
Contract.Requires(engineCache != null);
Contract.Requires(executionLog != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
m_engineCache = engineCache;
m_executionLog = executionLog;
m_buildManifestGenerator = buildManifestGenerator;
m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, ContentHash>();
m_buildManifestHashCacheSalt = string.IsNullOrEmpty(Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt)
? string.Empty
: Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt;
}
/// <summary>
/// Constructor.
/// </summary>
private FingerprintStoreExecutionLogTarget(
LoggingContext loggingContext,
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
FingerprintStore fingerprintStore,
FingerprintStore cacheLookupFingerprintStore,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_context = context;
m_pipTable = pipTable;
LoggingContext = loggingContext;
PipContentFingerprinter = pipContentFingerprinter;
ExecutionFingerprintStore = fingerprintStore;
CacheLookupFingerprintStore = cacheLookupFingerprintStore;
// Cache lookup store is per-build state and doesn't need to be garbage collected (vs. execution fignerprint store which is persisted build-over-build)
CacheLookupFingerprintStore?.GarbageCollectCancellationToken.Cancel();
m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>();
Counters = counters;
FingerprintStoreMode = configuration.Logging.FingerprintStoreMode;
m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync(
this,
loggingContext,
context,
configuration,
cache,
graph,
runnablePipPerformance);
Contract.Assume(FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly || CacheLookupFingerprintStore != null, "Unless /storeFingerprints flag is set to /storeFingerprints:ExecutionFingerprintsOnly, the cache lookup FingerprintStore must exist.");
}
/// <summary>
/// Retrieve the running time table from the cache
/// </summary>
public static async Task <Possible <bool> > TryRetrieveRunningTimeTableAsync(
this EngineCache cache,
LoggingContext loggingContext,
string path,
PathTable pathTable,
ContentFingerprint graphSemistableFingerprint,
DateTime?time = null)
{
var possibleCacheEntry =
await cache.TwoPhaseFingerprintStore.TryGetLatestCacheEntryAsync(loggingContext, graphSemistableFingerprint, time);
if (!possibleCacheEntry.Succeeded)
{
Logger.Log.PerformanceDataCacheTrace(
loggingContext,
I($"Failed loading running time table entry from cache: Failure:{possibleCacheEntry.Failure.DescribeIncludingInnerFailures()}"));
return(possibleCacheEntry.Failure);
}
Logger.Log.PerformanceDataCacheTrace(
loggingContext,
I($"Loaded running time table entry from cache: Fingerprint='{graphSemistableFingerprint}' MetadataHash={possibleCacheEntry.Result?.MetadataHash ?? ContentHashingUtilities.ZeroHash}"));
if (!possibleCacheEntry.Result.HasValue)
{
return(false);
}
var runningTimeTableHash = possibleCacheEntry.Result.Value.MetadataHash;
var absolutePath = AbsolutePath.Create(pathTable, path);
var maybePinned = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { runningTimeTableHash });
var result = await maybePinned.ThenAsync(
async pinResult =>
{
if (!pinResult.AllContentAvailable)
{
return(new Failure <string>(I($"Could not pin content for running time table '{runningTimeTableHash}'")));
}
return(await cache.ArtifactContentCache.TryMaterializeAsync(
FileRealizationMode.Copy,
absolutePath.Expand(pathTable),
runningTimeTableHash));
});
if (!result.Succeeded)
{
Logger.Log.PerformanceDataCacheTrace(
loggingContext,
I($"Failed loading running time table from cache: Failure:{result.Failure.DescribeIncludingInnerFailures()}"));
return(result.Failure);
}
Logger.Log.PerformanceDataCacheTrace(loggingContext, I($"Loaded running time table from cache: Path='{path}'"));
return(true);
}
private static Scheduler CreateInternal(
PipExecutionContext context,
PipGraph pipGraph,
IPipQueue queue,
EngineCache cache,
IConfiguration configuration)
{
Contract.Requires(context != null);
Contract.Requires(queue != null);
Contract.Requires(cache != null);
Contract.Requires(configuration != null);
var fileContentTable = FileContentTable.CreateNew();
var fileAccessWhiteList = new FileAccessWhitelist(context);
var testHooks = new SchedulerTestHooks();
return(new Scheduler(
pipGraph,
queue,
context,
fileContentTable,
cache: cache,
loggingContext: Events.StaticContext,
configuration: configuration,
fileAccessWhitelist: fileAccessWhiteList,
testHooks: testHooks,
buildEngineFingerprint: null));
}
/// <summary>
/// Store the running time table to the cache
/// </summary>
public static async Task <Possible <Unit> > TryStoreRunningTimeTableAsync(
this EngineCache cache,
LoggingContext loggingContext,
string path,
PathTable pathTable,
ContentFingerprint performanceDataFingerprint,
DateTime?storeTime = null)
{
var absolutePath = AbsolutePath.Create(pathTable, path);
var storeResult = await cache.ArtifactContentCache.TryStoreAsync(
FileRealizationMode.Copy,
absolutePath.Expand(pathTable));
Logger.Log.HistoricPerfDataCacheTrace(loggingContext, I($"Storing running time table to cache: Success='{storeResult.Succeeded}'"));
if (!storeResult.Succeeded)
{
return(storeResult.Failure);
}
ContentHash hash = storeResult.Result;
var cacheEntry = new CacheEntry(hash, null, ArrayView <ContentHash> .Empty);
var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, performanceDataFingerprint, cacheEntry, storeTime);
Logger.Log.HistoricPerfDataCacheTrace(loggingContext, I($"Publishing running time table from cache: Fingerprint='{performanceDataFingerprint}' Hash={hash}"));
return(publishResult);
}
public static Tuple <Scheduler, EngineCache> CreateWithCaching(
PipExecutionContext context,
LoggingContext loggingContext,
IConfiguration configuration,
PipGraph.Builder graphBuilder,
IPipQueue queue)
{
Contract.Requires(graphBuilder != null);
Contract.Requires(context != null);
Contract.Requires(queue != null);
var cacheLayer = new EngineCache(
new InMemoryArtifactContentCache(),
new InMemoryTwoPhaseFingerprintStore());
Scheduler scheduler = CreateInternal(
context,
loggingContext,
graphBuilder.Build(),
queue,
cacheLayer,
configuration);
return(Tuple.Create(scheduler, cacheLayer));
}
private static Scheduler CreateInternal(
PipExecutionContext context,
PipGraph pipGraph,
IPipQueue queue,
EngineCache cache,
IConfiguration configuration)
{
Contract.Requires(context != null);
Contract.Requires(queue != null);
Contract.Requires(cache != null);
Contract.Requires(configuration != null);
var fileContentTable = FileContentTable.CreateNew();
var fileAccessWhiteList = new FileAccessWhitelist(context);
var testHooks = new SchedulerTestHooks();
return(new Scheduler(
pipGraph,
queue,
context,
fileContentTable,
cache: cache,
loggingContext: Events.StaticContext,
configuration: configuration,
fileAccessWhitelist: fileAccessWhiteList,
testHooks: testHooks,
buildEngineFingerprint: null,
tempCleaner: new TestMoveDeleteCleaner(Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "MoveDeletionTemp"))));
}
/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config,
EngineCache engineCache,
Tracing.IExecutionLogTarget executionLog,
Tracing.BuildManifestGenerator buildManifestGenerator)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
Contract.Requires(engineCache != null);
Contract.Requires(executionLog != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
m_engineCache = engineCache;
m_executionLog = executionLog;
m_buildManifestGenerator = buildManifestGenerator;
m_inMemoryBuildManifestStore = new ConcurrentDictionary <ContentHash, ContentHash>();
}
private FrontEndHostController CreateHost()
{
var factory = new FrontEndFactory();
factory.AddFrontEnd(new DummyFrontEnd1());
factory.TrySeal(new LoggingContext("UnitTest"));
var controller = new FrontEndHostController(factory, new DScriptWorkspaceResolverFactory(), new EvaluationScheduler(degreeOfParallelism: 8), collectMemoryAsSoonAsPossible: false);
var context = BuildXLContext.CreateInstanceForTesting();
var fileSystem = new InMemoryFileSystem(context.PathTable);
((IFrontEndController)controller).InitializeHost(
new FrontEndContext(context, new LoggingContext("UnitTest"), fileSystem),
new ConfigurationImpl()
{
FrontEnd = new FrontEndConfiguration()
{
MaxFrontEndConcurrency = 1,
}
});
var inMemoryCache = new EngineCache(
new InMemoryArtifactContentCache(context),
new InMemoryTwoPhaseFingerprintStore());
controller.InitializeInternalForTesting(
Task.FromResult(new Possible <EngineCache>(inMemoryCache)),
AbsolutePath.Create(context.PathTable, TestOutputDirectory));
return(controller);
}
EngineContainer LoadFromCache(string key)
{
if (EngineCache == null || !EngineCache.ContainsKey(key))
{
return(null);
}
return(EngineCache[key]);
}
public void GetEngine_PYandRBEngine_CachedEnginesCountEquals2()
{
LanguageSettings ruby = Helper.CreateIronRubySettings();
LanguageSettings python = Helper.CreateIronPythonSettings();
EngineCache cache = new EngineCache();
cache.GetEngine(ruby, null);
cache.GetEngine(python, null);
Assert.AreEqual(2, cache.CachedEngines);
}
/// <summary>
/// Tries to fetch symlink file from the cache.
/// </summary>
public static async Task <bool> TryFetchWorkerSymlinkFileAsync(
LoggingContext loggingContext,
PathTable pathTable,
EngineCache cache,
ILayoutConfiguration layoutConfiguration,
WorkerService workerService,
AsyncOut <AbsolutePath> symlinkPathAsyncOut)
{
Contract.Requires(loggingContext != null);
Contract.Requires(pathTable != null);
Contract.Requires(cache != null);
Contract.Requires(workerService != null);
Contract.Requires(symlinkPathAsyncOut != null);
symlinkPathAsyncOut.Value = AbsolutePath.Invalid;
var symlinkFileContentHash = workerService.BuildStartData.SymlinkFileContentHash.ToContentHash();
if (symlinkFileContentHash == WellKnownContentHashes.AbsentFile)
{
// Absent file meaning there is no symlink file to use
return(true);
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Attempting to retrieve symlink file with hash '{symlinkFileContentHash}'."));
var maybeLoaded = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { symlinkFileContentHash });
if (!maybeLoaded.Succeeded)
{
Tracing.Logger.Log.FailedLoadSymlinkFileFromCache(loggingContext, maybeLoaded.Failure.DescribeIncludingInnerFailures());
return(false);
}
var destinationPath = layoutConfiguration.EngineCacheDirectory.Combine(pathTable, SymlinkFileName).Expand(pathTable);
var materializedFile =
await
cache.ArtifactContentCache.TryMaterializeAsync(
FileRealizationMode.HardLinkOrCopy,
destinationPath,
symlinkFileContentHash);
if (!materializedFile.Succeeded)
{
Tracing.Logger.Log.FailedMaterializeSymlinkFileFromCache(
loggingContext,
destinationPath.ExpandedPath,
materializedFile.Failure.DescribeIncludingInnerFailures());
return(false);
}
Logger.Log.SymlinkFileTraceMessage(loggingContext, I($"Symlink file with hash '{symlinkFileContentHash}' materialized to location '{destinationPath}'."));
symlinkPathAsyncOut.Value = destinationPath.Path;
return(true);
}
public void AppendPathToEngines_ValidPath_EngineHavePathAdded()
{
string path = Environment.CurrentDirectory;
LanguageSettings ruby = Helper.CreateIronRubySettings();
EngineCache cache = new EngineCache();
cache.AppendPathToEngines(path);
IEngine engine = cache.GetEngine(ruby, null);
bool found = EngineHasPath(path, engine);
Assert.IsTrue(found);
}
public TestScheduler(
PipGraph graph,
TestPipQueue pipQueue,
PipExecutionContext context,
FileContentTable fileContentTable,
EngineCache cache,
IConfiguration configuration,
FileAccessWhitelist fileAccessWhitelist,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null,
ITempCleaner tempCleaner = null,
PipRuntimeTimeTable runningTimeTable = null,
JournalState journalState = null,
PerformanceCollector performanceCollector = null,
string fingerprintSalt = null,
PreserveOutputsInfo?previousInputsSalt = null,
IEnumerable <Pip> successfulPips = null,
IEnumerable <Pip> failedPips = null,
LoggingContext loggingContext = null,
IIpcProvider ipcProvider = null,
DirectoryTranslator directoryTranslator = null,
VmInitializer vmInitializer = null,
SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache,
configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules,
tempCleaner, AsyncLazy <PipRuntimeTimeTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt,
ipcProvider: ipcProvider,
directoryTranslator: directoryTranslator,
journalState: journalState,
vmInitializer: vmInitializer,
testHooks: testHooks)
{
m_testPipQueue = pipQueue;
if (successfulPips != null)
{
foreach (var pip in successfulPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded);
}
}
if (failedPips != null)
{
foreach (var pip in failedPips)
{
Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler");
m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed);
}
}
m_loggingContext = loggingContext;
}
/// <nodoc />
public PipTwoPhaseCache(LoggingContext loggingContext, EngineCache cache, PipExecutionContext context, PathExpander pathExpander)
{
Contract.Requires(loggingContext != null);
Contract.Requires(cache != null);
Contract.Requires(context != null);
LoggingContext = loggingContext;
ArtifactContentCache = cache.ArtifactContentCache;
TwoPhaseFingerprintStore = cache.TwoPhaseFingerprintStore;
Context = context;
Counters = new CounterCollection <PipCachingCounter>();
m_pathExpander = pathExpander;
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
(string substSource, string substTarget)?subst = default,
/// <nodoc/>
public SchedulerIntegrationTestBase(ITestOutputHelper output) : base(output)
{
m_testOutputHelper = output;
CaptureAllDiagnosticMessages = false;
// Each event listener that we want to capture events from must be listed here
foreach (var eventSource in BuildXLApp.GeneratedEventSources)
{
RegisterEventSource(eventSource);
}
// Go through the command line configuration handling used by the real app to get the appropriate defaults
ICommandLineConfiguration config;
XAssert.IsTrue(Args.TryParseArguments(new[] { "/c:" + Path.Combine(TemporaryDirectory, "config.dc") }, Context.PathTable, null, out config), "Failed to construct arguments");
Configuration = new CommandLineConfiguration(config);
Cache = InMemoryCacheFactory.Create();
FileContentTable = FileContentTable.CreateNew(LoggingContext);
// Disable defaults that write disk IO but are not critical to correctness or performance
Configuration.Logging.StoreFingerprints = false;
Configuration.Logging.LogExecution = false;
Configuration.Engine.TrackBuildsInUserFolder = false;
Configuration.Engine.CleanTempDirectories = false;
Configuration.Sandbox.EnsureTempDirectoriesExistenceBeforePipExecution = true;
// Skip hash source file pips
// Some error becomes a contract exception when SkipHashSourceFile is enabled.
// Mostly on methods that traverse graph, e.g., DirectedGraph.GetOutgoingEdges();
// BUG 1472567
// Configuration.Schedule.SkipHashSourceFile = true;
// Compute static pip fingerprints for incremental scheduling tests.
Configuration.Schedule.ComputePipStaticFingerprints = true;
// Disable currently enabled unsafe option.
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.IgnoreCreateProcessReport = false;
// Populate file system capabilities.
// Here, for example, we use copy-on-write instead of hardlinks when Unix file system supports copy-on-write.
// Particular tests can override this by setting Configuration.Engine.UseHardlinks.
BuildXLEngine.PopulateFileSystemCapabilities(Configuration, Configuration, Context.PathTable, LoggingContext);
// Reset pip graph builder to use the populated configuration.
ResetPipGraphBuilder();
}
public void AppendPathToEngines_ValidPath_2EnginesHavePathAdded()
{
string path = Environment.CurrentDirectory;
LanguageSettings ruby = Helper.CreateIronRubySettings();
LanguageSettings python = Helper.CreateIronPythonSettings();
EngineCache cache = new EngineCache();
cache.AppendPathToEngines(path);
IEngine rubyEngine = cache.GetEngine(ruby, null);
IEngine pythonEngine = cache.GetEngine(python, null);
bool rubyFound = EngineHasPath(path, rubyEngine);
bool pythonFound = EngineHasPath(path, pythonEngine);
Assert.IsTrue(rubyFound);
Assert.IsTrue(pythonFound);
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
string fingerprintStoreDirectory,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleStore = FingerprintStore.Open(
fingerprintStoreDirectory,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
testHooks: testHooks);
if (possibleStore.Succeeded)
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleStore.Result,
configuration,
cache,
graph,
runnablePipPerformance));
}
else
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleStore.Failure.DescribeIncludingInnerFailures());
}
return(null);
}
/// <summary>
/// Run the test
/// </summary>
public bool Run(string testFolder, string specFile, string fullIdentifier, string shortName, string lkgFile, params string[] sdksToResolve)
{
Contract.Requires(!string.IsNullOrEmpty(testFolder));
Contract.Requires(!string.IsNullOrEmpty(specFile));
Contract.Requires(sdksToResolve != null);
// Sadly the frontend doesn't use the engine abstractions file api's so we have to materialize stuff on disk for now...
// TODO: Fix this code once the frontend supports a proper virtual FileSystem.
// TODO: Change the package semantics to implicit when we expose a way to evaluate a single value
var testFileName = Path.GetFileName(specFile);
var mainFileName = "testMain.bp";
var testMainFile = Path.Combine(testFolder, mainFileName);
Directory.CreateDirectory(testFolder);
File.WriteAllText(Path.Combine(testFolder, Names.ModuleConfigBm), I($@"module(
{{
name: 'TestPackage',
nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences,
projects: [
f`{mainFileName}`,
f`{testFileName}`,
],
}});"));
File.WriteAllText(testMainFile, I($@"
export const testFolder = d`{Path.GetDirectoryName(specFile).Replace('\\', '/')}`;
@@public
export const main = {fullIdentifier}();"));
File.Copy(specFile, Path.Combine(testFolder, testFileName));
// Create a fake package for Sdk.TestRunner so that you can safely test packages that have the tests embedded in them.
var testRunnerFolder = Path.Combine(testFolder, "Sdk.TestRunner");
Directory.CreateDirectory(testRunnerFolder);
File.WriteAllText(Path.Combine(testRunnerFolder, Names.ModuleConfigBm), I($"module({{\n\tname: 'Sdk.TestRunner',\n}});"));
File.WriteAllText(Path.Combine(testRunnerFolder, "package" + Names.DotDscExtension), I($@"
export interface TestArguments {{
testFiles: File[];
sdkFolders?: (Directory|StaticDirectory)[];
autoFixLkgs?: boolean;
}}
export interface TestResult {{
xmlResults: File;
}}
export function test(args: TestArguments): TestResult {{
Contract.fail(""Can't run a DScript UnitTest inside of a DScript UnitTest"");
}}"));
// Setup Context and configuration
var frontEndContext = FrontEndContext.CreateInstanceForTesting();
var pipContext = new SchedulerContext(CancellationToken.None, frontEndContext.StringTable, frontEndContext.PathTable, frontEndContext.SymbolTable, frontEndContext.QualifierTable);
var pathTable = frontEndContext.PathTable;
var testFolderPath = AbsolutePath.Create(pathTable, testFolder);
var configuration = CreateConfiguration(sdksToResolve.Union(new[] { testRunnerFolder }), pathTable, testFolderPath);
var engineAbstraction = new TestEngineAbstraction(pathTable, frontEndContext.StringTable, testFolderPath, new PassThroughFileSystem(pathTable));
var frontEndStatistics = new FrontEndStatistics();
if (!CreateFactories(
frontEndContext,
engineAbstraction,
frontEndStatistics,
configuration,
out var ambientTesting,
out var moduleRegistry,
out var frontEndFactory))
{
return(false);
}
// Set the timeout to a large number to avoid useless performance collections in tests.
using (var performanceCollector = new PerformanceCollector(TimeSpan.FromHours(1)))
using (var frontEndHostController = new FrontEndHostController(
frontEndFactory,
new EvaluationScheduler(1),
moduleRegistry,
frontEndStatistics,
m_tracingLogger,
performanceCollector,
collectMemoryAsSoonAsPossible: true))
{
var frontEndController = (IFrontEndController)frontEndHostController;
frontEndController.InitializeHost(frontEndContext, configuration);
frontEndController.ParseConfig(configuration);
// Populate the graph
using (var pipTable = new PipTable(
pipContext.PathTable,
pipContext.SymbolTable,
initialBufferSize: 16384,
maxDegreeOfParallelism: 1,
debug: true))
{
var mountPathExpander = new MountPathExpander(pathTable);
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "testFolder"), testFolderPath, allowHashing: true, readable: true, writable: false));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "src"), testFolderPath.Combine(pathTable, "src"), allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "out"), testFolderPath.Combine(pathTable, "out"), allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "noRead"), testFolderPath.Combine(pathTable, "noRead"), allowHashing: true, readable: false, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "temp"), engineAbstraction.Layout.TempDirectory, allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "obj"), engineAbstraction.Layout.ObjectDirectory, allowHashing: true, readable: true, writable: true));
var graph = new PipGraph.Builder(
pipTable,
pipContext,
m_pipLogger,
frontEndContext.LoggingContext,
configuration,
mountPathExpander);
using (var cacheLayer = new EngineCache(
new InMemoryArtifactContentCache(),
new InMemoryTwoPhaseFingerprintStore()))
{
var cache = Task.FromResult(Possible.Create(cacheLayer));
try
{
var evaluationFilter = new EvaluationFilter(
pipContext.SymbolTable,
pipContext.PathTable,
new FullSymbol[0],
new[]
{
AbsolutePath.Create(frontEndContext.PathTable, testMainFile),
},
CollectionUtilities.EmptyArray <StringId>());
if (!frontEndController.PopulateGraph(cache, graph, engineAbstraction, evaluationFilter, configuration, configuration.Startup))
{
HandleDiagnostics();
return(false);
}
}
catch (AggregateException e)
{
var baseException = e.GetBaseException();
if (baseException is XunitException)
{
// If it is an XUnit assert, then unwrap the exception and throw that because XUnit other doesn't display the error nicely.
ExceptionDispatchInfo.Capture(baseException).Throw();
}
throw;
}
}
if (!ValidatePips(frontEndContext, graph, testFolderPath, specFile, shortName, lkgFile, ambientTesting.DontValidatePipsEnabled))
{
return(false);
}
}
}
HandleDiagnostics();
return(true);
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
FingerprintStoreTestHooks testHooks = null)
{
// Unblock caller
await Task.Yield();
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
string downLoadedPriviousFingerprintStoreSavedPath = null;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
PathTable newPathTable = new PathTable();
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var fingerprintsLogDirectoryStr = configuration.Logging.FingerprintsLogDirectory.ToString(context.PathTable);
var fingerprintsLogDirectory = AbsolutePath.Create(newPathTable, fingerprintsLogDirectoryStr);
var cacheSavePath = fingerprintsLogDirectory.Combine(newPathTable, Scheduler.FingerprintStoreDirectory + "." + key);
var result = await cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, newPathTable, key, configuration, context.CancellationToken);
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(newPathTable);
downLoadedPriviousFingerprintStoreSavedPath = path;
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(
logTarget,
loggingContext,
context,
possibleStore.Result,
graph,
runnablePipPerformance,
configuration,
downLoadedPriviousFingerprintStoreSavedPath,
testHooks: testHooks));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}
public void GetEngine_PYEngine_TwoCalls_SameObjectReturned()
{
LanguageSettings python = Helper.CreateIronPythonSettings();
EngineCache cache = new EngineCache();
IEngine engine1 = cache.GetEngine(python, null);
IEngine engine2 = cache.GetEngine(python, null);
Assert.AreEqual(engine1, engine2);
Assert.AreEqual(1, cache.CachedEngines);
}
public void GetEngine_PYEngine_CachedEnginesCountEquals1()
{
LanguageSettings python = Helper.CreateIronPythonSettings();
EngineCache cache = new EngineCache();
cache.GetEngine(python, null);
Assert.AreEqual(1, cache.CachedEngines);
}
/// <summary>
/// Retrieve the fingerprint store from the cache
/// </summary>
public static async Task <Possible <bool> > TryRetrieveFingerprintStoreAsync(
this EngineCache cache,
LoggingContext loggingContext,
AbsolutePath path,
PathTable pathTable,
string key,
string environment)
{
var fingerprint = ComputeFingerprint(pathTable, key, environment);
var possibleCacheEntry = await cache.TwoPhaseFingerprintStore.TryGetLatestCacheEntryAsync(loggingContext, fingerprint);
if (!possibleCacheEntry.Succeeded)
{
Logger.Log.GettingFingerprintStoreTrace(
loggingContext,
I($"Failed loading fingerprint store cache entry from cache: Key='{key}' Failure: {possibleCacheEntry.Failure.DescribeIncludingInnerFailures()}"));
return(possibleCacheEntry.Failure);
}
Logger.Log.GettingFingerprintStoreTrace(
loggingContext,
I($"Loaded fingerprint store entry from cache: Key='{key}' Fingerprint='{fingerprint}' MetadataHash='{possibleCacheEntry.Result?.MetadataHash ?? ContentHashingUtilities.ZeroHash}'"));
if (!possibleCacheEntry.Result.HasValue)
{
return(false);
}
var fingerprintStoreDescriptorHash = possibleCacheEntry.Result.Value.MetadataHash;
var maybePinned = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(possibleCacheEntry.Result.Value.ToArray());
var result = await maybePinned.ThenAsync <Unit>(
async pinResult =>
{
if (!pinResult.AllContentAvailable)
{
return(new Failure <string>(I($"Could not pin content for fingerprint store '{string.Join(", ", pinResult.Results.Where(r => !r.IsAvailable).Select(r => r.Hash))}'")));
}
var maybeLoadedDescriptor = await cache.ArtifactContentCache.TryLoadAndDeserializeContent <PackageDownloadDescriptor>(fingerprintStoreDescriptorHash);
if (!maybeLoadedDescriptor.Succeeded)
{
return(maybeLoadedDescriptor.Failure);
}
Logger.Log.GettingFingerprintStoreTrace(
loggingContext,
I($"Loaded fingerprint store cache descriptor from cache: Key='{key}' Hash='{fingerprintStoreDescriptorHash}'"));
PackageDownloadDescriptor descriptor = maybeLoadedDescriptor.Result;
SemaphoreSlim concurrencyLimiter = new SemaphoreSlim(8);
var materializedFiles = await Task.WhenAll(descriptor.Contents.Select(async subPathKeyedHash =>
{
await Task.Yield();
using (await concurrencyLimiter.AcquireAsync())
{
var filePath = path.Combine(pathTable, subPathKeyedHash.Key);
var maybeMaterialized = await cache.ArtifactContentCache.TryMaterializeAsync(
FileRealizationMode.Copy,
filePath.Expand(pathTable),
subPathKeyedHash.ContentHash.ToContentHash());
return(maybeMaterialized);
}
}).ToList());
var failure = materializedFiles.Where(p => !p.Succeeded).Select(p => p.Failure).FirstOrDefault();
if (failure != null)
{
return(failure);
}
return(Unit.Void);
});
if (!result.Succeeded)
{
Logger.Log.GettingFingerprintStoreTrace(
loggingContext,
I($"Failed loading fingerprint store from cache: Key='{key}' Failure: {result.Failure.DescribeIncludingInnerFailures()}"));
return(result.Failure);
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Loaded fingerprint store from cache: Key='{key}' Path='{path.ToString(pathTable)}'"));
return(true);
}
/// <summary>
/// Store the fingerprint store directory to the cache
/// </summary>
public static async Task <Possible <long> > TrySaveFingerprintStoreAsync(
this EngineCache cache,
LoggingContext loggingContext,
AbsolutePath path,
PathTable pathTable,
string key,
string environment)
{
var fingerprint = ComputeFingerprint(pathTable, key, environment);
var pathStr = path.ToString(pathTable);
BoxRef <long> size = 0;
SemaphoreSlim concurrencyLimiter = new SemaphoreSlim(8);
var tasks = new List <Task <Possible <StringKeyedHash, Failure> > >();
FileUtilities.EnumerateDirectoryEntries(pathStr, (name, attr) =>
{
var task = Task.Run(async() =>
{
using (await concurrencyLimiter.AcquireAsync())
{
var filePath = path.Combine(pathTable, name);
var storeResult = await cache.ArtifactContentCache.TryStoreAsync(
FileRealizationMode.Copy,
filePath.Expand(pathTable));
if (storeResult.Succeeded)
{
Interlocked.Add(ref size.Value, new FileInfo(filePath.ToString(pathTable)).Length);
}
return(storeResult.Then(result => new StringKeyedHash()
{
Key = path.ExpandRelative(pathTable, filePath),
ContentHash = result.ToBondContentHash()
}));
}
});
tasks.Add(task);
});
var storedFiles = await Task.WhenAll(tasks);
var failure = storedFiles.Where(p => !p.Succeeded).Select(p => p.Failure).FirstOrDefault();
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store to cache: Success='{failure == null}', FileCount={storedFiles.Length} Size={size.Value}"));
if (failure != null)
{
return(failure);
}
PackageDownloadDescriptor descriptor = new PackageDownloadDescriptor()
{
TraceInfo = loggingContext.Session.Environment,
FriendlyName = nameof(FingerprintStore),
Contents = storedFiles.Select(p => p.Result).ToList()
};
var storeDescriptorResult = await cache.ArtifactContentCache.TrySerializeAndStoreContent(descriptor);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Saving fingerprint store descriptor to cache: Success='{storeDescriptorResult.Succeeded}'"));
if (!storeDescriptorResult.Succeeded)
{
return(storeDescriptorResult.Failure);
}
var associatedFileHashes = descriptor.Contents.Select(s => s.ContentHash.ToContentHash()).ToArray().ToReadOnlyArray().GetSubView(0);
var cacheEntry = new CacheEntry(storeDescriptorResult.Result, null, associatedFileHashes);
var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, fingerprint, cacheEntry);
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Publishing fingerprint store to cache: Fingerprint='{fingerprint}' Hash={storeDescriptorResult.Result}"));
return(size.Value);
}
public void GetEngine_RBEngine_CreatesEngine_ReturnsObject()
{
LanguageSettings ruby = Helper.CreateIronRubySettings();
EngineCache cache = new EngineCache();
IEngine engine = cache.GetEngine(ruby, null);
Assert.AreEqual("IronRuby", engine.LanguageName);
}
public void GetEngine_PYandRBEngine_TwoDifferentObjectsReturned()
{
LanguageSettings ruby = Helper.CreateIronRubySettings();
LanguageSettings python = Helper.CreateIronPythonSettings();
EngineCache cache = new EngineCache();
IEngine rubyEngine = cache.GetEngine(ruby, null);
IEngine pythonEngine = cache.GetEngine(python, null);
Assert.AreNotEqual(rubyEngine, pythonEngine);
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var cacheSavePath = configuration.Logging.FingerprintsLogDirectory
.Combine(context.PathTable, Scheduler.FingerprintStoreDirectory + "." + key);
#pragma warning disable AsyncFixer02 // This should explicitly happen synchronously since it interacts with the PathTable and StringTable
var result = cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, context.PathTable, key, configuration.Schedule.EnvironmentFingerprint).Result;
#pragma warning restore AsyncFixer02
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(context.PathTable);
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
// Unblock caller
// WARNING: The rest can simultenously happen with saving the graph files to disk.
// We should not create any paths or strings by using PathTable and StringTable.
await Task.Yield();
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}
public void GetEngine_PYEngine_ByLanguageSettingsObject_ReturnsDLREngine()
{
LanguageSettings python = Helper.CreateIronPythonSettings();
EngineCache cache = new EngineCache();
IEngine engine = cache.GetEngine(python, null);
Assert.AreEqual("IronPython 2.0 Beta", engine.LanguageName);
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable);
var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable);
try
{
FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleExecutionStore = FingerprintStore.Open(
fingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet.");
if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly)
{
try
{
FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
possibleCacheLookupStore = FingerprintStore.Open(
cacheLookupFingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
}
if (possibleExecutionStore.Succeeded &&
(possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly))
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleExecutionStore.Result,
possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null,
configuration,
cache,
graph,
counters,
runnablePipPerformance));
}
else
{
if (!possibleExecutionStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures());
}
if (!possibleCacheLookupStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures());
}
}
return(null);
}
/// <summary>
/// Check if pip graph can be reused.
///
/// There are 3 opportunities to determine a graph match. The applicability of each depends on the distributed build roles.
/// (1) from engine cache,
/// (2) from content cache, and
/// (3) from master node (if running on a worker node in a distributed build)
/// </summary>
private GraphCacheCheckStatistics CheckGraphCacheReuse(
LoggingContext outerLoggingContext,
int maxDegreeOfParallelism,
GraphFingerprint graphFingerprint,
IReadOnlyDictionary <string, string> properties,
CacheInitializationTask cacheInitializationTask,
JournalState journalState,
out EngineSerializer serializer,
out InputTracker.InputChanges inputChanges)
{
serializer = CreateEngineSerializer(outerLoggingContext);
inputChanges = null;
var cacheGraphStats = default(GraphCacheCheckStatistics);
using (var timeBlock = TimedBlock <EmptyStruct, GraphCacheCheckStatistics> .Start(
outerLoggingContext,
Statistics.GraphCacheReuseCheck,
(context, emptyStruct) => Logger.Log.CheckingForPipGraphReuseStart(context),
default(EmptyStruct),
(loggingContext, stats) =>
{
Logger.Log.CheckingForPipGraphReuseComplete(loggingContext, stats);
// On misses we want to give the user a message for why there was a miss
if (!stats.WasHit)
{
Contract.Assume(stats.MissReason != GraphCacheMissReason.NoMiss);
Logger.Log.GraphNotReusedDueToChangedInput(loggingContext, stats.MissMessageForConsole, stats.MissDescription);
}
m_enginePerformanceInfo.GraphCacheCheckDurationMs = stats.ElapsedMilliseconds;
m_enginePerformanceInfo.GraphCacheCheckJournalEnabled = stats.JournalEnabled;
},
() => cacheGraphStats))
{
var loggingContext = timeBlock.LoggingContext;
var effectiveEnvironmentVariables = FrontEndEngineImplementation.PopulateFromEnvironmentAndApplyOverrides(properties);
var availableMounts = MountsTable.CreateAndRegister(loggingContext, Context, Configuration, m_initialCommandLineConfiguration.Startup.Properties);
if (!AddConfigurationMountsAndCompleteInitialization(loggingContext, availableMounts))
{
return(cacheGraphStats);
}
cacheGraphStats.JournalEnabled = journalState.IsEnabled;
// ************************************************************
// 1. Engine cache check:
// ************************************************************
// * Single machine builds
// Distributed builds rely on the graph being available via the cache for it to be shared between master
// and workers. So even if the master could have had a hit from the engine cache, it must be ignored
// since the workers would not be able to retrieve it.
if (!HasExplicitlyLoadedGraph(Configuration.Cache) &&
!Configuration.Schedule.ForceUseEngineInfoFromCache &&
Configuration.Distribution.BuildRole == DistributedBuildRoles.None)
{
Contract.Assume(
graphFingerprint != null,
"When looking up a cached graph on a distributed master or single-machine build, a graph fingerprint must be computed");
InputTracker.MatchResult engineCacheMatchResult = CheckIfAvailableInputsToGraphMatchPreviousRun(
loggingContext,
serializer,
graphFingerprint: graphFingerprint,
availableEnvironmentVariables: effectiveEnvironmentVariables,
availableMounts: availableMounts,
journalState: journalState,
maxDegreeOfParallelism: maxDegreeOfParallelism);
cacheGraphStats.ObjectDirectoryHit = engineCacheMatchResult.Matches;
cacheGraphStats.ObjectDirectoryMissReason = engineCacheMatchResult.MissType;
cacheGraphStats.MissReason = engineCacheMatchResult.MissType;
cacheGraphStats.MissDescription = engineCacheMatchResult.FirstMissIdentifier;
cacheGraphStats.WasHit = engineCacheMatchResult.Matches;
cacheGraphStats.InputFilesChecked = engineCacheMatchResult.FilesChecked;
// Checking the engine cache may have used a FileChangeTracker and provided information about
// files/ContentHash pairs that were unchanged from the previous run. Hold onto this information as it may
// be useful when eventually parsing files.
// The FileChangeTracker is now up to date. All changed files have been removed from it. It can be reused
// for a future build with the same fingerprint, though it may be tracking extra files, for example if
// a spec was removed since the previous build.
inputChanges = engineCacheMatchResult.InputChanges;
}
var shouldTryContentCache =
!cacheGraphStats.WasHit &&
Configuration.Distribution.BuildRole != DistributedBuildRoles.Worker &&
Configuration.Cache.AllowFetchingCachedGraphFromContentCache &&
!HasExplicitlyLoadedGraph(Configuration.Cache) &&
(!Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.HasValue ||
!Configuration.FrontEnd.UseSpecPublicFacadeAndAstWhenAvailable.Value);
// ************************************************************
// 2. Content cache check:
// ************************************************************
// * Single machine builds that missed earlier
// * Distributed masters
// This is the only valid place for the master to get a hit since it must be in the cache for the
// workers to get it.
if (shouldTryContentCache)
{
// Since an in-place match did not succeed, we need a readied cache to try again.
// TODO: This logs an error if it fails. We are assuming some later thing will ensure that, if failed,
// the engine fails overall.
Possible <CacheInitializer> possibleCacheInitializerForFallback = cacheInitializationTask.GetAwaiter().GetResult();
if (possibleCacheInitializerForFallback.Succeeded)
{
CacheInitializer cacheInitializerForFallback = possibleCacheInitializerForFallback.Result;
using (EngineCache cacheForFallback = cacheInitializerForFallback.CreateCacheForContext())
{
var cacheGraphProvider = new CachedGraphProvider(
loggingContext,
Context,
cacheForFallback,
FileContentTable,
maxDegreeOfParallelism);
var cachedGraphDescriptor =
cacheGraphProvider.TryGetPipGraphCacheDescriptorAsync(graphFingerprint, effectiveEnvironmentVariables, availableMounts.MountsByName).Result;
if (cachedGraphDescriptor == null)
{
// There was no matching fingerprint in the cache. Record the status for logging before returning.
cacheGraphStats.CacheMissReason = GraphCacheMissReason.FingerprintChanged;
SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason);
return(cacheGraphStats);
}
var fetchEngineScheduleContent = EngineSchedule.TryFetchFromCacheAsync(
loggingContext,
Context,
cacheForFallback,
cachedGraphDescriptor,
serializer,
FileContentTable,
m_tempCleaner).Result;
if (!fetchEngineScheduleContent)
{
cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoPreviousRunToCheck;
SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason);
return(cacheGraphStats);
}
// If a distributed master, take note of the graph fingerprint
if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Master)
{
Contract.Assert(cachedGraphDescriptor != null);
m_masterService.CachedGraphDescriptor = cachedGraphDescriptor;
}
Logger.Log.FetchedSerializedGraphFromCache(outerLoggingContext);
cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss;
cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason;
cacheGraphStats.WasHit = true;
}
}
else
{
cacheGraphStats.CacheMissReason = GraphCacheMissReason.CacheFailure;
SetMissReasonIfUnset(ref cacheGraphStats, cacheGraphStats.CacheMissReason);
return(cacheGraphStats);
}
}
// ************************************************************
// 3. Query distributed master
// ************************************************************
// * Distributed workers only
if (Configuration.Distribution.BuildRole == DistributedBuildRoles.Worker)
{
Contract.Assume(
graphFingerprint == null,
"Distributed workers should request a graph fingerprint from the master (not compute one locally)");
Possible <CacheInitializer> possibleCacheInitializerForWorker = cacheInitializationTask.GetAwaiter().GetResult();
Contract.Assume(possibleCacheInitializerForWorker.Succeeded, "Workers must have a valid cache");
CacheInitializer cacheInitializerForWorker = possibleCacheInitializerForWorker.Result;
using (EngineCache cacheForWorker = cacheInitializerForWorker.CreateCacheForContext())
{
PipGraphCacheDescriptor schedulerStateDescriptor;
if (!m_workerService.TryGetBuildScheduleDescriptor(out schedulerStateDescriptor) ||
!EngineSchedule.TryFetchFromCacheAsync(
outerLoggingContext,
Context,
cacheForWorker,
schedulerStateDescriptor,
serializer,
FileContentTable,
m_tempCleaner).Result)
{
cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster;
cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason;
return(cacheGraphStats);
}
AsyncOut <AbsolutePath> symlinkFileLocation = new AsyncOut <AbsolutePath>();
if (!SymlinkDefinitionFileProvider.TryFetchWorkerSymlinkFileAsync(
outerLoggingContext,
Context.PathTable,
cacheForWorker,
Configuration.Layout,
m_workerService,
symlinkFileLocation).Result)
{
cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoFingerprintFromMaster;
cacheGraphStats.MissReason = cacheGraphStats.CacheMissReason;
return(cacheGraphStats);
}
m_workerSymlinkDefinitionFile = symlinkFileLocation.Value;
// Success. Populate the stats
cacheGraphStats.WasHit = true;
cacheGraphStats.WorkerHit = true;
cacheGraphStats.MissReason = GraphCacheMissReason.NoMiss;
cacheGraphStats.CacheMissReason = GraphCacheMissReason.NoMiss;
}
}
}
return(cacheGraphStats);
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create(context);
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
