protected virtual BuildXLEngineResult CreateAndRunEngine(
ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
out BuildXLEngine engine,
Action <EngineTestHooksData> verifyEngineTestHooksData = null,
TestCache testCache = null)
{
testCache = testCache ?? TestCache;
using (EngineTestHooksData testHooks = new EngineTestHooksData
{
AppDeployment = appDeployment,
CacheFactory = (context) => new EngineCache(
testCache.GetArtifacts(context),
testCache.Fingerprints)
})
{
engine = CreateEngine(config, appDeployment, testRootDirectory, rememberAllChangedTrackedInputs, verifyEngineTestHooksData);
// Ignore DX222 for csc.exe being outside of src directory
IgnoreWarnings();
engine.TestHooks = testHooks;
var result = engine.RunForFrontEndTests(LoggingContext);
return(result);
}
}
public void EnabledUnsafeOptionsLogWarnings()
{
ICommandLineConfiguration config;
PathTable pt = new PathTable();
var argsParser = new Args();
argsParser.TryParse(new[] { "/c:" + m_specFilePath }, pt, out config);
// Get a list of all possible options
var options = argsParser.GetParsedOptionNames();
// Compile unsafe options while removing duplicates
var unsafeOptions = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (var opt in options)
{
if (opt.StartsWith("unsafe_", StringComparison.OrdinalIgnoreCase))
{
unsafeOptions.Add(opt);
}
}
// Get all the unsafe options that also associated loggers
var unsafeOptionLoggers = new HashSet <string>(BuildXLEngine.CreateUnsafeOptionLoggers().Keys, StringComparer.OrdinalIgnoreCase);
// After all unsafe options are removed, there should be no unsafe options left
unsafeOptions.ExceptWith(unsafeOptionLoggers);
XAssert.IsTrue(
unsafeOptions.Count == 0,
"The following unsafe options are not logging warnings to users when enabled: "
+ XAssert.SetToString(unsafeOptions)
+ Environment.NewLine
+ "Add an associated logger function in CreateUnsafeOptionLoggers()");
}
protected virtual BuildXLEngine CreateEngine(ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
Action <EngineTestHooksData> verifyEngineTestHooksData = null)
{
var engineContext = EngineContext.CreateNew(CancellationToken.None, PathTable, FileSystem);
var factory = FrontEndControllerFactory.Create(
FrontEndMode.NormalMode,
LoggingContext,
config,
// Set the timeout to a large number to avoid useless performance collections in tests.
new PerformanceCollector(TimeSpan.FromHours(1)),
collectMemoryAsSoonAsPossible: false);
var engine = BuildXLEngine.Create(
LoggingContext,
engineContext,
config,
factory,
rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs
);
return(engine);
}
private BuildXLEngine CreateEngine(bool rememberAllChangedTrackedInputs)
{
IFrontEndController Create(PathTable pathTable, SymbolTable symbolTable)
{
var frontEndStatistics = new FrontEndStatistics();
var moduleRegistry = new ModuleRegistry(symbolTable);
var frontEndFactory = FrontEndFactory.CreateInstanceForTesting(
() => new ConfigurationProcessor(new FrontEndStatistics(), ParseAndEvaluateLogger),
new DScriptFrontEnd(frontEndStatistics, ParseAndEvaluateLogger));
var evaluationScheduler = new EvaluationScheduler(degreeOfParallelism: 1);
return(new FrontEndHostController(
frontEndFactory,
evaluationScheduler,
moduleRegistry,
new FrontEndStatistics(),
logger: InitializationLogger,
collector: null,
collectMemoryAsSoonAsPossible: false));
}
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(Configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
var successfulValidation = BuildXLEngine.PopulateAndValidateConfiguration(Configuration, Configuration, Context.PathTable, LoggingContext);
Assert.True(successfulValidation);
var engine = BuildXLEngine.Create(LoggingContext, Context, Configuration, new LambdaBasedFrontEndControllerFactory(Create), new BuildViewModel(), rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs);
engine.TestHooks = TestHooks;
return(engine);
}
/// <summary>
/// Configures the engine to use an in-memory cache for testing
/// </summary>
public static BuildXLEngine ConfigureInMemoryCache(this BuildXLEngine engine, TestCache testCache = null)
{
if (engine.TestHooks == null)
{
engine.TestHooks = new EngineTestHooksData();
}
if (testCache != null)
{
engine.TestHooks.CacheFactory = (context) =>
{
return(new EngineCache(
testCache.GetArtifacts(context),
testCache.Fingerprints));
};
}
else
{
engine.TestHooks.CacheFactory = (context) =>
{
return(new EngineCache(
new InMemoryArtifactContentCache(context),
new InMemoryTwoPhaseFingerprintStore()));
};
}
return(engine);
}
public void TestSpotlightCheck()
{
PathTable pt = new PathTable();
AbsolutePath trailingNoindex = AbsolutePath.Create(pt, Path.Combine(TestRoot, ".noindex"));
AbsolutePath intermediateNoindex = AbsolutePath.Create(pt, Path.Combine(TestRoot, ".noindex", "subdirectory"));
BuildXLEngine.CheckArtifactFolersAndEmitNoIndexWarning(pt, LoggingContext, trailingNoindex, intermediateNoindex);
XAssert.AreEqual(0, EventListener.GetEventCount((int)LogEventId.EmitSpotlightIndexingWarning), "No warning should be logged for either path containing noindex");
AbsolutePath indexablePath = AbsolutePath.Create(pt, Path.Combine(TestRoot, "subdirectory"));
BuildXLEngine.CheckArtifactFolersAndEmitNoIndexWarning(pt, LoggingContext, indexablePath);
AssertWarningEventLogged(LogEventId.EmitSpotlightIndexingWarning);
}
/// <nodoc/>
public SchedulerIntegrationTestBase(ITestOutputHelper output) : base(output)
{
m_testOutputHelper = output;
CaptureAllDiagnosticMessages = false;
// Each event listener that we want to capture events from must be listed here
foreach (var eventSource in BuildXLApp.GeneratedEventSources)
{
RegisterEventSource(eventSource);
}
// Go through the command line configuration handling used by the real app to get the appropriate defaults
ICommandLineConfiguration config;
XAssert.IsTrue(Args.TryParseArguments(new[] { "/c:" + Path.Combine(TemporaryDirectory, "config.dc") }, Context.PathTable, null, out config), "Failed to construct arguments");
Configuration = new CommandLineConfiguration(config);
Cache = InMemoryCacheFactory.Create();
FileContentTable = FileContentTable.CreateNew(LoggingContext);
// Disable defaults that write disk IO but are not critical to correctness or performance
Configuration.Logging.StoreFingerprints = false;
Configuration.Logging.LogExecution = false;
Configuration.Engine.TrackBuildsInUserFolder = false;
Configuration.Engine.CleanTempDirectories = false;
Configuration.Sandbox.EnsureTempDirectoriesExistenceBeforePipExecution = true;
// Skip hash source file pips
// Some error becomes a contract exception when SkipHashSourceFile is enabled.
// Mostly on methods that traverse graph, e.g., DirectedGraph.GetOutgoingEdges();
// BUG 1472567
// Configuration.Schedule.SkipHashSourceFile = true;
// Compute static pip fingerprints for incremental scheduling tests.
Configuration.Schedule.ComputePipStaticFingerprints = true;
// Disable currently enabled unsafe option.
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.IgnoreCreateProcessReport = false;
// Populate file system capabilities.
// Here, for example, we use copy-on-write instead of hardlinks when Unix file system supports copy-on-write.
// Particular tests can override this by setting Configuration.Engine.UseHardlinks.
BuildXLEngine.PopulateFileSystemCapabilities(Configuration, Configuration, Context.PathTable, LoggingContext);
// Reset pip graph builder to use the populated configuration.
ResetPipGraphBuilder();
}
public void PreserveOutputsOpaqueTest(bool storeOutputsToCache, bool ignorePrivatization)
{
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.PreserveOutputs = PreserveOutputsMode.Enabled;
Configuration.Schedule.StoreOutputsToCache = storeOutputsToCache;
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.IgnorePreserveOutputsPrivatization = ignorePrivatization;
if (storeOutputsToCache && ignorePrivatization)
{
// Invalid configuration.
var config = new CommandLineConfiguration(Configuration);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, Context.PathTable, bxlExeLocation: null, inTestMode: true);
XAssert.IsFalse(BuildXLEngine.PopulateAndValidateConfiguration(config, config, Context.PathTable, LoggingContext));
AssertErrorEventLogged(global::BuildXL.Engine.Tracing.LogEventId.ConfigIncompatibleOptionIgnorePreserveOutputsPrivatization);
return;
}
// Output is in opaque dir and Unsafe.AllowPreservedOutputs = true
var pipA = ScheduleAndGetPip(out var input, out var output, opaque: true, pipPreserveOutputsFlag: true);
// No cache hit
string outputContents = RunSchedulerAndGetOutputContents(output, cacheHitAssert: false, id: pipA.Process.PipId);
XAssert.AreEqual(CONTENT, outputContents);
// Change input
ModifyFile(input);
// No cache hit
outputContents = RunSchedulerAndGetOutputContents(output, cacheHitAssert: false, id: pipA.Process.PipId);
// As the opaque output is preserved, the pip appended the existing file.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
if (ignorePrivatization)
{
AssertVerboseEventLogged(global::BuildXL.Processes.Tracing.LogEventId.PipProcessPreserveOutputDirectorySkipMakeFilesPrivate);
}
outputContents = RunSchedulerAndGetOutputContents(output, cacheHitAssert: true, id: pipA.Process.PipId);
// Cache hit and the appended file (CONTENT_TWICE) should remain the same.
XAssert.AreEqual(CONTENT_TWICE, outputContents);
}
private void CreateContextAndConfiguration(out BuildXLContext context, out IConfiguration configuration)
{
context = BuildXLContext.CreateInstanceForTesting();
var cmdLineConfiguration = new CommandLineConfiguration();
cmdLineConfiguration.Engine.TrackBuildsInUserFolder = false;
cmdLineConfiguration.Startup.ConfigFile = AbsolutePath.Create(context.PathTable, Path.Combine(TemporaryDirectory, "config.dc"));
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(cmdLineConfiguration, context.PathTable, bxlExeLocation: null, inTestMode: true);
var cacheConfigPath = Path.Combine(TemporaryDirectory, "CacheConfig.json");
const string CacheConfig = @"
{
""Assembly"":""BuildXL.Cache.MemoizationStoreAdapter"",
""Type"":""BuildXL.Cache.MemoizationStoreAdapter.MemoizationStoreCacheFactory"",
""CacheRootPath"":""[BuildXLSelectedRootPath]"",
""CacheLogPath"":""[BuildXLSelectedLogPath]""
}";
File.WriteAllText(cacheConfigPath, CacheConfig);
cmdLineConfiguration.Cache.CacheConfigFile = AbsolutePath.Create(context.PathTable, cacheConfigPath);
configuration = cmdLineConfiguration;
}
private static bool TryBuildPipGraphFragment(
ICommandLineConfiguration commandLineConfig,
PipGraphFragmentGeneratorConfiguration pipGraphFragmentGeneratorConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter)
{
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = CompleteCommandLineConfiguration(commandLineConfig);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics();
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
FrontEndHostController frontEndHostController = (FrontEndHostController)controller;
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
FrontEndEngineAbstraction frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var pipGraphBuilder = pipGraphFragmentGeneratorConfig.TopSort
? new PipGraphFragmentBuilderTopSort(engineContext, config, mountsTable.MountPathExpander)
: new PipGraphFragmentBuilder(engineContext, config, mountsTable.MountPathExpander);
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out var moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
// Error should have been reported already
return(false);
}
if (!SerializeFragmentIfRequested(pipGraphFragmentGeneratorConfig, frontEndContext, pipGraphBuilder))
{
// Error should have been reported already
return(false);
}
}
}
return(true);
}
/// <summary>
/// Helper function that parses provided configuration.
/// </summary>
protected TestResult <IConfiguration> ParseConfiguration(string configuration, ICommandLineConfiguration template = null)
{
var testWriter = CreateTestWriter();
AddPrelude(testWriter);
var pathTable = FrontEndContext.PathTable;
// Need to write configuration on disk
testWriter.AddExtraFile(Names.ConfigDsc, configuration);
testWriter.Write(TestRoot);
var configFile = AbsolutePath.Create(pathTable, Path.Combine(TestRoot, Names.ConfigDsc));
var config = template == null
? new CommandLineConfiguration()
{
Startup =
{
ConfigFile = configFile,
},
Schedule =
{
MaxProcesses = DegreeOfParallelism,
},
FrontEnd =
{
MaxFrontEndConcurrency = DegreeOfParallelism,
EnableIncrementalFrontEnd = false,
LogStatistics = false,
},
Engine =
{
LogStatistics = false,
},
Cache =
{
CacheSpecs = SpecCachingOption.Disabled,
}
}
: new CommandLineConfiguration(template)
{
Startup =
{
ConfigFile = configFile,
},
FrontEnd =
{
EnableIncrementalFrontEnd = false,
}
};
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, pathTable, bxlExeLocation: null, inTestMode: true);
var frontEndFactory = CreateFrontEndFactoryForParsingConfig(ParseAndEvaluateLogger);
var moduleRegistry = new ModuleRegistry(FrontEndContext.SymbolTable);
CreateFrontEndHost(config, frontEndFactory, moduleRegistry, configFile, out var finalConfig, out _);
if (finalConfig != null)
{
return(new TestResult <IConfiguration>(finalConfig));
}
return(new TestResult <IConfiguration>(CapturedWarningsAndErrors));
}
private static ICommandLineConfiguration CreateConfiguration(
IEnumerable <string> sdksToResolve,
PathTable pathTable,
AbsolutePath testFolderPath)
{
var configFilePath = testFolderPath.Combine(pathTable, Names.ConfigDsc);
var packageFilePath = testFolderPath.Combine(pathTable, Names.ModuleConfigBm);
var sdkPackages = sdksToResolve
.SelectMany(folder =>
Directory.EnumerateFiles(folder, Names.PackageConfigDsc, SearchOption.AllDirectories).Concat(
Directory.EnumerateFiles(folder, Names.ModuleConfigBm, SearchOption.AllDirectories).Concat(
Directory.EnumerateFiles(folder, Names.ModuleConfigDsc, SearchOption.AllDirectories))))
.Select(packageFile =>
new DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition>(AbsolutePath.Create(pathTable, packageFile)))
.ToList();
var configuration =
new CommandLineConfiguration
{
// Have to new up the list because we have some bad semantics dealing with the list being null or not.
Packages = new List <AbsolutePath>
{
packageFilePath,
},
Resolvers =
{
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = sdkPackages,
},
},
FrontEnd =
{
MaxFrontEndConcurrency = 1, // Single threaded for deterministic evaluation
NameResolutionSemantics = NameResolutionSemantics.ImplicitProjectReferences,
// PreserveFullNames = true, Some comment in code as not to turn on, check with folks....
UsePartialEvaluation = true,
UseSpecPublicFacadeAndAstWhenAvailable = false,
ConstructAndSaveBindingFingerprint = false,
// Some of the DS tests fail when incremental frontend is not used
EnableIncrementalFrontEnd = true,
},
Engine =
{
TrackBuildsInUserFolder = false,
},
Layout =
{
OutputDirectory = testFolderPath.Combine(pathTable, "Out"),
ObjectDirectory = testFolderPath.Combine(pathTable, "Out").Combine(pathTable, "Objects"),
PrimaryConfigFile = configFilePath,
SourceDirectory = testFolderPath,
TempDirectory = testFolderPath.Combine(pathTable, "Out").Combine(pathTable, "Temp"),
},
Mounts =
{
},
Startup =
{
ConfigFile = configFilePath,
},
DisableDefaultSourceResolver = false,
};
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, pathTable, Assembly.GetExecutingAssembly().Location, inTestMode: true);
return(configuration);
}
public static bool TryBuildWorkspace(
EnginePhases phase,
FrontEndContext frontEndContext,
PipExecutionContext engineContext,
AbsolutePath configFile,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(configFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var commandlineConfig = GetCommandLineConfiguration(configuration, phase, configFile);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(commandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: commandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, commandlineConfig);
frontEndHostController = controller as FrontEndHostController;
// If there is an explicit engine abstraction, we set it
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: commandlineConfig);
}
var config = controller.ParseConfig(commandlineConfig);
if (config == null)
{
frontEndHostController = null;
return(false);
}
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(
new SchedulerContext(
CancellationToken.None,
frontEndContext.StringTable,
frontEndContext.PathTable,
frontEndContext.SymbolTable,
frontEndContext.QualifierTable)),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: null /* No need to create pips */,
engineAbstraction: frontEndEngineAbstraction ?? new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config),
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: commandlineConfig.Startup))
{
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
}
Contract.Assert(frontEndHostController != null);
// If workspace construction is successfull, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
frontEndHostController.GetWorkspace(),
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
return(true);
}
/// <summary>
/// Runs the scheduler allowing various options to be specifically set
/// </summary>
public ScheduleRunResult RunSchedulerSpecific(
PipGraph graph,
SchedulerTestHooks testHooks = null,
SchedulerState schedulerState = null,
RootFilter filter = null,
TempCleaner tempCleaner = null)
{
var config = new CommandLineConfiguration(Configuration);
// Populating the configuration may modify the configuration, so it should occur first.
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(config, config, Context.PathTable, LoggingContext);
FileAccessWhitelist whitelist = new FileAccessWhitelist(Context);
whitelist.Initialize(config);
IReadOnlyList <string> junctionRoots = Configuration.Engine.DirectoriesToTranslate?.Select(a => a.ToPath.ToString(Context.PathTable)).ToList();
var map = VolumeMap.TryCreateMapOfAllLocalVolumes(LoggingContext, junctionRoots);
var optionalAccessor = TryGetJournalAccessor(map);
// Although scan change journal is enabled, but if we cannot create an enabled journal accessor, then create a disabled one.
m_journalState = map == null || !optionalAccessor.IsValid
? JournalState.DisabledJournal
: JournalState.CreateEnabledJournal(map, optionalAccessor.Value);
if (config.Schedule.IncrementalScheduling)
{
// Ensure that we can scan the journal when incremental scheduling is enabled.
XAssert.IsTrue(m_journalState.IsEnabled, "Incremental scheduling requires that journal is enabled");
}
// Seal the translator if not sealed
DirectoryTranslator.Seal();
// .....................................................................................
// some dummy setup in order to get a PreserveOutputsSalt.txt file and an actual salt
// .....................................................................................
string dummyCacheDir = Path.Combine(TemporaryDirectory, "Out", "Cache");
Directory.CreateDirectory(dummyCacheDir); // EngineSchedule tries to put the PreserveOutputsSalt.txt here
ContentHash?previousOutputsSalt =
EngineSchedule.PreparePreviousOutputsSalt(LoggingContext, Context.PathTable, config);
Contract.Assert(previousOutputsSalt.HasValue);
// .....................................................................................
testHooks = testHooks ?? new SchedulerTestHooks();
Contract.Assert(!(config.Engine.CleanTempDirectories && tempCleaner == null));
using (var queue = new PipQueue(config.Schedule))
using (var testQueue = new TestPipQueue(queue, LoggingContext, initiallyPaused: false))
using (var testScheduler = new TestScheduler(
graph: graph,
pipQueue: testQueue,
context: Context,
fileContentTable: FileContentTable,
loggingContext: LoggingContext,
cache: Cache,
configuration: config,
journalState: m_journalState,
fileAccessWhitelist: whitelist,
fingerprintSalt: Configuration.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable),
tempCleaner: tempCleaner,
previousInputsSalt: previousOutputsSalt.Value,
successfulPips: null,
failedPips: null,
ipcProvider: null,
directoryTranslator: DirectoryTranslator,
testHooks: testHooks))
{
if (filter == null)
{
EngineSchedule.TryGetPipFilter(LoggingContext, Context, config, config, Expander.TryGetRootByMountName, out filter);
}
XAssert.IsTrue(testScheduler.InitForMaster(LoggingContext, filter, schedulerState), "Failed to initialized test scheduler");
testScheduler.Start(LoggingContext);
bool success = testScheduler.WhenDone().GetAwaiter().GetResult();
testScheduler.SaveFileChangeTrackerAsync(LoggingContext).Wait();
return(new ScheduleRunResult
{
Graph = graph,
Config = config,
Success = success,
PipResults = testScheduler.PipResults,
PipExecutorCounters = testScheduler.PipExecutionCounters,
PathSets = testScheduler.PathSets,
ProcessPipCountersByFilter = testScheduler.ProcessPipCountersByFilter,
ProcessPipCountersByTelemetryTag = testScheduler.ProcessPipCountersByTelemetryTag,
SchedulerState = new SchedulerState(testScheduler)
});
}
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IPipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by IDE test.
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IPipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(),
5000,
false);
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Scheduler.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable));
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
public TestEnv(
string name,
string rootPath,
bool enableLazyOutputMaterialization = false,
int maxRelativeOutputDirectoryLength = 260,
List <IMount> mounts = null,
PathTable pathTable = null)
{
Contract.Requires(name != null);
Contract.Requires(!string.IsNullOrEmpty(rootPath));
LoggingContext = new LoggingContext("TestLogger." + name);
PathTable = pathTable ?? new PathTable();
PipDataBuilderPool = new ObjectPool <PipDataBuilder>(() => new PipDataBuilder(PathTable.StringTable), _ => { });
// The tests that use TestEnv need to be modernized to take a filesystem
var fileSystem = new PassThroughFileSystem(PathTable);
Context = EngineContext.CreateNew(CancellationToken.None, PathTable, fileSystem);
// Add some well-known paths with fixed casing to the Context.PathTable
AbsolutePath.Create(Context.PathTable, rootPath.ToLowerInvariant());
var root = AbsolutePath.Create(Context.PathTable, rootPath);
var configuration = ConfigHelpers.CreateDefaultForXml(Context.PathTable, root);
configuration.Layout.SourceDirectory = root.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "src")); // These tests have non-standard src folder
configuration.Engine.MaxRelativeOutputDirectoryLength = maxRelativeOutputDirectoryLength;
configuration.Schedule.EnableLazyOutputMaterialization = enableLazyOutputMaterialization;
configuration.Schedule.UnsafeDisableGraphPostValidation = false;
configuration.Schedule.ComputePipStaticFingerprints = true;
configuration.Sandbox.FileAccessIgnoreCodeCoverage = true;
BuildXLEngine.PopulateFileSystemCapabilities(configuration, configuration, Context.PathTable, LoggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(configuration, configuration, Context.PathTable, LoggingContext);
Configuration = configuration;
var mountsTable = MountsTable.CreateAndRegister(LoggingContext, Context, Configuration, null);
if (mounts != null)
{
foreach (var mount in mounts)
{
mountsTable.AddResolvedMount(mount);
}
}
AbsolutePath specFile = SourceRoot.CreateRelative(Context.PathTable, "TestSpecFile.dsc");
var graph = TestSchedulerFactory.CreateEmptyPipGraph(Context, configuration, mountsTable.MountPathExpander);
PipTable = graph.PipTable;
PipGraph = graph;
var locationData = new LocationData(specFile, 0, 0);
var modulePip = ModulePip.CreateForTesting(Context.StringTable, specFile);
PipGraph.AddModule(modulePip);
PipGraph.AddSpecFile(new SpecFilePip(FileArtifact.CreateSourceFile(specFile), locationData, modulePip.Module));
PipConstructionHelper = PipConstructionHelper.CreateForTesting(
Context,
ObjectRoot,
redirectedRoot: Configuration.Layout.RedirectedDirectory,
pipGraph: PipGraph,
moduleName: modulePip.Identity.ToString(Context.StringTable),
symbol: name,
specPath: specFile);
Paths = new Paths(PathTable);
mountsTable.CompleteInitialization();
}
private static bool TryBuildWorkspaceInternal(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
bool forIDE,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by the IDE.
if (frontEndEngineAbstraction != null)
{
// The IDE engine typically doesn't have mounts configured. We do it here if they haven't been configured yet.
// Observe these are just the default mounts used for config evaluation.
if (frontEndEngineAbstraction is BasicFrontEndEngineAbstraction basicEngine && !frontEndEngineAbstraction.GetMountNames("Script", BuildXL.Utilities.ModuleId.Invalid).Any())
{
// If this fails we just ignore the failure. Mounts not being properly configured doesn't prevent the IDE plugin from working.
basicEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties);
}
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
else
{
// Otherwise we construct one with all mounts populated for config evaluation
var configurationEngine = new BasicFrontEndEngineAbstraction(engineContext.PathTable, engineContext.FileSystem, mutableCommandlineConfig);
if (!configurationEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties))
{
// Errors are logged already
return(false);
}
frontEndEngineAbstraction = configurationEngine;
}
var config = controller.ParseConfig(frontEndEngineAbstraction, mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IMutablePipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
// For the IDE case, we want to make sure all config-specific mounts are properly populated
if (forIDE && frontEndEngineAbstraction is BasicFrontEndEngineAbstraction languageServiceEngine)
{
Contract.AssertNotNull(frontEndEngineAbstraction);
AddConfigurationMounts(config, mountsTable);
languageServiceEngine.SetMountsTable(mountsTable);
}
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var searchPathToolsHash = new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable).ComputeSearchPathToolsHash();
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Pips.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
searchPathToolsHash: searchPathToolsHash);
// Observe mount table is completed during workspace construction
AddConfigurationMounts(config, mountsTable);
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
public void TestRedirectUserProfileDirectory()
{
// first run to create all necessary directories leading to obj directory
SetupTestData();
RunEngine();
string currentUserProfile = SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile);
string junctionPath = Path.Combine(Configuration.Layout.ObjectDirectory.ToString(Context.PathTable), "buildXLUserProfile");
bool specialFolderInitializerWasCalled = false;
var translatedDirectory = new List <TranslateDirectoryData>();
var properties = new Dictionary <string, string>();
var expectedProperties = new Dictionary <string, string>
{
{ "APPDATA", Path.Combine(junctionPath, "AppData", "Roaming") },
{ "LOCALAPPDATA", Path.Combine(junctionPath, "AppData", "Local") },
{ "USERPROFILE", junctionPath },
{ "USERNAME", "buildXLUserProfile" },
{ "HOMEDRIVE", Path.GetPathRoot(junctionPath).TrimEnd('\\') },
{ "HOMEPATH", junctionPath.Substring(Path.GetPathRoot(junctionPath).TrimEnd('\\').Length) },
{ "INTERNETCACHE", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCache") },
{ "INTERNETHISTORY", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "History") },
{ "INETCOOKIES", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCookies") },
{ "LOCALLOW", Path.Combine(junctionPath, "AppData", "LocalLow") },
};
// add the variables to the dictionary, so we can verify that the method overrides the existing values
foreach (var envVar in expectedProperties.Keys)
{
properties.Add(envVar, string.Empty);
}
try
{
var success = BuildXLEngine.RedirectUserProfileDirectory(
Configuration.Layout.ObjectDirectory,
translatedDirectory,
properties,
dict => { specialFolderInitializerWasCalled = true; },
true,
Context.PathTable,
LoggingContext);
// must have finished successfully
XAssert.IsTrue(success);
// verify the env block is properly populated
XAssert.AreSetsEqual(expectedProperties.Keys, properties.Keys, true);
XAssert.IsFalse(expectedProperties.Any(kvp => properties[kvp.Key] != kvp.Value));
XAssert.IsTrue(specialFolderInitializerWasCalled);
// verify junction
var openResult = FileUtilities.TryOpenDirectory(junctionPath, FileDesiredAccess.FileReadAttributes, FileShare.ReadWrite, FileFlagsAndAttributes.FileFlagOpenReparsePoint, out var handle);
XAssert.IsTrue(openResult.Succeeded);
using (handle)
{
var possibleTarget = FileUtilities.TryGetReparsePointTarget(handle, junctionPath);
XAssert.IsTrue(possibleTarget.Succeeded);
XAssert.AreEqual(FileSystemWin.NtPathPrefix + currentUserProfile, possibleTarget.Result);
}
// verify that we added a new directory translation
AbsolutePath.TryCreate(Context.PathTable, currentUserProfile, out var fromPath);
AbsolutePath.TryCreate(Context.PathTable, junctionPath, out var toPath);
XAssert.IsTrue(translatedDirectory.Count == 1);
XAssert.IsTrue(translatedDirectory[0].FromPath == fromPath && translatedDirectory[0].ToPath == toPath);
}
finally
{
// clean the junction after the test
var possibleProbe = FileUtilities.TryProbePathExistence(junctionPath, false);
if (possibleProbe.Succeeded && possibleProbe.Result != PathExistence.Nonexistent)
{
// we attempt to delete the junction, but we do not care if we failed to do
FileUtilities.TryRemoveDirectory(junctionPath, out int errCode);
}
}
}
