protected virtual BuildXLEngine CreateEngine(ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
Action <EngineTestHooksData> verifyEngineTestHooksData = null)
{
var engineContext = EngineContext.CreateNew(CancellationToken.None, PathTable, FileSystem);
var factory = FrontEndControllerFactory.Create(
FrontEndMode.NormalMode,
LoggingContext,
config,
// Set the timeout to a large number to avoid useless performance collections in tests.
new PerformanceCollector(TimeSpan.FromHours(1)),
collectMemoryAsSoonAsPossible: false);
var engine = BuildXLEngine.Create(
LoggingContext,
engineContext,
config,
factory,
rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs
);
return(engine);
}
private static bool TryBuildPipGraphFragment(
ICommandLineConfiguration commandLineConfig,
PipGraphFragmentGeneratorConfiguration pipGraphFragmentGeneratorConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter)
{
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = CompleteCommandLineConfiguration(commandLineConfig);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics();
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
FrontEndHostController frontEndHostController = (FrontEndHostController)controller;
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
FrontEndEngineAbstraction frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var pipGraphBuilder = pipGraphFragmentGeneratorConfig.TopSort
? new PipGraphFragmentBuilderTopSort(engineContext, config, mountsTable.MountPathExpander)
: new PipGraphFragmentBuilder(engineContext, config, mountsTable.MountPathExpander);
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out var moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
// Error should have been reported already
return(false);
}
if (!SerializeFragmentIfRequested(pipGraphFragmentGeneratorConfig, frontEndContext, pipGraphBuilder))
{
// Error should have been reported already
return(false);
}
}
}
return(true);
}
public static bool TryBuildWorkspace(
EnginePhases phase,
FrontEndContext frontEndContext,
PipExecutionContext engineContext,
AbsolutePath configFile,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(configFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var commandlineConfig = GetCommandLineConfiguration(configuration, phase, configFile);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(commandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: commandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, commandlineConfig);
frontEndHostController = controller as FrontEndHostController;
// If there is an explicit engine abstraction, we set it
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: commandlineConfig);
}
var config = controller.ParseConfig(commandlineConfig);
if (config == null)
{
frontEndHostController = null;
return(false);
}
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(
new SchedulerContext(
CancellationToken.None,
frontEndContext.StringTable,
frontEndContext.PathTable,
frontEndContext.SymbolTable,
frontEndContext.QualifierTable)),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: null /* No need to create pips */,
engineAbstraction: frontEndEngineAbstraction ?? new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config),
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: commandlineConfig.Startup))
{
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
}
Contract.Assert(frontEndHostController != null);
// If workspace construction is successfull, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
frontEndHostController.GetWorkspace(),
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
return(true);
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IPipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by IDE test.
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IPipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(),
5000,
false);
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Scheduler.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable));
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
private static bool TryBuildWorkspaceInternal(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
bool forIDE,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by the IDE.
if (frontEndEngineAbstraction != null)
{
// The IDE engine typically doesn't have mounts configured. We do it here if they haven't been configured yet.
// Observe these are just the default mounts used for config evaluation.
if (frontEndEngineAbstraction is BasicFrontEndEngineAbstraction basicEngine && !frontEndEngineAbstraction.GetMountNames("Script", BuildXL.Utilities.ModuleId.Invalid).Any())
{
// If this fails we just ignore the failure. Mounts not being properly configured doesn't prevent the IDE plugin from working.
basicEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties);
}
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
else
{
// Otherwise we construct one with all mounts populated for config evaluation
var configurationEngine = new BasicFrontEndEngineAbstraction(engineContext.PathTable, engineContext.FileSystem, mutableCommandlineConfig);
if (!configurationEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties))
{
// Errors are logged already
return(false);
}
frontEndEngineAbstraction = configurationEngine;
}
var config = controller.ParseConfig(frontEndEngineAbstraction, mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IMutablePipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
// For the IDE case, we want to make sure all config-specific mounts are properly populated
if (forIDE && frontEndEngineAbstraction is BasicFrontEndEngineAbstraction languageServiceEngine)
{
Contract.AssertNotNull(frontEndEngineAbstraction);
AddConfigurationMounts(config, mountsTable);
languageServiceEngine.SetMountsTable(mountsTable);
}
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var searchPathToolsHash = new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable).ComputeSearchPathToolsHash();
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Pips.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
searchPathToolsHash: searchPathToolsHash);
// Observe mount table is completed during workspace construction
AddConfigurationMounts(config, mountsTable);
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}