private bool CreateFactories(
FrontEndContext frontEndContext,
TestEngineAbstraction engineAbstraction,
FrontEndStatistics frontEndStatistics,
ICommandLineConfiguration configuration,
out AmbientTesting ambientTesting,
out ModuleRegistry moduleRegistry,
out FrontEndFactory frontEndFactory)
{
moduleRegistry = new ModuleRegistry(frontEndContext.SymbolTable);
ambientTesting = new AmbientTesting(engineAbstraction, GetAllDiagnostics, moduleRegistry.PrimitiveTypes);
ambientTesting.Initialize(moduleRegistry.GlobalLiteral);
var ambientAssert = new AmbientAssert(moduleRegistry.PrimitiveTypes);
ambientAssert.Initialize(moduleRegistry.GlobalLiteral);
// Create the controller
frontEndFactory = new FrontEndFactory();
frontEndFactory.SetConfigurationProcessor(new TestConfigProcessor(configuration));
frontEndFactory.AddFrontEnd(
new DScriptFrontEnd(
frontEndStatistics,
logger: m_astLogger));
if (!frontEndFactory.TrySeal(frontEndContext.LoggingContext))
{
HandleDiagnostics();
return(false);
}
return(true);
}
private IFrontEndController CreateControllerWithProfiler(PathTable pathTable, SymbolTable symbolTable)
{
var frontEndFactory = new FrontEndFactory();
var profilerDecorator = new ProfilerDecorator();
// When evaluation is done we materialize the result of the profiler
frontEndFactory.AddPhaseEndHook(EnginePhases.Evaluate, () =>
{
var entries = profilerDecorator.GetProfiledEntries();
var materializer = new ProfilerMaterializer(pathTable);
var reportDestination = Configuration.FrontEnd.ProfileReportDestination(pathTable);
Logger.Log.MaterializingProfilerReport(LoggingContext, reportDestination.ToString(pathTable));
try
{
materializer.Materialize(entries, reportDestination);
}
catch (BuildXLException ex)
{
Logger.Log.ErrorMaterializingProfilerReport(LoggingContext, ex.LogEventErrorCode, ex.LogEventMessage);
}
});
return(TryCreateFrontEndController(
frontEndFactory,
profilerDecorator,
Configuration,
symbolTable,
LoggingContext,
Collector,
collectMemoryAsSoonAsPossible: CollectMemoryAsSoonAsPossible,
statistics: m_statistics));
}
private BuildXLEngine CreateEngine(bool rememberAllChangedTrackedInputs)
{
IFrontEndController Create(PathTable pathTable, SymbolTable symbolTable)
{
var frontEndStatistics = new FrontEndStatistics();
var moduleRegistry = new ModuleRegistry(symbolTable);
var frontEndFactory = FrontEndFactory.CreateInstanceForTesting(
() => new ConfigurationProcessor(new FrontEndStatistics(), ParseAndEvaluateLogger),
new DScriptFrontEnd(frontEndStatistics, ParseAndEvaluateLogger));
var evaluationScheduler = new EvaluationScheduler(degreeOfParallelism: 1);
return(new FrontEndHostController(
frontEndFactory,
evaluationScheduler,
moduleRegistry,
new FrontEndStatistics(),
logger: InitializationLogger,
collector: null,
collectMemoryAsSoonAsPossible: false));
}
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(Configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
var successfulValidation = BuildXLEngine.PopulateAndValidateConfiguration(Configuration, Configuration, Context.PathTable, LoggingContext);
Assert.True(successfulValidation);
var engine = BuildXLEngine.Create(LoggingContext, Context, Configuration, new LambdaBasedFrontEndControllerFactory(Create), new BuildViewModel(), rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs);
engine.TestHooks = TestHooks;
return(engine);
}
private FrontEndHostController CreateHost()
{
var factory = new FrontEndFactory();
factory.AddFrontEnd(new DummyFrontEnd1());
factory.TrySeal(new LoggingContext("UnitTest"));
var controller = new FrontEndHostController(factory, new DScriptWorkspaceResolverFactory(), new EvaluationScheduler(degreeOfParallelism: 8), collectMemoryAsSoonAsPossible: false);
var context = BuildXLContext.CreateInstanceForTesting();
var fileSystem = new InMemoryFileSystem(context.PathTable);
((IFrontEndController)controller).InitializeHost(
new FrontEndContext(context, new LoggingContext("UnitTest"), fileSystem),
new ConfigurationImpl()
{
FrontEnd = new FrontEndConfiguration()
{
MaxFrontEndConcurrency = 1,
}
});
var inMemoryCache = new EngineCache(
new InMemoryArtifactContentCache(context),
new InMemoryTwoPhaseFingerprintStore());
controller.InitializeInternalForTesting(
Task.FromResult(new Possible <EngineCache>(inMemoryCache)),
AbsolutePath.Create(context.PathTable, TestOutputDirectory));
return(controller);
}
private static IFrontEndController TryCreateFrontEndController(
FrontEndFactory frontEndFactory,
IDecorator <EvaluationResult> decorator,
ICommandLineConfiguration configuration,
SymbolTable symbolTable,
LoggingContext loggingContext,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible,
IFrontEndStatistics statistics)
{
Contract.Requires(frontEndFactory != null && !frontEndFactory.IsSealed);
// Statistic should be global for all front-ends, not per an instance.
var frontEndStatistics = statistics ?? new FrontEndStatistics();
var sharedModuleRegistry = new ModuleRegistry(symbolTable);
// Note, that the following code is absolutely critical for detecting that front-end related objects
// are freed successfully after evaluation.
// ModuleRegistry was picked intentionally because it holds vast amount of front-end data.
FrontEndControllerMemoryObserver.CaptureFrontEndReference(sharedModuleRegistry);
frontEndFactory.SetConfigurationProcessor(
new ConfigurationProcessor(
new FrontEndStatistics(), // Configuration processing is so lightweight that it won't affect overall perf statistics
logger: null));
frontEndFactory.AddFrontEnd(new DScriptFrontEnd(
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new NugetFrontEnd(
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new DownloadFrontEnd());
#if PLATFORM_WIN
frontEndFactory.AddFrontEnd(new MsBuildFrontEnd());
frontEndFactory.AddFrontEnd(new NinjaFrontEnd());
frontEndFactory.AddFrontEnd(new CMakeFrontEnd());
frontEndFactory.AddFrontEnd(new RushFrontEnd());
frontEndFactory.AddFrontEnd(new YarnFrontEnd());
frontEndFactory.AddFrontEnd(new LageFrontEnd());
#endif
if (!frontEndFactory.TrySeal(loggingContext))
{
return(null);
}
return(new FrontEndHostController(
frontEndFactory,
evaluationScheduler: EvaluationScheduler.Default,
moduleRegistry: sharedModuleRegistry,
frontEndStatistics: frontEndStatistics,
logger: BuildXL.FrontEnd.Core.Tracing.Logger.CreateLogger(),
collector: collector,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible));
}
private IFrontEndController CreateRegularController(SymbolTable symbolTable)
{
var frontEndFactory = new FrontEndFactory();
return(TryCreateFrontEndController(
frontEndFactory,
decorator: null,
configuration: Configuration,
symbolTable: symbolTable,
loggingContext: LoggingContext,
collector: Collector,
collectMemoryAsSoonAsPossible: CollectMemoryAsSoonAsPossible,
statistics: m_statistics));
}
private bool CreateFactories(
FrontEndContext frontEndContext,
TestEngineAbstraction engineAbstraction,
FrontEndStatistics frontEndStatistics,
ICommandLineConfiguration configuration,
out AmbientTesting ambientTesting,
out DScriptWorkspaceResolverFactory workspaceFactory,
out FrontEndFactory frontEndFactory)
{
var globalConstants = new GlobalConstants(frontEndContext.SymbolTable);
ambientTesting = new AmbientTesting(engineAbstraction, GetAllDiagnostics, globalConstants.KnownTypes);
ambientTesting.Initialize(globalConstants.Global);
var ambientAssert = new AmbientAssert(globalConstants.KnownTypes);
ambientAssert.Initialize(globalConstants.Global);
var sharedModuleRegistry = new ModuleRegistry();
workspaceFactory = new DScriptWorkspaceResolverFactory();
workspaceFactory.RegisterResolver(
KnownResolverKind.DScriptResolverKind,
() => new WorkspaceSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
workspaceFactory.RegisterResolver(
KnownResolverKind.SourceResolverKind,
() => new WorkspaceSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
workspaceFactory.RegisterResolver(
KnownResolverKind.DefaultSourceResolverKind,
() => new WorkspaceDefaultSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
// Create the controller
frontEndFactory = new FrontEndFactory();
frontEndFactory.SetConfigurationProcessor(new TestConfigProcessor(configuration));
frontEndFactory.AddFrontEnd(
new DScriptFrontEnd(
globalConstants,
sharedModuleRegistry,
frontEndStatistics,
logger: m_astLogger));
if (!frontEndFactory.TrySeal(frontEndContext.LoggingContext))
{
HandleDiagnostics();
return(false);
}
return(true);
}
private static bool TryCreateResolvers(
[NotNull] FrontEndFactory frontEndFactory,
WorkspaceConfiguration configuration,
PathTable pathTable,
AbsolutePath mainConfigurationFile,
bool addBuiltInPreludeResolver,
out List <IWorkspaceModuleResolver> resolvers,
out IEnumerable <Failure> failures)
{
Contract.Ensures(Contract.Result <List <IWorkspaceModuleResolver> >() != null);
Contract.EnsuresForAll(Contract.Result <List <IWorkspaceModuleResolver> >(), r => r != null);
resolvers = new List <IWorkspaceModuleResolver>(configuration.ResolverSettings.Count + (addBuiltInPreludeResolver ? 1 : 0));
var resolverFailures = new List <Failure>();
var resolverSettings = new List <IResolverSettings>(configuration.ResolverSettings);
// The built in resolver is generally not added only for some tests. Regular spec processing always adds it.
if (addBuiltInPreludeResolver)
{
// We add a resolver that points to the built-in prelude at the end of the resolver collection
// so the built-in prelude is used if no prelude is specified explicitly
var builtInPreludeSettings = PreludeManager.GetResolverSettingsForBuiltInPrelude(mainConfigurationFile, pathTable);
resolverSettings.Add(builtInPreludeSettings);
}
foreach (var resolverConfiguration in resolverSettings)
{
var kind = resolverConfiguration.Kind;
if (!frontEndFactory.TryGetFrontEnd(kind, out var frontEnd))
{
resolverFailures.Add(new WorkspaceModuleResolverGenericInitializationFailure(kind));
continue;
}
if (!frontEnd.TryCreateWorkspaceResolver(resolverConfiguration, out var resolver))
{
resolverFailures.Add(new WorkspaceModuleResolverGenericInitializationFailure(resolverConfiguration.Kind));
}
else
{
resolvers.Add(resolver);
}
}
failures = resolverFailures;
return(resolverFailures.Count == 0);
}
/// <summary>
/// Each element in <param name="moduleRepositoryArray"/> represents the extent of a resolver,
/// containing module references (keys) to spec content (values). So a workspace is created accordingly: so many
/// resolvers as array elements.
/// </summary>
public WorkspaceProvider CreateWorkspaceProviderFromContentWithFileSystem(
IFileSystem fileSystem,
bool cancelOnFirstFailure,
bool preserveTrivia = false,
params ModuleRepository[] moduleRepositoryArray)
{
var resolverSettings = new List <IResolverSettings>();
foreach (var modulesWithContent in moduleRepositoryArray)
{
resolverSettings.Add(CreateResolverSettingsFromModulesWithContent(modulesWithContent, fileSystem));
}
var workspaceConfiguration = new WorkspaceConfiguration(
resolverSettings,
constructFingerprintDuringParsing: false,
maxDegreeOfParallelismForParsing: DataflowBlockOptions.Unbounded,
parsingOptions: ParsingOptions.DefaultParsingOptions.WithTrivia(preserveTrivia),
maxDegreeOfParallelismForTypeChecking: 1,
cancelOnFirstFailure: cancelOnFirstFailure,
includePreludeWithName: PreludeName,
cancellationToken: m_cancellationToken);
var frontEndFactory = new FrontEndFactory();
frontEndFactory.AddFrontEnd(new SimpleDScriptFrontEnd());
frontEndFactory.TrySeal(new LoggingContext("test", "Test"));
var result = WorkspaceProvider.TryCreate(
mainConfigurationWorkspace: null,
workspaceStatistics: new WorkspaceStatistics(),
frontEndFactory,
configuration: workspaceConfiguration,
pathTable: PathTable,
symbolTable: new SymbolTable(),
useDecorator: false,
addBuiltInPreludeResolver: false,
workspaceProvider: out var workspaceProvider,
failures: out var failures);
// We assume workspace provider does not fail here
Contract.Assert(result);
return((WorkspaceProvider)workspaceProvider);
}
private IFrontEndController CreateControllerWithDebugger(PathTable pathTable, SymbolTable symbolTable)
{
var confPort = Configuration.FrontEnd.DebuggerPort();
var debugServerPort = confPort != 0 ? confPort : DebugServer.DefaultDebugPort;
var pathTranslator = GetPathTranslator(Configuration.Logging, pathTable);
var debugState = new DebuggerState(pathTable, LoggingContext, DScriptDebugerRenderer.Render, new DScriptExprEvaluator(LoggingContext));
var debugServer = new DebugServer(LoggingContext, debugServerPort,
(debugger) => new DebugSession(debugState, pathTranslator, debugger));
Task <IDebugger> debuggerTask = debugServer.StartAsync();
var evaluationDecorator = new LazyDecorator(debuggerTask, Configuration.FrontEnd.DebuggerBreakOnExit());
var frontEndFactory = new FrontEndFactory();
frontEndFactory.AddPhaseStartHook(EnginePhases.Evaluate, () =>
{
if (!debuggerTask.IsCompleted)
{
Logger.Log.WaitingForClientDebuggerToConnect(LoggingContext, debugServer.Port);
}
debuggerTask.Result?.Session.WaitSessionInitialized();
});
frontEndFactory.AddPhaseEndHook(EnginePhases.Evaluate, () =>
{
// make sure the debugger is shut down at the end (unnecessary in most cases, as the debugger will shut itself down after completion)
debugServer.ShutDown();
debuggerTask.Result?.ShutDown();
});
return(TryCreateFrontEndController(
frontEndFactory,
evaluationDecorator,
Configuration,
symbolTable,
LoggingContext,
Collector,
collectMemoryAsSoonAsPossible: CollectMemoryAsSoonAsPossible,
statistics: m_statistics));
}
/// <nodoc/>
public static bool TryCreate(
[CanBeNull] Workspace mainConfigurationWorkspace,
IWorkspaceStatistics workspaceStatistics,
FrontEndFactory frontEndFactory,
PathTable pathTable,
SymbolTable symbolTable,
WorkspaceConfiguration configuration,
bool useDecorator,
bool addBuiltInPreludeResolver,
out IWorkspaceProvider workspaceProvider,
out IEnumerable <Failure> failures)
{
// mainConfigurationWorkspace can be null for some tests
var mainFile = mainConfigurationWorkspace != null ?
mainConfigurationWorkspace.ConfigurationModule.Definition.MainFile :
AbsolutePath.Invalid;
if (!TryCreateResolvers(
frontEndFactory,
configuration,
pathTable,
mainFile,
addBuiltInPreludeResolver,
out var resolvers,
out failures))
{
workspaceProvider = default(IWorkspaceProvider);
return(false);
}
var provider = new WorkspaceProvider(workspaceStatistics, resolvers, configuration, pathTable, symbolTable);
provider.m_mainConfigurationWorkspace = mainConfigurationWorkspace;
workspaceProvider = useDecorator
? (IWorkspaceProvider) new WorkspaceProviderStatisticsDecorator(workspaceStatistics, provider)
: provider;
return(true);
}
private static IFrontEndController TryCreateFrontEndController(
FrontEndFactory frontEndFactory,
IDecorator <EvaluationResult> decorator,
ICommandLineConfiguration configuration,
SymbolTable symbolTable,
LoggingContext loggingContext,
PerformanceCollector collector,
bool collectMemoryAsSoonAsPossible,
IFrontEndStatistics statistics)
{
var workspaceResolverFactory = new DScriptWorkspaceResolverFactory();
Contract.Requires(frontEndFactory != null && !frontEndFactory.IsSealed);
// Statistic should be global for all front-ends, not per an instance.
var frontEndStatistics = statistics ?? new FrontEndStatistics();
var globalConstants = new GlobalConstants(symbolTable);
var sharedModuleRegistry = new ModuleRegistry();
// Note, that the following code is absolutely critical for detecting that front-end related objects
// are freed successfully after evaluation.
// ModuleRegistry was picked intentionally because it holds vast amount of front-end data.
FrontEndControllerMemoryObserver.CaptureFrontEndReference(sharedModuleRegistry);
frontEndFactory.SetConfigurationProcessor(
new ConfigurationProcessor(globalConstants, sharedModuleRegistry, logger: null));
var msBuildFrontEnd = new MsBuildFrontEnd(
globalConstants,
sharedModuleRegistry,
frontEndStatistics);
var ninjaFrontEnd = new NinjaFrontEnd(
globalConstants,
sharedModuleRegistry,
frontEndStatistics);
var cmakeFrontEnd = new CMakeFrontEnd(
globalConstants,
sharedModuleRegistry,
frontEndStatistics);
// TODO: Workspace resolvers and frontends are registered in separate factories. Consider
// adding a main coordinator/registry
RegisterKnownWorkspaceResolvers(
workspaceResolverFactory,
globalConstants,
sharedModuleRegistry,
frontEndStatistics,
msBuildFrontEnd,
ninjaFrontEnd,
cmakeFrontEnd);
frontEndFactory.AddFrontEnd(new DScriptFrontEnd(
globalConstants,
sharedModuleRegistry,
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new NugetFrontEnd(
globalConstants,
sharedModuleRegistry,
frontEndStatistics,
evaluationDecorator: decorator));
frontEndFactory.AddFrontEnd(new DownloadFrontEnd(
globalConstants,
sharedModuleRegistry));
frontEndFactory.AddFrontEnd(msBuildFrontEnd);
frontEndFactory.AddFrontEnd(ninjaFrontEnd);
frontEndFactory.AddFrontEnd(cmakeFrontEnd);
if (!frontEndFactory.TrySeal(loggingContext))
{
return(null);
}
return(new FrontEndHostController(frontEndFactory, workspaceResolverFactory,
frontEndStatistics: frontEndStatistics, collector: collector, collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible));
}
private async Task TestDownloadResolver(DownloadData data, Func <DownloadResolver, Task> performTest, bool useHttpServer = true)
{
var dummyConfigFile = Path.Combine(TemporaryDirectory, m_uniqueTestFolder, "config.dsc");
var statistics = new Statistics();
var moduleRegistry = new ModuleRegistry(FrontEndContext.SymbolTable);
var workspaceFactory = CreateWorkspaceFactoryForTesting(FrontEndContext, ParseAndEvaluateLogger);
var configuration = ConfigurationHelpers.GetDefaultForTesting(FrontEndContext.PathTable, AbsolutePath.Create(FrontEndContext.PathTable, dummyConfigFile));
var resolverSettings = new ResolverSettings();
var frontEndFactory = new FrontEndFactory();
frontEndFactory.AddFrontEnd(new DownloadFrontEnd());
frontEndFactory.TrySeal(new LoggingContext("UnitTest"));
using (var host = new FrontEndHostController(
frontEndFactory,
workspaceFactory,
new EvaluationScheduler(degreeOfParallelism: 1),
moduleRegistry,
new FrontEndStatistics(),
global::BuildXL.FrontEnd.Core.Tracing.Logger.CreateLogger(),
collector: null,
collectMemoryAsSoonAsPossible: false))
{
var frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(
FrontEndContext.PathTable,
FrontEndContext.FileSystem,
configuration);
((IFrontEndController)host).InitializeHost(FrontEndContext, configuration);
host.SetState(frontEndEngineAbstraction, new TestEnv.TestPipGraph(), configuration);
var resolver = new DownloadResolver(
statistics,
host,
FrontEndContext,
Logger.Log,
"TestFrontEnd"
);
var workspaceResolver = new DownloadWorkspaceResolver();
workspaceResolver.UpdateDataForDownloadData(data, FrontEndContext);
await resolver.InitResolverAsync(resolverSettings, workspaceResolver);
if (useHttpServer)
{
using (var listener = new HttpListener())
{
// This test relies on the mutex in the build engine to only run one unittest at a time and this assembly to be single thread
// if any of those assumptions will be broken we will have to either dynamically (remind you globally) get unique ports.
// HttpListner doesn't have this built-in so there will always be a race. Just spam the ports utnill one doesn't fail
// use a global mutex (This is not honored by qtest since it can run in a different session on cloudbuild).
listener.Prefixes.Add(TestServer);
listener.Start();
StartRequestHandler(listener);
await performTest(resolver);
listener.Stop();
listener.Close();
}
}
else
{
await performTest(resolver);
}
}
}
