예제 #1
0
        private bool CreateFactories(
            FrontEndContext frontEndContext,
            TestEngineAbstraction engineAbstraction,
            FrontEndStatistics frontEndStatistics,
            ICommandLineConfiguration configuration,
            out AmbientTesting ambientTesting,
            out ModuleRegistry moduleRegistry,
            out FrontEndFactory frontEndFactory)
        {
            moduleRegistry = new ModuleRegistry(frontEndContext.SymbolTable);
            ambientTesting = new AmbientTesting(engineAbstraction, GetAllDiagnostics, moduleRegistry.PrimitiveTypes);
            ambientTesting.Initialize(moduleRegistry.GlobalLiteral);

            var ambientAssert = new AmbientAssert(moduleRegistry.PrimitiveTypes);

            ambientAssert.Initialize(moduleRegistry.GlobalLiteral);

            // Create the controller
            frontEndFactory = new FrontEndFactory();
            frontEndFactory.SetConfigurationProcessor(new TestConfigProcessor(configuration));
            frontEndFactory.AddFrontEnd(
                new DScriptFrontEnd(
                    frontEndStatistics,
                    logger: m_astLogger));

            if (!frontEndFactory.TrySeal(frontEndContext.LoggingContext))
            {
                HandleDiagnostics();
                return(false);
            }

            return(true);
        }
        private IFrontEndController CreateControllerWithProfiler(PathTable pathTable, SymbolTable symbolTable)
        {
            var frontEndFactory   = new FrontEndFactory();
            var profilerDecorator = new ProfilerDecorator();

            // When evaluation is done we materialize the result of the profiler
            frontEndFactory.AddPhaseEndHook(EnginePhases.Evaluate, () =>
            {
                var entries           = profilerDecorator.GetProfiledEntries();
                var materializer      = new ProfilerMaterializer(pathTable);
                var reportDestination = Configuration.FrontEnd.ProfileReportDestination(pathTable);

                Logger.Log.MaterializingProfilerReport(LoggingContext, reportDestination.ToString(pathTable));

                try
                {
                    materializer.Materialize(entries, reportDestination);
                }
                catch (BuildXLException ex)
                {
                    Logger.Log.ErrorMaterializingProfilerReport(LoggingContext, ex.LogEventErrorCode, ex.LogEventMessage);
                }
            });

            return(TryCreateFrontEndController(
                       frontEndFactory,
                       profilerDecorator,
                       Configuration,
                       symbolTable,
                       LoggingContext,
                       Collector,
                       collectMemoryAsSoonAsPossible: CollectMemoryAsSoonAsPossible,
                       statistics: m_statistics));
        }
예제 #3
0
        private BuildXLEngine CreateEngine(bool rememberAllChangedTrackedInputs)
        {
            IFrontEndController Create(PathTable pathTable, SymbolTable symbolTable)
            {
                var frontEndStatistics = new FrontEndStatistics();
                var moduleRegistry     = new ModuleRegistry(symbolTable);

                var frontEndFactory = FrontEndFactory.CreateInstanceForTesting(
                    () => new ConfigurationProcessor(new FrontEndStatistics(), ParseAndEvaluateLogger),
                    new DScriptFrontEnd(frontEndStatistics, ParseAndEvaluateLogger));

                var evaluationScheduler = new EvaluationScheduler(degreeOfParallelism: 1);

                return(new FrontEndHostController(
                           frontEndFactory,
                           evaluationScheduler,
                           moduleRegistry,
                           new FrontEndStatistics(),
                           logger: InitializationLogger,
                           collector: null,
                           collectMemoryAsSoonAsPossible: false));
            }

            BuildXLEngine.PopulateLoggingAndLayoutConfiguration(Configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
            var successfulValidation = BuildXLEngine.PopulateAndValidateConfiguration(Configuration, Configuration, Context.PathTable, LoggingContext);

            Assert.True(successfulValidation);

            var engine = BuildXLEngine.Create(LoggingContext, Context, Configuration, new LambdaBasedFrontEndControllerFactory(Create), new BuildViewModel(), rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs);

            engine.TestHooks = TestHooks;

            return(engine);
        }
예제 #4
0
        private FrontEndHostController CreateHost()
        {
            var factory = new FrontEndFactory();

            factory.AddFrontEnd(new DummyFrontEnd1());
            factory.TrySeal(new LoggingContext("UnitTest"));

            var controller = new FrontEndHostController(factory, new DScriptWorkspaceResolverFactory(), new EvaluationScheduler(degreeOfParallelism: 8), collectMemoryAsSoonAsPossible: false);

            var context = BuildXLContext.CreateInstanceForTesting();

            var fileSystem = new InMemoryFileSystem(context.PathTable);

            ((IFrontEndController)controller).InitializeHost(
                new FrontEndContext(context, new LoggingContext("UnitTest"), fileSystem),
                new ConfigurationImpl()
            {
                FrontEnd = new FrontEndConfiguration()
                {
                    MaxFrontEndConcurrency = 1,
                }
            });

            var inMemoryCache = new EngineCache(
                new InMemoryArtifactContentCache(context),
                new InMemoryTwoPhaseFingerprintStore());

            controller.InitializeInternalForTesting(
                Task.FromResult(new Possible <EngineCache>(inMemoryCache)),
                AbsolutePath.Create(context.PathTable, TestOutputDirectory));

            return(controller);
        }
예제 #5
0
        private static IFrontEndController TryCreateFrontEndController(
            FrontEndFactory frontEndFactory,
            IDecorator <EvaluationResult> decorator,
            ICommandLineConfiguration configuration,
            SymbolTable symbolTable,
            LoggingContext loggingContext,
            PerformanceCollector collector,
            bool collectMemoryAsSoonAsPossible,
            IFrontEndStatistics statistics)
        {
            Contract.Requires(frontEndFactory != null && !frontEndFactory.IsSealed);

            // Statistic should be global for all front-ends, not per an instance.
            var frontEndStatistics = statistics ?? new FrontEndStatistics();

            var sharedModuleRegistry = new ModuleRegistry(symbolTable);

            // Note, that the following code is absolutely critical for detecting that front-end related objects
            // are freed successfully after evaluation.
            // ModuleRegistry was picked intentionally because it holds vast amount of front-end data.
            FrontEndControllerMemoryObserver.CaptureFrontEndReference(sharedModuleRegistry);

            frontEndFactory.SetConfigurationProcessor(
                new ConfigurationProcessor(
                    new FrontEndStatistics(), // Configuration processing is so lightweight that it won't affect overall perf statistics
                    logger: null));

            frontEndFactory.AddFrontEnd(new DScriptFrontEnd(
                                            frontEndStatistics,
                                            evaluationDecorator: decorator));

            frontEndFactory.AddFrontEnd(new NugetFrontEnd(
                                            frontEndStatistics,
                                            evaluationDecorator: decorator));

            frontEndFactory.AddFrontEnd(new DownloadFrontEnd());

#if PLATFORM_WIN
            frontEndFactory.AddFrontEnd(new MsBuildFrontEnd());
            frontEndFactory.AddFrontEnd(new NinjaFrontEnd());
            frontEndFactory.AddFrontEnd(new CMakeFrontEnd());
            frontEndFactory.AddFrontEnd(new RushFrontEnd());
            frontEndFactory.AddFrontEnd(new YarnFrontEnd());
            frontEndFactory.AddFrontEnd(new LageFrontEnd());
#endif

            if (!frontEndFactory.TrySeal(loggingContext))
            {
                return(null);
            }

            return(new FrontEndHostController(
                       frontEndFactory,
                       evaluationScheduler: EvaluationScheduler.Default,
                       moduleRegistry: sharedModuleRegistry,
                       frontEndStatistics: frontEndStatistics,
                       logger: BuildXL.FrontEnd.Core.Tracing.Logger.CreateLogger(),
                       collector: collector,
                       collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible));
        }
        private IFrontEndController CreateRegularController(SymbolTable symbolTable)
        {
            var frontEndFactory = new FrontEndFactory();

            return(TryCreateFrontEndController(
                       frontEndFactory,
                       decorator: null,
                       configuration: Configuration,
                       symbolTable: symbolTable,
                       loggingContext: LoggingContext,
                       collector: Collector,
                       collectMemoryAsSoonAsPossible: CollectMemoryAsSoonAsPossible,
                       statistics: m_statistics));
        }
예제 #7
0
        private bool CreateFactories(
            FrontEndContext frontEndContext,
            TestEngineAbstraction engineAbstraction,
            FrontEndStatistics frontEndStatistics,
            ICommandLineConfiguration configuration,
            out AmbientTesting ambientTesting,
            out DScriptWorkspaceResolverFactory workspaceFactory,
            out FrontEndFactory frontEndFactory)
        {
            var globalConstants = new GlobalConstants(frontEndContext.SymbolTable);

            ambientTesting = new AmbientTesting(engineAbstraction, GetAllDiagnostics, globalConstants.KnownTypes);
            ambientTesting.Initialize(globalConstants.Global);

            var ambientAssert = new AmbientAssert(globalConstants.KnownTypes);

            ambientAssert.Initialize(globalConstants.Global);

            var sharedModuleRegistry = new ModuleRegistry();

            workspaceFactory = new DScriptWorkspaceResolverFactory();
            workspaceFactory.RegisterResolver(
                KnownResolverKind.DScriptResolverKind,
                () => new WorkspaceSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
            workspaceFactory.RegisterResolver(
                KnownResolverKind.SourceResolverKind,
                () => new WorkspaceSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));
            workspaceFactory.RegisterResolver(
                KnownResolverKind.DefaultSourceResolverKind,
                () => new WorkspaceDefaultSourceModuleResolver(globalConstants, sharedModuleRegistry, frontEndStatistics));

            // Create the controller
            frontEndFactory = new FrontEndFactory();
            frontEndFactory.SetConfigurationProcessor(new TestConfigProcessor(configuration));
            frontEndFactory.AddFrontEnd(
                new DScriptFrontEnd(
                    globalConstants,
                    sharedModuleRegistry,
                    frontEndStatistics,
                    logger: m_astLogger));

            if (!frontEndFactory.TrySeal(frontEndContext.LoggingContext))
            {
                HandleDiagnostics();
                return(false);
            }

            return(true);
        }
예제 #8
0
        private static bool TryCreateResolvers(
            [NotNull] FrontEndFactory frontEndFactory,
            WorkspaceConfiguration configuration,
            PathTable pathTable,
            AbsolutePath mainConfigurationFile,
            bool addBuiltInPreludeResolver,
            out List <IWorkspaceModuleResolver> resolvers,
            out IEnumerable <Failure> failures)
        {
            Contract.Ensures(Contract.Result <List <IWorkspaceModuleResolver> >() != null);
            Contract.EnsuresForAll(Contract.Result <List <IWorkspaceModuleResolver> >(), r => r != null);

            resolvers = new List <IWorkspaceModuleResolver>(configuration.ResolverSettings.Count + (addBuiltInPreludeResolver ? 1 : 0));

            var resolverFailures = new List <Failure>();

            var resolverSettings = new List <IResolverSettings>(configuration.ResolverSettings);

            // The built in resolver is generally not added only for some tests. Regular spec processing always adds it.
            if (addBuiltInPreludeResolver)
            {
                // We add a resolver that points to the built-in prelude at the end of the resolver collection
                // so the built-in prelude is used if no prelude is specified explicitly
                var builtInPreludeSettings = PreludeManager.GetResolverSettingsForBuiltInPrelude(mainConfigurationFile, pathTable);
                resolverSettings.Add(builtInPreludeSettings);
            }

            foreach (var resolverConfiguration in resolverSettings)
            {
                var kind = resolverConfiguration.Kind;
                if (!frontEndFactory.TryGetFrontEnd(kind, out var frontEnd))
                {
                    resolverFailures.Add(new WorkspaceModuleResolverGenericInitializationFailure(kind));
                    continue;
                }

                if (!frontEnd.TryCreateWorkspaceResolver(resolverConfiguration, out var resolver))
                {
                    resolverFailures.Add(new WorkspaceModuleResolverGenericInitializationFailure(resolverConfiguration.Kind));
                }
                else
                {
                    resolvers.Add(resolver);
                }
            }

            failures = resolverFailures;
            return(resolverFailures.Count == 0);
        }
예제 #9
0
        /// <summary>
        /// Each element in <param name="moduleRepositoryArray"/> represents the extent of a resolver,
        /// containing module references (keys) to spec content (values). So a workspace is created accordingly: so many
        /// resolvers as array elements.
        /// </summary>
        public WorkspaceProvider CreateWorkspaceProviderFromContentWithFileSystem(
            IFileSystem fileSystem,
            bool cancelOnFirstFailure,
            bool preserveTrivia = false,
            params ModuleRepository[] moduleRepositoryArray)
        {
            var resolverSettings = new List <IResolverSettings>();

            foreach (var modulesWithContent in moduleRepositoryArray)
            {
                resolverSettings.Add(CreateResolverSettingsFromModulesWithContent(modulesWithContent, fileSystem));
            }

            var workspaceConfiguration = new WorkspaceConfiguration(
                resolverSettings,
                constructFingerprintDuringParsing: false,
                maxDegreeOfParallelismForParsing: DataflowBlockOptions.Unbounded,
                parsingOptions: ParsingOptions.DefaultParsingOptions.WithTrivia(preserveTrivia),
                maxDegreeOfParallelismForTypeChecking: 1,
                cancelOnFirstFailure: cancelOnFirstFailure,
                includePreludeWithName: PreludeName,
                cancellationToken: m_cancellationToken);

            var frontEndFactory = new FrontEndFactory();

            frontEndFactory.AddFrontEnd(new SimpleDScriptFrontEnd());
            frontEndFactory.TrySeal(new LoggingContext("test", "Test"));

            var result = WorkspaceProvider.TryCreate(
                mainConfigurationWorkspace: null,
                workspaceStatistics: new WorkspaceStatistics(),
                frontEndFactory,
                configuration: workspaceConfiguration,
                pathTable: PathTable,
                symbolTable: new SymbolTable(),
                useDecorator: false,
                addBuiltInPreludeResolver: false,
                workspaceProvider: out var workspaceProvider,
                failures: out var failures);

            // We assume workspace provider does not fail here
            Contract.Assert(result);

            return((WorkspaceProvider)workspaceProvider);
        }
        private IFrontEndController CreateControllerWithDebugger(PathTable pathTable, SymbolTable symbolTable)
        {
            var confPort        = Configuration.FrontEnd.DebuggerPort();
            var debugServerPort = confPort != 0 ? confPort : DebugServer.DefaultDebugPort;
            var pathTranslator  = GetPathTranslator(Configuration.Logging, pathTable);
            var debugState      = new DebuggerState(pathTable, LoggingContext, DScriptDebugerRenderer.Render, new DScriptExprEvaluator(LoggingContext));
            var debugServer     = new DebugServer(LoggingContext, debugServerPort,
                                                  (debugger) => new DebugSession(debugState, pathTranslator, debugger));
            Task <IDebugger> debuggerTask = debugServer.StartAsync();
            var evaluationDecorator       = new LazyDecorator(debuggerTask, Configuration.FrontEnd.DebuggerBreakOnExit());
            var frontEndFactory           = new FrontEndFactory();

            frontEndFactory.AddPhaseStartHook(EnginePhases.Evaluate, () =>
            {
                if (!debuggerTask.IsCompleted)
                {
                    Logger.Log.WaitingForClientDebuggerToConnect(LoggingContext, debugServer.Port);
                }

                debuggerTask.Result?.Session.WaitSessionInitialized();
            });

            frontEndFactory.AddPhaseEndHook(EnginePhases.Evaluate, () =>
            {
                // make sure the debugger is shut down at the end (unnecessary in most cases, as the debugger will shut itself down after completion)
                debugServer.ShutDown();
                debuggerTask.Result?.ShutDown();
            });

            return(TryCreateFrontEndController(
                       frontEndFactory,
                       evaluationDecorator,
                       Configuration,
                       symbolTable,
                       LoggingContext,
                       Collector,
                       collectMemoryAsSoonAsPossible: CollectMemoryAsSoonAsPossible,
                       statistics: m_statistics));
        }
예제 #11
0
        /// <nodoc/>
        public static bool TryCreate(
            [CanBeNull] Workspace mainConfigurationWorkspace,
            IWorkspaceStatistics workspaceStatistics,
            FrontEndFactory frontEndFactory,
            PathTable pathTable,
            SymbolTable symbolTable,
            WorkspaceConfiguration configuration,
            bool useDecorator,
            bool addBuiltInPreludeResolver,
            out IWorkspaceProvider workspaceProvider,
            out IEnumerable <Failure> failures)
        {
            // mainConfigurationWorkspace can be null for some tests
            var mainFile = mainConfigurationWorkspace != null ?
                           mainConfigurationWorkspace.ConfigurationModule.Definition.MainFile :
                           AbsolutePath.Invalid;

            if (!TryCreateResolvers(
                    frontEndFactory,
                    configuration,
                    pathTable,
                    mainFile,
                    addBuiltInPreludeResolver,
                    out var resolvers,
                    out failures))
            {
                workspaceProvider = default(IWorkspaceProvider);
                return(false);
            }

            var provider = new WorkspaceProvider(workspaceStatistics, resolvers, configuration, pathTable, symbolTable);

            provider.m_mainConfigurationWorkspace = mainConfigurationWorkspace;

            workspaceProvider = useDecorator
                ? (IWorkspaceProvider) new WorkspaceProviderStatisticsDecorator(workspaceStatistics, provider)
                : provider;
            return(true);
        }
예제 #12
0
        private static IFrontEndController TryCreateFrontEndController(
            FrontEndFactory frontEndFactory,
            IDecorator <EvaluationResult> decorator,
            ICommandLineConfiguration configuration,
            SymbolTable symbolTable,
            LoggingContext loggingContext,
            PerformanceCollector collector,
            bool collectMemoryAsSoonAsPossible,
            IFrontEndStatistics statistics)
        {
            var workspaceResolverFactory = new DScriptWorkspaceResolverFactory();

            Contract.Requires(frontEndFactory != null && !frontEndFactory.IsSealed);

            // Statistic should be global for all front-ends, not per an instance.
            var frontEndStatistics = statistics ?? new FrontEndStatistics();

            var globalConstants      = new GlobalConstants(symbolTable);
            var sharedModuleRegistry = new ModuleRegistry();

            // Note, that the following code is absolutely critical for detecting that front-end related objects
            // are freed successfully after evaluation.
            // ModuleRegistry was picked intentionally because it holds vast amount of front-end data.
            FrontEndControllerMemoryObserver.CaptureFrontEndReference(sharedModuleRegistry);

            frontEndFactory.SetConfigurationProcessor(
                new ConfigurationProcessor(globalConstants, sharedModuleRegistry, logger: null));

            var msBuildFrontEnd = new MsBuildFrontEnd(
                globalConstants,
                sharedModuleRegistry,
                frontEndStatistics);

            var ninjaFrontEnd = new NinjaFrontEnd(
                globalConstants,
                sharedModuleRegistry,
                frontEndStatistics);

            var cmakeFrontEnd = new CMakeFrontEnd(
                globalConstants,
                sharedModuleRegistry,
                frontEndStatistics);

            // TODO: Workspace resolvers and frontends are registered in separate factories. Consider
            // adding a main coordinator/registry
            RegisterKnownWorkspaceResolvers(
                workspaceResolverFactory,
                globalConstants,
                sharedModuleRegistry,
                frontEndStatistics,
                msBuildFrontEnd,
                ninjaFrontEnd,
                cmakeFrontEnd);

            frontEndFactory.AddFrontEnd(new DScriptFrontEnd(
                                            globalConstants,
                                            sharedModuleRegistry,
                                            frontEndStatistics,
                                            evaluationDecorator: decorator));

            frontEndFactory.AddFrontEnd(new NugetFrontEnd(
                                            globalConstants,
                                            sharedModuleRegistry,
                                            frontEndStatistics,
                                            evaluationDecorator: decorator));

            frontEndFactory.AddFrontEnd(new DownloadFrontEnd(
                                            globalConstants,
                                            sharedModuleRegistry));

            frontEndFactory.AddFrontEnd(msBuildFrontEnd);
            frontEndFactory.AddFrontEnd(ninjaFrontEnd);
            frontEndFactory.AddFrontEnd(cmakeFrontEnd);

            if (!frontEndFactory.TrySeal(loggingContext))
            {
                return(null);
            }

            return(new FrontEndHostController(frontEndFactory, workspaceResolverFactory,
                                              frontEndStatistics: frontEndStatistics, collector: collector, collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible));
        }
예제 #13
0
        private async Task TestDownloadResolver(DownloadData data, Func <DownloadResolver, Task> performTest, bool useHttpServer = true)
        {
            var dummyConfigFile = Path.Combine(TemporaryDirectory, m_uniqueTestFolder, "config.dsc");

            var statistics       = new Statistics();
            var moduleRegistry   = new ModuleRegistry(FrontEndContext.SymbolTable);
            var workspaceFactory = CreateWorkspaceFactoryForTesting(FrontEndContext, ParseAndEvaluateLogger);
            var configuration    = ConfigurationHelpers.GetDefaultForTesting(FrontEndContext.PathTable, AbsolutePath.Create(FrontEndContext.PathTable, dummyConfigFile));
            var resolverSettings = new ResolverSettings();

            var frontEndFactory = new FrontEndFactory();

            frontEndFactory.AddFrontEnd(new DownloadFrontEnd());
            frontEndFactory.TrySeal(new LoggingContext("UnitTest"));

            using (var host = new FrontEndHostController(
                       frontEndFactory,
                       workspaceFactory,
                       new EvaluationScheduler(degreeOfParallelism: 1),
                       moduleRegistry,
                       new FrontEndStatistics(),
                       global::BuildXL.FrontEnd.Core.Tracing.Logger.CreateLogger(),
                       collector: null,
                       collectMemoryAsSoonAsPossible: false))
            {
                var frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(
                    FrontEndContext.PathTable,
                    FrontEndContext.FileSystem,
                    configuration);

                ((IFrontEndController)host).InitializeHost(FrontEndContext, configuration);
                host.SetState(frontEndEngineAbstraction, new TestEnv.TestPipGraph(), configuration);

                var resolver = new DownloadResolver(
                    statistics,
                    host,
                    FrontEndContext,
                    Logger.Log,
                    "TestFrontEnd"
                    );

                var workspaceResolver = new DownloadWorkspaceResolver();
                workspaceResolver.UpdateDataForDownloadData(data, FrontEndContext);
                await resolver.InitResolverAsync(resolverSettings, workspaceResolver);

                if (useHttpServer)
                {
                    using (var listener = new HttpListener())
                    {
                        // This test relies on the mutex in the build engine to only run one unittest at a time and this assembly to be single thread
                        // if any of those assumptions will be broken we will have to either dynamically (remind you globally) get unique ports.
                        // HttpListner doesn't have this built-in so there will always be a race. Just spam the ports utnill one doesn't fail
                        // use a global mutex (This is not honored by qtest since it can run in a different session on cloudbuild).
                        listener.Prefixes.Add(TestServer);
                        listener.Start();

                        StartRequestHandler(listener);

                        await performTest(resolver);

                        listener.Stop();
                        listener.Close();
                    }
                }
                else
                {
                    await performTest(resolver);
                }
            }
        }