Ejemplo n.º 1
0
        private void RunGraphFingerprintValueTest(string[] expectedValues, string commandLineFilter, string[] commandLineValues, bool expectSuccess = true)
        {
            var pathTable = new PathTable();
            // EngineSchedulTest operate on the real filesystem for now
            var            fileSystem = new PassThroughFileSystem(pathTable);
            BuildXLContext context    = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);

            var config = new CommandLineConfiguration
            {
                Filter  = commandLineFilter,
                Startup = { ImplicitFilters = new List <string>(commandLineValues) },
                Engine  = { DefaultFilter = null }
            };
            EvaluationFilter evaluationFilter;

            if (!expectSuccess)
            {
                XAssert.IsFalse(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));
            }
            else
            {
                XAssert.IsTrue(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));

                foreach (FullSymbol symbol in evaluationFilter.ValueNamesToResolve)
                {
                    string value = symbol.ToString(context.SymbolTable);
                    XAssert.IsTrue(expectedValues.Contains(value));
                }

                XAssert.AreEqual(expectedValues.Length, evaluationFilter.ValueNamesToResolve.Count());
            }
        }
Ejemplo n.º 2
0
        protected virtual BuildXLEngine CreateEngine(ICommandLineConfiguration config,
                                                     AppDeployment appDeployment,
                                                     string testRootDirectory,
                                                     bool rememberAllChangedTrackedInputs,
                                                     Action <EngineTestHooksData> verifyEngineTestHooksData = null)
        {
            var engineContext = EngineContext.CreateNew(CancellationToken.None, PathTable, FileSystem);

            var factory = FrontEndControllerFactory.Create(
                FrontEndMode.NormalMode,
                LoggingContext,
                config,
                // Set the timeout to a large number to avoid useless performance collections in tests.
                new PerformanceCollector(TimeSpan.FromHours(1)),
                collectMemoryAsSoonAsPossible: false);

            var engine = BuildXLEngine.Create(
                LoggingContext,
                engineContext,
                config,
                factory,
                rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs
                );

            return(engine);
        }
Ejemplo n.º 3
0
        private RootFilter ParseFilter(IEnumerable <string> values, string commandLineFilter, string defaultFilter, out SymbolTable symbolTable, out PathTable pathTable)
        {
            var loggingContext = new LoggingContext("EngineScheduleTests.ParseFilter");

            pathTable = new PathTable();
            // EngineSchedulTest operate on the real filesystem for now
            var            fileSystem = new PassThroughFileSystem(pathTable);
            BuildXLContext context    = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);

            symbolTable = context.SymbolTable;

            var config = new CommandLineConfiguration
            {
                Filter  = commandLineFilter,
                Startup =
                {
                    ImplicitFilters = new List <string>(values ?? Enumerable.Empty <string>()),
                },
                Engine =
                {
                    DefaultFilter = defaultFilter,
                }
            };

            RootFilter rootFilter;

            XAssert.IsTrue(EngineSchedule.TryGetPipFilter(loggingContext, context, config, config, TryGetPathByMountName,
                                                          rootFilter: out rootFilter));
            return(rootFilter);
        }
Ejemplo n.º 4
0
        public static AppState TryCreateWorkspace(
            TextDocumentManager documentManager,
            Uri rootFolder,
            EventHandler <WorkspaceProgressEventArgs> progressHandler,
            TestContext?testContext  = null,
            DScriptSettings settings = null)
        {
            documentManager = documentManager ?? new TextDocumentManager(new PathTable());
            // Need to clear event handlers to detach old instances like IncrementalWorkspaceProvider from the new events.
            documentManager.ClearEventHandlers();

            var pathTable = documentManager.PathTable;

            // Check if we need to prepopulate the document manager
            // with some documents (for testing)
            if (testContext.HasValue)
            {
                foreach (var document in testContext.Value.PrePopulatedDocuments)
                {
                    documentManager.Add(AbsolutePath.Create(pathTable, document.Uri), document);
                }
            }

            // Bootstrap the DScript frontend to parse/type-check the workspace given a build config
            var loggingContext = new LoggingContext("LanguageServer");
            var fileSystem     = new PassThroughFileSystem(pathTable);
            var engineContext  = EngineContext.CreateNew(
                cancellationToken: System.Threading.CancellationToken.None,
                pathTable: pathTable,
                fileSystem: fileSystem);
            var engineAbstraction = new LanguageServiceEngineAbstraction(documentManager, pathTable, engineContext.FileSystem);

            var frontEndContext = engineContext.ToFrontEndContext(loggingContext);

            var rootFolderAsAbsolutePath = rootFolder.ToAbsolutePath(pathTable);

            if (!WorkspaceBuilder.TryBuildWorkspaceForIde(
                    frontEndContext: frontEndContext,
                    engineContext: engineContext,
                    frontEndEngineAbstraction: engineAbstraction,
                    rootFolder: rootFolderAsAbsolutePath,
                    progressHandler: progressHandler,
                    workspace: out var workspace,
                    skipNuget: settings?.SkipNuget ?? false, // Do not skip nuget by default.
                    controller: out var controller))
            {
                // The workspace builder fails when the workspace contains any errors
                // however, we bail out only if the workspace is null, which happens only
                // when the config could not be parsed
                if (workspace == null || controller == null)
                {
                    return(null);
                }
            }

            var incrementalWorkspaceProvider = new IncrementalWorkspaceProvider(controller, pathTable, workspace, documentManager, testContext);

            return(new AppState(engineAbstraction, documentManager, pathTable, incrementalWorkspaceProvider, workspace));
        }
Ejemplo n.º 5
0
        /// <summary>
        /// Generates pip graph fragment.
        /// </summary>
        public static bool TryGeneratePipGraphFragment(
            PathTable pathTable,
            ICommandLineConfiguration commandLineConfig,
            PipGraphFragmentGeneratorConfiguration pipGraphFragmentConfig)
        {
            var loggingContext = new LoggingContext(nameof(PipGraphFragmentGenerator));
            var fileSystem     = new PassThroughFileSystem(pathTable);
            var engineContext  = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);

            FrontEndContext context = engineContext.ToFrontEndContext(loggingContext);

            // Parse filter string.

            var evaluationFilter = EvaluationFilter.Empty;

            if (!string.IsNullOrWhiteSpace(commandLineConfig.Filter))
            {
                if (!TryGetEvaluationFilter(loggingContext, engineContext, commandLineConfig.Filter, out evaluationFilter))
                {
                    // Error should have been been reported already.
                    return(false);
                }
            }

            if (!TryBuildPipGraphFragment(
                    commandLineConfig,
                    pipGraphFragmentConfig,
                    context,
                    engineContext,
                    evaluationFilter))
            {
                return(false);
            }

            return(true);
        }
Ejemplo n.º 6
0
        protected void RestartEngine()
        {
            var pathTable = new PathTable();

            FileSystem = new PassThroughMutableFileSystem(pathTable);
            Context    = EngineContext.CreateNew(CancellationToken.None, pathTable, FileSystem);
            MainSourceResolverModules = new List <DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition> >();

            var rootPath = AbsolutePath.Create(Context.PathTable, TestRoot);
            var logsPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "logs");

            Configuration = new CommandLineConfiguration()
            {
                DisableDefaultSourceResolver = true,
                Resolvers = new List <IResolverSettings>
                {
                    new SourceResolverSettings
                    {
                        Kind    = "SourceResolver",
                        Modules = MainSourceResolverModules,
                    },
                    new SourceResolverSettings
                    {
                        Kind    = "SourceResolver",
                        Modules = new List <DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition> >
                        {
                            new DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition>(
                                AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Prelude", "package.config.dsc"))),
                            new DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition>(
                                AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Transformers", "package.config.dsc"))),
                            new DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition>(
                                AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Deployment", "module.config.dsc"))),
                        },
                    },
                },
                Layout =
                {
                    SourceDirectory = rootPath,
                    OutputDirectory = Combine(rootPath,                             "out"),
                    ObjectDirectory = Combine(rootPath,                             "obj"),
                    CacheDirectory  = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "cache"),
                },
                Cache =
                {
                    CacheSpecs       = SpecCachingOption.Disabled,
                    CacheLogFilePath = logsPath.Combine(Context.PathTable,PathAtom.Create(Context.StringTable,  "cache.log")),
                },
                Engine =
                {
                    ReuseEngineState        = false,
                    LogStatistics           = false,
                    TrackBuildsInUserFolder = false,
                },
                FrontEnd =
                {
                    MaxFrontEndConcurrency =     1,
                    LogStatistics          = false,
                },
                Schedule =
                {
                    MaxIO        = 1,
                    MaxProcesses = 1,
                },
                Sandbox =
                {
                    FileSystemMode      = FileSystemMode.RealAndMinimalPipGraph,
                    OutputReportingMode = OutputReportingMode.FullOutputOnError,
                },
                Logging =
                {
                    LogsDirectory     = logsPath,
                    LogStats          = false,
                    LogExecution      = false,
                    LogCounters       = false,
                    LogMemory         = false,
                    StoreFingerprints = false,
                    NoWarnings        =
                    {
                        909,     // Disable warnings about experimental feature
                    },
                }
            };

            if (TryGetSubstSourceAndTarget(out string substSource, out string substTarget))
            {
                // Directory translation is needed here particularly when the test temporary directory
                // is inside a directory that is actually a junction to another place.
                // For example, the temporary directory is D:\src\BuildXL\Out\Object\abc\t_1, but
                // the path D:\src\BuildXL\Out or D:\src\BuildXL\Out\Object is a junction to K:\Out.
                // Some tool, like cmd, can access the path in K:\Out, and thus the test will have a DFA
                // if there's no directory translation.
                // This problem does not occur when only substs are involved, but no junctions. The method
                // TryGetSubstSourceAndTarget works to get translations due to substs or junctions.
                AbsolutePath substSourcePath = AbsolutePath.Create(Context.PathTable, substSource);
                AbsolutePath substTargetPath = AbsolutePath.Create(Context.PathTable, substTarget);
                Configuration.Engine.DirectoriesToTranslate.Add(
                    new TranslateDirectoryData(I($"{substSource}<{substTarget}"), substSourcePath, substTargetPath));
            }
        }
Ejemplo n.º 7
0
        protected BaseEngineTest(ITestOutputHelper output)
            : base(output)
        {
            m_testOutput     = output;
            m_ignoreWarnings = OperatingSystemHelper.IsUnixOS; // ignoring /bin/sh is being used as a source file

            RegisterEventSource(global::BuildXL.Scheduler.ETWLogger.Log);
            RegisterEventSource(global::BuildXL.FrontEnd.Script.ETWLogger.Log);
            RegisterEventSource(global::BuildXL.FrontEnd.Core.ETWLogger.Log);
            RegisterEventSource(global::BuildXL.Engine.ETWLogger.Log);
            RegisterEventSource(global::BuildXL.Processes.ETWLogger.Log);

            ParseAndEvaluateLogger = Logger.CreateLogger();
            InitializationLogger   = InitializationLogger.CreateLogger();

            var pathTable = new PathTable();

            FileSystem = new PassThroughMutableFileSystem(pathTable);
            Context    = EngineContext.CreateNew(CancellationToken.None, pathTable, FileSystem);
            MainSourceResolverModules = new List <AbsolutePath>();

            var rootPath = AbsolutePath.Create(Context.PathTable, TestRoot);
            var logsPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "logs");

            Configuration = new CommandLineConfiguration()
            {
                DisableDefaultSourceResolver = true,
                Resolvers = new List <IResolverSettings>
                {
                    new SourceResolverSettings
                    {
                        Kind    = "SourceResolver",
                        Modules = MainSourceResolverModules,
                    },
                    new SourceResolverSettings
                    {
                        Kind    = "SourceResolver",
                        Modules = new List <AbsolutePath>
                        {
                            AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Prelude", "package.config.dsc")),
                            AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Transformers", "package.config.dsc")),
                            AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Deployment", "module.config.dsc")),
                        },
                    },
                },
                Layout =
                {
                    SourceDirectory = rootPath,
                    OutputDirectory = Combine(rootPath,                             "out"),
                    ObjectDirectory = Combine(rootPath,                             "obj"),
                    CacheDirectory  = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "cache"),
                },
                Cache =
                {
                    CacheSpecs       = SpecCachingOption.Disabled,
                    CacheLogFilePath = logsPath.Combine(Context.PathTable,PathAtom.Create(Context.StringTable,  "cache.log")),
                },
                Engine =
                {
                    ReuseEngineState        = false,
                    LogStatistics           = false,
                    TrackBuildsInUserFolder = false,
                },
                FrontEnd =
                {
                    MaxFrontEndConcurrency =     1,
                    LogStatistics          = false,
                },
                Schedule =
                {
                    MaxIO        = 1,
                    MaxProcesses = 1,
                },
                Sandbox =
                {
                    FileSystemMode      = FileSystemMode.RealAndMinimalPipGraph,
                    OutputReportingMode = OutputReportingMode.FullOutputOnError,
                },
                Logging =
                {
                    LogsDirectory     = logsPath,
                    LogStats          = false,
                    LogExecution      = false,
                    LogCounters       = false,
                    LogMemory         = false,
                    StoreFingerprints = false,
                    NoWarnings        =
                    {
                        909,     // Disable warnings about experimental feature
                    },
                }
            };

            AbsolutePath Combine(AbsolutePath parent, string name)
            {
                return(parent.Combine(Context.PathTable, PathAtom.Create(Context.StringTable, name)));
            }
        }
Ejemplo n.º 8
0
        public TestEnv(
            string name,
            string rootPath,
            bool enableLazyOutputMaterialization = false,
            int maxRelativeOutputDirectoryLength = 260,
            List <IMount> mounts = null,
            PathTable pathTable  = null)
        {
            Contract.Requires(name != null);
            Contract.Requires(!string.IsNullOrEmpty(rootPath));

            LoggingContext = new LoggingContext("TestLogger." + name);
            PathTable      = pathTable ?? new PathTable();

            PipDataBuilderPool = new ObjectPool <PipDataBuilder>(() => new PipDataBuilder(PathTable.StringTable), _ => { });

            // The tests that use TestEnv need to be modernized to take a filesystem
            var fileSystem = new PassThroughFileSystem(PathTable);

            Context = EngineContext.CreateNew(CancellationToken.None, PathTable, fileSystem);

            // Add some well-known paths with fixed casing to the Context.PathTable
            AbsolutePath.Create(Context.PathTable, rootPath.ToLowerInvariant());
            var root = AbsolutePath.Create(Context.PathTable, rootPath);

            var configuration = ConfigHelpers.CreateDefaultForXml(Context.PathTable, root);

            configuration.Layout.SourceDirectory = root.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "src")); // These tests have non-standard src folder
            configuration.Engine.MaxRelativeOutputDirectoryLength   = maxRelativeOutputDirectoryLength;
            configuration.Schedule.EnableLazyOutputMaterialization  = enableLazyOutputMaterialization;
            configuration.Schedule.UnsafeDisableGraphPostValidation = false;
            configuration.Schedule.ComputePipStaticFingerprints     = true;
            configuration.Sandbox.FileAccessIgnoreCodeCoverage      = true;

            BuildXLEngine.PopulateFileSystemCapabilities(configuration, configuration, Context.PathTable, LoggingContext);
            BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
            BuildXLEngine.PopulateAndValidateConfiguration(configuration, configuration, Context.PathTable, LoggingContext);

            Configuration = configuration;

            var mountsTable = MountsTable.CreateAndRegister(LoggingContext, Context, Configuration, null);

            if (mounts != null)
            {
                foreach (var mount in mounts)
                {
                    mountsTable.AddResolvedMount(mount);
                }
            }

            AbsolutePath specFile = SourceRoot.CreateRelative(Context.PathTable, "TestSpecFile.dsc");

            var graph = TestSchedulerFactory.CreateEmptyPipGraph(Context, configuration, mountsTable.MountPathExpander);

            PipTable = graph.PipTable;
            PipGraph = graph;

            var locationData = new LocationData(specFile, 0, 0);
            var modulePip    = ModulePip.CreateForTesting(Context.StringTable, specFile);

            PipGraph.AddModule(modulePip);
            PipGraph.AddSpecFile(new SpecFilePip(FileArtifact.CreateSourceFile(specFile), locationData, modulePip.Module));

            PipConstructionHelper = PipConstructionHelper.CreateForTesting(
                Context,
                ObjectRoot,
                redirectedRoot: Configuration.Layout.RedirectedDirectory,
                pipGraph: PipGraph,
                moduleName: modulePip.Identity.ToString(Context.StringTable),
                symbol: name,
                specPath: specFile);

            Paths = new Paths(PathTable);

            mountsTable.CompleteInitialization();
        }
Ejemplo n.º 9
0
        /// <summary>
        /// Builds a workspace and uses filter to find specs to evaluate.
        /// </summary>
        public static bool TryBuildWorkspaceAndCollectFilesToAnalyze(
            Tracing.Logger logger,
            PathTable pathTable,
            ICommandLineConfiguration configuation,
            out Workspace workspace,
            out IPipGraph pipGraph,
            out IReadOnlyDictionary <AbsolutePath, ISourceFile> filesToAnalyze,
            out FrontEndContext context)
        {
            workspace      = null;
            pipGraph       = null;
            filesToAnalyze = null;

            var loggingContext = new LoggingContext("DScriptAnalyzer");
            var fileSystem     = new PassThroughFileSystem(pathTable);
            var engineContext  = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);

            context = engineContext.ToFrontEndContext(loggingContext);

            // Parse filter string into EvaluationFilter
            var evaluationFilter = EvaluationFilter.Empty;

            if (!string.IsNullOrEmpty(configuation.Filter))
            {
                if (!TryGetEvaluationFilter(logger, loggingContext, engineContext, configuation.Filter, out evaluationFilter))
                {
                    // Error has been reported already
                    return(false);
                }
            }

            // Try parsing the workspace from config and evaluation filter
            if (!TryBuildWorkspace(
                    configuation,
                    context,
                    engineContext,
                    evaluationFilter,
                    progressHandler: null,
                    workspace: out workspace,
                    frontEndHostController: out _,
                    pipGraph: out pipGraph,
                    configuration: GetDefaultConfiguration()))
            {
                return(false);
            }

            if (configuation.Engine.Phase == EnginePhases.AnalyzeWorkspace)
            {
                // Find strict subset of specs in workspace that should be analyzed
                var collectedFilesToAnalyze = CollectFilesToAnalyze(
                    workspace,
                    pathTable,
                    configuation.Startup.ConfigFile,
                    evaluationFilter);

                if (collectedFilesToAnalyze.Count == 0)
                {
                    logger.ErrorFilterHasNoMatchingSpecs(loggingContext, configuation.Filter);
                    return(false);
                }

                filesToAnalyze = collectedFilesToAnalyze;
            }
            else
            {
                filesToAnalyze = new Dictionary <AbsolutePath, ISourceFile>();
            }

            return(true);
        }