/// <summary> /// Initialize the test /// </summary> public DirectoryMembershipFingerprinterTests(ITestOutputHelper output) : base(output) { m_context = BuildXLContext.CreateInstanceForTesting(); m_pipTable = new PipTable(m_context.PathTable, m_context.SymbolTable, initialBufferSize: 1024, maxDegreeOfParallelism: 1, debug: true); m_mountInfo = new MountPathExpander(m_context.PathTable); RegisterEventSource(global::BuildXL.Scheduler.ETWLogger.Log); }
public void ScrubbingDirectoriesWithMounts() { string rootDirectory = Path.Combine(TestOutputDirectory, nameof(ScrubbingDirectoriesWithMounts)); const string NonScrubbableMountName = "NonScrubbable"; string nonScrubbableMountPath = Path.Combine(rootDirectory, NonScrubbableMountName); const string ScrubbableMountName = "Scrubbable"; string scrubbableMountPath = Path.Combine(rootDirectory, ScrubbableMountName); const string NestedScrubbableMountName = "NestedScrubbable"; string nestedScrubbableMountPath = Path.Combine(scrubbableMountPath, NestedScrubbableMountName); MountPathExpander mountPathExpander = CreateMountPathExpander( new TestMount(NonScrubbableMountName, nonScrubbableMountPath, MountFeatures.Writable | MountFeatures.Readable), new TestMount(ScrubbableMountName, scrubbableMountPath, MountFeatures.Scrubbable), new TestMount(NestedScrubbableMountName, nestedScrubbableMountPath, MountFeatures.Scrubbable)); string f = WriteFile(Path.Combine(nonScrubbableMountPath, "D", "f")); string g1 = WriteFile(Path.Combine(scrubbableMountPath, "D", "g1")); string g2 = WriteFile(Path.Combine(scrubbableMountPath, "D", "g2")); string h = WriteFile(Path.Combine(scrubbableMountPath, "D", "E", "h")); string i = WriteFile(Path.Combine(scrubbableMountPath, "D", "F", "i")); string j = WriteFile(Path.Combine(nestedScrubbableMountPath, "D", "j")); var inBuild = new HashSet <string>(StringComparer.OrdinalIgnoreCase) { Path.Combine(scrubbableMountPath, "D", "E"), g1 }; var scrubber = new DirectoryScrubber( LoggingContext, m_loggingConfiguration, path => inBuild.Contains(path), pathsToScrub: new[] { Path.Combine(nonScrubbableMountPath, "D"), scrubbableMountPath }, blockedPaths: new string[0], nonDeletableRootDirectories: new string[0], mountPathExpander: mountPathExpander, maxDegreeParallelism: 2); scrubber.RemoveExtraneousFilesAndDirectories(new CancellationToken()); // NonScrubbable\D produces a warning. AssertWarningEventLogged(EventId.ScrubbingFailedBecauseDirectoryIsNotScrubbable); // f is in NonScrubbable. XAssert.IsTrue(File.Exists(f)); // g1 & h (via Scrubbable\D\E) is in build. XAssert.IsTrue(File.Exists(g1)); XAssert.IsTrue(File.Exists(h)); // NestedScrubbable, although not in build, but is a mount root. XAssert.IsTrue(Directory.Exists(nestedScrubbableMountPath)); // Scrubbed files/directories. XAssert.IsFalse(File.Exists(g2)); XAssert.IsFalse(File.Exists(i)); XAssert.IsFalse(Directory.Exists(Path.Combine(scrubbableMountPath, "D", "F"))); XAssert.IsFalse(File.Exists(j)); XAssert.IsFalse(Directory.Exists(Path.Combine(nestedScrubbableMountPath, "D"))); }
private static void AddMount(MountPathExpander tokenizer, PathTable pathTable, AbsolutePath path, string name, bool isSystem = false) { tokenizer.Add( pathTable, new Configuration.Mutable.Mount() { Name = PathAtom.Create(pathTable.StringTable, name), Path = path, IsSystem = isSystem }); }
public void ScrubbingDirectoriesWithMounts() { string rootDirectory = Path.Combine(TemporaryDirectory, nameof(ScrubbingDirectoriesWithMounts)); const string NonScrubbableMountName = "NonScrubbable"; string nonScrubbableMountPath = Path.Combine(rootDirectory, NonScrubbableMountName); const string ScrubbableMountName = "Scrubbable"; string scrubbableMountPath = Path.Combine(rootDirectory, ScrubbableMountName); const string NestedScrubbableMountName = "NestedScrubbable"; string nestedScrubbableMountPath = Path.Combine(scrubbableMountPath, NestedScrubbableMountName); MountPathExpander mountPathExpander = CreateMountPathExpander( new TestMount(NonScrubbableMountName, nonScrubbableMountPath, MountFeatures.Writable | MountFeatures.Readable), new TestMount(ScrubbableMountName, scrubbableMountPath, MountFeatures.Scrubbable), new TestMount(NestedScrubbableMountName, nestedScrubbableMountPath, MountFeatures.Scrubbable)); string f = WriteFile(Path.Combine(nonScrubbableMountPath, "D", "f")); string g1 = WriteFile(Path.Combine(scrubbableMountPath, "D", "g1")); string g2 = WriteFile(Path.Combine(scrubbableMountPath, "D", "g2")); string h = WriteFile(Path.Combine(scrubbableMountPath, "D", "E", "h")); string i = WriteFile(Path.Combine(scrubbableMountPath, "D", "F", "i")); string j = WriteFile(Path.Combine(nestedScrubbableMountPath, "D", "j")); var inBuild = new HashSet <string>(OperatingSystemHelper.PathComparer) { Path.Combine(scrubbableMountPath, "D", "E"), g1 }; Scrubber.RemoveExtraneousFilesAndDirectories( path => inBuild.Contains(path), pathsToScrub: new[] { Path.Combine(nonScrubbableMountPath, "D"), scrubbableMountPath }, blockedPaths: new string[0], nonDeletableRootDirectories: new string[0], mountPathExpander: mountPathExpander); // NonScrubbable\D produces a warning. AssertWarningEventLogged(global::BuildXL.Engine.Tracing.LogEventId.ScrubbingFailedBecauseDirectoryIsNotScrubbable); // f is in NonScrubbable. XAssert.IsTrue(File.Exists(f)); // g1 & h (via Scrubbable\D\E) is in build. XAssert.IsTrue(File.Exists(g1)); XAssert.IsTrue(File.Exists(h)); // NestedScrubbable, although not in build, but is a mount root. XAssert.IsTrue(Directory.Exists(nestedScrubbableMountPath)); // Scrubbed files/directories. XAssert.IsFalse(File.Exists(g2)); XAssert.IsFalse(File.Exists(i)); XAssert.IsFalse(Directory.Exists(Path.Combine(scrubbableMountPath, "D", "F"))); XAssert.IsFalse(File.Exists(j)); XAssert.IsFalse(Directory.Exists(Path.Combine(nestedScrubbableMountPath, "D"))); }
/// <summary> /// Creates an instance of <see cref="TestPipGraphFragment"/>. /// </summary> public TestPipGraphFragment(LoggingContext loggingContext, string sourceRoot, string objectRoot, string redirectedRoot, string moduleName, bool useTopSort = false) { Contract.Requires(loggingContext != null); Contract.Requires(!string.IsNullOrEmpty(sourceRoot)); Contract.Requires(!string.IsNullOrEmpty(objectRoot)); Contract.Requires(!string.IsNullOrEmpty(moduleName)); Context = BuildXLContext.CreateInstanceForTesting(); m_loggingContext = loggingContext; m_sourceRoot = AbsolutePath.Create(Context.PathTable, sourceRoot); m_objectRoot = AbsolutePath.Create(Context.PathTable, objectRoot); m_expander = new MountPathExpander(Context.PathTable); var configuration = new ConfigurationImpl() { Schedule = { UseFixedApiServerMoniker = true, ComputePipStaticFingerprints = true, } }; m_useTopSort = useTopSort; PipGraph = m_useTopSort ? new PipGraphFragmentBuilderTopSort(Context, configuration, m_expander) : new PipGraphFragmentBuilder(Context, configuration, m_expander); ModuleName = moduleName; var specFileName = moduleName + ".dsc"; m_specPath = m_sourceRoot.Combine(Context.PathTable, specFileName); m_moduleId = ModuleId.Create(StringId.Create(Context.StringTable, moduleName)); var modulePip = ModulePip.CreateForTesting( Context.StringTable, m_specPath, m_moduleId); PipGraph.AddModule(modulePip); PipGraph.AddSpecFile(new SpecFilePip(new FileArtifact(m_specPath), new LocationData(m_specPath, 0, 0), modulePip.Module)); m_defaultConstructionHelper = PipConstructionHelper.CreateForTesting( Context, objectRoot: m_objectRoot, redirectedRoot: AbsolutePath.Create(Context.PathTable, redirectedRoot), pipGraph: PipGraph, moduleName: moduleName, specRelativePath: Path.Combine(m_sourceRoot.GetName(Context.PathTable).ToString(Context.StringTable), specFileName), specPath: m_specPath, symbol: moduleName + "_defaultValue"); }
public async Task TokenizedPathSet() { var pathTable = new PathTable(); var pathExpanderA = new MountPathExpander(pathTable); AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/users/AUser")), "UserProfile", isSystem: true); AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/windows")), "Windows", isSystem: true); AddMount(pathExpanderA, pathTable, AbsolutePath.Create(pathTable, X("/x/test")), "TestRoot", isSystem: false); var pathSetA = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/x/abc"), X("/x/users/AUser/def"), X("/x/windows"), X("/x/test/abc")); ObservedPathSet roundtripA = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetA); XAssert.AreEqual(4, roundtripA.Paths.Length); ContentHash pathSetHashA = await pathSetA.ToContentHash(pathTable, pathExpanderA); var pathExpanderB = new MountPathExpander(pathTable); AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/users/BUser")), "UserProfile", isSystem: true); AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/windows")), "Windows", isSystem: true); AddMount(pathExpanderB, pathTable, AbsolutePath.Create(pathTable, X("/y/abc/test")), "TestRoot", isSystem: false); var pathSetB = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/x/abc"), X("/y/users/BUser/def"), X("/y/windows"), X("/y/abc/test/abc")); ObservedPathSet roundtripB = SerializeRoundTripAndAssertEquivalent(pathTable, pathSetB); XAssert.AreEqual(4, roundtripB.Paths.Length); ContentHash pathSetHashB = await pathSetB.ToContentHash(pathTable, pathExpanderB); AssertTrue(pathSetHashA == pathSetHashB); }
private void RunScrubberWithPipGraph( TestEnv env, PipGraph pipGraph, string[] pathsToScrub, MountPathExpander mountPathExpander = null) { var scrubber = new DirectoryScrubber( LoggingContext, m_loggingConfiguration, isPathInBuild: p => pipGraph.IsPathInBuild(env.Paths.CreateAbsolutePath(p)), pathsToScrub: pathsToScrub, blockedPaths: new string[0], nonDeletableRootDirectories: pipGraph.AllDirectoriesContainingOutputs().Select(p => env.Paths.Expand(p)), mountPathExpander: mountPathExpander, maxDegreeParallelism: 2); bool removed = scrubber.RemoveExtraneousFilesAndDirectories(new CancellationToken()); XAssert.IsTrue(removed); }
private static MountPathExpander CreateMountPathExpander(params TestMount[] mounts) { var pathTable = new PathTable(); var mountPathExpander = new MountPathExpander(pathTable); foreach (var mount in mounts) { mountPathExpander.Add( pathTable, new Mount { Name = PathAtom.Create(pathTable.StringTable, mount.Name), Path = AbsolutePath.Create(pathTable, mount.Path), IsReadable = mount.Features.HasFlag(MountFeatures.Readable), IsWritable = mount.Features.HasFlag(MountFeatures.Writable), IsScrubbable = mount.Features.HasFlag(MountFeatures.Scrubbable), TrackSourceFileChanges = mount.Features.HasFlag(MountFeatures.Hashable) }); } return(mountPathExpander); }
public override void Prepare() { base.Prepare(); if (!m_tokenizeByMounts) { return; } m_mountPathExpander = new MountPathExpander(PathTable); if (InclusionMountNames.Count == 0) { InclusionMountNames.AddRange(CachedGraph.MountPathExpander.GetAllRoots().Select(r => CachedGraph.MountPathExpander.GetSemanticPathInfo(r).RootName.ToString(StringTable))); } foreach (var name in InclusionMountNames.Distinct(StringComparer.OrdinalIgnoreCase)) { if (CachedGraph.MountPathExpander.TryGetRootByMountName(name, out var mountRoot)) { m_mountPathExpander.Add(PathTable, CachedGraph.MountPathExpander.GetSemanticPathInfo(mountRoot), forceTokenize: true); } } }
/// <summary> /// Run the test /// </summary> public bool Run(string testFolder, string specFile, string fullIdentifier, string shortName, string lkgFile, params string[] sdksToResolve) { Contract.Requires(!string.IsNullOrEmpty(testFolder)); Contract.Requires(!string.IsNullOrEmpty(specFile)); Contract.Requires(sdksToResolve != null); // Sadly the frontend doesn't use the engine abstractions file api's so we have to materialize stuff on disk for now... // TODO: Fix this code once the frontend supports a proper virtual FileSystem. // TODO: Change the package semantics to implicit when we expose a way to evaluate a single value var testFileName = Path.GetFileName(specFile); var mainFileName = "testMain.bp"; var testMainFile = Path.Combine(testFolder, mainFileName); Directory.CreateDirectory(testFolder); File.WriteAllText(Path.Combine(testFolder, Names.ModuleConfigBm), I($@"module( {{ name: 'TestPackage', nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences, projects: [ f`{mainFileName}`, f`{testFileName}`, ], }});")); File.WriteAllText(testMainFile, I($@" export const testFolder = d`{Path.GetDirectoryName(specFile).Replace('\\', '/')}`; @@public export const main = {fullIdentifier}();")); File.Copy(specFile, Path.Combine(testFolder, testFileName)); // Create a fake package for Sdk.TestRunner so that you can safely test packages that have the tests embedded in them. var testRunnerFolder = Path.Combine(testFolder, "Sdk.TestRunner"); Directory.CreateDirectory(testRunnerFolder); File.WriteAllText(Path.Combine(testRunnerFolder, Names.ModuleConfigBm), I($"module({{\n\tname: 'Sdk.TestRunner',\n}});")); File.WriteAllText(Path.Combine(testRunnerFolder, "package" + Names.DotDscExtension), I($@" export interface TestArguments {{ testFiles: File[]; sdkFolders?: (Directory|StaticDirectory)[]; autoFixLkgs?: boolean; }} export interface TestResult {{ xmlResults: File; }} export function test(args: TestArguments): TestResult {{ Contract.fail(""Can't run a DScript UnitTest inside of a DScript UnitTest""); }}")); // Setup Context and configuration var frontEndContext = FrontEndContext.CreateInstanceForTesting(); var pipContext = new SchedulerContext(CancellationToken.None, frontEndContext.StringTable, frontEndContext.PathTable, frontEndContext.SymbolTable, frontEndContext.QualifierTable); var pathTable = frontEndContext.PathTable; var testFolderPath = AbsolutePath.Create(pathTable, testFolder); var configuration = CreateConfiguration(sdksToResolve.Union(new[] { testRunnerFolder }), pathTable, testFolderPath); var engineAbstraction = new TestEngineAbstraction(pathTable, frontEndContext.StringTable, testFolderPath, new PassThroughFileSystem(pathTable)); var frontEndStatistics = new FrontEndStatistics(); if (!CreateFactories( frontEndContext, engineAbstraction, frontEndStatistics, configuration, out var ambientTesting, out var moduleRegistry, out var frontEndFactory)) { return(false); } // Set the timeout to a large number to avoid useless performance collections in tests. using (var performanceCollector = new PerformanceCollector(TimeSpan.FromHours(1))) using (var frontEndHostController = new FrontEndHostController( frontEndFactory, new EvaluationScheduler(1), moduleRegistry, frontEndStatistics, m_tracingLogger, performanceCollector, collectMemoryAsSoonAsPossible: true)) { var frontEndController = (IFrontEndController)frontEndHostController; frontEndController.InitializeHost(frontEndContext, configuration); frontEndController.ParseConfig(configuration); // Populate the graph using (var pipTable = new PipTable( pipContext.PathTable, pipContext.SymbolTable, initialBufferSize: 16384, maxDegreeOfParallelism: 1, debug: true)) { var mountPathExpander = new MountPathExpander(pathTable); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "testFolder"), testFolderPath, allowHashing: true, readable: true, writable: false)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "src"), testFolderPath.Combine(pathTable, "src"), allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "out"), testFolderPath.Combine(pathTable, "out"), allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "noRead"), testFolderPath.Combine(pathTable, "noRead"), allowHashing: true, readable: false, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "temp"), engineAbstraction.Layout.TempDirectory, allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "obj"), engineAbstraction.Layout.ObjectDirectory, allowHashing: true, readable: true, writable: true)); var graph = new PipGraph.Builder( pipTable, pipContext, m_pipLogger, frontEndContext.LoggingContext, configuration, mountPathExpander); using (var cacheLayer = new EngineCache( new InMemoryArtifactContentCache(), new InMemoryTwoPhaseFingerprintStore())) { var cache = Task.FromResult(Possible.Create(cacheLayer)); try { var evaluationFilter = new EvaluationFilter( pipContext.SymbolTable, pipContext.PathTable, new FullSymbol[0], new[] { AbsolutePath.Create(frontEndContext.PathTable, testMainFile), }, CollectionUtilities.EmptyArray <StringId>()); if (!frontEndController.PopulateGraph(cache, graph, engineAbstraction, evaluationFilter, configuration, configuration.Startup)) { HandleDiagnostics(); return(false); } } catch (AggregateException e) { var baseException = e.GetBaseException(); if (baseException is XunitException) { // If it is an XUnit assert, then unwrap the exception and throw that because XUnit other doesn't display the error nicely. ExceptionDispatchInfo.Capture(baseException).Throw(); } throw; } } if (!ValidatePips(frontEndContext, graph, testFolderPath, specFile, shortName, lkgFile, ambientTesting.DontValidatePipsEnabled)) { return(false); } } } HandleDiagnostics(); return(true); }
/// <summary> /// Runs the scheduler allowing various options to be specifically set /// </summary> public ScheduleRunResult RunSchedulerSpecific( PipGraph graph, SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, TempCleaner tempCleaner = null) { var config = new CommandLineConfiguration(Configuration); // Populating the configuration may modify the configuration, so it should occur first. BuildXLEngine.PopulateLoggingAndLayoutConfiguration(config, Context.PathTable, bxlExeLocation: null, inTestMode: true); BuildXLEngine.PopulateAndValidateConfiguration(config, config, Context.PathTable, LoggingContext); FileAccessWhitelist whitelist = new FileAccessWhitelist(Context); whitelist.Initialize(config); IReadOnlyList <string> junctionRoots = Configuration.Engine.DirectoriesToTranslate?.Select(a => a.ToPath.ToString(Context.PathTable)).ToList(); var map = VolumeMap.TryCreateMapOfAllLocalVolumes(LoggingContext, junctionRoots); var optionalAccessor = TryGetJournalAccessor(map); // Although scan change journal is enabled, but if we cannot create an enabled journal accessor, then create a disabled one. m_journalState = map == null || !optionalAccessor.IsValid ? JournalState.DisabledJournal : JournalState.CreateEnabledJournal(map, optionalAccessor.Value); if (config.Schedule.IncrementalScheduling) { // Ensure that we can scan the journal when incremental scheduling is enabled. XAssert.IsTrue(m_journalState.IsEnabled, "Incremental scheduling requires that journal is enabled"); } // Seal the translator if not sealed DirectoryTranslator.Seal(); // ..................................................................................... // some dummy setup in order to get a PreserveOutputsSalt.txt file and an actual salt // ..................................................................................... string dummyCacheDir = Path.Combine(TemporaryDirectory, "Out", "Cache"); Directory.CreateDirectory(dummyCacheDir); // EngineSchedule tries to put the PreserveOutputsSalt.txt here ContentHash?previousOutputsSalt = EngineSchedule.PreparePreviousOutputsSalt(LoggingContext, Context.PathTable, config); Contract.Assert(previousOutputsSalt.HasValue); // ..................................................................................... testHooks = testHooks ?? new SchedulerTestHooks(); Contract.Assert(!(config.Engine.CleanTempDirectories && tempCleaner == null)); using (var queue = new PipQueue(config.Schedule)) using (var testQueue = new TestPipQueue(queue, LoggingContext, initiallyPaused: false)) using (var testScheduler = new TestScheduler( graph: graph, pipQueue: testQueue, context: Context, fileContentTable: FileContentTable, loggingContext: LoggingContext, cache: Cache, configuration: config, journalState: m_journalState, fileAccessWhitelist: whitelist, fingerprintSalt: Configuration.Cache.CacheSalt, directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable), tempCleaner: tempCleaner, previousInputsSalt: previousOutputsSalt.Value, successfulPips: null, failedPips: null, ipcProvider: null, directoryTranslator: DirectoryTranslator, testHooks: testHooks)) { MountPathExpander mountPathExpander = null; var frontEndNonScrubbablePaths = CollectionUtilities.EmptyArray <string>(); var nonScrubbablePaths = EngineSchedule.GetNonScrubbablePaths(Context.PathTable, config, frontEndNonScrubbablePaths, tempCleaner); EngineSchedule.ScrubExtraneousFilesAndDirectories(mountPathExpander, testScheduler, LoggingContext, config, nonScrubbablePaths, tempCleaner); if (filter == null) { EngineSchedule.TryGetPipFilter(LoggingContext, Context, config, config, Expander.TryGetRootByMountName, out filter); } XAssert.IsTrue(testScheduler.InitForMaster(LoggingContext, filter, schedulerState), "Failed to initialized test scheduler"); testScheduler.Start(LoggingContext); bool success = testScheduler.WhenDone().GetAwaiter().GetResult(); testScheduler.SaveFileChangeTrackerAsync(LoggingContext).Wait(); if (ShouldLogSchedulerStats) { // Logs are not written out normally during these tests, but LogStats depends on the existence of the logs directory // to write out the stats perf JSON file var logsDir = config.Logging.LogsDirectory.ToString(Context.PathTable); Directory.CreateDirectory(logsDir); testScheduler.LogStats(LoggingContext); } return(new ScheduleRunResult { Graph = graph, Config = config, Success = success, PipResults = testScheduler.PipResults, PipExecutorCounters = testScheduler.PipExecutionCounters, PathSets = testScheduler.PathSets, ProcessPipCountersByFilter = testScheduler.ProcessPipCountersByFilter, ProcessPipCountersByTelemetryTag = testScheduler.ProcessPipCountersByTelemetryTag, SchedulerState = new SchedulerState(testScheduler) }); } }