private void RunGraphFingerprintValueTest(string[] expectedValues, string commandLineFilter, string[] commandLineValues, bool expectSuccess = true)
{
var pathTable = new PathTable();
// EngineSchedulTest operate on the real filesystem for now
var fileSystem = new PassThroughFileSystem(pathTable);
BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
var config = new CommandLineConfiguration
{
Filter = commandLineFilter,
Startup = { ImplicitFilters = new List <string>(commandLineValues) },
Engine = { DefaultFilter = null }
};
EvaluationFilter evaluationFilter;
if (!expectSuccess)
{
XAssert.IsFalse(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));
}
else
{
XAssert.IsTrue(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));
foreach (FullSymbol symbol in evaluationFilter.ValueNamesToResolve)
{
string value = symbol.ToString(context.SymbolTable);
XAssert.IsTrue(expectedValues.Contains(value));
}
XAssert.AreEqual(expectedValues.Length, evaluationFilter.ValueNamesToResolve.Count());
}
}
protected virtual BuildXLEngine CreateEngine(ICommandLineConfiguration config,
AppDeployment appDeployment,
string testRootDirectory,
bool rememberAllChangedTrackedInputs,
Action <EngineTestHooksData> verifyEngineTestHooksData = null)
{
var engineContext = EngineContext.CreateNew(CancellationToken.None, PathTable, FileSystem);
var factory = FrontEndControllerFactory.Create(
FrontEndMode.NormalMode,
LoggingContext,
config,
// Set the timeout to a large number to avoid useless performance collections in tests.
new PerformanceCollector(TimeSpan.FromHours(1)),
collectMemoryAsSoonAsPossible: false);
var engine = BuildXLEngine.Create(
LoggingContext,
engineContext,
config,
factory,
rememberAllChangedTrackedInputs: rememberAllChangedTrackedInputs
);
return(engine);
}
private RootFilter ParseFilter(IEnumerable <string> values, string commandLineFilter, string defaultFilter, out SymbolTable symbolTable, out PathTable pathTable)
{
var loggingContext = new LoggingContext("EngineScheduleTests.ParseFilter");
pathTable = new PathTable();
// EngineSchedulTest operate on the real filesystem for now
var fileSystem = new PassThroughFileSystem(pathTable);
BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
symbolTable = context.SymbolTable;
var config = new CommandLineConfiguration
{
Filter = commandLineFilter,
Startup =
{
ImplicitFilters = new List <string>(values ?? Enumerable.Empty <string>()),
},
Engine =
{
DefaultFilter = defaultFilter,
}
};
RootFilter rootFilter;
XAssert.IsTrue(EngineSchedule.TryGetPipFilter(loggingContext, context, config, config, TryGetPathByMountName,
rootFilter: out rootFilter));
return(rootFilter);
}
public static AppState TryCreateWorkspace(
TextDocumentManager documentManager,
Uri rootFolder,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
TestContext?testContext = null,
DScriptSettings settings = null)
{
documentManager = documentManager ?? new TextDocumentManager(new PathTable());
// Need to clear event handlers to detach old instances like IncrementalWorkspaceProvider from the new events.
documentManager.ClearEventHandlers();
var pathTable = documentManager.PathTable;
// Check if we need to prepopulate the document manager
// with some documents (for testing)
if (testContext.HasValue)
{
foreach (var document in testContext.Value.PrePopulatedDocuments)
{
documentManager.Add(AbsolutePath.Create(pathTable, document.Uri), document);
}
}
// Bootstrap the DScript frontend to parse/type-check the workspace given a build config
var loggingContext = new LoggingContext("LanguageServer");
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(
cancellationToken: System.Threading.CancellationToken.None,
pathTable: pathTable,
fileSystem: fileSystem);
var engineAbstraction = new LanguageServiceEngineAbstraction(documentManager, pathTable, engineContext.FileSystem);
var frontEndContext = engineContext.ToFrontEndContext(loggingContext);
var rootFolderAsAbsolutePath = rootFolder.ToAbsolutePath(pathTable);
if (!WorkspaceBuilder.TryBuildWorkspaceForIde(
frontEndContext: frontEndContext,
engineContext: engineContext,
frontEndEngineAbstraction: engineAbstraction,
rootFolder: rootFolderAsAbsolutePath,
progressHandler: progressHandler,
workspace: out var workspace,
skipNuget: settings?.SkipNuget ?? false, // Do not skip nuget by default.
controller: out var controller))
{
// The workspace builder fails when the workspace contains any errors
// however, we bail out only if the workspace is null, which happens only
// when the config could not be parsed
if (workspace == null || controller == null)
{
return(null);
}
}
var incrementalWorkspaceProvider = new IncrementalWorkspaceProvider(controller, pathTable, workspace, documentManager, testContext);
return(new AppState(engineAbstraction, documentManager, pathTable, incrementalWorkspaceProvider, workspace));
}
/// <summary>
/// Generates pip graph fragment.
/// </summary>
public static bool TryGeneratePipGraphFragment(
PathTable pathTable,
ICommandLineConfiguration commandLineConfig,
PipGraphFragmentGeneratorConfiguration pipGraphFragmentConfig)
{
var loggingContext = new LoggingContext(nameof(PipGraphFragmentGenerator));
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
FrontEndContext context = engineContext.ToFrontEndContext(loggingContext);
// Parse filter string.
var evaluationFilter = EvaluationFilter.Empty;
if (!string.IsNullOrWhiteSpace(commandLineConfig.Filter))
{
if (!TryGetEvaluationFilter(loggingContext, engineContext, commandLineConfig.Filter, out evaluationFilter))
{
// Error should have been been reported already.
return(false);
}
}
if (!TryBuildPipGraphFragment(
commandLineConfig,
pipGraphFragmentConfig,
context,
engineContext,
evaluationFilter))
{
return(false);
}
return(true);
}
protected void RestartEngine()
{
var pathTable = new PathTable();
FileSystem = new PassThroughMutableFileSystem(pathTable);
Context = EngineContext.CreateNew(CancellationToken.None, pathTable, FileSystem);
MainSourceResolverModules = new List <DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition> >();
var rootPath = AbsolutePath.Create(Context.PathTable, TestRoot);
var logsPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "logs");
Configuration = new CommandLineConfiguration()
{
DisableDefaultSourceResolver = true,
Resolvers = new List <IResolverSettings>
{
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = MainSourceResolverModules,
},
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = new List <DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition> >
{
new DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition>(
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Prelude", "package.config.dsc"))),
new DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition>(
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Transformers", "package.config.dsc"))),
new DiscriminatingUnion <AbsolutePath, IInlineModuleDefinition>(
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Deployment", "module.config.dsc"))),
},
},
},
Layout =
{
SourceDirectory = rootPath,
OutputDirectory = Combine(rootPath, "out"),
ObjectDirectory = Combine(rootPath, "obj"),
CacheDirectory = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "cache"),
},
Cache =
{
CacheSpecs = SpecCachingOption.Disabled,
CacheLogFilePath = logsPath.Combine(Context.PathTable,PathAtom.Create(Context.StringTable, "cache.log")),
},
Engine =
{
ReuseEngineState = false,
LogStatistics = false,
TrackBuildsInUserFolder = false,
},
FrontEnd =
{
MaxFrontEndConcurrency = 1,
LogStatistics = false,
},
Schedule =
{
MaxIO = 1,
MaxProcesses = 1,
},
Sandbox =
{
FileSystemMode = FileSystemMode.RealAndMinimalPipGraph,
OutputReportingMode = OutputReportingMode.FullOutputOnError,
},
Logging =
{
LogsDirectory = logsPath,
LogStats = false,
LogExecution = false,
LogCounters = false,
LogMemory = false,
StoreFingerprints = false,
NoWarnings =
{
909, // Disable warnings about experimental feature
},
}
};
if (TryGetSubstSourceAndTarget(out string substSource, out string substTarget))
{
// Directory translation is needed here particularly when the test temporary directory
// is inside a directory that is actually a junction to another place.
// For example, the temporary directory is D:\src\BuildXL\Out\Object\abc\t_1, but
// the path D:\src\BuildXL\Out or D:\src\BuildXL\Out\Object is a junction to K:\Out.
// Some tool, like cmd, can access the path in K:\Out, and thus the test will have a DFA
// if there's no directory translation.
// This problem does not occur when only substs are involved, but no junctions. The method
// TryGetSubstSourceAndTarget works to get translations due to substs or junctions.
AbsolutePath substSourcePath = AbsolutePath.Create(Context.PathTable, substSource);
AbsolutePath substTargetPath = AbsolutePath.Create(Context.PathTable, substTarget);
Configuration.Engine.DirectoriesToTranslate.Add(
new TranslateDirectoryData(I($"{substSource}<{substTarget}"), substSourcePath, substTargetPath));
}
}
protected BaseEngineTest(ITestOutputHelper output)
: base(output)
{
m_testOutput = output;
m_ignoreWarnings = OperatingSystemHelper.IsUnixOS; // ignoring /bin/sh is being used as a source file
RegisterEventSource(global::BuildXL.Scheduler.ETWLogger.Log);
RegisterEventSource(global::BuildXL.FrontEnd.Script.ETWLogger.Log);
RegisterEventSource(global::BuildXL.FrontEnd.Core.ETWLogger.Log);
RegisterEventSource(global::BuildXL.Engine.ETWLogger.Log);
RegisterEventSource(global::BuildXL.Processes.ETWLogger.Log);
ParseAndEvaluateLogger = Logger.CreateLogger();
InitializationLogger = InitializationLogger.CreateLogger();
var pathTable = new PathTable();
FileSystem = new PassThroughMutableFileSystem(pathTable);
Context = EngineContext.CreateNew(CancellationToken.None, pathTable, FileSystem);
MainSourceResolverModules = new List <AbsolutePath>();
var rootPath = AbsolutePath.Create(Context.PathTable, TestRoot);
var logsPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "logs");
Configuration = new CommandLineConfiguration()
{
DisableDefaultSourceResolver = true,
Resolvers = new List <IResolverSettings>
{
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = MainSourceResolverModules,
},
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = new List <AbsolutePath>
{
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Prelude", "package.config.dsc")),
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Transformers", "package.config.dsc")),
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Deployment", "module.config.dsc")),
},
},
},
Layout =
{
SourceDirectory = rootPath,
OutputDirectory = Combine(rootPath, "out"),
ObjectDirectory = Combine(rootPath, "obj"),
CacheDirectory = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "cache"),
},
Cache =
{
CacheSpecs = SpecCachingOption.Disabled,
CacheLogFilePath = logsPath.Combine(Context.PathTable,PathAtom.Create(Context.StringTable, "cache.log")),
},
Engine =
{
ReuseEngineState = false,
LogStatistics = false,
TrackBuildsInUserFolder = false,
},
FrontEnd =
{
MaxFrontEndConcurrency = 1,
LogStatistics = false,
},
Schedule =
{
MaxIO = 1,
MaxProcesses = 1,
},
Sandbox =
{
FileSystemMode = FileSystemMode.RealAndMinimalPipGraph,
OutputReportingMode = OutputReportingMode.FullOutputOnError,
},
Logging =
{
LogsDirectory = logsPath,
LogStats = false,
LogExecution = false,
LogCounters = false,
LogMemory = false,
StoreFingerprints = false,
NoWarnings =
{
909, // Disable warnings about experimental feature
},
}
};
AbsolutePath Combine(AbsolutePath parent, string name)
{
return(parent.Combine(Context.PathTable, PathAtom.Create(Context.StringTable, name)));
}
}
public TestEnv(
string name,
string rootPath,
bool enableLazyOutputMaterialization = false,
int maxRelativeOutputDirectoryLength = 260,
List <IMount> mounts = null,
PathTable pathTable = null)
{
Contract.Requires(name != null);
Contract.Requires(!string.IsNullOrEmpty(rootPath));
LoggingContext = new LoggingContext("TestLogger." + name);
PathTable = pathTable ?? new PathTable();
PipDataBuilderPool = new ObjectPool <PipDataBuilder>(() => new PipDataBuilder(PathTable.StringTable), _ => { });
// The tests that use TestEnv need to be modernized to take a filesystem
var fileSystem = new PassThroughFileSystem(PathTable);
Context = EngineContext.CreateNew(CancellationToken.None, PathTable, fileSystem);
// Add some well-known paths with fixed casing to the Context.PathTable
AbsolutePath.Create(Context.PathTable, rootPath.ToLowerInvariant());
var root = AbsolutePath.Create(Context.PathTable, rootPath);
var configuration = ConfigHelpers.CreateDefaultForXml(Context.PathTable, root);
configuration.Layout.SourceDirectory = root.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "src")); // These tests have non-standard src folder
configuration.Engine.MaxRelativeOutputDirectoryLength = maxRelativeOutputDirectoryLength;
configuration.Schedule.EnableLazyOutputMaterialization = enableLazyOutputMaterialization;
configuration.Schedule.UnsafeDisableGraphPostValidation = false;
configuration.Schedule.ComputePipStaticFingerprints = true;
configuration.Sandbox.FileAccessIgnoreCodeCoverage = true;
BuildXLEngine.PopulateFileSystemCapabilities(configuration, configuration, Context.PathTable, LoggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(configuration, configuration, Context.PathTable, LoggingContext);
Configuration = configuration;
var mountsTable = MountsTable.CreateAndRegister(LoggingContext, Context, Configuration, null);
if (mounts != null)
{
foreach (var mount in mounts)
{
mountsTable.AddResolvedMount(mount);
}
}
AbsolutePath specFile = SourceRoot.CreateRelative(Context.PathTable, "TestSpecFile.dsc");
var graph = TestSchedulerFactory.CreateEmptyPipGraph(Context, configuration, mountsTable.MountPathExpander);
PipTable = graph.PipTable;
PipGraph = graph;
var locationData = new LocationData(specFile, 0, 0);
var modulePip = ModulePip.CreateForTesting(Context.StringTable, specFile);
PipGraph.AddModule(modulePip);
PipGraph.AddSpecFile(new SpecFilePip(FileArtifact.CreateSourceFile(specFile), locationData, modulePip.Module));
PipConstructionHelper = PipConstructionHelper.CreateForTesting(
Context,
ObjectRoot,
redirectedRoot: Configuration.Layout.RedirectedDirectory,
pipGraph: PipGraph,
moduleName: modulePip.Identity.ToString(Context.StringTable),
symbol: name,
specPath: specFile);
Paths = new Paths(PathTable);
mountsTable.CompleteInitialization();
}
/// <summary>
/// Builds a workspace and uses filter to find specs to evaluate.
/// </summary>
public static bool TryBuildWorkspaceAndCollectFilesToAnalyze(
Tracing.Logger logger,
PathTable pathTable,
ICommandLineConfiguration configuation,
out Workspace workspace,
out IPipGraph pipGraph,
out IReadOnlyDictionary <AbsolutePath, ISourceFile> filesToAnalyze,
out FrontEndContext context)
{
workspace = null;
pipGraph = null;
filesToAnalyze = null;
var loggingContext = new LoggingContext("DScriptAnalyzer");
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
context = engineContext.ToFrontEndContext(loggingContext);
// Parse filter string into EvaluationFilter
var evaluationFilter = EvaluationFilter.Empty;
if (!string.IsNullOrEmpty(configuation.Filter))
{
if (!TryGetEvaluationFilter(logger, loggingContext, engineContext, configuation.Filter, out evaluationFilter))
{
// Error has been reported already
return(false);
}
}
// Try parsing the workspace from config and evaluation filter
if (!TryBuildWorkspace(
configuation,
context,
engineContext,
evaluationFilter,
progressHandler: null,
workspace: out workspace,
frontEndHostController: out _,
pipGraph: out pipGraph,
configuration: GetDefaultConfiguration()))
{
return(false);
}
if (configuation.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// Find strict subset of specs in workspace that should be analyzed
var collectedFilesToAnalyze = CollectFilesToAnalyze(
workspace,
pathTable,
configuation.Startup.ConfigFile,
evaluationFilter);
if (collectedFilesToAnalyze.Count == 0)
{
logger.ErrorFilterHasNoMatchingSpecs(loggingContext, configuation.Filter);
return(false);
}
filesToAnalyze = collectedFilesToAnalyze;
}
else
{
filesToAnalyze = new Dictionary <AbsolutePath, ISourceFile>();
}
return(true);
}