private IPipGraphBuilder CreatePipGraphBuilder(
LoggingContext loggingContext,
MountsTable mountsTable,
[CanBeNull] GraphReuseResult reuseResult)
{
var builder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(Context),
Context,
Scheduler.Tracing.Logger.Log,
loggingContext,
Configuration,
mountsTable.MountPathExpander,
fingerprintSalt: Configuration.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new Scheduler.DirectoryMembershipFingerprinterRuleSet(Configuration, Context.StringTable));
PatchablePipGraph patchableGraph = null;
if (Configuration.FrontEnd.UseGraphPatching() && reuseResult?.IsPartialReuse == true)
{
Logger.Log.UsingPatchableGraphBuilder(loggingContext);
patchableGraph = new PatchablePipGraph(
oldPipGraph: reuseResult.PipGraph.DataflowGraph,
oldPipTable: reuseResult.PipGraph.PipTable,
graphBuilder: builder,
maxDegreeOfParallelism: Configuration.FrontEnd.MaxFrontEndConcurrency());
}
return((IPipGraphBuilder)patchableGraph ?? builder);
}
public static Tuple <Scheduler, EngineCache> CreateWithCaching(
PipExecutionContext context,
LoggingContext loggingContext,
IConfiguration configuration,
PipGraph.Builder graphBuilder,
IPipQueue queue)
{
Contract.Requires(graphBuilder != null);
Contract.Requires(context != null);
Contract.Requires(queue != null);
var cacheLayer = new EngineCache(
new InMemoryArtifactContentCache(),
new InMemoryTwoPhaseFingerprintStore());
Scheduler scheduler = CreateInternal(
context,
loggingContext,
graphBuilder.Build(),
queue,
cacheLayer,
configuration);
return(Tuple.Create(scheduler, cacheLayer));
}
/// <nodoc />
public MacOsDefaults(PathTable pathTable, PipGraph.Builder pipGraph)
{
m_provenance = new PipProvenance(
0,
ModuleId.Invalid,
StringId.Invalid,
FullSymbol.Invalid,
LocationData.Invalid,
QualifierId.Unqualified,
PipData.Invalid);
m_inputFiles =
new[]
{
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.Etc)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.TmpDir)),
};
// Sealed Source inputs
m_inputDirectories =
new[]
{
GetSourceSeal(pathTable, pipGraph, MacPaths.Applications),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrBin),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrInclude),
GetSourceSeal(pathTable, pipGraph, MacPaths.UsrLib),
GetSourceSeal(pathTable, pipGraph, MacPaths.Library),
GetSourceSeal(pathTable, pipGraph, MacPaths.UserProvisioning),
};
m_untrackedFiles =
new[]
{
// login.keychain is created by the OS the first time any process invokes an OS API that references the keychain.
// Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChainsDb)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserKeyChains)),
FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, MacPaths.UserCFTextEncoding)),
};
m_untrackedDirectories =
new[]
{
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Bin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Dev),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Private),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Sbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.SystemLibrary),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrLibexec),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrShare),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrStandalone),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UsrSbin),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.Var),
DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, MacPaths.UserPreferences),
};
}
/// <summary>
/// Conceptually, converts a given <paramref name="workspace"/> into "evaluation AST" (which can next be evaluated/interpreted).
///
/// In reality, this "evaluation AST" is so tightly coupled with the engine, so this method has no choice but to
/// create a big hairball of hosts/controllers/resolvers/contexts/configurations/etc to make evaluation possible.
///
/// This method tries to bypass as much of the front-end stuff as possible. For example, it doesn't start evaluation from
/// <see cref="FrontEndHost"/>, but instead it creates a single resolver (namely <see cref="DScriptSourceResolver"/>
/// and uses that resolver directly to evaluate the AST.
///
/// Any errors can be retrieved via the <see cref="CreateTestResult"/> method.
/// </summary>
public async Task <TestResult <Interpreter> > ConvertNoErrorCheckAsync(Workspace workspace, [CanBeNull] PipGraph oldPipGraph)
{
var nonPreludeModules = NonPreludeModules(workspace).ToArray();
var moduleRegistry = new ModuleRegistry(SymbolTable);
var configStringPath = Path.Combine(SrcRoot.ToString(PathTable), Names.ConfigDsc);
var configuration = new ConfigurationImpl()
{
FrontEnd =
{
EnableIncrementalFrontEnd = false,
ReloadPartialEngineStateWhenPossible = false,
UseSpecPublicFacadeAndAstWhenAvailable = false,
ConstructAndSaveBindingFingerprint = false,
UsePartialEvaluation = false,
}
};
var frontEndHost = FrontEndHostController.CreateForTesting(FrontEndContext, Engine, moduleRegistry, configStringPath, FrontEndLogger);
var frontEnd = new DScriptFrontEnd(FrontEndStatistics, AstLogger, null);
frontEnd.InitializeFrontEnd(frontEndHost, FrontEndContext, configuration);
var resolver = (DScriptSourceResolver)frontEnd.CreateResolver(KnownResolverKind.DScriptResolverKind);
var packages = nonPreludeModules.Select(module => CreatePackageForModule(module)).ToList();
resolver.InitResolverForTesting("Test", packages);
frontEndHost.InitializeResolvers(new[] { resolver });
// convert all modules and assert it succeeds
var convertTasks = nonPreludeModules.Select(module => frontEndHost.ConvertWorkspaceToEvaluationAsync(workspace));
await Task.WhenAll(convertTasks);
// prepare for evaluation
var graphBuilder = new PipGraph.Builder(
new PipTable(PathTable, SymbolTable, initialBufferSize: 16, maxDegreeOfParallelism: Environment.ProcessorCount, debug: false),
new EngineContext(CancellationToken.None, PathTable, SymbolTable, new QualifierTable(PathTable.StringTable), FrontEndContext.FileSystem, new TokenTextTable()),
global::BuildXL.Pips.Tracing.Logger.Log,
FrontEndContext.LoggingContext,
// For tests, allow writes outside of mounts unles defined otherwise
new ConfigurationImpl()
{
Engine = { UnsafeAllowOutOfMountWrites = true }
},
new MountPathExpander(PathTable));
IMutablePipGraph pipGraph = oldPipGraph != null
? new PatchablePipGraph(oldPipGraph.DataflowGraph, oldPipGraph.PipTable, graphBuilder, maxDegreeOfParallelism: Environment.ProcessorCount)
: (IMutablePipGraph)graphBuilder;
frontEndHost.SetState(Engine, pipGraph, configuration);
return(new TestResult <Interpreter>(frontEndHost, Diagnostics));
}
public ISourceFile Generate(PipGraph.Builder pipGraph)
{
var statements = new List <IStatement>();
foreach (var pip in pipGraph.RetrieveScheduledPips())
{
statements.Add(Generate(pip));
}
return(new TypeScript.Net.Types.SourceFile(statements.ToArray()));
}
private DirectoryArtifact GetSourceSeal(PathTable pathTable, PipGraph.Builder pipGraph, string path)
{
var sealDirectory = new SealDirectory(
AbsolutePath.Create(pathTable, path),
contents: s_emptySealContents,
kind: SealDirectoryKind.SourceAllDirectories,
provenance: m_provenance,
tags: ReadOnlyArray <StringId> .Empty,
patterns: ReadOnlyArray <StringId> .Empty,
scrub: false);
return(pipGraph.AddSealDirectory(sealDirectory));
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="oldPipGraph">Old pip graph.</param>
/// <param name="oldPipTable">Old pip table.</param>
/// <param name="graphBuilder">Pip graph builder to which to delegate all "add pip" operations.</param>
/// <param name="maxDegreeOfParallelism">Max concurrency for graph reloading (<see cref="PartiallyReloadGraph"/>).</param>
public PatchablePipGraph(
IReadonlyDirectedGraph oldPipGraph,
PipTable oldPipTable,
PipGraph.Builder graphBuilder,
int maxDegreeOfParallelism)
{
m_oldPipGraph = oldPipGraph;
m_oldPipTable = oldPipTable;
m_builder = graphBuilder;
m_maxDegreeOfParallelism = maxDegreeOfParallelism;
m_reloadedSealDirectories = new ConcurrentBigMap <long, DirectoryArtifact>();
m_reloadedServicePips = new ConcurrentBigMap <long, PipId>();
m_pipIdMap = new ConcurrentBigMap <PipId, PipId>();
}
/// <summary>
/// Run the test
/// </summary>
public bool Run(string testFolder, string specFile, string fullIdentifier, string shortName, string lkgFile, params string[] sdksToResolve)
{
Contract.Requires(!string.IsNullOrEmpty(testFolder));
Contract.Requires(!string.IsNullOrEmpty(specFile));
Contract.Requires(sdksToResolve != null);
// Sadly the frontend doesn't use the engine abstractions file api's so we have to materialize stuff on disk for now...
// TODO: Fix this code once the frontend supports a proper virtual FileSystem.
// TODO: Change the package semantics to implicit when we expose a way to evaluate a single value
var testFileName = Path.GetFileName(specFile);
var mainFileName = "testMain.bp";
var testMainFile = Path.Combine(testFolder, mainFileName);
Directory.CreateDirectory(testFolder);
File.WriteAllText(Path.Combine(testFolder, Names.ModuleConfigBm), I($@"module(
{{
name: 'TestPackage',
nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences,
projects: [
f`{mainFileName}`,
f`{testFileName}`,
],
}});"));
File.WriteAllText(testMainFile, I($@"
export const testFolder = d`{Path.GetDirectoryName(specFile).Replace('\\', '/')}`;
@@public
export const main = {fullIdentifier}();"));
File.Copy(specFile, Path.Combine(testFolder, testFileName));
// Create a fake package for Sdk.TestRunner so that you can safely test packages that have the tests embedded in them.
var testRunnerFolder = Path.Combine(testFolder, "Sdk.TestRunner");
Directory.CreateDirectory(testRunnerFolder);
File.WriteAllText(Path.Combine(testRunnerFolder, Names.ModuleConfigBm), I($"module({{\n\tname: 'Sdk.TestRunner',\n}});"));
File.WriteAllText(Path.Combine(testRunnerFolder, "package" + Names.DotDscExtension), I($@"
export interface TestArguments {{
testFiles: File[];
sdkFolders?: (Directory|StaticDirectory)[];
autoFixLkgs?: boolean;
}}
export interface TestResult {{
xmlResults: File;
}}
export function test(args: TestArguments): TestResult {{
Contract.fail(""Can't run a DScript UnitTest inside of a DScript UnitTest"");
}}"));
// Setup Context and configuration
var frontEndContext = FrontEndContext.CreateInstanceForTesting();
var pipContext = new SchedulerContext(CancellationToken.None, frontEndContext.StringTable, frontEndContext.PathTable, frontEndContext.SymbolTable, frontEndContext.QualifierTable);
var pathTable = frontEndContext.PathTable;
var testFolderPath = AbsolutePath.Create(pathTable, testFolder);
var configuration = CreateConfiguration(sdksToResolve.Union(new[] { testRunnerFolder }), pathTable, testFolderPath);
var engineAbstraction = new TestEngineAbstraction(pathTable, frontEndContext.StringTable, testFolderPath, new PassThroughFileSystem(pathTable));
var frontEndStatistics = new FrontEndStatistics();
if (!CreateFactories(
frontEndContext,
engineAbstraction,
frontEndStatistics,
configuration,
out var ambientTesting,
out var moduleRegistry,
out var frontEndFactory))
{
return(false);
}
// Set the timeout to a large number to avoid useless performance collections in tests.
using (var performanceCollector = new PerformanceCollector(TimeSpan.FromHours(1)))
using (var frontEndHostController = new FrontEndHostController(
frontEndFactory,
new EvaluationScheduler(1),
moduleRegistry,
frontEndStatistics,
m_tracingLogger,
performanceCollector,
collectMemoryAsSoonAsPossible: true))
{
var frontEndController = (IFrontEndController)frontEndHostController;
frontEndController.InitializeHost(frontEndContext, configuration);
frontEndController.ParseConfig(configuration);
// Populate the graph
using (var pipTable = new PipTable(
pipContext.PathTable,
pipContext.SymbolTable,
initialBufferSize: 16384,
maxDegreeOfParallelism: 1,
debug: true))
{
var mountPathExpander = new MountPathExpander(pathTable);
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "testFolder"), testFolderPath, allowHashing: true, readable: true, writable: false));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "src"), testFolderPath.Combine(pathTable, "src"), allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "out"), testFolderPath.Combine(pathTable, "out"), allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "noRead"), testFolderPath.Combine(pathTable, "noRead"), allowHashing: true, readable: false, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "temp"), engineAbstraction.Layout.TempDirectory, allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "obj"), engineAbstraction.Layout.ObjectDirectory, allowHashing: true, readable: true, writable: true));
var graph = new PipGraph.Builder(
pipTable,
pipContext,
m_pipLogger,
frontEndContext.LoggingContext,
configuration,
mountPathExpander);
using (var cacheLayer = new EngineCache(
new InMemoryArtifactContentCache(),
new InMemoryTwoPhaseFingerprintStore()))
{
var cache = Task.FromResult(Possible.Create(cacheLayer));
try
{
var evaluationFilter = new EvaluationFilter(
pipContext.SymbolTable,
pipContext.PathTable,
new FullSymbol[0],
new[]
{
AbsolutePath.Create(frontEndContext.PathTable, testMainFile),
},
CollectionUtilities.EmptyArray <StringId>());
if (!frontEndController.PopulateGraph(cache, graph, engineAbstraction, evaluationFilter, configuration, configuration.Startup))
{
HandleDiagnostics();
return(false);
}
}
catch (AggregateException e)
{
var baseException = e.GetBaseException();
if (baseException is XunitException)
{
// If it is an XUnit assert, then unwrap the exception and throw that because XUnit other doesn't display the error nicely.
ExceptionDispatchInfo.Capture(baseException).Throw();
}
throw;
}
}
if (!ValidatePips(frontEndContext, graph, testFolderPath, specFile, shortName, lkgFile, ambientTesting.DontValidatePipsEnabled))
{
return(false);
}
}
}
HandleDiagnostics();
return(true);
}
private bool ValidatePips(FrontEndContext context, PipGraph.Builder pipGraph, AbsolutePath testFolder, string specFile, string shortName, string lkgFile, bool dontValidatePipsEnabled)
{
bool hasLkgFile = !string.IsNullOrEmpty(lkgFile);
var hasPips = pipGraph.PipTable.Keys.Count(pipId => !pipGraph.PipTable.GetPipType(pipId).IsMetaPip()) > 0;
if (dontValidatePipsEnabled)
{
if (hasLkgFile)
{
m_diagnosticHandler(
new Diagnostic(
0,
EventLevel.Error,
C($"Test calls 'Testing.dontValidatePips', but lkg file '{lkgFile}' is present. Either remove that call or delete the file.{GetAutoFixString()}"),
default(Location)));
return(FailOrDeleteLkg(lkgFile));
}
if (!hasPips)
{
m_diagnosticHandler(
new Diagnostic(
0,
EventLevel.Error,
C($"Test calls 'Testing.dontValidatePips', but no pips were created. Remove that call.{GetAutoFixString()}"),
default(Location)));
return(FailOrDeleteLkg(lkgFile));
}
}
else
{
if (!hasPips)
{
if (hasLkgFile)
{
m_diagnosticHandler(
new Diagnostic(
0,
EventLevel.Error,
C($"No pips were created in the test, but lkg file '{lkgFile}' is present. Delete the file.{GetAutoFixString()}"),
default(Location)));
return(FailOrDeleteLkg(lkgFile));
}
}
else
{
var printer = new TestPipPrinter(context.PathTable, context.StringTable, testFolder);
var actual = printer.Print(pipGraph);
if (!hasLkgFile)
{
lkgFile = Path.Combine(Path.GetDirectoryName(specFile), Path.GetFileNameWithoutExtension(specFile), shortName + ".lkg");
m_diagnosticHandler(
new Diagnostic(
0,
EventLevel.Error,
C($"This test creates pip, but no lkgFile was encountered. Either create file '{lkgFile}' or add a call to 'Testing.dontValidatePips'.{GetAutoFixString()}"),
default(Location)));
return(FailOrUpdateLkg(lkgFile, actual));
}
if (!File.Exists(lkgFile))
{
m_diagnosticHandler(
new Diagnostic(
0,
EventLevel.Error,
C($"File '{lkgFile}' not found.{GetAutoFixString()}"),
default(Location)));
return(FailOrUpdateLkg(lkgFile, actual));
}
var expected = TranslateLkgContentForCurrentPlatform(File.ReadAllText(lkgFile));
string message;
if (!FileComparison.ValidateContentsAreEqual(expected, actual, lkgFile, out message))
{
m_diagnosticHandler(
new Diagnostic(
0,
EventLevel.Error,
C($"Pips don't match '{lkgFile}: {message}'.{GetAutoFixString()}"),
default(Location)));
return(FailOrUpdateLkg(lkgFile, actual));
}
}
}
return(true);
}
/// <summary>
/// Prints a pipGraph. Relies on the TypeScript.Net pretty printer
/// </summary>
public string Print(PipGraph.Builder pipGraph)
{
var sourceFile = Generate(pipGraph);
return(sourceFile.ToDisplayString());
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IPipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by IDE test.
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IPipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(),
5000,
false);
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Scheduler.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable));
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
private static bool TryBuildWorkspaceInternal(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
bool forIDE,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by the IDE.
if (frontEndEngineAbstraction != null)
{
// The IDE engine typically doesn't have mounts configured. We do it here if they haven't been configured yet.
// Observe these are just the default mounts used for config evaluation.
if (frontEndEngineAbstraction is BasicFrontEndEngineAbstraction basicEngine && !frontEndEngineAbstraction.GetMountNames("Script", BuildXL.Utilities.ModuleId.Invalid).Any())
{
// If this fails we just ignore the failure. Mounts not being properly configured doesn't prevent the IDE plugin from working.
basicEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties);
}
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
else
{
// Otherwise we construct one with all mounts populated for config evaluation
var configurationEngine = new BasicFrontEndEngineAbstraction(engineContext.PathTable, engineContext.FileSystem, mutableCommandlineConfig);
if (!configurationEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties))
{
// Errors are logged already
return(false);
}
frontEndEngineAbstraction = configurationEngine;
}
var config = controller.ParseConfig(frontEndEngineAbstraction, mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IMutablePipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
// For the IDE case, we want to make sure all config-specific mounts are properly populated
if (forIDE && frontEndEngineAbstraction is BasicFrontEndEngineAbstraction languageServiceEngine)
{
Contract.AssertNotNull(frontEndEngineAbstraction);
AddConfigurationMounts(config, mountsTable);
languageServiceEngine.SetMountsTable(mountsTable);
}
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var searchPathToolsHash = new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable).ComputeSearchPathToolsHash();
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Pips.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
searchPathToolsHash: searchPathToolsHash);
// Observe mount table is completed during workspace construction
AddConfigurationMounts(config, mountsTable);
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
/// <nodoc />
public MacOsDefaults(PathTable pathTable, PipGraph.Builder pipGraph)
{
m_provenance = new PipProvenance(
0,
ModuleId.Invalid,
StringId.Invalid,
FullSymbol.Invalid,
LocationData.Invalid,
QualifierId.Unqualified,
PipData.Invalid);
// Sealed Source inputs
// (using Lazy so that these directories are sealed and added to the graph only if explicitly requested by a process)
m_lazySourceSealDirectories = Lazy.Create(() =>
new DefaultSourceSealDirectories(new[]
{
MacPaths.Applications,
MacPaths.Library,
MacPaths.UserProvisioning,
// consider untracking /usr/bin and /usr/include because they are not writable by default
MacPaths.UsrBin,
MacPaths.UsrInclude,
}
.Select(p => GetSourceSeal(pathTable, pipGraph, p))
.ToArray()));
m_untrackedFiles =
new[]
{
// login.keychain is created by the OS the first time any process invokes an OS API that references the keychain.
// Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain
MacPaths.Etc,
MacPaths.UserKeyChainsDb,
MacPaths.UserKeyChains,
MacPaths.UserCFTextEncoding,
MacPaths.TmpDir
}
.Select(p => FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, p)))
.ToArray();
m_untrackedDirectories =
new[]
{
MacPaths.Bin,
MacPaths.Dev,
MacPaths.Private,
MacPaths.Sbin,
MacPaths.SystemLibrary,
MacPaths.UsrLibexec,
MacPaths.UsrShare,
MacPaths.UsrStandalone,
MacPaths.UsrSbin,
MacPaths.Var,
MacPaths.UserPreferences,
// it's important to untrack /usr/lib instead of creating a sealed source directory
// - the set of dynamically loaded libraries during an execution of a process is
// not necessarily deterministic, i.e., when the same process---which itself is
// deterministic---is executed multiple times on same inputs, the set of
// dynamically loaded libraries is not necessarily going to stay the same.
MacPaths.UsrLib
}
.Select(p => DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, p))
.ToArray();
}
