/// <nodoc />
private PipConstructionHelper(
PipExecutionContext context,
AbsolutePath objectRoot,
AbsolutePath redirectedRoot,
AbsolutePath tempRoot,
IMutablePipGraph pipGraph,
ModuleId moduleId,
string moduleName,
ValuePip valuePip,
RelativePath pipRelativePath,
long semiStableHashSeed)
{
Context = context;
m_objectRoot = objectRoot;
m_redirectedRoot = redirectedRoot;
m_tempRoot = tempRoot.IsValid ? tempRoot : objectRoot;
PipGraph = pipGraph;
m_moduleId = moduleId;
m_moduleName = moduleName;
m_valuePip = valuePip;
PipRelativePath = pipRelativePath;
m_semiStableHash = semiStableHashSeed;
m_folderIdResolver = new ReserveFoldersResolver(this);
}
/// <summary>
/// Helper method with defaults for convenient creation from unit tests
/// </summary>
public static PipConstructionHelper CreateForTesting(
PipExecutionContext context,
AbsolutePath?objectRoot = null,
AbsolutePath?redirectedRoot = null,
AbsolutePath?tempRoot = null,
IMutablePipGraph pipGraph = null,
string moduleName = null,
string specRelativePath = null,
string symbol = null,
AbsolutePath?specPath = null,
QualifierId?qualifierId = null)
{
moduleName = moduleName ?? "TestModule";
return(Create(
context,
objectRoot ?? AbsolutePath.Create(context.PathTable, "d:\\test\\obj"),
redirectedRoot ?? AbsolutePath.Create(context.PathTable, "d:\\test\\redirected"),
tempRoot ?? objectRoot ?? AbsolutePath.Create(context.PathTable, "d:\\test\\tmp"),
pipGraph,
ModuleId.Create(context.StringTable, moduleName),
moduleName,
RelativePath.Create(context.StringTable, specRelativePath ?? "spec"),
FullSymbol.Create(context.SymbolTable, symbol ?? "testValue"),
new LocationData(specPath ?? AbsolutePath.Create(context.PathTable, "d:\\src\\spec.dsc"), 0, 0),
qualifierId ?? QualifierId.Unqualified));
}
/// <summary>
/// Conceptually, converts a given <paramref name="workspace"/> into "evaluation AST" (which can next be evaluated/interpreted).
///
/// In reality, this "evaluation AST" is so tightly coupled with the engine, so this method has no choice but to
/// create a big hairball of hosts/controllers/resolvers/contexts/configurations/etc to make evaluation possible.
///
/// This method tries to bypass as much of the front-end stuff as possible. For example, it doesn't start evaluation from
/// <see cref="FrontEndHost"/>, but instead it creates a single resolver (namely <see cref="DScriptSourceResolver"/>
/// and uses that resolver directly to evaluate the AST.
///
/// Any errors can be retrieved via the <see cref="CreateTestResult"/> method.
/// </summary>
public async Task <TestResult <Interpreter> > ConvertNoErrorCheckAsync(Workspace workspace, [CanBeNull] PipGraph oldPipGraph)
{
var nonPreludeModules = NonPreludeModules(workspace).ToArray();
var moduleRegistry = new ModuleRegistry(SymbolTable);
var configStringPath = Path.Combine(SrcRoot.ToString(PathTable), Names.ConfigDsc);
var configuration = new ConfigurationImpl()
{
FrontEnd =
{
EnableIncrementalFrontEnd = false,
ReloadPartialEngineStateWhenPossible = false,
UseSpecPublicFacadeAndAstWhenAvailable = false,
ConstructAndSaveBindingFingerprint = false,
UsePartialEvaluation = false,
}
};
var frontEndHost = FrontEndHostController.CreateForTesting(FrontEndContext, Engine, moduleRegistry, configStringPath, FrontEndLogger);
var frontEnd = new DScriptFrontEnd(FrontEndStatistics, AstLogger, null);
frontEnd.InitializeFrontEnd(frontEndHost, FrontEndContext, configuration);
var resolver = (DScriptSourceResolver)frontEnd.CreateResolver(KnownResolverKind.DScriptResolverKind);
var packages = nonPreludeModules.Select(module => CreatePackageForModule(module)).ToList();
resolver.InitResolverForTesting("Test", packages);
frontEndHost.InitializeResolvers(new[] { resolver });
// convert all modules and assert it succeeds
var convertTasks = nonPreludeModules.Select(module => frontEndHost.ConvertWorkspaceToEvaluationAsync(workspace));
await Task.WhenAll(convertTasks);
// prepare for evaluation
var graphBuilder = new PipGraph.Builder(
new PipTable(PathTable, SymbolTable, initialBufferSize: 16, maxDegreeOfParallelism: Environment.ProcessorCount, debug: false),
new EngineContext(CancellationToken.None, PathTable, SymbolTable, new QualifierTable(PathTable.StringTable), FrontEndContext.FileSystem, new TokenTextTable()),
global::BuildXL.Pips.Tracing.Logger.Log,
FrontEndContext.LoggingContext,
// For tests, allow writes outside of mounts unles defined otherwise
new ConfigurationImpl()
{
Engine = { UnsafeAllowOutOfMountWrites = true }
},
new MountPathExpander(PathTable));
IMutablePipGraph pipGraph = oldPipGraph != null
? new PatchablePipGraph(oldPipGraph.DataflowGraph, oldPipGraph.PipTable, graphBuilder, maxDegreeOfParallelism: Environment.ProcessorCount)
: (IMutablePipGraph)graphBuilder;
frontEndHost.SetState(Engine, pipGraph, configuration);
return(new TestResult <Interpreter>(frontEndHost, Diagnostics));
}
private DirectoryArtifact GetSourceSeal(IMutablePipGraph pipGraph, AbsolutePath path)
{
var sealDirectory = new SealDirectory(
path,
contents: s_emptySealContents,
kind: SealDirectoryKind.SourceAllDirectories,
provenance: m_provenance,
tags: ReadOnlyArray <StringId> .Empty,
patterns: ReadOnlyArray <StringId> .Empty,
scrub: false);
return(pipGraph.AddSealDirectory(sealDirectory, default));
}
/// <summary>
/// Helper that stores some shared state on the Analyzer
/// </summary>
internal bool SetSharedState(Args arguments, FrontEndContext context, Logger logger, Workspace workspace, IMutablePipGraph pipGraph)
{
m_arguments = arguments;
Context = context;
Workspace = workspace;
PipGraph = pipGraph;
Logger = logger;
Initializing = true;
var result = Initialize();
Initializing = false;
return(result);
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
return(TryBuildWorkspaceInternal(commandLineConfig, frontEndContext, engineContext, evaluationFilter, progressHandler, out workspace, out frontEndHostController, out pipGraph,
configuration, forIDE: false, frontEndEngineAbstraction, collectMemoryAsSoonAsPossible));
}
/// <summary>
/// Creates a new PipConstructionHelper
/// </summary>
/// <remarks>
/// Ideally this function would take ModuleId, FullSymbol QualifierId and compute uniqueOutputLocation itself. Unfortunately today the data is not yet
/// exposed via IPipGraph, therefore the responsibility is on the call site for now.
/// </remarks>
public static PipConstructionHelper Create(
PipExecutionContext context,
AbsolutePath objectRoot,
AbsolutePath redirectedRoot,
AbsolutePath tempRoot,
IMutablePipGraph pipGraph,
ModuleId moduleId,
string moduleName,
RelativePath specRelativePath,
FullSymbol symbol,
LocationData thunkLocation,
QualifierId qualifierId)
{
var stringTable = context.StringTable;
var pathTable = context.PathTable;
// We have to manually compute the pipPipUniqueString here, Ideally we pass PackageId, SpecFile, FullSymbol and qualiferId and have it computed inside, but the IPipGraph does not allow querying it for now.
string hashString;
long semiStableHashSeed = 0;
using (var builderWrapper = Pools.GetStringBuilder())
{
var builder = builderWrapper.Instance;
builder.Append(moduleName);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.GetOrdinalHashCode64(moduleName);
if (specRelativePath.IsValid)
{
string specPath = specRelativePath.ToString(stringTable);
builder.Append(specPath);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(specPath));
}
var symbolName = symbol.ToStringAsCharArray(context.SymbolTable);
builder.Append(symbolName);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(symbolName));
var qualifierDisplayValue = context.QualifierTable.GetCanonicalDisplayString(qualifierId);
builder.Append(qualifierDisplayValue);
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(qualifierDisplayValue));
var pipPipUniqueString = builder.ToString();
hashString = Hash(pipPipUniqueString);
}
var pipRelativePath = RelativePath.Create(
PathAtom.Create(stringTable, hashString.Substring(0, 1)),
PathAtom.Create(stringTable, hashString.Substring(1, 1)),
PathAtom.Create(stringTable, hashString.Substring(2)));
var valuePip = new ValuePip(symbol, qualifierId, thunkLocation);
return(new PipConstructionHelper(
context,
objectRoot,
redirectedRoot,
tempRoot,
pipGraph,
moduleId,
moduleName,
valuePip,
pipRelativePath,
semiStableHashSeed));
}
/// <nodoc />
public MacOsDefaults(PathTable pathTable, IMutablePipGraph pipGraph)
{
m_provenance = new PipProvenance(
0,
ModuleId.Invalid,
StringId.Invalid,
FullSymbol.Invalid,
LocationData.Invalid,
QualifierId.Unqualified,
PipData.Invalid);
m_sourceSealDirectoryPaths =
new[]
{
MacPaths.Applications,
MacPaths.Library,
MacPaths.UserProvisioning
}
.Select(p => AbsolutePath.Create(pathTable, p))
.ToArray();
// Sealed Source inputs
// (using Lazy so that these directories are sealed and added to the graph only if explicitly requested by a process)
m_lazySourceSealDirectories = Lazy.Create(() =>
new DefaultSourceSealDirectories(m_sourceSealDirectoryPaths.Select(p => GetSourceSeal(pipGraph, p)).ToArray()));
m_untrackedFiles =
new[]
{
// login.keychain is created by the OS the first time any process invokes an OS API that references the keychain.
// Untracked because build state will not be stored there and code signing will fail if required certs are in the keychain
MacPaths.Etc,
MacPaths.UserKeyChainsDb,
MacPaths.UserKeyChains,
MacPaths.UserCFTextEncoding,
MacPaths.TmpDir
}
.Select(p => FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, p)))
.ToArray();
m_untrackedDirectories =
new[]
{
MacPaths.Bin,
MacPaths.Dev,
MacPaths.Private,
MacPaths.Sbin,
MacPaths.SystemLibrary,
MacPaths.UsrBin,
MacPaths.UsrInclude,
MacPaths.UsrLibexec,
MacPaths.UsrShare,
MacPaths.UsrStandalone,
MacPaths.UsrSbin,
MacPaths.Var,
MacPaths.UserPreferences,
// it's important to untrack /usr/lib instead of creating a sealed source directory
// - the set of dynamically loaded libraries during an execution of a process is
// not necessarily deterministic, i.e., when the same process---which itself is
// deterministic---is executed multiple times on same inputs, the set of
// dynamically loaded libraries is not necessarily going to stay the same.
MacPaths.UsrLib
}
.Select(p => DirectoryArtifact.CreateWithZeroPartialSealId(pathTable, p))
.ToArray();
}
/// <summary>
/// Builds a workspace and uses filter to find specs to evaluate.
/// </summary>
public static bool TryBuildWorkspaceAndCollectFilesToAnalyze(
Tracing.Logger logger,
PathTable pathTable,
ICommandLineConfiguration configuation,
out Workspace workspace,
out IMutablePipGraph pipGraph,
out IReadOnlyDictionary <AbsolutePath, ISourceFile> filesToAnalyze,
out FrontEndContext context)
{
workspace = null;
pipGraph = null;
filesToAnalyze = null;
var loggingContext = new LoggingContext("DScriptAnalyzer");
var fileSystem = new PassThroughFileSystem(pathTable);
var engineContext = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
context = engineContext.ToFrontEndContext(loggingContext);
// Parse filter string into EvaluationFilter
var evaluationFilter = EvaluationFilter.Empty;
if (!string.IsNullOrEmpty(configuation.Filter))
{
if (!TryGetEvaluationFilter(logger, loggingContext, engineContext, configuation.Filter, out evaluationFilter))
{
// Error has been reported already
return(false);
}
}
// Try parsing the workspace from config and evaluation filter
if (!TryBuildWorkspace(
configuation,
context,
engineContext,
evaluationFilter,
progressHandler: null,
workspace: out workspace,
frontEndHostController: out _,
pipGraph: out pipGraph,
configuration: GetDefaultConfiguration()))
{
return(false);
}
if (configuation.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// Find strict subset of specs in workspace that should be analyzed
var collectedFilesToAnalyze = CollectFilesToAnalyze(
workspace,
pathTable,
configuation.Startup.ConfigFile,
evaluationFilter);
if (collectedFilesToAnalyze.Count == 0)
{
logger.ErrorFilterHasNoMatchingSpecs(loggingContext, configuation.Filter);
return(false);
}
filesToAnalyze = collectedFilesToAnalyze;
}
else
{
filesToAnalyze = new Dictionary <AbsolutePath, ISourceFile>();
}
return(true);
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by IDE test.
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IMutablePipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(),
5000,
false);
var searchPathToolsHash = new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable).ComputeSearchPathToolsHash();
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Pips.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
searchPathToolsHash: searchPathToolsHash);
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
