/// <nodoc/>
public FrontEndPublicFacadeAndAstProvider(
FrontEndEngineAbstraction engine,
Logger logger,
LoggingContext loggingContext,
string frontEndEngineDirectory,
bool logFrontEndStatistics,
PathTable pathTable,
IFrontEndStatistics statistics,
CancellationToken cancellationToken)
{
Contract.Requires(engine != null);
Contract.Requires(loggingContext != null);
Contract.Requires(!string.IsNullOrEmpty(frontEndEngineDirectory));
Contract.Requires(pathTable != null);
Contract.Requires(statistics != null);
m_engine = engine;
m_pathTable = pathTable;
m_statistics = statistics;
m_fileCombiner = new FileCombiner(
loggingContext,
SpecCacheFullPath(frontEndEngineDirectory),
FileCombiner.FileCombinerUsage.IncrementalScriptFrontEnd,
logFrontEndStatistics);
m_logger = logger;
m_loggingContext = loggingContext;
var queueOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1, CancellationToken = cancellationToken
};
Action <FileContentWithHash> action = SaveFile;
m_filesToSaveQueue = new ActionBlock <FileContentWithHash>(action, queueOptions);
}
/// <nodoc />
public PreludeManager(FrontEndEngineAbstraction engine, PathTable pathTable, Func <AbsolutePath, Task <Possible <ISourceFile> > > parser, int maxParseConcurrency, bool useOfficeBackCompatHacks)
{
Contract.Requires(engine != null);
Contract.Requires(pathTable.IsValid);
Contract.Requires(parser != null);
Contract.Requires(maxParseConcurrency > 0);
m_engine = engine;
m_pathTable = pathTable;
m_parser = parser;
m_maxParseConcurrency = maxParseConcurrency;
m_useOfficeBackCompatHacks = useOfficeBackCompatHacks;
}
private AppState(
FrontEndEngineAbstraction engineAbstraction,
TextDocumentManager documentManager,
PathTable pathTable,
IncrementalWorkspaceProvider incrementalWorkspaceProvider,
Workspace workspace)
{
EngineAbstraction = engineAbstraction;
DocumentManager = documentManager;
PathTable = pathTable;
IncrementalWorkspaceProvider = incrementalWorkspaceProvider;
m_workspace = workspace;
}
/// <nodoc />
public ConfigurationConversionHelper(
[CanBeNull] FrontEndEngineAbstraction engine,
ConfigurationKind kind,
Logger logger,
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
IFrontEndStatistics statistics)
: base(statistics, logger, host, context, configuration)
{
Engine = engine ?? new SimpleFrontEndEngineAbstraction(context.PathTable, context.FileSystem, configuration);
Kind = kind;
ConversionConfiguration = AstConversionConfiguration.ForConfiguration(FrontEndConfiguration);
Linter = Lazy.Create(() => CreateLinter(ConversionConfiguration));
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
return(TryBuildWorkspaceInternal(commandLineConfig, frontEndContext, engineContext, evaluationFilter, progressHandler, out workspace, out frontEndHostController, out pipGraph,
configuration, forIDE: false, frontEndEngineAbstraction, collectMemoryAsSoonAsPossible));
}
/// <summary>
/// Get all the environment exposed to the process, with the values overriden by the engine
/// </summary>
public static IDictionary <string, string> GetEngineEnvironment(FrontEndEngineAbstraction engine, string frontEndName)
{
var engineEnvironment = new Dictionary <string, string>();
IDictionary environment = Environment.GetEnvironmentVariables();
foreach (string environmentVariable in environment.Keys)
{
// Expose as much of the environment as we can -- use the ones overriden by the Engine
if (engine.TryGetBuildParameter(environmentVariable, frontEndName, out var value))
{
engineEnvironment[environmentVariable] = value;
}
}
return(engineEnvironment);
}
/// <summary>
/// Tries to build a workspace that an IDE-like language service can use
/// </summary>
public static bool TryBuildWorkspaceForIde(
FrontEndContext frontEndContext,
EngineContext engineContext,
FrontEndEngineAbstraction frontEndEngineAbstraction,
AbsolutePath rootFolder,
bool skipNuget,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController controller)
{
Contract.Requires(frontEndEngineAbstraction != null);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(rootFolder.IsValid);
var config = FindPrimaryConfiguration(rootFolder, frontEndEngineAbstraction, frontEndContext.PathTable);
if (!config.IsValid)
{
workspace = null;
controller = null;
return(false);
}
return(TryBuildWorkspace(
new CommandLineConfiguration()
{
Startup = { ConfigFile = config }
},
frontEndContext,
engineContext,
EvaluationFilter.Empty, // TODO: consider passing a filter that scopes down the build to the root folder
progressHandler,
false,
out workspace,
out controller,
out _,
new WorkspaceBuilderConfiguration()
{
CancelOnFirstParsingFailure = false, // We always want to do as much as we can for the IDE,
PublicFacadeOptimization = false, // The IDE never wants public facades to be on, since this swaps specs under the hood.
SaveBindingFingerprint = false, // Off for IDE.
SkipNuget = skipNuget,
},
frontEndEngineAbstraction: frontEndEngineAbstraction,
collectMemoryAsSoonAsPossible: false));
}
/// <nodoc />
public ConfigurationConversionHelper(
[CanBeNull] FrontEndEngineAbstraction engine,
ConfigurationKind kind,
GlobalConstants constants,
ModuleRegistry sharedModuleRegistry,
Logger logger,
FrontEndHost host,
FrontEndContext context,
IConfiguration configuration,
IFrontEndStatistics statistics = null)
: base(constants, sharedModuleRegistry, statistics ?? new FrontEndStatistics(), logger, host, context, configuration)
{
Engine = engine ?? new SimpleFrontEndEngineAbstraction(context.PathTable, context.FileSystem, configuration);
Kind = kind;
ConversionConfiguration = AstConversionConfiguration.ForConfiguration(FrontEndConfiguration);
Linter = Lazy.Create(() => CreateLinter(ConversionConfiguration));
}
/// <summary>
/// Retrieves a list of search locations for an executable, inspecting a list of explicit candidates first or using PATH.
/// The onEmptyResult action will be invoked if there are no search locations available
/// The onPathParseFailure action will be invoked with the PATH as an argument if the PATH is malformed
/// </summary>
public static bool TryRetrieveExecutableSearchLocations(
string frontEnd,
FrontEndContext context,
FrontEndEngineAbstraction engine,
IReadOnlyCollection <AbsolutePath> explicitCandidates,
out IEnumerable <AbsolutePath> searchLocations,
Action onEmptyResult = null,
Action <string> onPathParseFailure = null)
{
// If there are explicit search locations specified, use those
if (explicitCandidates?.Count > 0)
{
searchLocations = explicitCandidates;
return(true);
}
// Otherwise use %PATH%
if (!engine.TryGetBuildParameter("PATH", frontEnd, out string paths))
{
onEmptyResult?.Invoke();
searchLocations = null;
return(false);
}
var locations = new List <AbsolutePath>();
foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
var nonEscapedPath = path.Trim('"');
if (AbsolutePath.TryCreate(context.PathTable, nonEscapedPath, out var absolutePath))
{
locations.Add(absolutePath);
}
}
if (locations.Count == 0)
{
onPathParseFailure?.Invoke(paths);
searchLocations = null;
return(false);
}
searchLocations = locations;
return(true);
}
public LanguageServiceProviders(ProviderContext providerContext, FrontEndEngineAbstraction engineAbstraction, IProgressReporter progressReporter)
{
Contract.Requires(providerContext != null);
Contract.Requires(engineAbstraction != null);
m_autoCompleteProvider = new AutoCompleteProvider(providerContext);
m_gotoDefinitionProvider = new GotoDefinitionProvider(providerContext);
m_findReferencesProvider = new FindReferencesProvider(providerContext, progressReporter);
m_formattingProvider = new FormattingProvider(providerContext, engineAbstraction);
m_executeCommandProvider = new ExecuteCommandProvider(providerContext);
m_codeActionProvider = new CodeActionProvider(providerContext, m_executeCommandProvider);
m_codeLensProvider = new CodeLensProvider(providerContext);
m_hoverProvider = new HoverProvider(providerContext);
m_diagnosticProvider = new DiagnosticProvider(providerContext);
m_signatureHelpProvider = new SignatureHelpProvider(providerContext);
m_symbolProvider = new SymbolProvider(providerContext);
m_renameProvider = new RenameProvider(providerContext, m_findReferencesProvider);
}
/// <nodoc/>
public FrontEndArtifactManager(
FrontEndEngineAbstraction engine,
string frontEndEngineDirectory,
Logger logger,
LoggingContext loggingContext,
IFrontEndStatistics frontEndStatistics,
PathTable pathTable,
IFrontEndConfiguration configuration,
CancellationToken cancellationToken)
{
Contract.Requires(engine != null);
Contract.Requires(loggingContext != null);
Contract.Requires(frontEndStatistics != null);
Contract.Requires(!string.IsNullOrEmpty(frontEndEngineDirectory));
Contract.Requires(pathTable != null);
m_engine = engine;
m_frontEndCache = new FrontEndCache(frontEndEngineDirectory, logger, loggingContext, frontEndStatistics, pathTable);
// If engine state changed, delete facade and ast cache, because from one run to another the serialized AST bytes
// can be different even if corresponding DScript sourced didn't change (reason: PathTable can change in the meantime)
if (!engine.IsEngineStatePartiallyReloaded())
{
FrontEndPublicFacadeAndAstProvider.PurgeCache(frontEndEngineDirectory);
}
// If public facades are not used, then we don't create the public facade provider. This avoids
// creating the public facade cache.
// In particular, the IDE never uses this optimization, and the cache needs an exclusive lock.
if (configuration.UseSpecPublicFacadeAndAstWhenAvailable())
{
m_publicFacadeAndAstProvider = new FrontEndPublicFacadeAndAstProvider(
engine,
logger,
loggingContext,
frontEndEngineDirectory,
configuration.LogStatistics,
pathTable,
frontEndStatistics,
cancellationToken);
}
}
internal Workspace DoPhaseBuildWorkspace(IConfiguration configuration, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
if (!TryGetWorkspaceProvider(configuration, out var workspaceProvider, out var failures))
{
var workspaceConfiguration = GetWorkspaceConfiguration(configuration);
return(Workspace.Failure(workspaceConfiguration: workspaceConfiguration, failures: failures.ToArray()));
}
var result = TaskUtilities.WithCancellationHandlingAsync(
FrontEndContext.LoggingContext,
BuildAndFilterWorkspaceAsync(workspaceProvider, engineAbstraction, evaluationFilter),
m_logger.FrontEndBuildWorkspacePhaseCanceled,
GetOrCreateComputationCancelledWorkspace(workspaceProvider),
FrontEndContext.CancellationToken).GetAwaiter().GetResult();
ReportWorkspaceParsingAndBindingErrorsIfNeeded(result);
return(result);
}
private static AbsolutePath TryFindConfig(AbsolutePath startDir, FrontEndEngineAbstraction engine, PathTable pathTable)
{
Contract.Requires(startDir.IsValid);
for (; startDir != AbsolutePath.Invalid; startDir = startDir.GetParent(pathTable))
{
var configFilename = startDir.Combine(pathTable, Names.ConfigBc);
var legacyConfigFilename = startDir.Combine(pathTable, Names.ConfigDsc);
if (engine.FileExists(legacyConfigFilename))
{
return(legacyConfigFilename);
}
if (engine.FileExists(configFilename))
{
return(configFilename);
}
}
return(AbsolutePath.Invalid);
}
/// <summary>
/// The FrontEnds that use an out-of-proc tool should sandbox that process and call this method
/// in order to tack the tool's file accesses, enumerations, etc. in order to make graph caching sound
/// </summary>
public static void TrackToolFileAccesses(FrontEndEngineAbstraction engine, FrontEndContext context, string frontEndName, ISet <ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
{
// Compute all parseable paths
// TODO: does it make sense to consider enumerations, or as a result the graph will be too unstable? Does it matter for MsBuild graph construction?
foreach (var access in fileAccesses)
{
string accessPath = access.GetPath(context.PathTable);
if (AbsolutePath.TryCreate(context.PathTable, accessPath, out AbsolutePath path))
{
// Ignore accesses under the frontend folder: these are files used for internal communication between
// BuildXL and the graph builder tool, and they are never files that MSBuild itself interacted with
if (path.IsWithin(context.PathTable, frontEndFolder))
{
continue;
}
if ((access.RequestedAccess & RequestedAccess.Enumerate) != 0)
{
engine.TrackDirectory(path.ToString(context.PathTable));
}
if ((access.RequestedAccess & RequestedAccess.Probe) != 0)
{
engine.FileExists(path);
}
if ((access.RequestedAccess & RequestedAccess.Read) != 0)
{
// Two things are happening here: we want to register if the file is present or absent. Engine.FileExists takes
// care of that. And in the case the file exists, record the content.
// There are apparently some repos that create and delete files during graph construction :(
// So we cannot trust detours and check for IsNonexistent on the access itself. Even though there were read/write accesses on a given file,
// the file may not exist at this point
if (engine.FileExists(path))
{
engine.RecordFrontEndFile(path, frontEndName);
}
}
}
}
}
/// <summary>
/// Expands environment variables in a string by querying the engine for the corresponding build parameters
/// </summary>
/// <remarks>
/// Even though there is a Environment.ExpandEnvironmentVariables(String) available in C# this function only works
/// for Windows (https://github.com/dotnet/runtime/issues/25792). Furthermore, we need the expansion to consider
/// the configured build parameters in addition to the actual environment, and to properly register the access to those parameters in order
/// for incrementality to safely work.
/// </remarks>
/// <returns>Whether there is at least one environment variable in the string that was expanded</returns>
public static bool TryExpandVariablesInString(string stringToExpand, FrontEndEngineAbstraction engine, out string expandedString)
{
// if the path does not contain an env variable, just return the original path to avoid extra allocations
#if NETCOREAPP
if (stringToExpand?.Contains(s_variableSeparator) != true)
#else
if (stringToExpand?.Contains(s_variableSeparator.ToString()) != true)
#endif
{
expandedString = stringToExpand;
return(false);
}
// Here we build the result, containing all variables expanded
StringBuilder result = new StringBuilder(stringToExpand.Length);
// Here we build the variable name we'll query the environment with. Reused across each variable
StringBuilder variableName = new StringBuilder();
// Whether we are traversing a variable name or a regular part of the string
bool inAVariableName = false;
// Whether there has been at least one successful expansion of the provided string
bool atLeastOneExpansion = false;
foreach (char c in stringToExpand)
{
// If the character is a variable separator and we are not parsing a variable name, then
// this is the start of a variable name. Both Windows and non-Windows case start with a distinguished separator
if (c == s_variableSeparator && !inAVariableName)
{
inAVariableName = true;
// The separator is always discarded and not part of the name
continue;
}
// We are not parsing a variable name and this is a regular character of a string. Just make it part of the result
else if (c != s_variableSeparator && !inAVariableName)
{
result.Append(c);
}
// We are parsing a variable name and we are past the initial separator
else
{
// This is the end of the variable name
if (IsEndOfVariableName(c))
{
// Get the name and query the engine for the result. If the variable is not defined, just append the name unexpanded
// Observe that even when the variable is not defined, by querying the engine we will record this as an anti-dependency
var name = variableName.ToString();
if (!string.IsNullOrEmpty(name) && engine.TryGetBuildParameter(name, "DScript", out string value) && !string.IsNullOrEmpty(value))
{
result.Append(value);
atLeastOneExpansion = true;
}
else
{
AppendVariableUnexpanded(result, name);
}
// We are done parsing the variable name. Clear up the builder so subsequent names can be constructed.
variableName.Clear();
inAVariableName = false;
// If the end of the variable name is flagged by an explicit separator (e.g. %PATH%), then there is nothing else to do
// and this ending separator won't be part of the result.
// Otherwise the ending character can be either the start of a new variable or a regular part of the string
if (!EndOfVariableNameIsSeparator())
{
if (c == s_variableSeparator)
{
inAVariableName = true;
// The separator is always discarded and not part of the name
continue;
}
else
{
result.Append(c);
}
}
}
// We are parsing a variable name and the current char is part of the name
else
{
variableName.Append(c);
}
}
}
// We finished traversing the string while interpreting a variable name
if (inAVariableName)
{
// If a separator is needed to end a variable name, then this is a malformed name. E.g."C:\foo\%PATH"
// So we leave it unexpanded
if (EndOfVariableNameIsSeparator())
{
result.Append(s_variableSeparator);
result.Append(variableName);
}
else
{
// Otherwise, the end of the string defines the end of the name. Expand it as needed
var name = variableName.ToString();
if (!string.IsNullOrEmpty(name) && engine.TryGetBuildParameter(name, "DScript", out string value) && !string.IsNullOrEmpty(value))
{
result.Append(value);
atLeastOneExpansion = true;
}
else
{
AppendVariableUnexpanded(result, name);
}
}
}
expandedString = result.ToString();
return(atLeastOneExpansion);
}
private static bool TryBuildWorkspaceInternal(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IMutablePipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
bool forIDE,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by the IDE.
if (frontEndEngineAbstraction != null)
{
// The IDE engine typically doesn't have mounts configured. We do it here if they haven't been configured yet.
// Observe these are just the default mounts used for config evaluation.
if (frontEndEngineAbstraction is BasicFrontEndEngineAbstraction basicEngine && !frontEndEngineAbstraction.GetMountNames("Script", BuildXL.Utilities.ModuleId.Invalid).Any())
{
// If this fails we just ignore the failure. Mounts not being properly configured doesn't prevent the IDE plugin from working.
basicEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties);
}
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
else
{
// Otherwise we construct one with all mounts populated for config evaluation
var configurationEngine = new BasicFrontEndEngineAbstraction(engineContext.PathTable, engineContext.FileSystem, mutableCommandlineConfig);
if (!configurationEngine.TryPopulateWithDefaultMountsTable(loggingContext, engineContext, mutableCommandlineConfig, mutableCommandlineConfig.Startup.Properties))
{
// Errors are logged already
return(false);
}
frontEndEngineAbstraction = configurationEngine;
}
var config = controller.ParseConfig(frontEndEngineAbstraction, mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IMutablePipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
// For the IDE case, we want to make sure all config-specific mounts are properly populated
if (forIDE && frontEndEngineAbstraction is BasicFrontEndEngineAbstraction languageServiceEngine)
{
Contract.AssertNotNull(frontEndEngineAbstraction);
AddConfigurationMounts(config, mountsTable);
languageServiceEngine.SetMountsTable(mountsTable);
}
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(loggingContext),
5000,
false,
controller.RegisteredFrontEnds);
var searchPathToolsHash = new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable).ComputeSearchPathToolsHash();
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Pips.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
searchPathToolsHash: searchPathToolsHash);
// Observe mount table is completed during workspace construction
AddConfigurationMounts(config, mountsTable);
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
/// <summary>
/// Tries to create a filtered workspace based on a front-end snapshot from the previous BuildXL invocation.
/// </summary>
/// <returns>
/// * Possibke<ValidConstructedWorkspace> when the workspace was successfully constructed.
/// * Possible<null> when the snapshot was unaiable.
/// * Failure when the snapshot was available but parsing failed.
/// </returns>
private async Task <Possible <Workspace> > TryCreateFilteredWorkspaceAsync(Possible <WorkspaceDefinition> workspaceDefinition, IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
if (!FrontEndConfiguration.ConstructAndSaveBindingFingerprint())
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
"Binding fingerprint is disabled. Please use 'constructAndSaveBindingFingerprint' option to turn it on");
return(default(Possible <Workspace>));
}
// If a filter cannot be performed and public facade + AST is not to be used, then there is no point in continuing and we can
// go to full mode
if (!evaluationFilter.CanPerformPartialEvaluationScript(PrimaryConfigFile) && !CanUseSpecPublicFacadeAndAst())
{
var message = !CanUseSpecPublicFacadeAndAst()
? "Engine state was not reloaded"
: "User filter was not specified";
Logger.FailToReuseFrontEndSnapshot(LoggingContext, message);
return(default(Possible <Workspace>));
}
var changedFiles = engineAbstraction.GetChangedFiles()?.ToList();
if (changedFiles == null)
{
Logger.FailToReuseFrontEndSnapshot(LoggingContext, "Change journal is not available");
return(default(Possible <Workspace>));
}
using (var sw = Watch.Start())
{
// We're potentially in incremental mode.
var filteredDefinitionResult = await TryFilterWorkspaceDefinitionIncrementallyAsync(
changedFiles,
workspaceProvider,
workspaceDefinition.Result,
evaluationFilter);
if (filteredDefinitionResult.Failed)
{
return(filteredDefinitionResult.Failure);
}
if (filteredDefinitionResult.Filtered)
{
var filteredDefinition = filteredDefinitionResult.FilteredDefinition;
Logger.WorkspaceDefinitionFiltered(
LoggingContext,
filteredDefinition.SpecCount,
workspaceDefinition.Result.SpecCount,
sw.ElapsedMilliseconds);
// TODO: with C# 7, use tuple instead of changing the workspace to carry the information about the filtering.
return(await workspaceProvider.CreateWorkspaceAsync(filteredDefinition, userFilterWasApplied : true));
}
}
return(default(Possible <Workspace>));
}
private async Task <Workspace> BuildAndFilterWorkspaceAsync(WorkspaceDefinition workspaceDefinition, IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
// First, trying to filter workspace based on information from the previous run
var possibleFilteredWorkspace = await TryCreateFilteredWorkspaceAsync(workspaceDefinition, workspaceProvider, engineAbstraction, evaluationFilter);
if (!possibleFilteredWorkspace.Succeeded)
{
// Error was already logged
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, possibleFilteredWorkspace.Failure));
}
// If the filtered workspace is not null, just return it.
// Otherwise falling back to the full parse mode.
if (possibleFilteredWorkspace.Result != null)
{
return(possibleFilteredWorkspace.Result);
}
// "Incremental" workspace construction has failed, but we still can try to use module filter to build a smaller workspace.
if (evaluationFilter.ModulesToResolve.Count != 0)
{
var filteredDefinition = this.ApplyModuleFilter(workspaceDefinition, evaluationFilter.ModulesToResolve);
return(await workspaceProvider.CreateWorkspaceAsync(filteredDefinition, userFilterWasApplied : true));
}
Logger.BuildingFullWorkspace(LoggingContext);
return(await workspaceProvider.CreateWorkspaceAsync(workspaceDefinition, userFilterWasApplied : false));
}
/// <summary>
/// Builds and filters the worksapce.
/// </summary>
/// <remarks>
/// This method not just builds the workspace from scratch, but it also tries to compute it in an efficient way.
/// If there is a front end snapshot from the previous BuildXL run and the engine gives us a set of changed files,
/// then we can build a filtered workspace based on the old spec-2-spec map without parsing the entire world.
/// </remarks>
private async Task <Workspace> BuildAndFilterWorkspaceAsync(IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
// this step downloads nugets too, and that's why we want to do it outside of the progress reporting block below
Possible <WorkspaceDefinition> workspaceDefinition = await TryGetWorkspaceDefinitionAsync(workspaceProvider);
if (!workspaceDefinition.Succeeded)
{
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, workspaceDefinition.Failure));
}
return(await WithWorkspaceProgressReportingAsync(
numSpecs : workspaceDefinition.Result.SpecCount,
task : BuildAndFilterWorkspaceAsync(workspaceDefinition.Result, workspaceProvider, engineAbstraction, evaluationFilter)));
}
private bool ConstructAndEvaluateGraph(
LoggingContext loggingContext,
FrontEndEngineAbstraction frontEndEngineAbstration,
CacheInitializationTask engineCacheTask,
MountsTable mountsTable,
EvaluationFilter evaluationFilter,
[CanBeNull] GraphReuseResult reuseResult,
out PipGraph pipGraph)
{
Contract.Requires(frontEndEngineAbstration != null);
Contract.Requires(engineCacheTask != null);
Contract.Requires(mountsTable != null);
pipGraph = null;
IPipGraphBuilder pipGraphBuilder = null;
if (!AddConfigurationMountsAndCompleteInitialization(loggingContext, mountsTable))
{
return(false);
}
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(Configuration.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
m_visualization?.MountsTable.MakeAvailable(mountsTable);
if ((Configuration.Engine.Phase & EnginePhases.Schedule) != 0)
{
pipGraphBuilder = CreatePipGraphBuilder(loggingContext, mountsTable, reuseResult);
}
// Have to do some horrible magic here to get to a proper Task<T> with the BuildXL cache since
// someone updated the engine cache to be an await style pattern, and there is no way to get to the EngineCache
// If the cache was fast to startup, but perhaps blocked itself on first access we wouldn't have to do all these hoops.
Func <Task <Possible <EngineCache> > > getBuildCacheTask =
async() =>
{
return((await engineCacheTask).Then(engineCache => engineCache.CreateCacheForContext()));
};
if (!FrontEndController.PopulateGraph(
getBuildCacheTask(),
pipGraphBuilder,
frontEndEngineAbstration,
evaluationFilter,
Configuration,
m_initialCommandLineConfiguration.Startup))
{
LogFrontEndStats(loggingContext);
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after FrontEndController.PopulateGraph()");
return(false);
}
LogFrontEndStats(loggingContext);
// Pip graph must become immutable now that evaluation is done (required to construct a scheduler).
return(pipGraphBuilder == null || (pipGraph = pipGraphBuilder.Build()) != null);
}
/// <summary>
/// Create a simple wrapper representing the real file system
/// </summary>
public EngineFileSystem(PathTable pathTable, FrontEndEngineAbstraction engine)
: base(pathTable)
{
Contract.Requires(pathTable != null);
m_engine = engine;
}
/// <nodoc />
public FormattingProvider(ProviderContext providerContext, FrontEndEngineAbstraction engineAbstraction)
: base(providerContext)
{
m_engineAbstraction = engineAbstraction;
}
/// <summary>
/// Returns <code>true</code> IFF information about both changed and unchanged files is available.
/// <see cref="FrontEndEngineAbstraction.GetChangedFiles"/>
/// <see cref="FrontEndEngineAbstraction.GetUnchangedFiles"/>
/// </summary>
/// <remarks>
/// Information about changed files may not always be available, e.g., because engine cache is missing,
/// engine cache is corrupted, USNJournal is not working, etc.
/// </remarks>
public static bool IsSpecChangeInformationAvailable(this FrontEndEngineAbstraction engineAbstraction)
{
return
(engineAbstraction.GetChangedFiles() != null &&
engineAbstraction.GetUnchangedFiles() != null);
}
public static bool TryBuildWorkspace(
EnginePhases phase,
FrontEndContext frontEndContext,
PipExecutionContext engineContext,
AbsolutePath configFile,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(configFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var commandlineConfig = GetCommandLineConfiguration(configuration, phase, configFile);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(commandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: commandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, commandlineConfig);
frontEndHostController = controller as FrontEndHostController;
// If there is an explicit engine abstraction, we set it
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: commandlineConfig);
}
var config = controller.ParseConfig(commandlineConfig);
if (config == null)
{
frontEndHostController = null;
return(false);
}
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(
new SchedulerContext(
CancellationToken.None,
frontEndContext.StringTable,
frontEndContext.PathTable,
frontEndContext.SymbolTable,
frontEndContext.QualifierTable)),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: null /* No need to create pips */,
engineAbstraction: frontEndEngineAbstraction ?? new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config),
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: commandlineConfig.Startup))
{
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
}
Contract.Assert(frontEndHostController != null);
// If workspace construction is successfull, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
frontEndHostController.GetWorkspace(),
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
return(true);
}
public static bool TryBuildWorkspace(
ICommandLineConfiguration commandLineConfig,
FrontEndContext frontEndContext,
EngineContext engineContext,
EvaluationFilter evaluationFilter,
EventHandler <WorkspaceProgressEventArgs> progressHandler,
out Workspace workspace,
out FrontEndHostController frontEndHostController,
out IPipGraph pipGraph,
WorkspaceBuilderConfiguration configuration,
FrontEndEngineAbstraction frontEndEngineAbstraction = null,
bool collectMemoryAsSoonAsPossible = true)
{
Contract.Requires((commandLineConfig.Engine.Phase & (EnginePhases.ParseWorkspace | EnginePhases.AnalyzeWorkspace)) != EnginePhases.None);
Contract.Requires(frontEndContext != null);
Contract.Requires(engineContext != null);
Contract.Requires(commandLineConfig.Startup.ConfigFile.IsValid);
Contract.Requires(evaluationFilter != null);
workspace = null;
frontEndHostController = null;
pipGraph = null;
var pathTable = engineContext.PathTable;
var loggingContext = frontEndContext.LoggingContext;
var mutableCommandlineConfig = GetCommandLineConfiguration(
commandLineConfig,
configuration);
BuildXLEngine.ModifyConfigurationForCloudbuild(mutableCommandlineConfig, false, pathTable, loggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(mutableCommandlineConfig, pathTable, bxlExeLocation: null);
var statistics = new FrontEndStatistics(progressHandler);
var frontEndControllerFactory = FrontEndControllerFactory.Create(
mode: FrontEndMode.NormalMode,
loggingContext: loggingContext,
configuration: mutableCommandlineConfig,
collector: null,
statistics: statistics,
collectMemoryAsSoonAsPossible: collectMemoryAsSoonAsPossible);
var controller = frontEndControllerFactory.Create(engineContext.PathTable, engineContext.SymbolTable);
controller.InitializeHost(frontEndContext, mutableCommandlineConfig);
frontEndHostController = (FrontEndHostController)controller;
// If there is an explicit engine abstraction, we set it. This is used by IDE test.
if (frontEndEngineAbstraction != null)
{
frontEndHostController.SetState(frontEndEngineAbstraction, pipGraph: null, configuration: mutableCommandlineConfig);
}
var config = controller.ParseConfig(mutableCommandlineConfig);
if (config == null)
{
return(false);
}
IPipGraph pipGraphBuilder = null;
using (var cache = Task.FromResult <Possible <EngineCache> >(
new EngineCache(
new InMemoryArtifactContentCache(),
// Note that we have an 'empty' store (no hits ever) rather than a normal in memory one.
new EmptyTwoPhaseFingerprintStore())))
{
if (frontEndEngineAbstraction == null)
{
if (mutableCommandlineConfig.Engine.Phase.HasFlag(EnginePhases.Schedule))
{
var mountsTable = MountsTable.CreateAndRegister(loggingContext, engineContext, config, mutableCommandlineConfig.Startup.Properties);
frontEndEngineAbstraction = new FrontEndEngineImplementation(
loggingContext,
frontEndContext.PathTable,
config,
mutableCommandlineConfig.Startup,
mountsTable,
InputTracker.CreateDisabledTracker(loggingContext),
null,
null,
() => FileContentTable.CreateStub(),
5000,
false);
pipGraphBuilder = new PipGraph.Builder(
EngineSchedule.CreateEmptyPipTable(engineContext),
engineContext,
Scheduler.Tracing.Logger.Log,
loggingContext,
config,
mountsTable.MountPathExpander,
fingerprintSalt: config.Cache.CacheSalt,
directoryMembershipFingerprinterRules: new DirectoryMembershipFingerprinterRuleSet(config, engineContext.StringTable));
if (!AddConfigurationMountsAndCompleteInitialization(config, loggingContext, mountsTable))
{
return(false);
}
IDictionary <ModuleId, MountsTable> moduleMountsTableMap;
if (!mountsTable.PopulateModuleMounts(config.ModulePolicies.Values, out moduleMountsTableMap))
{
Contract.Assume(loggingContext.ErrorWasLogged, "An error should have been logged after MountTable.PopulateModuleMounts()");
return(false);
}
}
else
{
frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(frontEndContext.PathTable, frontEndContext.FileSystem, config);
}
}
using (frontEndEngineAbstraction is IDisposable ? (IDisposable)frontEndEngineAbstraction : null)
{
// Attempt to build and/or analyze the workspace
if (!controller.PopulateGraph(
cache: cache,
graph: pipGraphBuilder,
engineAbstraction: frontEndEngineAbstraction,
evaluationFilter: evaluationFilter,
configuration: config,
startupConfiguration: mutableCommandlineConfig.Startup))
{
workspace = frontEndHostController.GetWorkspace();
// Error has been reported already
return(false);
}
pipGraph = pipGraphBuilder;
}
}
Contract.Assert(frontEndHostController != null);
workspace = frontEndHostController.GetWorkspace();
if (mutableCommandlineConfig.Engine.Phase == EnginePhases.AnalyzeWorkspace)
{
// If workspace construction is successful, we run the linter on all specs.
// This makes sure the workspace will carry all the errors that will occur when running the same specs in the regular engine path
workspace = CreateLintedWorkspace(
workspace,
frontEndContext.LoggingContext,
config.FrontEnd,
pathTable);
}
return(true);
}
/// <summary>
/// TODO: This is code duplicated from the configuration processor. Consider refactoring
/// </summary>
private static AbsolutePath FindPrimaryConfiguration(AbsolutePath rootFolder, FrontEndEngineAbstraction engine, PathTable pathTable)
{
return(TryFindConfig(rootFolder, engine, pathTable));
}
/// <summary>
/// Builds and filters the worksapce.
/// </summary>
/// <remarks>
/// This method not just builds the workspace from scratch, but it also tries to compute it in an efficient way.
/// If there is a front end snapshot from the previous BuildXL run and the engine gives us a set of changed files,
/// then we can build a filtered workspace based on the old spec-2-spec map without parsing the entire world.
/// </remarks>
private async Task <Workspace> BuildAndFilterWorkspaceAsync(IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
// this step downloads nugets too, and that's why we want to do it outside of the progress reporting block below
Possible <WorkspaceDefinition> workspaceDefinition = await TryGetWorkspaceDefinitionAsync(workspaceProvider);
if (!workspaceDefinition.Succeeded)
{
// In some cases even if the workspace failed to build some of the pipeline still tries to continue
// Complete mount initialization to enable mount related queries downstream
Engine.CompleteMountInitialization();
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, workspaceDefinition.Failure));
}
// As soon as we get the workspace definition, we can configure the mount table with the additional mounts modules may have defined and seal it
foreach (var module in workspaceDefinition.Result.Modules.Where(module => module.Mounts != null))
{
foreach (var mount in module.Mounts)
{
Engine.AddResolvedModuleDefinedMount(mount, LocationData.Create(module.ModuleConfigFile));
}
}
// At this point the mount table can be completed
if (!Engine.CompleteMountInitialization())
{
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, new GenericWorkspaceFailure("Mount points not properly defined. Detailed errors should have been logged.")));
}
return(await WithWorkspaceProgressReportingAsync(
numSpecs : workspaceDefinition.Result.SpecCount,
task : BuildAndFilterWorkspaceAsync(workspaceDefinition.Result, workspaceProvider, engineAbstraction, evaluationFilter)));
}
