/// <nodoc/>
public Workspace(
[CanBeNull] IWorkspaceProvider provider,
WorkspaceConfiguration workspaceConfiguration,
IEnumerable <ParsedModule> modules,
IEnumerable <Failure> failures,
[CanBeNull] ParsedModule preludeModule,
[CanBeNull] ParsedModule configurationModule)
{
Contract.Requires(workspaceConfiguration != null);
Contract.Requires(modules != null);
Contract.Requires(failures != null);
Contract.RequiresForAll(modules, m => m != null);
WorkspaceProvider = provider;
WorkspaceConfiguration = workspaceConfiguration;
var allModules = GetAllParsedModules(modules, preludeModule, configurationModule);
m_specModules = allModules.Where(m => m != preludeModule && m != configurationModule).ToArray();
// Double ownership is not allowed: specs are already validated for double ownership
m_specSources = CreateSpecsFromModules(m_specModules, allowDoubleOwnership: false);
m_specialSpecs = CreateSpecsForPreludeAndConfiguration(preludeModule, configurationModule);
// Spec array contains all the specs for the workspace.
m_specArray = m_specSources.ToDictionary().AddRange(m_specialSpecs).Select(s => s.Value.SourceFile).ToArray();
m_allModulesByDescriptor = AllModulesByDescriptor(allModules);
Failures = failures.ToArray();
PreludeModule = preludeModule;
ConfigurationModule = configurationModule;
}
/// <summary>
/// Creates a queue that starts with some already parsed module and a pending module under construction
/// </summary>
public static ModuleParsingQueue CreateIncrementalQueue(
WorkspaceProvider workspaceProvider,
WorkspaceConfiguration workspaceConfiguration,
IModuleReferenceResolver moduleReferenceResolver,
ModuleDefinition designatedPrelude,
ParsedModule configurationModule,
IEnumerable <ParsedModule> parsedModules,
IEnumerable <Failure> failures)
{
Contract.Requires(workspaceProvider != null);
Contract.Requires(moduleReferenceResolver != null);
Contract.Requires(parsedModules != null);
Contract.Requires(failures != null);
var parsedModulesDictionary =
new ConcurrentDictionary <ModuleDescriptor, ParsedModule>(parsedModules.Select(parsedModule => new KeyValuePair <ModuleDescriptor, ParsedModule>(parsedModule.Descriptor, parsedModule)));
var failureBag = new ConcurrentQueue <Failure>(failures);
// For IDE mode it is very crucial to preserve trivias. For instance, without it, there is no way to check that the current position is inside a comment.
var queue = new ModuleParsingQueue(
workspaceProvider,
workspaceConfiguration,
moduleReferenceResolver,
designatedPrelude,
configurationModule,
parsedModulesDictionary,
failureBag,
preserveTrivias: true);
return(queue);
}
private void CheckForCompletionAndSignalQueueIfDone(ModuleUnderConstruction owningModule)
{
// Now that all dependencies of the owning module have been scheduled,
// we check if the module for which we just added a spec is complete
if (owningModule.IsModuleFirstTimeComplete())
{
var moduleId = owningModule.Definition.Descriptor;
// We create a parsed module out of a module under construction.
// Observe many threads can try to create the same completed module. But that's ok, the method is thread safe
// and always return the same instance.
// It is important to notice that we should add this module to m_modulesAlreadyParsed before removing it from
// m_modulesToBeParsed. This is so no producer will think the module is not there at any point in time (which
// would cause the producer to try to schedule that module again)
var success = owningModule.TryCreateCompletedModule(m_moduleReferenceResolver, m_workspaceConfiguration, m_queueOptions.CancelOnFirstFailure, out ParsedModule module, out Failure[] failures);
if (success)
{
m_modulesAlreadyParsed[moduleId] = module;
// Now the module is completed and we can bind the changed file as well.
var modifiedSourceFile = owningModule.GetSourceFileForInjectingQualifiers();
if (modifiedSourceFile != null)
{
ScheduleSourceFileBinding(new ParsedSpecWithOwningModule(parsedFile: modifiedSourceFile, owningModule: owningModule.Definition));
}
}
else
{
// This is a bogus module we put here when failures occurr. This guarantees that modules
// are not scheduled more than once (detailed reasons above). When there are failures
// we want to report them only once, so leave that to whoever successfully removes
// the module from m_modulesToBeParsed to report them.
// Observe that this (empty) module will be left in case of failures, but that's not
// likely to be a problem
m_modulesAlreadyParsed[moduleId] = new ParsedModule(owningModule.Definition, hasFailures: true);
}
// If removing fails, that means that somebody else successfully removed the module, so we do nothing
// If removing succeeds, we are responsible for signaling if the queue is done
if (m_modulesToBeParsed.TryRemove(moduleId, out ModuleUnderConstruction dummy))
{
if (!success)
{
ReportFailureAndCancelParsingIfNeeded(failures);
}
// Now we decide if we are done with parsing the transitive closure.
// This happens when the set of modules to be parsed is empty
// and EnqueuingCompleted has been called.
// Observe that it is important that we first add all dependencies to be parsed
// before removing a completed module. This ensures
// that if anybody sees an empty collection, we are done for sure
if (m_modulesToBeParsed.IsEmpty && m_enqueingComplete)
{
cancellationTokenChain.Dispose();
m_parseQueue.Complete();
}
}
}
}
private ModuleParsingQueue(
WorkspaceProvider workspaceProvider,
WorkspaceConfiguration workspaceConfiguration,
IModuleReferenceResolver moduleReferenceResolver,
ModuleDefinition designatedPrelude,
ParsedModule configurationModule,
ConcurrentDictionary <ModuleDescriptor, ParsedModule> alreadyParsedModules,
ConcurrentQueue <Failure> failures,
bool preserveTrivias = false)
{
Contract.Requires(workspaceProvider != null);
Contract.Requires(workspaceConfiguration != null);
Contract.Requires(moduleReferenceResolver != null);
Contract.Requires(alreadyParsedModules != null, "alreadyParsedModules != null");
Contract.Requires(failures != null, "failures != null");
m_modulesToBeParsed = new ConcurrentDictionary <ModuleDescriptor, ModuleUnderConstruction>();
m_modulesAlreadyParsed = alreadyParsedModules;
m_failures = failures;
m_workspaceProvider = workspaceProvider;
m_moduleReferenceResolver = moduleReferenceResolver;
m_designatedPrelude = designatedPrelude;
m_configurationModule = configurationModule;
m_workspaceConfiguration = workspaceConfiguration;
m_parsingOptions = workspaceConfiguration.ParsingOptions;
if (preserveTrivias)
{
m_parsingOptions = (m_parsingOptions ?? ParsingOptions.DefaultParsingOptions).WithTrivia(true);
}
DegreeOfParallelism = workspaceConfiguration.MaxDegreeOfParallelismForParsing;
// WARNING: this is extremely subtle.
// We need to keep a 'registration token' from the chained operation we are doing next to avoid memory leak.
// The instance of this class stores the reference to key front-end objects, like resolvers,
// that keeps the entire front-end in memory.
// CancellationToken.Register registers the call back, that lead to a closure allocation of the current instance.
// And this means that the lifetime of this instance is coupled to the lifetime of the the workspaceConfiguration.CancellationToken which is global.
// This means that if we won't dispose the resistration we'll keep the entire front-end in memory for the entire app life time.
cancellationTokenChain = workspaceConfiguration.CancellationToken.Register(() => m_cancellationTokenSource.Cancel());
m_queueOptions = new ModuleParsingQueueOptions()
{
CancelOnFirstFailure = workspaceConfiguration.CancelOnFirstFailure,
MaxDegreeOfParallelism = DegreeOfParallelism,
CancellationToken = CancellationToken,
};
m_parseQueue = new ActionBlock <SpecWithOwningModule>(ProcessQueuedItemForParsing, m_queueOptions);
Action <ParsedSpecWithOwningModule> action = ProcessQueueItemForBinding;
m_bindQueue = new ActionBlock <ParsedSpecWithOwningModule>(action, m_queueOptions);
}
/// <nodoc />
public SpecFileWithMetadata([NotNull] ISourceFile sourceFile, [NotNull] ParsedModule owningModule, SpecState state)
{
Contract.Requires(sourceFile != null);
Contract.Requires(owningModule != null);
SourceFile = sourceFile;
OwningModule = owningModule;
State = state;
}
/// <summary>
/// Creates a module parsing queue. The queue options are specified by the provided queueOptions.
/// </summary>
public ModuleParsingQueue(
[NotNull] WorkspaceProvider workspaceProvider,
WorkspaceConfiguration workspaceConfiguration,
IModuleReferenceResolver moduleReferenceResolver,
ModuleDefinition designatedPrelude,
ParsedModule configurationModule)
: this(workspaceProvider, workspaceConfiguration, moduleReferenceResolver, designatedPrelude, configurationModule,
new ConcurrentDictionary <ModuleDescriptor, ParsedModule>(), new ConcurrentQueue <Failure>())
{
Contract.Requires(moduleReferenceResolver != null);
}
/// <summary>
/// Creates a workspace from all known modules in an incremental way
/// </summary>
/// <remarks>
/// It reuses already parsed modules so they don't get recomputed again
/// </remarks>
public async Task <Workspace> CreateIncrementalWorkspaceForAllKnownModulesAsync(
IEnumerable <ParsedModule> parsedModules,
ModuleUnderConstruction moduleUnderConstruction,
IEnumerable <Failure> failures,
[CanBeNull] ParsedModule preludeModule)
{
var maybeModuleDefinitions = await GetModuleDefinitionsForAllResolversAsync();
if (!maybeModuleDefinitions.Succeeded)
{
return(Failure(maybeModuleDefinitions.Failure));
}
var moduleDefinitions = maybeModuleDefinitions.Result;
ModuleDefinition preludeDefinition;
if (preludeModule != null)
{
// Need to add prelude to a list of parsed modules if awailable
var parsedModuleList = parsedModules.ToList();
parsedModuleList.Add(preludeModule);
parsedModules = parsedModuleList;
preludeDefinition = preludeModule.Definition;
}
else
{
var possiblePreludeDefinition = await TryGetPreludeModuleDefinitionAsync();
if (!possiblePreludeDefinition.Succeeded)
{
return(Failure(possiblePreludeDefinition.Failure));
}
preludeDefinition = possiblePreludeDefinition.Result;
moduleDefinitions.Add(preludeDefinition);
}
if (!WorkspaceValidator.ValidateModuleDefinitions(moduleDefinitions, PathTable, out var validationFailures))
{
return(Failure(validationFailures));
}
var queue = ModuleParsingQueue.CreateIncrementalQueue(
this,
Configuration,
m_moduleReferenceResolver,
preludeDefinition,
GetConfigurationModule(),
parsedModules,
failures);
return(await queue.ProcessIncrementalAsync(moduleUnderConstruction));
}
/// <inheritdoc />
public async Task <Workspace> CreateIncrementalWorkspaceForAllKnownModulesAsync(
IEnumerable <ParsedModule> parsedModules,
ModuleUnderConstruction moduleUnderConstruction,
IEnumerable <Failure> failures,
ParsedModule preludeModule)
{
using (m_statistics.EndToEndParsing.Start())
{
return(await m_decoratee.CreateIncrementalWorkspaceForAllKnownModulesAsync(parsedModules, moduleUnderConstruction, failures, preludeModule));
}
}
private ParsedModule GetProcessedPreludeOrDefault()
{
// If there is a designated prelude module, we identify it for the workspace
ParsedModule prelude = null;
if (m_designatedPrelude != null)
{
m_modulesAlreadyParsed.TryGetValue(m_designatedPrelude.Descriptor, out prelude);
}
return(prelude);
}
private ParsedModule GetProcessedConfigurationModuleOrDefault()
{
ParsedModule result = m_configurationModule;
if (m_configurationModule != null)
{
if (m_modulesAlreadyParsed.TryGetValue(m_configurationModule.Descriptor, out var parsedConfigurationModule))
{
result = parsedConfigurationModule;
}
}
return(result);
}
public SemanticWorkspace(
IWorkspaceProvider workspaceProvider,
WorkspaceConfiguration workspaceConfiguration,
IEnumerable <ParsedModule> modules,
[CanBeNull] ParsedModule preludeModule,
[CanBeNull] ParsedModule configurationModule,
ISemanticModel semanticModel,
IReadOnlyCollection <Failure> failures)
: base(workspaceProvider, workspaceConfiguration, modules, failures, preludeModule, configurationModule)
{
Contract.Requires(semanticModel != null);
m_semanticModel = semanticModel;
}
/// <inheritdoc/>
public override bool TryGetOwningModule(string fileName, out ModuleName moduleName)
{
var path = AbsolutePath.Create(m_pathTable, fileName);
ParsedModule parsedModule = m_workspace.TryGetModuleBySpecFileName(path);
if (parsedModule != null)
{
moduleName = CreateModuleName(parsedModule.Definition);
return(true);
}
moduleName = ModuleName.Invalid;
return(false);
}
private static List <ParsedModule> SpecialModules(ParsedModule preludeModule, ParsedModule configurationModule)
{
var modules = new List <ParsedModule>();
if (preludeModule != null)
{
modules.Add(preludeModule);
}
if (configurationModule != null)
{
modules.Add(configurationModule);
}
return(modules);
}
/// <summary>
/// Creates a parsing queue for parsing specs in a regular BuildXL invocation.
/// </summary>
public static ModuleParsingQueue Create(
[NotNull] WorkspaceProvider workspaceProvider,
[NotNull] WorkspaceConfiguration workspaceConfiguration,
[NotNull] IModuleReferenceResolver moduleReferenceResolver,
[CanBeNull] ModuleDefinition designatedPrelude,
[CanBeNull] ParsedModule configurationModule)
{
Contract.Requires(workspaceProvider != null);
return(new ModuleParsingQueue(
workspaceProvider,
workspaceConfiguration,
moduleReferenceResolver,
designatedPrelude,
configurationModule));
}
private IEnumerable <ParsedModule> GetAllSourceModules(ParsedModule prelude, ParsedModule configurationModule)
{
var result = new HashSet <ParsedModule>(m_modulesAlreadyParsed.Values);
if (prelude != null)
{
result.Remove(prelude);
}
if (configurationModule != null)
{
result.Remove(configurationModule);
}
return(result);
}
/// <inheritdoc />
public ParsedModule GetConfigurationModule()
{
// In some scenarios (like tests) main config workspace is unavailable.
if (m_mainConfigurationWorkspace == null)
{
return(null);
}
if (m_configurationModule == null)
{
m_configurationModule = CreateConfigurationModule();
}
return(m_configurationModule);
ParsedModule CreateConfigurationModule()
{
// Module with all configuration files includes all the files obtained during main config processing
// as well as other module configuration files that each resolver used to compute their modules.
var moduleFiles = GetModuleConfigurationFiles();
var moduleFileNames = new HashSet <AbsolutePath>(moduleFiles.Keys);
var mainConfigurationModule = m_mainConfigurationWorkspace.ConfigurationModule;
Contract.Assert(mainConfigurationModule != null);
if (moduleFileNames.Count == 0)
{
// If there is no module configuration files, we can use main configuration as a result.
return(m_mainConfigurationWorkspace.ConfigurationModule);
}
// For the full configuration module using the main configuration file.
// Main config should be part of the module definition as well.
moduleFileNames.AddRange(mainConfigurationModule.PathToSpecs);
var module = CreateConfigModuleDefinition(mainConfigurationModule.Definition.MainFile, moduleFileNames, m_configurationResolver);
// Need to add files from the main configuration to the final set of files.
moduleFiles.AddRange(mainConfigurationModule.Specs);
var parsedModule = new ParsedModule(module, moduleFiles);
return(parsedModule);
}
}
private static HashSet <ParsedModule> GetAllParsedModules(
IEnumerable <ParsedModule> specModules,
ParsedModule preludeModule,
ParsedModule configurationModule)
{
var result = new HashSet <ParsedModule>(specModules);
if (preludeModule != null)
{
result.Add(preludeModule);
}
if (configurationModule != null)
{
result.Add(configurationModule);
}
return(result);
}
private async Task <Workspace> EnqueuingFinishedAndWaitForCompletion()
{
// No more new module definition may be added for processing
EnqueuingCompleted();
try
{
await m_parseQueue.Completion;
}
catch (TaskCanceledException)
{
// Expected. This means that at least one task failed parsing
}
// Parsing is complete. Just need to wait for pending items to finish binding.
m_bindQueue.Complete();
try
{
await m_bindQueue.Completion;
}
catch (TaskCanceledException)
{
// Expected. This means that at least one task failed binding
}
// Need to check if the queue reparsed prelude or configuration module.
ParsedModule prelude = GetProcessedPreludeOrDefault();
ParsedModule configurationModule = GetProcessedConfigurationModuleOrDefault();
// The workspace is ready to be constructed. So at this point we can run some workspace-level validations.
var workspaceFailures = WorkspaceValidator.ValidateParsedModules(m_modulesAlreadyParsed.Values, m_workspaceProvider.PathTable);
// Create the result. Observe that if the task is cancelled, we don't propagate it
// but reflect it in the result.
return(new Workspace(
m_workspaceProvider,
m_workspaceConfiguration,
GetAllSourceModules(prelude, configurationModule),
m_failures.Union(workspaceFailures),
preludeModule: prelude,
configurationModule: configurationModule));
}
private static Dictionary <AbsolutePath, SpecFileWithMetadata> CreateSpecsForPreludeAndConfiguration(ParsedModule preludeModule, ParsedModule configurationModule)
{
// The prelude module used for configuration parsing and the prelude module used for regular spec parsing may be the same
// So we allow double ownership in this case. The first module being processed will win spec ownership. It shouldn't matter which one.
return(CreateSpecsFromModules(SpecialModules(preludeModule, configurationModule), allowDoubleOwnership: true));
}
/// <nodoc/>
public bool TryGetModuleByModuleDescriptor(ModuleDescriptor moduleDescriptor, out ParsedModule parsedModule)
{
return(m_allModulesByDescriptor.TryGetValue(moduleDescriptor, out parsedModule));
}
/// <summary>
/// Creates a workspace for configuration processing.
/// </summary>
public static Workspace CreateConfigurationWorkspace(WorkspaceConfiguration configuration, ParsedModule configurationModule, ParsedModule preludeModule)
{
return(new Workspace(
provider: null,
workspaceConfiguration: configuration,
modules: CollectionUtilities.EmptyArray <ParsedModule>(),
failures: CollectionUtilities.EmptyArray <Failure>(),
preludeModule: preludeModule,
configurationModule: configurationModule));
}
/// <nodoc />
public static SpecFileWithMetadata CreateNew(ParsedModule owningModule, ISourceFile sourceFile)
{
return(new SpecFileWithMetadata(sourceFile, owningModule, SpecState.Changed));
}