/// <nodoc/> public Workspace( [CanBeNull] IWorkspaceProvider provider, WorkspaceConfiguration workspaceConfiguration, IEnumerable <ParsedModule> modules, IEnumerable <Failure> failures, [CanBeNull] ParsedModule preludeModule, [CanBeNull] ParsedModule configurationModule) { Contract.Requires(workspaceConfiguration != null); Contract.Requires(modules != null); Contract.Requires(failures != null); Contract.RequiresForAll(modules, m => m != null); WorkspaceProvider = provider; WorkspaceConfiguration = workspaceConfiguration; var allModules = GetAllParsedModules(modules, preludeModule, configurationModule); m_specModules = allModules.Where(m => m != preludeModule && m != configurationModule).ToArray(); // Double ownership is not allowed: specs are already validated for double ownership m_specSources = CreateSpecsFromModules(m_specModules, allowDoubleOwnership: false); m_specialSpecs = CreateSpecsForPreludeAndConfiguration(preludeModule, configurationModule); // Spec array contains all the specs for the workspace. m_specArray = m_specSources.ToDictionary().AddRange(m_specialSpecs).Select(s => s.Value.SourceFile).ToArray(); m_allModulesByDescriptor = AllModulesByDescriptor(allModules); Failures = failures.ToArray(); PreludeModule = preludeModule; ConfigurationModule = configurationModule; }
/// <summary> /// Creates a queue that starts with some already parsed module and a pending module under construction /// </summary> public static ModuleParsingQueue CreateIncrementalQueue( WorkspaceProvider workspaceProvider, WorkspaceConfiguration workspaceConfiguration, IModuleReferenceResolver moduleReferenceResolver, ModuleDefinition designatedPrelude, ParsedModule configurationModule, IEnumerable <ParsedModule> parsedModules, IEnumerable <Failure> failures) { Contract.Requires(workspaceProvider != null); Contract.Requires(moduleReferenceResolver != null); Contract.Requires(parsedModules != null); Contract.Requires(failures != null); var parsedModulesDictionary = new ConcurrentDictionary <ModuleDescriptor, ParsedModule>(parsedModules.Select(parsedModule => new KeyValuePair <ModuleDescriptor, ParsedModule>(parsedModule.Descriptor, parsedModule))); var failureBag = new ConcurrentQueue <Failure>(failures); // For IDE mode it is very crucial to preserve trivias. For instance, without it, there is no way to check that the current position is inside a comment. var queue = new ModuleParsingQueue( workspaceProvider, workspaceConfiguration, moduleReferenceResolver, designatedPrelude, configurationModule, parsedModulesDictionary, failureBag, preserveTrivias: true); return(queue); }
private void CheckForCompletionAndSignalQueueIfDone(ModuleUnderConstruction owningModule) { // Now that all dependencies of the owning module have been scheduled, // we check if the module for which we just added a spec is complete if (owningModule.IsModuleFirstTimeComplete()) { var moduleId = owningModule.Definition.Descriptor; // We create a parsed module out of a module under construction. // Observe many threads can try to create the same completed module. But that's ok, the method is thread safe // and always return the same instance. // It is important to notice that we should add this module to m_modulesAlreadyParsed before removing it from // m_modulesToBeParsed. This is so no producer will think the module is not there at any point in time (which // would cause the producer to try to schedule that module again) var success = owningModule.TryCreateCompletedModule(m_moduleReferenceResolver, m_workspaceConfiguration, m_queueOptions.CancelOnFirstFailure, out ParsedModule module, out Failure[] failures); if (success) { m_modulesAlreadyParsed[moduleId] = module; // Now the module is completed and we can bind the changed file as well. var modifiedSourceFile = owningModule.GetSourceFileForInjectingQualifiers(); if (modifiedSourceFile != null) { ScheduleSourceFileBinding(new ParsedSpecWithOwningModule(parsedFile: modifiedSourceFile, owningModule: owningModule.Definition)); } } else { // This is a bogus module we put here when failures occurr. This guarantees that modules // are not scheduled more than once (detailed reasons above). When there are failures // we want to report them only once, so leave that to whoever successfully removes // the module from m_modulesToBeParsed to report them. // Observe that this (empty) module will be left in case of failures, but that's not // likely to be a problem m_modulesAlreadyParsed[moduleId] = new ParsedModule(owningModule.Definition, hasFailures: true); } // If removing fails, that means that somebody else successfully removed the module, so we do nothing // If removing succeeds, we are responsible for signaling if the queue is done if (m_modulesToBeParsed.TryRemove(moduleId, out ModuleUnderConstruction dummy)) { if (!success) { ReportFailureAndCancelParsingIfNeeded(failures); } // Now we decide if we are done with parsing the transitive closure. // This happens when the set of modules to be parsed is empty // and EnqueuingCompleted has been called. // Observe that it is important that we first add all dependencies to be parsed // before removing a completed module. This ensures // that if anybody sees an empty collection, we are done for sure if (m_modulesToBeParsed.IsEmpty && m_enqueingComplete) { cancellationTokenChain.Dispose(); m_parseQueue.Complete(); } } } }
private ModuleParsingQueue( WorkspaceProvider workspaceProvider, WorkspaceConfiguration workspaceConfiguration, IModuleReferenceResolver moduleReferenceResolver, ModuleDefinition designatedPrelude, ParsedModule configurationModule, ConcurrentDictionary <ModuleDescriptor, ParsedModule> alreadyParsedModules, ConcurrentQueue <Failure> failures, bool preserveTrivias = false) { Contract.Requires(workspaceProvider != null); Contract.Requires(workspaceConfiguration != null); Contract.Requires(moduleReferenceResolver != null); Contract.Requires(alreadyParsedModules != null, "alreadyParsedModules != null"); Contract.Requires(failures != null, "failures != null"); m_modulesToBeParsed = new ConcurrentDictionary <ModuleDescriptor, ModuleUnderConstruction>(); m_modulesAlreadyParsed = alreadyParsedModules; m_failures = failures; m_workspaceProvider = workspaceProvider; m_moduleReferenceResolver = moduleReferenceResolver; m_designatedPrelude = designatedPrelude; m_configurationModule = configurationModule; m_workspaceConfiguration = workspaceConfiguration; m_parsingOptions = workspaceConfiguration.ParsingOptions; if (preserveTrivias) { m_parsingOptions = (m_parsingOptions ?? ParsingOptions.DefaultParsingOptions).WithTrivia(true); } DegreeOfParallelism = workspaceConfiguration.MaxDegreeOfParallelismForParsing; // WARNING: this is extremely subtle. // We need to keep a 'registration token' from the chained operation we are doing next to avoid memory leak. // The instance of this class stores the reference to key front-end objects, like resolvers, // that keeps the entire front-end in memory. // CancellationToken.Register registers the call back, that lead to a closure allocation of the current instance. // And this means that the lifetime of this instance is coupled to the lifetime of the the workspaceConfiguration.CancellationToken which is global. // This means that if we won't dispose the resistration we'll keep the entire front-end in memory for the entire app life time. cancellationTokenChain = workspaceConfiguration.CancellationToken.Register(() => m_cancellationTokenSource.Cancel()); m_queueOptions = new ModuleParsingQueueOptions() { CancelOnFirstFailure = workspaceConfiguration.CancelOnFirstFailure, MaxDegreeOfParallelism = DegreeOfParallelism, CancellationToken = CancellationToken, }; m_parseQueue = new ActionBlock <SpecWithOwningModule>(ProcessQueuedItemForParsing, m_queueOptions); Action <ParsedSpecWithOwningModule> action = ProcessQueueItemForBinding; m_bindQueue = new ActionBlock <ParsedSpecWithOwningModule>(action, m_queueOptions); }
/// <nodoc /> public SpecFileWithMetadata([NotNull] ISourceFile sourceFile, [NotNull] ParsedModule owningModule, SpecState state) { Contract.Requires(sourceFile != null); Contract.Requires(owningModule != null); SourceFile = sourceFile; OwningModule = owningModule; State = state; }
/// <summary> /// Creates a module parsing queue. The queue options are specified by the provided queueOptions. /// </summary> public ModuleParsingQueue( [NotNull] WorkspaceProvider workspaceProvider, WorkspaceConfiguration workspaceConfiguration, IModuleReferenceResolver moduleReferenceResolver, ModuleDefinition designatedPrelude, ParsedModule configurationModule) : this(workspaceProvider, workspaceConfiguration, moduleReferenceResolver, designatedPrelude, configurationModule, new ConcurrentDictionary <ModuleDescriptor, ParsedModule>(), new ConcurrentQueue <Failure>()) { Contract.Requires(moduleReferenceResolver != null); }
/// <summary> /// Creates a workspace from all known modules in an incremental way /// </summary> /// <remarks> /// It reuses already parsed modules so they don't get recomputed again /// </remarks> public async Task <Workspace> CreateIncrementalWorkspaceForAllKnownModulesAsync( IEnumerable <ParsedModule> parsedModules, ModuleUnderConstruction moduleUnderConstruction, IEnumerable <Failure> failures, [CanBeNull] ParsedModule preludeModule) { var maybeModuleDefinitions = await GetModuleDefinitionsForAllResolversAsync(); if (!maybeModuleDefinitions.Succeeded) { return(Failure(maybeModuleDefinitions.Failure)); } var moduleDefinitions = maybeModuleDefinitions.Result; ModuleDefinition preludeDefinition; if (preludeModule != null) { // Need to add prelude to a list of parsed modules if awailable var parsedModuleList = parsedModules.ToList(); parsedModuleList.Add(preludeModule); parsedModules = parsedModuleList; preludeDefinition = preludeModule.Definition; } else { var possiblePreludeDefinition = await TryGetPreludeModuleDefinitionAsync(); if (!possiblePreludeDefinition.Succeeded) { return(Failure(possiblePreludeDefinition.Failure)); } preludeDefinition = possiblePreludeDefinition.Result; moduleDefinitions.Add(preludeDefinition); } if (!WorkspaceValidator.ValidateModuleDefinitions(moduleDefinitions, PathTable, out var validationFailures)) { return(Failure(validationFailures)); } var queue = ModuleParsingQueue.CreateIncrementalQueue( this, Configuration, m_moduleReferenceResolver, preludeDefinition, GetConfigurationModule(), parsedModules, failures); return(await queue.ProcessIncrementalAsync(moduleUnderConstruction)); }
/// <inheritdoc /> public async Task <Workspace> CreateIncrementalWorkspaceForAllKnownModulesAsync( IEnumerable <ParsedModule> parsedModules, ModuleUnderConstruction moduleUnderConstruction, IEnumerable <Failure> failures, ParsedModule preludeModule) { using (m_statistics.EndToEndParsing.Start()) { return(await m_decoratee.CreateIncrementalWorkspaceForAllKnownModulesAsync(parsedModules, moduleUnderConstruction, failures, preludeModule)); } }
private ParsedModule GetProcessedPreludeOrDefault() { // If there is a designated prelude module, we identify it for the workspace ParsedModule prelude = null; if (m_designatedPrelude != null) { m_modulesAlreadyParsed.TryGetValue(m_designatedPrelude.Descriptor, out prelude); } return(prelude); }
private ParsedModule GetProcessedConfigurationModuleOrDefault() { ParsedModule result = m_configurationModule; if (m_configurationModule != null) { if (m_modulesAlreadyParsed.TryGetValue(m_configurationModule.Descriptor, out var parsedConfigurationModule)) { result = parsedConfigurationModule; } } return(result); }
public SemanticWorkspace( IWorkspaceProvider workspaceProvider, WorkspaceConfiguration workspaceConfiguration, IEnumerable <ParsedModule> modules, [CanBeNull] ParsedModule preludeModule, [CanBeNull] ParsedModule configurationModule, ISemanticModel semanticModel, IReadOnlyCollection <Failure> failures) : base(workspaceProvider, workspaceConfiguration, modules, failures, preludeModule, configurationModule) { Contract.Requires(semanticModel != null); m_semanticModel = semanticModel; }
/// <inheritdoc/> public override bool TryGetOwningModule(string fileName, out ModuleName moduleName) { var path = AbsolutePath.Create(m_pathTable, fileName); ParsedModule parsedModule = m_workspace.TryGetModuleBySpecFileName(path); if (parsedModule != null) { moduleName = CreateModuleName(parsedModule.Definition); return(true); } moduleName = ModuleName.Invalid; return(false); }
private static List <ParsedModule> SpecialModules(ParsedModule preludeModule, ParsedModule configurationModule) { var modules = new List <ParsedModule>(); if (preludeModule != null) { modules.Add(preludeModule); } if (configurationModule != null) { modules.Add(configurationModule); } return(modules); }
/// <summary> /// Creates a parsing queue for parsing specs in a regular BuildXL invocation. /// </summary> public static ModuleParsingQueue Create( [NotNull] WorkspaceProvider workspaceProvider, [NotNull] WorkspaceConfiguration workspaceConfiguration, [NotNull] IModuleReferenceResolver moduleReferenceResolver, [CanBeNull] ModuleDefinition designatedPrelude, [CanBeNull] ParsedModule configurationModule) { Contract.Requires(workspaceProvider != null); return(new ModuleParsingQueue( workspaceProvider, workspaceConfiguration, moduleReferenceResolver, designatedPrelude, configurationModule)); }
private IEnumerable <ParsedModule> GetAllSourceModules(ParsedModule prelude, ParsedModule configurationModule) { var result = new HashSet <ParsedModule>(m_modulesAlreadyParsed.Values); if (prelude != null) { result.Remove(prelude); } if (configurationModule != null) { result.Remove(configurationModule); } return(result); }
/// <inheritdoc /> public ParsedModule GetConfigurationModule() { // In some scenarios (like tests) main config workspace is unavailable. if (m_mainConfigurationWorkspace == null) { return(null); } if (m_configurationModule == null) { m_configurationModule = CreateConfigurationModule(); } return(m_configurationModule); ParsedModule CreateConfigurationModule() { // Module with all configuration files includes all the files obtained during main config processing // as well as other module configuration files that each resolver used to compute their modules. var moduleFiles = GetModuleConfigurationFiles(); var moduleFileNames = new HashSet <AbsolutePath>(moduleFiles.Keys); var mainConfigurationModule = m_mainConfigurationWorkspace.ConfigurationModule; Contract.Assert(mainConfigurationModule != null); if (moduleFileNames.Count == 0) { // If there is no module configuration files, we can use main configuration as a result. return(m_mainConfigurationWorkspace.ConfigurationModule); } // For the full configuration module using the main configuration file. // Main config should be part of the module definition as well. moduleFileNames.AddRange(mainConfigurationModule.PathToSpecs); var module = CreateConfigModuleDefinition(mainConfigurationModule.Definition.MainFile, moduleFileNames, m_configurationResolver); // Need to add files from the main configuration to the final set of files. moduleFiles.AddRange(mainConfigurationModule.Specs); var parsedModule = new ParsedModule(module, moduleFiles); return(parsedModule); } }
private static HashSet <ParsedModule> GetAllParsedModules( IEnumerable <ParsedModule> specModules, ParsedModule preludeModule, ParsedModule configurationModule) { var result = new HashSet <ParsedModule>(specModules); if (preludeModule != null) { result.Add(preludeModule); } if (configurationModule != null) { result.Add(configurationModule); } return(result); }
private async Task <Workspace> EnqueuingFinishedAndWaitForCompletion() { // No more new module definition may be added for processing EnqueuingCompleted(); try { await m_parseQueue.Completion; } catch (TaskCanceledException) { // Expected. This means that at least one task failed parsing } // Parsing is complete. Just need to wait for pending items to finish binding. m_bindQueue.Complete(); try { await m_bindQueue.Completion; } catch (TaskCanceledException) { // Expected. This means that at least one task failed binding } // Need to check if the queue reparsed prelude or configuration module. ParsedModule prelude = GetProcessedPreludeOrDefault(); ParsedModule configurationModule = GetProcessedConfigurationModuleOrDefault(); // The workspace is ready to be constructed. So at this point we can run some workspace-level validations. var workspaceFailures = WorkspaceValidator.ValidateParsedModules(m_modulesAlreadyParsed.Values, m_workspaceProvider.PathTable); // Create the result. Observe that if the task is cancelled, we don't propagate it // but reflect it in the result. return(new Workspace( m_workspaceProvider, m_workspaceConfiguration, GetAllSourceModules(prelude, configurationModule), m_failures.Union(workspaceFailures), preludeModule: prelude, configurationModule: configurationModule)); }
private static Dictionary <AbsolutePath, SpecFileWithMetadata> CreateSpecsForPreludeAndConfiguration(ParsedModule preludeModule, ParsedModule configurationModule) { // The prelude module used for configuration parsing and the prelude module used for regular spec parsing may be the same // So we allow double ownership in this case. The first module being processed will win spec ownership. It shouldn't matter which one. return(CreateSpecsFromModules(SpecialModules(preludeModule, configurationModule), allowDoubleOwnership: true)); }
/// <nodoc/> public bool TryGetModuleByModuleDescriptor(ModuleDescriptor moduleDescriptor, out ParsedModule parsedModule) { return(m_allModulesByDescriptor.TryGetValue(moduleDescriptor, out parsedModule)); }
/// <summary> /// Creates a workspace for configuration processing. /// </summary> public static Workspace CreateConfigurationWorkspace(WorkspaceConfiguration configuration, ParsedModule configurationModule, ParsedModule preludeModule) { return(new Workspace( provider: null, workspaceConfiguration: configuration, modules: CollectionUtilities.EmptyArray <ParsedModule>(), failures: CollectionUtilities.EmptyArray <Failure>(), preludeModule: preludeModule, configurationModule: configurationModule)); }
/// <nodoc /> public static SpecFileWithMetadata CreateNew(ParsedModule owningModule, ISourceFile sourceFile) { return(new SpecFileWithMetadata(sourceFile, owningModule, SpecState.Changed)); }