/// <summary>
/// Builds and filters the worksapce.
/// </summary>
/// <remarks>
/// This method not just builds the workspace from scratch, but it also tries to compute it in an efficient way.
/// If there is a front end snapshot from the previous BuildXL run and the engine gives us a set of changed files,
/// then we can build a filtered workspace based on the old spec-2-spec map without parsing the entire world.
/// </remarks>
private async Task <Workspace> BuildAndFilterWorkspaceAsync(IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
// this step downloads nugets too, and that's why we want to do it outside of the progress reporting block below
Possible <WorkspaceDefinition> workspaceDefinition = await TryGetWorkspaceDefinitionAsync(workspaceProvider);
if (!workspaceDefinition.Succeeded)
{
// In some cases even if the workspace failed to build some of the pipeline still tries to continue
// Complete mount initialization to enable mount related queries downstream
Engine.CompleteMountInitialization();
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, workspaceDefinition.Failure));
}
// As soon as we get the workspace definition, we can configure the mount table with the additional mounts modules may have defined and seal it
foreach (var module in workspaceDefinition.Result.Modules.Where(module => module.Mounts != null))
{
foreach (var mount in module.Mounts)
{
Engine.AddResolvedModuleDefinedMount(mount, LocationData.Create(module.ModuleConfigFile));
}
}
// At this point the mount table can be completed
if (!Engine.CompleteMountInitialization())
{
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, new GenericWorkspaceFailure("Mount points not properly defined. Detailed errors should have been logged.")));
}
return(await WithWorkspaceProgressReportingAsync(
numSpecs : workspaceDefinition.Result.SpecCount,
task : BuildAndFilterWorkspaceAsync(workspaceDefinition.Result, workspaceProvider, engineAbstraction, evaluationFilter)));
}
/// <nodoc/>
public Workspace(
[CanBeNull] IWorkspaceProvider provider,
WorkspaceConfiguration workspaceConfiguration,
IEnumerable <ParsedModule> modules,
IEnumerable <Failure> failures,
[CanBeNull] ParsedModule preludeModule,
[CanBeNull] ParsedModule configurationModule)
{
Contract.Requires(workspaceConfiguration != null);
Contract.Requires(modules != null);
Contract.Requires(failures != null);
Contract.RequiresForAll(modules, m => m != null);
WorkspaceProvider = provider;
WorkspaceConfiguration = workspaceConfiguration;
var allModules = GetAllParsedModules(modules, preludeModule, configurationModule);
m_specModules = allModules.Where(m => m != preludeModule && m != configurationModule).ToArray();
// Double ownership is not allowed: specs are already validated for double ownership
m_specSources = CreateSpecsFromModules(m_specModules, allowDoubleOwnership: false);
m_specialSpecs = CreateSpecsForPreludeAndConfiguration(preludeModule, configurationModule);
// Spec array contains all the specs for the workspace.
m_specArray = m_specSources.ToDictionary().AddRange(m_specialSpecs).Select(s => s.Value.SourceFile).ToArray();
m_allModulesByDescriptor = AllModulesByDescriptor(allModules);
Failures = failures.ToArray();
PreludeModule = preludeModule;
ConfigurationModule = configurationModule;
}
/// <summary>
/// Builds and filters the worksapce.
/// </summary>
/// <remarks>
/// This method not just builds the workspace from scratch, but it also tries to compute it in an efficient way.
/// If there is a front end snapshot from the previous BuildXL run and the engine gives us a set of changed files,
/// then we can build a filtered workspace based on the old spec-2-spec map without parsing the entire world.
/// </remarks>
private async Task <Workspace> BuildAndFilterWorkspaceAsync(IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
// this step downloads nugets too, and that's why we want to do it outside of the progress reporting block below
Possible <WorkspaceDefinition> workspaceDefinition = await TryGetWorkspaceDefinitionAsync(workspaceProvider);
if (!workspaceDefinition.Succeeded)
{
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, workspaceDefinition.Failure));
}
return(await WithWorkspaceProgressReportingAsync(
numSpecs : workspaceDefinition.Result.SpecCount,
task : BuildAndFilterWorkspaceAsync(workspaceDefinition.Result, workspaceProvider, engineAbstraction, evaluationFilter)));
}
public SemanticWorkspace(
IWorkspaceProvider workspaceProvider,
WorkspaceConfiguration workspaceConfiguration,
IEnumerable <ParsedModule> modules,
[CanBeNull] ParsedModule preludeModule,
[CanBeNull] ParsedModule configurationModule,
ISemanticModel semanticModel,
IReadOnlyCollection <Failure> failures)
: base(workspaceProvider, workspaceConfiguration, modules, failures, preludeModule, configurationModule)
{
Contract.Requires(semanticModel != null);
m_semanticModel = semanticModel;
}
public WorkspaceBrowserViewModel(IWorkspaceProvider workspaceProvider, ILog logger)
{
try{
_workspaceProvider = workspaceProvider;
Logger = logger;
Logger.Debug("Calling WorkspaceService to get all template Categories");
_categories = _workspaceProvider.GetCategories();
Logger.Debug(String.Format("Got {0} Categories from WorkspaceService", _categories.Count));
}catch (Exception ex) {
Logger.Error("Call to WorkspaceService failed while fetching Categories due to following exception =>", ex);
throw ex;
}
}
/// <nodoc/>
public static bool TryCreate(
[CanBeNull] Workspace mainConfigurationWorkspace,
IWorkspaceStatistics workspaceStatistics,
FrontEndFactory frontEndFactory,
PathTable pathTable,
SymbolTable symbolTable,
WorkspaceConfiguration configuration,
bool useDecorator,
bool addBuiltInPreludeResolver,
out IWorkspaceProvider workspaceProvider,
out IEnumerable <Failure> failures)
{
// mainConfigurationWorkspace can be null for some tests
var mainFile = mainConfigurationWorkspace != null ?
mainConfigurationWorkspace.ConfigurationModule.Definition.MainFile :
AbsolutePath.Invalid;
if (!TryCreateResolvers(
frontEndFactory,
configuration,
pathTable,
mainFile,
addBuiltInPreludeResolver,
out var resolvers,
out failures))
{
workspaceProvider = default(IWorkspaceProvider);
return(false);
}
var provider = new WorkspaceProvider(workspaceStatistics, resolvers, configuration, pathTable, symbolTable);
provider.m_mainConfigurationWorkspace = mainConfigurationWorkspace;
workspaceProvider = useDecorator
? (IWorkspaceProvider) new WorkspaceProviderStatisticsDecorator(workspaceStatistics, provider)
: provider;
return(true);
}
/// <summary>
/// Tries to filter a given workspace definition by reusing information from the previous BuildXL invocation.
/// </summary>
/// <returns>
/// 1. Failure if the error occurred during parsing/binding one of the changed specs.
/// 2. Result(null) when the filtering failed due to symbols mismatch or due to another reason.
/// 3. Result(WorkspaceDefinition) when the filtering succeeded.
/// </returns>
/// <remarks>
/// If the previous binding information can be reused, then the set of specs that are safe to use as public facades + serialized AST
/// are identified as well
/// </remarks>
private async Task <FilteredWorkspaceDefinition> TryFilterWorkspaceDefinitionIncrementallyAsync(
List <string> changedFiles,
IWorkspaceProvider workspaceProvider,
WorkspaceDefinition workspaceDefinition,
EvaluationFilter evaluationFilter)
{
Logger.TryingToReuseFrontEndSnapshot(LoggingContext);
// TODO: potentially, we could check the number of changes compared to the workspace definition size.
// If the number of changes is too big, maybe we should go into the full parse mode.
// But we need to check the perf implications before making this decision.
var changedSpecs = changedFiles.Select(
p =>
{
var fullPath = AbsolutePath.Create(FrontEndContext.PathTable, p);
var containingModule = workspaceDefinition.TryGetModuleDefinition(fullPath);
return(new SpecWithOwningModule(fullPath, containingModule));
}).ToArray();
// Need to check if the spec does not belong to the current workspace
// or the changed spec belongs to the prelude.
foreach (var changedSpec in changedSpecs)
{
if (changedSpec.OwningModule == null)
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
I($"Changed spec file '{changedSpec.Path.ToString(FrontEndContext.PathTable)}' is not part of the computed workspace."));
return(FilteredWorkspaceDefinition.CanNotFilter());
}
if (changedSpec.OwningModule.Descriptor == workspaceDefinition.PreludeModule.Descriptor)
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
I($"Changed spec file '{changedSpec.Path.ToString(FrontEndContext.PathTable)}' is part of the prelude."));
return(FilteredWorkspaceDefinition.CanNotFilter());
}
}
// Getting the snapshot from the previous run.
// Binding snapshot contains all the specs as well as all the configuration files.
// Need to adjust the count.
var expectedNumberOfSpecs = workspaceDefinition.SpecCount + (workspaceProvider.GetConfigurationModule()?.Specs.Count ?? 0);
var snapshot = FrontEndArtifactManager.TryLoadFrontEndSnapshot(expectedNumberOfSpecs);
if (snapshot == null)
{
// The error message was already logged.
return(FilteredWorkspaceDefinition.CanNotFilter());
}
// Parsing and binding all the changed specs.
var possibleParseResult = await workspaceProvider.ParseAndBindSpecsAsync(changedSpecs);
var firstFailure = LogParseOrBindingErrorsIfAny(possibleParseResult);
if (firstFailure != null)
{
// This is actual failure.
// Instead of switching to the full mode, we can actually stop here.
return(FilteredWorkspaceDefinition.Error(firstFailure));
}
// Snapshot is valid and parse/binding is completed successfully.
var snapshotState = GetSnapshotReuseState(possibleParseResult, snapshot);
if (snapshotState.State == SnapshotState.NoMatch)
{
// NoMatch is returned if the snapshot is unavailable.
if (snapshotState.SpecsWithIncompatiblePublicSurface.Count != 0)
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
I($"Spec file '{snapshotState.SpecsWithIncompatiblePublicSurface.First().Path.AbsolutePath}' changed its binding symbols."));
}
return(FilteredWorkspaceDefinition.CanNotFilter());
}
// Changed file could get different symbols.
// Need to re-save it within the front-end snapshot.
UpdateAndSaveSnapshot(possibleParseResult, snapshot);
var snapshotProvider = new SnapshotBasedSpecProvider(snapshot);
// Now we know exactly which are all the files that need to go through parsing/type checking/AST conversion. So we
// inform that to the artifact manager so the public surface and AST serialization
// can be resued for the rest, if available.
// Observe these set of files are not reflecting a potential user filter, but that's fine. If there is a dirty spec
// that is outside of the filter, that spec won't be requested by the workspace anyway
NotifyDirtySpecsForPublicFacadeAndAstReuse(
snapshotProvider,
workspaceDefinition,
changedSpecs.Select(f => f.Path).ToList());
// The fingerprints for all changed specs are still the same,
// so we can filter the workspace definition provided that the filter allows it.
if (snapshotState.State == SnapshotState.FullMatch)
{
var filter = new WorkspaceFilter(FrontEndContext.PathTable);
var filteredWorkspace = evaluationFilter.CanPerformPartialEvaluationScript(PrimaryConfigFile)
? filter.FilterWorkspaceDefinition(workspaceDefinition, evaluationFilter, snapshotProvider)
: workspaceDefinition.Modules;
return(FilteredWorkspaceDefinition.Filter(new WorkspaceDefinition(filteredWorkspace, workspaceDefinition.PreludeModule)));
}
// Specs are not the same, but we would be able to load public facades for all unaffected specs.
var dirtySpecNames = string.Join(
", ",
snapshotState.SpecsWithTheSamePublicSurface.Take(10).Select(p => Path.GetFileName(p.Path.AbsolutePath)));
Logger.FailedToFilterWorkspaceDefinition(
LoggingContext,
I($"{dirtySpecNames} changed one or more declarations."));
return(FilteredWorkspaceDefinition.CanNotFilter());
}
private async Task <Possible <WorkspaceDefinition> > TryGetWorkspaceDefinitionAsync(IWorkspaceProvider workspaceProvider)
{
Possible <WorkspaceDefinition> workspaceDefinition;
using (var sw = Watch.Start())
{
workspaceDefinition = await workspaceProvider.GetWorkspaceDefinitionForAllResolversAsync();
if (workspaceDefinition.Succeeded)
{
var configurationFiles = workspaceProvider.GetConfigurationModule()?.Specs.Count ?? 0;
Logger.WorkspaceDefinitionCreated(
LoggingContext,
workspaceDefinition.Result.ModuleCount,
workspaceDefinition.Result.SpecCount,
configurationFiles,
sw.ElapsedMilliseconds);
}
}
return(workspaceDefinition);
}
/// <summary>
/// Tries to create a filtered workspace based on a front-end snapshot from the previous BuildXL invocation.
/// </summary>
/// <returns>
/// * Possibke<ValidConstructedWorkspace> when the workspace was successfully constructed.
/// * Possible<null> when the snapshot was unaiable.
/// * Failure when the snapshot was available but parsing failed.
/// </returns>
private async Task <Possible <Workspace> > TryCreateFilteredWorkspaceAsync(Possible <WorkspaceDefinition> workspaceDefinition, IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
if (!FrontEndConfiguration.ConstructAndSaveBindingFingerprint())
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
"Binding fingerprint is disabled. Please use 'constructAndSaveBindingFingerprint' option to turn it on");
return(default(Possible <Workspace>));
}
// If a filter cannot be performed and public facade + AST is not to be used, then there is no point in continuing and we can
// go to full mode
if (!evaluationFilter.CanPerformPartialEvaluationScript(PrimaryConfigFile) && !CanUseSpecPublicFacadeAndAst())
{
var message = !CanUseSpecPublicFacadeAndAst()
? "Engine state was not reloaded"
: "User filter was not specified";
Logger.FailToReuseFrontEndSnapshot(LoggingContext, message);
return(default(Possible <Workspace>));
}
var changedFiles = engineAbstraction.GetChangedFiles()?.ToList();
if (changedFiles == null)
{
Logger.FailToReuseFrontEndSnapshot(LoggingContext, "Change journal is not available");
return(default(Possible <Workspace>));
}
using (var sw = Watch.Start())
{
// We're potentially in incremental mode.
var filteredDefinitionResult = await TryFilterWorkspaceDefinitionIncrementallyAsync(
changedFiles,
workspaceProvider,
workspaceDefinition.Result,
evaluationFilter);
if (filteredDefinitionResult.Failed)
{
return(filteredDefinitionResult.Failure);
}
if (filteredDefinitionResult.Filtered)
{
var filteredDefinition = filteredDefinitionResult.FilteredDefinition;
Logger.WorkspaceDefinitionFiltered(
LoggingContext,
filteredDefinition.SpecCount,
workspaceDefinition.Result.SpecCount,
sw.ElapsedMilliseconds);
// TODO: with C# 7, use tuple instead of changing the workspace to carry the information about the filtering.
return(await workspaceProvider.CreateWorkspaceAsync(filteredDefinition, userFilterWasApplied : true));
}
}
return(default(Possible <Workspace>));
}
private async Task <Workspace> BuildAndFilterWorkspaceAsync(WorkspaceDefinition workspaceDefinition, IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
// First, trying to filter workspace based on information from the previous run
var possibleFilteredWorkspace = await TryCreateFilteredWorkspaceAsync(workspaceDefinition, workspaceProvider, engineAbstraction, evaluationFilter);
if (!possibleFilteredWorkspace.Succeeded)
{
// Error was already logged
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, possibleFilteredWorkspace.Failure));
}
// If the filtered workspace is not null, just return it.
// Otherwise falling back to the full parse mode.
if (possibleFilteredWorkspace.Result != null)
{
return(possibleFilteredWorkspace.Result);
}
// "Incremental" workspace construction has failed, but we still can try to use module filter to build a smaller workspace.
if (evaluationFilter.ModulesToResolve.Count != 0)
{
var filteredDefinition = this.ApplyModuleFilter(workspaceDefinition, evaluationFilter.ModulesToResolve);
return(await workspaceProvider.CreateWorkspaceAsync(filteredDefinition, userFilterWasApplied : true));
}
Logger.BuildingFullWorkspace(LoggingContext);
return(await workspaceProvider.CreateWorkspaceAsync(workspaceDefinition, userFilterWasApplied : false));
}
/// <nodoc />
public WorkspaceProviderStatisticsDecorator(IWorkspaceStatistics statistics, IWorkspaceProvider decoratee)
{
m_statistics = statistics;
m_decoratee = decoratee;
}
/// <summary>
/// Adds the provider that will provide information to the workspace when the information is requested.
/// </summary>
/// <param name="workspaceProvider">The workspace provider.</param>
public void AddProvider(IWorkspaceProvider workspaceProvider)
{
Argument.IsNotNull(() => workspaceProvider);
_workspaceProviders.Add(workspaceProvider);
}
public Worker(IServiceScopeFactory scopeFactory, IWorkspaceProvider workspaceProvider, ILogger <Worker> logger)
{
_scopeFactory = scopeFactory;
_workspaceProvider = workspaceProvider;
_logger = logger;
}
public WorkspaceProviderEventArgs(IWorkspaceProvider workspaceProvider)
{
WorkspaceProvider = workspaceProvider;
}
public CsharpCollectionController(ICSharpCollectionView view, IWorkspaceProvider workSpaceProvider)
{
this.view = view;
this.workSpaceProvider = workSpaceProvider;
HookupEvents();
}
/// <summary>
/// Constructs the workspace with given errors.
/// </summary>
public static Workspace Failure(IWorkspaceProvider provider, WorkspaceConfiguration workspaceConfiguration, params Failure[] failures)
{
Contract.Requires(failures.Length != 0);
return(new Workspace(provider, workspaceConfiguration, new List <ParsedModule>(), failures, preludeModule: null, configurationModule: null));
}
/// <summary>
/// Removes the provider that will provide information to the workspace when the information is requested.
/// </summary>
/// <param name="workspaceProvider">The workspace provider.</param>
/// <returns><c>true</c> if the workspace provider is deleted; otherwise <c>false</c>.</returns>
public bool RemoveProvider(IWorkspaceProvider workspaceProvider)
{
Argument.IsNotNull(() => workspaceProvider);
return _workspaceProviders.Remove(workspaceProvider);
}
