private async Task <Workspace> BuildAndFilterWorkspaceAsync(WorkspaceDefinition workspaceDefinition, IWorkspaceProvider workspaceProvider, FrontEndEngineAbstraction engineAbstraction, EvaluationFilter evaluationFilter)
{
// First, trying to filter workspace based on information from the previous run
var possibleFilteredWorkspace = await TryCreateFilteredWorkspaceAsync(workspaceDefinition, workspaceProvider, engineAbstraction, evaluationFilter);
if (!possibleFilteredWorkspace.Succeeded)
{
// Error was already logged
return(Workspace.Failure(workspaceProvider, workspaceProvider.Configuration, possibleFilteredWorkspace.Failure));
}
// If the filtered workspace is not null, just return it.
// Otherwise falling back to the full parse mode.
if (possibleFilteredWorkspace.Result != null)
{
return(possibleFilteredWorkspace.Result);
}
// "Incremental" workspace construction has failed, but we still can try to use module filter to build a smaller workspace.
if (evaluationFilter.ModulesToResolve.Count != 0)
{
var filteredDefinition = this.ApplyModuleFilter(workspaceDefinition, evaluationFilter.ModulesToResolve);
return(await workspaceProvider.CreateWorkspaceAsync(filteredDefinition, userFilterWasApplied : true));
}
Logger.BuildingFullWorkspace(LoggingContext);
return(await workspaceProvider.CreateWorkspaceAsync(workspaceDefinition, userFilterWasApplied : false));
}
/// <summary>
/// Filter a workspace definition.
/// </summary>
public List <ModuleDefinition> FilterWorkspaceDefinition(
[NotNull] WorkspaceDefinition workspace,
[NotNull] EvaluationFilter evaluationFilter,
[NotNull] ISpecDependencyProvider provider)
{
// Resulting list should always have a prelude.
var modulesToInclude = new HashSet <ModuleDefinition> {
workspace.PreludeModule
};
// Getting all the files relevant to the build.
var filesToInclude = GetFilesToInclude(modulesToInclude, workspace, provider, evaluationFilter);
// Keep modules with a set of files relevant for a given filter.
var partiallyFilteredModules = new Dictionary <ModuleDefinition, HashSet <AbsolutePath> >();
foreach (var kvp in workspace.Specs)
{
// File is not part of 'must have' module and is part of 'must have' spec.
if (filesToInclude.Contains(kvp.Path))
{
var map = partiallyFilteredModules.GetOrAdd(kvp.OwningModule, k => new HashSet <AbsolutePath>());
map.Add(kvp.Path);
}
}
foreach (var kvp in partiallyFilteredModules)
{
var moduleDefinition = kvp.Key.WithSpecs(kvp.Value.ToReadOnlySet());
modulesToInclude.Add(moduleDefinition);
}
return(modulesToInclude.ToList());
}
private void UpdateWorkspace(WorkspaceDefinition definition, bool resetSettings, SortDirection sortDirection, OrderBy orderBy, Serializer serializer)
{
Settings settings;
var workspaceFilePath = Path.Combine(_options.WorkspacePath, definition.FileName);
if (resetSettings)
{
settings = new Settings();
}
else
{
var currentWorkspace = serializer.DeserializeFromJson <Workspace>(workspaceFilePath);
settings = currentWorkspace.Settings;
}
IEnumerable <Snippet> filtered = _snippets.ToList();
if (definition.Languages.Any())
{
filtered = filtered.Where(x => x.Files.Any(f => definition.Languages.Contains(_fileAssociations.Lookup(f))));
}
if (definition.Tags.Any())
{
filtered = filtered.Where(x => x.Meta.Tags.Any(t => definition.Tags.Contains(t)));
}
if (!filtered.Any())
{
File.Delete(workspaceFilePath);
return;
}
var ascending = sortDirection == SortDirection.Ascending;
var ordered = orderBy switch
{
OrderBy.Alphabetical => ascending?filtered.OrderBy(x => x.Meta.Title) : filtered.OrderByDescending(x => x.Meta.Title),
OrderBy.Created => ascending?filtered.OrderBy(x => x.Meta.CreatedUtc) : filtered.OrderByDescending(x => x.Meta.CreatedUtc),
_ => throw new NotSupportedException($"Unable to sort by {nameof(orderBy)}")
};
var workspace = new Workspace {
Settings = settings
};
foreach (var snippet in ordered)
{
var folder = new Folder(snippet.Meta.Title, Path.GetRelativePath(_options.WorkspacePath, snippet.DirectoryPath));
workspace.Add(folder);
}
serializer.SerializeToJson(workspace, workspaceFilePath);
}
/// <summary>
/// Filters a <paramref name="workspace"/> using <paramref name="modules"/>.
/// </summary>
/// <remarks>
/// Currently only module filter is supported.
/// </remarks>
private WorkspaceDefinition ApplyModuleFilter(WorkspaceDefinition workspace, IReadOnlyList <StringId> modules)
{
using (var sw = Watch.Start())
{
// WorkspaceFilter updates the existing workspace instead of creating brand new one.
// This is crucial to avoid redundant type checking required for semantic workspace creation.
var filter = new WorkspaceFilter(FrontEndContext.PathTable);
WorkspaceDefinition filteredDefinition = filter.ApplyModuleFilter(workspace, modules);
Logger.WorkspaceDefinitionFilteredBasedOnModuleFilter(LoggingContext,
workspace.ModuleCount - filteredDefinition.ModuleCount,
workspace.SpecCount - filteredDefinition.SpecCount,
workspace.ModuleCount,
filteredDefinition.ModuleCount,
sw.ElapsedMilliseconds);
return(filteredDefinition);
}
}
/// <summary>
/// Applies module filter for a given <paramref name="workspace"/>.
/// </summary>
public WorkspaceDefinition ApplyModuleFilter(WorkspaceDefinition workspace, IReadOnlyList <StringId> modulesToKeep)
{
HashSet <string> requiredModules = new HashSet <string>(modulesToKeep.Select(m => m.ToString(m_pathTable.StringTable)));
List <ModuleDefinition> finalModules = new List <ModuleDefinition>();
var preludeModule = workspace.PreludeModule;
foreach (var module in workspace.Modules)
{
// Should keep special modules and the prelude.
if (module.Descriptor.IsSpecialConfigModule() ||
module.Equals(preludeModule) ||
requiredModules.Contains(module.Descriptor.Name))
{
finalModules.Add(module);
}
}
return(new WorkspaceDefinition(finalModules, workspace.PreludeModule));
}
public Manifest ToManifest(OrderBy order, SortDirection direction)
{
var manifest = new Manifest
{
SortDirection = direction,
OrderBy = order
};
foreach (var package in _packages)
{
var definition = new WorkspaceDefinition
{
FileName = package.FileName,
Languages = package.Languages.ToList(),
Tags = package.Tags.ToList()
};
manifest.Definitions.Add(definition);
}
return(manifest);
}
private HashSet <AbsolutePath> GetFilesToInclude(HashSet <ModuleDefinition> modulesToInclude, WorkspaceDefinition workspace, ISpecDependencyProvider provider, EvaluationFilter evaluationFilter)
{
var filesToInclude = new HashSet <AbsolutePath>();
foreach (var kvp in workspace.Specs)
{
var specPath = kvp.Path;
// If the spec belonged to a module that is required, then the entire module will be evaluated.
// No need to keep the spec separately
if (!modulesToInclude.Contains(kvp.OwningModule))
{
foreach (var specRootToResolve in evaluationFilter.ValueDefinitionRootsToResolve)
{
if (specPath == specRootToResolve || specPath.IsWithin(m_pathTable, specRootToResolve))
{
filesToInclude.Add(specPath);
AddUpStreamDependencies(filesToInclude, specPath, provider);
}
}
}
}
return(filesToInclude);
}
/// <summary>
/// Computes the transitive closure of all dependents of the changed specs and report them as 'dirty' to the artifact manager,
/// so they are not replaced by their public facade version and their serialized AST is not used
/// </summary>
private void NotifyDirtySpecsForPublicFacadeAndAstReuse(ISpecDependencyProvider snapshotProvider, WorkspaceDefinition workspaceDefinition, IReadOnlyList <AbsolutePath> changedSpecs)
{
if (CanUseSpecPublicFacadeAndAst())
{
var changedSpecsWithDependents = snapshotProvider.ComputeReflectiveClosureOfDependentFiles(changedSpecs);
FrontEndArtifactManager.NotifySpecsCannotBeUsedAsFacades(changedSpecsWithDependents);
var requiredSpecs = snapshotProvider.ComputeReflectiveClosureOfDependencyFiles(changedSpecsWithDependents);
Logger.ReportDestructionCone(LoggingContext, changedSpecs.Count, changedSpecsWithDependents.Count, requiredSpecs.Count, workspaceDefinition.SpecCount);
}
}
/// <summary>
/// Tries to filter a given workspace definition by reusing information from the previous BuildXL invocation.
/// </summary>
/// <returns>
/// 1. Failure if the error occurred during parsing/binding one of the changed specs.
/// 2. Result(null) when the filtering failed due to symbols mismatch or due to another reason.
/// 3. Result(WorkspaceDefinition) when the filtering succeeded.
/// </returns>
/// <remarks>
/// If the previous binding information can be reused, then the set of specs that are safe to use as public facades + serialized AST
/// are identified as well
/// </remarks>
private async Task <FilteredWorkspaceDefinition> TryFilterWorkspaceDefinitionIncrementallyAsync(
List <string> changedFiles,
IWorkspaceProvider workspaceProvider,
WorkspaceDefinition workspaceDefinition,
EvaluationFilter evaluationFilter)
{
Logger.TryingToReuseFrontEndSnapshot(LoggingContext);
// TODO: potentially, we could check the number of changes compared to the workspace definition size.
// If the number of changes is too big, maybe we should go into the full parse mode.
// But we need to check the perf implications before making this decision.
var changedSpecs = changedFiles.Select(
p =>
{
var fullPath = AbsolutePath.Create(FrontEndContext.PathTable, p);
var containingModule = workspaceDefinition.TryGetModuleDefinition(fullPath);
return(new SpecWithOwningModule(fullPath, containingModule));
}).ToArray();
// Need to check if the spec does not belong to the current workspace
// or the changed spec belongs to the prelude.
foreach (var changedSpec in changedSpecs)
{
if (changedSpec.OwningModule == null)
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
I($"Changed spec file '{changedSpec.Path.ToString(FrontEndContext.PathTable)}' is not part of the computed workspace."));
return(FilteredWorkspaceDefinition.CanNotFilter());
}
if (changedSpec.OwningModule.Descriptor == workspaceDefinition.PreludeModule.Descriptor)
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
I($"Changed spec file '{changedSpec.Path.ToString(FrontEndContext.PathTable)}' is part of the prelude."));
return(FilteredWorkspaceDefinition.CanNotFilter());
}
}
// Getting the snapshot from the previous run.
// Binding snapshot contains all the specs as well as all the configuration files.
// Need to adjust the count.
var expectedNumberOfSpecs = workspaceDefinition.SpecCount + (workspaceProvider.GetConfigurationModule()?.Specs.Count ?? 0);
var snapshot = FrontEndArtifactManager.TryLoadFrontEndSnapshot(expectedNumberOfSpecs);
if (snapshot == null)
{
// The error message was already logged.
return(FilteredWorkspaceDefinition.CanNotFilter());
}
// Parsing and binding all the changed specs.
var possibleParseResult = await workspaceProvider.ParseAndBindSpecsAsync(changedSpecs);
var firstFailure = LogParseOrBindingErrorsIfAny(possibleParseResult);
if (firstFailure != null)
{
// This is actual failure.
// Instead of switching to the full mode, we can actually stop here.
return(FilteredWorkspaceDefinition.Error(firstFailure));
}
// Snapshot is valid and parse/binding is completed successfully.
var snapshotState = GetSnapshotReuseState(possibleParseResult, snapshot);
if (snapshotState.State == SnapshotState.NoMatch)
{
// NoMatch is returned if the snapshot is unavailable.
if (snapshotState.SpecsWithIncompatiblePublicSurface.Count != 0)
{
Logger.FailToReuseFrontEndSnapshot(
LoggingContext,
I($"Spec file '{snapshotState.SpecsWithIncompatiblePublicSurface.First().Path.AbsolutePath}' changed its binding symbols."));
}
return(FilteredWorkspaceDefinition.CanNotFilter());
}
// Changed file could get different symbols.
// Need to re-save it within the front-end snapshot.
UpdateAndSaveSnapshot(possibleParseResult, snapshot);
var snapshotProvider = new SnapshotBasedSpecProvider(snapshot);
// Now we know exactly which are all the files that need to go through parsing/type checking/AST conversion. So we
// inform that to the artifact manager so the public surface and AST serialization
// can be resued for the rest, if available.
// Observe these set of files are not reflecting a potential user filter, but that's fine. If there is a dirty spec
// that is outside of the filter, that spec won't be requested by the workspace anyway
NotifyDirtySpecsForPublicFacadeAndAstReuse(
snapshotProvider,
workspaceDefinition,
changedSpecs.Select(f => f.Path).ToList());
// The fingerprints for all changed specs are still the same,
// so we can filter the workspace definition provided that the filter allows it.
if (snapshotState.State == SnapshotState.FullMatch)
{
var filter = new WorkspaceFilter(FrontEndContext.PathTable);
var filteredWorkspace = evaluationFilter.CanPerformPartialEvaluationScript(PrimaryConfigFile)
? filter.FilterWorkspaceDefinition(workspaceDefinition, evaluationFilter, snapshotProvider)
: workspaceDefinition.Modules;
return(FilteredWorkspaceDefinition.Filter(new WorkspaceDefinition(filteredWorkspace, workspaceDefinition.PreludeModule)));
}
// Specs are not the same, but we would be able to load public facades for all unaffected specs.
var dirtySpecNames = string.Join(
", ",
snapshotState.SpecsWithTheSamePublicSurface.Take(10).Select(p => Path.GetFileName(p.Path.AbsolutePath)));
Logger.FailedToFilterWorkspaceDefinition(
LoggingContext,
I($"{dirtySpecNames} changed one or more declarations."));
return(FilteredWorkspaceDefinition.CanNotFilter());
}
/// <nodoc />
public static FilteredWorkspaceDefinition Filter(WorkspaceDefinition workspaceDefinition)
{
Contract.Requires(workspaceDefinition != null);
return(new FilteredWorkspaceDefinition(null, workspaceDefinition));
}
/// <summary>
/// Constructor for a failure case.
/// </summary>
private FilteredWorkspaceDefinition(Failure failure, WorkspaceDefinition workspaceDefinition)
: this()
{
Failure = failure;
FilteredDefinition = workspaceDefinition;
}
