public IFileDatabaseSnapshot BuildWithChangedFiles(IFileDatabaseSnapshot previousFileDatabaseSnapshot,
FileSystemSnapshot fileSystemSnapshot, IEnumerable <ProjectFileName> changedFiles,
Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and list of changed files", cancellationToken, InfoLogger.Instance)) {
Invariants.Assert(previousFileDatabaseSnapshot is FileDatabaseSnapshot);
var previousFileDatabase = (FileDatabaseSnapshot)previousFileDatabaseSnapshot;
// Update file contents of file data entries of changed files.
var filesToRead = changedFiles
.Where(x => x.Project.IsFileSearchable(x.FileName) && previousFileDatabase.Files.ContainsKey(x.FileName))
.ToList();
if (filesToRead.Count == 0)
{
Logger.LogInfo("None of the changed file is searchable, return previous database snapshot");
return(previousFileDatabaseSnapshot);
}
// Read file contents.
onLoading();
filesToRead.ForAll(x => {
var newContents = _fileContentsFactory.ReadFileContents(x.FileName.FullPath);
DangerousUpdateFileTableEntry(previousFileDatabase, x.FileName, newContents);
});
onLoaded();
// Return new file database with updated file contents.
return(new FileDatabaseSnapshot(
previousFileDatabase.ProjectHashes,
previousFileDatabase.Directories,
previousFileDatabase.Files));
}
}
public static FileSystemTree ToIpcCompactFileSystemTree(this FileSystemSnapshot tree)
{
return(new FileSystemTree {
Version = tree.Version,
Projects = BuildCompactProjectEntries(tree)
});
}
public static void VisitDirectories(FileSystemSnapshot snapshot, Action <IProject, DirectorySnapshot> callback)
{
foreach (var project in snapshot.ProjectRoots.ToForeachEnum())
{
VisitDirectory(project.Project, project.Directory, callback);
}
}
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges,
Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.Build(previousDatabase, newFileSystemSnapshot, fullPathChanges, onIntermadiateResult, cancellationToken));
}
public static FileSystemTree ToIpcFileSystemTree(this FileSystemSnapshot tree)
{
return(new FileSystemTree {
Version = tree.Version,
Root = BuildFileSystemTreeRoot(tree)
});
}
private void ComputeNewStateLongTask(FileSystemSnapshot previousSnapshot, FileSystemSnapshot newSnapshot,
FullPathChanges fullPathChanges,
CancellationToken cancellationToken)
{
Invariants.Assert(_inTaskQueueTask);
UpdateFileDatabase(newSnapshot, options => {
// We only allow incremental updates if the last update was successfully completed
// and if the file system snapshot version we are based on is the same as the
// new file system snapshot we are processing
if (previousSnapshot.Version == _currentFileSystemSnapshotVersion &&
options.PreviousUpdateCompleted &&
fullPathChanges != null)
{
return(CreateWithFileSystemChanges(newSnapshot, fullPathChanges, options, cancellationToken));
}
else
{
Logger.LogInfo($"Starting a full database update: " +
$"CurrentSnapshotVersion={_currentFileSystemSnapshotVersion}, " +
$"PreviousUpdateCompleted={options.PreviousUpdateCompleted}, " +
$"PreviousSnapshotVersion={previousSnapshot.Version}, " +
$"FullPathChanges={fullPathChanges?.Entries.Count ?? -1}.");
return(CreateFullScan(newSnapshot, options, cancellationToken));
}
});
}
public FileSystemSnapshot Compute(IFileSystemNameFactory fileNameFactory,
FileSystemSnapshot oldSnapshot,
FullPathChanges pathChanges /* may be null */,
IList <IProject> projects,
int version,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested(); // cancellation
using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) {
var projectRoots =
projects
.Distinct(new ProjectPathComparer())
.Select(project => {
cancellationToken.ThrowIfCancellationRequested(); // cancellation
var projectSnapshotBuilder = new ProjectRootSnapshotBuilder(
_fileSystem, fileNameFactory, oldSnapshot, project, progress,
(pathChanges == null ? null : new ProjectPathChanges(project.RootPath, pathChanges.Entries)),
cancellationToken);
var rootSnapshot = projectSnapshotBuilder.Build();
return(new ProjectRootSnapshot(project, rootSnapshot));
})
.OrderBy(projectRoot => projectRoot.Directory.DirectoryName)
.ToReadOnlyCollection();
return(new FileSystemSnapshot(version, projectRoots));
}
}
private static List <ProjectEntry> BuildCompactProjectEntries(FileSystemSnapshot tree)
{
return(tree.ProjectRoots
.Select(x => new ProjectEntry {
RootPath = x.Project.RootPath.Value
})
.ToList());
}
private void ActivateCurrentDatabase(FileSystemSnapshot fileSystemSnapshot, IFileDatabaseSnapshot databaseSnapshot, bool complete)
{
Invariants.Assert(_inTaskQueueTask);
_currentFileDatabase = databaseSnapshot;
_currentFileSystemSnapshotVersion = fileSystemSnapshot.Version;
_previousUpdateCompleted = complete; // Success => we allow incremtal updates next time
}
public IFileDatabaseSnapshot CreateIncrementalWithFileSystemUpdates(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges, Action onLoading, Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken)
{
return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, fullPathChanges, onLoading, onLoaded,
onIntermadiateResult,
cancellationToken));
}
private static DirectoryEntry BuildFileSystemTreeRoot(FileSystemSnapshot fileSystemSnapshot)
{
return(new DirectoryEntry {
Name = null,
Data = null,
Entries = fileSystemSnapshot.ProjectRoots.Select(x => BuildDirectoryEntry(x.Directory)).Cast <FileSystemEntry>().ToList()
});
}
/// <summary>
/// Atomically updates the file contents of <paramref name="changedFiles"/>
/// with the new file contents on disk. This method violates the "pure
/// snapshot" semantics but enables efficient updates for the most common
/// type of file change events.
/// </summary>
public IFileDatabaseSnapshot CreateIncrementalWithModifiedFiles(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot fileSystemSnapshot, IList <ProjectFileName> changedFiles,
Action onLoading, Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.BuildWithChangedFiles(previousDatabase, fileSystemSnapshot, changedFiles,
onLoading, onLoaded, onIntermadiateResult, cancellationToken));
}
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newFileSystemSnapshot, Action onLoading, Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, null, onLoading, onLoaded,
onIntermadiateResult,
cancellationToken));
}
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action onLoading,
Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot",
cancellationToken, InfoLogger.Instance)) {
using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) {
onLoading();
progress.DisplayProgress((i, n) => "Preparing list of files to load from disk");
var fileDatabase = (FileDatabaseSnapshot)previousDatabase;
// Compute list of files from tree
var entities = ComputeFileSystemEntities(newSnapshot, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
cancellationToken.ThrowIfCancellationRequested();
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
cancellationToken.ThrowIfCancellationRequested();
var loadingContext = new FileContentsLoadingContext {
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabaseSnapshot = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database for partial progress reporting");
var database = CreateFileDatabase(entities, false, cancellationToken);
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(entities, loadingContext, cancellationToken);
var result = CreateFileDatabase(entities, true, cancellationToken);
onLoaded();
return(result);
}
}
}
private FileSystemEntities ComputeFileSystemEntities(FileSystemSnapshot snapshot,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Computing tables of directory names and file names from FileSystemTree",
cancellationToken)) {
var directories = FileSystemSnapshotVisitor.GetDirectories(snapshot);
//var directoryNames = new Dictionary<DirectoryName, DirectoryData>(
var directoryNames = new SlimHashTable <DirectoryName, DirectoryData>(
v => v.DirectoryName,
directories.Count,
// Note: We can use reference equality here because the directory
// names are contructed unique.
new ReferenceEqualityComparer <DirectoryName>());
foreach (var kvp in directories.ToForeachEnum())
{
directoryNames.Add(
kvp.Value.DirectoryName,
new DirectoryData(kvp.Value.DirectoryName, kvp.Value.IsSymLink));
}
var files = new SlimHashTable <FileName, ProjectFileData>(
v => v.FileName,
directories.Count * 2,
// Note: We can use reference equality here because the file names are
// constructed unique and the dictionary will be discarded once we are
// done building this snapshot.
new FileNameReferenceEqualityComparer());
foreach (var directory in directories.ToForeachEnum())
{
foreach (var fileName in directory.Value.ChildFiles.ToForeachEnum())
{
if (directory.Key.IsFileSearchable(fileName))
{
files.Add(fileName, new ProjectFileData(directory.Key, fileName, null));
}
}
}
return(new FileSystemEntities {
Files = files,
Directories = directoryNames,
ProjectHashes = snapshot.ProjectRoots.ToDictionary(x => x.Project.RootPath, x => x.Project.VersionHash)
});
}
}
public ProjectRootSnapshotBuilder(IFileSystem fileSystem,
IFileSystemNameFactory fileSystemNameFactory,
FileSystemSnapshot oldSnapshot,
IProject project,
IProgressTracker progress,
ProjectPathChanges pathChanges,
CancellationToken cancellationToken)
{
_fileSystem = fileSystem;
_fileSystemNameFactory = fileSystemNameFactory;
_oldSnapshot = oldSnapshot;
_project = project;
_progress = progress;
_pathChanges = pathChanges;
_cancellationToken = cancellationToken;
}
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newSnapshot,
FullPathChanges fullPathChanges, Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabaseSnapshot)previousDatabase;
// Compute list of files from tree
var entities = ComputeFileSystemEntities(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingContext = new FileContentsLoadingContext {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabaseSnapshot = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database for partial progress reporting");
var database = CreateFileDatabse(entities);
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(entities, loadingContext, cancellationToken);
return(CreateFileDatabse(entities));
}
}
private IFileDatabaseSnapshot CreateWithModifiedFiles(FileSystemSnapshot fileSystemSnapshot, IList <ProjectFileName> changedFiles, UpdateFileDatabaseOptions options, CancellationToken cancellationToken)
{
Invariants.Assert(_inTaskQueueTask);
return(_fileDatabaseSnapshotFactory.CreateIncrementalWithModifiedFiles(
options.CurrentFileDatabase,
fileSystemSnapshot,
changedFiles,
onIntermadiateResult: fileDatabase => {
ActivateCurrentDatabase(fileSystemSnapshot, fileDatabase, false);
OnFilesLoadingProgress(options.OperationInfo);
},
onLoading: () => OnFilesLoading(options.OperationInfo),
onLoaded: () => OnFilesLoaded(new FilesLoadedResult {
OperationInfo = options.OperationInfo,
TreeVersion = _currentFileSystemSnapshotVersion,
}),
cancellationToken: cancellationToken));
}
private void ComputeNewState(FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges,
CancellationToken cancellationToken)
{
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
OnFilesLoading(info),
OnError = (info, error) => {
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
Error = error,
TreeVersion = newSnapshot.Version
});
},
Execute = info => {
using (new TimeElapsedLogger("Computing new state of file database")) {
var oldState = _currentFileDatabaseSnapshot;
var newState = _fileDatabaseSnapshotFactory.CreateIncremental(
oldState,
newSnapshot,
fullPathChanges,
fileDatabase => {
// Store and activate intermediate new state (atomic operation).
_currentFileDatabaseSnapshot = fileDatabase;
OnFilesLoadingProgress(info);
},
cancellationToken);
// Store and activate final new state (atomic operation).
_currentFileDatabaseSnapshot = newState;
}
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
TreeVersion = newSnapshot.Version
});
}
});
}
public FileSystemSnapshot Compute(IFileSystemNameFactory fileNameFactory,
FileSystemSnapshot oldSnapshot,
FullPathChanges pathChanges /* may be null */,
IList <IProject> projects,
int version,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested(); // cancellation
using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) {
// Clear file name factory intern tables if no projects. We could be
// more aggressive at the expense of decreasing string interning.
if (projects.Count == 0)
{
fileNameFactory.ClearInternedStrings();
}
var projectRoots =
projects
.Distinct(new ProjectPathComparer())
.Select(project => {
cancellationToken.ThrowIfCancellationRequested(); // cancellation
var projectSnapshotBuilder = new ProjectRootSnapshotBuilder(
_fileSystem, fileNameFactory, oldSnapshot, project, progress,
(pathChanges == null ? null : new ProjectPathChanges(project.RootPath, pathChanges.Entries)),
cancellationToken);
var rootSnapshot = projectSnapshotBuilder.Build();
return(new ProjectRootSnapshot(project, rootSnapshot));
})
.OrderBy(projectRoot => projectRoot.Directory.DirectoryName)
.ToReadOnlyCollection();
return(new FileSystemSnapshot(version, projectRoots));
}
}
private void UpdateFileDatabase(FileSystemSnapshot fileSystemSnapshot, Func <UpdateFileDatabaseOptions, IFileDatabaseSnapshot> updater)
{
Invariants.Assert(_inTaskQueueTask);
var options = new UpdateFileDatabaseOptions();
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info => {
options.OperationInfo = info;
options.PreviousUpdateCompleted = _previousUpdateCompleted;
_previousUpdateCompleted = false;
},
OnError = (info, error) => {
_currentFileSystemSnapshotVersion = fileSystemSnapshot.Version;
_previousUpdateCompleted = false;
OnFilesLoading(info);
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
Error = error,
TreeVersion = _currentFileSystemSnapshotVersion,
});
},
Execute = info => {
options.CurrentFileDatabase = _currentFileDatabase;
var newFileDatabase = updater(options);
ActivateCurrentDatabase(fileSystemSnapshot, newFileDatabase, true);
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
TreeVersion = fileSystemSnapshot.Version
});
}
});
}
private IEnumerable <ProjectDetails> CreateProjectsDetails(GetDatabaseDetailsRequest request,
FileSystemSnapshot snapshot, IFileDatabaseSnapshot database)
{
return(snapshot.ProjectRoots.Select(project => GetProjectDetailsRequestHandler.CreateProjectDetails(database,
project, request.MaxFilesByExtensionDetailsCount, request.MaxLargeFilesDetailsCount)));
}
public static List <KeyValuePair <IProject, DirectorySnapshot> > GetDirectories(FileSystemSnapshot snapshot)
{
var result = new List <KeyValuePair <IProject, DirectorySnapshot> >();
VisitDirectories(snapshot, (project, directory) => {
result.Add(new KeyValuePair <IProject, DirectorySnapshot>(project, directory));
});
return(result);
}
public FileSystemTreeSnapshotNameFactory(FileSystemSnapshot snapshot, IFileSystemNameFactory previous)
{
_snapshot = snapshot;
_previous = previous;
}