private void UpdateFileContents(IEnumerable <ProjectFileName> files)
{
OperationInfo operationInfo = null;
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
operationInfo = info,
OnError = (info, error) => {
OnFilesLoading(info);
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
Error = error,
TreeVersion = _currentTreeVersion,
});
},
Execute = info => {
_currentFileDatabaseSnapshot = _fileDatabaseSnapshotFactory.CreateWithChangedFiles(
_currentFileDatabaseSnapshot,
files,
onLoading: () => OnFilesLoading(operationInfo),
onLoaded: () => OnFilesLoaded(new FilesLoadedResult {
OperationInfo = operationInfo,
TreeVersion = _currentTreeVersion,
}));
}
});
}
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges,
Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.Build(previousDatabase, newFileSystemSnapshot, fullPathChanges, onIntermadiateResult, cancellationToken));
}
public IFileDatabaseSnapshot BuildWithChangedFiles(IFileDatabaseSnapshot previousFileDatabaseSnapshot,
FileSystemSnapshot fileSystemSnapshot, IEnumerable <ProjectFileName> changedFiles,
Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and list of changed files", cancellationToken, InfoLogger.Instance)) {
Invariants.Assert(previousFileDatabaseSnapshot is FileDatabaseSnapshot);
var previousFileDatabase = (FileDatabaseSnapshot)previousFileDatabaseSnapshot;
// Update file contents of file data entries of changed files.
var filesToRead = changedFiles
.Where(x => x.Project.IsFileSearchable(x.FileName) && previousFileDatabase.Files.ContainsKey(x.FileName))
.ToList();
if (filesToRead.Count == 0)
{
Logger.LogInfo("None of the changed file is searchable, return previous database snapshot");
return(previousFileDatabaseSnapshot);
}
// Read file contents.
onLoading();
filesToRead.ForAll(x => {
var newContents = _fileContentsFactory.ReadFileContents(x.FileName.FullPath);
DangerousUpdateFileTableEntry(previousFileDatabase, x.FileName, newContents);
});
onLoaded();
// Return new file database with updated file contents.
return(new FileDatabaseSnapshot(
previousFileDatabase.ProjectHashes,
previousFileDatabase.Directories,
previousFileDatabase.Files));
}
}
public IFileDatabaseSnapshot BuildWithChangedFiles(IFileDatabaseSnapshot previousFileDatabaseSnapshot,
IEnumerable <ProjectFileName> changedFiles, Action onLoading, Action onLoaded)
{
using (new TimeElapsedLogger("Building file database from previous one and list of changed files")) {
var fileDatabase = (FileDatabaseSnapshot)previousFileDatabaseSnapshot;
// Update file contents of file data entries of changed files.
var filesToRead = changedFiles
.Where(x => x.Project.IsFileSearchable(x.FileName) && fileDatabase.Files.ContainsKey(x.FileName))
.ToList();
if (filesToRead.Count == 0)
{
return(previousFileDatabaseSnapshot);
}
// Read file contents.
onLoading();
filesToRead.ForAll(x => {
var newContents = _fileContentsFactory.ReadFileContents(x.FileName.FullPath);
fileDatabase.Files[x.FileName].UpdateContents(newContents);
});
onLoaded();
// Return new file database with updated file contents.
var filesWithContents = FilterFilesWithContents(fileDatabase.Files.Values);
return(new FileDatabaseSnapshot(
fileDatabase.ProjectHashes,
fileDatabase.Files,
fileDatabase.FileNames,
fileDatabase.Directories,
CreateFilePieces(filesWithContents),
filesWithContents.Count));
}
}
private void ActivateCurrentDatabase(FileSystemSnapshot fileSystemSnapshot, IFileDatabaseSnapshot databaseSnapshot, bool complete)
{
Invariants.Assert(_inTaskQueueTask);
_currentFileDatabase = databaseSnapshot;
_currentFileSystemSnapshotVersion = fileSystemSnapshot.Version;
_previousUpdateCompleted = complete; // Success => we allow incremtal updates next time
}
public IFileDatabaseSnapshot CreateIncrementalWithFileSystemUpdates(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges, Action onLoading, Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken)
{
return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, fullPathChanges, onLoading, onLoaded,
onIntermadiateResult,
cancellationToken));
}
/// <summary>
/// Atomically updates the file contents of <paramref name="changedFiles"/>
/// with the new file contents on disk. This method violates the "pure
/// snapshot" semantics but enables efficient updates for the most common
/// type of file change events.
/// </summary>
public IFileDatabaseSnapshot CreateIncrementalWithModifiedFiles(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot fileSystemSnapshot, IList <ProjectFileName> changedFiles,
Action onLoading, Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.BuildWithChangedFiles(previousDatabase, fileSystemSnapshot, changedFiles,
onLoading, onLoaded, onIntermadiateResult, cancellationToken));
}
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newFileSystemSnapshot, Action onLoading, Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, null, onLoading, onLoaded,
onIntermadiateResult,
cancellationToken));
}
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action onLoading,
Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot",
cancellationToken, InfoLogger.Instance)) {
using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) {
onLoading();
progress.DisplayProgress((i, n) => "Preparing list of files to load from disk");
var fileDatabase = (FileDatabaseSnapshot)previousDatabase;
// Compute list of files from tree
var entities = ComputeFileSystemEntities(newSnapshot, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
cancellationToken.ThrowIfCancellationRequested();
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
cancellationToken.ThrowIfCancellationRequested();
var loadingContext = new FileContentsLoadingContext {
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabaseSnapshot = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database for partial progress reporting");
var database = CreateFileDatabase(entities, false, cancellationToken);
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(entities, loadingContext, cancellationToken);
var result = CreateFileDatabase(entities, true, cancellationToken);
onLoaded();
return(result);
}
}
}
public static ProjectDetails CreateProjectDetails(IFileDatabaseSnapshot database, ProjectRootSnapshot project,
int maxFilesByExtensionDetailsCount, int maxLargeFilesDetailsCount)
{
return(new ProjectDetails {
RootPath = project.Project.RootPath.Value,
DirectoryDetails = GetDirectoryDetailsRequestHandler.CreateDirectoryDetails(database, project,
project.Directory,
maxFilesByExtensionDetailsCount, maxLargeFilesDetailsCount),
ConfigurationDetails = CreateProjectConfigurationDetails(project)
});
}
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newSnapshot,
FullPathChanges fullPathChanges, Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabaseSnapshot)previousDatabase;
// Compute list of files from tree
var entities = ComputeFileSystemEntities(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingContext = new FileContentsLoadingContext {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabaseSnapshot = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database for partial progress reporting");
var database = CreateFileDatabse(entities);
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(entities, loadingContext, cancellationToken);
return(CreateFileDatabse(entities));
}
}
private void ComputeNewState(FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges,
CancellationToken cancellationToken)
{
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
OnFilesLoading(info),
OnError = (info, error) => {
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
Error = error,
TreeVersion = newSnapshot.Version
});
},
Execute = info => {
using (new TimeElapsedLogger("Computing new state of file database")) {
var oldState = _currentFileDatabaseSnapshot;
var newState = _fileDatabaseSnapshotFactory.CreateIncremental(
oldState,
newSnapshot,
fullPathChanges,
fileDatabase => {
// Store and activate intermediate new state (atomic operation).
_currentFileDatabaseSnapshot = fileDatabase;
OnFilesLoadingProgress(info);
},
cancellationToken);
// Store and activate final new state (atomic operation).
_currentFileDatabaseSnapshot = newState;
}
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
TreeVersion = newSnapshot.Version
});
}
});
}
public SearchEngine(
IFileSystemSnapshotManager fileSystemSnapshotManager,
IFileSystemNameFactory fileSystemNameFactory,
ILongRunningFileSystemTaskQueue taskQueue,
IFileDatabaseSnapshotFactory fileDatabaseSnapshotFactory,
IProjectDiscovery projectDiscovery,
ICompiledTextSearchDataFactory compiledTextSearchDataFactory,
IOperationProcessor operationProcessor)
{
_fileSystemNameFactory = fileSystemNameFactory;
_taskQueue = taskQueue;
_fileDatabaseSnapshotFactory = fileDatabaseSnapshotFactory;
_projectDiscovery = projectDiscovery;
_compiledTextSearchDataFactory = compiledTextSearchDataFactory;
_operationProcessor = operationProcessor;
// Create a "Null" state
_currentFileDatabase = _fileDatabaseSnapshotFactory.CreateEmpty();
// Setup computing a new state everytime a new tree is computed.
fileSystemSnapshotManager.SnapshotScanFinished += FileSystemSnapshotManagerOnSnapshotScanFinished;
fileSystemSnapshotManager.FilesChanged += FileSystemSnapshotManagerOnFilesChanged;
}
private IEnumerable <ProjectDetails> CreateProjectsDetails(GetDatabaseDetailsRequest request,
FileSystemSnapshot snapshot, IFileDatabaseSnapshot database)
{
return(snapshot.ProjectRoots.Select(project => GetProjectDetailsRequestHandler.CreateProjectDetails(database,
project, request.MaxFilesByExtensionDetailsCount, request.MaxLargeFilesDetailsCount)));
}
public static DirectoryDetails CreateDirectoryDetails(IFileDatabaseSnapshot database,
ProjectRootSnapshot projectSnapshot,
DirectorySnapshot baseDirectory, int maxFilesByExtensionDetailsCount, int maxLargeFilesDetailsCount)
{
var directoryPath = baseDirectory.DirectoryName.FullPath;
var fileDatabse = (FileDatabaseSnapshot)database;
var directoryFileNames = fileDatabse.FileNames
.Where(x => directoryPath.ContainsPath(x.FullPath))
.ToHashSet();
var directoryFiles = fileDatabse.Files.Values
.Where(x => directoryFileNames.Contains(x.FileName))
.ToList();
var searchableFiles = directoryFiles
.Where(x => x.HasContents())
.ToList();
var binaryFiles = directoryFiles
.Where(x => (x.Contents is BinaryFileContents) && (((BinaryFileContents)x.Contents).BinaryFileSize > 0))
.ToList();
return(new DirectoryDetails {
Path = directoryPath.Value,
DirectoryCount = FileSystemSnapshotManager.CountDirectoryEntries(baseDirectory),
FileCount = FileSystemSnapshotManager.CountFileEntries(baseDirectory),
SearchableFilesCount = searchableFiles.Count,
SearchableFilesByteLength = searchableFiles.Aggregate(0L, (acc, x) => acc + x.Contents.ByteLength),
BinaryFilesCount = binaryFiles.Count,
BinaryFilesByteLength = binaryFiles.Aggregate(0L, (acc, x) => acc + ((BinaryFileContents)x.Contents).BinaryFileSize),
SearchableFilesByExtensionDetails = searchableFiles
.GroupBy(x => GetFileExtension(x.FileName))
.Select(g => new FileByExtensionDetails {
FileExtension = g.Key,
FileCount = g.Count(),
FilesByteLength = g.Aggregate(0L, (acc, x) => acc + x.Contents.ByteLength)
})
.OrderByDescendingThenTake(maxFilesByExtensionDetailsCount, x => x.FilesByteLength)
.ToList(),
LargeSearchableFilesDetails = searchableFiles
.OrderByDescendingThenTake(maxLargeFilesDetailsCount, x => x.Contents.ByteLength)
.Select(x => new LargeFileDetails {
RelativePath = GetRelativePath(directoryPath, x.FileName.FullPath),
ByteLength = x.Contents.ByteLength
})
.ToList(),
BinaryFilesByExtensionDetails = binaryFiles
.GroupBy(x => GetFileExtension(x.FileName))
.Select(g => new FileByExtensionDetails {
FileExtension = g.Key,
FileCount = g.Count(),
FilesByteLength = g.Aggregate(0L, (acc, x) => acc + ((BinaryFileContents)x.Contents).BinaryFileSize)
})
.OrderByDescendingThenTake(maxFilesByExtensionDetailsCount, x => x.FilesByteLength)
.ToList(),
LargeBinaryFilesDetails = binaryFiles
.OrderByDescendingThenTake(maxLargeFilesDetailsCount, x => ((BinaryFileContents)x.Contents).BinaryFileSize)
.Select(x => new LargeFileDetails {
RelativePath = GetRelativePath(directoryPath, x.FileName.FullPath),
ByteLength = ((BinaryFileContents)x.Contents).BinaryFileSize
})
.ToList()
});
}
/// <summary>
/// Atomically updates the file contents of <paramref name="changedFiles"/>
/// with the new file contents on disk. This method violates the "pure
/// snapshot" semantics but enables efficient updates for the most common
/// type of file change events.
/// </summary>
public IFileDatabaseSnapshot CreateWithChangedFiles(IFileDatabaseSnapshot previousDatabase,
IEnumerable <ProjectFileName> changedFiles, Action onLoading, Action onLoaded)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.BuildWithChangedFiles(previousDatabase, changedFiles, onLoading, onLoaded));
}
public static bool IsContainedInSymLink(this IFileDatabaseSnapshot snapshot, FileName name)
{
return(snapshot.IsContainedInSymLink(name.Parent));
}
