private void UpdateFileContents(IEnumerable <ProjectFileName> files) { OperationInfo operationInfo = null; _operationProcessor.Execute(new OperationHandlers { OnBeforeExecute = info => operationInfo = info, OnError = (info, error) => { OnFilesLoading(info); OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, Error = error, TreeVersion = _currentTreeVersion, }); }, Execute = info => { _currentFileDatabaseSnapshot = _fileDatabaseSnapshotFactory.CreateWithChangedFiles( _currentFileDatabaseSnapshot, files, onLoading: () => OnFilesLoading(operationInfo), onLoaded: () => OnFilesLoaded(new FilesLoadedResult { OperationInfo = operationInfo, TreeVersion = _currentTreeVersion, })); } }); }
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory) .Build(previousDatabase, newFileSystemSnapshot, fullPathChanges, onIntermadiateResult, cancellationToken)); }
public IFileDatabaseSnapshot BuildWithChangedFiles(IFileDatabaseSnapshot previousFileDatabaseSnapshot, FileSystemSnapshot fileSystemSnapshot, IEnumerable <ProjectFileName> changedFiles, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { using (new TimeElapsedLogger("Building file database from previous one and list of changed files", cancellationToken, InfoLogger.Instance)) { Invariants.Assert(previousFileDatabaseSnapshot is FileDatabaseSnapshot); var previousFileDatabase = (FileDatabaseSnapshot)previousFileDatabaseSnapshot; // Update file contents of file data entries of changed files. var filesToRead = changedFiles .Where(x => x.Project.IsFileSearchable(x.FileName) && previousFileDatabase.Files.ContainsKey(x.FileName)) .ToList(); if (filesToRead.Count == 0) { Logger.LogInfo("None of the changed file is searchable, return previous database snapshot"); return(previousFileDatabaseSnapshot); } // Read file contents. onLoading(); filesToRead.ForAll(x => { var newContents = _fileContentsFactory.ReadFileContents(x.FileName.FullPath); DangerousUpdateFileTableEntry(previousFileDatabase, x.FileName, newContents); }); onLoaded(); // Return new file database with updated file contents. return(new FileDatabaseSnapshot( previousFileDatabase.ProjectHashes, previousFileDatabase.Directories, previousFileDatabase.Files)); } }
public IFileDatabaseSnapshot BuildWithChangedFiles(IFileDatabaseSnapshot previousFileDatabaseSnapshot, IEnumerable <ProjectFileName> changedFiles, Action onLoading, Action onLoaded) { using (new TimeElapsedLogger("Building file database from previous one and list of changed files")) { var fileDatabase = (FileDatabaseSnapshot)previousFileDatabaseSnapshot; // Update file contents of file data entries of changed files. var filesToRead = changedFiles .Where(x => x.Project.IsFileSearchable(x.FileName) && fileDatabase.Files.ContainsKey(x.FileName)) .ToList(); if (filesToRead.Count == 0) { return(previousFileDatabaseSnapshot); } // Read file contents. onLoading(); filesToRead.ForAll(x => { var newContents = _fileContentsFactory.ReadFileContents(x.FileName.FullPath); fileDatabase.Files[x.FileName].UpdateContents(newContents); }); onLoaded(); // Return new file database with updated file contents. var filesWithContents = FilterFilesWithContents(fileDatabase.Files.Values); return(new FileDatabaseSnapshot( fileDatabase.ProjectHashes, fileDatabase.Files, fileDatabase.FileNames, fileDatabase.Directories, CreateFilePieces(filesWithContents), filesWithContents.Count)); } }
private void ActivateCurrentDatabase(FileSystemSnapshot fileSystemSnapshot, IFileDatabaseSnapshot databaseSnapshot, bool complete) { Invariants.Assert(_inTaskQueueTask); _currentFileDatabase = databaseSnapshot; _currentFileSystemSnapshotVersion = fileSystemSnapshot.Version; _previousUpdateCompleted = complete; // Success => we allow incremtal updates next time }
public IFileDatabaseSnapshot CreateIncrementalWithFileSystemUpdates(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, fullPathChanges, onLoading, onLoaded, onIntermadiateResult, cancellationToken)); }
/// <summary> /// Atomically updates the file contents of <paramref name="changedFiles"/> /// with the new file contents on disk. This method violates the "pure /// snapshot" semantics but enables efficient updates for the most common /// type of file change events. /// </summary> public IFileDatabaseSnapshot CreateIncrementalWithModifiedFiles(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot fileSystemSnapshot, IList <ProjectFileName> changedFiles, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory) .BuildWithChangedFiles(previousDatabase, fileSystemSnapshot, changedFiles, onLoading, onLoaded, onIntermadiateResult, cancellationToken)); }
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newFileSystemSnapshot, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, null, onLoading, onLoaded, onIntermadiateResult, cancellationToken)); }
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot", cancellationToken, InfoLogger.Instance)) { using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) { onLoading(); progress.DisplayProgress((i, n) => "Preparing list of files to load from disk"); var fileDatabase = (FileDatabaseSnapshot)previousDatabase; // Compute list of files from tree var entities = ComputeFileSystemEntities(newSnapshot, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); var unchangedProjects = newSnapshot .ProjectRoots.Where(x => fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) && fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash) .Select(x => x.Project); cancellationToken.ThrowIfCancellationRequested(); var unchangedProjectSet = new HashSet <IProject>(unchangedProjects, // Use reference equality for IProject is safe, as we keep this // dictionary only for the duration of this "Build" call. new ReferenceEqualityComparer <IProject>()); cancellationToken.ThrowIfCancellationRequested(); var loadingContext = new FileContentsLoadingContext { FullPathChanges = fullPathChanges, LoadedTextFileCount = 0, OldFileDatabaseSnapshot = fileDatabase, UnchangedProjects = unchangedProjectSet, PartialProgressReporter = new PartialProgressReporter( TimeSpan.FromSeconds(5.0), () => { Logger.LogInfo("Creating intermedidate file database for partial progress reporting"); var database = CreateFileDatabase(entities, false, cancellationToken); onIntermadiateResult(database); }) }; // Merge old state in new state and load all missing files LoadFileContents(entities, loadingContext, cancellationToken); var result = CreateFileDatabase(entities, true, cancellationToken); onLoaded(); return(result); } } }
public static ProjectDetails CreateProjectDetails(IFileDatabaseSnapshot database, ProjectRootSnapshot project, int maxFilesByExtensionDetailsCount, int maxLargeFilesDetailsCount) { return(new ProjectDetails { RootPath = project.Project.RootPath.Value, DirectoryDetails = GetDirectoryDetailsRequestHandler.CreateDirectoryDetails(database, project, project.Directory, maxFilesByExtensionDetailsCount, maxLargeFilesDetailsCount), ConfigurationDetails = CreateProjectConfigurationDetails(project) }); }
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) { var fileDatabase = (FileDatabaseSnapshot)previousDatabase; // Compute list of files from tree var entities = ComputeFileSystemEntities(newSnapshot); var unchangedProjects = newSnapshot .ProjectRoots.Where(x => fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) && fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash) .Select(x => x.Project); var unchangedProjectSet = new HashSet <IProject>(unchangedProjects, // Use reference equality for IProject is safe, as we keep this // dictionary only for the duration of this "Build" call. new ReferenceEqualityComparer <IProject>()); // Don't use file memoization for now, as benefit is dubvious. //IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization(); IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization(); var loadingContext = new FileContentsLoadingContext { FileContentsMemoization = fileContentsMemoization, FullPathChanges = fullPathChanges, LoadedTextFileCount = 0, OldFileDatabaseSnapshot = fileDatabase, UnchangedProjects = unchangedProjectSet, PartialProgressReporter = new PartialProgressReporter( TimeSpan.FromSeconds(5.0), () => { Logger.LogInfo("Creating intermedidate file database for partial progress reporting"); var database = CreateFileDatabse(entities); onIntermadiateResult(database); }) }; // Merge old state in new state and load all missing files LoadFileContents(entities, loadingContext, cancellationToken); return(CreateFileDatabse(entities)); } }
private void ComputeNewState(FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges, CancellationToken cancellationToken) { _operationProcessor.Execute(new OperationHandlers { OnBeforeExecute = info => OnFilesLoading(info), OnError = (info, error) => { _currentTreeVersion = newSnapshot.Version; OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, Error = error, TreeVersion = newSnapshot.Version }); }, Execute = info => { using (new TimeElapsedLogger("Computing new state of file database")) { var oldState = _currentFileDatabaseSnapshot; var newState = _fileDatabaseSnapshotFactory.CreateIncremental( oldState, newSnapshot, fullPathChanges, fileDatabase => { // Store and activate intermediate new state (atomic operation). _currentFileDatabaseSnapshot = fileDatabase; OnFilesLoadingProgress(info); }, cancellationToken); // Store and activate final new state (atomic operation). _currentFileDatabaseSnapshot = newState; } _currentTreeVersion = newSnapshot.Version; OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, TreeVersion = newSnapshot.Version }); } }); }
public SearchEngine( IFileSystemSnapshotManager fileSystemSnapshotManager, IFileSystemNameFactory fileSystemNameFactory, ILongRunningFileSystemTaskQueue taskQueue, IFileDatabaseSnapshotFactory fileDatabaseSnapshotFactory, IProjectDiscovery projectDiscovery, ICompiledTextSearchDataFactory compiledTextSearchDataFactory, IOperationProcessor operationProcessor) { _fileSystemNameFactory = fileSystemNameFactory; _taskQueue = taskQueue; _fileDatabaseSnapshotFactory = fileDatabaseSnapshotFactory; _projectDiscovery = projectDiscovery; _compiledTextSearchDataFactory = compiledTextSearchDataFactory; _operationProcessor = operationProcessor; // Create a "Null" state _currentFileDatabase = _fileDatabaseSnapshotFactory.CreateEmpty(); // Setup computing a new state everytime a new tree is computed. fileSystemSnapshotManager.SnapshotScanFinished += FileSystemSnapshotManagerOnSnapshotScanFinished; fileSystemSnapshotManager.FilesChanged += FileSystemSnapshotManagerOnFilesChanged; }
private IEnumerable <ProjectDetails> CreateProjectsDetails(GetDatabaseDetailsRequest request, FileSystemSnapshot snapshot, IFileDatabaseSnapshot database) { return(snapshot.ProjectRoots.Select(project => GetProjectDetailsRequestHandler.CreateProjectDetails(database, project, request.MaxFilesByExtensionDetailsCount, request.MaxLargeFilesDetailsCount))); }
public static DirectoryDetails CreateDirectoryDetails(IFileDatabaseSnapshot database, ProjectRootSnapshot projectSnapshot, DirectorySnapshot baseDirectory, int maxFilesByExtensionDetailsCount, int maxLargeFilesDetailsCount) { var directoryPath = baseDirectory.DirectoryName.FullPath; var fileDatabse = (FileDatabaseSnapshot)database; var directoryFileNames = fileDatabse.FileNames .Where(x => directoryPath.ContainsPath(x.FullPath)) .ToHashSet(); var directoryFiles = fileDatabse.Files.Values .Where(x => directoryFileNames.Contains(x.FileName)) .ToList(); var searchableFiles = directoryFiles .Where(x => x.HasContents()) .ToList(); var binaryFiles = directoryFiles .Where(x => (x.Contents is BinaryFileContents) && (((BinaryFileContents)x.Contents).BinaryFileSize > 0)) .ToList(); return(new DirectoryDetails { Path = directoryPath.Value, DirectoryCount = FileSystemSnapshotManager.CountDirectoryEntries(baseDirectory), FileCount = FileSystemSnapshotManager.CountFileEntries(baseDirectory), SearchableFilesCount = searchableFiles.Count, SearchableFilesByteLength = searchableFiles.Aggregate(0L, (acc, x) => acc + x.Contents.ByteLength), BinaryFilesCount = binaryFiles.Count, BinaryFilesByteLength = binaryFiles.Aggregate(0L, (acc, x) => acc + ((BinaryFileContents)x.Contents).BinaryFileSize), SearchableFilesByExtensionDetails = searchableFiles .GroupBy(x => GetFileExtension(x.FileName)) .Select(g => new FileByExtensionDetails { FileExtension = g.Key, FileCount = g.Count(), FilesByteLength = g.Aggregate(0L, (acc, x) => acc + x.Contents.ByteLength) }) .OrderByDescendingThenTake(maxFilesByExtensionDetailsCount, x => x.FilesByteLength) .ToList(), LargeSearchableFilesDetails = searchableFiles .OrderByDescendingThenTake(maxLargeFilesDetailsCount, x => x.Contents.ByteLength) .Select(x => new LargeFileDetails { RelativePath = GetRelativePath(directoryPath, x.FileName.FullPath), ByteLength = x.Contents.ByteLength }) .ToList(), BinaryFilesByExtensionDetails = binaryFiles .GroupBy(x => GetFileExtension(x.FileName)) .Select(g => new FileByExtensionDetails { FileExtension = g.Key, FileCount = g.Count(), FilesByteLength = g.Aggregate(0L, (acc, x) => acc + ((BinaryFileContents)x.Contents).BinaryFileSize) }) .OrderByDescendingThenTake(maxFilesByExtensionDetailsCount, x => x.FilesByteLength) .ToList(), LargeBinaryFilesDetails = binaryFiles .OrderByDescendingThenTake(maxLargeFilesDetailsCount, x => ((BinaryFileContents)x.Contents).BinaryFileSize) .Select(x => new LargeFileDetails { RelativePath = GetRelativePath(directoryPath, x.FileName.FullPath), ByteLength = ((BinaryFileContents)x.Contents).BinaryFileSize }) .ToList() }); }
/// <summary> /// Atomically updates the file contents of <paramref name="changedFiles"/> /// with the new file contents on disk. This method violates the "pure /// snapshot" semantics but enables efficient updates for the most common /// type of file change events. /// </summary> public IFileDatabaseSnapshot CreateWithChangedFiles(IFileDatabaseSnapshot previousDatabase, IEnumerable <ProjectFileName> changedFiles, Action onLoading, Action onLoaded) { return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory) .BuildWithChangedFiles(previousDatabase, changedFiles, onLoading, onLoaded)); }
public static bool IsContainedInSymLink(this IFileDatabaseSnapshot snapshot, FileName name) { return(snapshot.IsContainedInSymLink(name.Parent)); }