public IFileDatabaseSnapshot BuildWithChangedFiles(IFileDatabaseSnapshot previousFileDatabaseSnapshot, FileSystemSnapshot fileSystemSnapshot, IEnumerable <ProjectFileName> changedFiles, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { using (new TimeElapsedLogger("Building file database from previous one and list of changed files", cancellationToken, InfoLogger.Instance)) { Invariants.Assert(previousFileDatabaseSnapshot is FileDatabaseSnapshot); var previousFileDatabase = (FileDatabaseSnapshot)previousFileDatabaseSnapshot; // Update file contents of file data entries of changed files. var filesToRead = changedFiles .Where(x => x.Project.IsFileSearchable(x.FileName) && previousFileDatabase.Files.ContainsKey(x.FileName)) .ToList(); if (filesToRead.Count == 0) { Logger.LogInfo("None of the changed file is searchable, return previous database snapshot"); return(previousFileDatabaseSnapshot); } // Read file contents. onLoading(); filesToRead.ForAll(x => { var newContents = _fileContentsFactory.ReadFileContents(x.FileName.FullPath); DangerousUpdateFileTableEntry(previousFileDatabase, x.FileName, newContents); }); onLoaded(); // Return new file database with updated file contents. return(new FileDatabaseSnapshot( previousFileDatabase.ProjectHashes, previousFileDatabase.Directories, previousFileDatabase.Files)); } }
public static FileSystemTree ToIpcCompactFileSystemTree(this FileSystemSnapshot tree) { return(new FileSystemTree { Version = tree.Version, Projects = BuildCompactProjectEntries(tree) }); }
public static void VisitDirectories(FileSystemSnapshot snapshot, Action <IProject, DirectorySnapshot> callback) { foreach (var project in snapshot.ProjectRoots.ToForeachEnum()) { VisitDirectory(project.Project, project.Directory, callback); } }
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory) .Build(previousDatabase, newFileSystemSnapshot, fullPathChanges, onIntermadiateResult, cancellationToken)); }
public static FileSystemTree ToIpcFileSystemTree(this FileSystemSnapshot tree) { return(new FileSystemTree { Version = tree.Version, Root = BuildFileSystemTreeRoot(tree) }); }
private void ComputeNewStateLongTask(FileSystemSnapshot previousSnapshot, FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges, CancellationToken cancellationToken) { Invariants.Assert(_inTaskQueueTask); UpdateFileDatabase(newSnapshot, options => { // We only allow incremental updates if the last update was successfully completed // and if the file system snapshot version we are based on is the same as the // new file system snapshot we are processing if (previousSnapshot.Version == _currentFileSystemSnapshotVersion && options.PreviousUpdateCompleted && fullPathChanges != null) { return(CreateWithFileSystemChanges(newSnapshot, fullPathChanges, options, cancellationToken)); } else { Logger.LogInfo($"Starting a full database update: " + $"CurrentSnapshotVersion={_currentFileSystemSnapshotVersion}, " + $"PreviousUpdateCompleted={options.PreviousUpdateCompleted}, " + $"PreviousSnapshotVersion={previousSnapshot.Version}, " + $"FullPathChanges={fullPathChanges?.Entries.Count ?? -1}."); return(CreateFullScan(newSnapshot, options, cancellationToken)); } }); }
public FileSystemSnapshot Compute(IFileSystemNameFactory fileNameFactory, FileSystemSnapshot oldSnapshot, FullPathChanges pathChanges /* may be null */, IList <IProject> projects, int version, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); // cancellation using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) { var projectRoots = projects .Distinct(new ProjectPathComparer()) .Select(project => { cancellationToken.ThrowIfCancellationRequested(); // cancellation var projectSnapshotBuilder = new ProjectRootSnapshotBuilder( _fileSystem, fileNameFactory, oldSnapshot, project, progress, (pathChanges == null ? null : new ProjectPathChanges(project.RootPath, pathChanges.Entries)), cancellationToken); var rootSnapshot = projectSnapshotBuilder.Build(); return(new ProjectRootSnapshot(project, rootSnapshot)); }) .OrderBy(projectRoot => projectRoot.Directory.DirectoryName) .ToReadOnlyCollection(); return(new FileSystemSnapshot(version, projectRoots)); } }
private static List <ProjectEntry> BuildCompactProjectEntries(FileSystemSnapshot tree) { return(tree.ProjectRoots .Select(x => new ProjectEntry { RootPath = x.Project.RootPath.Value }) .ToList()); }
private void ActivateCurrentDatabase(FileSystemSnapshot fileSystemSnapshot, IFileDatabaseSnapshot databaseSnapshot, bool complete) { Invariants.Assert(_inTaskQueueTask); _currentFileDatabase = databaseSnapshot; _currentFileSystemSnapshotVersion = fileSystemSnapshot.Version; _previousUpdateCompleted = complete; // Success => we allow incremtal updates next time }
public IFileDatabaseSnapshot CreateIncrementalWithFileSystemUpdates(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newFileSystemSnapshot, FullPathChanges fullPathChanges, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, fullPathChanges, onLoading, onLoaded, onIntermadiateResult, cancellationToken)); }
private static DirectoryEntry BuildFileSystemTreeRoot(FileSystemSnapshot fileSystemSnapshot) { return(new DirectoryEntry { Name = null, Data = null, Entries = fileSystemSnapshot.ProjectRoots.Select(x => BuildDirectoryEntry(x.Directory)).Cast <FileSystemEntry>().ToList() }); }
/// <summary> /// Atomically updates the file contents of <paramref name="changedFiles"/> /// with the new file contents on disk. This method violates the "pure /// snapshot" semantics but enables efficient updates for the most common /// type of file change events. /// </summary> public IFileDatabaseSnapshot CreateIncrementalWithModifiedFiles(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot fileSystemSnapshot, IList <ProjectFileName> changedFiles, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory) .BuildWithChangedFiles(previousDatabase, fileSystemSnapshot, changedFiles, onLoading, onLoaded, onIntermadiateResult, cancellationToken)); }
public IFileDatabaseSnapshot CreateIncremental(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newFileSystemSnapshot, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { return(CreateIncrementalWorker(previousDatabase, newFileSystemSnapshot, null, onLoading, onLoaded, onIntermadiateResult, cancellationToken)); }
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges, Action onLoading, Action onLoaded, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot", cancellationToken, InfoLogger.Instance)) { using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) { onLoading(); progress.DisplayProgress((i, n) => "Preparing list of files to load from disk"); var fileDatabase = (FileDatabaseSnapshot)previousDatabase; // Compute list of files from tree var entities = ComputeFileSystemEntities(newSnapshot, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); var unchangedProjects = newSnapshot .ProjectRoots.Where(x => fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) && fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash) .Select(x => x.Project); cancellationToken.ThrowIfCancellationRequested(); var unchangedProjectSet = new HashSet <IProject>(unchangedProjects, // Use reference equality for IProject is safe, as we keep this // dictionary only for the duration of this "Build" call. new ReferenceEqualityComparer <IProject>()); cancellationToken.ThrowIfCancellationRequested(); var loadingContext = new FileContentsLoadingContext { FullPathChanges = fullPathChanges, LoadedTextFileCount = 0, OldFileDatabaseSnapshot = fileDatabase, UnchangedProjects = unchangedProjectSet, PartialProgressReporter = new PartialProgressReporter( TimeSpan.FromSeconds(5.0), () => { Logger.LogInfo("Creating intermedidate file database for partial progress reporting"); var database = CreateFileDatabase(entities, false, cancellationToken); onIntermadiateResult(database); }) }; // Merge old state in new state and load all missing files LoadFileContents(entities, loadingContext, cancellationToken); var result = CreateFileDatabase(entities, true, cancellationToken); onLoaded(); return(result); } } }
private FileSystemEntities ComputeFileSystemEntities(FileSystemSnapshot snapshot, CancellationToken cancellationToken) { using (new TimeElapsedLogger("Computing tables of directory names and file names from FileSystemTree", cancellationToken)) { var directories = FileSystemSnapshotVisitor.GetDirectories(snapshot); //var directoryNames = new Dictionary<DirectoryName, DirectoryData>( var directoryNames = new SlimHashTable <DirectoryName, DirectoryData>( v => v.DirectoryName, directories.Count, // Note: We can use reference equality here because the directory // names are contructed unique. new ReferenceEqualityComparer <DirectoryName>()); foreach (var kvp in directories.ToForeachEnum()) { directoryNames.Add( kvp.Value.DirectoryName, new DirectoryData(kvp.Value.DirectoryName, kvp.Value.IsSymLink)); } var files = new SlimHashTable <FileName, ProjectFileData>( v => v.FileName, directories.Count * 2, // Note: We can use reference equality here because the file names are // constructed unique and the dictionary will be discarded once we are // done building this snapshot. new FileNameReferenceEqualityComparer()); foreach (var directory in directories.ToForeachEnum()) { foreach (var fileName in directory.Value.ChildFiles.ToForeachEnum()) { if (directory.Key.IsFileSearchable(fileName)) { files.Add(fileName, new ProjectFileData(directory.Key, fileName, null)); } } } return(new FileSystemEntities { Files = files, Directories = directoryNames, ProjectHashes = snapshot.ProjectRoots.ToDictionary(x => x.Project.RootPath, x => x.Project.VersionHash) }); } }
public ProjectRootSnapshotBuilder(IFileSystem fileSystem, IFileSystemNameFactory fileSystemNameFactory, FileSystemSnapshot oldSnapshot, IProject project, IProgressTracker progress, ProjectPathChanges pathChanges, CancellationToken cancellationToken) { _fileSystem = fileSystem; _fileSystemNameFactory = fileSystemNameFactory; _oldSnapshot = oldSnapshot; _project = project; _progress = progress; _pathChanges = pathChanges; _cancellationToken = cancellationToken; }
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges, Action <IFileDatabaseSnapshot> onIntermadiateResult, CancellationToken cancellationToken) { using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) { var fileDatabase = (FileDatabaseSnapshot)previousDatabase; // Compute list of files from tree var entities = ComputeFileSystemEntities(newSnapshot); var unchangedProjects = newSnapshot .ProjectRoots.Where(x => fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) && fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash) .Select(x => x.Project); var unchangedProjectSet = new HashSet <IProject>(unchangedProjects, // Use reference equality for IProject is safe, as we keep this // dictionary only for the duration of this "Build" call. new ReferenceEqualityComparer <IProject>()); // Don't use file memoization for now, as benefit is dubvious. //IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization(); IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization(); var loadingContext = new FileContentsLoadingContext { FileContentsMemoization = fileContentsMemoization, FullPathChanges = fullPathChanges, LoadedTextFileCount = 0, OldFileDatabaseSnapshot = fileDatabase, UnchangedProjects = unchangedProjectSet, PartialProgressReporter = new PartialProgressReporter( TimeSpan.FromSeconds(5.0), () => { Logger.LogInfo("Creating intermedidate file database for partial progress reporting"); var database = CreateFileDatabse(entities); onIntermadiateResult(database); }) }; // Merge old state in new state and load all missing files LoadFileContents(entities, loadingContext, cancellationToken); return(CreateFileDatabse(entities)); } }
private IFileDatabaseSnapshot CreateWithModifiedFiles(FileSystemSnapshot fileSystemSnapshot, IList <ProjectFileName> changedFiles, UpdateFileDatabaseOptions options, CancellationToken cancellationToken) { Invariants.Assert(_inTaskQueueTask); return(_fileDatabaseSnapshotFactory.CreateIncrementalWithModifiedFiles( options.CurrentFileDatabase, fileSystemSnapshot, changedFiles, onIntermadiateResult: fileDatabase => { ActivateCurrentDatabase(fileSystemSnapshot, fileDatabase, false); OnFilesLoadingProgress(options.OperationInfo); }, onLoading: () => OnFilesLoading(options.OperationInfo), onLoaded: () => OnFilesLoaded(new FilesLoadedResult { OperationInfo = options.OperationInfo, TreeVersion = _currentFileSystemSnapshotVersion, }), cancellationToken: cancellationToken)); }
private void ComputeNewState(FileSystemSnapshot newSnapshot, FullPathChanges fullPathChanges, CancellationToken cancellationToken) { _operationProcessor.Execute(new OperationHandlers { OnBeforeExecute = info => OnFilesLoading(info), OnError = (info, error) => { _currentTreeVersion = newSnapshot.Version; OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, Error = error, TreeVersion = newSnapshot.Version }); }, Execute = info => { using (new TimeElapsedLogger("Computing new state of file database")) { var oldState = _currentFileDatabaseSnapshot; var newState = _fileDatabaseSnapshotFactory.CreateIncremental( oldState, newSnapshot, fullPathChanges, fileDatabase => { // Store and activate intermediate new state (atomic operation). _currentFileDatabaseSnapshot = fileDatabase; OnFilesLoadingProgress(info); }, cancellationToken); // Store and activate final new state (atomic operation). _currentFileDatabaseSnapshot = newState; } _currentTreeVersion = newSnapshot.Version; OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, TreeVersion = newSnapshot.Version }); } }); }
public FileSystemSnapshot Compute(IFileSystemNameFactory fileNameFactory, FileSystemSnapshot oldSnapshot, FullPathChanges pathChanges /* may be null */, IList <IProject> projects, int version, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); // cancellation using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) { // Clear file name factory intern tables if no projects. We could be // more aggressive at the expense of decreasing string interning. if (projects.Count == 0) { fileNameFactory.ClearInternedStrings(); } var projectRoots = projects .Distinct(new ProjectPathComparer()) .Select(project => { cancellationToken.ThrowIfCancellationRequested(); // cancellation var projectSnapshotBuilder = new ProjectRootSnapshotBuilder( _fileSystem, fileNameFactory, oldSnapshot, project, progress, (pathChanges == null ? null : new ProjectPathChanges(project.RootPath, pathChanges.Entries)), cancellationToken); var rootSnapshot = projectSnapshotBuilder.Build(); return(new ProjectRootSnapshot(project, rootSnapshot)); }) .OrderBy(projectRoot => projectRoot.Directory.DirectoryName) .ToReadOnlyCollection(); return(new FileSystemSnapshot(version, projectRoots)); } }
private void UpdateFileDatabase(FileSystemSnapshot fileSystemSnapshot, Func <UpdateFileDatabaseOptions, IFileDatabaseSnapshot> updater) { Invariants.Assert(_inTaskQueueTask); var options = new UpdateFileDatabaseOptions(); _operationProcessor.Execute(new OperationHandlers { OnBeforeExecute = info => { options.OperationInfo = info; options.PreviousUpdateCompleted = _previousUpdateCompleted; _previousUpdateCompleted = false; }, OnError = (info, error) => { _currentFileSystemSnapshotVersion = fileSystemSnapshot.Version; _previousUpdateCompleted = false; OnFilesLoading(info); OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, Error = error, TreeVersion = _currentFileSystemSnapshotVersion, }); }, Execute = info => { options.CurrentFileDatabase = _currentFileDatabase; var newFileDatabase = updater(options); ActivateCurrentDatabase(fileSystemSnapshot, newFileDatabase, true); OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, TreeVersion = fileSystemSnapshot.Version }); } }); }
private IEnumerable <ProjectDetails> CreateProjectsDetails(GetDatabaseDetailsRequest request, FileSystemSnapshot snapshot, IFileDatabaseSnapshot database) { return(snapshot.ProjectRoots.Select(project => GetProjectDetailsRequestHandler.CreateProjectDetails(database, project, request.MaxFilesByExtensionDetailsCount, request.MaxLargeFilesDetailsCount))); }
public static List <KeyValuePair <IProject, DirectorySnapshot> > GetDirectories(FileSystemSnapshot snapshot) { var result = new List <KeyValuePair <IProject, DirectorySnapshot> >(); VisitDirectories(snapshot, (project, directory) => { result.Add(new KeyValuePair <IProject, DirectorySnapshot>(project, directory)); }); return(result); }
public FileSystemTreeSnapshotNameFactory(FileSystemSnapshot snapshot, IFileSystemNameFactory previous) { _snapshot = snapshot; _previous = previous; }