private void ComputeFileCollection(FileSystemTreeSnapshot snapshot) { Logger.Log("Computing list of searchable files from FileSystemTree."); var ssw = new ScopedStopWatch(); var directories = FileSystemSnapshotVisitor.GetDirectories(snapshot).ToList(); ssw.Step(sw => Logger.Log("Done flattening file system tree snapshot in {0:n0} msec.", sw.ElapsedMilliseconds)); var directoryNames = directories .Select(x => x.Value.DirectoryName) .ToArray(); ssw.Step(sw => Logger.Log("Done creating array of directory names in {0:n0} msec.", sw.ElapsedMilliseconds)); var searchableFiles = directories .AsParallel() .SelectMany(x => x.Value.Files.Select(y => KeyValuePair.Create(x.Key, y))) .Select(x => new FileInfo(new FileData(x.Value, null), x.Key.IsFileSearchable(x.Value))) .ToList(); ssw.Step(sw => Logger.Log("Done creating list of files in {0:n0} msec.", sw.ElapsedMilliseconds)); var files = searchableFiles .ToDictionary(x => x.FileData.FileName, x => x); ssw.Step(sw => Logger.Log("Done creating dictionary of files in {0:n0} msec.", sw.ElapsedMilliseconds)); Logger.Log("Done computing list of searchable files from FileSystemTree."); Logger.LogMemoryStats(); _files = new FileNameDictionary <FileInfo>(files); _directoryNames = directoryNames; }
private void RecomputeGraph(FullPathChanges pathChanges) { _operationProcessor.Execute(new OperationHandlers { OnBeforeExecute = info => OnSnapshotComputing(info), OnError = (info, error) => OnSnapshotComputed(new SnapshotComputedResult { OperationInfo = info, Error = error }), Execute = info => { // Compute and assign new snapshot var oldSnapshot = _fileSystemSnapshot; var newSnapshot = ComputeNewSnapshot(oldSnapshot, pathChanges); // Update of new tree (assert calls are serialized). Debug.Assert(ReferenceEquals(oldSnapshot, _fileSystemSnapshot)); _fileSystemSnapshot = newSnapshot; if (Logger.Info) { Logger.LogInfo("+++++++++++ Collected {0:n0} files in {1:n0} directories", newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountFileEntries(x.Directory)), newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountDirectoryEntries(x.Directory))); } // Post event OnSnapshotComputed(new SnapshotComputedResult { OperationInfo = info, PreviousSnapshot = oldSnapshot, FullPathChanges = pathChanges, NewSnapshot = newSnapshot }); } }); }
private void ComputeNewState(FileSystemTreeSnapshot newSnapshot) { _fileLoadingOperationProcessor.Execute(new OperationInfo <OperationResultEventArgs> { OnBeforeExecute = args => OnFilesLoading(args), OnAfterExecute = args => OnFilesLoaded(args), Execute = _ => { Logger.Log("Computing new state of file database from file system tree."); var sw = Stopwatch.StartNew(); var oldState = _currentFileDatabase; var newState = _fileDatabaseFactory.CreateIncremental(oldState, newSnapshot); sw.Stop(); Logger.Log(">>>>>>>> Done computing new state of file database from file system tree in {0:n0} msec.", sw.ElapsedMilliseconds); Logger.LogMemoryStats(); // Swap states lock (_lock) { _currentFileDatabase = newState; } return(new OperationResultEventArgs()); } }); }
public IFileDatabase CreateIncremental( IFileDatabase previousFileDatabase, FileSystemTreeSnapshot previousSnapshot, FileSystemTreeSnapshot newSnapshot, FullPathChanges fullPathChanges, Action <IFileDatabase> onIntermadiateResult) { return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory) .Build(previousFileDatabase, newSnapshot, fullPathChanges, onIntermadiateResult)); }
private void RecomputeGraph() { _snapshotOperationProcessor.Execute(new OperationInfo <SnapshotComputedEventArgs> { OnBeforeExecute = args => OnSnapshotComputing(args), OnAfterExecute = args => OnSnapshotComputed(args), Execute = args => { Logger.Log("Collecting list of files from file system."); Logger.LogMemoryStats(); var sw = Stopwatch.StartNew(); var files = new List <FullPathName>(); lock (_lock) { ValidateKnownFiles(); files.AddRange(_addedFiles); } IFileSystemNameFactory fileNameFactory = _fileSystemNameFactory; if (ReuseFileNameInstances) { if (_fileSystemSnapshot.ProjectRoots.Count > 0) { fileNameFactory = new FileSystemTreeSnapshotNameFactory(_fileSystemSnapshot, fileNameFactory); } } var newSnapshot = _fileSystemSnapshotBuilder.Compute(fileNameFactory, files, Interlocked.Increment(ref _version)); // Monitor all the Chromium directories for changes. var newRoots = newSnapshot.ProjectRoots .Select(entry => entry.Directory.DirectoryName); _directoryChangeWatcher.WatchDirectories(newRoots); // Update current tree atomically FileSystemTreeSnapshot previousSnapshot; lock (_lock) { previousSnapshot = _fileSystemSnapshot; _fileSystemSnapshot = newSnapshot; } sw.Stop(); Logger.Log(">>>>>>>> Done collecting list of files: {0:n0} files in {1:n0} directories collected in {2:n0} msec.", newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountFileEntries(x.Directory)), newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountDirectoryEntries(x.Directory)), sw.ElapsedMilliseconds); Logger.LogMemoryStats(); return(new SnapshotComputedEventArgs { PreviousSnapshot = previousSnapshot, NewSnapshot = newSnapshot }); } }); }
public IFileDatabase Build( IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot, FullPathChanges fullPathChanges, Action <IFileDatabase> onIntermadiateResult) { using (var logger = new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) { var fileDatabase = (FileDatabase)previousFileDatabase; // Compute list of files from tree ComputeFileCollection(newSnapshot); var unchangedProjects = newSnapshot .ProjectRoots.Where(x => fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) && fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash) .Select(x => x.Project); var unchangedProjectSet = new HashSet <IProject>(unchangedProjects, // Use reference equality for IProject is safe, as we keep this // dictionary only for the duration of this "Build" call. new ReferenceEqualityComparer <IProject>()); // Don't use file memoization for now, as benefit is dubvious. //IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization(); IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization(); var loadingInfo = new FileContentsLoadingInfo { FileContentsMemoization = fileContentsMemoization, FullPathChanges = fullPathChanges, LoadedTextFileCount = 0, OldFileDatabase = fileDatabase, UnchangedProjects = unchangedProjectSet, PartialProgressReporter = new PartialProgressReporter( TimeSpan.FromSeconds(5.0), () => { Logger.LogInfo("Creating intermedidate file database"); var database = this.CreateFileDatabse(); onIntermadiateResult(database); }) }; // Merge old state in new state and load all missing files LoadFileContents(loadingInfo); return(CreateFileDatabse()); } }
public FileSystemProcessor( IFileSystemNameFactory fileSystemNameFactory, IProjectDiscovery projectDiscovery, IDirectoryChangeWatcherFactory directoryChangeWatcherFactory, ITaskQueueFactory taskQueueFactory, IFileSystemSnapshotBuilder fileSystemSnapshotBuilder, IOperationProcessor <SnapshotComputedEventArgs> snapshotOperationProcessor) { _fileSystemNameFactory = fileSystemNameFactory; _directoryChangeWatcher = directoryChangeWatcherFactory.CreateWatcher(); _fileSystemSnapshotBuilder = fileSystemSnapshotBuilder; _snapshotOperationProcessor = snapshotOperationProcessor; _projectDiscovery = projectDiscovery; _taskQueue = taskQueueFactory.CreateQueue(); _directoryChangeWatcher.PathsChanged += DirectoryChangeWatcherOnPathsChanged; _fileSystemSnapshot = FileSystemTreeSnapshot.Empty; }
private void ComputeNewState( FileSystemTreeSnapshot previousSnapshot, FileSystemTreeSnapshot newSnapshot, FullPathChanges fullPathChanges) { _operationProcessor.Execute(new OperationHandlers { OnBeforeExecute = info => OnFilesLoading(info), OnError = (info, error) => { _currentTreeVersion = newSnapshot.Version; OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, Error = error, TreeVersion = newSnapshot.Version }); }, Execute = info => { using (new TimeElapsedLogger("Computing new state of file database")) { var oldState = _currentFileDatabase; var newState = _fileDatabaseFactory.CreateIncremental( oldState, previousSnapshot, newSnapshot, fullPathChanges, fileDatabase => { // Store and activate intermediate new state (atomic operation). _currentFileDatabase = fileDatabase; OnFilesLoadingProgress(info); }); // Store and activate final new state (atomic operation). _currentFileDatabase = newState; } _currentTreeVersion = newSnapshot.Version; OnFilesLoaded(new FilesLoadedResult { OperationInfo = info, TreeVersion = newSnapshot.Version }); } }); }
public FileSystemProcessor( IFileSystemNameFactory fileSystemNameFactory, IFileSystem fileSystem, IFileSystemSnapshotBuilder fileSystemSnapshotBuilder, IOperationProcessor operationProcessor, IProjectDiscovery projectDiscovery, IDirectoryChangeWatcherFactory directoryChangeWatcherFactory, ITaskQueueFactory taskQueueFactory) { _fileSystemNameFactory = fileSystemNameFactory; _fileSystem = fileSystem; _fileSystemSnapshotBuilder = fileSystemSnapshotBuilder; _operationProcessor = operationProcessor; _projectDiscovery = projectDiscovery; _taskQueue = taskQueueFactory.CreateQueue("FileSystemProcessor Task Queue"); _fileSystemSnapshot = FileSystemTreeSnapshot.Empty; _directoryChangeWatcher = directoryChangeWatcherFactory.CreateWatcher(); _directoryChangeWatcher.PathsChanged += DirectoryChangeWatcherOnPathsChanged; _directoryChangeWatcher.Error += DirectoryChangeWatcherOnError; }
private void ComputeFileCollection(FileSystemTreeSnapshot snapshot) { using (new TimeElapsedLogger("Computing tables of directory names and file names from FileSystemTree")) { var directories = FileSystemSnapshotVisitor.GetDirectories(snapshot); var directoryNames = new Dictionary <DirectoryName, DirectoryData>( directories.Count, // Note: We can use reference equality here because the directory // names are contructed unique. new ReferenceEqualityComparer <DirectoryName>()); foreach (var kvp in directories.ToForeachEnum()) { directoryNames.Add( kvp.Value.DirectoryName, new DirectoryData(kvp.Value.DirectoryName, kvp.Value.IsSymLink)); } var files = new Dictionary <FileName, ProjectFileData>( directories.Count * 4, // Note: We can use reference equality here because the file names are // constructed unique and the dictionary will be discarded once we are // done building this snapshot. new ReferenceEqualityComparer <FileName>()); foreach (var directory in directories.ToForeachEnum()) { foreach (var fileName in directory.Value.ChildFiles.ToForeachEnum()) { files.Add(fileName, new ProjectFileData(directory.Key, new FileData(fileName, null))); } } _files = files; _directories = directoryNames; _projectHashes = snapshot.ProjectRoots.ToDictionary( x => x.Project.RootPath, x => x.Project.VersionHash); } }
/// <summary> /// Prepares this instance for searches by computing various snapshots from /// the previous <see cref="FileDatabase"/> snapshot and the new current /// <see cref="FileSystemTreeSnapshot"/> instance. /// </summary> private FileDatabase ComputeFileDatabase(FileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot) { if (previousFileDatabase == null) { throw new ArgumentNullException("previousFileDatabase"); } if (newSnapshot == null) { throw new ArgumentNullException("newSnapshot"); } // Compute list of files from tree ComputeFileCollection(newSnapshot); // Merge old state in new state TransferUnchangedFileContents(previousFileDatabase); // Load file contents into newState ReadMissingFileContents(); return(CreateFileDatabse()); }
private FileSystemTreeSnapshot ComputeNewSnapshot(FileSystemTreeSnapshot oldSnapshot, FullPathChanges pathChanges) { using (new TimeElapsedLogger("Computing snapshot delta from list of file changes")) { // Get list of currently registered files. var rootFiles = new List <FullPath>(); lock (_lock) { ValidateKnownFiles(); rootFiles.AddRange(_registeredFiles); } // file name factory var fileNameFactory = _fileSystemNameFactory; if (ReuseFileNameInstances) { if (_fileSystemSnapshot.ProjectRoots.Count > 0) { fileNameFactory = new FileSystemTreeSnapshotNameFactory(_fileSystemSnapshot, fileNameFactory); } } // Compute new snapshot var newSnapshot = _fileSystemSnapshotBuilder.Compute( fileNameFactory, oldSnapshot, pathChanges, rootFiles, Interlocked.Increment(ref _version)); // Monitor all the Chromium directories for changes. var newRoots = newSnapshot.ProjectRoots .Select(entry => entry.Directory.DirectoryName.FullPath); _directoryChangeWatcher.WatchDirectories(newRoots); return(newSnapshot); } }
public IFileDatabase Build(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot) { return(ComputeFileDatabase((FileDatabase)previousFileDatabase, newSnapshot)); }
public IFileDatabase CreateIncremental(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot) { return(new FileDatabaseBuilder(_fileContentsFactory, _progressTrackerFactory).Build(previousFileDatabase, newSnapshot)); }