private void ComputeFileCollection(FileSystemTreeSnapshot snapshot)
{
Logger.Log("Computing list of searchable files from FileSystemTree.");
var ssw = new ScopedStopWatch();
var directories = FileSystemSnapshotVisitor.GetDirectories(snapshot).ToList();
ssw.Step(sw => Logger.Log("Done flattening file system tree snapshot in {0:n0} msec.", sw.ElapsedMilliseconds));
var directoryNames = directories
.Select(x => x.Value.DirectoryName)
.ToArray();
ssw.Step(sw => Logger.Log("Done creating array of directory names in {0:n0} msec.", sw.ElapsedMilliseconds));
var searchableFiles = directories
.AsParallel()
.SelectMany(x => x.Value.Files.Select(y => KeyValuePair.Create(x.Key, y)))
.Select(x => new FileInfo(new FileData(x.Value, null), x.Key.IsFileSearchable(x.Value)))
.ToList();
ssw.Step(sw => Logger.Log("Done creating list of files in {0:n0} msec.", sw.ElapsedMilliseconds));
var files = searchableFiles
.ToDictionary(x => x.FileData.FileName, x => x);
ssw.Step(sw => Logger.Log("Done creating dictionary of files in {0:n0} msec.", sw.ElapsedMilliseconds));
Logger.Log("Done computing list of searchable files from FileSystemTree.");
Logger.LogMemoryStats();
_files = new FileNameDictionary <FileInfo>(files);
_directoryNames = directoryNames;
}
private void RecomputeGraph(FullPathChanges pathChanges)
{
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
OnSnapshotComputing(info),
OnError = (info, error) =>
OnSnapshotComputed(new SnapshotComputedResult {
OperationInfo = info,
Error = error
}),
Execute = info => {
// Compute and assign new snapshot
var oldSnapshot = _fileSystemSnapshot;
var newSnapshot = ComputeNewSnapshot(oldSnapshot, pathChanges);
// Update of new tree (assert calls are serialized).
Debug.Assert(ReferenceEquals(oldSnapshot, _fileSystemSnapshot));
_fileSystemSnapshot = newSnapshot;
if (Logger.Info)
{
Logger.LogInfo("+++++++++++ Collected {0:n0} files in {1:n0} directories",
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountFileEntries(x.Directory)),
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountDirectoryEntries(x.Directory)));
}
// Post event
OnSnapshotComputed(new SnapshotComputedResult {
OperationInfo = info,
PreviousSnapshot = oldSnapshot,
FullPathChanges = pathChanges,
NewSnapshot = newSnapshot
});
}
});
}
private void ComputeNewState(FileSystemTreeSnapshot newSnapshot)
{
_fileLoadingOperationProcessor.Execute(new OperationInfo <OperationResultEventArgs> {
OnBeforeExecute = args => OnFilesLoading(args),
OnAfterExecute = args => OnFilesLoaded(args),
Execute = _ => {
Logger.Log("Computing new state of file database from file system tree.");
var sw = Stopwatch.StartNew();
var oldState = _currentFileDatabase;
var newState = _fileDatabaseFactory.CreateIncremental(oldState, newSnapshot);
sw.Stop();
Logger.Log(">>>>>>>> Done computing new state of file database from file system tree in {0:n0} msec.",
sw.ElapsedMilliseconds);
Logger.LogMemoryStats();
// Swap states
lock (_lock) {
_currentFileDatabase = newState;
}
return(new OperationResultEventArgs());
}
});
}
public IFileDatabase CreateIncremental(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot previousSnapshot,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action <IFileDatabase> onIntermadiateResult)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.Build(previousFileDatabase, newSnapshot, fullPathChanges, onIntermadiateResult));
}
private void RecomputeGraph()
{
_snapshotOperationProcessor.Execute(new OperationInfo <SnapshotComputedEventArgs> {
OnBeforeExecute = args => OnSnapshotComputing(args),
OnAfterExecute = args => OnSnapshotComputed(args),
Execute = args => {
Logger.Log("Collecting list of files from file system.");
Logger.LogMemoryStats();
var sw = Stopwatch.StartNew();
var files = new List <FullPathName>();
lock (_lock) {
ValidateKnownFiles();
files.AddRange(_addedFiles);
}
IFileSystemNameFactory fileNameFactory = _fileSystemNameFactory;
if (ReuseFileNameInstances)
{
if (_fileSystemSnapshot.ProjectRoots.Count > 0)
{
fileNameFactory = new FileSystemTreeSnapshotNameFactory(_fileSystemSnapshot, fileNameFactory);
}
}
var newSnapshot = _fileSystemSnapshotBuilder.Compute(fileNameFactory, files, Interlocked.Increment(ref _version));
// Monitor all the Chromium directories for changes.
var newRoots = newSnapshot.ProjectRoots
.Select(entry => entry.Directory.DirectoryName);
_directoryChangeWatcher.WatchDirectories(newRoots);
// Update current tree atomically
FileSystemTreeSnapshot previousSnapshot;
lock (_lock) {
previousSnapshot = _fileSystemSnapshot;
_fileSystemSnapshot = newSnapshot;
}
sw.Stop();
Logger.Log(">>>>>>>> Done collecting list of files: {0:n0} files in {1:n0} directories collected in {2:n0} msec.",
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountFileEntries(x.Directory)),
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountDirectoryEntries(x.Directory)),
sw.ElapsedMilliseconds);
Logger.LogMemoryStats();
return(new SnapshotComputedEventArgs {
PreviousSnapshot = previousSnapshot,
NewSnapshot = newSnapshot
});
}
});
}
public IFileDatabase Build(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action <IFileDatabase> onIntermadiateResult)
{
using (var logger = new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Compute list of files from tree
ComputeFileCollection(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingInfo = new FileContentsLoadingInfo {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabase = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database");
var database = this.CreateFileDatabse();
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(loadingInfo);
return(CreateFileDatabse());
}
}
public FileSystemProcessor(
IFileSystemNameFactory fileSystemNameFactory,
IProjectDiscovery projectDiscovery,
IDirectoryChangeWatcherFactory directoryChangeWatcherFactory,
ITaskQueueFactory taskQueueFactory,
IFileSystemSnapshotBuilder fileSystemSnapshotBuilder,
IOperationProcessor <SnapshotComputedEventArgs> snapshotOperationProcessor)
{
_fileSystemNameFactory = fileSystemNameFactory;
_directoryChangeWatcher = directoryChangeWatcherFactory.CreateWatcher();
_fileSystemSnapshotBuilder = fileSystemSnapshotBuilder;
_snapshotOperationProcessor = snapshotOperationProcessor;
_projectDiscovery = projectDiscovery;
_taskQueue = taskQueueFactory.CreateQueue();
_directoryChangeWatcher.PathsChanged += DirectoryChangeWatcherOnPathsChanged;
_fileSystemSnapshot = FileSystemTreeSnapshot.Empty;
}
private void ComputeNewState(
FileSystemTreeSnapshot previousSnapshot,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges)
{
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
OnFilesLoading(info),
OnError = (info, error) => {
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
Error = error,
TreeVersion = newSnapshot.Version
});
},
Execute = info => {
using (new TimeElapsedLogger("Computing new state of file database")) {
var oldState = _currentFileDatabase;
var newState = _fileDatabaseFactory.CreateIncremental(
oldState,
previousSnapshot,
newSnapshot,
fullPathChanges,
fileDatabase => {
// Store and activate intermediate new state (atomic operation).
_currentFileDatabase = fileDatabase;
OnFilesLoadingProgress(info);
});
// Store and activate final new state (atomic operation).
_currentFileDatabase = newState;
}
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
TreeVersion = newSnapshot.Version
});
}
});
}
public FileSystemProcessor(
IFileSystemNameFactory fileSystemNameFactory,
IFileSystem fileSystem,
IFileSystemSnapshotBuilder fileSystemSnapshotBuilder,
IOperationProcessor operationProcessor,
IProjectDiscovery projectDiscovery,
IDirectoryChangeWatcherFactory directoryChangeWatcherFactory,
ITaskQueueFactory taskQueueFactory)
{
_fileSystemNameFactory = fileSystemNameFactory;
_fileSystem = fileSystem;
_fileSystemSnapshotBuilder = fileSystemSnapshotBuilder;
_operationProcessor = operationProcessor;
_projectDiscovery = projectDiscovery;
_taskQueue = taskQueueFactory.CreateQueue("FileSystemProcessor Task Queue");
_fileSystemSnapshot = FileSystemTreeSnapshot.Empty;
_directoryChangeWatcher = directoryChangeWatcherFactory.CreateWatcher();
_directoryChangeWatcher.PathsChanged += DirectoryChangeWatcherOnPathsChanged;
_directoryChangeWatcher.Error += DirectoryChangeWatcherOnError;
}
private void ComputeFileCollection(FileSystemTreeSnapshot snapshot)
{
using (new TimeElapsedLogger("Computing tables of directory names and file names from FileSystemTree")) {
var directories = FileSystemSnapshotVisitor.GetDirectories(snapshot);
var directoryNames = new Dictionary <DirectoryName, DirectoryData>(
directories.Count,
// Note: We can use reference equality here because the directory
// names are contructed unique.
new ReferenceEqualityComparer <DirectoryName>());
foreach (var kvp in directories.ToForeachEnum())
{
directoryNames.Add(
kvp.Value.DirectoryName,
new DirectoryData(kvp.Value.DirectoryName, kvp.Value.IsSymLink));
}
var files = new Dictionary <FileName, ProjectFileData>(
directories.Count * 4,
// Note: We can use reference equality here because the file names are
// constructed unique and the dictionary will be discarded once we are
// done building this snapshot.
new ReferenceEqualityComparer <FileName>());
foreach (var directory in directories.ToForeachEnum())
{
foreach (var fileName in directory.Value.ChildFiles.ToForeachEnum())
{
files.Add(fileName, new ProjectFileData(directory.Key, new FileData(fileName, null)));
}
}
_files = files;
_directories = directoryNames;
_projectHashes = snapshot.ProjectRoots.ToDictionary(
x => x.Project.RootPath,
x => x.Project.VersionHash);
}
}
/// <summary>
/// Prepares this instance for searches by computing various snapshots from
/// the previous <see cref="FileDatabase"/> snapshot and the new current
/// <see cref="FileSystemTreeSnapshot"/> instance.
/// </summary>
private FileDatabase ComputeFileDatabase(FileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot)
{
if (previousFileDatabase == null)
{
throw new ArgumentNullException("previousFileDatabase");
}
if (newSnapshot == null)
{
throw new ArgumentNullException("newSnapshot");
}
// Compute list of files from tree
ComputeFileCollection(newSnapshot);
// Merge old state in new state
TransferUnchangedFileContents(previousFileDatabase);
// Load file contents into newState
ReadMissingFileContents();
return(CreateFileDatabse());
}
private FileSystemTreeSnapshot ComputeNewSnapshot(FileSystemTreeSnapshot oldSnapshot, FullPathChanges pathChanges)
{
using (new TimeElapsedLogger("Computing snapshot delta from list of file changes")) {
// Get list of currently registered files.
var rootFiles = new List <FullPath>();
lock (_lock) {
ValidateKnownFiles();
rootFiles.AddRange(_registeredFiles);
}
// file name factory
var fileNameFactory = _fileSystemNameFactory;
if (ReuseFileNameInstances)
{
if (_fileSystemSnapshot.ProjectRoots.Count > 0)
{
fileNameFactory = new FileSystemTreeSnapshotNameFactory(_fileSystemSnapshot, fileNameFactory);
}
}
// Compute new snapshot
var newSnapshot = _fileSystemSnapshotBuilder.Compute(
fileNameFactory,
oldSnapshot,
pathChanges,
rootFiles,
Interlocked.Increment(ref _version));
// Monitor all the Chromium directories for changes.
var newRoots = newSnapshot.ProjectRoots
.Select(entry => entry.Directory.DirectoryName.FullPath);
_directoryChangeWatcher.WatchDirectories(newRoots);
return(newSnapshot);
}
}
public IFileDatabase Build(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot)
{
return(ComputeFileDatabase((FileDatabase)previousFileDatabase, newSnapshot));
}
public IFileDatabase CreateIncremental(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot)
{
return(new FileDatabaseBuilder(_fileContentsFactory, _progressTrackerFactory).Build(previousFileDatabase, newSnapshot));
}
