public FileSystemTreeSnapshot Compute(
IFileSystemNameFactory fileNameFactory,
FileSystemTreeSnapshot oldSnapshot,
FullPathChanges pathChanges /* may be null */,
List <FullPath> rootFiles,
int version)
{
using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) {
var projectRoots =
rootFiles
.Select(filename => _projectDiscovery.GetProject(filename))
.Where(project => project != null)
.Distinct(new ProjectPathComparer())
.Select(project => {
var data = new ProjectProcessingData {
FileSystemNameFactory = fileNameFactory,
Project = project,
Progress = progress,
OldSnapshot = oldSnapshot,
PathChanges = (pathChanges == null ? null : new ProjectPathChanges(project.RootPath, pathChanges.Entries)),
};
var rootSnapshot = ProcessProject(data);
return(new ProjectRootSnapshot(project, rootSnapshot));
})
.OrderBy(projectRoot => projectRoot.Directory.DirectoryName)
.ToReadOnlyCollection();
return(new FileSystemTreeSnapshot(version, projectRoots));
}
}
public static void VisitDirectories(
FileSystemTreeSnapshot snapshot,
Action<IProject, DirectorySnapshot> callback) {
foreach (var project in snapshot.ProjectRoots.ToForeachEnum()) {
VisitDirectory(project.Project, project.Directory, callback);
}
}
private static DirectoryEntry BuildFileSystemTreeRoot(FileSystemTreeSnapshot fileSystemSnapshot) {
return new DirectoryEntry {
Name = null,
Data = null,
Entries = fileSystemSnapshot.ProjectRoots.Select(x => BuildDirectoryEntry(x.Directory)).Cast<FileSystemEntry>().ToList()
};
}
public static List<KeyValuePair<IProject, DirectorySnapshot>> GetDirectories(FileSystemTreeSnapshot snapshot) {
var result = new List<KeyValuePair<IProject, DirectorySnapshot>>();
VisitDirectories(snapshot, (project, directory) => {
result.Add(new KeyValuePair<IProject, DirectorySnapshot>(project, directory));
});
return result;
}
public static FileSystemTree ToIpcFileSystemTree(this FileSystemTreeSnapshot tree)
{
return(new FileSystemTree {
Version = tree.Version,
Root = BuildFileSystemTreeRoot(tree)
});
}
private static DirectoryEntry BuildFileSystemTreeRoot(FileSystemTreeSnapshot fileSystemSnapshot)
{
return(new DirectoryEntry {
Name = null,
Data = null,
Entries = fileSystemSnapshot.ProjectRoots.Select(x => BuildDirectoryEntry(x.Directory)).Cast <FileSystemEntry>().ToList()
});
}
public static IEnumerable<KeyValuePair<IProject, DirectorySnapshot>> GetDirectories(FileSystemTreeSnapshot snapshot)
{
var result = new List<KeyValuePair<IProject, DirectorySnapshot>>();
foreach (var project in snapshot.ProjectRoots) {
ProcessRoot(project, result);
}
return result;
}
public static void VisitDirectories(
FileSystemTreeSnapshot snapshot,
Action <IProject, DirectorySnapshot> callback)
{
foreach (var project in snapshot.ProjectRoots.ToForeachEnum())
{
VisitDirectory(project.Project, project.Directory, callback);
}
}
public IFileDatabase CreateIncremental(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot previousSnapshot,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action<IFileDatabase> onIntermadiateResult) {
return new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.Build(previousFileDatabase, newSnapshot, fullPathChanges, onIntermadiateResult);
}
public IFileDatabase Build(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action<IFileDatabase> onIntermadiateResult)
{
using (var logger = new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Compute list of files from tree
ComputeFileCollection(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet<IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer<IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingInfo = new FileContentsLoadingInfo {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabase = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database");
var database = this.CreateFileDatabse();
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(loadingInfo);
return CreateFileDatabse();
}
}
public FileSystemProcessor(
IFileSystemNameFactory fileSystemNameFactory,
IProjectDiscovery projectDiscovery,
IDirectoryChangeWatcherFactory directoryChangeWatcherFactory,
ITaskQueueFactory taskQueueFactory,
IFileSystemSnapshotBuilder fileSystemSnapshotBuilder,
IOperationProcessor<SnapshotComputedEventArgs> snapshotOperationProcessor)
{
_fileSystemNameFactory = fileSystemNameFactory;
_directoryChangeWatcher = directoryChangeWatcherFactory.CreateWatcher();
_fileSystemSnapshotBuilder = fileSystemSnapshotBuilder;
_snapshotOperationProcessor = snapshotOperationProcessor;
_projectDiscovery = projectDiscovery;
_taskQueue = taskQueueFactory.CreateQueue();
_directoryChangeWatcher.PathsChanged += DirectoryChangeWatcherOnPathsChanged;
_fileSystemSnapshot = FileSystemTreeSnapshot.Empty;
}
public FileSystemProcessor(
IFileSystemNameFactory fileSystemNameFactory,
IFileSystem fileSystem,
IFileSystemSnapshotBuilder fileSystemSnapshotBuilder,
IOperationProcessor operationProcessor,
IProjectDiscovery projectDiscovery,
IDirectoryChangeWatcherFactory directoryChangeWatcherFactory,
ITaskQueueFactory taskQueueFactory)
{
_fileSystemNameFactory = fileSystemNameFactory;
_fileSystem = fileSystem;
_fileSystemSnapshotBuilder = fileSystemSnapshotBuilder;
_operationProcessor = operationProcessor;
_projectDiscovery = projectDiscovery;
_taskQueue = taskQueueFactory.CreateQueue("FileSystemProcessor Task Queue");
_fileSystemSnapshot = FileSystemTreeSnapshot.Empty;
_directoryChangeWatcher = directoryChangeWatcherFactory.CreateWatcher();
_directoryChangeWatcher.PathsChanged += DirectoryChangeWatcherOnPathsChanged;
}
private void RecomputeGraph()
{
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info => OnSnapshotComputing(info),
OnError = (info, error) => OnSnapshotComputed(new SnapshotComputedResult { OperationInfo = info, Error = error }),
Execute = info => {
Logger.Log("Collecting list of files from file system.");
Logger.LogMemoryStats();
var sw = Stopwatch.StartNew();
var files = new List<FullPath>();
lock (_lock) {
ValidateKnownFiles();
files.AddRange(_addedFiles);
}
IFileSystemNameFactory fileNameFactory = _fileSystemNameFactory;
if (ReuseFileNameInstances) {
if (_fileSystemSnapshot.ProjectRoots.Count > 0) {
fileNameFactory = new FileSystemTreeSnapshotNameFactory(_fileSystemSnapshot, fileNameFactory);
}
}
var newSnapshot = _fileSystemSnapshotBuilder.Compute(fileNameFactory, files, Interlocked.Increment(ref _version));
// Monitor all the Chromium directories for changes.
var newRoots = newSnapshot.ProjectRoots
.Select(entry => entry.Directory.DirectoryName.FullPath);
_directoryChangeWatcher.WatchDirectories(newRoots);
// Update current tree atomically
FileSystemTreeSnapshot previousSnapshot;
lock (_lock) {
previousSnapshot = _fileSystemSnapshot;
_fileSystemSnapshot = newSnapshot;
}
sw.Stop();
Logger.Log(">>>>>>>> Done collecting list of files: {0:n0} files in {1:n0} directories collected in {2:n0} msec.",
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountFileEntries(x.Directory)),
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountDirectoryEntries(x.Directory)),
sw.ElapsedMilliseconds);
Logger.LogMemoryStats();
OnSnapshotComputed(new SnapshotComputedResult {
OperationInfo = info,
PreviousSnapshot = previousSnapshot,
NewSnapshot = newSnapshot
});
}
});
}
private void ComputeFileCollection(FileSystemTreeSnapshot snapshot)
{
using (new TimeElapsedLogger("Computing tables of directory names and file names from FileSystemTree")) {
var directories = FileSystemSnapshotVisitor.GetDirectories(snapshot);
var directoryNames = new Dictionary<DirectoryName, DirectoryData>(
directories.Count,
// Note: We can use reference equality here because the directory
// names are contructed unique.
new ReferenceEqualityComparer<DirectoryName>());
foreach (var kvp in directories.ToForeachEnum()) {
directoryNames.Add(
kvp.Value.DirectoryName,
new DirectoryData(kvp.Value.DirectoryName, kvp.Value.IsSymLink));
}
var files = new Dictionary<FileName, ProjectFileData>(
directories.Count * 4,
// Note: We can use reference equality here because the file names are
// constructed unique and the dictionary will be discarded once we are
// done building this snapshot.
new ReferenceEqualityComparer<FileName>());
foreach (var directory in directories.ToForeachEnum()) {
foreach (var fileName in directory.Value.ChildFiles.ToForeachEnum()) {
files.Add(fileName, new ProjectFileData(directory.Key, new FileData(fileName, null)));
}
}
_files = files;
_directories = directoryNames;
_projectHashes = snapshot.ProjectRoots.ToDictionary(
x => x.Project.RootPath,
x => x.Project.VersionHash);
}
}
public FileSystemTreeSnapshotNameFactory(FileSystemTreeSnapshot snapshot, IFileSystemNameFactory previous)
{
_snapshot = snapshot;
_previous = previous;
}
public FileSystemTreeSnapshotNameFactory(FileSystemTreeSnapshot snapshot, IFileSystemNameFactory previous)
{
_snapshot = snapshot;
_previous = previous;
}
private FileSystemTreeSnapshot ComputeNewSnapshot(FileSystemTreeSnapshot oldSnapshot, FullPathChanges pathChanges) {
using (new TimeElapsedLogger("Computing snapshot delta from list of file changes")) {
// Get list of currently registered files.
var rootFiles = new List<FullPath>();
lock (_lock) {
ValidateKnownFiles();
rootFiles.AddRange(_registeredFiles);
}
// file name factory
var fileNameFactory = _fileSystemNameFactory;
if (ReuseFileNameInstances) {
if (_fileSystemSnapshot.ProjectRoots.Count > 0) {
fileNameFactory = new FileSystemTreeSnapshotNameFactory(_fileSystemSnapshot, fileNameFactory);
}
}
// Compute new snapshot
var newSnapshot = _fileSystemSnapshotBuilder.Compute(
fileNameFactory,
oldSnapshot,
pathChanges,
rootFiles,
Interlocked.Increment(ref _version));
// Monitor all the Chromium directories for changes.
var newRoots = newSnapshot.ProjectRoots
.Select(entry => entry.Directory.DirectoryName.FullPath);
_directoryChangeWatcher.WatchDirectories(newRoots);
return newSnapshot;
}
}
public static IEnumerable <KeyValuePair <IProject, DirectorySnapshot> > GetDirectories(FileSystemTreeSnapshot snapshot)
{
var result = new List <KeyValuePair <IProject, DirectorySnapshot> >();
foreach (var project in snapshot.ProjectRoots)
{
ProcessRoot(project, result);
}
return(result);
}
public static List <KeyValuePair <IProject, DirectorySnapshot> > GetDirectories(FileSystemTreeSnapshot snapshot)
{
var result = new List <KeyValuePair <IProject, DirectorySnapshot> >();
VisitDirectories(snapshot, (project, directory) => {
result.Add(new KeyValuePair <IProject, DirectorySnapshot>(project, directory));
});
return(result);
}
public IFileDatabase CreateIncremental(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot)
{
return new FileDatabaseBuilder(_fileContentsFactory, _progressTrackerFactory).Build(previousFileDatabase, newSnapshot);
}
private void RecomputeGraph(FullPathChanges pathChanges) {
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
OnSnapshotComputing(info),
OnError = (info, error) =>
OnSnapshotComputed(new SnapshotComputedResult {
OperationInfo = info,
Error = error
}),
Execute = info => {
// Compute and assign new snapshot
var oldSnapshot = _fileSystemSnapshot;
var newSnapshot = ComputeNewSnapshot(oldSnapshot, pathChanges);
// Update of new tree (assert calls are serialized).
Debug.Assert(ReferenceEquals(oldSnapshot, _fileSystemSnapshot));
_fileSystemSnapshot = newSnapshot;
if (Logger.Info) {
Logger.LogInfo("+++++++++++ Collected {0:n0} files in {1:n0} directories",
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountFileEntries(x.Directory)),
newSnapshot.ProjectRoots.Aggregate(0, (acc, x) => acc + CountDirectoryEntries(x.Directory)));
}
// Post event
OnSnapshotComputed(new SnapshotComputedResult {
OperationInfo = info,
PreviousSnapshot = oldSnapshot,
FullPathChanges = pathChanges,
NewSnapshot = newSnapshot
});
}
});
}
