private void LoadFileContents(FileSystemEntities entities, FileContentsLoadingContext loadingContext, CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Loading file contents from disk")) {
using (var progress = _progressTrackerFactory.CreateTracker(entities.Files.Count)) {
entities.Files.AsParallelWrapper().ForAll(fileEntry => {
Debug.Assert(fileEntry.Value.FileWithContents.Contents == null);
if (progress.Step())
{
progress.DisplayProgress((i, n) =>
string.Format("Reading file {0:n0} of {1:n0}: {2}", i, n, fileEntry.Value.FileName.FullPath));
// Check for cancellation
if (cancellationToken.IsCancellationRequested)
{
loadingContext.PartialProgressReporter.ReportProgressNow();
cancellationToken.ThrowIfCancellationRequested();
}
}
var contents = LoadSingleFileContents(entities, loadingContext, fileEntry.Value);
if (contents != null)
{
fileEntry.Value.FileWithContents.UpdateContents(contents);
}
});
}
}
Logger.LogInfo("Loaded {0:n0} text files from disk, skipped {1:n0} binary files.",
loadingContext.LoadedTextFileCount,
loadingContext.LoadedBinaryFileCount);
}
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase,
FileSystemSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action onLoading,
Action onLoaded,
Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot",
cancellationToken, InfoLogger.Instance)) {
using (var progress = _progressTrackerFactory.CreateIndeterminateTracker()) {
onLoading();
progress.DisplayProgress((i, n) => "Preparing list of files to load from disk");
var fileDatabase = (FileDatabaseSnapshot)previousDatabase;
// Compute list of files from tree
var entities = ComputeFileSystemEntities(newSnapshot, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
cancellationToken.ThrowIfCancellationRequested();
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
cancellationToken.ThrowIfCancellationRequested();
var loadingContext = new FileContentsLoadingContext {
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabaseSnapshot = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database for partial progress reporting");
var database = CreateFileDatabase(entities, false, cancellationToken);
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(entities, loadingContext, cancellationToken);
var result = CreateFileDatabase(entities, true, cancellationToken);
onLoaded();
return(result);
}
}
}
private FileContents LoadSingleFileContents(
FileSystemEntities entities,
FileContentsLoadingContext loadingContext,
ProjectFileData projectFileData)
{
var fileName = projectFileData.FileName;
var oldFileData = loadingContext.OldFileDatabaseSnapshot.Files.GetValue(fileName);
// If the file was never loaded before, just load it
if (oldFileData == null || oldFileData.Contents == null)
{
return(LoadSingleFileContentsWorker(loadingContext, projectFileData));
}
bool isSearchable;
// If the project configuration is unchanged from the previous file
// database (and the file was present in it), then it is certainly
// searchable, so no need to make an expensive call to "IsSearchable"
// again.
if (loadingContext.UnchangedProjects.Contains(projectFileData.Project))
{
isSearchable = true;
}
else
{
// If the file is not searchable in the current project, it should be
// ignored too. Note that "IsSearachable" is a somewhat expensive
// operation, as the filename is checked against potentially many glob
// patterns.
isSearchable = projectFileData.IsSearchable;
}
if (!isSearchable)
{
return(null);
}
// If the file has not changed since the previous snapshot, we can re-use
// the former file contents snapshot.
if (IsFileContentsUpToDate(entities, loadingContext.FullPathChanges, oldFileData))
{
return(oldFileData.Contents);
}
return(LoadSingleFileContentsWorker(loadingContext, projectFileData));
}
public IFileDatabaseSnapshot Build(IFileDatabaseSnapshot previousDatabase, FileSystemSnapshot newSnapshot,
FullPathChanges fullPathChanges, Action <IFileDatabaseSnapshot> onIntermadiateResult,
CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabaseSnapshot)previousDatabase;
// Compute list of files from tree
var entities = ComputeFileSystemEntities(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingContext = new FileContentsLoadingContext {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabaseSnapshot = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database for partial progress reporting");
var database = CreateFileDatabse(entities);
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(entities, loadingContext, cancellationToken);
return(CreateFileDatabse(entities));
}
}
private FileContents LoadSingleFileContentsWorker(
FileContentsLoadingContext loadingContext,
ProjectFileData projectFileData)
{
// If project configuration has not changed, the file is still not
// searchable, irrelevant to calling "IsSearchable".
if (loadingContext.FullPathChanges != null)
{
if (loadingContext.FullPathChanges.ShouldSkipLoadFileContents(projectFileData.FileName.FullPath))
{
if (loadingContext.UnchangedProjects.Contains(projectFileData.Project))
{
return(null);
}
}
}
// This is an expensive call, hopefully avoided by the code above.
if (!projectFileData.IsSearchable)
{
return(null);
}
loadingContext.PartialProgressReporter.ReportProgress();
var fileContents = _fileContentsFactory.ReadFileContents(projectFileData.FileName.FullPath);
if (fileContents is BinaryFileContents)
{
Interlocked.Increment(ref loadingContext.LoadedBinaryFileCount);
}
else
{
Interlocked.Increment(ref loadingContext.LoadedTextFileCount);
}
return(loadingContext.FileContentsMemoization.Get(projectFileData.FileName, fileContents));
}
