private void LoadFileContents(FileContentsLoadingInfo loadingInfo)
{
using (var logger = new TimeElapsedLogger("Loading file contents from disk")) {
using (var progress = _progressTrackerFactory.CreateTracker(_files.Count)) {
_files.AsParallel().ForAll(fileEntry => {
Debug.Assert(fileEntry.Value.FileData.Contents == null);
if (progress.Step())
{
progress.DisplayProgress((i, n) =>
string.Format("Reading file {0:n0} of {1:n0}: {2}", i, n, fileEntry.Value.FileName.FullPath));
}
var contents = LoadSingleFileContents(loadingInfo, fileEntry.Value);
if (contents != null)
{
fileEntry.Value.FileData.UpdateContents(contents);
}
});
}
}
Logger.LogInfo("Loaded {0:n0} text files from disk, skipped {1:n0} binary files.",
loadingInfo.LoadedTextFileCount,
loadingInfo.LoadedBinaryFileCount);
}
public IFileDatabase Build(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action <IFileDatabase> onIntermadiateResult)
{
using (var logger = new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Compute list of files from tree
ComputeFileCollection(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingInfo = new FileContentsLoadingInfo {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabase = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database");
var database = this.CreateFileDatabse();
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(loadingInfo);
return(CreateFileDatabse());
}
}
private FileContents LoadSingleFileContents(
FileContentsLoadingInfo loadingInfo,
ProjectFileData projectFileData)
{
var fileName = projectFileData.FileName;
var oldFileData = loadingInfo.OldFileDatabase.Files.GetValue(fileName);
// If the file was never loaded before, just load it
if (oldFileData == null || oldFileData.Contents == null)
{
return(LoadSingleFileContentsWorker(loadingInfo, projectFileData));
}
bool isSearchable;
// If the project configuration is unchanged from the previous file
// database (and the file was present in it), then it is certainly
// searchable, so no need to make an expensive call to "IsSearchable"
// again.
if (loadingInfo.UnchangedProjects.Contains(projectFileData.Project))
{
isSearchable = true;
}
else
{
// If the file is not searchable in the current project, it should be
// ignored too. Note that "IsSearachable" is a somewhat expensive
// operation, as the filename is checked against potentially many glob
// patterns.
isSearchable = projectFileData.IsSearchable;
}
if (!isSearchable)
{
return(null);
}
// If the file has not changed since the previous snapshot, we can re-use
// the former file contents snapshot.
if (IsFileContentsUpToDate(loadingInfo.FullPathChanges, oldFileData))
{
return(oldFileData.Contents);
}
return(LoadSingleFileContentsWorker(loadingInfo, projectFileData));
}
private FileContents LoadSingleFileContentsWorker(
FileContentsLoadingInfo loadingInfo,
ProjectFileData projectFileData)
{
// If project configuration has not changed, the file is still not
// searchable, irrelevant to calling "IsSearchable".
if (loadingInfo.FullPathChanges != null)
{
if (loadingInfo.FullPathChanges.GetPathChangeKind(projectFileData.FileName.FullPath) == PathChangeKind.None)
{
if (loadingInfo.UnchangedProjects.Contains(projectFileData.Project))
{
return(null);
}
}
}
// This is an expensive call, hopefully avoided by the code above.
if (!projectFileData.IsSearchable)
{
return(null);
}
loadingInfo.PartialProgressReporter.ReportProgress();
var fileContents = _fileContentsFactory.ReadFileContents(projectFileData.FileName.FullPath);
if (fileContents is BinaryFileContents)
{
Interlocked.Increment(ref loadingInfo.LoadedBinaryFileCount);
}
else
{
Interlocked.Increment(ref loadingInfo.LoadedTextFileCount);
}
return(loadingInfo.FileContentsMemoization.Get(projectFileData.FileName, fileContents));
}
public IFileDatabase Build(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action<IFileDatabase> onIntermadiateResult)
{
using (var logger = new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Compute list of files from tree
ComputeFileCollection(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet<IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer<IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingInfo = new FileContentsLoadingInfo {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabase = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database");
var database = this.CreateFileDatabse();
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(loadingInfo);
return CreateFileDatabse();
}
}
private FileContents LoadSingleFileContentsWorker(
FileContentsLoadingInfo loadingInfo,
ProjectFileData projectFileData)
{
// If project configuration has not changed, the file is still not
// searchable, irrelevant to calling "IsSearchable".
if (loadingInfo.FullPathChanges != null) {
if (loadingInfo.FullPathChanges.GetPathChangeKind(projectFileData.FileName.FullPath) == PathChangeKind.None) {
if (loadingInfo.UnchangedProjects.Contains(projectFileData.Project))
return null;
}
}
// This is an expensive call, hopefully avoided by the code above.
if (!projectFileData.IsSearchable)
return null;
loadingInfo.PartialProgressReporter.ReportProgress();
var fileContents = _fileContentsFactory.GetFileContents(projectFileData.FileName.FullPath);
if (fileContents is BinaryFileContents) {
Interlocked.Increment(ref loadingInfo.LoadedBinaryFileCount);
}
else {
Interlocked.Increment(ref loadingInfo.LoadedTextFileCount);
}
return loadingInfo.FileContentsMemoization.Get(projectFileData.FileName, fileContents);
}
private FileContents LoadSingleFileContents(
FileContentsLoadingInfo loadingInfo,
ProjectFileData projectFileData)
{
var fileName = projectFileData.FileName;
var oldFileData = loadingInfo.OldFileDatabase.Files.GetValue(fileName);
// If the file was never loaded before, just load it
if (oldFileData == null || oldFileData.Contents == null) {
return LoadSingleFileContentsWorker(loadingInfo, projectFileData);
}
bool isSearchable;
// If the project configuration is unchanged from the previous file
// database (and the file was present in it), then it is certainly
// searchable, so no need to make an expensive call to "IsSearchable"
// again.
if (loadingInfo.UnchangedProjects.Contains(projectFileData.Project)) {
isSearchable = true;
} else {
// If the file is not searchable in the current project, it should be
// ignored too. Note that "IsSearachable" is a somewhat expensive
// operation, as the filename is checked against potentially many glob
// patterns.
isSearchable = projectFileData.IsSearchable;
}
if (!isSearchable)
return null;
// If the file has not changed since the previous snapshot, we can re-use
// the former file contents snapshot.
if (IsFileContentsUpToDate(loadingInfo.FullPathChanges, oldFileData)) {
return oldFileData.Contents;
}
return LoadSingleFileContentsWorker(loadingInfo, projectFileData);
}
private void LoadFileContents(FileContentsLoadingInfo loadingInfo)
{
using (var logger = new TimeElapsedLogger("Loading file contents from disk")) {
using (var progress = _progressTrackerFactory.CreateTracker(_files.Count)) {
_files.AsParallel().ForAll(fileEntry => {
Debug.Assert(fileEntry.Value.FileData.Contents == null);
if (progress.Step()) {
progress.DisplayProgress((i, n) =>
string.Format("Reading file {0:n0} of {1:n0}: {2}", i, n, fileEntry.Value.FileName.FullPath));
}
var contents = LoadSingleFileContents(loadingInfo, fileEntry.Value);
if (contents != null) {
fileEntry.Value.FileData.UpdateContents(contents);
}
});
}
}
Logger.LogInfo("Loaded {0:n0} text files from disk, skipped {1:n0} binary files.",
loadingInfo.LoadedTextFileCount,
loadingInfo.LoadedBinaryFileCount);
}
