private void LoadFileContents(FileSystemEntities entities, FileContentsLoadingContext loadingContext, CancellationToken cancellationToken)
{
using (new TimeElapsedLogger("Loading file contents from disk")) {
using (var progress = _progressTrackerFactory.CreateTracker(entities.Files.Count)) {
entities.Files.AsParallelWrapper().ForAll(fileEntry => {
Debug.Assert(fileEntry.Value.FileWithContents.Contents == null);
if (progress.Step())
{
progress.DisplayProgress((i, n) =>
string.Format("Reading file {0:n0} of {1:n0}: {2}", i, n, fileEntry.Value.FileName.FullPath));
// Check for cancellation
if (cancellationToken.IsCancellationRequested)
{
loadingContext.PartialProgressReporter.ReportProgressNow();
cancellationToken.ThrowIfCancellationRequested();
}
}
var contents = LoadSingleFileContents(entities, loadingContext, fileEntry.Value);
if (contents != null)
{
fileEntry.Value.FileWithContents.UpdateContents(contents);
}
});
}
}
Logger.LogInfo("Loaded {0:n0} text files from disk, skipped {1:n0} binary files.",
loadingContext.LoadedTextFileCount,
loadingContext.LoadedBinaryFileCount);
}
private bool IsFileContentsUpToDate(FileSystemEntities entities, FullPathChanges fullPathChanges, FileWithContents existingFileWithContents)
{
Debug.Assert(existingFileWithContents.Contents != null);
var fullPath = existingFileWithContents.FileName.FullPath;
if (fullPathChanges != null)
{
// We don't get file change events for file in symlinks, so we can't
// rely on fullPathChanges contents for our heuristic of avoiding file
// system access.
if (!FileDatabaseSnapshot.IsContainedInSymLinkHelper(entities.Directories, existingFileWithContents.FileName))
{
return(fullPathChanges.ShouldSkipLoadFileContents(fullPath));
}
}
// Do the "expensive" check by going to the file system.
var fi = _fileSystem.GetFileInfoSnapshot(fullPath);
return
((fi.Exists) &&
(fi.IsFile) &&
(fi.LastWriteTimeUtc == existingFileWithContents.Contents.UtcLastModified));
}
private FileDatabaseSnapshot CreateFileDatabse(FileSystemEntities entities)
{
using (new TimeElapsedLogger("Freezing file database state")) {
var directories = entities.Directories;
// Note: We cannot use "ReferenceEqualityComparer<FileName>" here because
// the dictionary will be used in incremental updates where FileName instances
// may be new instances from a complete file system enumeration.
var files = new Dictionary <FileName, FileWithContents>(entities.Files.Count);
var filesWithContentsArray = new FileWithContents[entities.Files.Count];
int filesWithContentsIndex = 0;
foreach (var kvp in entities.Files)
{
var fileData = kvp.Value.FileWithContents;
files.Add(kvp.Key, fileData);
if (fileData.Contents != null && fileData.Contents.ByteLength > 0)
{
filesWithContentsArray[filesWithContentsIndex++] = fileData;
}
}
var filesWithContents = new ListSegment <FileWithContents>(filesWithContentsArray, 0, filesWithContentsIndex);
var searchableContentsCollection = CreateFilePieces(filesWithContents);
FileDatabaseDebugLogger.LogFileContentsStats(filesWithContents);
return(new FileDatabaseSnapshot(
entities.ProjectHashes,
files,
files.Keys.ToArray(),
directories,
searchableContentsCollection,
filesWithContents.Count));
}
}
private bool IsFileContentsUpToDate(FileSystemEntities entities,
FullPathChanges fullPathChanges,
FileWithContents existingFileWithContents)
{
Invariants.Assert(existingFileWithContents.Contents != null);
var fullPath = existingFileWithContents.FileName.FullPath;
if (fullPathChanges != null)
{
// We don't get file change events for file in symlinks, so we can't
// rely on fullPathChanges contents for our heuristic of avoiding file
// system access.
// Actually, since we cannot reliable detect changes in symlinks, we enable this
// optimization anyways, as we rely on the user to manually refresh the index
// (that results in a "null" value for fullPathChanges)
//if (!FileDatabaseSnapshot.IsContainedInSymLinkHelper(entities.Directories, existingFileWithContents.FileName)) {
return(fullPathChanges.ShouldSkipLoadFileContents(fullPath));
//}
}
// Do the "expensive" check by going to the file system.
var fi = _fileSystem.GetFileInfoSnapshot(fullPath);
return
((fi.Exists) &&
(fi.IsFile) &&
(fi.LastWriteTimeUtc == existingFileWithContents.Contents.UtcLastModified));
}
private FileDatabaseSnapshot CreateFileDatabase(FileSystemEntities entities,
bool notifyProgress,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
using (new TimeElapsedLogger("Freezing file database state", cancellationToken)) {
var progress = notifyProgress ? _progressTrackerFactory.CreateIndeterminateTracker() : null;
try {
progress?.DisplayProgress((i, n) => "Finalizing index update");
var directories = entities.Directories;
// Note: We cannot use "ReferenceEqualityComparer<FileName>" here because
// the dictionary will be used in incremental updates where FileName instances
// may be new instances from a complete file system enumeration.
var files = new SlimHashTable <FileName, FileWithContents>(v => v.FileName, entities.Files.Count);
foreach (var kvp in entities.Files)
{
cancellationToken.ThrowIfCancellationRequested();
files.Add(kvp.Key, new FileWithContents(kvp.Value.FileName, kvp.Value.Contents));
}
return(new FileDatabaseSnapshot(entities.ProjectHashes, directories, files));
}
finally {
progress?.Dispose();
}
}
}
private FileContents LoadSingleFileContents(
FileSystemEntities entities,
FileContentsLoadingContext loadingContext,
ProjectFileData projectFileData)
{
var fileName = projectFileData.FileName;
var oldFileData = loadingContext.OldFileDatabaseSnapshot.Files.GetValue(fileName);
// If the file was never loaded before, just load it
if (oldFileData == null || oldFileData.Contents == null)
{
return(LoadSingleFileContentsWorker(loadingContext, projectFileData));
}
bool isSearchable;
// If the project configuration is unchanged from the previous file
// database (and the file was present in it), then it is certainly
// searchable, so no need to make an expensive call to "IsSearchable"
// again.
if (loadingContext.UnchangedProjects.Contains(projectFileData.Project))
{
isSearchable = true;
}
else
{
// If the file is not searchable in the current project, it should be
// ignored too. Note that "IsSearachable" is a somewhat expensive
// operation, as the filename is checked against potentially many glob
// patterns.
isSearchable = projectFileData.IsSearchable;
}
if (!isSearchable)
{
return(null);
}
// If the file has not changed since the previous snapshot, we can re-use
// the former file contents snapshot.
if (IsFileContentsUpToDate(entities, loadingContext.FullPathChanges, oldFileData))
{
return(oldFileData.Contents);
}
return(LoadSingleFileContentsWorker(loadingContext, projectFileData));
}
/// <summary>
/// Note: Concurrency: Updating an entry of the <see cref="FileSystemEntities.Files"/> table
/// in this method assumes that 1) the entry exists and 2) the hash table implementation does not
/// mutate any of its internal state when updating an existing entry.
///
/// <para>
/// 1) is verified by virtue of the <see cref="LoadFileContents"/> method implementation.</para>
///
/// <para>
/// 2) is verified because the two possible implementations, <see cref="Dictionary{TKey,TValue}"/>
/// and <see cref="SlimHashTable{TKey,TValue}"/> behave that way.</para>
/// </summary>
private static void DangerousUpdateFileSystemEntitiesEntry(FileSystemEntities entities,
KeyValuePair <FileName, ProjectFileData> fileEntry,
FileContents contents)
{
entities.Files[fileEntry.Key] = fileEntry.Value.WithContents(contents);
}