/// <summary>
/// Adjusts the modifed paths and placeholders list for an index entry.
/// </summary>
/// <param name="gitIndexEntry">Index entry</param>
/// <param name="filePlaceholders">
/// Dictionary of file placeholders. AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded will
/// remove enties from filePlaceholders as they are found in the index. After
/// AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded is called for all entries in the index
/// filePlaceholders will contain only those placeholders that are not in the index.
/// </param>
private FileSystemTaskResult AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded(
GitIndexEntry gitIndexEntry,
HashSet <string> filePlaceholders)
{
gitIndexEntry.BackgroundTask_ParsePath();
string placeholderRelativePath = gitIndexEntry.BackgroundTask_GetPlatformRelativePath();
FileSystemTaskResult result = FileSystemTaskResult.Success;
if (!gitIndexEntry.SkipWorktree)
{
// A git command (e.g. 'git reset --mixed') may have cleared a file's skip worktree bit without
// triggering an update to the projection. If git cleared the skip-worktree bit then git will
// be responsible for updating the file and we need to:
// - Ensure this file is in GVFS's modified files database
// - Remove this path from the placeholders list (if present)
result = this.projection.AddModifiedPath(placeholderRelativePath);
if (result == FileSystemTaskResult.Success)
{
if (filePlaceholders.Remove(placeholderRelativePath))
{
this.projection.RemoveFromPlaceholderList(placeholderRelativePath);
}
}
}
else
{
filePlaceholders.Remove(placeholderRelativePath);
}
return(result);
}
public static void ValidateIndex(ITracer tracer, Stream indexStream)
{
GitIndexParser indexParser = new GitIndexParser(null);
FileSystemTaskResult result = indexParser.ParseIndex(tracer, indexStream, indexParser.resuableProjectionBuildingIndexEntry, ValidateIndexEntry);
if (result != FileSystemTaskResult.Success)
{
// ValidateIndex should always result in FileSystemTaskResult.Success (or a thrown exception)
throw new InvalidOperationException($"{nameof(ValidateIndex)} failed: {result.ToString()}");
}
}
public void RebuildProjection(Stream indexStream)
{
if (this.projection == null)
{
throw new InvalidOperationException($"{nameof(this.projection)} cannot be null when calling {nameof(RebuildProjection)}");
}
this.projection.ClearProjectionCaches();
FileSystemTaskResult result = this.ParseIndex(indexStream, this.AddToProjection);
if (result != FileSystemTaskResult.Success)
{
// RebuildProjection should always result in FileSystemTaskResult.Success (or a thrown exception)
throw new InvalidOperationException($"{nameof(RebuildProjection)}: {nameof(GitIndexParser.ParseIndex)} failed to {nameof(this.AddToProjection)}");
}
}
public FileSystemTaskResult AddMissingModifiedFilesAndRemoveThemFromPlaceholderList(
ITracer tracer,
Stream indexStream)
{
if (this.projection == null)
{
throw new InvalidOperationException($"{nameof(this.projection)} cannot be null when calling {nameof(AddMissingModifiedFilesAndRemoveThemFromPlaceholderList)}");
}
Dictionary <string, PlaceholderListDatabase.PlaceholderData> filePlaceholders =
this.projection.placeholderList.GetAllFileEntries();
tracer.RelatedEvent(
EventLevel.Informational,
$"{nameof(this.AddMissingModifiedFilesAndRemoveThemFromPlaceholderList)}_FilePlaceholderCount",
new EventMetadata
{
{ "FilePlaceholderCount", filePlaceholders.Count }
});
FileSystemTaskResult result = this.ParseIndex(
tracer,
indexStream,
this.resuableBackgroundTaskThreadIndexEntry,
(data) => this.AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded(data, filePlaceholders));
if (result != FileSystemTaskResult.Success)
{
return(result);
}
// Any paths that were not found in the index need to be added to ModifiedPaths
// and removed from the placeholder list
foreach (string path in filePlaceholders.Keys)
{
result = this.projection.AddModifiedPath(path);
if (result != FileSystemTaskResult.Success)
{
return(result);
}
this.projection.RemoveFromPlaceholderList(path);
}
return(FileSystemTaskResult.Success);
}
private FileSystemTaskResult AddModifiedPathAndRemoveFromPlaceholderList(string virtualPath)
{
FileSystemTaskResult result = this.TryAddModifiedPath(virtualPath, isFolder: false);
if (result != FileSystemTaskResult.Success)
{
return(result);
}
bool isFolder;
string fileName;
// We don't want to fill the placeholder list with deletes for files that are
// not in the projection so we make sure it is in the projection before removing.
if (this.GitIndexProjection.IsPathProjected(virtualPath, out fileName, out isFolder))
{
this.GitIndexProjection.RemoveFromPlaceholderList(virtualPath);
}
return(result);
}
private void RunCallbackUntilSuccess(Func <FileSystemTaskResult> callback, string errorHeader)
{
int attempts = 0;
while (true)
{
FileSystemTaskResult callbackResult = callback();
switch (callbackResult)
{
case FileSystemTaskResult.Success:
return;
case FileSystemTaskResult.RetryableError:
if (this.isStopping)
{
return;
}
++attempts;
if (attempts > RetryFailuresLogThreshold)
{
this.context.Tracer.RelatedWarning("RunCallbackUntilSuccess(" + errorHeader + "): callback failed, retrying");
attempts = 0;
}
Thread.Sleep(ActionRetryDelayMS);
break;
case FileSystemTaskResult.FatalError:
this.LogErrorAndExit(errorHeader + " encountered fatal error, exiting process");
return;
default:
this.LogErrorAndExit(errorHeader + " result could not be found");
return;
}
}
}
private void ProcessBackgroundTasks()
{
FileSystemTask backgroundTask;
while (true)
{
AcquireGVFSLockResult acquireLockResult = AcquireGVFSLockResult.ShuttingDown;
try
{
this.wakeUpThread.WaitOne();
if (this.isStopping)
{
return;
}
acquireLockResult = this.WaitToAcquireGVFSLock();
switch (acquireLockResult)
{
case AcquireGVFSLockResult.LockAcquired:
break;
case AcquireGVFSLockResult.ShuttingDown:
return;
default:
this.LogErrorAndExit("Invalid " + nameof(AcquireGVFSLockResult) + " result");
return;
}
this.RunCallbackUntilSuccess(this.preCallback, "PreCallback");
int tasksProcessed = 0;
while (this.backgroundTasks.TryPeek(out backgroundTask))
{
if (tasksProcessed % LogUpdateTaskThreshold == 0 &&
(tasksProcessed >= LogUpdateTaskThreshold || this.backgroundTasks.Count >= LogUpdateTaskThreshold))
{
this.LogTaskProcessingStatus(tasksProcessed);
}
if (this.isStopping)
{
// If we are stopping, then ProjFS has already been shut down
// Some of the queued background tasks may require ProjFS, and so it is unsafe to
// proceed. GVFS will resume any queued tasks next time it is mounted
return;
}
FileSystemTaskResult callbackResult = this.callback(backgroundTask);
switch (callbackResult)
{
case FileSystemTaskResult.Success:
this.backgroundTasks.DequeueAndFlush(backgroundTask);
++tasksProcessed;
break;
case FileSystemTaskResult.RetryableError:
if (!this.isStopping)
{
Thread.Sleep(ActionRetryDelayMS);
}
break;
case FileSystemTaskResult.FatalError:
this.LogErrorAndExit("Callback encountered fatal error, exiting process");
break;
default:
this.LogErrorAndExit("Invalid background operation result");
break;
}
}
if (tasksProcessed >= LogUpdateTaskThreshold)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("TasksProcessed", tasksProcessed);
metadata.Add(TracingConstants.MessageKey.InfoMessage, "Processing background tasks complete");
this.context.Tracer.RelatedEvent(EventLevel.Informational, "TaskProcessingStatus", metadata);
}
if (this.isStopping)
{
return;
}
}
catch (Exception e)
{
this.LogErrorAndExit($"{nameof(this.ProcessBackgroundTasks)} caught unhandled exception, exiting process", e);
}
finally
{
this.PerformPostTaskProcessing(acquireLockResult);
}
}
}
/// <summary>
/// Takes an action on a GitIndexEntry using the index in indexStream
/// </summary>
/// <param name="indexStream">Stream for reading a git index file</param>
/// <param name="entryAction">Action to take on each GitIndexEntry from the index</param>
/// <returns>
/// FileSystemTaskResult indicating success or failure of the specified action
/// </returns>
/// <remarks>
/// Only the AddToModifiedFiles method because it updates the modified paths file can result
/// in TryIndexAction returning a FileSystemTaskResult other than Success. All other actions result in success (or an exception in the
/// case of a corrupt index)
/// </remarks>
private FileSystemTaskResult ParseIndex(
ITracer tracer,
Stream indexStream,
GitIndexEntry resuableParsedIndexEntry,
Func <GitIndexEntry, FileSystemTaskResult> entryAction)
{
this.indexStream = indexStream;
this.indexStream.Position = 0;
this.ReadNextPage();
if (this.page[0] != 'D' ||
this.page[1] != 'I' ||
this.page[2] != 'R' ||
this.page[3] != 'C')
{
throw new InvalidDataException("Incorrect magic signature for index: " + string.Join(string.Empty, this.page.Take(4).Select(c => (char)c)));
}
this.Skip(4);
uint indexVersion = this.ReadFromIndexHeader();
if (indexVersion != 4)
{
throw new InvalidDataException("Unsupported index version: " + indexVersion);
}
uint entryCount = this.ReadFromIndexHeader();
// Don't want to flood the logs on large indexes so only log every 500ms
const int LoggingTicksThreshold = 5000000;
long nextLogTicks = DateTime.UtcNow.Ticks + LoggingTicksThreshold;
SortedFolderEntries.InitializePools(tracer, entryCount);
LazyUTF8String.InitializePools(tracer, entryCount);
resuableParsedIndexEntry.ClearLastParent();
int previousPathLength = 0;
bool parseMode = GVFSPlatform.Instance.FileSystem.SupportsFileMode;
FileSystemTaskResult result = FileSystemTaskResult.Success;
for (int i = 0; i < entryCount; i++)
{
if (parseMode)
{
this.Skip(26);
// 4-bit object type
// valid values in binary are 1000(regular file), 1010(symbolic link) and 1110(gitlink)
// 3-bit unused
// 9-bit unix permission. Only 0755 and 0644 are valid for regular files. (Legacy repos can also contain 664)
// Symbolic links and gitlinks have value 0 in this field.
ushort indexFormatTypeAndMode = this.ReadUInt16();
FileTypeAndMode typeAndMode = new FileTypeAndMode(indexFormatTypeAndMode);
switch (typeAndMode.Type)
{
case FileType.Regular:
if (typeAndMode.Mode != FileMode755 &&
typeAndMode.Mode != FileMode644 &&
typeAndMode.Mode != FileMode664)
{
throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for regular file in index");
}
break;
case FileType.SymLink:
case FileType.GitLink:
if (typeAndMode.Mode != 0)
{
throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for link file({typeAndMode.Type:X}) in index");
}
break;
default:
throw new InvalidDataException($"Invalid file type {typeAndMode.Type:X} found in index");
}
resuableParsedIndexEntry.TypeAndMode = typeAndMode;
this.Skip(12);
}
else
{
this.Skip(40);
}
this.ReadSha(resuableParsedIndexEntry);
ushort flags = this.ReadUInt16();
if (flags == 0)
{
throw new InvalidDataException("Invalid flags found in index");
}
resuableParsedIndexEntry.MergeState = (MergeStage)((flags >> 12) & 3);
bool isExtended = (flags & ExtendedBit) == ExtendedBit;
resuableParsedIndexEntry.PathLength = (ushort)(flags & 0xFFF);
resuableParsedIndexEntry.SkipWorktree = false;
if (isExtended)
{
ushort extendedFlags = this.ReadUInt16();
resuableParsedIndexEntry.SkipWorktree = (extendedFlags & SkipWorktreeBit) == SkipWorktreeBit;
}
int replaceLength = this.ReadReplaceLength();
resuableParsedIndexEntry.ReplaceIndex = previousPathLength - replaceLength;
int bytesToRead = resuableParsedIndexEntry.PathLength - resuableParsedIndexEntry.ReplaceIndex + 1;
this.ReadPath(resuableParsedIndexEntry, resuableParsedIndexEntry.ReplaceIndex, bytesToRead);
previousPathLength = resuableParsedIndexEntry.PathLength;
result = entryAction.Invoke(resuableParsedIndexEntry);
if (result != FileSystemTaskResult.Success)
{
return(result);
}
if (DateTime.UtcNow.Ticks > nextLogTicks)
{
tracer.RelatedInfo($"{i}/{entryCount} index entries parsed.");
nextLogTicks = DateTime.UtcNow.Ticks + LoggingTicksThreshold;
}
}
tracer.RelatedInfo($"Finished parsing {entryCount} index entries.");
return(result);
}
/// <summary>
/// Takes an action on a GitIndexEntry using the index in indexStream
/// </summary>
/// <param name="indexStream">Stream for reading a git index file</param>
/// <param name="entryAction">Action to take on each GitIndexEntry from the index</param>
/// <returns>
/// FileSystemTaskResult indicating success or failure of the specified action
/// </returns>
/// <remarks>
/// Only the AddToModifiedFiles method because it updates the modified paths file can result
/// in TryIndexAction returning a FileSystemTaskResult other than Success. All other actions result in success (or an exception in the
/// case of a corrupt index)
/// </remarks>
private FileSystemTaskResult ParseIndex(Stream indexStream, Func <GitIndexEntry, FileSystemTaskResult> entryAction)
{
this.indexStream = indexStream;
this.indexStream.Position = 0;
this.ReadNextPage();
if (this.page[0] != 'D' ||
this.page[1] != 'I' ||
this.page[2] != 'R' ||
this.page[3] != 'C')
{
throw new InvalidDataException("Incorrect magic signature for index: " + string.Join(string.Empty, this.page.Take(4).Select(c => (char)c)));
}
this.Skip(4);
uint indexVersion = this.ReadFromIndexHeader();
if (indexVersion != 4)
{
throw new InvalidDataException("Unsupported index version: " + indexVersion);
}
uint entryCount = this.ReadFromIndexHeader();
this.resuableParsedIndexEntry.ClearLastParent();
int previousPathLength = 0;
bool parseMode = GVFSPlatform.Instance.FileSystem.SupportsFileMode;
FileSystemTaskResult result = FileSystemTaskResult.Success;
for (int i = 0; i < entryCount; i++)
{
if (parseMode)
{
this.Skip(26);
// 4-bit object type
// valid values in binary are 1000(regular file), 1010(symbolic link) and 1110(gitlink)
// 3-bit unused
// 9-bit unix permission. Only 0755 and 0644 are valid for regular files. (Legacy repos can also contain 664)
// Symbolic links and gitlinks have value 0 in this field.
ushort mode = this.ReadUInt16();
this.resuableParsedIndexEntry.FileMode = (ushort)(mode & 0x1FF);
this.Skip(12);
}
else
{
this.Skip(40);
}
this.ReadSha(this.resuableParsedIndexEntry);
ushort flags = this.ReadUInt16();
if (flags == 0)
{
throw new InvalidDataException("Invalid flags found in index");
}
this.resuableParsedIndexEntry.MergeState = (MergeStage)((flags >> 12) & 3);
bool isExtended = (flags & ExtendedBit) == ExtendedBit;
this.resuableParsedIndexEntry.PathLength = (ushort)(flags & 0xFFF);
this.resuableParsedIndexEntry.SkipWorktree = false;
if (isExtended)
{
ushort extendedFlags = this.ReadUInt16();
this.resuableParsedIndexEntry.SkipWorktree = (extendedFlags & SkipWorktreeBit) == SkipWorktreeBit;
}
int replaceLength = this.ReadReplaceLength();
this.resuableParsedIndexEntry.ReplaceIndex = previousPathLength - replaceLength;
int bytesToRead = this.resuableParsedIndexEntry.PathLength - this.resuableParsedIndexEntry.ReplaceIndex + 1;
this.ReadPath(this.resuableParsedIndexEntry, this.resuableParsedIndexEntry.ReplaceIndex, bytesToRead);
previousPathLength = this.resuableParsedIndexEntry.PathLength;
result = entryAction.Invoke(this.resuableParsedIndexEntry);
if (result != FileSystemTaskResult.Success)
{
return(result);
}
}
return(result);
}
