private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string line)
{
if (!line.StartsWith(":"))
{
// Diff-tree starts with metadata we can ignore.
// Real diff lines always start with a colon
return;
}
DiffTreeResult result = DiffTreeResult.ParseFromDiffTreeLine(line);
if (!this.ShouldIncludeResult(result))
{
return;
}
if (result.Operation == DiffTreeResult.Operations.Unknown ||
result.Operation == DiffTreeResult.Operations.Unmerged ||
result.Operation == DiffTreeResult.Operations.CopyEdit ||
result.Operation == DiffTreeResult.Operations.RenameEdit)
{
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(result.TargetPath), result.TargetPath);
metadata.Add(nameof(line), line);
activity.RelatedError(metadata, "Unexpected diff operation: " + result.Operation);
this.HasFailures = true;
return;
}
// Separate and enqueue all directory operations first.
if (result.SourceIsDirectory || result.TargetIsDirectory)
{
switch (result.Operation)
{
case DiffTreeResult.Operations.Delete:
if (!this.stagedDirectoryOperations.Add(result))
{
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(result.TargetPath), result.TargetPath);
metadata.Add(TracingConstants.MessageKey.WarningMessage, "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
}
break;
case DiffTreeResult.Operations.Add:
case DiffTreeResult.Operations.Modify:
if (!this.stagedDirectoryOperations.Add(result))
{
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(result.TargetPath), result.TargetPath);
metadata.Add(TracingConstants.MessageKey.WarningMessage, "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
// Replace the delete with the add to make sure we don't delete a folder from under ourselves
this.stagedDirectoryOperations.Remove(result);
this.stagedDirectoryOperations.Add(result);
}
break;
default:
activity.RelatedError("Unexpected diff operation from line: {0}", line);
break;
}
}
else
{
switch (result.Operation)
{
case DiffTreeResult.Operations.Delete:
this.EnqueueFileDeleteOperation(activity, result.TargetPath);
break;
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.Add:
this.EnqueueFileAddOperation(activity, result);
break;
default:
activity.RelatedError("Unexpected diff operation from line: {0}", line);
break;
}
}
}
/// <param name="endOfFile">Length of the file, not required on the Mac platform</param>
public override FileSystemResult UpdatePlaceholderIfNeeded(
string relativePath,
DateTime creationTime,
DateTime lastAccessTime,
DateTime lastWriteTime,
DateTime changeTime,
FileAttributes fileAttributes,
long endOfFile,
string shaContentId,
UpdatePlaceholderType updateFlags,
out UpdateFailureReason failureReason)
{
UpdateFailureCause failureCause = UpdateFailureCause.NoFailure;
// TODO(#223): Add functional tests that include:
// - Mode + content changes between commits
// - Mode only changes (without any change to content, see issue #223)
GitIndexProjection.FileType fileType;
ushort fileMode;
this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath, out fileType, out fileMode);
if (fileType == GitIndexProjection.FileType.Regular)
{
Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded(
relativePath,
PlaceholderVersionId,
ToVersionIdByteArray(ConvertShaToContentId(shaContentId)),
fileMode,
(UpdateType)updateFlags,
out failureCause);
failureReason = (UpdateFailureReason)failureCause;
return(new FileSystemResult(ResultToFSResult(result), unchecked ((int)result)));
}
else if (fileType == GitIndexProjection.FileType.SymLink)
{
string symLinkTarget;
if (this.TryGetSymLinkTarget(shaContentId, out symLinkTarget))
{
Result result = this.virtualizationInstance.ReplacePlaceholderFileWithSymLink(
relativePath,
symLinkTarget,
(UpdateType)updateFlags,
out failureCause);
this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath);
failureReason = (UpdateFailureReason)failureCause;
return(new FileSystemResult(ResultToFSResult(result), unchecked ((int)result)));
}
EventMetadata metadata = this.CreateEventMetadata(relativePath);
metadata.Add(nameof(shaContentId), shaContentId);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Failed to read contents of symlink object");
failureReason = UpdateFailureReason.NoFailure;
return(new FileSystemResult(FSResult.IOError, 0));
}
else
{
EventMetadata metadata = this.CreateEventMetadata(relativePath);
metadata.Add(nameof(fileType), fileType);
metadata.Add(nameof(fileMode), fileMode);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType");
failureReason = UpdateFailureReason.NoFailure;
return(new FileSystemResult(FSResult.IOError, 0));
}
}
private Result OnGetFileStream(
ulong commandId,
string relativePath,
byte[] providerId,
byte[] contentId,
int triggeringProcessId,
string triggeringProcessName,
IntPtr fileHandle)
{
try
{
if (contentId == null)
{
this.Context.Tracer.RelatedError($"{nameof(this.OnGetFileStream)} called with null contentId, path: " + relativePath);
return(Result.EInvalidOperation);
}
if (providerId == null)
{
this.Context.Tracer.RelatedError($"{nameof(this.OnGetFileStream)} called with null epochId, path: " + relativePath);
return(Result.EInvalidOperation);
}
string sha = GetShaFromContentId(contentId);
byte placeholderVersion = GetPlaceholderVersionFromProviderId(providerId);
EventMetadata metadata = this.CreateEventMetadata(relativePath);
metadata.Add(nameof(triggeringProcessId), triggeringProcessId);
metadata.Add(nameof(triggeringProcessName), triggeringProcessName);
metadata.Add(nameof(sha), sha);
metadata.Add(nameof(placeholderVersion), placeholderVersion);
metadata.Add(nameof(commandId), commandId);
ITracer activity = this.Context.Tracer.StartActivity("GetFileStream", EventLevel.Verbose, Keywords.Telemetry, metadata);
if (placeholderVersion != FileSystemVirtualizer.PlaceholderVersion)
{
activity.RelatedError(metadata, nameof(this.OnGetFileStream) + ": Unexpected placeholder version");
activity.Dispose();
// TODO(#1362): Is this the correct Result to return?
return(Result.EIOError);
}
try
{
if (!this.GitObjects.TryCopyBlobContentStream(
sha,
CancellationToken.None,
GVFSGitObjects.RequestSource.FileStreamCallback,
(stream, blobLength) =>
{
// TODO(#1361): Find a better solution than reading from the stream one byte at at time
byte[] buffer = new byte[4096];
uint bufferIndex = 0;
int nextByte = stream.ReadByte();
int bytesWritten = 0;
while (nextByte != -1)
{
while (bufferIndex < buffer.Length && nextByte != -1)
{
buffer[bufferIndex] = (byte)nextByte;
nextByte = stream.ReadByte();
++bufferIndex;
}
Result result = this.virtualizationInstance.WriteFileContents(
fileHandle,
buffer,
bufferIndex);
if (result != Result.Success)
{
activity.RelatedError(metadata, $"{nameof(this.virtualizationInstance.WriteFileContents)} failed, error: " + result.ToString("X") + "(" + result.ToString("G") + ")");
throw new GetFileStreamException(result);
}
if (bufferIndex == buffer.Length)
{
bufferIndex = 0;
bytesWritten += buffer.Length;
}
}
bytesWritten += Convert.ToInt32(bufferIndex);
if (bytesWritten != blobLength)
{
// If the read size does not match the expected size print an error and add the file to ModifiedPaths.dat
// This allows the user to see that something went wrong with file hydration
// Unfortunitely we must do this check *after* the file is hydrated since the header isn't corrupt for trunctated objects on mac
this.Context.Tracer.RelatedError($"Read {relativePath} to {bytesWritten}, not expected size of {blobLength}");
this.FileSystemCallbacks.OnFailedFileHydration(relativePath);
}
}))
{
activity.RelatedError(metadata, $"{nameof(this.OnGetFileStream)}: TryCopyBlobContentStream failed");
// TODO(#1362): Is this the correct Result to return?
return(Result.EFileNotFound);
}
}
catch (GetFileStreamException e)
{
return(e.Result);
}
this.FileSystemCallbacks.OnPlaceholderFileHydrated(triggeringProcessName);
return(Result.Success);
}
catch (Exception e)
{
EventMetadata metadata = this.CreateEventMetadata(relativePath, e);
metadata.Add(nameof(triggeringProcessId), triggeringProcessId);
metadata.Add(nameof(triggeringProcessName), triggeringProcessName);
metadata.Add(nameof(commandId), commandId);
this.LogUnhandledExceptionAndExit(nameof(this.OnGetFileStream), metadata);
}
return(Result.EIOError);
}
public virtual GitProcess.Result IndexPackFile(string packfilePath, GitProcess gitProcess)
{
string tempIdxPath = Path.ChangeExtension(packfilePath, TempIdxExtension);
string idxPath = Path.ChangeExtension(packfilePath, ".idx");
Exception indexPackException = null;
try
{
if (gitProcess == null)
{
gitProcess = new GitProcess(this.Enlistment);
}
GitProcess.Result result = gitProcess.IndexPack(packfilePath, tempIdxPath);
if (result.ExitCodeIsFailure)
{
Exception exception;
if (!this.fileSystem.TryDeleteFile(tempIdxPath, exception: out exception))
{
EventMetadata metadata = CreateEventMetadata(exception);
metadata.Add("tempIdxPath", tempIdxPath);
this.Tracer.RelatedWarning(metadata, $"{nameof(this.IndexPackFile)}: Failed to cleanup temp idx file after index pack failure");
}
}
else
{
if (this.Enlistment.FlushFileBuffersForPacks)
{
Exception exception;
string error;
if (!this.TryFlushFileBuffers(tempIdxPath, out exception, out error))
{
EventMetadata metadata = CreateEventMetadata(exception);
metadata.Add("packfilePath", packfilePath);
metadata.Add("tempIndexPath", tempIdxPath);
metadata.Add("error", error);
this.Tracer.RelatedWarning(metadata, $"{nameof(this.IndexPackFile)}: Failed to flush temp idx file buffers");
}
}
this.fileSystem.MoveAndOverwriteFile(tempIdxPath, idxPath);
}
return(result);
}
catch (Win32Exception e)
{
indexPackException = e;
}
catch (IOException e)
{
indexPackException = e;
}
catch (UnauthorizedAccessException e)
{
indexPackException = e;
}
EventMetadata failureMetadata = CreateEventMetadata(indexPackException);
failureMetadata.Add("packfilePath", packfilePath);
failureMetadata.Add("tempIdxPath", tempIdxPath);
failureMetadata.Add("idxPath", idxPath);
this.fileSystem.TryDeleteFile(tempIdxPath, metadataKey: nameof(tempIdxPath), metadata: failureMetadata);
this.fileSystem.TryDeleteFile(idxPath, metadataKey: nameof(idxPath), metadata: failureMetadata);
this.Tracer.RelatedWarning(failureMetadata, $"{nameof(this.IndexPackFile): Exception caught while trying to index pack file}");
return(new GitProcess.Result(
string.Empty,
indexPackException != null ? indexPackException.Message : "Failed to index pack file",
GitProcess.Result.GenericFailureCode));
}
private Result CreateClone(
ITracer tracer,
GVFSEnlistment enlistment,
GitObjectsHttpRequestor objectRequestor,
GitRefs refs,
string branch)
{
Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string errorMessage;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage))
{
return(new Result("Error configuring alternate: " + errorMessage));
}
GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem);
GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment);
GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor);
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage))
{
return(new Result(errorMessage));
}
if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) ||
!GVFSVerb.TrySetOptionalGitConfigSettings(enlistment))
{
return(new Result("Unable to configure git repo"));
}
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage))
{
return(new Result("Unable to configure cache server: " + errorMessage));
}
GitProcess git = new GitProcess(enlistment);
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.ExitCodeIsFailure)
{
return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Head),
"ref: refs/heads/" + branch);
if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage))
{
return(new Result(errorMessage));
}
this.CreateGitScript(enlistment);
string installHooksError;
if (!HooksInstaller.InstallHooks(context, out installHooksError))
{
tracer.RelatedError(installHooksError);
return(new Result(installHooksError));
}
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0)
{
// It is possible to have the above TryDownloadCommit() fail because we
// already have the commit and root tree we intend to check out, but
// don't have a tree further down the working directory. If we fail
// checkout here, its' because we don't have these trees and the
// read-object hook is not available yet. Force downloading the commit
// again and retry the checkout.
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage,
checkLocalObjectCache: false))
{
return(new Result(errorMessage));
}
forceCheckoutResult = git.ForceCheckout(branch);
}
if (forceCheckoutResult.ExitCodeIsFailure)
{
string[] errorLines = forceCheckoutResult.Errors.Split('\n');
StringBuilder checkoutErrors = new StringBuilder();
foreach (string gitError in errorLines)
{
if (IsForceCheckoutErrorCloneFailure(gitError))
{
checkoutErrors.AppendLine(gitError);
}
}
if (checkoutErrors.Length > 0)
{
string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString();
tracer.RelatedError(error);
return(new Result(error));
}
}
if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage))
{
tracer.RelatedError(errorMessage);
return(new Result(errorMessage));
}
try
{
RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment);
this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment);
}
catch (Exception e)
{
tracer.RelatedError(e.ToString());
return(new Result(e.Message));
}
finally
{
RepoMetadata.Shutdown();
}
// Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed
Exception exception;
string prepFileSystemError;
if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryBackingRoot, out prepFileSystemError, out exception))
{
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(prepFileSystemError), prepFileSystemError);
if (exception != null)
{
metadata.Add("Exception", exception.ToString());
}
tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed");
return(new Result(prepFileSystemError));
}
return(new Result(true));
}
protected override void Execute(GVFSEnlistment enlistment)
{
using (JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "Prefetch"))
{
if (this.Verbose)
{
tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment);
tracer.AddLogFileEventListener(
GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Prefetch),
EventLevel.Informational,
Keywords.Any);
tracer.WriteStartEvent(
enlistment.EnlistmentRoot,
enlistment.RepoUrl,
cacheServerUrl);
RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
CacheServerInfo cacheServer = this.ResolvedCacheServer;
GVFSConfig gvfsConfig = this.GVFSConfig;
if (!this.SkipVersionCheck)
{
string authErrorMessage;
if (!this.ShowStatusWhileRunning(
() => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage),
"Authenticating"))
{
this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed");
}
if (gvfsConfig == null)
{
gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig);
}
if (cacheServer == null)
{
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig);
}
this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: false);
this.Output.WriteLine("Configured cache server: " + cacheServer);
}
this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig, cacheServer);
try
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Commits", this.Commits);
metadata.Add("Files", this.Files);
metadata.Add("Folders", this.Folders);
metadata.Add("FoldersListFile", this.FoldersListFile);
metadata.Add("HydrateFiles", this.HydrateFiles);
tracer.RelatedEvent(EventLevel.Informational, "PerformPrefetch", metadata);
GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig);
if (this.Commits)
{
if (!string.IsNullOrWhiteSpace(this.Files) ||
!string.IsNullOrWhiteSpace(this.Folders) ||
!string.IsNullOrWhiteSpace(this.FoldersListFile))
{
this.ReportErrorAndExit(tracer, "You cannot prefetch commits and blobs at the same time.");
}
if (this.HydrateFiles)
{
this.ReportErrorAndExit(tracer, "You can only specify --hydrate with --files or --folders");
}
this.PrefetchCommits(tracer, enlistment, objectRequestor, cacheServer);
}
else
{
this.PrefetchBlobs(tracer, enlistment, objectRequestor, cacheServer);
}
}
catch (VerbAbortedException)
{
throw;
}
catch (AggregateException aggregateException)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
foreach (Exception innerException in aggregateException.Flatten().InnerExceptions)
{
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", innerException.ToString() }
},
$"Unhandled {innerException.GetType().Name}: {innerException.Message}");
}
Environment.ExitCode = (int)ReturnCode.GenericError;
}
catch (Exception e)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", e.ToString() }
},
$"Unhandled {e.GetType().Name}: {e.Message}");
Environment.ExitCode = (int)ReturnCode.GenericError;
}
}
}
private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate)
{
EventMetadata metadata = new EventMetadata();
FileSystemTaskResult result;
switch (gitUpdate.Operation)
{
case FileSystemTask.OperationType.OnFileCreated:
case FileSystemTask.OperationType.OnFailedPlaceholderDelete:
case FileSystemTask.OperationType.OnFileHardLinkCreated:
case FileSystemTask.OperationType.OnFileSymLinkCreated:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFileRenamed:
metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = FileSystemTaskResult.Success;
if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) && !IsPathInsideDotGit(gitUpdate.OldVirtualPath))
{
if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath))
{
result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: false);
}
else
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.OldVirtualPath);
}
}
if (result == FileSystemTaskResult.Success &&
!string.IsNullOrEmpty(gitUpdate.VirtualPath) &&
!IsPathInsideDotGit(gitUpdate.VirtualPath))
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
}
break;
case FileSystemTask.OperationType.OnFilePreDelete:
// This code assumes that the current implementations of FileSystemVirtualizer will call either
// the PreDelete or the Delete not both so if a new implementation starts calling both
// this will need to be cleaned up to not duplicate the work that is being done.
metadata.Add("virtualPath", gitUpdate.VirtualPath);
if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath))
{
string fullPathToFile = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, gitUpdate.VirtualPath);
// Because this is a predelete message the file could still be on disk when we make this check
// so we retry for a limited time before deciding the delete didn't happen
bool fileDeleted = CheckConditionWithRetry(() => !this.context.FileSystem.FileExists(fullPathToFile), NumberOfRetriesCheckingForDeleted, MillisecondsToSleepBeforeCheckingForDeleted);
if (fileDeleted)
{
result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: false);
}
else
{
result = FileSystemTaskResult.Success;
}
}
else
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
}
break;
case FileSystemTask.OperationType.OnFileDeleted:
// This code assumes that the current implementations of FileSystemVirtualizer will call either
// the PreDelete or the Delete not both so if a new implementation starts calling both
// this will need to be cleaned up to not duplicate the work that is being done.
metadata.Add("virtualPath", gitUpdate.VirtualPath);
if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath))
{
result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: false);
}
else
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
}
break;
case FileSystemTask.OperationType.OnFileOverwritten:
case FileSystemTask.OperationType.OnFileSuperseded:
case FileSystemTask.OperationType.OnFileConvertedToFull:
case FileSystemTask.OperationType.OnFailedPlaceholderUpdate:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFolderCreated:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
break;
case FileSystemTask.OperationType.OnFolderRenamed:
result = FileSystemTaskResult.Success;
metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
metadata.Add("virtualPath", gitUpdate.VirtualPath);
if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) &&
this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath))
{
result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true);
}
// An empty destination path means the folder was renamed to somewhere outside of the repo
// Note that only full folders can be moved\renamed, and so there will already be a recursive
// sparse-checkout entry for the virtualPath of the folder being moved (meaning that no
// additional work is needed for any files\folders inside the folder being moved)
if (result == FileSystemTaskResult.Success && !string.IsNullOrEmpty(gitUpdate.VirtualPath))
{
this.AddToNewlyCreatedList(gitUpdate.VirtualPath, isFolder: true);
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
if (result == FileSystemTaskResult.Success)
{
Queue <string> relativeFolderPaths = new Queue <string>();
relativeFolderPaths.Enqueue(gitUpdate.VirtualPath);
// Remove old paths from modified paths if in the newly created list
while (relativeFolderPaths.Count > 0)
{
string folderPath = relativeFolderPaths.Dequeue();
if (result == FileSystemTaskResult.Success)
{
try
{
foreach (DirectoryItemInfo itemInfo in this.context.FileSystem.ItemsInDirectory(Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, folderPath)))
{
string itemVirtualPath = Path.Combine(folderPath, itemInfo.Name);
string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length);
this.AddToNewlyCreatedList(itemVirtualPath, isFolder: itemInfo.IsDirectory);
if (this.newlyCreatedFileAndFolderPaths.Contains(oldItemVirtualPath))
{
result = this.TryRemoveModifiedPath(oldItemVirtualPath, isFolder: itemInfo.IsDirectory);
}
if (itemInfo.IsDirectory)
{
relativeFolderPaths.Enqueue(itemVirtualPath);
}
}
}
catch (DirectoryNotFoundException)
{
// DirectoryNotFoundException can occur when the renamed folder (or one of its children) is
// deleted prior to the background thread running
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Area", "ExecuteBackgroundOperation");
exceptionMetadata.Add("Operation", gitUpdate.Operation.ToString());
exceptionMetadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
exceptionMetadata.Add("virtualPath", gitUpdate.VirtualPath);
exceptionMetadata.Add(TracingConstants.MessageKey.InfoMessage, "DirectoryNotFoundException while traversing folder path");
exceptionMetadata.Add("folderPath", folderPath);
this.context.Tracer.RelatedEvent(EventLevel.Informational, "DirectoryNotFoundWhileUpdatingModifiedPaths", exceptionMetadata);
}
catch (IOException e)
{
metadata.Add("Details", "IOException while traversing folder path");
metadata.Add("folderPath", folderPath);
metadata.Add("Exception", e.ToString());
result = FileSystemTaskResult.RetryableError;
break;
}
catch (UnauthorizedAccessException e)
{
metadata.Add("Details", "UnauthorizedAccessException while traversing folder path");
metadata.Add("folderPath", folderPath);
metadata.Add("Exception", e.ToString());
result = FileSystemTaskResult.RetryableError;
break;
}
}
else
{
break;
}
}
}
}
break;
case FileSystemTask.OperationType.OnFolderPreDelete:
// This code assumes that the current implementations of FileSystemVirtualizer will call either
// the PreDelete or the Delete not both so if a new implementation starts calling both
// this will need to be cleaned up to not duplicate the work that is being done.
metadata.Add("virtualPath", gitUpdate.VirtualPath);
if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath))
{
string fullPathToFolder = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, gitUpdate.VirtualPath);
// Because this is a predelete message the file could still be on disk when we make this check
// so we retry for a limited time before deciding the delete didn't happen
bool folderDeleted = CheckConditionWithRetry(() => !this.context.FileSystem.DirectoryExists(fullPathToFolder), NumberOfRetriesCheckingForDeleted, MillisecondsToSleepBeforeCheckingForDeleted);
if (folderDeleted)
{
result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: true);
}
else
{
result = FileSystemTaskResult.Success;
}
}
else
{
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
}
break;
case FileSystemTask.OperationType.OnFolderDeleted:
// This code assumes that the current implementations of FileSystemVirtualizer will call either
// the PreDelete or the Delete not both so if a new implementation starts calling both
// this will need to be cleaned up to not duplicate the work that is being done.
metadata.Add("virtualPath", gitUpdate.VirtualPath);
if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath))
{
result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: true);
}
else
{
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
}
break;
case FileSystemTask.OperationType.OnFolderFirstWrite:
result = FileSystemTaskResult.Success;
break;
case FileSystemTask.OperationType.OnIndexWriteRequiringModifiedPathsValidation:
result = this.GitIndexProjection.AddMissingModifiedFiles();
break;
case FileSystemTask.OperationType.OnPlaceholderCreationsBlockedForGit:
this.GitIndexProjection.ClearNegativePathCacheIfPollutedByGit();
result = FileSystemTaskResult.Success;
break;
default:
throw new InvalidOperationException("Invalid background operation");
}
if (result != FileSystemTaskResult.Success)
{
metadata.Add("Area", "ExecuteBackgroundOperation");
metadata.Add("Operation", gitUpdate.Operation.ToString());
metadata.Add(TracingConstants.MessageKey.WarningMessage, "Background operation failed");
metadata.Add(nameof(result), result.ToString());
this.context.Tracer.RelatedEvent(EventLevel.Warning, "FailedBackgroundOperation", metadata);
}
return(result);
}
private void HandleDehydrateFolders(NamedPipeMessages.Message message, NamedPipeServer.Connection connection)
{
NamedPipeMessages.DehydrateFolders.Request request = new NamedPipeMessages.DehydrateFolders.Request(message);
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(request.Folders), request.Folders);
metadata.Add(TracingConstants.MessageKey.InfoMessage, "Received dehydrate folders request");
this.tracer.RelatedEvent(EventLevel.Informational, nameof(this.HandleDehydrateFolders), metadata);
NamedPipeMessages.DehydrateFolders.Response response;
if (this.currentState == MountState.Ready)
{
response = new NamedPipeMessages.DehydrateFolders.Response(NamedPipeMessages.DehydrateFolders.DehydratedResult);
string[] folders = request.Folders.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
StringBuilder resetFolderPaths = new StringBuilder();
foreach (string folder in folders)
{
if (this.fileSystemCallbacks.TryDehydrateFolder(folder, out string errorMessage))
{
response.SuccessfulFolders.Add(folder);
}
else
{
response.FailedFolders.Add($"{folder}\0{errorMessage}");
}
resetFolderPaths.Append($"\"{folder.Replace(Path.DirectorySeparatorChar, GVFSConstants.GitPathSeparator)}\" ");
}
// Since modified paths could have changed with the dehydrate, the paths that were dehydrated need to be reset in the index
string resetPaths = resetFolderPaths.ToString();
GitProcess gitProcess = new GitProcess(this.enlistment);
EventMetadata resetIndexMetadata = new EventMetadata();
resetIndexMetadata.Add(nameof(resetPaths), resetPaths);
GitProcess.Result refreshIndexResult;
this.resetForDehydrateInProgress = true;
try
{
// Because we've set resetForDehydrateInProgress to true, this call to 'git reset' will also force
// the projection to be updated (required because 'git reset' will adjust the skip worktree bits in
// the index).
refreshIndexResult = gitProcess.Reset(GVFSConstants.DotGit.HeadName, resetPaths);
}
finally
{
this.resetForDehydrateInProgress = false;
}
resetIndexMetadata.Add(nameof(refreshIndexResult.ExitCode), refreshIndexResult.ExitCode);
resetIndexMetadata.Add(nameof(refreshIndexResult.Output), refreshIndexResult.Output);
resetIndexMetadata.Add(nameof(refreshIndexResult.Errors), refreshIndexResult.Errors);
resetIndexMetadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(this.HandleDehydrateFolders)}: Reset git index");
this.tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.HandleDehydrateFolders)}_ResetIndex", resetIndexMetadata);
}
else
{
response = new NamedPipeMessages.DehydrateFolders.Response(NamedPipeMessages.DehydrateFolders.MountNotReadyResult);
}
connection.TrySendResponse(response.CreateMessage());
}
private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot, string line)
{
if (!line.StartsWith(":"))
{
// Diff-tree starts with metadata we can ignore.
// Real diff lines always start with a colon
return;
}
DiffTreeResult result = DiffTreeResult.ParseFromDiffTreeLine(line, repoRoot);
if (!this.ResultIsInWhitelist(result))
{
return;
}
if (result.Operation == DiffTreeResult.Operations.Unknown ||
result.Operation == DiffTreeResult.Operations.Unmerged)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Path", result.TargetFilename);
metadata.Add("ErrorMessage", "Unexpected diff operation: " + result.Operation);
activity.RelatedError(metadata);
this.HasFailures = true;
return;
}
// Separate and enqueue all directory operations first.
if (result.SourceIsDirectory || result.TargetIsDirectory)
{
switch (result.Operation)
{
case DiffTreeResult.Operations.Delete:
if (!this.stagedDirectoryOperations.Add(result))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Filename", result.TargetFilename);
metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
}
break;
case DiffTreeResult.Operations.RenameEdit:
if (!this.stagedDirectoryOperations.Add(result))
{
// This could happen if a directory was deleted and an existing directory was renamed to replace it, but with a different case.
EventMetadata metadata = new EventMetadata();
metadata.Add("Filename", result.TargetFilename);
metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
// The target of RenameEdit is always akin to an Add, so replacing the delete is the safer thing to do.
this.stagedDirectoryOperations.Remove(result);
this.stagedDirectoryOperations.Add(result);
}
if (!result.TargetIsDirectory)
{
// Handle when a directory becomes a file.
// Files becoming directories is handled by HandleAllDirectoryOperations
this.EnqueueFileAddOperation(activity, result);
}
break;
case DiffTreeResult.Operations.Add:
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.CopyEdit:
if (!this.stagedDirectoryOperations.Add(result))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Filename", result.TargetFilename);
metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
// Replace the delete with the add to make sure we don't delete a folder from under ourselves
this.stagedDirectoryOperations.Remove(result);
this.stagedDirectoryOperations.Add(result);
}
break;
default:
activity.RelatedError("Unexpected diff operation from line: {0}", line);
break;
}
}
else
{
switch (result.Operation)
{
case DiffTreeResult.Operations.Delete:
this.EnqueueFileDeleteOperation(activity, result.TargetFilename);
break;
case DiffTreeResult.Operations.RenameEdit:
this.EnqueueFileAddOperation(activity, result);
this.EnqueueFileDeleteOperation(activity, result.SourceFilename);
break;
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.CopyEdit:
case DiffTreeResult.Operations.Add:
this.EnqueueFileAddOperation(activity, result);
break;
default:
activity.RelatedError("Unexpected diff operation from line: {0}", line);
break;
}
}
}
private void EnsureLocalCacheIsHealthy(
ITracer tracer,
GVFSEnlistment enlistment,
RetryConfig retryConfig,
GVFSConfig gvfsConfig,
CacheServerInfo cacheServer)
{
if (!Directory.Exists(enlistment.LocalCacheRoot))
{
try
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Local cache root: {enlistment.LocalCacheRoot} missing, recreating it");
Directory.CreateDirectory(enlistment.LocalCacheRoot);
}
catch (Exception e)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Exception", e.ToString());
metadata.Add("enlistment.LocalCacheRoot", enlistment.LocalCacheRoot);
tracer.RelatedError(metadata, $"{nameof(this.EnsureLocalCacheIsHealthy)}: Exception while trying to create local cache root");
this.ReportErrorAndExit(tracer, "Failed to create local cache: " + enlistment.LocalCacheRoot);
}
}
// Validate that the GitObjectsRoot directory is on disk, and that the GVFS repo is configured to use it.
// If the directory is missing (and cannot be found in the mapping file) a new key for the repo will be added
// to the mapping file and used for BOTH the GitObjectsRoot and BlobSizesRoot
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
if (Directory.Exists(enlistment.GitObjectsRoot))
{
bool gitObjectsRootInAlternates = false;
string alternatesFilePath = this.GetAlternatesPath(enlistment);
if (File.Exists(alternatesFilePath))
{
try
{
using (Stream stream = fileSystem.OpenFileStream(
alternatesFilePath,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite,
callFlushFileBuffers: false))
{
using (StreamReader reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
string alternatesLine = reader.ReadLine();
if (string.Equals(alternatesLine, enlistment.GitObjectsRoot, StringComparison.OrdinalIgnoreCase))
{
gitObjectsRootInAlternates = true;
}
}
}
}
}
catch (Exception e)
{
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Exception", e.ToString());
tracer.RelatedError(exceptionMetadata, $"{nameof(this.EnsureLocalCacheIsHealthy)}: Exception while trying to validate alternates file");
this.ReportErrorAndExit(tracer, $"Failed to validate that alternates file includes git objects root: {e.Message}");
}
}
else
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Alternates file not found");
}
if (!gitObjectsRootInAlternates)
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: GitObjectsRoot ({enlistment.GitObjectsRoot}) missing from alternates files, recreating alternates");
string error;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out error))
{
this.ReportErrorAndExit(tracer, $"Failed to update alternates file to include git objects root: {error}");
}
}
}
else
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: GitObjectsRoot ({enlistment.GitObjectsRoot}) missing, determining new root");
if (cacheServer == null)
{
cacheServer = CacheServerResolver.GetCacheServerFromConfig(enlistment);
}
string error;
if (gvfsConfig == null)
{
if (retryConfig == null)
{
if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error))
{
this.ReportErrorAndExit(tracer, "Failed to determine GVFS timeout and max retries: " + error);
}
}
gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig);
}
string localCacheKey;
LocalCacheResolver localCacheResolver = new LocalCacheResolver(enlistment);
if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers(
tracer,
gvfsConfig,
cacheServer,
enlistment.LocalCacheRoot,
localCacheKey: out localCacheKey,
errorMessage: out error))
{
this.ReportErrorAndExit(tracer, $"Previous git objects root ({enlistment.GitObjectsRoot}) not found, and failed to determine new local cache key: {error}");
}
EventMetadata metadata = new EventMetadata();
metadata.Add("localCacheRoot", enlistment.LocalCacheRoot);
metadata.Add("localCacheKey", localCacheKey);
metadata.Add(TracingConstants.MessageKey.InfoMessage, "Initializing and persisting updated paths");
tracer.RelatedEvent(EventLevel.Informational, "GVFSVerb_EnsureLocalCacheIsHealthy_InitializePathsFromKey", metadata);
enlistment.InitializeCachePathsFromKey(enlistment.LocalCacheRoot, localCacheKey);
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Creating GitObjectsRoot ({enlistment.GitObjectsRoot}), GitPackRoot ({enlistment.GitPackRoot}), and BlobSizesRoot ({enlistment.BlobSizesRoot})");
try
{
Directory.CreateDirectory(enlistment.GitObjectsRoot);
Directory.CreateDirectory(enlistment.GitPackRoot);
}
catch (Exception e)
{
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Exception", e.ToString());
exceptionMetadata.Add("enlistment.LocalCacheRoot", enlistment.LocalCacheRoot);
exceptionMetadata.Add("enlistment.GitObjectsRoot", enlistment.GitObjectsRoot);
exceptionMetadata.Add("enlistment.GitPackRoot", enlistment.GitPackRoot);
exceptionMetadata.Add("enlistment.BlobSizesRoot", enlistment.BlobSizesRoot);
tracer.RelatedError(exceptionMetadata, $"{nameof(this.InitializeLocalCacheAndObjectsPaths)}: Exception while trying to create objects, pack, and sizes folders");
this.ReportErrorAndExit(tracer, "Failed to create objects, pack, and sizes folders");
}
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Creating new alternates file");
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out error))
{
this.ReportErrorAndExit(tracer, $"Failed to update alterates file with new objects path: {error}");
}
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Saving git objects root ({enlistment.GitObjectsRoot}) in repo metadata");
RepoMetadata.Instance.SetGitObjectsRoot(enlistment.GitObjectsRoot);
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Saving blob sizes root ({enlistment.BlobSizesRoot}) in repo metadata");
RepoMetadata.Instance.SetBlobSizesRoot(enlistment.BlobSizesRoot);
}
// Validate that the BlobSizesRoot folder is on disk.
// Note that if a user performed an action that resulted in the entire .gvfscache being deleted, the code above
// for validating GitObjectsRoot will have already taken care of generating a new key and setting a new enlistment.BlobSizesRoot path
if (!Directory.Exists(enlistment.BlobSizesRoot))
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: BlobSizesRoot ({enlistment.BlobSizesRoot}) not found, re-creating");
try
{
Directory.CreateDirectory(enlistment.BlobSizesRoot);
}
catch (Exception e)
{
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Exception", e.ToString());
exceptionMetadata.Add("enlistment.BlobSizesRoot", enlistment.BlobSizesRoot);
tracer.RelatedError(exceptionMetadata, $"{nameof(this.InitializeLocalCacheAndObjectsPaths)}: Exception while trying to create blob sizes folder");
this.ReportErrorAndExit(tracer, "Failed to create blob sizes folder");
}
}
}
public virtual bool TryCopyBlobToFile(string sha, IEnumerable <string> destinations, out long bytesWritten)
{
IntPtr objHandle;
if (Native.RevParseSingle(out objHandle, this.repoHandle, sha) != Native.SuccessCode)
{
bytesWritten = 0;
EventMetadata metadata = new EventMetadata();
metadata.Add("ObjectSha", sha);
metadata.Add("ErrorMessage", "Couldn't find object");
this.tracer.RelatedError(metadata);
return(false);
}
try
{
// Avoid marshalling raw content by using byte* and native writes
unsafe
{
switch (Native.Object.GetType(objHandle))
{
case Native.ObjectTypes.Blob:
byte *originalData = Native.Blob.GetRawContent(objHandle);
long originalSize = Native.Blob.GetRawSize(objHandle);
foreach (string destination in destinations)
{
try
{
using (SafeFileHandle fileHandle = OpenForWrite(this.tracer, destination))
{
if (fileHandle.IsInvalid)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
byte *data = originalData;
long size = originalSize;
uint written = 0;
while (size > 0)
{
uint toWrite = size < uint.MaxValue ? (uint)size : uint.MaxValue;
if (!Native.WriteFile(fileHandle, data, toWrite, out written, IntPtr.Zero))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
size -= written;
data = data + written;
}
}
}
catch (Exception e)
{
this.tracer.RelatedError("Exception writing {0}: {1}", destination, e);
throw;
}
}
bytesWritten = originalSize * destinations.Count();
break;
default:
throw new NotSupportedException("Copying object types other than blobs is not supported.");
}
}
}
finally
{
Native.Object.Free(objHandle);
}
return(true);
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public override void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitId(branchOrCommit);
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);
// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
// Since pack indexer is the last to finish before checkout finishes, it should propagate completion.
// This prevents availableObjects from completing before packIndexer can push its objects through this link.
checkout.AvailableBlobShas.CompleteAdding();
checkout.WaitForCompletion();
this.HasFailures |= checkout.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
// Update the refspec before setting the upstream or git will complain the remote branch doesn't exist
this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational))
{
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch);
if (result.HasErrors)
{
activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors);
this.HasFailures = true;
}
}
}
bool shouldSignIndex = !this.GetIsIndexSigningOff();
// Update the index
EventMetadata updateIndexMetadata = new EventMetadata();
updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex);
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata))
{
Index sourceIndex = this.GetSourceIndex();
GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex);
indexGen.CreateFromHeadTree(indexVersion: 2);
this.HasFailures |= indexGen.HasFailures;
if (!indexGen.HasFailures)
{
Index newIndex = new Index(
this.Enlistment.EnlistmentRoot,
this.Tracer,
Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName),
readOnly: false);
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;
newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex);
}
}
}
}
private void HandleAllDirectoryOperations()
{
DiffTreeResult treeOp;
while (this.diff.DirectoryOperations.TryDequeue(out treeOp))
{
if (this.HasFailures)
{
return;
}
switch (treeOp.Operation)
{
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.Add:
try
{
Directory.CreateDirectory(treeOp.TargetFilename);
}
catch (Exception ex)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "CreateDirectory");
metadata.Add("Path", treeOp.TargetFilename);
metadata.Add("ErrorMessage", ex.Message);
this.tracer.RelatedError(metadata);
this.HasFailures = true;
}
break;
case DiffTreeResult.Operations.Delete:
try
{
if (Directory.Exists(treeOp.TargetFilename))
{
PhysicalFileSystem.RecursiveDelete(treeOp.TargetFilename);
}
}
catch (Exception ex)
{
// We are deleting directories and subdirectories in parallel
if (Directory.Exists(treeOp.TargetFilename))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "DeleteDirectory");
metadata.Add("Path", treeOp.TargetFilename);
metadata.Add("ErrorMessage", ex.Message);
this.tracer.RelatedError(metadata);
this.HasFailures = true;
}
}
break;
case DiffTreeResult.Operations.RenameEdit:
try
{
// If target is file, just delete the source.
if (!treeOp.TargetIsDirectory)
{
if (Directory.Exists(treeOp.SourceFilename))
{
PhysicalFileSystem.RecursiveDelete(treeOp.SourceFilename);
}
}
else
{
// If target is directory, delete any source file and add
if (!treeOp.SourceIsDirectory)
{
if (File.Exists(treeOp.SourceFilename))
{
File.Delete(treeOp.SourceFilename);
}
goto case DiffTreeResult.Operations.Add;
}
else
{
// Source and target are directory, do a move and let later steps handle any sub-edits.
Directory.Move(treeOp.SourceFilename, treeOp.TargetFilename);
}
}
}
catch (Exception ex)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "RenameDirectory");
metadata.Add("Path", treeOp.TargetFilename);
metadata.Add("ErrorMessage", ex.Message);
this.tracer.RelatedError(metadata);
this.HasFailures = true;
}
break;
default:
this.tracer.RelatedError("Ignoring unexpected Tree Operation {0}: {1}", treeOp.TargetFilename, treeOp.Operation);
continue;
}
if (Interlocked.Increment(ref this.directoryOpCount) % NumOperationsPerStatus == 0)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("DirectoryOperationsQueued", this.diff.DirectoryOperations.Count);
metadata.Add("DirectoryOperationsCompleted", this.directoryOpCount);
this.tracer.RelatedEvent(EventLevel.Informational, "CheckoutStatus", metadata);
}
}
}
private void HandleRequest(ITracer tracer, string request, NamedPipeServer.Connection connection)
{
NamedPipeMessages.Message message = NamedPipeMessages.Message.FromString(request);
if (string.IsNullOrWhiteSpace(message.Header))
{
return;
}
using (ITracer activity = this.tracer.StartActivity(message.Header, EventLevel.Informational, new EventMetadata {
{ "request", request }
}))
{
switch (message.Header)
{
case NamedPipeMessages.RegisterRepoRequest.Header:
try
{
NamedPipeMessages.RegisterRepoRequest mountRequest = NamedPipeMessages.RegisterRepoRequest.FromMessage(message);
RegisterRepoHandler mountHandler = new RegisterRepoHandler(activity, this.repoRegistry, connection, mountRequest);
mountHandler.Run();
}
catch (SerializationException ex)
{
activity.RelatedError("Could not deserialize mount request: {0}", ex.Message);
}
break;
case NamedPipeMessages.UnregisterRepoRequest.Header:
try
{
NamedPipeMessages.UnregisterRepoRequest unmountRequest = NamedPipeMessages.UnregisterRepoRequest.FromMessage(message);
UnregisterRepoHandler unmountHandler = new UnregisterRepoHandler(activity, this.repoRegistry, connection, unmountRequest);
unmountHandler.Run();
}
catch (SerializationException ex)
{
activity.RelatedError("Could not deserialize unmount request: {0}", ex.Message);
}
break;
case NamedPipeMessages.AttachGvFltRequest.Header:
try
{
NamedPipeMessages.AttachGvFltRequest attachRequest = NamedPipeMessages.AttachGvFltRequest.FromMessage(message);
AttachGvFltHandler attachHandler = new AttachGvFltHandler(activity, connection, attachRequest);
attachHandler.Run();
}
catch (SerializationException ex)
{
activity.RelatedError("Could not deserialize attach volume request: {0}", ex.Message);
}
break;
case NamedPipeMessages.ExcludeFromAntiVirusRequest.Header:
try
{
NamedPipeMessages.ExcludeFromAntiVirusRequest excludeFromAntiVirusRequest = NamedPipeMessages.ExcludeFromAntiVirusRequest.FromMessage(message);
ExcludeFromAntiVirusHandler excludeHandler = new ExcludeFromAntiVirusHandler(activity, connection, excludeFromAntiVirusRequest);
excludeHandler.Run();
}
catch (SerializationException ex)
{
activity.RelatedError("Could not deserialize exclude from antivirus request: {0}", ex.Message);
}
break;
default:
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("Header", message.Header);
this.tracer.RelatedWarning(metadata, "HandleNewConnection: Unknown request", Keywords.Telemetry);
connection.TrySendResponse(NamedPipeMessages.UnknownRequest);
break;
}
}
}
protected GitEndPointResponseData SendRequest(
long requestId,
Uri requestUri,
HttpMethod httpMethod,
string requestContent,
MediaTypeWithQualityHeaderValue acceptType = null)
{
string authString;
string errorMessage;
if (!this.authentication.TryGetCredentials(this.Tracer, out authString, out errorMessage))
{
return(new GitEndPointResponseData(
HttpStatusCode.Unauthorized,
new GitObjectsHttpException(HttpStatusCode.Unauthorized, errorMessage),
shouldRetry: false));
}
HttpRequestMessage request = new HttpRequestMessage(httpMethod, requestUri);
request.Headers.UserAgent.Add(this.userAgentHeader);
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", authString);
if (acceptType != null)
{
request.Headers.Accept.Add(acceptType);
}
if (requestContent != null)
{
request.Content = new StringContent(requestContent, Encoding.UTF8, "application/json");
}
EventMetadata responseMetadata = new EventMetadata();
responseMetadata.Add("RequestId", requestId);
try
{
HttpResponseMessage response = this.client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).GetAwaiter().GetResult();
responseMetadata.Add("CacheName", GetSingleHeaderOrEmpty(response.Headers, "X-Cache-Name"));
responseMetadata.Add("StatusCode", response.StatusCode);
if (response.StatusCode == HttpStatusCode.OK)
{
string contentType = GetSingleHeaderOrEmpty(response.Content.Headers, "Content-Type");
responseMetadata.Add("ContentType", contentType);
this.authentication.ConfirmCredentialsWorked(authString);
Stream responseStream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult();
return(new GitEndPointResponseData(response.StatusCode, contentType, responseStream));
}
else
{
errorMessage = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
int statusInt = (int)response.StatusCode;
if (string.IsNullOrWhiteSpace(errorMessage))
{
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
if (this.authentication.RevokeAndCheckCanRetry(authString))
{
return(new GitEndPointResponseData(
response.StatusCode,
new GitObjectsHttpException(response.StatusCode, "Server returned error code 401 (Unauthorized). Your PAT may be expired."),
shouldRetry: true));
}
else
{
return(new GitEndPointResponseData(
response.StatusCode,
new GitObjectsHttpException(response.StatusCode, "Server returned error code 401 (Unauthorized) after successfully renewing your PAT. You may not have access to this repo"),
shouldRetry: false));
}
}
else
{
errorMessage = string.Format("Server returned error code {0} ({1})", statusInt, response.StatusCode);
}
}
return(new GitEndPointResponseData(response.StatusCode, new GitObjectsHttpException(response.StatusCode, errorMessage), ShouldRetry(response.StatusCode)));
}
}
catch (TaskCanceledException)
{
errorMessage = string.Format("Request to {0} timed out", requestUri);
return(new GitEndPointResponseData(HttpStatusCode.RequestTimeout, new GitObjectsHttpException(HttpStatusCode.RequestTimeout, errorMessage), shouldRetry: true));
}
catch (WebException ex)
{
return(new GitEndPointResponseData(HttpStatusCode.InternalServerError, ex, shouldRetry: true));
}
finally
{
this.Tracer.RelatedEvent(EventLevel.Informational, "NetworkResponse", responseMetadata);
}
}
public virtual bool TryCopyBlobContentStream(string blobSha, Action <Stream, long> writeAction)
{
string blobPath = Path.Combine(
this.enlistment.GitObjectsRoot,
blobSha.Substring(0, 2),
blobSha.Substring(2));
bool corruptLooseObject = false;
try
{
if (File.Exists(blobPath))
{
using (Stream file = new FileStream(blobPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
// The DeflateStream header starts 2 bytes into the gzip header, but they are otherwise compatible
file.Position = 2;
using (DeflateStream deflate = new DeflateStream(file, CompressionMode.Decompress))
{
long size;
if (!ReadLooseObjectHeader(deflate, out size))
{
corruptLooseObject = true;
return(false);
}
writeAction(deflate, size);
return(true);
}
}
}
}
catch (InvalidDataException ex)
{
corruptLooseObject = true;
EventMetadata metadata = new EventMetadata();
metadata.Add("blobPath", blobPath);
metadata.Add("Exception", ex.ToString());
metadata.Add("ErrorMessage", "TryCopyBlobContentStream: Failed to stream blob (InvalidDataException)");
this.tracer.RelatedError(metadata);
return(false);
}
catch (IOException ex)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("blobPath", blobPath);
metadata.Add("Exception", ex.ToString());
metadata.Add("ErrorMessage", "TryCopyBlobContentStream: Failed to stream blob from disk");
this.tracer.RelatedError(metadata);
return(false);
}
finally
{
if (corruptLooseObject)
{
string corruptBlobsFolderPath = Path.Combine(this.enlistment.EnlistmentRoot, GVFSConstants.DotGVFS.CorruptObjectsPath);
string corruptBlobPath = Path.Combine(corruptBlobsFolderPath, Path.GetRandomFileName());
EventMetadata metadata = new EventMetadata();
metadata.Add("blobPath", blobPath);
metadata.Add("corruptBlobPath", corruptBlobPath);
metadata.Add("Message", "TryCopyBlobContentStream: Renaming corrupt loose object");
this.tracer.RelatedEvent(EventLevel.Informational, "TryCopyBlobContentStream_RenameCorruptObject", metadata);
try
{
this.fileSystem.CreateDirectory(corruptBlobsFolderPath);
File.Move(blobPath, corruptBlobPath);
}
catch (Exception e)
{
metadata = new EventMetadata();
metadata.Add("blobPath", blobPath);
metadata.Add("blobBackupPath", corruptBlobPath);
metadata.Add("Exception", e.ToString());
metadata.Add("Message", "TryCopyBlobContentStream: Failed to rename corrupt loose object");
this.tracer.RelatedEvent(EventLevel.Warning, "TryCopyBlobContentStream_RenameCorruptObjectFailed", metadata);
}
}
}
bool copyBlobResult;
if (!this.libgit2RepoPool.TryInvoke(repo => repo.TryCopyBlob(blobSha, writeAction), out copyBlobResult))
{
return(false);
}
return(copyBlobResult);
}
public Dictionary <string, RepoRegistration> ReadRegistry()
{
Dictionary <string, RepoRegistration> allRepos = new Dictionary <string, RepoRegistration>(StringComparer.OrdinalIgnoreCase);
using (Stream stream = this.fileSystem.OpenFileStream(
Path.Combine(this.registryParentFolderPath, RegistryName),
FileMode.OpenOrCreate,
FileAccess.Read,
FileShare.Read,
callFlushFileBuffers: false))
{
using (StreamReader reader = new StreamReader(stream))
{
string versionString = reader.ReadLine();
int version;
if (!int.TryParse(versionString, out version) ||
version > RegistryVersion)
{
if (versionString != null)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("OnDiskVersion", versionString);
metadata.Add("ExpectedVersion", versionString);
this.tracer.RelatedError(metadata, "ReadRegistry: Unsupported version");
}
return(allRepos);
}
while (!reader.EndOfStream)
{
string entry = reader.ReadLine();
if (entry.Length > 0)
{
try
{
RepoRegistration registration = RepoRegistration.FromJson(entry);
string errorMessage;
string normalizedEnlistmentRootPath = registration.EnlistmentRoot;
if (GVFSPlatform.Instance.FileSystem.TryGetNormalizedPath(registration.EnlistmentRoot, out normalizedEnlistmentRootPath, out errorMessage))
{
if (!normalizedEnlistmentRootPath.Equals(registration.EnlistmentRoot, StringComparison.OrdinalIgnoreCase))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("registration.EnlistmentRoot", registration.EnlistmentRoot);
metadata.Add(nameof(normalizedEnlistmentRootPath), normalizedEnlistmentRootPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(this.ReadRegistry)}: Mapping registered enlistment root to final path");
this.tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.ReadRegistry)}_NormalizedPathMapping", metadata);
}
}
else
{
EventMetadata metadata = new EventMetadata();
metadata.Add("registration.EnlistmentRoot", registration.EnlistmentRoot);
metadata.Add("NormalizedEnlistmentRootPath", normalizedEnlistmentRootPath);
metadata.Add("ErrorMessage", errorMessage);
this.tracer.RelatedWarning(metadata, $"{nameof(this.ReadRegistry)}: Failed to get normalized path name for registed enlistment root");
}
if (normalizedEnlistmentRootPath != null)
{
allRepos[normalizedEnlistmentRootPath] = registration;
}
}
catch (Exception e)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("entry", entry);
metadata.Add("Exception", e.ToString());
this.tracer.RelatedError(metadata, "ReadRegistry: Failed to read entry");
}
}
}
}
}
return(allRepos);
}
private void HandleAllDirectoryOperations()
{
DiffTreeResult treeOp;
while (this.diff.DirectoryOperations.TryDequeue(out treeOp))
{
if (this.HasFailures)
{
return;
}
switch (treeOp.Operation)
{
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.Add:
try
{
Directory.CreateDirectory(treeOp.TargetPath);
}
catch (Exception ex)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "CreateDirectory");
metadata.Add(nameof(treeOp.TargetPath), treeOp.TargetPath);
this.tracer.RelatedError(metadata, ex.Message);
this.HasFailures = true;
}
break;
case DiffTreeResult.Operations.Delete:
try
{
if (Directory.Exists(treeOp.TargetPath))
{
PhysicalFileSystem.RecursiveDelete(treeOp.TargetPath);
}
}
catch (Exception ex)
{
// We are deleting directories and subdirectories in parallel
if (Directory.Exists(treeOp.TargetPath))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "DeleteDirectory");
metadata.Add(nameof(treeOp.TargetPath), treeOp.TargetPath);
this.tracer.RelatedError(metadata, ex.Message);
this.HasFailures = true;
}
}
break;
default:
this.tracer.RelatedError("Ignoring unexpected Tree Operation {0}: {1}", treeOp.TargetPath, treeOp.Operation);
continue;
}
if (Interlocked.Increment(ref this.directoryOpCount) % NumOperationsPerStatus == 0)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("DirectoryOperationsQueued", this.diff.DirectoryOperations.Count);
metadata.Add("DirectoryOperationsCompleted", this.directoryOpCount);
this.tracer.RelatedEvent(EventLevel.Informational, "CheckoutStatus", metadata);
}
}
}
public FileSystemCallbacks(
GVFSContext context,
GVFSGitObjects gitObjects,
RepoMetadata repoMetadata,
BlobSizes blobSizes,
GitIndexProjection gitIndexProjection,
BackgroundFileSystemTaskRunner backgroundFileSystemTaskRunner,
FileSystemVirtualizer fileSystemVirtualizer,
GitStatusCache gitStatusCache = null)
{
this.logsHeadFileProperties = null;
this.context = context;
this.fileSystemVirtualizer = fileSystemVirtualizer;
this.placeHolderCreationCount = new ConcurrentDictionary <string, PlaceHolderCreateCounter>(StringComparer.OrdinalIgnoreCase);
this.newlyCreatedFileAndFolderPaths = new ConcurrentHashSet <string>(StringComparer.OrdinalIgnoreCase);
string error;
if (!ModifiedPathsDatabase.TryLoadOrCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths),
this.context.FileSystem,
out this.modifiedPaths,
out error))
{
throw new InvalidRepoException(error);
}
this.BlobSizes = blobSizes;
this.BlobSizes.Initialize();
PlaceholderListDatabase placeholders;
if (!PlaceholderListDatabase.TryCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList),
this.context.FileSystem,
out placeholders,
out error))
{
throw new InvalidRepoException(error);
}
this.GitIndexProjection = gitIndexProjection ?? new GitIndexProjection(
context,
gitObjects,
this.BlobSizes,
repoMetadata,
fileSystemVirtualizer,
placeholders,
this.modifiedPaths);
if (backgroundFileSystemTaskRunner != null)
{
this.backgroundFileSystemTaskRunner = backgroundFileSystemTaskRunner;
this.backgroundFileSystemTaskRunner.SetCallbacks(
this.PreBackgroundOperation,
this.ExecuteBackgroundOperation,
this.PostBackgroundOperation);
}
else
{
this.backgroundFileSystemTaskRunner = new BackgroundFileSystemTaskRunner(
this.context,
this.PreBackgroundOperation,
this.ExecuteBackgroundOperation,
this.PostBackgroundOperation,
Path.Combine(context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.BackgroundFileSystemTasks));
}
this.enableGitStatusCache = gitStatusCache != null;
// If the status cache is not enabled, create a dummy GitStatusCache that will never be initialized
// This lets us from having to add null checks to callsites into GitStatusCache.
this.gitStatusCache = gitStatusCache ?? new GitStatusCache(context, TimeSpan.Zero);
this.logsHeadPath = Path.Combine(this.context.Enlistment.LocalStorageRoot, GVFSConstants.DotGit.Logs.Head);
EventMetadata metadata = new EventMetadata();
metadata.Add("placeholders.Count", placeholders.EstimatedCount);
metadata.Add("background.Count", this.backgroundFileSystemTaskRunner.Count);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(FileSystemCallbacks)} created");
this.context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(FileSystemCallbacks)}_Constructor", metadata);
}
private static bool TryCopyNativeLibToAppDirectory(ITracer tracer, PhysicalFileSystem fileSystem, string gvfsAppDirectory)
{
string installFilePath;
string appFilePath;
GetNativeLibPaths(gvfsAppDirectory, out installFilePath, out appFilePath);
EventMetadata pathMetadata = CreateEventMetadata();
pathMetadata.Add(nameof(gvfsAppDirectory), gvfsAppDirectory);
pathMetadata.Add(nameof(installFilePath), installFilePath);
pathMetadata.Add(nameof(appFilePath), appFilePath);
if (fileSystem.FileExists(installFilePath))
{
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryCopyNativeLibToAppDirectory)}_CopyingNativeLib", pathMetadata);
try
{
fileSystem.CopyFile(installFilePath, appFilePath, overwrite: true);
try
{
Common.NativeMethods.FlushFileBuffers(appFilePath);
}
catch (Win32Exception e)
{
EventMetadata metadata = CreateEventMetadata(e);
metadata.Add(nameof(appFilePath), appFilePath);
metadata.Add(nameof(installFilePath), installFilePath);
tracer.RelatedWarning(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: Win32Exception while trying to flush file buffers", Keywords.Telemetry);
}
}
catch (UnauthorizedAccessException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedError(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: UnauthorizedAccessException caught while trying to copy native lib");
return(false);
}
catch (DirectoryNotFoundException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedError(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: DirectoryNotFoundException caught while trying to copy native lib");
return(false);
}
catch (FileNotFoundException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedError(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: FileNotFoundException caught while trying to copy native lib");
return(false);
}
catch (IOException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedWarning(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: IOException caught while trying to copy native lib");
if (fileSystem.FileExists(appFilePath))
{
tracer.RelatedWarning(
CreateEventMetadata(),
"Could not copy native lib to app directory, but file already exists, continuing with install",
Keywords.Telemetry);
}
else
{
tracer.RelatedError($"{nameof(TryCopyNativeLibToAppDirectory)}: Failed to copy native lib to app directory");
return(false);
}
}
}
else
{
tracer.RelatedError(pathMetadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: Native lib does not exist in install directory");
return(false);
}
return(true);
}
private int ExecuteWithExitCode()
{
// CmdParser doesn't strip quotes, and Path.Combine will throw
this.GitBinPath = this.GitBinPath.Replace("\"", string.Empty);
if (!GVFSPlatform.Instance.GitInstallation.GitExists(this.GitBinPath))
{
Console.WriteLine(
"Could not find git.exe {0}",
!string.IsNullOrWhiteSpace(this.GitBinPath) ? "at " + this.GitBinPath : "on %PATH%");
return(ExitFailure);
}
if (this.Commit != null && this.Branch != null)
{
Console.WriteLine("Cannot specify both a commit sha and a branch name.");
return(ExitFailure);
}
if (this.ForceCheckout && !this.Checkout)
{
Console.WriteLine("Cannot use --force-checkout option without --checkout option.");
return(ExitFailure);
}
this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount;
this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads);
this.IndexThreadCount = this.IndexThreadCount > 0 ? this.IndexThreadCount : Environment.ProcessorCount;
this.CheckoutThreadCount = this.CheckoutThreadCount > 0 ? this.CheckoutThreadCount : Environment.ProcessorCount;
this.GitBinPath = !string.IsNullOrWhiteSpace(this.GitBinPath) ? this.GitBinPath : GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath();
GitEnlistment enlistment = GitEnlistment.CreateFromCurrentDirectory(this.GitBinPath);
if (enlistment == null)
{
Console.WriteLine("Must be run within a git repo");
return(ExitFailure);
}
string commitish = this.Commit ?? this.Branch;
if (string.IsNullOrWhiteSpace(commitish))
{
GitProcess.Result result = new GitProcess(enlistment).GetCurrentBranchName();
if (result.HasErrors || string.IsNullOrWhiteSpace(result.Output))
{
Console.WriteLine("Could not retrieve current branch name: " + result.Errors);
return(ExitFailure);
}
commitish = result.Output.Trim();
}
Guid parentActivityId = Guid.Empty;
if (!string.IsNullOrWhiteSpace(this.ParentActivityId) && !Guid.TryParse(this.ParentActivityId, out parentActivityId))
{
Console.WriteLine("The ParentActivityId provided (" + this.ParentActivityId + ") is not a valid GUID.");
}
using (JsonTracer tracer = new JsonTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch", enlistmentId: null, mountId: null, disableTelemetry: true))
{
if (this.Verbose)
{
tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
else
{
tracer.AddPrettyConsoleEventListener(EventLevel.Error, Keywords.Any);
}
string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch");
tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any);
CacheServerInfo cacheServer = new CacheServerInfo(this.GetRemoteUrl(enlistment), null);
tracer.WriteStartEvent(
enlistment.EnlistmentRoot,
enlistment.RepoUrl,
cacheServer.Url,
new EventMetadata
{
{ "TargetCommitish", commitish },
{ "Checkout", this.Checkout },
});
string error;
if (!enlistment.Authentication.TryInitialize(tracer, enlistment, out error))
{
tracer.RelatedError(error);
Console.WriteLine(error);
return(ExitFailure);
}
RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
BlobPrefetcher prefetcher = this.GetFolderPrefetcher(tracer, enlistment, cacheServer, retryConfig);
if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, out error))
{
tracer.RelatedError(error);
Console.WriteLine(error);
return(ExitFailure);
}
bool isSuccess;
try
{
Func <bool> doPrefetch =
() =>
{
try
{
bool isBranch = this.Commit == null;
prefetcher.Prefetch(commitish, isBranch);
return(!prefetcher.HasFailures);
}
catch (BlobPrefetcher.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
isSuccess = doPrefetch();
}
else
{
isSuccess = ConsoleHelper.ShowStatusWhileRunning(
doPrefetch,
"Fetching",
output: Console.Out,
showSpinner: !Console.IsOutputRedirected,
gvfsLogEnlistmentRoot: null);
Console.WriteLine();
Console.WriteLine("See the full log at " + fastfetchLogFile);
}
isSuccess &= !prefetcher.HasFailures;
}
catch (AggregateException e)
{
isSuccess = false;
foreach (Exception ex in e.Flatten().InnerExceptions)
{
tracer.RelatedError(ex.ToString());
}
}
catch (Exception e)
{
isSuccess = false;
tracer.RelatedError(e.ToString());
}
EventMetadata stopMetadata = new EventMetadata();
stopMetadata.Add("Success", isSuccess);
tracer.Stop(stopMetadata);
return(isSuccess ? ExitSuccess : ExitFailure);
}
}
private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileSystem fileSystem)
{
EventMetadata metadata = CreateEventMetadata();
const int ProjFSNotAnOptionalFeature = 2;
const int ProjFSEnabled = 3;
const int ProjFSDisabled = 4;
ProcessResult getOptionalFeatureResult = CallPowershellCommand(
"$var=(Get-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + "); if($var -eq $null){exit " +
ProjFSNotAnOptionalFeature + "}else{if($var.State -eq 'Enabled'){exit " + ProjFSEnabled + "}else{exit " + ProjFSDisabled + "}}");
bool projFSEnabled = false;
switch (getOptionalFeatureResult.ExitCode)
{
case ProjFSNotAnOptionalFeature:
tracer.RelatedError($"{nameof(TryEnableProjFSOptionalFeature)}: {OptionalFeatureName} optional feature is missing");
break;
case ProjFSEnabled:
tracer.RelatedEvent(
EventLevel.Informational,
$"{nameof(TryEnableProjFSOptionalFeature)}_ClientProjFSAlreadyEnabled",
metadata,
Keywords.Network);
projFSEnabled = true;
break;
case ProjFSDisabled:
ProcessResult enableOptionalFeatureResult = CallPowershellCommand("try {Enable-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + " -NoRestart}catch{exit 1}");
if (enableOptionalFeatureResult.ExitCode == 0)
{
metadata.Add(TracingConstants.MessageKey.InfoMessage, "Enabled ProjFS optional feature");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryEnableProjFSOptionalFeature)}_ClientProjFSDisabled", metadata);
projFSEnabled = true;
break;
}
metadata.Add("enableOptionalFeatureResult.ExitCode", enableOptionalFeatureResult.ExitCode);
metadata.Add("enableOptionalFeatureResult.Output", enableOptionalFeatureResult.Output);
metadata.Add("enableOptionalFeatureResult.Errors", enableOptionalFeatureResult.Errors);
tracer.RelatedError(metadata, $"{nameof(TryEnableProjFSOptionalFeature)}: Failed to enable optional feature");
break;
default:
metadata.Add("getOptionalFeatureResult.ExitCode", getOptionalFeatureResult.ExitCode);
metadata.Add("getOptionalFeatureResult.Output", getOptionalFeatureResult.Output);
metadata.Add("getOptionalFeatureResult.Errors", getOptionalFeatureResult.Errors);
tracer.RelatedError(metadata, $"{nameof(TryEnableProjFSOptionalFeature)}: Unexpected result");
break;
}
if (projFSEnabled)
{
if (IsNativeLibInstalled(tracer, fileSystem))
{
return(true);
}
tracer.RelatedError($"{nameof(TryEnableProjFSOptionalFeature)}: {OptionalFeatureName} enabled, but native ProjFS library is not on path");
}
return(false);
}
private Result TryClone(
JsonTracer tracer,
GVFSEnlistment enlistment,
CacheServerInfo cacheServer,
RetryConfig retryConfig,
ServerGVFSConfig serverGVFSConfig,
string resolvedLocalCacheRoot)
{
Result pipeResult;
using (NamedPipeServer pipeServer = this.StartNamedPipe(tracer, enlistment, out pipeResult))
{
if (!pipeResult.Success)
{
return(pipeResult);
}
using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig))
{
GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null);
if (refs == null)
{
return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl)));
}
if (this.Branch == null)
{
this.Branch = refs.GetDefaultBranch();
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata);
}
else
{
if (!refs.HasBranch(this.Branch))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata);
string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch);
return(new Result(errorMessage));
}
}
if (!enlistment.TryCreateEnlistmentSubFolders())
{
string error = "Could not create enlistment directories";
tracer.RelatedError(error);
return(new Result(error));
}
if (!GVFSPlatform.Instance.FileSystem.IsFileSystemSupported(enlistment.EnlistmentRoot, out string fsError))
{
string error = $"FileSystem unsupported: {fsError}";
tracer.RelatedError(error);
return(new Result(error));
}
string localCacheError;
if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, serverGVFSConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError))
{
tracer.RelatedError(localCacheError);
return(new Result(localCacheError));
}
// There's no need to use CreateDirectoryAccessibleByAuthUsers as these directories will inherit
// the ACLs used to create LocalCacheRoot
Directory.CreateDirectory(enlistment.GitObjectsRoot);
Directory.CreateDirectory(enlistment.GitPackRoot);
Directory.CreateDirectory(enlistment.BlobSizesRoot);
return(this.CreateClone(tracer, enlistment, objectRequestor, refs, this.Branch));
}
}
}
private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate)
{
EventMetadata metadata = new EventMetadata();
FileSystemTaskResult result;
switch (gitUpdate.Operation)
{
case FileSystemTask.OperationType.OnFileCreated:
case FileSystemTask.OperationType.OnFailedPlaceholderDelete:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFileRenamed:
metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = FileSystemTaskResult.Success;
if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) && !IsPathInsideDotGit(gitUpdate.OldVirtualPath))
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.OldVirtualPath);
}
if (result == FileSystemTaskResult.Success &&
!string.IsNullOrEmpty(gitUpdate.VirtualPath) &&
!IsPathInsideDotGit(gitUpdate.VirtualPath))
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
}
break;
case FileSystemTask.OperationType.OnFileDeleted:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFileOverwritten:
case FileSystemTask.OperationType.OnFileSuperseded:
case FileSystemTask.OperationType.OnFileConvertedToFull:
case FileSystemTask.OperationType.OnFailedPlaceholderUpdate:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFolderCreated:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
break;
case FileSystemTask.OperationType.OnFolderRenamed:
result = FileSystemTaskResult.Success;
metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
metadata.Add("virtualPath", gitUpdate.VirtualPath);
// An empty destination path means the folder was renamed to somewhere outside of the repo
// Note that only full folders can be moved\renamed, and so there will already be a recursive
// sparse-checkout entry for the virtualPath of the folder being moved (meaning that no
// additional work is needed for any files\folders inside the folder being moved)
if (!string.IsNullOrEmpty(gitUpdate.VirtualPath))
{
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
if (result == FileSystemTaskResult.Success)
{
Queue <string> relativeFolderPaths = new Queue <string>();
relativeFolderPaths.Enqueue(gitUpdate.VirtualPath);
// Add all the files in the renamed folder to the always_exclude file
while (relativeFolderPaths.Count > 0)
{
string folderPath = relativeFolderPaths.Dequeue();
if (result == FileSystemTaskResult.Success)
{
try
{
foreach (DirectoryItemInfo itemInfo in this.context.FileSystem.ItemsInDirectory(Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, folderPath)))
{
string itemVirtualPath = Path.Combine(folderPath, itemInfo.Name);
if (itemInfo.IsDirectory)
{
relativeFolderPaths.Enqueue(itemVirtualPath);
}
else
{
string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length);
result = this.TryAddModifiedPath(itemVirtualPath, isFolder: false);
}
}
}
catch (DirectoryNotFoundException)
{
// DirectoryNotFoundException can occur when the renamed folder (or one of its children) is
// deleted prior to the background thread running
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Area", "ExecuteBackgroundOperation");
exceptionMetadata.Add("Operation", gitUpdate.Operation.ToString());
exceptionMetadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
exceptionMetadata.Add("virtualPath", gitUpdate.VirtualPath);
exceptionMetadata.Add(TracingConstants.MessageKey.InfoMessage, "DirectoryNotFoundException while traversing folder path");
exceptionMetadata.Add("folderPath", folderPath);
this.context.Tracer.RelatedEvent(EventLevel.Informational, "DirectoryNotFoundWhileUpdatingAlwaysExclude", exceptionMetadata);
}
catch (IOException e)
{
metadata.Add("Details", "IOException while traversing folder path");
metadata.Add("folderPath", folderPath);
metadata.Add("Exception", e.ToString());
result = FileSystemTaskResult.RetryableError;
break;
}
catch (UnauthorizedAccessException e)
{
metadata.Add("Details", "UnauthorizedAccessException while traversing folder path");
metadata.Add("folderPath", folderPath);
metadata.Add("Exception", e.ToString());
result = FileSystemTaskResult.RetryableError;
break;
}
}
else
{
break;
}
}
}
}
break;
case FileSystemTask.OperationType.OnFolderDeleted:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
break;
case FileSystemTask.OperationType.OnFolderFirstWrite:
result = FileSystemTaskResult.Success;
break;
case FileSystemTask.OperationType.OnIndexWriteWithoutProjectionChange:
result = this.GitIndexProjection.AddMissingModifiedFiles();
break;
case FileSystemTask.OperationType.OnPlaceholderCreationsBlockedForGit:
this.GitIndexProjection.ClearNegativePathCacheIfPollutedByGit();
result = FileSystemTaskResult.Success;
break;
default:
throw new InvalidOperationException("Invalid background operation");
}
if (result != FileSystemTaskResult.Success)
{
metadata.Add("Area", "ExecuteBackgroundOperation");
metadata.Add("Operation", gitUpdate.Operation.ToString());
metadata.Add(TracingConstants.MessageKey.WarningMessage, "Background operation failed");
metadata.Add(nameof(result), result.ToString());
this.context.Tracer.RelatedEvent(EventLevel.Warning, "FailedBackgroundOperation", metadata);
}
return(result);
}
public static bool TryGetMaxGoodPrefetchTimestamp(
ITracer tracer,
GVFSEnlistment enlistment,
PhysicalFileSystem fileSystem,
GitObjects gitObjects,
out long maxGoodTimestamp,
out string error)
{
fileSystem.CreateDirectory(enlistment.GitPackRoot);
string[] packs = gitObjects.ReadPackFileNames(enlistment.GitPackRoot, GVFSConstants.PrefetchPackPrefix);
List <PrefetchPackInfo> orderedPacks = packs
.Where(pack => GetTimestamp(pack).HasValue)
.Select(pack => new PrefetchPackInfo(GetTimestamp(pack).Value, pack))
.OrderBy(packInfo => packInfo.Timestamp)
.ToList();
maxGoodTimestamp = -1;
int firstBadPack = -1;
for (int i = 0; i < orderedPacks.Count; ++i)
{
long timestamp = orderedPacks[i].Timestamp;
string packPath = orderedPacks[i].Path;
string idxPath = Path.ChangeExtension(packPath, ".idx");
if (!fileSystem.FileExists(idxPath))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("pack", packPath);
metadata.Add("idxPath", idxPath);
metadata.Add("timestamp", timestamp);
GitProcess.Result indexResult = gitObjects.IndexPackFile(packPath);
if (indexResult.ExitCodeIsFailure)
{
firstBadPack = i;
metadata.Add("Errors", indexResult.Errors);
tracer.RelatedWarning(metadata, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}: Found pack file that's missing idx file, and failed to regenerate idx");
break;
}
else
{
maxGoodTimestamp = timestamp;
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}: Found pack file that's missing idx file, and regenerated idx");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_RebuildIdx", metadata);
}
}
else
{
maxGoodTimestamp = timestamp;
}
}
if (firstBadPack != -1)
{
const int MaxDeleteRetries = 200; // 200 * IoFailureRetryDelayMS (50ms) = 10 seconds
const int RetryLoggingThreshold = 40; // 40 * IoFailureRetryDelayMS (50ms) = 2 seconds
// Before we delete _any_ pack-files, we need to delete the multi-pack-index, which
// may refer to those packs.
EventMetadata metadata = new EventMetadata();
string midxPath = Path.Combine(enlistment.GitPackRoot, "multi-pack-index");
metadata.Add("path", midxPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting multi-pack-index");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteMultiPack_index", metadata);
if (!fileSystem.TryWaitForDelete(tracer, midxPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {midxPath}";
return(false);
}
// Delete packs and indexes in reverse order so that if prefetch is killed, subseqeuent prefetch commands will
// find the right starting spot.
for (int i = orderedPacks.Count - 1; i >= firstBadPack; --i)
{
string packPath = orderedPacks[i].Path;
string idxPath = Path.ChangeExtension(packPath, ".idx");
metadata = new EventMetadata();
metadata.Add("path", idxPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting bad idx file");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteBadIdx", metadata);
if (!fileSystem.TryWaitForDelete(tracer, idxPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {idxPath}";
return(false);
}
metadata = new EventMetadata();
metadata.Add("path", packPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting bad pack file");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteBadPack", metadata);
if (!fileSystem.TryWaitForDelete(tracer, packPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {packPath}";
return(false);
}
}
}
error = null;
return(true);
}
private void HandleRequest(ITracer tracer, string request, NamedPipeServer.Connection connection)
{
NamedPipeMessages.Message message = NamedPipeMessages.Message.FromString(request);
if (string.IsNullOrWhiteSpace(message.Header))
{
return;
}
using (ITracer activity = this.tracer.StartActivity(message.Header, EventLevel.Informational, new EventMetadata {
{ "request", request }
}))
{
switch (message.Header)
{
case NamedPipeMessages.MountRepoRequest.Header:
try
{
NamedPipeMessages.MountRepoRequest mountRequest = NamedPipeMessages.MountRepoRequest.FromMessage(message);
MountHandler mountHandler = new MountHandler(activity, this.repoRegistry, connection, mountRequest);
mountHandler.Run();
}
catch (SerializationException ex)
{
activity.RelatedError("Could not deserialize mount request: {0}", ex.Message);
}
break;
case NamedPipeMessages.UnmountRepoRequest.Header:
try
{
NamedPipeMessages.UnmountRepoRequest unmountRequest = NamedPipeMessages.UnmountRepoRequest.FromMessage(message);
UnmountHandler unmountHandler = new UnmountHandler(activity, this.repoRegistry, connection, unmountRequest);
unmountHandler.Run();
}
catch (SerializationException ex)
{
activity.RelatedError("Could not deserialize unmount request: {0}", ex.Message);
}
break;
case NamedPipeMessages.Notification.Request.Header:
try
{
NamedPipeMessages.Notification.Request notificationRequest = NamedPipeMessages.Notification.Request.FromMessage(message);
NotificationHandler.Instance.SendNotification(activity, notificationRequest);
}
catch (SerializationException ex)
{
activity.RelatedError("Could not deserialize notification request: {0}", ex.Message);
}
break;
default:
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("Header", message.Header);
metadata.Add("ErrorMessage", "HandleNewConnection: Unknown request");
this.tracer.RelatedError(metadata);
connection.TrySendResponse(NamedPipeMessages.UnknownRequest);
break;
}
}
}
/// <summary>
/// Gets the target of the symbolic link.
/// </summary>
/// <param name="sha">SHA of the loose object containing the target path of the symbolic link</param>
/// <param name="symLinkTarget">Target path of the symbolic link</param>
private bool TryGetSymLinkTarget(string sha, out string symLinkTarget)
{
symLinkTarget = null;
string symLinkBlobContents = null;
try
{
if (!this.GitObjects.TryCopyBlobContentStream(
sha,
CancellationToken.None,
GVFSGitObjects.RequestSource.SymLinkCreation,
(stream, blobLength) =>
{
byte[] buffer = new byte[SymLinkTargetBufferSize];
uint bufferIndex = 0;
// TODO(#1361): Find a better solution than reading from the stream one byte at at time
int nextByte = stream.ReadByte();
while (nextByte != -1)
{
while (bufferIndex < buffer.Length && nextByte != -1)
{
buffer[bufferIndex] = (byte)nextByte;
nextByte = stream.ReadByte();
++bufferIndex;
}
if (bufferIndex < buffer.Length)
{
buffer[bufferIndex] = 0;
symLinkBlobContents = Encoding.UTF8.GetString(buffer);
}
else
{
buffer[bufferIndex - 1] = 0;
EventMetadata metadata = this.CreateEventMetadata();
metadata.Add(nameof(sha), sha);
metadata.Add("bufferContents", Encoding.UTF8.GetString(buffer));
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: SymLink target exceeds buffer size");
throw new GetSymLinkTargetException("SymLink target exceeds buffer size");
}
}
}))
{
EventMetadata metadata = this.CreateEventMetadata();
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream failed");
return(false);
}
}
catch (GetSymLinkTargetException e)
{
EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e);
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught GetSymLinkTargetException");
return(false);
}
catch (DecoderFallbackException e)
{
EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e);
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught DecoderFallbackException");
return(false);
}
symLinkTarget = symLinkBlobContents;
return(true);
}
public virtual void Initialize()
{
string folderPath = Path.GetDirectoryName(this.databasePath);
this.fileSystem.CreateDirectory(folderPath);
using (SqliteConnection connection = new SqliteConnection(this.sqliteConnectionString))
{
connection.Open();
using (SqliteCommand pragmaWalCommand = connection.CreateCommand())
{
// Advantages of using WAL ("Write-Ahead Log")
// 1. WAL is significantly faster in most scenarios.
// 2. WAL provides more concurrency as readers do not block writers and a writer does not block readers.
// Reading and writing can proceed concurrently.
// 3. Disk I/O operations tends to be more sequential using WAL.
// 4. WAL uses many fewer fsync() operations and is thus less vulnerable to problems on systems
// where the fsync() system call is broken.
// http://www.sqlite.org/wal.html
pragmaWalCommand.CommandText = $"PRAGMA journal_mode=WAL;";
pragmaWalCommand.ExecuteNonQuery();
}
using (SqliteCommand pragmaCacheSizeCommand = connection.CreateCommand())
{
// If the argument N is negative, then the number of cache pages is adjusted to use approximately abs(N*1024) bytes of memory
// -40000 => 40,000 * 1024 bytes => ~39MB
pragmaCacheSizeCommand.CommandText = $"PRAGMA cache_size=-40000;";
pragmaCacheSizeCommand.ExecuteNonQuery();
}
EventMetadata databaseMetadata = this.CreateEventMetadata();
using (SqliteCommand userVersionCommand = connection.CreateCommand())
{
// The user_version pragma will to get or set the value of the user-version integer at offset 60 in the database header.
// The user-version is an integer that is available to applications to use however they want. SQLite makes no use of the user-version itself.
// https://sqlite.org/pragma.html#pragma_user_version
userVersionCommand.CommandText = $"PRAGMA user_version;";
object userVersion = userVersionCommand.ExecuteScalar();
if (userVersion == null || Convert.ToInt64(userVersion) < 1)
{
userVersionCommand.CommandText = $"PRAGMA user_version=1;";
userVersionCommand.ExecuteNonQuery();
this.tracer.RelatedInfo($"{nameof(BlobSize)}.{nameof(this.Initialize)}: setting user_version to 1");
}
else
{
databaseMetadata.Add("user_version", Convert.ToInt64(userVersion));
}
}
using (SqliteCommand pragmaSynchronousCommand = connection.CreateCommand())
{
// GVFS uses the default value (FULL) to reduce the risks of corruption
// http://www.sqlite.org/pragma.html#pragma_synchronous
// (Note: This call is to retrieve the value of 'synchronous' and log it)
pragmaSynchronousCommand.CommandText = $"PRAGMA synchronous;";
object synchronous = pragmaSynchronousCommand.ExecuteScalar();
if (synchronous != null)
{
databaseMetadata.Add("synchronous", Convert.ToInt64(synchronous));
}
}
this.tracer.RelatedEvent(EventLevel.Informational, $"{nameof(BlobSize)}_{nameof(this.Initialize)}_db_settings", databaseMetadata);
using (SqliteCommand createTableCommand = connection.CreateCommand())
{
// Use a BLOB for sha rather than a string to reduce the size of the database
createTableCommand.CommandText = @"CREATE TABLE IF NOT EXISTS [BlobSizes] (sha BLOB, size INT, PRIMARY KEY (sha));";
createTableCommand.ExecuteNonQuery();
}
}
this.flushDataThread = new Thread(this.FlushDbThreadMain);
this.flushDataThread.IsBackground = true;
this.flushDataThread.Start();
}
private Result CreatePlaceholders(string directoryRelativePath, IEnumerable <ProjectedFileInfo> projectedItems, string triggeringProcessName)
{
foreach (ProjectedFileInfo fileInfo in projectedItems)
{
string childRelativePath = Path.Combine(directoryRelativePath, fileInfo.Name);
string sha;
FileSystemResult fileSystemResult;
if (fileInfo.IsFolder)
{
sha = string.Empty;
fileSystemResult = this.WritePlaceholderDirectory(childRelativePath);
}
else
{
sha = fileInfo.Sha.ToString();
// Writing placeholders on Mac does not require a file size
fileSystemResult = this.WritePlaceholderFile(childRelativePath, DummyFileSize, sha);
}
Result result = (Result)fileSystemResult.RawResult;
if (result != Result.Success)
{
EventMetadata metadata = this.CreateEventMetadata(childRelativePath);
metadata.Add("fileInfo.Name", fileInfo.Name);
metadata.Add("fileInfo.Size", fileInfo.Size);
metadata.Add("fileInfo.IsFolder", fileInfo.IsFolder);
metadata.Add(nameof(result), result.ToString());
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.CreatePlaceholders)}: Write placeholder failed");
if (result == Result.EIOError)
{
// If there is an IO error writing the placeholder then the file might already exist and it needs to
// be added to the modified paths so that git will show any differences or errors when interacting with the file
// This will happen in the include mode when the user creates a file that is already in the files that
// should be projected but we are trying to create the placeholder after it has already been created
this.FileSystemCallbacks.OnFileConvertedToFull(childRelativePath);
}
else
{
return(result);
}
}
else
{
if (fileInfo.IsFolder)
{
this.FileSystemCallbacks.OnPlaceholderFolderCreated(childRelativePath, triggeringProcessName);
}
else
{
this.FileSystemCallbacks.OnPlaceholderFileCreated(childRelativePath, sha, triggeringProcessName);
}
}
}
this.FileSystemCallbacks.OnPlaceholderFolderExpanded(directoryRelativePath);
return(Result.Success);
}
public void PerformDiff(string sourceTreeSha, string targetTreeSha)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("TargetTreeSha", targetTreeSha);
metadata.Add("HeadTreeSha", sourceTreeSha);
using (ITracer activity = this.tracer.StartActivity("PerformDiff", EventLevel.Informational, Keywords.Telemetry, metadata))
{
metadata = new EventMetadata();
if (sourceTreeSha == null)
{
this.UpdatedWholeTree = true;
// Nothing is checked out (fresh git init), so we must search the entire tree.
GitProcess.Result result = this.git.LsTree(
targetTreeSha,
line => this.EnqueueOperationsFromLsTreeLine(activity, line),
recursive: true,
showAllTrees: true);
if (result.ExitCodeIsFailure)
{
this.HasFailures = true;
metadata.Add("Errors", result.Errors);
metadata.Add("Output", result.Output.Length > 1024 ? result.Output.Substring(1024) : result.Output);
}
metadata.Add("Operation", "LsTree");
}
else
{
// Diff head and target, determine what needs to be done.
GitProcess.Result result = this.git.DiffTree(
sourceTreeSha,
targetTreeSha,
line => this.EnqueueOperationsFromDiffTreeLine(this.tracer, line));
if (result.ExitCodeIsFailure)
{
this.HasFailures = true;
metadata.Add("Errors", result.Errors);
metadata.Add("Output", result.Output.Length > 1024 ? result.Output.Substring(1024) : result.Output);
}
metadata.Add("Operation", "DiffTree");
}
this.FlushStagedQueues();
metadata.Add("Success", !this.HasFailures);
metadata.Add("DirectoryOperationsCount", this.TotalDirectoryOperations);
metadata.Add("FileDeleteOperationsCount", this.TotalFileDeletes);
metadata.Add("RequiredBlobsCount", this.RequiredBlobs.Count);
metadata.Add("FileAddOperationsCount", this.FileAddOperations.Sum(kvp => kvp.Value.Count));
activity.Stop(metadata);
}
}