private int GetParameterCount(EventMetadata eventData)
{
int paramCount;
if(eventData.Parameters == null)
{
paramCount = eventData.ParameterTypes.Length;
}
else
{
paramCount = eventData.Parameters.Length;
}
return paramCount;
}
public EventEnvelope(TEvent @event, EventMetadata metadata) : base(@event, metadata)
{
}
private void EnsureLocalCacheIsHealthy(
ITracer tracer,
GVFSEnlistment enlistment,
RetryConfig retryConfig,
ServerGVFSConfig serverGVFSConfig,
CacheServerInfo cacheServer)
{
if (!Directory.Exists(enlistment.LocalCacheRoot))
{
try
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Local cache root: {enlistment.LocalCacheRoot} missing, recreating it");
Directory.CreateDirectory(enlistment.LocalCacheRoot);
}
catch (Exception e)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Exception", e.ToString());
metadata.Add("enlistment.LocalCacheRoot", enlistment.LocalCacheRoot);
tracer.RelatedError(metadata, $"{nameof(this.EnsureLocalCacheIsHealthy)}: Exception while trying to create local cache root");
this.ReportErrorAndExit(tracer, "Failed to create local cache: " + enlistment.LocalCacheRoot);
}
}
// Validate that the GitObjectsRoot directory is on disk, and that the GVFS repo is configured to use it.
// If the directory is missing (and cannot be found in the mapping file) a new key for the repo will be added
// to the mapping file and used for BOTH the GitObjectsRoot and BlobSizesRoot
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
if (Directory.Exists(enlistment.GitObjectsRoot))
{
bool gitObjectsRootInAlternates = false;
string alternatesFilePath = this.GetAlternatesPath(enlistment);
if (File.Exists(alternatesFilePath))
{
try
{
using (Stream stream = fileSystem.OpenFileStream(
alternatesFilePath,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite,
callFlushFileBuffers: false))
{
using (StreamReader reader = new StreamReader(stream))
{
while (!reader.EndOfStream)
{
string alternatesLine = reader.ReadLine();
if (string.Equals(alternatesLine, enlistment.GitObjectsRoot, StringComparison.OrdinalIgnoreCase))
{
gitObjectsRootInAlternates = true;
}
}
}
}
}
catch (Exception e)
{
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Exception", e.ToString());
tracer.RelatedError(exceptionMetadata, $"{nameof(this.EnsureLocalCacheIsHealthy)}: Exception while trying to validate alternates file");
this.ReportErrorAndExit(tracer, $"Failed to validate that alternates file includes git objects root: {e.Message}");
}
}
else
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Alternates file not found");
}
if (!gitObjectsRootInAlternates)
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: GitObjectsRoot ({enlistment.GitObjectsRoot}) missing from alternates files, recreating alternates");
string error;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out error))
{
this.ReportErrorAndExit(tracer, $"Failed to update alternates file to include git objects root: {error}");
}
}
}
else
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: GitObjectsRoot ({enlistment.GitObjectsRoot}) missing, determining new root");
if (cacheServer == null)
{
cacheServer = CacheServerResolver.GetCacheServerFromConfig(enlistment);
}
string error;
if (serverGVFSConfig == null)
{
if (retryConfig == null)
{
if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error))
{
this.ReportErrorAndExit(tracer, "Failed to determine GVFS timeout and max retries: " + error);
}
}
serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig);
}
string localCacheKey;
LocalCacheResolver localCacheResolver = new LocalCacheResolver(enlistment);
if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers(
tracer,
serverGVFSConfig,
cacheServer,
enlistment.LocalCacheRoot,
localCacheKey: out localCacheKey,
errorMessage: out error))
{
this.ReportErrorAndExit(tracer, $"Previous git objects root ({enlistment.GitObjectsRoot}) not found, and failed to determine new local cache key: {error}");
}
EventMetadata metadata = new EventMetadata();
metadata.Add("localCacheRoot", enlistment.LocalCacheRoot);
metadata.Add("localCacheKey", localCacheKey);
metadata.Add(TracingConstants.MessageKey.InfoMessage, "Initializing and persisting updated paths");
tracer.RelatedEvent(EventLevel.Informational, "GVFSVerb_EnsureLocalCacheIsHealthy_InitializePathsFromKey", metadata);
enlistment.InitializeCachePathsFromKey(enlistment.LocalCacheRoot, localCacheKey);
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Creating GitObjectsRoot ({enlistment.GitObjectsRoot}), GitPackRoot ({enlistment.GitPackRoot}), and BlobSizesRoot ({enlistment.BlobSizesRoot})");
try
{
Directory.CreateDirectory(enlistment.GitObjectsRoot);
Directory.CreateDirectory(enlistment.GitPackRoot);
}
catch (Exception e)
{
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Exception", e.ToString());
exceptionMetadata.Add("enlistment.LocalCacheRoot", enlistment.LocalCacheRoot);
exceptionMetadata.Add("enlistment.GitObjectsRoot", enlistment.GitObjectsRoot);
exceptionMetadata.Add("enlistment.GitPackRoot", enlistment.GitPackRoot);
exceptionMetadata.Add("enlistment.BlobSizesRoot", enlistment.BlobSizesRoot);
tracer.RelatedError(exceptionMetadata, $"{nameof(this.InitializeLocalCacheAndObjectsPaths)}: Exception while trying to create objects, pack, and sizes folders");
this.ReportErrorAndExit(tracer, "Failed to create objects, pack, and sizes folders");
}
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Creating new alternates file");
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out error))
{
this.ReportErrorAndExit(tracer, $"Failed to update alterates file with new objects path: {error}");
}
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Saving git objects root ({enlistment.GitObjectsRoot}) in repo metadata");
RepoMetadata.Instance.SetGitObjectsRoot(enlistment.GitObjectsRoot);
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: Saving blob sizes root ({enlistment.BlobSizesRoot}) in repo metadata");
RepoMetadata.Instance.SetBlobSizesRoot(enlistment.BlobSizesRoot);
}
// Validate that the BlobSizesRoot folder is on disk.
// Note that if a user performed an action that resulted in the entire .gvfscache being deleted, the code above
// for validating GitObjectsRoot will have already taken care of generating a new key and setting a new enlistment.BlobSizesRoot path
if (!Directory.Exists(enlistment.BlobSizesRoot))
{
tracer.RelatedInfo($"{nameof(this.EnsureLocalCacheIsHealthy)}: BlobSizesRoot ({enlistment.BlobSizesRoot}) not found, re-creating");
try
{
Directory.CreateDirectory(enlistment.BlobSizesRoot);
}
catch (Exception e)
{
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Exception", e.ToString());
exceptionMetadata.Add("enlistment.BlobSizesRoot", enlistment.BlobSizesRoot);
tracer.RelatedError(exceptionMetadata, $"{nameof(this.InitializeLocalCacheAndObjectsPaths)}: Exception while trying to create blob sizes folder");
this.ReportErrorAndExit(tracer, "Failed to create blob sizes folder");
}
}
}
private static bool TryCopyNativeLibToAppDirectory(ITracer tracer, PhysicalFileSystem fileSystem, string gvfsAppDirectory)
{
string installFilePath;
string appFilePath;
GetNativeLibPaths(gvfsAppDirectory, out installFilePath, out appFilePath);
EventMetadata pathMetadata = CreateEventMetadata();
pathMetadata.Add(nameof(gvfsAppDirectory), gvfsAppDirectory);
pathMetadata.Add(nameof(installFilePath), installFilePath);
pathMetadata.Add(nameof(appFilePath), appFilePath);
if (fileSystem.FileExists(installFilePath))
{
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryCopyNativeLibToAppDirectory)}_CopyingNativeLib", pathMetadata);
try
{
fileSystem.CopyFile(installFilePath, appFilePath, overwrite: true);
try
{
Common.NativeMethods.FlushFileBuffers(appFilePath);
}
catch (Win32Exception e)
{
EventMetadata metadata = CreateEventMetadata(e);
metadata.Add(nameof(appFilePath), appFilePath);
metadata.Add(nameof(installFilePath), installFilePath);
tracer.RelatedWarning(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: Win32Exception while trying to flush file buffers", Keywords.Telemetry);
}
}
catch (UnauthorizedAccessException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedError(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: UnauthorizedAccessException caught while trying to copy native lib");
return(false);
}
catch (DirectoryNotFoundException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedError(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: DirectoryNotFoundException caught while trying to copy native lib");
return(false);
}
catch (FileNotFoundException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedError(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: FileNotFoundException caught while trying to copy native lib");
return(false);
}
catch (IOException e)
{
EventMetadata metadata = CreateEventMetadata(e);
tracer.RelatedWarning(metadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: IOException caught while trying to copy native lib");
if (fileSystem.FileExists(appFilePath))
{
tracer.RelatedWarning(
CreateEventMetadata(),
"Could not copy native lib to app directory, but file already exists, continuing with install",
Keywords.Telemetry);
}
else
{
tracer.RelatedError($"{nameof(TryCopyNativeLibToAppDirectory)}: Failed to copy native lib to app directory");
return(false);
}
}
}
else
{
tracer.RelatedError(pathMetadata, $"{nameof(TryCopyNativeLibToAppDirectory)}: Native lib does not exist in install directory");
return(false);
}
return(true);
}
private void OnNewConnection(IAsyncResult ar, bool createNewThreadIfSynchronous)
{
if (createNewThreadIfSynchronous &&
ar.CompletedSynchronously)
{
// if this callback got called synchronously, we must not do any blocking IO on this thread
// or we will block the original caller. Moving to a new thread so that it will be safe
// to call a blocking Read on the NamedPipeServerStream
new Thread(() => this.OnNewConnection(ar, createNewThreadIfSynchronous: false)).Start();
return;
}
this.listeningPipe = null;
bool connectionBroken = false;
NamedPipeServerStream pipe = (NamedPipeServerStream)ar.AsyncState;
try
{
try
{
pipe.EndWaitForConnection(ar);
}
catch (IOException e)
{
connectionBroken = true;
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", "NamedPipeServer");
metadata.Add("Exception", e.ToString());
metadata.Add(TracingConstants.MessageKey.WarningMessage, "OnNewConnection: Connection broken");
this.tracer.RelatedEvent(EventLevel.Warning, "OnNewConnectionn_EndWaitForConnection_IOException", metadata);
}
catch (Exception e)
{
this.LogErrorAndExit("OnNewConnection caught unhandled exception, exiting process", e);
}
if (!this.isStopping)
{
this.OpenListeningPipe();
if (!connectionBroken)
{
try
{
this.handleConnection(new Connection(pipe, this.tracer, () => this.isStopping));
}
catch (Exception e)
{
this.LogErrorAndExit("Unhandled exception in connection handler", e);
}
}
}
}
finally
{
pipe.Dispose();
}
}
private Type GetDataType(EventMetadata eventData, int parameterId)
{
Type dataType;
if(eventData.Parameters == null)
{
dataType = EventTypeToType(eventData.ParameterTypes[parameterId]);
}
else
{
dataType = eventData.Parameters[parameterId].ParameterType;
}
return dataType;
}
/// <summary>
/// Uses a <see cref="PrefetchPacksDeserializer"/> to read the packs from the stream.
/// </summary>
private RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult DeserializePrefetchPacks(
GitEndPointResponseData response,
ref long latestTimestamp,
ref long bytesDownloaded,
ref List <string> packIndexes,
GitProcess gitProcess)
{
if (packIndexes == null)
{
packIndexes = new List <string>();
}
using (ITracer activity = this.Tracer.StartActivity("DeserializePrefetchPacks", EventLevel.Informational))
{
PrefetchPacksDeserializer deserializer = new PrefetchPacksDeserializer(response.Stream);
string tempPackFolderPath = Path.Combine(this.Enlistment.GitPackRoot, TempPackFolder);
this.fileSystem.CreateDirectory(tempPackFolderPath);
List <TempPrefetchPackAndIdx> tempPacks = new List <TempPrefetchPackAndIdx>();
foreach (PrefetchPacksDeserializer.PackAndIndex pack in deserializer.EnumeratePacks())
{
// The advertised size may not match the actual, on-disk size.
long indexLength = 0;
long packLength;
// Write the temp and index to a temp folder to avoid putting corrupt files in the pack folder
// Once the files are validated and flushed they can be moved to the pack folder
string packName = string.Format("{0}-{1}-{2}.pack", GVFSConstants.PrefetchPackPrefix, pack.Timestamp, pack.UniqueId);
string packTempPath = Path.Combine(tempPackFolderPath, packName);
string idxName = string.Format("{0}-{1}-{2}.idx", GVFSConstants.PrefetchPackPrefix, pack.Timestamp, pack.UniqueId);
string idxTempPath = Path.Combine(tempPackFolderPath, idxName);
EventMetadata data = CreateEventMetadata();
data["timestamp"] = pack.Timestamp.ToString();
data["uniqueId"] = pack.UniqueId;
activity.RelatedEvent(EventLevel.Informational, "Receiving Pack/Index", data);
// Write the pack
// If it fails, TryWriteTempFile cleans up the file and we retry the prefetch
Task packFlushTask;
if (!this.TryWriteTempFile(activity, pack.PackStream, packTempPath, out packLength, out packFlushTask))
{
bytesDownloaded += packLength;
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(null, true));
}
bytesDownloaded += packLength;
// We will try to build an index if the server does not send one
if (pack.IndexStream == null)
{
GitProcess.Result result;
if (!this.TryBuildIndex(activity, packTempPath, out result, gitProcess))
{
if (packFlushTask != null)
{
packFlushTask.Wait();
}
// Move whatever has been successfully downloaded so far
Exception moveException;
this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out moveException);
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(null, true));
}
tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, idxFlushTask: null));
}
else
{
Task indexFlushTask;
if (this.TryWriteTempFile(activity, pack.IndexStream, idxTempPath, out indexLength, out indexFlushTask))
{
tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, indexFlushTask));
}
else
{
bytesDownloaded += indexLength;
// Try to build the index manually, then retry the prefetch
GitProcess.Result result;
if (this.TryBuildIndex(activity, packTempPath, out result, gitProcess))
{
// If we were able to recreate the failed index
// we can start the prefetch at the next timestamp
tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, idxFlushTask: null));
}
else
{
if (packFlushTask != null)
{
packFlushTask.Wait();
}
}
// Move whatever has been successfully downloaded so far
Exception moveException;
this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out moveException);
// The download stream will not be in a good state if the index download fails.
// So we have to restart the prefetch
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(null, true));
}
}
bytesDownloaded += indexLength;
}
Exception exception = null;
if (!this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out exception))
{
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(exception, true));
}
foreach (TempPrefetchPackAndIdx tempPack in tempPacks)
{
packIndexes.Add(tempPack.IdxName);
}
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(
new GitObjectsHttpRequestor.GitObjectTaskResult(success: true)));
}
}
private Result CreatePlaceholders(string directoryRelativePath, IEnumerable <ProjectedFileInfo> projectedItems, string triggeringProcessName)
{
foreach (ProjectedFileInfo fileInfo in projectedItems)
{
string childRelativePath = Path.Combine(directoryRelativePath, fileInfo.Name);
string sha;
FileSystemResult fileSystemResult;
if (fileInfo.IsFolder)
{
sha = string.Empty;
fileSystemResult = this.WritePlaceholderDirectory(childRelativePath);
}
else
{
sha = fileInfo.Sha.ToString();
// Writing placeholders on Mac does not require a file size
fileSystemResult = this.WritePlaceholderFile(childRelativePath, DummyFileSize, sha);
}
Result result = (Result)fileSystemResult.RawResult;
if (result != Result.Success)
{
EventMetadata metadata = this.CreateEventMetadata(childRelativePath);
metadata.Add("fileInfo.Name", fileInfo.Name);
metadata.Add("fileInfo.Size", fileInfo.Size);
metadata.Add("fileInfo.IsFolder", fileInfo.IsFolder);
metadata.Add(nameof(result), result.ToString());
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.CreatePlaceholders)}: Write placeholder failed");
if (result == Result.EIOError)
{
// If there is an IO error writing the placeholder then the file might already exist and it needs to
// be added to the modified paths so that git will show any differences or errors when interacting with the file
// This will happen in the include mode when the user creates a file that is already in the files that
// should be projected but we are trying to create the placeholder after it has already been created
this.FileSystemCallbacks.OnFileConvertedToFull(childRelativePath);
}
else
{
return(result);
}
}
else
{
if (fileInfo.IsFolder)
{
this.FileSystemCallbacks.OnPlaceholderFolderCreated(childRelativePath, triggeringProcessName);
}
else
{
this.FileSystemCallbacks.OnPlaceholderFileCreated(childRelativePath, sha, triggeringProcessName);
}
}
}
this.FileSystemCallbacks.OnPlaceholderFolderExpanded(directoryRelativePath);
return(Result.Success);
}
private void HandleAllDirectoryOperations()
{
DiffTreeResult treeOp;
while (this.diff.DirectoryOperations.TryDequeue(out treeOp))
{
if (this.HasFailures)
{
return;
}
switch (treeOp.Operation)
{
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.Add:
try
{
Directory.CreateDirectory(treeOp.TargetPath);
}
catch (Exception ex)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "CreateDirectory");
metadata.Add(nameof(treeOp.TargetPath), treeOp.TargetPath);
this.tracer.RelatedError(metadata, ex.Message);
this.HasFailures = true;
}
break;
case DiffTreeResult.Operations.Delete:
try
{
if (Directory.Exists(treeOp.TargetPath))
{
this.fileSystem.DeleteDirectory(treeOp.TargetPath);
}
}
catch (Exception ex)
{
// We are deleting directories and subdirectories in parallel
if (Directory.Exists(treeOp.TargetPath))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "DeleteDirectory");
metadata.Add(nameof(treeOp.TargetPath), treeOp.TargetPath);
this.tracer.RelatedError(metadata, ex.Message);
this.HasFailures = true;
}
}
break;
default:
this.tracer.RelatedError("Ignoring unexpected Tree Operation {0}: {1}", treeOp.TargetPath, treeOp.Operation);
continue;
}
if (Interlocked.Increment(ref this.directoryOpCount) % NumOperationsPerStatus == 0)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("DirectoryOperationsQueued", this.diff.DirectoryOperations.Count);
metadata.Add("DirectoryOperationsCompleted", this.directoryOpCount);
this.tracer.RelatedEvent(EventLevel.Informational, "CheckoutStatus", metadata);
}
}
}
private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot, string line)
{
if (!line.StartsWith(":"))
{
// Diff-tree starts with metadata we can ignore.
// Real diff lines always start with a colon
return;
}
DiffTreeResult result = DiffTreeResult.ParseFromDiffTreeLine(line, repoRoot);
if (!this.ResultIsInWhitelist(result))
{
return;
}
if (result.Operation == DiffTreeResult.Operations.Unknown ||
result.Operation == DiffTreeResult.Operations.Unmerged)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Path", result.TargetFilename);
metadata.Add("ErrorMessage", "Unexpected diff operation: " + result.Operation);
activity.RelatedError(metadata);
this.HasFailures = true;
return;
}
// Separate and enqueue all directory operations first.
if (result.SourceIsDirectory || result.TargetIsDirectory)
{
switch (result.Operation)
{
case DiffTreeResult.Operations.Delete:
if (!this.stagedDirectoryOperations.Add(result))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Filename", result.TargetFilename);
metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
}
break;
case DiffTreeResult.Operations.RenameEdit:
if (!this.stagedDirectoryOperations.Add(result))
{
// This could happen if a directory was deleted and an existing directory was renamed to replace it, but with a different case.
EventMetadata metadata = new EventMetadata();
metadata.Add("Filename", result.TargetFilename);
metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
// The target of RenameEdit is always akin to an Add, so replacing the delete is the safer thing to do.
this.stagedDirectoryOperations.Remove(result);
this.stagedDirectoryOperations.Add(result);
}
if (!result.TargetIsDirectory)
{
// Handle when a directory becomes a file.
// Files becoming directories is handled by HandleAllDirectoryOperations
this.EnqueueFileAddOperation(activity, result);
}
break;
case DiffTreeResult.Operations.Add:
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.CopyEdit:
if (!this.stagedDirectoryOperations.Add(result))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Filename", result.TargetFilename);
metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory.");
activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata);
// Replace the delete with the add to make sure we don't delete a folder from under ourselves
this.stagedDirectoryOperations.Remove(result);
this.stagedDirectoryOperations.Add(result);
}
break;
default:
activity.RelatedError("Unexpected diff operation from line: {0}", line);
break;
}
}
else
{
switch (result.Operation)
{
case DiffTreeResult.Operations.Delete:
this.EnqueueFileDeleteOperation(activity, result.TargetFilename);
break;
case DiffTreeResult.Operations.RenameEdit:
this.EnqueueFileAddOperation(activity, result);
this.EnqueueFileDeleteOperation(activity, result.SourceFilename);
break;
case DiffTreeResult.Operations.Modify:
case DiffTreeResult.Operations.CopyEdit:
case DiffTreeResult.Operations.Add:
this.EnqueueFileAddOperation(activity, result);
break;
default:
activity.RelatedError("Unexpected diff operation from line: {0}", line);
break;
}
}
}
private Result OnGetFileStream(
ulong commandId,
string relativePath,
byte[] providerId,
byte[] contentId,
int triggeringProcessId,
string triggeringProcessName,
IntPtr fileHandle)
{
try
{
if (contentId == null)
{
this.Context.Tracer.RelatedError($"{nameof(this.OnGetFileStream)} called with null contentId, path: " + relativePath);
return(Result.EInvalidOperation);
}
if (providerId == null)
{
this.Context.Tracer.RelatedError($"{nameof(this.OnGetFileStream)} called with null epochId, path: " + relativePath);
return(Result.EInvalidOperation);
}
string sha = GetShaFromContentId(contentId);
byte placeholderVersion = GetPlaceholderVersionFromProviderId(providerId);
EventMetadata metadata = this.CreateEventMetadata(relativePath);
metadata.Add(nameof(triggeringProcessId), triggeringProcessId);
metadata.Add(nameof(triggeringProcessName), triggeringProcessName);
metadata.Add(nameof(sha), sha);
metadata.Add(nameof(placeholderVersion), placeholderVersion);
metadata.Add(nameof(commandId), commandId);
ITracer activity = this.Context.Tracer.StartActivity("GetFileStream", EventLevel.Verbose, Keywords.Telemetry, metadata);
if (placeholderVersion != FileSystemVirtualizer.PlaceholderVersion)
{
activity.RelatedError(metadata, nameof(this.OnGetFileStream) + ": Unexpected placeholder version");
activity.Dispose();
// TODO(#1362): Is this the correct Result to return?
return(Result.EIOError);
}
try
{
if (!this.GitObjects.TryCopyBlobContentStream(
sha,
CancellationToken.None,
GVFSGitObjects.RequestSource.FileStreamCallback,
(stream, blobLength) =>
{
// TODO(#1361): Find a better solution than reading from the stream one byte at at time
byte[] buffer = new byte[4096];
uint bufferIndex = 0;
int nextByte = stream.ReadByte();
int bytesWritten = 0;
while (nextByte != -1)
{
while (bufferIndex < buffer.Length && nextByte != -1)
{
buffer[bufferIndex] = (byte)nextByte;
nextByte = stream.ReadByte();
++bufferIndex;
}
Result result = this.virtualizationInstance.WriteFileContents(
fileHandle,
buffer,
bufferIndex);
if (result != Result.Success)
{
activity.RelatedError(metadata, $"{nameof(this.virtualizationInstance.WriteFileContents)} failed, error: " + result.ToString("X") + "(" + result.ToString("G") + ")");
throw new GetFileStreamException(result);
}
if (bufferIndex == buffer.Length)
{
bufferIndex = 0;
bytesWritten += buffer.Length;
}
}
bytesWritten += Convert.ToInt32(bufferIndex);
if (bytesWritten != blobLength)
{
// If the read size does not match the expected size print an error and add the file to ModifiedPaths.dat
// This allows the user to see that something went wrong with file hydration
// Unfortunitely we must do this check *after* the file is hydrated since the header isn't corrupt for trunctated objects on mac
this.Context.Tracer.RelatedError($"Read {relativePath} to {bytesWritten}, not expected size of {blobLength}");
this.FileSystemCallbacks.OnFailedFileHydration(relativePath);
}
}))
{
activity.RelatedError(metadata, $"{nameof(this.OnGetFileStream)}: TryCopyBlobContentStream failed");
// TODO(#1362): Is this the correct Result to return?
return(Result.EFileNotFound);
}
}
catch (GetFileStreamException e)
{
return(e.Result);
}
this.FileSystemCallbacks.OnPlaceholderFileHydrated(triggeringProcessName);
return(Result.Success);
}
catch (Exception e)
{
EventMetadata metadata = this.CreateEventMetadata(relativePath, e);
metadata.Add(nameof(triggeringProcessId), triggeringProcessId);
metadata.Add(nameof(triggeringProcessName), triggeringProcessName);
metadata.Add(nameof(commandId), commandId);
this.LogUnhandledExceptionAndExit(nameof(this.OnGetFileStream), metadata);
}
return(Result.EIOError);
}
private void UpdateHooks()
{
bool copyReadObjectHook = false;
string enlistmentReadObjectHookPath = Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Hooks.ReadObjectPath + ".exe");
string installedReadObjectHookPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), GVFSConstants.GVFSReadObjectHookExecutableName);
if (!File.Exists(installedReadObjectHookPath))
{
this.FailMountAndExit(GVFSConstants.GVFSReadObjectHookExecutableName + " cannot be found at {0}", installedReadObjectHookPath);
}
if (!File.Exists(enlistmentReadObjectHookPath))
{
copyReadObjectHook = true;
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", "Mount");
metadata.Add("enlistmentReadObjectHookPath", enlistmentReadObjectHookPath);
metadata.Add("installedReadObjectHookPath", installedReadObjectHookPath);
metadata.Add("Message", GVFSConstants.DotGit.Hooks.ReadObjectName + " not found in enlistment, copying from installation folder");
this.tracer.RelatedEvent(EventLevel.Warning, "ReadObjectMissingFromEnlistment", metadata);
}
else
{
try
{
FileVersionInfo enlistmentVersion = FileVersionInfo.GetVersionInfo(enlistmentReadObjectHookPath);
FileVersionInfo installedVersion = FileVersionInfo.GetVersionInfo(installedReadObjectHookPath);
copyReadObjectHook = enlistmentVersion.FileVersion != installedVersion.FileVersion;
}
catch (Exception e)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", "Mount");
metadata.Add("enlistmentReadObjectHookPath", enlistmentReadObjectHookPath);
metadata.Add("installedReadObjectHookPath", installedReadObjectHookPath);
metadata.Add("Exception", e.ToString());
metadata.Add("ErrorMessage", "Failed to compare " + GVFSConstants.DotGit.Hooks.ReadObjectName + " version");
this.tracer.RelatedError(metadata);
this.FailMountAndExit("Error comparing " + GVFSConstants.DotGit.Hooks.ReadObjectName + " versions, see log file for details");
}
}
if (copyReadObjectHook)
{
try
{
File.Copy(installedReadObjectHookPath, enlistmentReadObjectHookPath, overwrite: true);
}
catch (Exception e)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", "Mount");
metadata.Add("enlistmentReadObjectHookPath", enlistmentReadObjectHookPath);
metadata.Add("installedReadObjectHookPath", installedReadObjectHookPath);
metadata.Add("Exception", e.ToString());
metadata.Add("ErrorMessage", "Failed to copy " + GVFSConstants.DotGit.Hooks.ReadObjectName + " to enlistment");
this.tracer.RelatedError(metadata);
this.FailMountAndExit("Error copying " + GVFSConstants.DotGit.Hooks.ReadObjectName + " to enlistment, see log file for details");
}
}
}
private Type GetDataType(EventMetadata eventData, int parameterId)
{
return(eventData.Parameters[parameterId].ParameterType);
}
private int GetParameterCount(EventMetadata eventData)
{
return(eventData.Parameters.Length);
}
// Use reflection to look at the attributes of a class, and generate a manifest for it (as UTF8) and
// return the UTF8 bytes. It also sets up the code:EventData structures needed to dispatch events
// at run time. 'source' is the event source to place the descriptors. If it is null,
// then the descriptors are not creaed, and just the manifest is generated.
private static byte[] CreateManifestAndDescriptors(Type eventSourceType, string eventSourceDllName, EventSource source)
{
MethodInfo[] methods = eventSourceType.GetMethods(BindingFlags.DeclaredOnly | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance);
EventAttribute defaultEventAttribute;
int eventId = 1; // The number given to an event that does not have a explicitly given ID.
EventMetadata[] eventData = null;
Dictionary<string, string> eventsByName = null;
if (source != null)
eventData = new EventMetadata[methods.Length];
// See if we have localization information.
ResourceManager resources = null;
EventSourceAttribute eventSourceAttrib = (EventSourceAttribute)GetCustomAttributeHelper(eventSourceType, typeof(EventSourceAttribute));
if (eventSourceAttrib != null && eventSourceAttrib.LocalizationResources != null)
resources = new ResourceManager(eventSourceAttrib.LocalizationResources, eventSourceType.Assembly);
ManifestBuilder manifest = new ManifestBuilder(GetName(eventSourceType), GetGuid(eventSourceType), eventSourceDllName, resources);
// Collect task, opcode, keyword and channel information
#if FEATURE_MANAGED_ETW_CHANNELS
foreach (var providerEnumKind in new string[] { "Keywords", "Tasks", "Opcodes", "Channels" })
#else
foreach (var providerEnumKind in new string[] { "Keywords", "Tasks", "Opcodes" })
#endif
{
Type nestedType = eventSourceType.GetNestedType(providerEnumKind);
if (nestedType != null)
{
foreach (FieldInfo staticField in nestedType.GetFields(BindingFlags.DeclaredOnly | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static))
{
AddProviderEnumKind(manifest, staticField, providerEnumKind);
}
}
}
for (int i = 0; i < methods.Length; i++)
{
MethodInfo method = methods[i];
ParameterInfo[] args = method.GetParameters();
// Get the EventDescriptor (from the Custom attributes)
EventAttribute eventAttribute = (EventAttribute)GetCustomAttributeHelper(method, typeof(EventAttribute));
// Methods that don't return void can't be events.
if (method.ReturnType != typeof(void))
{
if (eventAttribute != null)
throw new ArgumentException(Environment.GetResourceString("EventSource_AttributeOnNonVoid", method.Name));
continue;
}
if (method.IsVirtual || method.IsStatic)
{
continue;
}
if (eventAttribute == null)
{
// If we explictly mark the method as not being an event, then honor that.
if (GetCustomAttributeHelper(method, typeof(NonEventAttribute)) != null)
continue;
defaultEventAttribute = new EventAttribute(eventId);
eventAttribute = defaultEventAttribute;
}
else if (eventAttribute.EventId <= 0)
throw new ArgumentException(Environment.GetResourceString("EventSource_NeedPositiveId"));
else if ((ulong)eventAttribute.Keywords >= 0x0000100000000000UL)
throw new ArgumentException(Environment.GetResourceString("EventSource_ReservedKeywords"));
eventId++;
// Auto-assign tasks, starting with the highest task number and working back
if (eventAttribute.Opcode == EventOpcode.Info && eventAttribute.Task == EventTask.None)
eventAttribute.Task = (EventTask)(0xFFFE - eventAttribute.EventId);
manifest.StartEvent(method.Name, eventAttribute);
for (int fieldIdx = 0; fieldIdx < args.Length; fieldIdx++)
{
// If the first parameter is 'RelatedActivityId' then skip it.
if (fieldIdx == 0 && args[fieldIdx].ParameterType == typeof(Guid) &&
string.Compare(args[fieldIdx].Name, "RelatedActivityId", StringComparison.OrdinalIgnoreCase) == 0)
continue;
manifest.AddEventParameter(args[fieldIdx].ParameterType, args[fieldIdx].Name);
}
manifest.EndEvent();
if (source != null)
{
// Do checking for user errors (optional, but nto a big deal so we do it).
DebugCheckEvent(ref eventsByName, eventData, method, eventAttribute);
AddEventDescriptor(ref eventData, method.Name, eventAttribute, args);
}
}
if (source != null)
{
TrimEventDescriptors(ref eventData);
source.m_eventData = eventData; // officaly initialize it. We do this at most once (it is racy otherwise).
}
return manifest.CreateManifest();
}
private void ProcessBackgroundOperations()
{
TBackgroundOperation backgroundOperation;
while (true)
{
AcquireGitLockResult acquireLockResult = AcquireGitLockResult.ShuttingDown;
try
{
this.wakeUpThread.WaitOne();
if (this.isStopping)
{
return;
}
acquireLockResult = this.WaitToAcquireGitLock();
switch (acquireLockResult)
{
case AcquireGitLockResult.LockAcquired:
break;
case AcquireGitLockResult.ShuttingDown:
return;
default:
this.LogErrorAndExit("Invalid AcquireGitLockResult result");
return;
}
this.RunCallbackUntilSuccess(this.preCallback, "PreCallback");
int tasksProcessed = 0;
while (this.backgroundOperations.TryPeek(out backgroundOperation))
{
if (tasksProcessed % LogUpdateTaskThreshold == 0 &&
(tasksProcessed >= LogUpdateTaskThreshold || this.backgroundOperations.Count >= LogUpdateTaskThreshold))
{
this.LogTaskProcessingStatus(tasksProcessed);
}
if (this.isStopping)
{
// If we are stopping, then GVFlt has already been shut down
// Some of the queued background tasks may require GVFlt, and so it is unsafe to
// proceed. GVFS will resume any queued tasks next time it is mounted
this.persistence.Flush();
return;
}
CallbackResult callbackResult = this.callback(backgroundOperation);
switch (callbackResult)
{
case CallbackResult.Success:
this.backgroundOperations.TryDequeue(out backgroundOperation);
this.persistence.Remove(backgroundOperation.Id);
++tasksProcessed;
break;
case CallbackResult.RetryableError:
if (!this.isStopping)
{
Thread.Sleep(ActionRetryDelayMS);
}
break;
case CallbackResult.FatalError:
this.LogErrorAndExit("Callback encountered fatal error, exiting process");
break;
default:
this.LogErrorAndExit("Invalid background operation result");
break;
}
}
this.persistence.Flush();
if (tasksProcessed >= LogUpdateTaskThreshold)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("BackgroundOperations", EtwArea);
metadata.Add("TasksProcessed", tasksProcessed);
metadata.Add("Message", "Processing background tasks complete");
this.context.Tracer.RelatedEvent(EventLevel.Informational, "TaskProcessingStatus", metadata);
}
if (this.isStopping)
{
return;
}
}
catch (Exception e)
{
this.LogErrorAndExit("ProcessBackgroundOperations caught unhandled exception, exiting process", e);
}
finally
{
if (acquireLockResult == AcquireGitLockResult.LockAcquired)
{
this.RunCallbackUntilSuccess(this.postCallback, "PostCallback");
this.ReleaseGitLockIfNecessary();
}
}
}
}
// Helper used by code:CreateManifestAndDescriptors that trims the m_eventData array to the correct
// size after all event descriptors have been added.
private static void TrimEventDescriptors(ref EventMetadata[] eventData)
{
int idx = eventData.Length;
while (0 < idx)
{
--idx;
if (eventData[idx].Descriptor.EventId != 0)
break;
}
if (eventData.Length - idx > 2) // allow one wasted slot.
{
EventMetadata[] newValues = new EventMetadata[idx + 1];
Array.Copy(eventData, newValues, newValues.Length);
eventData = newValues;
}
}
private Result TryClone(
JsonTracer tracer,
GVFSEnlistment enlistment,
CacheServerInfo cacheServer,
RetryConfig retryConfig,
ServerGVFSConfig serverGVFSConfig,
string resolvedLocalCacheRoot)
{
Result pipeResult;
using (NamedPipeServer pipeServer = this.StartNamedPipe(tracer, enlistment, out pipeResult))
{
if (!pipeResult.Success)
{
return(pipeResult);
}
using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig))
{
GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null);
if (refs == null)
{
return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl)));
}
if (this.Branch == null)
{
this.Branch = refs.GetDefaultBranch();
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata);
}
else
{
if (!refs.HasBranch(this.Branch))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata);
string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch);
return(new Result(errorMessage));
}
}
if (!enlistment.TryCreateEnlistmentFolders())
{
string error = "Could not create enlistment directory";
tracer.RelatedError(error);
return(new Result(error));
}
string localCacheError;
if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, serverGVFSConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError))
{
tracer.RelatedError(localCacheError);
return(new Result(localCacheError));
}
Directory.CreateDirectory(enlistment.GitObjectsRoot);
Directory.CreateDirectory(enlistment.GitPackRoot);
Directory.CreateDirectory(enlistment.BlobSizesRoot);
return(this.CreateClone(tracer, enlistment, objectRequestor, refs, this.Branch));
}
}
}
private Type GetDataType(EventMetadata eventData, int parameterId)
{
return eventData.Parameters[parameterId].ParameterType;
}
private Result CreateClone(
ITracer tracer,
GVFSEnlistment enlistment,
GitObjectsHttpRequestor objectRequestor,
GitRefs refs,
string branch)
{
Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string errorMessage;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage))
{
return(new Result("Error configuring alternate: " + errorMessage));
}
GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem);
GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment);
GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor);
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage))
{
return(new Result(errorMessage));
}
if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) ||
!GVFSVerb.TrySetOptionalGitConfigSettings(enlistment))
{
return(new Result("Unable to configure git repo"));
}
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage))
{
return(new Result("Unable to configure cache server: " + errorMessage));
}
GitProcess git = new GitProcess(enlistment);
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.HasErrors)
{
return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head),
"ref: refs/heads/" + branch);
if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage))
{
return(new Result(errorMessage));
}
this.CreateGitScript(enlistment);
string installHooksError;
if (!HooksInstaller.InstallHooks(context, out installHooksError))
{
tracer.RelatedError(installHooksError);
return(new Result(installHooksError));
}
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.HasErrors && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0)
{
// It is possible to have the above TryDownloadCommit() fail because we
// already have the commit and root tree we intend to check out, but
// don't have a tree further down the working directory. If we fail
// checkout here, its' because we don't have these trees and the
// read-object hook is not available yet. Force downloading the commit
// again and retry the checkout.
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage,
checkLocalObjectCache: false))
{
return(new Result(errorMessage));
}
forceCheckoutResult = git.ForceCheckout(branch);
}
if (forceCheckoutResult.HasErrors)
{
string[] errorLines = forceCheckoutResult.Errors.Split('\n');
StringBuilder checkoutErrors = new StringBuilder();
foreach (string gitError in errorLines)
{
if (IsForceCheckoutErrorCloneFailure(gitError))
{
checkoutErrors.AppendLine(gitError);
}
}
if (checkoutErrors.Length > 0)
{
string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString();
tracer.RelatedError(error);
return(new Result(error));
}
}
if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage))
{
tracer.RelatedError(errorMessage);
return(new Result(errorMessage));
}
try
{
RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment);
this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment);
}
catch (Exception e)
{
tracer.RelatedError(e.ToString());
return(new Result(e.Message));
}
finally
{
RepoMetadata.Shutdown();
}
// Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed
Exception exception;
string prepFileSystemError;
if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception))
{
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(prepFileSystemError), prepFileSystemError);
if (exception != null)
{
metadata.Add("Exception", exception.ToString());
}
tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed");
return(new Result(prepFileSystemError));
}
return(new Result(true));
}
public FileSystemCallbacks(
GVFSContext context,
GVFSGitObjects gitObjects,
RepoMetadata repoMetadata,
BlobSizes blobSizes,
GitIndexProjection gitIndexProjection,
BackgroundFileSystemTaskRunner backgroundFileSystemTaskRunner,
FileSystemVirtualizer fileSystemVirtualizer)
{
this.logsHeadFileProperties = null;
this.postFetchJobLock = new object();
this.context = context;
this.gitObjects = gitObjects;
this.fileSystemVirtualizer = fileSystemVirtualizer;
this.placeHolderCreationCount = new ConcurrentDictionary <string, PlaceHolderCreateCounter>(StringComparer.OrdinalIgnoreCase);
string error;
if (!ModifiedPathsDatabase.TryLoadOrCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths),
this.context.FileSystem,
out this.modifiedPaths,
out error))
{
throw new InvalidRepoException(error);
}
this.BlobSizes = blobSizes;
this.BlobSizes.Initialize();
PlaceholderListDatabase placeholders;
if (!PlaceholderListDatabase.TryCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList),
this.context.FileSystem,
out placeholders,
out error))
{
throw new InvalidRepoException(error);
}
this.GitIndexProjection = gitIndexProjection ?? new GitIndexProjection(
context,
gitObjects,
this.BlobSizes,
repoMetadata,
fileSystemVirtualizer,
placeholders,
this.modifiedPaths);
this.backgroundFileSystemTaskRunner = backgroundFileSystemTaskRunner ?? new BackgroundFileSystemTaskRunner(
this.context,
this.PreBackgroundOperation,
this.ExecuteBackgroundOperation,
this.PostBackgroundOperation,
Path.Combine(context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.BackgroundFileSystemTasks));
this.logsHeadPath = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Logs.Head);
EventMetadata metadata = new EventMetadata();
metadata.Add("placeholders.Count", placeholders.EstimatedCount);
metadata.Add("background.Count", this.backgroundFileSystemTaskRunner.Count);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(FileSystemCallbacks)} created");
this.context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(FileSystemCallbacks)}_Constructor", metadata);
}
/// <summary>
/// Add the standard ScalarVerb metadata to the specified EventMetadata
/// </summary>
/// <param name="metadata">
/// EventMetadata to which verb data will be added
/// </param>
/// <returns>
/// The specified EventMetadata (updated to include verb metadata)
/// </returns>
protected EventMetadata AddVerbDataToMetadata(EventMetadata metadata)
{
metadata["Area"] = $"{this.VerbName}_Verb";
metadata["Verb"] = this.VerbName;
return(metadata);
}
private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileSystem fileSystem, out bool isProjFSFeatureAvailable)
{
EventMetadata metadata = CreateEventMetadata();
const int ProjFSNotAnOptionalFeature = 2;
const int ProjFSEnabled = 3;
const int ProjFSDisabled = 4;
ProcessResult getOptionalFeatureResult = CallPowershellCommand(
"$var=(Get-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + "); if($var -eq $null){exit " +
ProjFSNotAnOptionalFeature + "}else{if($var.State -eq 'Enabled'){exit " + ProjFSEnabled + "}else{exit " + ProjFSDisabled + "}}");
isProjFSFeatureAvailable = true;
bool projFSEnabled = false;
switch (getOptionalFeatureResult.ExitCode)
{
case ProjFSNotAnOptionalFeature:
metadata.Add("getOptionalFeatureResult.Output", getOptionalFeatureResult.Output);
metadata.Add("getOptionalFeatureResult.Errors", getOptionalFeatureResult.Errors);
tracer.RelatedWarning(metadata, $"{nameof(TryEnableProjFSOptionalFeature)}: {OptionalFeatureName} optional feature is missing");
isProjFSFeatureAvailable = false;
break;
case ProjFSEnabled:
tracer.RelatedEvent(
EventLevel.Informational,
$"{nameof(TryEnableProjFSOptionalFeature)}_ClientProjFSAlreadyEnabled",
metadata,
Keywords.Network);
projFSEnabled = true;
break;
case ProjFSDisabled:
ProcessResult enableOptionalFeatureResult = CallPowershellCommand("try {Enable-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + " -NoRestart}catch{exit 1}");
metadata.Add("enableOptionalFeatureResult.Output", enableOptionalFeatureResult.Output.Trim().Replace("\r\n", ","));
metadata.Add("enableOptionalFeatureResult.Errors", enableOptionalFeatureResult.Errors);
if (enableOptionalFeatureResult.ExitCode == 0)
{
metadata.Add(TracingConstants.MessageKey.InfoMessage, "Enabled ProjFS optional feature");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryEnableProjFSOptionalFeature)}_ClientProjFSDisabled", metadata);
projFSEnabled = true;
break;
}
metadata.Add("enableOptionalFeatureResult.ExitCode", enableOptionalFeatureResult.ExitCode);
tracer.RelatedError(metadata, $"{nameof(TryEnableProjFSOptionalFeature)}: Failed to enable optional feature");
break;
default:
metadata.Add("getOptionalFeatureResult.ExitCode", getOptionalFeatureResult.ExitCode);
metadata.Add("getOptionalFeatureResult.Output", getOptionalFeatureResult.Output);
metadata.Add("getOptionalFeatureResult.Errors", getOptionalFeatureResult.Errors);
tracer.RelatedError(metadata, $"{nameof(TryEnableProjFSOptionalFeature)}: Unexpected result");
isProjFSFeatureAvailable = false;
break;
}
if (projFSEnabled)
{
if (IsNativeLibInstalled(tracer, fileSystem))
{
return(true);
}
tracer.RelatedError($"{nameof(TryEnableProjFSOptionalFeature)}: {OptionalFeatureName} enabled, but native ProjFS library is not on path");
}
return(false);
}
public static bool TryGetMaxGoodPrefetchTimestamp(
ITracer tracer,
GVFSEnlistment enlistment,
PhysicalFileSystem fileSystem,
GitObjects gitObjects,
out long maxGoodTimestamp,
out string error)
{
fileSystem.CreateDirectory(enlistment.GitPackRoot);
string[] packs = gitObjects.ReadPackFileNames(enlistment.GitPackRoot, GVFSConstants.PrefetchPackPrefix);
List <PrefetchPackInfo> orderedPacks = packs
.Where(pack => GetTimestamp(pack).HasValue)
.Select(pack => new PrefetchPackInfo(GetTimestamp(pack).Value, pack))
.OrderBy(packInfo => packInfo.Timestamp)
.ToList();
maxGoodTimestamp = -1;
int firstBadPack = -1;
for (int i = 0; i < orderedPacks.Count; ++i)
{
long timestamp = orderedPacks[i].Timestamp;
string packPath = orderedPacks[i].Path;
string idxPath = Path.ChangeExtension(packPath, ".idx");
if (!fileSystem.FileExists(idxPath))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("pack", packPath);
metadata.Add("idxPath", idxPath);
metadata.Add("timestamp", timestamp);
GitProcess.Result indexResult = gitObjects.IndexPackFile(packPath);
if (indexResult.HasErrors)
{
firstBadPack = i;
metadata.Add("Errors", indexResult.Errors);
tracer.RelatedWarning(metadata, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}: Found pack file that's missing idx file, and failed to regenerate idx");
break;
}
else
{
maxGoodTimestamp = timestamp;
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}: Found pack file that's missing idx file, and regenerated idx");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_RebuildIdx", metadata);
}
}
else
{
maxGoodTimestamp = timestamp;
}
}
if (firstBadPack != -1)
{
const int MaxDeleteRetries = 200; // 200 * IoFailureRetryDelayMS (50ms) = 10 seconds
const int RetryLoggingThreshold = 40; // 40 * IoFailureRetryDelayMS (50ms) = 2 seconds
// Delete packs and indexes in reverse order so that if prefetch is killed, subseqeuent prefetch commands will
// find the right starting spot.
for (int i = orderedPacks.Count - 1; i >= firstBadPack; --i)
{
string packPath = orderedPacks[i].Path;
string idxPath = Path.ChangeExtension(packPath, ".idx");
EventMetadata metadata = new EventMetadata();
metadata.Add("path", idxPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting bad idx file");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteBadIdx", metadata);
if (!fileSystem.TryWaitForDelete(tracer, idxPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {idxPath}";
return(false);
}
metadata = new EventMetadata();
metadata.Add("path", packPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting bad pack file");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteBadPack", metadata);
if (!fileSystem.TryWaitForDelete(tracer, packPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {packPath}";
return(false);
}
}
}
error = null;
return(true);
}
private int ExecuteWithExitCode()
{
// CmdParser doesn't strip quotes, and Path.Combine will throw
this.GitBinPath = this.GitBinPath.Replace("\"", string.Empty);
if (!GitProcess.GitExists(this.GitBinPath))
{
Console.WriteLine(
"Could not find git.exe {0}",
!string.IsNullOrWhiteSpace(this.GitBinPath) ? "at " + this.GitBinPath : "on %PATH%");
return(ExitFailure);
}
if (this.Commit != null && this.Branch != null)
{
Console.WriteLine("Cannot specify both a commit sha and a branch name.");
return(ExitFailure);
}
this.CacheServerUrl = Enlistment.StripObjectsEndpointSuffix(this.CacheServerUrl);
this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount;
this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads);
this.IndexThreadCount = this.IndexThreadCount > 0 ? this.IndexThreadCount : Environment.ProcessorCount;
this.CheckoutThreadCount = this.CheckoutThreadCount > 0 ? this.CheckoutThreadCount : Environment.ProcessorCount;
this.GitBinPath = !string.IsNullOrWhiteSpace(this.GitBinPath) ? this.GitBinPath : GitProcess.GetInstalledGitBinPath();
GitEnlistment enlistment = GitEnlistment.CreateFromCurrentDirectory(this.CacheServerUrl, this.GitBinPath);
if (enlistment == null)
{
Console.WriteLine("Must be run within a git repo");
return(ExitFailure);
}
string commitish = this.Commit ?? this.Branch;
if (string.IsNullOrWhiteSpace(commitish))
{
GitProcess.Result result = new GitProcess(enlistment).GetCurrentBranchName();
if (result.HasErrors || string.IsNullOrWhiteSpace(result.Output))
{
Console.WriteLine("Could not retrieve current branch name: " + result.Errors);
return(ExitFailure);
}
commitish = result.Output.Trim();
}
Guid parentActivityId = Guid.Empty;
if (!string.IsNullOrWhiteSpace(this.ParentActivityId) && !Guid.TryParse(this.ParentActivityId, out parentActivityId))
{
Console.WriteLine("The ParentActivityId provided (" + this.ParentActivityId + ") is not a valid GUID.");
}
using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch"))
{
if (this.Verbose)
{
tracer.AddConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch");
tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any);
tracer.WriteStartEvent(
enlistment.EnlistmentRoot,
enlistment.RepoUrl,
enlistment.CacheServerUrl,
new EventMetadata
{
{ "TargetCommitish", commitish },
{ "Checkout", this.Checkout },
});
FetchHelper fetchHelper = this.GetFetchHelper(tracer, enlistment);
fetchHelper.MaxRetries = this.MaxRetries;
if (!FetchHelper.TryLoadPathWhitelist(tracer, this.PathWhitelist, this.PathWhitelistFile, enlistment, fetchHelper.PathWhitelist))
{
return(ExitFailure);
}
bool isSuccess;
try
{
Func <bool> doPrefetch =
() =>
{
try
{
bool isBranch = this.Commit == null;
fetchHelper.FastFetch(commitish, isBranch);
return(!fetchHelper.HasFailures);
}
catch (FetchHelper.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
isSuccess = doPrefetch();
}
else
{
isSuccess = ConsoleHelper.ShowStatusWhileRunning(
doPrefetch,
"Fetching",
output: Console.Out,
showSpinner: !Console.IsOutputRedirected);
Console.WriteLine();
Console.WriteLine("FastFetch is complete. See the full logs at " + fastfetchLogFile);
}
isSuccess &= !fetchHelper.HasFailures;
}
catch (AggregateException e)
{
isSuccess = false;
foreach (Exception ex in e.Flatten().InnerExceptions)
{
tracer.RelatedError(ex.ToString());
}
}
catch (Exception e)
{
isSuccess = false;
tracer.RelatedError(e.ToString());
}
EventMetadata stopMetadata = new EventMetadata();
stopMetadata.Add("Success", isSuccess);
tracer.Stop(stopMetadata);
return(isSuccess ? ExitSuccess : ExitFailure);
}
}
private void LoadEventDescriptor()
{
if (_idSpecified)
{
List <EventMetadata> matchedEvents = new();
foreach (EventMetadata emd in _providerMetadata.Events)
{
if (emd.Id == _id)
{
matchedEvents.Add(emd);
}
}
if (matchedEvents.Count == 0)
{
string msg = string.Format(CultureInfo.InvariantCulture,
_resourceMgr.GetString("IncorrectEventId"),
_id,
_providerName);
throw new EventWriteException(msg);
}
EventMetadata matchedEvent = null;
if (!_versionSpecified && matchedEvents.Count == 1)
{
matchedEvent = matchedEvents[0];
}
else
{
if (_versionSpecified)
{
foreach (EventMetadata emd in matchedEvents)
{
if (emd.Version == _version)
{
matchedEvent = emd;
break;
}
}
if (matchedEvent == null)
{
string msg = string.Format(CultureInfo.InvariantCulture,
_resourceMgr.GetString("IncorrectEventVersion"),
_version,
_id,
_providerName);
throw new EventWriteException(msg);
}
}
else
{
string msg = string.Format(CultureInfo.InvariantCulture,
_resourceMgr.GetString("VersionNotSpecified"),
_id,
_providerName);
throw new EventWriteException(msg);
}
}
VerifyTemplate(matchedEvent);
_eventDescriptor = CreateEventDescriptor(_providerMetadata, matchedEvent);
}
else
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, _resourceMgr.GetString("EventIdNotSpecified")), "Id");
}
}
private Connected(Guid aggregateGuid, string effectiveDateTime, string baseContentGuid,
EventMetadata metadata, double temperature, double heatSetpoint, double coolSetpoint, string mode, string systemStatus, double?humidity, int version)
: this(aggregateGuid, DateTimeOffset.Parse(effectiveDateTime), metadata, temperature, heatSetpoint, coolSetpoint, mode, systemStatus, humidity)
{
ExpectedVersion = version;
}
private static EventDescriptor CreateEventDescriptor(ProviderMetadata providerMetaData, EventMetadata emd)
{
long keywords = 0;
foreach (EventKeyword keyword in emd.Keywords)
{
keywords |= keyword.Value;
}
byte channel = 0;
foreach (EventLogLink logLink in providerMetaData.LogLinks)
{
if (string.Equals(logLink.LogName, emd.LogLink.LogName, StringComparison.OrdinalIgnoreCase))
{
break;
}
channel++;
}
return(new EventDescriptor(
(int)emd.Id,
emd.Version,
channel,
(byte)emd.Level.Value,
(byte)emd.Opcode.Value,
emd.Task.Value,
keywords));
}
protected override void Execute(GVFSEnlistment enlistment)
{
using (JsonEtwTracer tracer = new JsonEtwTracer(GVFSConstants.GVFSEtwProviderName, "Prefetch"))
{
if (this.Verbose)
{
tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment);
tracer.AddLogFileEventListener(
GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Prefetch),
EventLevel.Informational,
Keywords.Any);
tracer.WriteStartEvent(
enlistment.EnlistmentRoot,
enlistment.RepoUrl,
cacheServerUrl,
enlistment.GitObjectsRoot);
RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
CacheServerInfo cacheServer = this.ResolvedCacheServer;
if (!this.SkipVersionCheck)
{
string authErrorMessage;
if (!this.ShowStatusWhileRunning(
() => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage),
"Authenticating"))
{
this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed");
}
GVFSConfig gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig);
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig);
this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: false);
}
try
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Commits", this.Commits);
metadata.Add("Files", this.Files);
metadata.Add("Folders", this.Folders);
metadata.Add("FoldersListFile", this.FoldersListFile);
tracer.RelatedEvent(EventLevel.Informational, "PerformPrefetch", metadata);
GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig);
if (this.Commits)
{
if (!string.IsNullOrWhiteSpace(this.Files) ||
!string.IsNullOrWhiteSpace(this.Folders) ||
!string.IsNullOrWhiteSpace(this.FoldersListFile))
{
this.ReportErrorAndExit(tracer, "You cannot prefetch commits and blobs at the same time.");
}
this.PrefetchCommits(tracer, enlistment, objectRequestor, cacheServer);
}
else
{
this.PrefetchBlobs(tracer, enlistment, objectRequestor, cacheServer);
}
}
catch (VerbAbortedException)
{
throw;
}
catch (AggregateException aggregateException)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
foreach (Exception innerException in aggregateException.Flatten().InnerExceptions)
{
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", innerException.ToString() }
},
$"Unhandled {innerException.GetType().Name}: {innerException.Message}");
}
Environment.ExitCode = (int)ReturnCode.GenericError;
}
catch (Exception e)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", e.ToString() }
},
$"Unhandled {e.GetType().Name}: {e.Message}");
}
}
}
public override FileSystemResult UpdatePlaceholderIfNeeded(
string relativePath,
DateTime creationTime,
DateTime lastAccessTime,
DateTime lastWriteTime,
DateTime changeTime,
uint fileAttributes,
long endOfFile,
string shaContentId,
UpdatePlaceholderType updateFlags,
out UpdateFailureReason failureReason)
{
UpdateFailureCause failureCause = UpdateFailureCause.NoFailure;
// TODO(Mac): Add functional tests that include:
// - Mode + content changes between commits
// - Mode only changes (without any change to content, see issue #223)
GitIndexProjection.FileType fileType;
ushort fileMode;
this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath, out fileType, out fileMode);
if (fileType == GitIndexProjection.FileType.Regular)
{
Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded(
relativePath,
PlaceholderVersionId,
ToVersionIdByteArray(ConvertShaToContentId(shaContentId)),
(ulong)endOfFile,
fileMode,
(UpdateType)updateFlags,
out failureCause);
failureReason = (UpdateFailureReason)failureCause;
return(new FileSystemResult(ResultToFSResult(result), unchecked ((int)result)));
}
else if (fileType == GitIndexProjection.FileType.SymLink)
{
string symLinkTarget;
if (this.TryGetSymLinkTarget(shaContentId, out symLinkTarget))
{
Result result = this.virtualizationInstance.ReplacePlaceholderFileWithSymLink(
relativePath,
symLinkTarget,
(UpdateType)updateFlags,
out failureCause);
this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath);
failureReason = (UpdateFailureReason)failureCause;
return(new FileSystemResult(ResultToFSResult(result), unchecked ((int)result)));
}
EventMetadata metadata = this.CreateEventMetadata(relativePath);
metadata.Add(nameof(shaContentId), shaContentId);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Failed to read contents of symlink object");
failureReason = UpdateFailureReason.NoFailure;
return(new FileSystemResult(FSResult.IOError, 0));
}
else
{
EventMetadata metadata = this.CreateEventMetadata(relativePath);
metadata.Add(nameof(fileType), fileType);
metadata.Add(nameof(fileMode), fileMode);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType");
failureReason = UpdateFailureReason.NoFailure;
return(new FileSystemResult(FSResult.IOError, 0));
}
}
private void ValidateEventOpcodeForTransfer(ref EventMetadata eventData)
{
if ((EventOpcode)eventData.Descriptor.Opcode != EventOpcode.Send &&
(EventOpcode)eventData.Descriptor.Opcode != EventOpcode.Receive)
{
ThrowEventSourceException();
}
}
/// <summary>
/// Gets the target of the symbolic link.
/// </summary>
/// <param name="sha">SHA of the loose object containing the target path of the symbolic link</param>
/// <param name="symLinkTarget">Target path of the symbolic link</param>
private bool TryGetSymLinkTarget(string sha, out string symLinkTarget)
{
symLinkTarget = null;
string symLinkBlobContents = null;
try
{
if (!this.GitObjects.TryCopyBlobContentStream(
sha,
CancellationToken.None,
GVFSGitObjects.RequestSource.SymLinkCreation,
(stream, blobLength) =>
{
byte[] buffer = new byte[SymLinkTargetBufferSize];
uint bufferIndex = 0;
// TODO(Mac): Find a better solution than reading from the stream one byte at at time
int nextByte = stream.ReadByte();
while (nextByte != -1)
{
while (bufferIndex < buffer.Length && nextByte != -1)
{
buffer[bufferIndex] = (byte)nextByte;
nextByte = stream.ReadByte();
++bufferIndex;
}
if (bufferIndex < buffer.Length)
{
buffer[bufferIndex] = 0;
symLinkBlobContents = Encoding.UTF8.GetString(buffer);
}
else
{
buffer[bufferIndex - 1] = 0;
EventMetadata metadata = this.CreateEventMetadata();
metadata.Add(nameof(sha), sha);
metadata.Add("bufferContents", Encoding.UTF8.GetString(buffer));
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: SymLink target exceeds buffer size");
throw new GetSymLinkTargetException("SymLink target exceeds buffer size");
}
}
}))
{
EventMetadata metadata = this.CreateEventMetadata();
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream failed");
return(false);
}
}
catch (GetSymLinkTargetException e)
{
EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e);
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught GetSymLinkTargetException");
return(false);
}
catch (DecoderFallbackException e)
{
EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e);
metadata.Add(nameof(sha), sha);
this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught DecoderFallbackException");
return(false);
}
symLinkTarget = symLinkBlobContents;
return(true);
}
// Helper used by code:CreateManifestAndDescriptors to add a code:EventData descriptor for a method
// with the code:EventAttribute 'eventAttribute'. resourceManger may be null in which case we populate it
// it is populated if we need to look up message resources
private static void AddEventDescriptor(ref EventMetadata[] eventData, string eventName,
EventAttribute eventAttribute, ParameterInfo[] eventParameters)
{
if (eventData == null || eventData.Length <= eventAttribute.EventId)
{
EventMetadata[] newValues = new EventMetadata[Math.Max(eventData.Length + 16, eventAttribute.EventId + 1)];
Array.Copy(eventData, newValues, eventData.Length);
eventData = newValues;
}
eventData[eventAttribute.EventId].Descriptor = new System.Diagnostics.Tracing.EventDescriptor(
eventAttribute.EventId,
eventAttribute.Version,
#if FEATURE_MANAGED_ETW_CHANNELS
(byte)eventAttribute.Channel,
#else
(byte)0,
#endif
(byte)eventAttribute.Level,
(byte)eventAttribute.Opcode,
(int)eventAttribute.Task,
(long)((ulong)eventAttribute.Keywords | SessionMask.All.ToEventKeywords()));
eventData[eventAttribute.EventId].Name = eventName;
eventData[eventAttribute.EventId].Parameters = eventParameters;
eventData[eventAttribute.EventId].Message = eventAttribute.Message;
}
private Result OnGetFileStream(
ulong commandId,
string relativePath,
byte[] providerId,
byte[] contentId,
int triggeringProcessId,
string triggeringProcessName,
IntPtr fileHandle)
{
try
{
if (contentId == null)
{
this.Context.Tracer.RelatedError($"{nameof(this.OnGetFileStream)} called with null contentId, path: " + relativePath);
return(Result.EInvalidOperation);
}
if (providerId == null)
{
this.Context.Tracer.RelatedError($"{nameof(this.OnGetFileStream)} called with null epochId, path: " + relativePath);
return(Result.EInvalidOperation);
}
string sha = GetShaFromContentId(contentId);
byte placeholderVersion = GetPlaceholderVersionFromProviderId(providerId);
EventMetadata metadata = this.CreateEventMetadata(relativePath);
metadata.Add(nameof(triggeringProcessId), triggeringProcessId);
metadata.Add(nameof(triggeringProcessName), triggeringProcessName);
metadata.Add(nameof(sha), sha);
metadata.Add(nameof(placeholderVersion), placeholderVersion);
metadata.Add(nameof(commandId), commandId);
ITracer activity = this.Context.Tracer.StartActivity("GetFileStream", EventLevel.Verbose, Keywords.Telemetry, metadata);
if (!this.FileSystemCallbacks.IsMounted)
{
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(this.OnGetFileStream)} failed, mount has not yet completed");
activity.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnGetFileStream)}_MountNotComplete", metadata);
activity.Dispose();
// TODO(Mac): Is this the correct Result to return?
return(Result.EIOError);
}
if (placeholderVersion != FileSystemVirtualizer.PlaceholderVersion)
{
activity.RelatedError(metadata, nameof(this.OnGetFileStream) + ": Unexpected placeholder version");
activity.Dispose();
// TODO(Mac): Is this the correct Result to return?
return(Result.EIOError);
}
try
{
if (!this.GitObjects.TryCopyBlobContentStream(
sha,
CancellationToken.None,
GVFSGitObjects.RequestSource.FileStreamCallback,
(stream, blobLength) =>
{
// TODO(Mac): Find a better solution than reading from the stream one byte at at time
byte[] buffer = new byte[4096];
uint bufferIndex = 0;
int nextByte = stream.ReadByte();
while (nextByte != -1)
{
while (bufferIndex < buffer.Length && nextByte != -1)
{
buffer[bufferIndex] = (byte)nextByte;
nextByte = stream.ReadByte();
++bufferIndex;
}
Result result = this.virtualizationInstance.WriteFileContents(
fileHandle,
buffer,
bufferIndex);
if (result != Result.Success)
{
activity.RelatedError(metadata, $"{nameof(this.virtualizationInstance.WriteFileContents)} failed, error: " + result.ToString("X") + "(" + result.ToString("G") + ")");
throw new GetFileStreamException(result);
}
if (bufferIndex == buffer.Length)
{
bufferIndex = 0;
}
}
}))
{
activity.RelatedError(metadata, $"{nameof(this.OnGetFileStream)}: TryCopyBlobContentStream failed");
// TODO(Mac): Is this the correct Result to return?
return(Result.EFileNotFound);
}
}
catch (GetFileStreamException e)
{
return(e.Result);
}
return(Result.Success);
}
catch (Exception e)
{
EventMetadata metadata = this.CreateEventMetadata(relativePath, e);
metadata.Add(nameof(triggeringProcessId), triggeringProcessId);
metadata.Add(nameof(triggeringProcessName), triggeringProcessName);
metadata.Add(nameof(commandId), commandId);
this.LogUnhandledExceptionAndExit(nameof(this.OnGetFileStream), metadata);
}
return(Result.EIOError);
}
// Helper used by code:CreateManifestAndDescriptors to find user mistakes like reusing an event
// index for two distinct events etc. Throws exceptions when it finds something wrong.
private static void DebugCheckEvent(ref Dictionary<string, string> eventsByName,
EventMetadata[] eventData, MethodInfo method, EventAttribute eventAttribute)
{
int eventArg = GetHelperCallFirstArg(method);
if (eventArg >= 0 && eventAttribute.EventId != eventArg)
{
throw new ArgumentException(Environment.GetResourceString("EventSource_MismatchIdToWriteEvent", method.Name, eventAttribute.EventId, eventArg));
}
if (eventAttribute.EventId < eventData.Length && eventData[eventAttribute.EventId].Descriptor.EventId != 0)
{
throw new ArgumentException(Environment.GetResourceString("EventSource_EventIdReused", method.Name, eventAttribute.EventId));
}
if (eventsByName == null)
eventsByName = new Dictionary<string, string>();
if (eventsByName.ContainsKey(method.Name))
throw new ArgumentException(Environment.GetResourceString("EventSource_EventNameReused", method.Name));
eventsByName[method.Name] = method.Name;
}
/// <summary>
/// Rebuild the status cache. This will run the background status to
/// generate status results, and update the serialized status cache
/// file.
/// </summary>
private bool TryRebuildStatusCache()
{
try
{
this.context.FileSystem.CreateDirectory(this.context.Enlistment.GitStatusCacheFolder);
}
catch (Exception ex) when(ex is IOException || ex is UnauthorizedAccessException)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("Exception", ex.ToString());
this.context.Tracer.RelatedWarning(
metadata,
string.Format("GitStatusCache is unable to create git status cache folder at {0}.", this.context.Enlistment.GitStatusCacheFolder));
return(false);
}
// The status cache is regenerated on mount. This means that even if the write to temp file
// and rename operation doesn't complete (due to a system crash), and there is a torn write,
// GSD is still protected because a new status cache file will be generated on mount.
string tmpStatusFilePath = Path.Combine(this.context.Enlistment.GitStatusCacheFolder, Path.GetRandomFileName() + "_status.tmp");
GitProcess.Result statusResult = null;
// Do not modify this block unless you completely understand the comments and code within
{
// We MUST set the state to Rebuilding _immediately before_ we call the `git status` command. That allows us to
// check afterwards if anything happened during the status command that should invalidate the cache, and we
// can discard its results if that happens.
this.cacheState = CacheState.Rebuilding;
GitProcess git = this.context.Enlistment.CreateGitProcess();
statusResult = git.SerializeStatus(
allowObjectDownloads: true,
serializePath: tmpStatusFilePath);
}
bool rebuildSucceeded = false;
if (statusResult.ExitCodeIsSuccess)
{
lock (this.cacheFileLock)
{
// Only update the cache if our state is still Rebuilding. Otherwise, this indicates that another call
// to Invalidate came in, and moved the state back to Dirty.
if (this.cacheState == CacheState.Rebuilding)
{
rebuildSucceeded = this.MoveCacheFileToFinalLocation(tmpStatusFilePath);
if (rebuildSucceeded)
{
// We have to check the state once again, because it could have been invalidated while we were
// copying the file in the previous step. Here we do it as a CompareExchange to minimize any further races.
if (Interlocked.CompareExchange(ref this.cacheState, CacheState.Clean, CacheState.Rebuilding) != CacheState.Rebuilding)
{
// We did not succeed in setting the state to Clean. Note that we have already overwritten the on disk cache,
// but all users of the cache file first check the cacheState, and since the cacheState is not Clean, no one
// should ever read it.
rebuildSucceeded = false;
}
}
if (!rebuildSucceeded)
{
this.cacheState = CacheState.Dirty;
}
}
}
if (!rebuildSucceeded)
{
try
{
this.context.FileSystem.DeleteFile(tmpStatusFilePath);
}
catch (Exception ex) when(ex is IOException || ex is UnauthorizedAccessException)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("Exception", ex.ToString());
this.context.Tracer.RelatedError(
metadata,
string.Format("GitStatusCache is unable to delete temporary status cache file at {0}.", tmpStatusFilePath));
}
}
}
else
{
this.statistics.RecordBackgroundStatusScanError();
this.context.Tracer.RelatedInfo("GitStatusCache.TryRebuildStatusCache: Error generating status: {0}", statusResult.Errors);
}
return(rebuildSucceeded);
}
private int GetParameterCount(EventMetadata eventData)
{
return eventData.Parameters.Length;
}
private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate)
{
EventMetadata metadata = new EventMetadata();
FileSystemTaskResult result;
switch (gitUpdate.Operation)
{
case FileSystemTask.OperationType.OnFileCreated:
case FileSystemTask.OperationType.OnFailedPlaceholderDelete:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFileRenamed:
metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = FileSystemTaskResult.Success;
if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) && !IsPathInsideDotGit(gitUpdate.OldVirtualPath))
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.OldVirtualPath);
}
if (result == FileSystemTaskResult.Success &&
!string.IsNullOrEmpty(gitUpdate.VirtualPath) &&
!IsPathInsideDotGit(gitUpdate.VirtualPath))
{
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
}
break;
case FileSystemTask.OperationType.OnFileDeleted:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFileOverwritten:
case FileSystemTask.OperationType.OnFileSuperseded:
case FileSystemTask.OperationType.OnFileConvertedToFull:
case FileSystemTask.OperationType.OnFailedPlaceholderUpdate:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath);
break;
case FileSystemTask.OperationType.OnFolderCreated:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
break;
case FileSystemTask.OperationType.OnFolderRenamed:
result = FileSystemTaskResult.Success;
metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
metadata.Add("virtualPath", gitUpdate.VirtualPath);
// An empty destination path means the folder was renamed to somewhere outside of the repo
// Note that only full folders can be moved\renamed, and so there will already be a recursive
// sparse-checkout entry for the virtualPath of the folder being moved (meaning that no
// additional work is needed for any files\folders inside the folder being moved)
if (!string.IsNullOrEmpty(gitUpdate.VirtualPath))
{
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
if (result == FileSystemTaskResult.Success)
{
Queue <string> relativeFolderPaths = new Queue <string>();
relativeFolderPaths.Enqueue(gitUpdate.VirtualPath);
// Add all the files in the renamed folder to the always_exclude file
while (relativeFolderPaths.Count > 0)
{
string folderPath = relativeFolderPaths.Dequeue();
if (result == FileSystemTaskResult.Success)
{
try
{
foreach (DirectoryItemInfo itemInfo in this.context.FileSystem.ItemsInDirectory(Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, folderPath)))
{
string itemVirtualPath = Path.Combine(folderPath, itemInfo.Name);
if (itemInfo.IsDirectory)
{
relativeFolderPaths.Enqueue(itemVirtualPath);
}
else
{
string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length);
result = this.TryAddModifiedPath(itemVirtualPath, isFolder: false);
}
}
}
catch (DirectoryNotFoundException)
{
// DirectoryNotFoundException can occur when the renamed folder (or one of its children) is
// deleted prior to the background thread running
EventMetadata exceptionMetadata = new EventMetadata();
exceptionMetadata.Add("Area", "ExecuteBackgroundOperation");
exceptionMetadata.Add("Operation", gitUpdate.Operation.ToString());
exceptionMetadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath);
exceptionMetadata.Add("virtualPath", gitUpdate.VirtualPath);
exceptionMetadata.Add(TracingConstants.MessageKey.InfoMessage, "DirectoryNotFoundException while traversing folder path");
exceptionMetadata.Add("folderPath", folderPath);
this.context.Tracer.RelatedEvent(EventLevel.Informational, "DirectoryNotFoundWhileUpdatingAlwaysExclude", exceptionMetadata);
}
catch (IOException e)
{
metadata.Add("Details", "IOException while traversing folder path");
metadata.Add("folderPath", folderPath);
metadata.Add("Exception", e.ToString());
result = FileSystemTaskResult.RetryableError;
break;
}
catch (UnauthorizedAccessException e)
{
metadata.Add("Details", "UnauthorizedAccessException while traversing folder path");
metadata.Add("folderPath", folderPath);
metadata.Add("Exception", e.ToString());
result = FileSystemTaskResult.RetryableError;
break;
}
}
else
{
break;
}
}
}
}
break;
case FileSystemTask.OperationType.OnFolderDeleted:
metadata.Add("virtualPath", gitUpdate.VirtualPath);
result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true);
break;
case FileSystemTask.OperationType.OnFolderFirstWrite:
result = FileSystemTaskResult.Success;
break;
case FileSystemTask.OperationType.OnIndexWriteWithoutProjectionChange:
result = this.GitIndexProjection.AddMissingModifiedFiles();
break;
case FileSystemTask.OperationType.OnPlaceholderCreationsBlockedForGit:
this.GitIndexProjection.ClearNegativePathCacheIfPollutedByGit();
result = FileSystemTaskResult.Success;
break;
default:
throw new InvalidOperationException("Invalid background operation");
}
if (result != FileSystemTaskResult.Success)
{
metadata.Add("Area", "ExecuteBackgroundOperation");
metadata.Add("Operation", gitUpdate.Operation.ToString());
metadata.Add(TracingConstants.MessageKey.WarningMessage, "Background operation failed");
metadata.Add(nameof(result), result.ToString());
this.context.Tracer.RelatedEvent(EventLevel.Warning, "FailedBackgroundOperation", metadata);
}
return(result);
}
public void Create( System.Collections.Generic.Dictionary<string,object> eventDict )
{
Debug.Log( "IgniteEvent:Create");
if( eventDict.ContainsKey( "id" ) ) {
this.Id = Convert.ToString( eventDict["id"] );
}
if( eventDict.ContainsKey( "startTime" ) ) {
var epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
long t = Convert.ToInt64 (eventDict["startTime"]);
this.StartTime = epoch.AddSeconds(t);
}
if( eventDict.ContainsKey( "authorized" ) ) {
this.Authorized = Convert.ToBoolean( eventDict["authorized"] );
}
if( eventDict.ContainsKey( "eventId" ) ) {
this.EventId = Convert.ToString( eventDict["eventId"] );
}
if( eventDict.ContainsKey( "state" ) ) {
this.State = Convert.ToString( eventDict["state"] );
}
if( eventDict.ContainsKey( "score" ) ) {
this.Score = (float)Convert.ToDouble( eventDict["score"] );
}
if( eventDict.ContainsKey( "type" ) ) {
this.Type = (IgniteEventType) Enum.Parse( typeof(IgniteEventType) , Convert.ToString( eventDict["type"] ) );
}
if( eventDict.ContainsKey( "endTime" ) ) {
var epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
long t = Convert.ToInt64 (eventDict["endTime"]);
this.EndTime = epoch.AddSeconds(t);
}
if( eventDict.ContainsKey( "metadata" ) ) {
System.Collections.Generic.Dictionary<string,object> eventMetadataDict = eventDict["metadata"] as System.Collections.Generic.Dictionary<string,object>;
EventMetadata eventMetadata = new EventMetadata();
if( eventMetadataDict.ContainsKey( "imageUrl" ) ) {
eventMetadata.imageUrl = Convert.ToString( eventMetadataDict["imageUrl"] );
}
if( eventMetadataDict.ContainsKey( "name" ) ) {
eventMetadata.Name = Convert.ToString( eventMetadataDict["name"] );
}
if( eventMetadataDict.ContainsKey( "gamedata" ) ) {
eventMetadata.GameData = Convert.ToString( eventMetadataDict["gamedata"] );
}
this.Metadata = eventMetadata;
}
switch( this.Type ) {
case IgniteEventType.leaderBoard:
FuelSDKGroovePlanetIntegration.Instance.GetLeaderBoard( this.Id );
activity = new IgniteLeaderBoard();
break;
case IgniteEventType.mission:
FuelSDKGroovePlanetIntegration.Instance.GetMission( this.Id );
activity = new IgniteMission();
break;
case IgniteEventType.quest:
FuelSDKGroovePlanetIntegration.Instance.GetQuest( this.Id );
activity = new IgniteQuest();
break;
default:
break;
}
}
public virtual GitProcess.Result IndexPackFile(string packfilePath, GitProcess gitProcess)
{
string tempIdxPath = Path.ChangeExtension(packfilePath, TempIdxExtension);
string idxPath = Path.ChangeExtension(packfilePath, ".idx");
Exception indexPackException = null;
try
{
if (gitProcess == null)
{
gitProcess = new GitProcess(this.Enlistment);
}
GitProcess.Result result = gitProcess.IndexPack(packfilePath, tempIdxPath);
if (result.ExitCodeIsFailure)
{
Exception exception;
if (!this.fileSystem.TryDeleteFile(tempIdxPath, exception: out exception))
{
EventMetadata metadata = CreateEventMetadata(exception);
metadata.Add("tempIdxPath", tempIdxPath);
this.Tracer.RelatedWarning(metadata, $"{nameof(this.IndexPackFile)}: Failed to cleanup temp idx file after index pack failure");
}
}
else
{
if (this.Enlistment.FlushFileBuffersForPacks)
{
Exception exception;
string error;
if (!this.TryFlushFileBuffers(tempIdxPath, out exception, out error))
{
EventMetadata metadata = CreateEventMetadata(exception);
metadata.Add("packfilePath", packfilePath);
metadata.Add("tempIndexPath", tempIdxPath);
metadata.Add("error", error);
this.Tracer.RelatedWarning(metadata, $"{nameof(this.IndexPackFile)}: Failed to flush temp idx file buffers");
}
}
this.fileSystem.MoveAndOverwriteFile(tempIdxPath, idxPath);
}
return(result);
}
catch (Win32Exception e)
{
indexPackException = e;
}
catch (IOException e)
{
indexPackException = e;
}
catch (UnauthorizedAccessException e)
{
indexPackException = e;
}
EventMetadata failureMetadata = CreateEventMetadata(indexPackException);
failureMetadata.Add("packfilePath", packfilePath);
failureMetadata.Add("tempIdxPath", tempIdxPath);
failureMetadata.Add("idxPath", idxPath);
this.fileSystem.TryDeleteFile(tempIdxPath, metadataKey: nameof(tempIdxPath), metadata: failureMetadata);
this.fileSystem.TryDeleteFile(idxPath, metadataKey: nameof(idxPath), metadata: failureMetadata);
this.Tracer.RelatedWarning(failureMetadata, $"{nameof(this.IndexPackFile): Exception caught while trying to index pack file}");
return(new GitProcess.Result(
string.Empty,
indexPackException != null ? indexPackException.Message : "Failed to index pack file",
GitProcess.Result.GenericFailureCode));
}