private Result CreateClone(
ITracer tracer,
GVFSEnlistment enlistment,
GitObjectsHttpRequestor objectRequestor,
GitRefs refs,
string branch)
{
Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string errorMessage;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage))
{
return(new Result("Error configuring alternate: " + errorMessage));
}
GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem);
GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment);
GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor);
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage))
{
return(new Result(errorMessage));
}
if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) ||
!GVFSVerb.TrySetOptionalGitConfigSettings(enlistment))
{
return(new Result("Unable to configure git repo"));
}
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage))
{
return(new Result("Unable to configure cache server: " + errorMessage));
}
GitProcess git = new GitProcess(enlistment);
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.HasErrors)
{
return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head),
"ref: refs/heads/" + branch);
if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage))
{
return(new Result(errorMessage));
}
this.CreateGitScript(enlistment);
string installHooksError;
if (!HooksInstaller.InstallHooks(context, out installHooksError))
{
tracer.RelatedError(installHooksError);
return(new Result(installHooksError));
}
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.HasErrors)
{
string[] errorLines = forceCheckoutResult.Errors.Split('\n');
StringBuilder checkoutErrors = new StringBuilder();
foreach (string gitError in errorLines)
{
if (IsForceCheckoutErrorCloneFailure(gitError))
{
checkoutErrors.AppendLine(gitError);
}
}
if (checkoutErrors.Length > 0)
{
string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString();
tracer.RelatedError(error);
return(new Result(error));
}
}
if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage))
{
tracer.RelatedError(errorMessage);
return(new Result(errorMessage));
}
try
{
RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment);
this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment);
}
catch (Exception e)
{
tracer.RelatedError(e.ToString());
return(new Result(e.Message));
}
finally
{
RepoMetadata.Shutdown();
}
// Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed
string prepFileSystemError;
if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError))
{
tracer.RelatedError(prepFileSystemError);
return(new Result(prepFileSystemError));
}
return(new Result(true));
}
public static bool TryGetMaxGoodPrefetchTimestamp(
ITracer tracer,
GVFSEnlistment enlistment,
PhysicalFileSystem fileSystem,
GitObjects gitObjects,
out long maxGoodTimestamp,
out string error)
{
fileSystem.CreateDirectory(enlistment.GitPackRoot);
string[] packs = gitObjects.ReadPackFileNames(enlistment.GitPackRoot, GVFSConstants.PrefetchPackPrefix);
List <PrefetchPackInfo> orderedPacks = packs
.Where(pack => GetTimestamp(pack).HasValue)
.Select(pack => new PrefetchPackInfo(GetTimestamp(pack).Value, pack))
.OrderBy(packInfo => packInfo.Timestamp)
.ToList();
maxGoodTimestamp = -1;
int firstBadPack = -1;
for (int i = 0; i < orderedPacks.Count; ++i)
{
long timestamp = orderedPacks[i].Timestamp;
string packPath = orderedPacks[i].Path;
string idxPath = Path.ChangeExtension(packPath, ".idx");
if (!fileSystem.FileExists(idxPath))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("pack", packPath);
metadata.Add("idxPath", idxPath);
metadata.Add("timestamp", timestamp);
GitProcess.Result indexResult = gitObjects.IndexPackFile(packPath);
if (indexResult.ExitCodeIsFailure)
{
firstBadPack = i;
metadata.Add("Errors", indexResult.Errors);
tracer.RelatedWarning(metadata, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}: Found pack file that's missing idx file, and failed to regenerate idx");
break;
}
else
{
maxGoodTimestamp = timestamp;
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}: Found pack file that's missing idx file, and regenerated idx");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_RebuildIdx", metadata);
}
}
else
{
maxGoodTimestamp = timestamp;
}
}
if (firstBadPack != -1)
{
const int MaxDeleteRetries = 200; // 200 * IoFailureRetryDelayMS (50ms) = 10 seconds
const int RetryLoggingThreshold = 40; // 40 * IoFailureRetryDelayMS (50ms) = 2 seconds
// Before we delete _any_ pack-files, we need to delete the multi-pack-index, which
// may refer to those packs.
EventMetadata metadata = new EventMetadata();
string midxPath = Path.Combine(enlistment.GitPackRoot, "multi-pack-index");
metadata.Add("path", midxPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting multi-pack-index");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteMultiPack_index", metadata);
if (!fileSystem.TryWaitForDelete(tracer, midxPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {midxPath}";
return(false);
}
// Delete packs and indexes in reverse order so that if prefetch is killed, subseqeuent prefetch commands will
// find the right starting spot.
for (int i = orderedPacks.Count - 1; i >= firstBadPack; --i)
{
string packPath = orderedPacks[i].Path;
string idxPath = Path.ChangeExtension(packPath, ".idx");
metadata = new EventMetadata();
metadata.Add("path", idxPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting bad idx file");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteBadIdx", metadata);
if (!fileSystem.TryWaitForDelete(tracer, idxPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {idxPath}";
return(false);
}
metadata = new EventMetadata();
metadata.Add("path", packPath);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting bad pack file");
tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteBadPack", metadata);
if (!fileSystem.TryWaitForDelete(tracer, packPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold))
{
error = $"Unable to delete {packPath}";
return(false);
}
}
}
error = null;
return(true);
}
private void PostFetchJob(List <string> packIndexes)
{
try
{
using (FileBasedLock postFetchFileLock = new FileBasedLock(
this.context.FileSystem,
this.context.Tracer,
Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock),
this.context.Enlistment.EnlistmentRoot,
overwriteExistingLock: true))
{
if (!postFetchFileLock.TryAcquireLockAndDeleteOnClose())
{
this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping post-fetch work since another process holds the lock");
return;
}
if (!this.gitObjects.TryWriteMultiPackIndex(this.context.Tracer, this.context.Enlistment, this.context.FileSystem))
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Failed to generate midx for new packfiles",
keywords: Keywords.Telemetry);
}
if (packIndexes == null || packIndexes.Count == 0)
{
this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping commit-graph write due to no new packfiles");
return;
}
using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null))
{
GitProcess process = new GitProcess(this.context.Enlistment);
GitProcess.Result result = process.WriteCommitGraph(this.context.Enlistment.GitObjectsRoot, packIndexes);
if (result.HasErrors)
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Failed to generate commit-graph for new packfiles:" + result.Errors,
keywords: Keywords.Telemetry);
return;
}
}
}
}
catch (ThreadAbortException)
{
this.context.Tracer.RelatedInfo("Aborting post-fetch job due to ThreadAbortException");
}
catch (IOException e)
{
this.context.Tracer.RelatedWarning(
metadata: this.CreateEventMetadata(null, e),
message: PostFetchTelemetryKey + ": IOException while running post-fetch job: " + e.Message,
keywords: Keywords.Telemetry);
}
catch (Exception e)
{
this.context.Tracer.RelatedError(
metadata: this.CreateEventMetadata(null, e),
message: PostFetchTelemetryKey + ": Exception while running post-fetch job: " + e.Message,
keywords: Keywords.Telemetry);
Environment.Exit((int)ReturnCode.GenericError);
}
}
private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor blobRequestor, CacheServerInfo cacheServer)
{
BlobPrefetcher blobPrefetcher = new BlobPrefetcher(
tracer,
enlistment,
blobRequestor,
ChunkSize,
SearchThreadCount,
DownloadThreadCount,
IndexThreadCount);
string error;
if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, blobPrefetcher.FolderList, out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (!BlobPrefetcher.TryLoadFileList(enlistment, this.Files, blobPrefetcher.FileList, out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (blobPrefetcher.FolderList.Count == 0 &&
blobPrefetcher.FileList.Count == 0)
{
this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files *` to confirm.");
}
if (this.HydrateFiles)
{
if (!this.CheckIsMounted(verbose: true))
{
this.ReportErrorAndExit("You can only specify --hydrate if the repo is mounted. Run 'gvfs mount' and try again.");
}
}
GitProcess gitProcess = new GitProcess(enlistment);
GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName);
if (result.HasErrors)
{
tracer.RelatedError(result.Errors);
this.Output.WriteLine(result.Errors);
Environment.ExitCode = (int)ReturnCode.GenericError;
return;
}
int matchedBlobCount = 0;
int downloadedBlobCount = 0;
int readFileCount = 0;
string headCommitId = result.Output;
Func <bool> doPrefetch =
() =>
{
try
{
blobPrefetcher.PrefetchWithStats(
headCommitId.Trim(),
isBranch: false,
readFilesAfterDownload: this.HydrateFiles,
matchedBlobCount: out matchedBlobCount,
downloadedBlobCount: out downloadedBlobCount,
readFileCount: out readFileCount);
return(!blobPrefetcher.HasFailures);
}
catch (BlobPrefetcher.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
doPrefetch();
}
else
{
string message =
this.HydrateFiles
? "Fetching blobs and hydrating files "
: "Fetching blobs ";
this.ShowStatusWhileRunning(doPrefetch, message + this.GetCacheServerDisplay(cacheServer));
}
if (blobPrefetcher.HasFailures)
{
Environment.ExitCode = 1;
}
else
{
Console.WriteLine();
Console.WriteLine("Stats:");
Console.WriteLine(" Matched blobs: " + matchedBlobCount);
Console.WriteLine(" Already cached: " + (matchedBlobCount - downloadedBlobCount));
Console.WriteLine(" Downloaded: " + downloadedBlobCount);
if (this.HydrateFiles)
{
Console.WriteLine(" Hydrated files: " + readFileCount);
}
}
}
private Result CreateClone(
ITracer tracer,
GVFSEnlistment enlistment,
GitObjectsHttpRequestor objectRequestor,
GitRefs refs,
string branch)
{
Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string errorMessage;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage))
{
return(new Result("Error configuring alternate: " + errorMessage));
}
GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem);
GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment);
GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor);
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage))
{
return(new Result(errorMessage));
}
if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) ||
!GVFSVerb.TrySetOptionalGitConfigSettings(enlistment))
{
return(new Result("Unable to configure git repo"));
}
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage))
{
return(new Result("Unable to configure cache server: " + errorMessage));
}
GitProcess git = new GitProcess(enlistment);
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.ExitCodeIsFailure)
{
return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Head),
"ref: refs/heads/" + branch);
if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage))
{
return(new Result(errorMessage));
}
this.CreateGitScript(enlistment);
string installHooksError;
if (!HooksInstaller.InstallHooks(context, out installHooksError))
{
tracer.RelatedError(installHooksError);
return(new Result(installHooksError));
}
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0)
{
// It is possible to have the above TryDownloadCommit() fail because we
// already have the commit and root tree we intend to check out, but
// don't have a tree further down the working directory. If we fail
// checkout here, its' because we don't have these trees and the
// read-object hook is not available yet. Force downloading the commit
// again and retry the checkout.
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage,
checkLocalObjectCache: false))
{
return(new Result(errorMessage));
}
forceCheckoutResult = git.ForceCheckout(branch);
}
if (forceCheckoutResult.ExitCodeIsFailure)
{
string[] errorLines = forceCheckoutResult.Errors.Split('\n');
StringBuilder checkoutErrors = new StringBuilder();
foreach (string gitError in errorLines)
{
if (IsForceCheckoutErrorCloneFailure(gitError))
{
checkoutErrors.AppendLine(gitError);
}
}
if (checkoutErrors.Length > 0)
{
string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString();
tracer.RelatedError(error);
return(new Result(error));
}
}
if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage))
{
tracer.RelatedError(errorMessage);
return(new Result(errorMessage));
}
try
{
RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment);
this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment);
}
catch (Exception e)
{
tracer.RelatedError(e.ToString());
return(new Result(e.Message));
}
finally
{
RepoMetadata.Shutdown();
}
// Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed
Exception exception;
string prepFileSystemError;
if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryBackingRoot, out prepFileSystemError, out exception))
{
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(prepFileSystemError), prepFileSystemError);
if (exception != null)
{
metadata.Add("Exception", exception.ToString());
}
tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed");
return(new Result(prepFileSystemError));
}
return(new Result(true));
}
private bool CheckGitStatus(ITracer tracer, GVFSEnlistment enlistment, bool fullDehydrate)
{
if (!this.NoStatus)
{
this.WriteMessage(tracer, "Running git status before dehydrating to make sure you don't have any pending changes.");
if (fullDehydrate)
{
this.WriteMessage(tracer, "If this takes too long, you can abort and run dehydrate with --no-status to skip this safety check.");
}
this.Output.WriteLine();
bool isMounted = false;
GitProcess.Result statusResult = null;
if (!this.ShowStatusWhileRunning(
() =>
{
if (this.ExecuteGVFSVerb <StatusVerb>(tracer) != ReturnCode.Success)
{
return(false);
}
isMounted = true;
GitProcess git = new GitProcess(enlistment);
statusResult = git.Status(allowObjectDownloads: false, useStatusCache: false, showUntracked: true);
if (statusResult.ExitCodeIsFailure)
{
return(false);
}
if (!statusResult.Output.Contains("nothing to commit, working tree clean"))
{
return(false);
}
return(true);
},
"Running git status",
suppressGvfsLogMessage: true))
{
this.Output.WriteLine();
if (!isMounted)
{
this.WriteMessage(tracer, "Failed to run git status because the repo is not mounted");
if (fullDehydrate)
{
this.WriteMessage(tracer, "Either mount first, or run with --no-status");
}
}
else if (statusResult.ExitCodeIsFailure)
{
this.WriteMessage(tracer, "Failed to run git status: " + statusResult.Errors);
}
else
{
this.WriteMessage(tracer, statusResult.Output);
this.WriteMessage(tracer, "git status reported that you have dirty files");
if (fullDehydrate)
{
this.WriteMessage(tracer, "Either commit your changes or run dehydrate with --no-status");
}
else
{
this.WriteMessage(tracer, "Either commit your changes or reset and clean your working directory.");
}
}
this.ReportErrorAndExit(tracer, "Dehydrate was aborted");
return(false);
}
else
{
return(true);
}
}
return(false);
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public override void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitId(branchOrCommit);
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);
// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
// Since pack indexer is the last to finish before checkout finishes, it should propagate completion.
// This prevents availableObjects from completing before packIndexer can push its objects through this link.
checkout.AvailableBlobShas.CompleteAdding();
checkout.WaitForCompletion();
this.HasFailures |= checkout.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
// Update the refspec before setting the upstream or git will complain the remote branch doesn't exist
this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational))
{
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch);
if (result.HasErrors)
{
activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors);
this.HasFailures = true;
}
}
}
bool shouldSignIndex = !this.GetIsIndexSigningOff();
// Update the index
EventMetadata updateIndexMetadata = new EventMetadata();
updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex);
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata))
{
Index sourceIndex = this.GetSourceIndex();
GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex);
indexGen.CreateFromHeadTree(indexVersion: 2);
this.HasFailures |= indexGen.HasFailures;
if (!indexGen.HasFailures)
{
Index newIndex = new Index(
this.Enlistment.EnlistmentRoot,
this.Tracer,
Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName),
readOnly: false);
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;
newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex);
}
}
}
}
private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor blobRequestor, CacheServerInfo cacheServer)
{
FetchHelper fetchHelper = new FetchHelper(
tracer,
enlistment,
blobRequestor,
ChunkSize,
SearchThreadCount,
DownloadThreadCount,
IndexThreadCount);
string error;
if (!FetchHelper.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, fetchHelper.FolderList, out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (!FetchHelper.TryLoadFileList(enlistment, this.Files, fetchHelper.FileList, out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (fetchHelper.FolderList.Count == 0 &&
fetchHelper.FileList.Count == 0)
{
this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files *` to confirm.");
}
GitProcess gitProcess = new GitProcess(enlistment);
GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName);
if (result.HasErrors)
{
tracer.RelatedError(result.Errors);
this.Output.WriteLine(result.Errors);
Environment.ExitCode = (int)ReturnCode.GenericError;
return;
}
int matchedBlobs = 0;
int downloadedBlobs = 0;
string headCommitId = result.Output;
Func <bool> doPrefetch =
() =>
{
try
{
fetchHelper.FastFetchWithStats(
headCommitId.Trim(),
isBranch: false,
matchedBlobs: out matchedBlobs,
downloadedBlobs: out downloadedBlobs);
return(!fetchHelper.HasFailures);
}
catch (FetchHelper.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
doPrefetch();
}
else
{
this.ShowStatusWhileRunning(doPrefetch, "Fetching blobs " + this.GetCacheServerDisplay(cacheServer));
}
if (fetchHelper.HasFailures)
{
Environment.ExitCode = 1;
}
else
{
Console.WriteLine("Your filter matched {0} blob(s), and {1} new blob(s) were downloaded.", matchedBlobs, downloadedBlobs);
}
}
/// <summary>
/// Rebuild the status cache. This will run the background status to
/// generate status results, and update the serialized status cache
/// file.
/// </summary>
private bool TryRebuildStatusCache()
{
try
{
this.context.FileSystem.CreateDirectory(this.context.Enlistment.GitStatusCacheFolder);
}
catch (Exception ex) when(ex is IOException || ex is UnauthorizedAccessException)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("Exception", ex.ToString());
this.context.Tracer.RelatedWarning(
metadata,
string.Format("GitStatusCache is unable to create git status cache folder at {0}.", this.context.Enlistment.GitStatusCacheFolder));
return(false);
}
// The status cache is regenerated on mount. This means that even if the write to temp file
// and rename operation doesn't complete (due to a system crash), and there is a torn write,
// GSD is still protected because a new status cache file will be generated on mount.
string tmpStatusFilePath = Path.Combine(this.context.Enlistment.GitStatusCacheFolder, Path.GetRandomFileName() + "_status.tmp");
GitProcess.Result statusResult = null;
// Do not modify this block unless you completely understand the comments and code within
{
// We MUST set the state to Rebuilding _immediately before_ we call the `git status` command. That allows us to
// check afterwards if anything happened during the status command that should invalidate the cache, and we
// can discard its results if that happens.
this.cacheState = CacheState.Rebuilding;
GitProcess git = this.context.Enlistment.CreateGitProcess();
statusResult = git.SerializeStatus(
allowObjectDownloads: true,
serializePath: tmpStatusFilePath);
}
bool rebuildSucceeded = false;
if (statusResult.ExitCodeIsSuccess)
{
lock (this.cacheFileLock)
{
// Only update the cache if our state is still Rebuilding. Otherwise, this indicates that another call
// to Invalidate came in, and moved the state back to Dirty.
if (this.cacheState == CacheState.Rebuilding)
{
rebuildSucceeded = this.MoveCacheFileToFinalLocation(tmpStatusFilePath);
if (rebuildSucceeded)
{
// We have to check the state once again, because it could have been invalidated while we were
// copying the file in the previous step. Here we do it as a CompareExchange to minimize any further races.
if (Interlocked.CompareExchange(ref this.cacheState, CacheState.Clean, CacheState.Rebuilding) != CacheState.Rebuilding)
{
// We did not succeed in setting the state to Clean. Note that we have already overwritten the on disk cache,
// but all users of the cache file first check the cacheState, and since the cacheState is not Clean, no one
// should ever read it.
rebuildSucceeded = false;
}
}
if (!rebuildSucceeded)
{
this.cacheState = CacheState.Dirty;
}
}
}
if (!rebuildSucceeded)
{
try
{
this.context.FileSystem.DeleteFile(tmpStatusFilePath);
}
catch (Exception ex) when(ex is IOException || ex is UnauthorizedAccessException)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Area", EtwArea);
metadata.Add("Exception", ex.ToString());
this.context.Tracer.RelatedError(
metadata,
string.Format("GitStatusCache is unable to delete temporary status cache file at {0}.", tmpStatusFilePath));
}
}
}
else
{
this.statistics.RecordBackgroundStatusScanError();
this.context.Tracer.RelatedInfo("GitStatusCache.TryRebuildStatusCache: Error generating status: {0}", statusResult.Errors);
}
return(rebuildSucceeded);
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public override void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitIds().Single();
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);
// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
// Since pack indexer is the last to finish before checkout finishes, it should propagate completion.
// This prevents availableObjects from completing before packIndexer can push its objects through this link.
checkout.AvailableBlobShas.CompleteAdding();
checkout.WaitForCompletion();
this.HasFailures |= checkout.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
// Update the refspec before setting the upstream or git will complain the remote branch doesn't exist
this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational))
{
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch);
if (result.HasErrors)
{
activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors);
this.HasFailures = true;
}
}
}
// Update the index
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational))
{
// The first bit of core.gvfs is set if index signing is turned off.
const uint CoreGvfsUnsignedIndexFlag = 1;
GitProcess git = new GitProcess(this.Enlistment);
// Only update the index if index signing is turned off.
// The first bit of core.gvfs is set if signing is turned off.
GitProcess.Result configCoreGvfs = git.GetFromConfig("core.gvfs");
uint valueCoreGvfs;
// No errors getting the configuration *and it is either "true" or numeric with the right bit set.
bool indexSigningIsTurnedOff =
!configCoreGvfs.HasErrors &&
!string.IsNullOrEmpty(configCoreGvfs.Output) &&
(configCoreGvfs.Output.Equals("true", StringComparison.OrdinalIgnoreCase) ||
(uint.TryParse(configCoreGvfs.Output, out valueCoreGvfs) &&
((valueCoreGvfs & CoreGvfsUnsignedIndexFlag) == CoreGvfsUnsignedIndexFlag)));
// Create the index object now so it can track the current index
Index index = indexSigningIsTurnedOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null;
GitProcess.Result result;
using (ITracer updateIndexActivity = this.Tracer.StartActivity("ReadTree", EventLevel.Informational))
{
result = git.ReadTree("HEAD");
}
if (result.HasErrors)
{
activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors);
this.HasFailures = true;
}
else
{
if (index != null)
{
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;
// Update
index.UpdateFileSizesAndTimes(allowIndexMetadataUpdateFromWorkingTree);
}
else
{
activity.RelatedEvent(EventLevel.Informational, "Core.gvfs is not set, so not updating index entries with file times and sizes.", null);
}
}
}
}
}