private Result TryInitRepo(ITracer tracer, GitRefs refs, Enlistment enlistmentToInit)
{
string repoPath = enlistmentToInit.LocalStorageRoot;
GitProcess.Result initResult = GitProcess.Init(enlistmentToInit);
if (initResult.ExitCodeIsFailure)
{
string error = string.Format("Could not init repo at to {0}: {1}", repoPath, initResult.Errors);
tracer.RelatedError(error);
return(new Result(error));
}
GitProcess.Result remoteAddResult = new GitProcess(enlistmentToInit).RemoteAdd("origin", enlistmentToInit.RepoUrl);
if (remoteAddResult.ExitCodeIsFailure)
{
string error = string.Format("Could not add remote to {0}: {1}", repoPath, remoteAddResult.Errors);
tracer.RelatedError(error);
return(new Result(error));
}
File.WriteAllText(
Path.Combine(repoPath, GVFSConstants.DotGit.PackedRefs),
refs.ToPackedRefs());
return(new Result(true));
}
private CloneVerb.Result TryInitRepo(GitRefs refs, Enlistment enlistmentToInit)
{
string repoPath = enlistmentToInit.WorkingDirectoryRoot;
GitProcess.Result initResult = GitProcess.Init(enlistmentToInit);
if (initResult.HasErrors)
{
string error = string.Format("Could not init repo at to {0}: {1}", repoPath, initResult.Errors);
this.tracer.RelatedError(error);
return(new CloneVerb.Result(error));
}
GitProcess.Result remoteAddResult = new GitProcess(enlistmentToInit).RemoteAdd("origin", enlistmentToInit.RepoUrl);
if (remoteAddResult.HasErrors)
{
string error = string.Format("Could not add remote to {0}: {1}", repoPath, remoteAddResult.Errors);
this.tracer.RelatedError(error);
return(new CloneVerb.Result(error));
}
File.WriteAllText(
Path.Combine(repoPath, GVFSConstants.DotGit.PackedRefs),
refs.ToPackedRefs());
return(new CloneVerb.Result(true));
}
private Result TryClone(JsonEtwTracer tracer, GVFSEnlistment enlistment)
{
this.CheckGitVersion(enlistment);
Result pipeResult;
using (NamedPipeServer pipeServer = this.StartNamedPipe(enlistment, out pipeResult))
{
if (!pipeResult.Success)
{
return(pipeResult);
}
HttpGitObjects httpGitObjects = new HttpGitObjects(tracer, enlistment, Environment.ProcessorCount);
this.ValidateGVFSVersion(enlistment, httpGitObjects, tracer);
GitRefs refs = httpGitObjects.QueryInfoRefs(this.SingleBranch ? this.Branch : null);
if (refs == null)
{
return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl)));
}
if (this.Branch == null)
{
this.Branch = refs.GetDefaultBranch();
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata);
}
else
{
if (!refs.HasBranch(this.Branch))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata);
string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch);
return(new Result(errorMessage));
}
}
if (!enlistment.TryCreateEnlistmentFolders())
{
string error = "Could not create enlistment directory";
tracer.RelatedError(error);
return(new Result(error));
}
this.CheckAntiVirusExclusion(enlistment);
CloneHelper cloneHelper = new CloneHelper(tracer, enlistment, httpGitObjects);
return(cloneHelper.CreateClone(refs, this.Branch));
}
}
private Result CreateScalarDirctories(string resolvedLocalCacheRoot)
{
this.refs = this.objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null);
if (this.refs == null)
{
return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(this.enlistment.RepoUrl)));
}
if (this.Branch == null)
{
this.Branch = this.refs.GetDefaultBranch();
EventMetadata metadata = this.CreateEventMetadata();
metadata.Add("Branch", this.Branch);
this.tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata);
}
else
{
if (!this.refs.HasBranch(this.Branch))
{
EventMetadata metadata = this.CreateEventMetadata();
metadata.Add("Branch", this.Branch);
this.tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata);
string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch);
return(new Result(errorMessage));
}
}
if (!this.enlistment.TryCreateEnlistmentFolders())
{
string error = "Could not create enlistment directory";
this.tracer.RelatedError(error);
return(new Result(error));
}
if (!ScalarPlatform.Instance.FileSystem.IsFileSystemSupported(this.enlistment.EnlistmentRoot, out string fsError))
{
string error = $"FileSystem unsupported: {fsError}";
this.tracer.RelatedError(error);
return(new Result(error));
}
string localCacheError;
if (!this.TryDetermineLocalCacheAndInitializePaths(resolvedLocalCacheRoot, out localCacheError))
{
this.tracer.RelatedError(localCacheError);
return(new Result(localCacheError));
}
Directory.CreateDirectory(this.enlistment.GitObjectsRoot);
Directory.CreateDirectory(this.enlistment.GitPackRoot);
return(new Result(true));
}
private Result TryClone(JsonEtwTracer tracer, GVFSEnlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig)
{
Result pipeResult;
using (NamedPipeServer pipeServer = this.StartNamedPipe(tracer, enlistment, out pipeResult))
{
if (!pipeResult.Success)
{
return(pipeResult);
}
using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig))
{
GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null);
if (refs == null)
{
return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl)));
}
if (this.Branch == null)
{
this.Branch = refs.GetDefaultBranch();
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata);
}
else
{
if (!refs.HasBranch(this.Branch))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata);
string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch);
return(new Result(errorMessage));
}
}
if (!enlistment.TryCreateEnlistmentFolders())
{
string error = "Could not create enlistment directory";
tracer.RelatedError(error);
return(new Result(error));
}
CloneHelper cloneHelper = new CloneHelper(tracer, enlistment, objectRequestor);
return(cloneHelper.CreateClone(refs, this.Branch));
}
}
}
/// <summary>
/// * Updates local branch (N/A for checkout to detached HEAD)
/// * Updates HEAD
/// * Calls base to update shallow file and remote branch.
/// </summary>
protected override void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs)
{
if (isBranch)
{
KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single();
string remoteBranch = remoteRef.Key;
string fullLocalBranchName = branchOrCommit.StartsWith(RefsHeadsGitPath) ? branchOrCommit : (RefsHeadsGitPath + branchOrCommit);
this.HasFailures |= !this.UpdateRef(this.Tracer, fullLocalBranchName, remoteRef.Value);
this.HasFailures |= !this.UpdateRef(this.Tracer, "HEAD", fullLocalBranchName);
}
else
{
this.HasFailures |= !this.UpdateRef(this.Tracer, "HEAD", branchOrCommit);
}
base.UpdateRefs(branchOrCommit, isBranch, refs);
}
/// <summary>
/// * Updates any remote branch (N/A for fetch of detached commit)
/// * Updates shallow file
/// </summary>
protected virtual void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs)
{
string commitSha = null;
if (isBranch)
{
KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single();
string remoteBranch = remoteRef.Key;
commitSha = remoteRef.Value;
this.HasFailures |= !this.UpdateRef(this.Tracer, remoteBranch, commitSha);
}
else
{
commitSha = branchOrCommit;
}
// Update shallow file to ensure this is a valid shallow repo
AppendToNewlineSeparatedFile(Path.Combine(this.Enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Shallow), commitSha);
}
/// <summary>
/// * Updates local branch (N/A for checkout to detached HEAD)
/// * Updates HEAD
/// * Calls base to update shallow file and remote branch.
/// </summary>
protected override void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs)
{
UpdateRefsHelper refHelper = new UpdateRefsHelper(this.Enlistment);
if (isBranch)
{
KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single();
string remoteBranch = remoteRef.Key;
string fullLocalBranchName = branchOrCommit.StartsWith("refs/heads/") ? branchOrCommit : ("refs/heads/" + branchOrCommit);
this.HasFailures |= !refHelper.UpdateRef(this.Tracer, fullLocalBranchName, remoteRef.Value);
this.HasFailures |= !refHelper.UpdateRef(this.Tracer, "HEAD", fullLocalBranchName);
}
else
{
this.HasFailures |= !refHelper.UpdateRef(this.Tracer, "HEAD", branchOrCommit);
}
base.UpdateRefs(branchOrCommit, isBranch, refs);
}
/// <summary>
/// * Updates any remote branch (N/A for fetch of detached commit)
/// * Updates shallow file
/// </summary>
protected virtual void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs)
{
UpdateRefsHelper refHelper = new UpdateRefsHelper(this.Enlistment);
string commitSha = null;
if (isBranch)
{
KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single();
string remoteBranch = remoteRef.Key;
commitSha = remoteRef.Value;
this.HasFailures |= !refHelper.UpdateRef(this.Tracer, remoteBranch, commitSha);
}
else
{
commitSha = branchOrCommit;
}
// Update shallow file to ensure this is a valid shallow repo
File.AppendAllText(Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow), commitSha + "\n");
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public override void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitIds().Single();
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);
// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
// Since pack indexer is the last to finish before checkout finishes, it should propagate completion.
// This prevents availableObjects from completing before packIndexer can push its objects through this link.
checkout.AvailableBlobShas.CompleteAdding();
checkout.WaitForCompletion();
this.HasFailures |= checkout.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
// Update the refspec before setting the upstream or git will complain the remote branch doesn't exist
this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational))
{
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch);
if (result.HasErrors)
{
activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors);
this.HasFailures = true;
}
}
}
bool indexSigningIsOff = this.GetIsIndexSigningOff();
// Update the index
EventMetadata updateIndexMetadata = new EventMetadata();
updateIndexMetadata.Add("IndexSigningIsOff", indexSigningIsOff);
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata))
{
// Create the index object now so it can track the current index
Index index = indexSigningIsOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null;
GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, !indexSigningIsOff);
indexGen.CreateFromHeadTree();
this.HasFailures = indexGen.HasFailures;
if (!indexGen.HasFailures && index != null)
{
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;
index.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree);
}
}
}
}
private Result CreateClone(
ITracer tracer,
GVFSEnlistment enlistment,
GitObjectsHttpRequestor objectRequestor,
GitRefs refs,
string branch)
{
Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string errorMessage;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage))
{
return(new Result("Error configuring alternate: " + errorMessage));
}
GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem);
GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment);
GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor);
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage))
{
return(new Result(errorMessage));
}
if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) ||
!GVFSVerb.TrySetOptionalGitConfigSettings(enlistment))
{
return(new Result("Unable to configure git repo"));
}
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage))
{
return(new Result("Unable to configure cache server: " + errorMessage));
}
GitProcess git = new GitProcess(enlistment);
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.HasErrors)
{
return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head),
"ref: refs/heads/" + branch);
File.AppendAllText(
Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath),
GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n");
if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage))
{
return(new Result(errorMessage));
}
this.CreateGitScript(enlistment);
string installHooksError;
if (!HooksInstaller.InstallHooks(context, out installHooksError))
{
tracer.RelatedError(installHooksError);
return(new Result(installHooksError));
}
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.HasErrors)
{
string[] errorLines = forceCheckoutResult.Errors.Split('\n');
StringBuilder checkoutErrors = new StringBuilder();
foreach (string gitError in errorLines)
{
if (IsForceCheckoutErrorCloneFailure(gitError))
{
checkoutErrors.AppendLine(gitError);
}
}
if (checkoutErrors.Length > 0)
{
string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString();
tracer.RelatedError(error);
return(new Result(error));
}
}
if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage))
{
tracer.RelatedError(errorMessage);
return(new Result(errorMessage));
}
try
{
RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment);
this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment);
}
catch (Exception e)
{
tracer.RelatedError(e.ToString());
return(new Result(e.Message));
}
finally
{
RepoMetadata.Shutdown();
}
// Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed
string prepFileSystemError;
if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError))
{
tracer.RelatedError(prepFileSystemError);
return(new Result(prepFileSystemError));
}
return(new Result(true));
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public override void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitIds().Single();
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);
// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
// Since pack indexer is the last to finish before checkout finishes, it should propagate completion.
// This prevents availableObjects from completing before packIndexer can push its objects through this link.
checkout.AvailableBlobShas.CompleteAdding();
checkout.WaitForCompletion();
this.HasFailures |= checkout.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
// Update the refspec before setting the upstream or git will complain the remote branch doesn't exist
this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational))
{
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch);
if (result.HasErrors)
{
activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors);
this.HasFailures = true;
}
}
}
// Update the index
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational))
{
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.ReadTree("HEAD");
if (result.HasErrors)
{
activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors);
this.HasFailures = true;
}
}
}
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public override void Prefetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitId(branchOrCommit);
}
else
{
commitToFetch = branchOrCommit;
}
using (new IndexLock(this.Enlistment.EnlistmentRoot, this.Tracer))
{
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper
// Checkout diff output => FindBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutStage checkout = new CheckoutStage(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, this.forceCheckout);
FindBlobsStage blobFinder = new FindBlobsStage(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadStage downloader = new BatchObjectDownloadStage(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackStage packIndexer = new IndexPackStage(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);
// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
// Since pack indexer is the last to finish before checkout finishes, it should propagate completion.
// This prevents availableObjects from completing before packIndexer can push its objects through this link.
checkout.AvailableBlobShas.CompleteAdding();
checkout.WaitForCompletion();
this.HasFailures |= checkout.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
bool shouldSignIndex = !this.GetIsIndexSigningOff();
// Update the index - note that this will take some time
EventMetadata updateIndexMetadata = new EventMetadata();
updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex);
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata))
{
Index sourceIndex = this.GetSourceIndex();
GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex);
indexGen.CreateFromRef(commitToFetch, indexVersion: 2, isFinal: false);
this.HasFailures |= indexGen.HasFailures;
if (!indexGen.HasFailures)
{
Index newIndex = new Index(
this.Enlistment.EnlistmentRoot,
this.Tracer,
indexGen.TemporaryIndexFilePath,
readOnly: false);
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;
newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex);
// All the slow stuff is over, so we will now move the final index into .git\index, shortly followed by
// updating the ref files and releasing index.lock.
string indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName);
this.Tracer.RelatedEvent(EventLevel.Informational, "MoveUpdatedIndexToFinalLocation", new EventMetadata()
{
{ "UpdatedIndex", indexGen.TemporaryIndexFilePath }, { "Index", indexPath }
});
File.Delete(indexPath);
File.Move(indexGen.TemporaryIndexFilePath, indexPath);
newIndex.WriteFastFetchIndexVersionMarker();
}
}
if (!this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
// Update the refspec before setting the upstream or git will complain the remote branch doesn't exist
this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational))
{
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch);
if (result.ExitCodeIsFailure)
{
activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors);
this.HasFailures = true;
}
}
}
}
}
}
}
private Result TryClone(JsonEtwTracer tracer, GVFSEnlistment enlistment)
{
this.CheckVolumeSupportsDeleteNotifications(tracer, enlistment);
this.CheckGitVersion(enlistment);
Result pipeResult;
using (NamedPipeServer pipeServer = this.StartNamedPipe(enlistment, out pipeResult))
{
if (!pipeResult.Success)
{
return(pipeResult);
}
using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment))
{
GVFSConfig config = configRequestor.QueryGVFSConfig();
this.ValidateGVFSVersion(enlistment, config, tracer);
}
using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, Environment.ProcessorCount))
{
GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null);
if (refs == null)
{
return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl)));
}
if (this.Branch == null)
{
this.Branch = refs.GetDefaultBranch();
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata);
}
else
{
if (!refs.HasBranch(this.Branch))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata);
string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch);
return(new Result(errorMessage));
}
}
if (!enlistment.TryCreateEnlistmentFolders())
{
string error = "Could not create enlistment directory";
tracer.RelatedError(error);
return(new Result(error));
}
// Only check Defender exclusions if not mounting, otherwise let mount take care of it.
if (this.NoMount)
{
this.CheckAntiVirusExclusion(enlistment);
}
CloneHelper cloneHelper = new CloneHelper(tracer, enlistment, objectRequestor);
return(cloneHelper.CreateClone(refs, this.Branch));
}
}
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public virtual void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitIds().Single();
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Dummy output queue since we don't need to checkout available blobs
BlockingCollection <string> availableBlobs = new BlockingCollection <string>();
// Configure pipeline
// LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack
LsTreeHelper blobEnumerator = new LsTreeHelper(this.PathWhitelist, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.BlobIdOutput, availableBlobs, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects);
blobFinder.Start();
downloader.Start();
this.HasFailures |= !blobEnumerator.EnqueueAllBlobs(commitToFetch);
// If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing.
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Index regardless of failures, it'll shorten the next fetch.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
if (!this.SkipConfigUpdate)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
}
}
}
private Result CreateClone(
ITracer tracer,
GVFSEnlistment enlistment,
GitObjectsHttpRequestor objectRequestor,
GitRefs refs,
string branch)
{
Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string errorMessage;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage))
{
return(new Result("Error configuring alternate: " + errorMessage));
}
GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem);
GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment);
GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor);
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage))
{
return(new Result(errorMessage));
}
if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) ||
!GVFSVerb.TrySetOptionalGitConfigSettings(enlistment))
{
return(new Result("Unable to configure git repo"));
}
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage))
{
return(new Result("Unable to configure cache server: " + errorMessage));
}
GitProcess git = new GitProcess(enlistment);
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.ExitCodeIsFailure)
{
return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head),
"ref: refs/heads/" + branch);
if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage))
{
return(new Result(errorMessage));
}
this.CreateGitScript(enlistment);
string installHooksError;
if (!HooksInstaller.InstallHooks(context, out installHooksError))
{
tracer.RelatedError(installHooksError);
return(new Result(installHooksError));
}
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0)
{
// It is possible to have the above TryDownloadCommit() fail because we
// already have the commit and root tree we intend to check out, but
// don't have a tree further down the working directory. If we fail
// checkout here, its' because we don't have these trees and the
// read-object hook is not available yet. Force downloading the commit
// again and retry the checkout.
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage,
checkLocalObjectCache: false))
{
return(new Result(errorMessage));
}
forceCheckoutResult = git.ForceCheckout(branch);
}
if (forceCheckoutResult.ExitCodeIsFailure)
{
string[] errorLines = forceCheckoutResult.Errors.Split('\n');
StringBuilder checkoutErrors = new StringBuilder();
foreach (string gitError in errorLines)
{
if (IsForceCheckoutErrorCloneFailure(gitError))
{
checkoutErrors.AppendLine(gitError);
}
}
if (checkoutErrors.Length > 0)
{
string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString();
tracer.RelatedError(error);
return(new Result(error));
}
}
if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage))
{
tracer.RelatedError(errorMessage);
return(new Result(errorMessage));
}
try
{
RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment);
this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment);
}
catch (Exception e)
{
tracer.RelatedError(e.ToString());
return(new Result(e.Message));
}
finally
{
RepoMetadata.Shutdown();
}
// Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed
Exception exception;
string prepFileSystemError;
if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception))
{
EventMetadata metadata = new EventMetadata();
metadata.Add(nameof(prepFileSystemError), prepFileSystemError);
if (exception != null)
{
metadata.Add("Exception", exception.ToString());
}
tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed");
return(new Result(prepFileSystemError));
}
return(new Result(true));
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public virtual void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitId(branchOrCommit);
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Dummy output queue since we don't need to checkout available blobs
BlockingCollection <string> availableBlobs = new BlockingCollection <string>();
// Configure pipeline
// LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack
string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow);
string previousCommit = null;
// Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check
DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.PathWhitelist);
if (File.Exists(shallowFile))
{
previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault();
if (string.IsNullOrWhiteSpace(previousCommit))
{
this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs.");
this.HasFailures = true;
return;
}
}
blobEnumerator.PerformDiff(previousCommit, commitToFetch);
this.HasFailures |= blobEnumerator.HasFailures;
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects);
blobFinder.Start();
downloader.Start();
// If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing.
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Index regardless of failures, it'll shorten the next fetch.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
}
}
}
public void FastFetchWithStats(
string branchOrCommit,
bool isBranch,
bool readFilesAfterDownload,
out int matchedBlobCount,
out int downloadedBlobCount,
out int readFileCount)
{
matchedBlobCount = 0;
downloadedBlobCount = 0;
readFileCount = 0;
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitId(branchOrCommit);
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack
string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow);
string previousCommit = null;
// Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check
DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList);
if (File.Exists(shallowFile))
{
previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault();
if (string.IsNullOrWhiteSpace(previousCommit))
{
this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs.");
this.HasFailures = true;
return;
}
}
ThreadStart performDiff = () =>
{
blobEnumerator.PerformDiff(previousCommit, commitToFetch);
this.HasFailures |= blobEnumerator.HasFailures;
};
if (readFilesAfterDownload)
{
// Call synchronously to ensure that blobEnumerator.FileAddOperations
// is completely populated when ReadFilesJob starts
performDiff();
}
else
{
new Thread(performDiff).Start();
}
BlockingCollection <string> availableBlobs = new BlockingCollection <string>();
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects);
ReadFilesJob readFiles = new ReadFilesJob(Environment.ProcessorCount * 2, blobEnumerator.FileAddOperations, availableBlobs, this.Tracer);
blobFinder.Start();
downloader.Start();
if (readFilesAfterDownload)
{
readFiles.Start();
}
// If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing.
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
availableBlobs.CompleteAdding();
if (readFilesAfterDownload)
{
readFiles.WaitForCompletion();
this.HasFailures |= readFiles.HasFailures;
}
matchedBlobCount = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount;
downloadedBlobCount = blobFinder.MissingBlobCount;
readFileCount = readFiles.ReadFileCount;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
}
}
}
private Result CreateClone(
ITracer tracer,
GSDEnlistment enlistment,
GitObjectsHttpRequestor objectRequestor,
GitRefs refs,
string branch)
{
Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string errorMessage;
if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage))
{
return(new Result("Error configuring alternate: " + errorMessage));
}
GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem);
GSDContext context = new GSDContext(tracer, fileSystem, gitRepo, enlistment);
GSDGitObjects gitObjects = new GSDGitObjects(context, objectRequestor);
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage))
{
return(new Result(errorMessage));
}
if (!GSDVerb.TrySetRequiredGitConfigSettings(enlistment) ||
!GSDVerb.TrySetOptionalGitConfigSettings(enlistment))
{
return(new Result("Unable to configure git repo"));
}
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage))
{
return(new Result("Unable to configure cache server: " + errorMessage));
}
GitProcess git = new GitProcess(enlistment);
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.ExitCodeIsFailure)
{
return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(enlistment.WorkingDirectoryBackingRoot, GSDConstants.DotGit.Head),
"ref: refs/heads/" + branch);
if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage))
{
return(new Result(errorMessage));
}
this.CreateGitScript(enlistment);
string installHooksError;
if (!HooksInstaller.InstallHooks(context, out installHooksError))
{
tracer.RelatedError(installHooksError);
return(new Result(installHooksError));
}
// TODO: Move this to be after the mount?
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0)
{
// It is possible to have the above TryDownloadCommit() fail because we
// already have the commit and root tree we intend to check out, but
// don't have a tree further down the working directory. If we fail
// checkout here, its' because we don't have these trees and the
// read-object hook is not available yet. Force downloading the commit
// again and retry the checkout.
if (!this.TryDownloadCommit(
refs.GetTipCommitId(branch),
enlistment,
objectRequestor,
gitObjects,
gitRepo,
out errorMessage,
checkLocalObjectCache: false))
{
return(new Result(errorMessage));
}
forceCheckoutResult = git.ForceCheckout(branch);
}
if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGSDRoot, out errorMessage))
{
tracer.RelatedError(errorMessage);
return(new Result(errorMessage));
}
try
{
RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment);
this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment);
}
catch (Exception e)
{
tracer.RelatedError(e.ToString());
return(new Result(e.Message));
}
finally
{
RepoMetadata.Shutdown();
}
return(new Result(true));
}
private Result TryClone(
JsonTracer tracer,
GSDEnlistment enlistment,
CacheServerInfo cacheServer,
RetryConfig retryConfig,
ServerGSDConfig serverGSDConfig,
string resolvedLocalCacheRoot)
{
Result pipeResult;
using (NamedPipeServer pipeServer = this.StartNamedPipe(tracer, enlistment, out pipeResult))
{
if (!pipeResult.Success)
{
return(pipeResult);
}
using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig))
{
GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null);
if (refs == null)
{
return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl)));
}
if (this.Branch == null)
{
this.Branch = refs.GetDefaultBranch();
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata);
}
else
{
if (!refs.HasBranch(this.Branch))
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Branch", this.Branch);
tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata);
string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch);
return(new Result(errorMessage));
}
}
if (!enlistment.TryCreateEnlistmentFolders())
{
string error = "Could not create enlistment directory";
tracer.RelatedError(error);
return(new Result(error));
}
string localCacheError;
if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, serverGSDConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError))
{
tracer.RelatedError(localCacheError);
return(new Result(localCacheError));
}
Directory.CreateDirectory(enlistment.GitObjectsRoot);
Directory.CreateDirectory(enlistment.GitPackRoot);
Directory.CreateDirectory(enlistment.BlobSizesRoot);
return(this.CreateClone(tracer, enlistment, objectRequestor, refs, this.Branch));
}
}
}
public void PrefetchWithStats(
string branchOrCommit,
bool isBranch,
bool hydrateFilesAfterDownload,
out int matchedBlobCount,
out int downloadedBlobCount,
out int hydratedFileCount)
{
matchedBlobCount = 0;
downloadedBlobCount = 0;
hydratedFileCount = 0;
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitId(branchOrCommit);
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// For FastFetch only, examine the shallow file to determine the previous commit that had been fetched
string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Shallow);
string previousCommit = null;
// Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check.
if (File.Exists(shallowFile))
{
previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault();
if (string.IsNullOrWhiteSpace(previousCommit))
{
this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs.");
this.HasFailures = true;
return;
}
}
BlockingCollection <string> availableBlobs = new BlockingCollection <string>();
////
// First create the pipeline
//
// diff ---> blobFinder ---> downloader ---> packIndexer
// | | | |
// ------------------------------------------------------> fileHydrator
////
// diff
// Inputs:
// * files/folders
// * commit id
// Outputs:
// * RequiredBlobs (property): Blob ids required to satisfy desired paths
// * FileAddOperations (property): Repo-relative paths corresponding to those blob ids
DiffHelper diff = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList, includeSymLinks: false);
// blobFinder
// Inputs:
// * requiredBlobs (in param): Blob ids from output of `diff`
// Outputs:
// * availableBlobs (out param): Locally available blob ids (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available)
// * MissingBlobs (property): Blob ids that are missing and need to be downloaded
// * AvailableBlobs (property): Same as availableBlobs
FindBlobsStage blobFinder = new FindBlobsStage(this.SearchThreadCount, diff.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment);
// downloader
// Inputs:
// * missingBlobs (in param): Blob ids from output of `blobFinder`
// Outputs:
// * availableBlobs (out param): Loose objects that have completed downloading (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available)
// * AvailableObjects (property): Same as availableBlobs
// * AvailablePacks (property): Packfiles that have completed downloading
BatchObjectDownloadStage downloader = new BatchObjectDownloadStage(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
// packIndexer
// Inputs:
// * availablePacks (in param): Packfiles that have completed downloading from output of `downloader`
// Outputs:
// * availableBlobs (out param): Blobs that have completed downloading and indexing (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available)
IndexPackStage packIndexer = new IndexPackStage(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects);
// fileHydrator
// Inputs:
// * workingDirectoryRoot (in param): the root of the working directory where hydration takes place
// * blobIdsToPaths (in param): paths of all blob ids that need to be hydrated from output of `diff`
// * availableBlobs (in param): blobs id that are available locally, from whatever source
// Outputs:
// * Hydrated files on disk.
HydrateFilesStage fileHydrator = new HydrateFilesStage(Environment.ProcessorCount * 2, this.Enlistment.WorkingDirectoryRoot, diff.FileAddOperations, availableBlobs, this.Tracer);
// All the stages of the pipeline are created and wired up, now kick them off in the proper sequence
ThreadStart performDiff = () =>
{
diff.PerformDiff(previousCommit, commitToFetch);
this.HasFailures |= diff.HasFailures;
};
if (hydrateFilesAfterDownload)
{
// Call synchronously to ensure that diff.FileAddOperations
// is completely populated when fileHydrator starts
performDiff();
}
else
{
new Thread(performDiff).Start();
}
blobFinder.Start();
downloader.Start();
if (hydrateFilesAfterDownload)
{
fileHydrator.Start();
}
// If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing.
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
availableBlobs.CompleteAdding();
if (hydrateFilesAfterDownload)
{
fileHydrator.WaitForCompletion();
this.HasFailures |= fileHydrator.HasFailures;
}
matchedBlobCount = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount;
downloadedBlobCount = blobFinder.MissingBlobCount;
hydratedFileCount = fileHydrator.ReadFileCount;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
}
}
if (!this.HasFailures)
{
this.SavePrefetchArgs(commitToFetch, hydrateFilesAfterDownload);
}
}
protected bool UpdateRefSpec(ITracer tracer, Enlistment enlistment, string branchOrCommit, GitRefs refs)
{
using (ITracer activity = tracer.StartActivity("UpdateRefSpec", EventLevel.Informational, Keywords.Telemetry, metadata: null))
{
const string OriginRefMapSettingName = "remote.origin.fetch";
// We must update the refspec to get proper "git pull" functionality.
string localBranch = branchOrCommit.StartsWith(RefsHeadsGitPath) ? branchOrCommit : (RefsHeadsGitPath + branchOrCommit);
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
string refSpec = "+" + localBranch + ":" + remoteBranch;
GitProcess git = new GitProcess(enlistment);
// Replace all ref-specs this
// * ensures the default refspec (remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*) is removed which avoids some "git fetch/pull" failures
// * gives added "git fetch" performance since git will only fetch the branch provided in the refspec.
GitProcess.Result setResult = git.SetInLocalConfig(OriginRefMapSettingName, refSpec, replaceAll: true);
if (setResult.ExitCodeIsFailure)
{
activity.RelatedError("Could not update ref spec to {0}: {1}", refSpec, setResult.Errors);
return(false);
}
}
return(true);
}
public CloneVerb.Result CreateClone(GitRefs refs, string branch)
{
GitObjects gitObjects = new GitObjects(this.tracer, this.enlistment, this.objectRequestor);
CloneVerb.Result initRepoResult = this.TryInitRepo(refs, this.enlistment);
if (!initRepoResult.Success)
{
return(initRepoResult);
}
string errorMessage;
if (!this.enlistment.TryConfigureAlternate(out errorMessage))
{
return(new CloneVerb.Result("Error configuring alternate: " + errorMessage));
}
if (!gitObjects.TryDownloadAndSaveCommit(refs.GetTipCommitId(branch), commitDepth: 2))
{
return(new CloneVerb.Result("Could not download tip commits from: " + Uri.EscapeUriString(this.objectRequestor.CacheServer.ObjectsEndpointUrl)));
}
GitProcess git = new GitProcess(this.enlistment);
if (!this.SetConfigSettings(git, this.objectRequestor.CacheServer))
{
return(new CloneVerb.Result("Unable to configure git repo"));
}
string originBranchName = "origin/" + branch;
GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName);
if (createBranchResult.HasErrors)
{
return(new CloneVerb.Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output));
}
File.WriteAllText(
Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head),
"ref: refs/heads/" + branch);
File.AppendAllText(
Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath),
GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n");
CloneVerb.Result hydrateResult = this.HydrateRootGitAttributes(gitObjects, branch);
if (!hydrateResult.Success)
{
return(hydrateResult);
}
this.CreateGitScript();
GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch);
if (forceCheckoutResult.HasErrors)
{
string[] errorLines = forceCheckoutResult.Errors.Split('\n');
StringBuilder checkoutErrors = new StringBuilder();
foreach (string gitError in errorLines)
{
if (IsForceCheckoutErrorCloneFailure(gitError))
{
checkoutErrors.AppendLine(gitError);
}
}
if (checkoutErrors.Length > 0)
{
string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString();
this.tracer.RelatedError(error);
return(new CloneVerb.Result(error));
}
}
GitProcess.Result updateIndexresult = git.UpdateIndexVersion4();
if (updateIndexresult.HasErrors)
{
string error = "Could not update index, error: " + updateIndexresult.Errors;
this.tracer.RelatedError(error);
return(new CloneVerb.Result(error));
}
string installHooksError;
if (!HooksInstaller.InstallHooks(this.enlistment, out installHooksError))
{
this.tracer.RelatedError(installHooksError);
return(new CloneVerb.Result(installHooksError));
}
using (RepoMetadata repoMetadata = new RepoMetadata(this.enlistment.DotGVFSRoot))
{
repoMetadata.SaveCurrentDiskLayoutVersion();
}
// Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed
string prepGVFltError;
if (!GVFltCallbacks.TryPrepareFolderForGVFltCallbacks(this.enlistment.WorkingDirectoryRoot, out prepGVFltError))
{
this.tracer.RelatedError(prepGVFltError);
return(new CloneVerb.Result(prepGVFltError));
}
return(new CloneVerb.Result(true));
}
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param>
public override void FastFetch(string branchOrCommit, bool isBranch)
{
if (string.IsNullOrWhiteSpace(branchOrCommit))
{
throw new FetchException("Must specify branch or commit to fetch");
}
GitRefs refs = null;
string commitToFetch;
if (isBranch)
{
refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
}
else if (refs.Count == 0)
{
throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl);
}
commitToFetch = refs.GetTipCommitIds().Single();
}
else
{
commitToFetch = branchOrCommit;
}
this.DownloadMissingCommit(commitToFetch, this.GitObjects);
// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);
// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();
blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;
// Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping.
packIndexer.Start();
downloader.WaitForCompletion();
this.HasFailures |= downloader.HasFailures;
packIndexer.WaitForCompletion();
this.HasFailures |= packIndexer.HasFailures;
// Since pack indexer is the last to finish before checkout finishes, it should propagate completion.
// This prevents availableObjects from completing before packIndexer can push its objects through this link.
checkout.AvailableBlobShas.CompleteAdding();
checkout.WaitForCompletion();
this.HasFailures |= checkout.HasFailures;
if (!this.SkipConfigUpdate && !this.HasFailures)
{
this.UpdateRefs(branchOrCommit, isBranch, refs);
if (isBranch)
{
// Update the refspec before setting the upstream or git will complain the remote branch doesn't exist
this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs);
using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational))
{
string remoteBranch = refs.GetBranchRefPairs().Single().Key;
GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch);
if (result.HasErrors)
{
activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors);
this.HasFailures = true;
}
}
}
// Update the index
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational))
{
// The first bit of core.gvfs is set if index signing is turned off.
const uint CoreGvfsUnsignedIndexFlag = 1;
GitProcess git = new GitProcess(this.Enlistment);
// Only update the index if index signing is turned off.
// The first bit of core.gvfs is set if signing is turned off.
GitProcess.Result configCoreGvfs = git.GetFromConfig("core.gvfs");
uint valueCoreGvfs;
// No errors getting the configuration *and it is either "true" or numeric with the right bit set.
bool indexSigningIsTurnedOff =
!configCoreGvfs.HasErrors &&
!string.IsNullOrEmpty(configCoreGvfs.Output) &&
(configCoreGvfs.Output.Equals("true", StringComparison.OrdinalIgnoreCase) ||
(uint.TryParse(configCoreGvfs.Output, out valueCoreGvfs) &&
((valueCoreGvfs & CoreGvfsUnsignedIndexFlag) == CoreGvfsUnsignedIndexFlag)));
// Create the index object now so it can track the current index
Index index = indexSigningIsTurnedOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null;
GitProcess.Result result;
using (ITracer updateIndexActivity = this.Tracer.StartActivity("ReadTree", EventLevel.Informational))
{
result = git.ReadTree("HEAD");
}
if (result.HasErrors)
{
activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors);
this.HasFailures = true;
}
else
{
if (index != null)
{
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;
// Update
index.UpdateFileSizesAndTimes(allowIndexMetadataUpdateFromWorkingTree);
}
else
{
activity.RelatedEvent(EventLevel.Informational, "Core.gvfs is not set, so not updating index entries with file times and sizes.", null);
}
}
}
}
}
