private Result TryInitRepo(ITracer tracer, GitRefs refs, Enlistment enlistmentToInit) { string repoPath = enlistmentToInit.LocalStorageRoot; GitProcess.Result initResult = GitProcess.Init(enlistmentToInit); if (initResult.ExitCodeIsFailure) { string error = string.Format("Could not init repo at to {0}: {1}", repoPath, initResult.Errors); tracer.RelatedError(error); return(new Result(error)); } GitProcess.Result remoteAddResult = new GitProcess(enlistmentToInit).RemoteAdd("origin", enlistmentToInit.RepoUrl); if (remoteAddResult.ExitCodeIsFailure) { string error = string.Format("Could not add remote to {0}: {1}", repoPath, remoteAddResult.Errors); tracer.RelatedError(error); return(new Result(error)); } File.WriteAllText( Path.Combine(repoPath, GVFSConstants.DotGit.PackedRefs), refs.ToPackedRefs()); return(new Result(true)); }
private CloneVerb.Result TryInitRepo(GitRefs refs, Enlistment enlistmentToInit) { string repoPath = enlistmentToInit.WorkingDirectoryRoot; GitProcess.Result initResult = GitProcess.Init(enlistmentToInit); if (initResult.HasErrors) { string error = string.Format("Could not init repo at to {0}: {1}", repoPath, initResult.Errors); this.tracer.RelatedError(error); return(new CloneVerb.Result(error)); } GitProcess.Result remoteAddResult = new GitProcess(enlistmentToInit).RemoteAdd("origin", enlistmentToInit.RepoUrl); if (remoteAddResult.HasErrors) { string error = string.Format("Could not add remote to {0}: {1}", repoPath, remoteAddResult.Errors); this.tracer.RelatedError(error); return(new CloneVerb.Result(error)); } File.WriteAllText( Path.Combine(repoPath, GVFSConstants.DotGit.PackedRefs), refs.ToPackedRefs()); return(new CloneVerb.Result(true)); }
private Result TryClone(JsonEtwTracer tracer, GVFSEnlistment enlistment) { this.CheckGitVersion(enlistment); Result pipeResult; using (NamedPipeServer pipeServer = this.StartNamedPipe(enlistment, out pipeResult)) { if (!pipeResult.Success) { return(pipeResult); } HttpGitObjects httpGitObjects = new HttpGitObjects(tracer, enlistment, Environment.ProcessorCount); this.ValidateGVFSVersion(enlistment, httpGitObjects, tracer); GitRefs refs = httpGitObjects.QueryInfoRefs(this.SingleBranch ? this.Branch : null); if (refs == null) { return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl))); } if (this.Branch == null) { this.Branch = refs.GetDefaultBranch(); EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata); } else { if (!refs.HasBranch(this.Branch)) { EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata); string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch); return(new Result(errorMessage)); } } if (!enlistment.TryCreateEnlistmentFolders()) { string error = "Could not create enlistment directory"; tracer.RelatedError(error); return(new Result(error)); } this.CheckAntiVirusExclusion(enlistment); CloneHelper cloneHelper = new CloneHelper(tracer, enlistment, httpGitObjects); return(cloneHelper.CreateClone(refs, this.Branch)); } }
private Result CreateScalarDirctories(string resolvedLocalCacheRoot) { this.refs = this.objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null); if (this.refs == null) { return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(this.enlistment.RepoUrl))); } if (this.Branch == null) { this.Branch = this.refs.GetDefaultBranch(); EventMetadata metadata = this.CreateEventMetadata(); metadata.Add("Branch", this.Branch); this.tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata); } else { if (!this.refs.HasBranch(this.Branch)) { EventMetadata metadata = this.CreateEventMetadata(); metadata.Add("Branch", this.Branch); this.tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata); string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch); return(new Result(errorMessage)); } } if (!this.enlistment.TryCreateEnlistmentFolders()) { string error = "Could not create enlistment directory"; this.tracer.RelatedError(error); return(new Result(error)); } if (!ScalarPlatform.Instance.FileSystem.IsFileSystemSupported(this.enlistment.EnlistmentRoot, out string fsError)) { string error = $"FileSystem unsupported: {fsError}"; this.tracer.RelatedError(error); return(new Result(error)); } string localCacheError; if (!this.TryDetermineLocalCacheAndInitializePaths(resolvedLocalCacheRoot, out localCacheError)) { this.tracer.RelatedError(localCacheError); return(new Result(localCacheError)); } Directory.CreateDirectory(this.enlistment.GitObjectsRoot); Directory.CreateDirectory(this.enlistment.GitPackRoot); return(new Result(true)); }
private Result TryClone(JsonEtwTracer tracer, GVFSEnlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig) { Result pipeResult; using (NamedPipeServer pipeServer = this.StartNamedPipe(tracer, enlistment, out pipeResult)) { if (!pipeResult.Success) { return(pipeResult); } using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig)) { GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null); if (refs == null) { return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl))); } if (this.Branch == null) { this.Branch = refs.GetDefaultBranch(); EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata); } else { if (!refs.HasBranch(this.Branch)) { EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata); string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch); return(new Result(errorMessage)); } } if (!enlistment.TryCreateEnlistmentFolders()) { string error = "Could not create enlistment directory"; tracer.RelatedError(error); return(new Result(error)); } CloneHelper cloneHelper = new CloneHelper(tracer, enlistment, objectRequestor); return(cloneHelper.CreateClone(refs, this.Branch)); } } }
/// <summary> /// * Updates local branch (N/A for checkout to detached HEAD) /// * Updates HEAD /// * Calls base to update shallow file and remote branch. /// </summary> protected override void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs) { if (isBranch) { KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single(); string remoteBranch = remoteRef.Key; string fullLocalBranchName = branchOrCommit.StartsWith(RefsHeadsGitPath) ? branchOrCommit : (RefsHeadsGitPath + branchOrCommit); this.HasFailures |= !this.UpdateRef(this.Tracer, fullLocalBranchName, remoteRef.Value); this.HasFailures |= !this.UpdateRef(this.Tracer, "HEAD", fullLocalBranchName); } else { this.HasFailures |= !this.UpdateRef(this.Tracer, "HEAD", branchOrCommit); } base.UpdateRefs(branchOrCommit, isBranch, refs); }
/// <summary> /// * Updates any remote branch (N/A for fetch of detached commit) /// * Updates shallow file /// </summary> protected virtual void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs) { string commitSha = null; if (isBranch) { KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single(); string remoteBranch = remoteRef.Key; commitSha = remoteRef.Value; this.HasFailures |= !this.UpdateRef(this.Tracer, remoteBranch, commitSha); } else { commitSha = branchOrCommit; } // Update shallow file to ensure this is a valid shallow repo AppendToNewlineSeparatedFile(Path.Combine(this.Enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Shallow), commitSha); }
/// <summary> /// * Updates local branch (N/A for checkout to detached HEAD) /// * Updates HEAD /// * Calls base to update shallow file and remote branch. /// </summary> protected override void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs) { UpdateRefsHelper refHelper = new UpdateRefsHelper(this.Enlistment); if (isBranch) { KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single(); string remoteBranch = remoteRef.Key; string fullLocalBranchName = branchOrCommit.StartsWith("refs/heads/") ? branchOrCommit : ("refs/heads/" + branchOrCommit); this.HasFailures |= !refHelper.UpdateRef(this.Tracer, fullLocalBranchName, remoteRef.Value); this.HasFailures |= !refHelper.UpdateRef(this.Tracer, "HEAD", fullLocalBranchName); } else { this.HasFailures |= !refHelper.UpdateRef(this.Tracer, "HEAD", branchOrCommit); } base.UpdateRefs(branchOrCommit, isBranch, refs); }
/// <summary> /// * Updates any remote branch (N/A for fetch of detached commit) /// * Updates shallow file /// </summary> protected virtual void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs) { UpdateRefsHelper refHelper = new UpdateRefsHelper(this.Enlistment); string commitSha = null; if (isBranch) { KeyValuePair <string, string> remoteRef = refs.GetBranchRefPairs().Single(); string remoteBranch = remoteRef.Key; commitSha = remoteRef.Value; this.HasFailures |= !refHelper.UpdateRef(this.Tracer, remoteBranch, commitSha); } else { commitSha = branchOrCommit; } // Update shallow file to ensure this is a valid shallow repo File.AppendAllText(Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow), commitSha + "\n"); }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.HasErrors) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } bool indexSigningIsOff = this.GetIsIndexSigningOff(); // Update the index EventMetadata updateIndexMetadata = new EventMetadata(); updateIndexMetadata.Add("IndexSigningIsOff", indexSigningIsOff); using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) { // Create the index object now so it can track the current index Index index = indexSigningIsOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null; GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, !indexSigningIsOff); indexGen.CreateFromHeadTree(); this.HasFailures = indexGen.HasFailures; if (!indexGen.HasFailures && index != null) { // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; index.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree); } } } }
private Result CreateClone( ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, GitRefs refs, string branch) { Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment); if (!initRepoResult.Success) { return(initRepoResult); } PhysicalFileSystem fileSystem = new PhysicalFileSystem(); string errorMessage; if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage)) { return(new Result("Error configuring alternate: " + errorMessage)); } GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem); GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment); GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor); if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) || !GVFSVerb.TrySetOptionalGitConfigSettings(enlistment)) { return(new Result("Unable to configure git repo")); } CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage)) { return(new Result("Unable to configure cache server: " + errorMessage)); } GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.HasErrors) { return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head), "ref: refs/heads/" + branch); File.AppendAllText( Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n"); if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } this.CreateGitScript(enlistment); string installHooksError; if (!HooksInstaller.InstallHooks(context, out installHooksError)) { tracer.RelatedError(installHooksError); return(new Result(installHooksError)); } GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.HasErrors) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); StringBuilder checkoutErrors = new StringBuilder(); foreach (string gitError in errorLines) { if (IsForceCheckoutErrorCloneFailure(gitError)) { checkoutErrors.AppendLine(gitError); } } if (checkoutErrors.Length > 0) { string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString(); tracer.RelatedError(error); return(new Result(error)); } } if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage)) { tracer.RelatedError(errorMessage); return(new Result(errorMessage)); } try { RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment); this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment); } catch (Exception e) { tracer.RelatedError(e.ToString()); return(new Result(e.Message)); } finally { RepoMetadata.Shutdown(); } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed string prepFileSystemError; if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) { tracer.RelatedError(prepFileSystemError); return(new Result(prepFileSystemError)); } return(new Result(true)); }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.HasErrors) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } // Update the index using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational)) { GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.ReadTree("HEAD"); if (result.HasErrors) { activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors); this.HasFailures = true; } } } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void Prefetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } using (new IndexLock(this.Enlistment.EnlistmentRoot, this.Tracer)) { this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper // Checkout diff output => FindBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutStage checkout = new CheckoutStage(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, this.forceCheckout); FindBlobsStage blobFinder = new FindBlobsStage(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadStage downloader = new BatchObjectDownloadStage(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackStage packIndexer = new IndexPackStage(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { bool shouldSignIndex = !this.GetIsIndexSigningOff(); // Update the index - note that this will take some time EventMetadata updateIndexMetadata = new EventMetadata(); updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex); using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) { Index sourceIndex = this.GetSourceIndex(); GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex); indexGen.CreateFromRef(commitToFetch, indexVersion: 2, isFinal: false); this.HasFailures |= indexGen.HasFailures; if (!indexGen.HasFailures) { Index newIndex = new Index( this.Enlistment.EnlistmentRoot, this.Tracer, indexGen.TemporaryIndexFilePath, readOnly: false); // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex); // All the slow stuff is over, so we will now move the final index into .git\index, shortly followed by // updating the ref files and releasing index.lock. string indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName); this.Tracer.RelatedEvent(EventLevel.Informational, "MoveUpdatedIndexToFinalLocation", new EventMetadata() { { "UpdatedIndex", indexGen.TemporaryIndexFilePath }, { "Index", indexPath } }); File.Delete(indexPath); File.Move(indexGen.TemporaryIndexFilePath, indexPath); newIndex.WriteFastFetchIndexVersionMarker(); } } if (!this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.ExitCodeIsFailure) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } } } } }
private Result TryClone(JsonEtwTracer tracer, GVFSEnlistment enlistment) { this.CheckVolumeSupportsDeleteNotifications(tracer, enlistment); this.CheckGitVersion(enlistment); Result pipeResult; using (NamedPipeServer pipeServer = this.StartNamedPipe(enlistment, out pipeResult)) { if (!pipeResult.Success) { return(pipeResult); } using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment)) { GVFSConfig config = configRequestor.QueryGVFSConfig(); this.ValidateGVFSVersion(enlistment, config, tracer); } using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, Environment.ProcessorCount)) { GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null); if (refs == null) { return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl))); } if (this.Branch == null) { this.Branch = refs.GetDefaultBranch(); EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata); } else { if (!refs.HasBranch(this.Branch)) { EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata); string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch); return(new Result(errorMessage)); } } if (!enlistment.TryCreateEnlistmentFolders()) { string error = "Could not create enlistment directory"; tracer.RelatedError(error); return(new Result(error)); } // Only check Defender exclusions if not mounting, otherwise let mount take care of it. if (this.NoMount) { this.CheckAntiVirusExclusion(enlistment); } CloneHelper cloneHelper = new CloneHelper(tracer, enlistment, objectRequestor); return(cloneHelper.CreateClone(refs, this.Branch)); } } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public virtual void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Dummy output queue since we don't need to checkout available blobs BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack LsTreeHelper blobEnumerator = new LsTreeHelper(this.PathWhitelist, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.BlobIdOutput, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); blobFinder.Start(); downloader.Start(); this.HasFailures |= !blobEnumerator.EnqueueAllBlobs(commitToFetch); // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Index regardless of failures, it'll shorten the next fetch. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; if (!this.SkipConfigUpdate) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
private Result CreateClone( ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, GitRefs refs, string branch) { Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment); if (!initRepoResult.Success) { return(initRepoResult); } PhysicalFileSystem fileSystem = new PhysicalFileSystem(); string errorMessage; if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage)) { return(new Result("Error configuring alternate: " + errorMessage)); } GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem); GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment); GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor); if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) || !GVFSVerb.TrySetOptionalGitConfigSettings(enlistment)) { return(new Result("Unable to configure git repo")); } CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage)) { return(new Result("Unable to configure cache server: " + errorMessage)); } GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.ExitCodeIsFailure) { return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head), "ref: refs/heads/" + branch); if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } this.CreateGitScript(enlistment); string installHooksError; if (!HooksInstaller.InstallHooks(context, out installHooksError)) { tracer.RelatedError(installHooksError); return(new Result(installHooksError)); } GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0) { // It is possible to have the above TryDownloadCommit() fail because we // already have the commit and root tree we intend to check out, but // don't have a tree further down the working directory. If we fail // checkout here, its' because we don't have these trees and the // read-object hook is not available yet. Force downloading the commit // again and retry the checkout. if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage, checkLocalObjectCache: false)) { return(new Result(errorMessage)); } forceCheckoutResult = git.ForceCheckout(branch); } if (forceCheckoutResult.ExitCodeIsFailure) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); StringBuilder checkoutErrors = new StringBuilder(); foreach (string gitError in errorLines) { if (IsForceCheckoutErrorCloneFailure(gitError)) { checkoutErrors.AppendLine(gitError); } } if (checkoutErrors.Length > 0) { string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString(); tracer.RelatedError(error); return(new Result(error)); } } if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage)) { tracer.RelatedError(errorMessage); return(new Result(errorMessage)); } try { RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment); this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment); } catch (Exception e) { tracer.RelatedError(e.ToString()); return(new Result(e.Message)); } finally { RepoMetadata.Shutdown(); } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed Exception exception; string prepFileSystemError; if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) { EventMetadata metadata = new EventMetadata(); metadata.Add(nameof(prepFileSystemError), prepFileSystemError); if (exception != null) { metadata.Add("Exception", exception.ToString()); } tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); return(new Result(prepFileSystemError)); } return(new Result(true)); }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public virtual void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Dummy output queue since we don't need to checkout available blobs BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.PathWhitelist); if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } blobEnumerator.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= blobEnumerator.HasFailures; FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); blobFinder.Start(); downloader.Start(); // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Index regardless of failures, it'll shorten the next fetch. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
public void FastFetchWithStats( string branchOrCommit, bool isBranch, bool readFilesAfterDownload, out int matchedBlobCount, out int downloadedBlobCount, out int readFileCount) { matchedBlobCount = 0; downloadedBlobCount = 0; readFileCount = 0; if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList); if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } ThreadStart performDiff = () => { blobEnumerator.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= blobEnumerator.HasFailures; }; if (readFilesAfterDownload) { // Call synchronously to ensure that blobEnumerator.FileAddOperations // is completely populated when ReadFilesJob starts performDiff(); } else { new Thread(performDiff).Start(); } BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); ReadFilesJob readFiles = new ReadFilesJob(Environment.ProcessorCount * 2, blobEnumerator.FileAddOperations, availableBlobs, this.Tracer); blobFinder.Start(); downloader.Start(); if (readFilesAfterDownload) { readFiles.Start(); } // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; availableBlobs.CompleteAdding(); if (readFilesAfterDownload) { readFiles.WaitForCompletion(); this.HasFailures |= readFiles.HasFailures; } matchedBlobCount = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount; downloadedBlobCount = blobFinder.MissingBlobCount; readFileCount = readFiles.ReadFileCount; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
private Result CreateClone( ITracer tracer, GSDEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, GitRefs refs, string branch) { Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment); if (!initRepoResult.Success) { return(initRepoResult); } PhysicalFileSystem fileSystem = new PhysicalFileSystem(); string errorMessage; if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage)) { return(new Result("Error configuring alternate: " + errorMessage)); } GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem); GSDContext context = new GSDContext(tracer, fileSystem, gitRepo, enlistment); GSDGitObjects gitObjects = new GSDGitObjects(context, objectRequestor); if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } if (!GSDVerb.TrySetRequiredGitConfigSettings(enlistment) || !GSDVerb.TrySetOptionalGitConfigSettings(enlistment)) { return(new Result("Unable to configure git repo")); } CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage)) { return(new Result("Unable to configure cache server: " + errorMessage)); } GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.ExitCodeIsFailure) { return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(enlistment.WorkingDirectoryBackingRoot, GSDConstants.DotGit.Head), "ref: refs/heads/" + branch); if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } this.CreateGitScript(enlistment); string installHooksError; if (!HooksInstaller.InstallHooks(context, out installHooksError)) { tracer.RelatedError(installHooksError); return(new Result(installHooksError)); } // TODO: Move this to be after the mount? GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0) { // It is possible to have the above TryDownloadCommit() fail because we // already have the commit and root tree we intend to check out, but // don't have a tree further down the working directory. If we fail // checkout here, its' because we don't have these trees and the // read-object hook is not available yet. Force downloading the commit // again and retry the checkout. if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage, checkLocalObjectCache: false)) { return(new Result(errorMessage)); } forceCheckoutResult = git.ForceCheckout(branch); } if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGSDRoot, out errorMessage)) { tracer.RelatedError(errorMessage); return(new Result(errorMessage)); } try { RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment); this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment); } catch (Exception e) { tracer.RelatedError(e.ToString()); return(new Result(e.Message)); } finally { RepoMetadata.Shutdown(); } return(new Result(true)); }
private Result TryClone( JsonTracer tracer, GSDEnlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig, ServerGSDConfig serverGSDConfig, string resolvedLocalCacheRoot) { Result pipeResult; using (NamedPipeServer pipeServer = this.StartNamedPipe(tracer, enlistment, out pipeResult)) { if (!pipeResult.Success) { return(pipeResult); } using (GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig)) { GitRefs refs = objectRequestor.QueryInfoRefs(this.SingleBranch ? this.Branch : null); if (refs == null) { return(new Result("Could not query info/refs from: " + Uri.EscapeUriString(enlistment.RepoUrl))); } if (this.Branch == null) { this.Branch = refs.GetDefaultBranch(); EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Informational, "CloneDefaultRemoteBranch", metadata); } else { if (!refs.HasBranch(this.Branch)) { EventMetadata metadata = new EventMetadata(); metadata.Add("Branch", this.Branch); tracer.RelatedEvent(EventLevel.Warning, "CloneBranchDoesNotExist", metadata); string errorMessage = string.Format("Remote branch {0} not found in upstream origin", this.Branch); return(new Result(errorMessage)); } } if (!enlistment.TryCreateEnlistmentFolders()) { string error = "Could not create enlistment directory"; tracer.RelatedError(error); return(new Result(error)); } string localCacheError; if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, serverGSDConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError)) { tracer.RelatedError(localCacheError); return(new Result(localCacheError)); } Directory.CreateDirectory(enlistment.GitObjectsRoot); Directory.CreateDirectory(enlistment.GitPackRoot); Directory.CreateDirectory(enlistment.BlobSizesRoot); return(this.CreateClone(tracer, enlistment, objectRequestor, refs, this.Branch)); } } }
public void PrefetchWithStats( string branchOrCommit, bool isBranch, bool hydrateFilesAfterDownload, out int matchedBlobCount, out int downloadedBlobCount, out int hydratedFileCount) { matchedBlobCount = 0; downloadedBlobCount = 0; hydratedFileCount = 0; if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // For FastFetch only, examine the shallow file to determine the previous commit that had been fetched string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check. if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); //// // First create the pipeline // // diff ---> blobFinder ---> downloader ---> packIndexer // | | | | // ------------------------------------------------------> fileHydrator //// // diff // Inputs: // * files/folders // * commit id // Outputs: // * RequiredBlobs (property): Blob ids required to satisfy desired paths // * FileAddOperations (property): Repo-relative paths corresponding to those blob ids DiffHelper diff = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList, includeSymLinks: false); // blobFinder // Inputs: // * requiredBlobs (in param): Blob ids from output of `diff` // Outputs: // * availableBlobs (out param): Locally available blob ids (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available) // * MissingBlobs (property): Blob ids that are missing and need to be downloaded // * AvailableBlobs (property): Same as availableBlobs FindBlobsStage blobFinder = new FindBlobsStage(this.SearchThreadCount, diff.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); // downloader // Inputs: // * missingBlobs (in param): Blob ids from output of `blobFinder` // Outputs: // * availableBlobs (out param): Loose objects that have completed downloading (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available) // * AvailableObjects (property): Same as availableBlobs // * AvailablePacks (property): Packfiles that have completed downloading BatchObjectDownloadStage downloader = new BatchObjectDownloadStage(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); // packIndexer // Inputs: // * availablePacks (in param): Packfiles that have completed downloading from output of `downloader` // Outputs: // * availableBlobs (out param): Blobs that have completed downloading and indexing (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available) IndexPackStage packIndexer = new IndexPackStage(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); // fileHydrator // Inputs: // * workingDirectoryRoot (in param): the root of the working directory where hydration takes place // * blobIdsToPaths (in param): paths of all blob ids that need to be hydrated from output of `diff` // * availableBlobs (in param): blobs id that are available locally, from whatever source // Outputs: // * Hydrated files on disk. HydrateFilesStage fileHydrator = new HydrateFilesStage(Environment.ProcessorCount * 2, this.Enlistment.WorkingDirectoryRoot, diff.FileAddOperations, availableBlobs, this.Tracer); // All the stages of the pipeline are created and wired up, now kick them off in the proper sequence ThreadStart performDiff = () => { diff.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= diff.HasFailures; }; if (hydrateFilesAfterDownload) { // Call synchronously to ensure that diff.FileAddOperations // is completely populated when fileHydrator starts performDiff(); } else { new Thread(performDiff).Start(); } blobFinder.Start(); downloader.Start(); if (hydrateFilesAfterDownload) { fileHydrator.Start(); } // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; availableBlobs.CompleteAdding(); if (hydrateFilesAfterDownload) { fileHydrator.WaitForCompletion(); this.HasFailures |= fileHydrator.HasFailures; } matchedBlobCount = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount; downloadedBlobCount = blobFinder.MissingBlobCount; hydratedFileCount = fileHydrator.ReadFileCount; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } if (!this.HasFailures) { this.SavePrefetchArgs(commitToFetch, hydrateFilesAfterDownload); } }
protected bool UpdateRefSpec(ITracer tracer, Enlistment enlistment, string branchOrCommit, GitRefs refs) { using (ITracer activity = tracer.StartActivity("UpdateRefSpec", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { const string OriginRefMapSettingName = "remote.origin.fetch"; // We must update the refspec to get proper "git pull" functionality. string localBranch = branchOrCommit.StartsWith(RefsHeadsGitPath) ? branchOrCommit : (RefsHeadsGitPath + branchOrCommit); string remoteBranch = refs.GetBranchRefPairs().Single().Key; string refSpec = "+" + localBranch + ":" + remoteBranch; GitProcess git = new GitProcess(enlistment); // Replace all ref-specs this // * ensures the default refspec (remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*) is removed which avoids some "git fetch/pull" failures // * gives added "git fetch" performance since git will only fetch the branch provided in the refspec. GitProcess.Result setResult = git.SetInLocalConfig(OriginRefMapSettingName, refSpec, replaceAll: true); if (setResult.ExitCodeIsFailure) { activity.RelatedError("Could not update ref spec to {0}: {1}", refSpec, setResult.Errors); return(false); } } return(true); }
public CloneVerb.Result CreateClone(GitRefs refs, string branch) { GitObjects gitObjects = new GitObjects(this.tracer, this.enlistment, this.objectRequestor); CloneVerb.Result initRepoResult = this.TryInitRepo(refs, this.enlistment); if (!initRepoResult.Success) { return(initRepoResult); } string errorMessage; if (!this.enlistment.TryConfigureAlternate(out errorMessage)) { return(new CloneVerb.Result("Error configuring alternate: " + errorMessage)); } if (!gitObjects.TryDownloadAndSaveCommit(refs.GetTipCommitId(branch), commitDepth: 2)) { return(new CloneVerb.Result("Could not download tip commits from: " + Uri.EscapeUriString(this.objectRequestor.CacheServer.ObjectsEndpointUrl))); } GitProcess git = new GitProcess(this.enlistment); if (!this.SetConfigSettings(git, this.objectRequestor.CacheServer)) { return(new CloneVerb.Result("Unable to configure git repo")); } string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.HasErrors) { return(new CloneVerb.Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head), "ref: refs/heads/" + branch); File.AppendAllText( Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n"); CloneVerb.Result hydrateResult = this.HydrateRootGitAttributes(gitObjects, branch); if (!hydrateResult.Success) { return(hydrateResult); } this.CreateGitScript(); GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.HasErrors) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); StringBuilder checkoutErrors = new StringBuilder(); foreach (string gitError in errorLines) { if (IsForceCheckoutErrorCloneFailure(gitError)) { checkoutErrors.AppendLine(gitError); } } if (checkoutErrors.Length > 0) { string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString(); this.tracer.RelatedError(error); return(new CloneVerb.Result(error)); } } GitProcess.Result updateIndexresult = git.UpdateIndexVersion4(); if (updateIndexresult.HasErrors) { string error = "Could not update index, error: " + updateIndexresult.Errors; this.tracer.RelatedError(error); return(new CloneVerb.Result(error)); } string installHooksError; if (!HooksInstaller.InstallHooks(this.enlistment, out installHooksError)) { this.tracer.RelatedError(installHooksError); return(new CloneVerb.Result(installHooksError)); } using (RepoMetadata repoMetadata = new RepoMetadata(this.enlistment.DotGVFSRoot)) { repoMetadata.SaveCurrentDiskLayoutVersion(); } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed string prepGVFltError; if (!GVFltCallbacks.TryPrepareFolderForGVFltCallbacks(this.enlistment.WorkingDirectoryRoot, out prepGVFltError)) { this.tracer.RelatedError(prepGVFltError); return(new CloneVerb.Result(prepGVFltError)); } return(new CloneVerb.Result(true)); }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.HasErrors) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } // Update the index using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational)) { // The first bit of core.gvfs is set if index signing is turned off. const uint CoreGvfsUnsignedIndexFlag = 1; GitProcess git = new GitProcess(this.Enlistment); // Only update the index if index signing is turned off. // The first bit of core.gvfs is set if signing is turned off. GitProcess.Result configCoreGvfs = git.GetFromConfig("core.gvfs"); uint valueCoreGvfs; // No errors getting the configuration *and it is either "true" or numeric with the right bit set. bool indexSigningIsTurnedOff = !configCoreGvfs.HasErrors && !string.IsNullOrEmpty(configCoreGvfs.Output) && (configCoreGvfs.Output.Equals("true", StringComparison.OrdinalIgnoreCase) || (uint.TryParse(configCoreGvfs.Output, out valueCoreGvfs) && ((valueCoreGvfs & CoreGvfsUnsignedIndexFlag) == CoreGvfsUnsignedIndexFlag))); // Create the index object now so it can track the current index Index index = indexSigningIsTurnedOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null; GitProcess.Result result; using (ITracer updateIndexActivity = this.Tracer.StartActivity("ReadTree", EventLevel.Informational)) { result = git.ReadTree("HEAD"); } if (result.HasErrors) { activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors); this.HasFailures = true; } else { if (index != null) { // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; // Update index.UpdateFileSizesAndTimes(allowIndexMetadataUpdateFromWorkingTree); } else { activity.RelatedEvent(EventLevel.Informational, "Core.gvfs is not set, so not updating index entries with file times and sizes.", null); } } } } }