public CloneVerb.Result CreateClone(GitRefs refs, string branch) { GitObjects gitObjects = new GitObjects(this.tracer, this.enlistment, this.objectRequestor); CloneVerb.Result initRepoResult = this.TryInitRepo(refs, this.enlistment); if (!initRepoResult.Success) { return(initRepoResult); } string errorMessage; if (!this.enlistment.TryConfigureAlternate(out errorMessage)) { return(new CloneVerb.Result("Error configuring alternate: " + errorMessage)); } if (!gitObjects.TryDownloadAndSaveCommit(refs.GetTipCommitId(branch), commitDepth: 2)) { return(new CloneVerb.Result("Could not download tip commits from: " + Uri.EscapeUriString(this.objectRequestor.CacheServer.ObjectsEndpointUrl))); } GitProcess git = new GitProcess(this.enlistment); if (!this.SetConfigSettings(git, this.objectRequestor.CacheServer)) { return(new CloneVerb.Result("Unable to configure git repo")); } string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.HasErrors) { return(new CloneVerb.Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head), "ref: refs/heads/" + branch); File.AppendAllText( Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n"); CloneVerb.Result hydrateResult = this.HydrateRootGitAttributes(gitObjects, branch); if (!hydrateResult.Success) { return(hydrateResult); } this.CreateGitScript(); GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.HasErrors) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); StringBuilder checkoutErrors = new StringBuilder(); foreach (string gitError in errorLines) { if (IsForceCheckoutErrorCloneFailure(gitError)) { checkoutErrors.AppendLine(gitError); } } if (checkoutErrors.Length > 0) { string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString(); this.tracer.RelatedError(error); return(new CloneVerb.Result(error)); } } GitProcess.Result updateIndexresult = git.UpdateIndexVersion4(); if (updateIndexresult.HasErrors) { string error = "Could not update index, error: " + updateIndexresult.Errors; this.tracer.RelatedError(error); return(new CloneVerb.Result(error)); } string installHooksError; if (!HooksInstaller.InstallHooks(this.enlistment, out installHooksError)) { this.tracer.RelatedError(installHooksError); return(new CloneVerb.Result(installHooksError)); } using (RepoMetadata repoMetadata = new RepoMetadata(this.enlistment.DotGVFSRoot)) { repoMetadata.SaveCurrentDiskLayoutVersion(); } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed string prepGVFltError; if (!GVFltCallbacks.TryPrepareFolderForGVFltCallbacks(this.enlistment.WorkingDirectoryRoot, out prepGVFltError)) { this.tracer.RelatedError(prepGVFltError); return(new CloneVerb.Result(prepGVFltError)); } return(new CloneVerb.Result(true)); }
private Result CreateClone( ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, GitRefs refs, string branch) { Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment); if (!initRepoResult.Success) { return(initRepoResult); } PhysicalFileSystem fileSystem = new PhysicalFileSystem(); string errorMessage; if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage)) { return(new Result("Error configuring alternate: " + errorMessage)); } GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem); GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment); GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor); if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) || !GVFSVerb.TrySetOptionalGitConfigSettings(enlistment)) { return(new Result("Unable to configure git repo")); } CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage)) { return(new Result("Unable to configure cache server: " + errorMessage)); } GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.HasErrors) { return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head), "ref: refs/heads/" + branch); File.AppendAllText( Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n"); if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } this.CreateGitScript(enlistment); string installHooksError; if (!HooksInstaller.InstallHooks(context, out installHooksError)) { tracer.RelatedError(installHooksError); return(new Result(installHooksError)); } GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.HasErrors) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); StringBuilder checkoutErrors = new StringBuilder(); foreach (string gitError in errorLines) { if (IsForceCheckoutErrorCloneFailure(gitError)) { checkoutErrors.AppendLine(gitError); } } if (checkoutErrors.Length > 0) { string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString(); tracer.RelatedError(error); return(new Result(error)); } } if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage)) { tracer.RelatedError(errorMessage); return(new Result(errorMessage)); } try { RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment); this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment); } catch (Exception e) { tracer.RelatedError(e.ToString()); return(new Result(e.Message)); } finally { RepoMetadata.Shutdown(); } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed string prepFileSystemError; if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) { tracer.RelatedError(prepFileSystemError); return(new Result(prepFileSystemError)); } return(new Result(true)); }
private Result CreateClone( ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, GitRefs refs, string branch) { Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment); if (!initRepoResult.Success) { return(initRepoResult); } PhysicalFileSystem fileSystem = new PhysicalFileSystem(); string errorMessage; if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage)) { return(new Result("Error configuring alternate: " + errorMessage)); } GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem); GVFSContext context = new GVFSContext(tracer, fileSystem, gitRepo, enlistment); GVFSGitObjects gitObjects = new GVFSGitObjects(context, objectRequestor); if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } if (!GVFSVerb.TrySetRequiredGitConfigSettings(enlistment) || !GVFSVerb.TrySetOptionalGitConfigSettings(enlistment)) { return(new Result("Unable to configure git repo")); } CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage)) { return(new Result("Unable to configure cache server: " + errorMessage)); } GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.ExitCodeIsFailure) { return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head), "ref: refs/heads/" + branch); if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } this.CreateGitScript(enlistment); string installHooksError; if (!HooksInstaller.InstallHooks(context, out installHooksError)) { tracer.RelatedError(installHooksError); return(new Result(installHooksError)); } GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0) { // It is possible to have the above TryDownloadCommit() fail because we // already have the commit and root tree we intend to check out, but // don't have a tree further down the working directory. If we fail // checkout here, its' because we don't have these trees and the // read-object hook is not available yet. Force downloading the commit // again and retry the checkout. if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage, checkLocalObjectCache: false)) { return(new Result(errorMessage)); } forceCheckoutResult = git.ForceCheckout(branch); } if (forceCheckoutResult.ExitCodeIsFailure) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); StringBuilder checkoutErrors = new StringBuilder(); foreach (string gitError in errorLines) { if (IsForceCheckoutErrorCloneFailure(gitError)) { checkoutErrors.AppendLine(gitError); } } if (checkoutErrors.Length > 0) { string error = "Could not complete checkout of branch: " + branch + ", " + checkoutErrors.ToString(); tracer.RelatedError(error); return(new Result(error)); } } if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGVFSRoot, out errorMessage)) { tracer.RelatedError(errorMessage); return(new Result(errorMessage)); } try { RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment); this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment); } catch (Exception e) { tracer.RelatedError(e.ToString()); return(new Result(e.Message)); } finally { RepoMetadata.Shutdown(); } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed Exception exception; string prepFileSystemError; if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) { EventMetadata metadata = new EventMetadata(); metadata.Add(nameof(prepFileSystemError), prepFileSystemError); if (exception != null) { metadata.Add("Exception", exception.ToString()); } tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); return(new Result(prepFileSystemError)); } return(new Result(true)); }
public void PrefetchWithStats( string branchOrCommit, bool isBranch, bool hydrateFilesAfterDownload, out int matchedBlobCount, out int downloadedBlobCount, out int hydratedFileCount) { matchedBlobCount = 0; downloadedBlobCount = 0; hydratedFileCount = 0; if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // For FastFetch only, examine the shallow file to determine the previous commit that had been fetched string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check. if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); //// // First create the pipeline // // diff ---> blobFinder ---> downloader ---> packIndexer // | | | | // ------------------------------------------------------> fileHydrator //// // diff // Inputs: // * files/folders // * commit id // Outputs: // * RequiredBlobs (property): Blob ids required to satisfy desired paths // * FileAddOperations (property): Repo-relative paths corresponding to those blob ids DiffHelper diff = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList, includeSymLinks: false); // blobFinder // Inputs: // * requiredBlobs (in param): Blob ids from output of `diff` // Outputs: // * availableBlobs (out param): Locally available blob ids (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available) // * MissingBlobs (property): Blob ids that are missing and need to be downloaded // * AvailableBlobs (property): Same as availableBlobs FindBlobsStage blobFinder = new FindBlobsStage(this.SearchThreadCount, diff.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); // downloader // Inputs: // * missingBlobs (in param): Blob ids from output of `blobFinder` // Outputs: // * availableBlobs (out param): Loose objects that have completed downloading (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available) // * AvailableObjects (property): Same as availableBlobs // * AvailablePacks (property): Packfiles that have completed downloading BatchObjectDownloadStage downloader = new BatchObjectDownloadStage(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); // packIndexer // Inputs: // * availablePacks (in param): Packfiles that have completed downloading from output of `downloader` // Outputs: // * availableBlobs (out param): Blobs that have completed downloading and indexing (shared between `blobFinder`, `downloader`, and `packIndexer`, all add blob ids to the list as they are locally available) IndexPackStage packIndexer = new IndexPackStage(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); // fileHydrator // Inputs: // * workingDirectoryRoot (in param): the root of the working directory where hydration takes place // * blobIdsToPaths (in param): paths of all blob ids that need to be hydrated from output of `diff` // * availableBlobs (in param): blobs id that are available locally, from whatever source // Outputs: // * Hydrated files on disk. HydrateFilesStage fileHydrator = new HydrateFilesStage(Environment.ProcessorCount * 2, this.Enlistment.WorkingDirectoryRoot, diff.FileAddOperations, availableBlobs, this.Tracer); // All the stages of the pipeline are created and wired up, now kick them off in the proper sequence ThreadStart performDiff = () => { diff.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= diff.HasFailures; }; if (hydrateFilesAfterDownload) { // Call synchronously to ensure that diff.FileAddOperations // is completely populated when fileHydrator starts performDiff(); } else { new Thread(performDiff).Start(); } blobFinder.Start(); downloader.Start(); if (hydrateFilesAfterDownload) { fileHydrator.Start(); } // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; availableBlobs.CompleteAdding(); if (hydrateFilesAfterDownload) { fileHydrator.WaitForCompletion(); this.HasFailures |= fileHydrator.HasFailures; } matchedBlobCount = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount; downloadedBlobCount = blobFinder.MissingBlobCount; hydratedFileCount = fileHydrator.ReadFileCount; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } if (!this.HasFailures) { this.SavePrefetchArgs(commitToFetch, hydrateFilesAfterDownload); } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void Prefetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.HasErrors) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } bool shouldSignIndex = !this.GetIsIndexSigningOff(); // Update the index EventMetadata updateIndexMetadata = new EventMetadata(); updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex); using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) { Index sourceIndex = this.GetSourceIndex(); GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex); indexGen.CreateFromHeadTree(indexVersion: 2); this.HasFailures |= indexGen.HasFailures; if (!indexGen.HasFailures) { Index newIndex = new Index( this.Enlistment.EnlistmentRoot, this.Tracer, Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName), readOnly: false); // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex); } } } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public virtual void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Dummy output queue since we don't need to checkout available blobs BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.PathWhitelist); if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } blobEnumerator.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= blobEnumerator.HasFailures; FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); blobFinder.Start(); downloader.Start(); // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Index regardless of failures, it'll shorten the next fetch. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
public void FastFetchWithStats( string branchOrCommit, bool isBranch, bool readFilesAfterDownload, out int matchedBlobCount, out int downloadedBlobCount, out int readFileCount) { matchedBlobCount = 0; downloadedBlobCount = 0; readFileCount = 0; if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList); if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } ThreadStart performDiff = () => { blobEnumerator.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= blobEnumerator.HasFailures; }; if (readFilesAfterDownload) { // Call synchronously to ensure that blobEnumerator.FileAddOperations // is completely populated when ReadFilesJob starts performDiff(); } else { new Thread(performDiff).Start(); } BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); ReadFilesJob readFiles = new ReadFilesJob(Environment.ProcessorCount * 2, blobEnumerator.FileAddOperations, availableBlobs, this.Tracer); blobFinder.Start(); downloader.Start(); if (readFilesAfterDownload) { readFiles.Start(); } // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; availableBlobs.CompleteAdding(); if (readFilesAfterDownload) { readFiles.WaitForCompletion(); this.HasFailures |= readFiles.HasFailures; } matchedBlobCount = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount; downloadedBlobCount = blobFinder.MissingBlobCount; readFileCount = readFiles.ReadFileCount; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
private Result CreateClone( ITracer tracer, GSDEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, GitRefs refs, string branch) { Result initRepoResult = this.TryInitRepo(tracer, refs, enlistment); if (!initRepoResult.Success) { return(initRepoResult); } PhysicalFileSystem fileSystem = new PhysicalFileSystem(); string errorMessage; if (!this.TryCreateAlternatesFile(fileSystem, enlistment, out errorMessage)) { return(new Result("Error configuring alternate: " + errorMessage)); } GitRepo gitRepo = new GitRepo(tracer, enlistment, fileSystem); GSDContext context = new GSDContext(tracer, fileSystem, gitRepo, enlistment); GSDGitObjects gitObjects = new GSDGitObjects(context, objectRequestor); if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } if (!GSDVerb.TrySetRequiredGitConfigSettings(enlistment) || !GSDVerb.TrySetOptionalGitConfigSettings(enlistment)) { return(new Result("Unable to configure git repo")); } CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); if (!cacheServerResolver.TrySaveUrlToLocalConfig(objectRequestor.CacheServer, out errorMessage)) { return(new Result("Unable to configure cache server: " + errorMessage)); } GitProcess git = new GitProcess(enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.ExitCodeIsFailure) { return(new Result("Unable to create branch '" + originBranchName + "': " + createBranchResult.Errors + "\r\n" + createBranchResult.Output)); } File.WriteAllText( Path.Combine(enlistment.WorkingDirectoryBackingRoot, GSDConstants.DotGit.Head), "ref: refs/heads/" + branch); if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage)) { return(new Result(errorMessage)); } this.CreateGitScript(enlistment); string installHooksError; if (!HooksInstaller.InstallHooks(context, out installHooksError)) { tracer.RelatedError(installHooksError); return(new Result(installHooksError)); } // TODO: Move this to be after the mount? GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); if (forceCheckoutResult.ExitCodeIsFailure && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0) { // It is possible to have the above TryDownloadCommit() fail because we // already have the commit and root tree we intend to check out, but // don't have a tree further down the working directory. If we fail // checkout here, its' because we don't have these trees and the // read-object hook is not available yet. Force downloading the commit // again and retry the checkout. if (!this.TryDownloadCommit( refs.GetTipCommitId(branch), enlistment, objectRequestor, gitObjects, gitRepo, out errorMessage, checkLocalObjectCache: false)) { return(new Result(errorMessage)); } forceCheckoutResult = git.ForceCheckout(branch); } if (!RepoMetadata.TryInitialize(tracer, enlistment.DotGSDRoot, out errorMessage)) { tracer.RelatedError(errorMessage); return(new Result(errorMessage)); } try { RepoMetadata.Instance.SaveCloneMetadata(tracer, enlistment); this.LogEnlistmentInfoAndSetConfigValues(tracer, git, enlistment); } catch (Exception e) { tracer.RelatedError(e.ToString()); return(new Result(e.Message)); } finally { RepoMetadata.Shutdown(); } return(new Result(true)); }