/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.HasErrors) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } bool indexSigningIsOff = this.GetIsIndexSigningOff(); // Update the index EventMetadata updateIndexMetadata = new EventMetadata(); updateIndexMetadata.Add("IndexSigningIsOff", indexSigningIsOff); using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) { // Create the index object now so it can track the current index Index index = indexSigningIsOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null; GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, !indexSigningIsOff); indexGen.CreateFromHeadTree(); this.HasFailures = indexGen.HasFailures; if (!indexGen.HasFailures && index != null) { // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; index.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree); } } } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.HasErrors) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } // Update the index using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational)) { GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.ReadTree("HEAD"); if (result.HasErrors) { activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors); this.HasFailures = true; } } } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public virtual void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Dummy output queue since we don't need to checkout available blobs BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.PathWhitelist); if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } blobEnumerator.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= blobEnumerator.HasFailures; FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); blobFinder.Start(); downloader.Start(); // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Index regardless of failures, it'll shorten the next fetch. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
public void FastFetchWithStats( string branchOrCommit, bool isBranch, bool readFilesAfterDownload, out int matchedBlobCount, out int downloadedBlobCount, out int readFileCount) { matchedBlobCount = 0; downloadedBlobCount = 0; readFileCount = 0; if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack string shallowFile = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow); string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList); if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) { this.Tracer.RelatedError("Shallow file exists, but contains no valid SHAs."); this.HasFailures = true; return; } } ThreadStart performDiff = () => { blobEnumerator.PerformDiff(previousCommit, commitToFetch); this.HasFailures |= blobEnumerator.HasFailures; }; if (readFilesAfterDownload) { // Call synchronously to ensure that blobEnumerator.FileAddOperations // is completely populated when ReadFilesJob starts performDiff(); } else { new Thread(performDiff).Start(); } BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); ReadFilesJob readFiles = new ReadFilesJob(Environment.ProcessorCount * 2, blobEnumerator.FileAddOperations, availableBlobs, this.Tracer); blobFinder.Start(); downloader.Start(); if (readFilesAfterDownload) { readFiles.Start(); } // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; availableBlobs.CompleteAdding(); if (readFilesAfterDownload) { readFiles.WaitForCompletion(); this.HasFailures |= readFiles.HasFailures; } matchedBlobCount = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount; downloadedBlobCount = blobFinder.MissingBlobCount; readFileCount = readFiles.ReadFileCount; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public virtual void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Dummy output queue since we don't need to checkout available blobs BlockingCollection <string> availableBlobs = new BlockingCollection <string>(); // Configure pipeline // LsTreeHelper output => FindMissingBlobs => BatchDownload => IndexPack LsTreeHelper blobEnumerator = new LsTreeHelper(this.PathWhitelist, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.BlobIdOutput, availableBlobs, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects); blobFinder.Start(); downloader.Start(); this.HasFailures |= !blobEnumerator.EnqueueAllBlobs(commitToFetch); // If indexing happens during searching, searching progressively gets slower, so wait on searching before indexing. blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Index regardless of failures, it'll shorten the next fetch. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; if (!this.SkipConfigUpdate) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); } } }
/// <param name="branchOrCommit">A specific branch to filter for, or null for all branches returned from info/refs</param> public override void FastFetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); } GitRefs refs = null; string commitToFetch; if (isBranch) { refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit); if (refs == null) { throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl); } else if (refs.Count == 0) { throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } commitToFetch = refs.GetTipCommitIds().Single(); } else { commitToFetch = branchOrCommit; } this.DownloadMissingCommit(commitToFetch, this.GitObjects); // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); // Start pipeline downloader.Start(); blobFinder.Start(); checkout.Start(); blobFinder.WaitForCompletion(); this.HasFailures |= blobFinder.HasFailures; // Delay indexing. It interferes with FindMissingBlobs, and doesn't help Bootstrapping. packIndexer.Start(); downloader.WaitForCompletion(); this.HasFailures |= downloader.HasFailures; packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; // Since pack indexer is the last to finish before checkout finishes, it should propagate completion. // This prevents availableObjects from completing before packIndexer can push its objects through this link. checkout.AvailableBlobShas.CompleteAdding(); checkout.WaitForCompletion(); this.HasFailures |= checkout.HasFailures; if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { string remoteBranch = refs.GetBranchRefPairs().Single().Key; GitProcess git = new GitProcess(this.Enlistment); GitProcess.Result result = git.SetUpstream(branchOrCommit, remoteBranch); if (result.HasErrors) { activity.RelatedError("Could not set upstream for {0} to {1}: {2}", branchOrCommit, remoteBranch, result.Errors); this.HasFailures = true; } } } // Update the index using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational)) { // The first bit of core.gvfs is set if index signing is turned off. const uint CoreGvfsUnsignedIndexFlag = 1; GitProcess git = new GitProcess(this.Enlistment); // Only update the index if index signing is turned off. // The first bit of core.gvfs is set if signing is turned off. GitProcess.Result configCoreGvfs = git.GetFromConfig("core.gvfs"); uint valueCoreGvfs; // No errors getting the configuration *and it is either "true" or numeric with the right bit set. bool indexSigningIsTurnedOff = !configCoreGvfs.HasErrors && !string.IsNullOrEmpty(configCoreGvfs.Output) && (configCoreGvfs.Output.Equals("true", StringComparison.OrdinalIgnoreCase) || (uint.TryParse(configCoreGvfs.Output, out valueCoreGvfs) && ((valueCoreGvfs & CoreGvfsUnsignedIndexFlag) == CoreGvfsUnsignedIndexFlag))); // Create the index object now so it can track the current index Index index = indexSigningIsTurnedOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null; GitProcess.Result result; using (ITracer updateIndexActivity = this.Tracer.StartActivity("ReadTree", EventLevel.Informational)) { result = git.ReadTree("HEAD"); } if (result.HasErrors) { activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors); this.HasFailures = true; } else { if (index != null) { // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; // Update index.UpdateFileSizesAndTimes(allowIndexMetadataUpdateFromWorkingTree); } else { activity.RelatedEvent(EventLevel.Informational, "Core.gvfs is not set, so not updating index entries with file times and sizes.", null); } } } } }