private void LoadBlobPrefetchArgs( ITracer tracer, GVFSEnlistment enlistment, out string headCommitId, out List <string> filesList, out List <string> foldersList, out FileBasedDictionary <string, string> lastPrefetchArgs) { string error; if (!FileBasedDictionary <string, string> .TryCreate( tracer, Path.Combine(enlistment.DotGVFSRoot, "LastBlobPrefetch.dat"), new PhysicalFileSystem(), out lastPrefetchArgs, out error)) { tracer.RelatedWarning("Unable to load last prefetch args: " + error); } filesList = new List <string>(); foldersList = new List <string>(); if (!BlobPrefetcher.TryLoadFileList(enlistment, this.Files, this.FilesListFile, filesList, readListFromStdIn: this.FilesFromStdIn, error: out error)) { this.ReportErrorAndExit(tracer, error); } if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, foldersList, readListFromStdIn: this.FoldersFromStdIn, error: out error)) { this.ReportErrorAndExit(tracer, error); } GitProcess gitProcess = new GitProcess(enlistment); GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName); if (result.ExitCodeIsFailure) { this.ReportErrorAndExit(tracer, result.Errors); } headCommitId = result.Output.Trim(); }
public void AppendToNewlineSeparatedFileTests() { MockFileSystem fileSystem = new MockFileSystem(new MockDirectory(Path.Combine("mock:", "GVFS", "UnitTests", "Repo"), null, null)); // Validate can write to a file that doesn't exist. string testFileName = Path.Combine("mock:", "GVFS", "UnitTests", "Repo", "appendTests"); BlobPrefetcher.AppendToNewlineSeparatedFile(fileSystem, testFileName, "expected content line 1"); fileSystem.ReadAllText(testFileName).ShouldEqual("expected content line 1\n"); // Validate that if the file doesn't end in a newline it gets a newline added. fileSystem.WriteAllText(testFileName, "existing content"); BlobPrefetcher.AppendToNewlineSeparatedFile(fileSystem, testFileName, "expected line 2"); fileSystem.ReadAllText(testFileName).ShouldEqual("existing content\nexpected line 2\n"); // Validate that if the file ends in a newline, we don't end up with two newlines fileSystem.WriteAllText(testFileName, "existing content\n"); BlobPrefetcher.AppendToNewlineSeparatedFile(fileSystem, testFileName, "expected line 2"); fileSystem.ReadAllText(testFileName).ShouldEqual("existing content\nexpected line 2\n"); }
private int ExecuteWithExitCode() { // CmdParser doesn't strip quotes, and Path.Combine will throw this.GitBinPath = this.GitBinPath.Replace("\"", string.Empty); if (!GVFSPlatform.Instance.GitInstallation.GitExists(this.GitBinPath)) { Console.WriteLine( "Could not find git.exe {0}", !string.IsNullOrWhiteSpace(this.GitBinPath) ? "at " + this.GitBinPath : "on %PATH%"); return(ExitFailure); } if (this.Commit != null && this.Branch != null) { Console.WriteLine("Cannot specify both a commit sha and a branch name."); return(ExitFailure); } if (this.ForceCheckout && !this.Checkout) { Console.WriteLine("Cannot use --force-checkout option without --checkout option."); return(ExitFailure); } this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount; this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads); this.IndexThreadCount = this.IndexThreadCount > 0 ? this.IndexThreadCount : Environment.ProcessorCount; this.CheckoutThreadCount = this.CheckoutThreadCount > 0 ? this.CheckoutThreadCount : Environment.ProcessorCount; this.GitBinPath = !string.IsNullOrWhiteSpace(this.GitBinPath) ? this.GitBinPath : GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); GitEnlistment enlistment = GitEnlistment.CreateFromCurrentDirectory(this.GitBinPath); if (enlistment == null) { Console.WriteLine("Must be run within a git repo"); return(ExitFailure); } string commitish = this.Commit ?? this.Branch; if (string.IsNullOrWhiteSpace(commitish)) { GitProcess.Result result = new GitProcess(enlistment).GetCurrentBranchName(); if (result.ExitCodeIsFailure || string.IsNullOrWhiteSpace(result.Output)) { Console.WriteLine("Could not retrieve current branch name: " + result.Errors); return(ExitFailure); } commitish = result.Output.Trim(); } Guid parentActivityId = Guid.Empty; if (!string.IsNullOrWhiteSpace(this.ParentActivityId) && !Guid.TryParse(this.ParentActivityId, out parentActivityId)) { Console.WriteLine("The ParentActivityId provided (" + this.ParentActivityId + ") is not a valid GUID."); } using (JsonTracer tracer = new JsonTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch", enlistmentId: null, mountId: null, disableTelemetry: true)) { if (this.Verbose) { tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any); } else { tracer.AddPrettyConsoleEventListener(EventLevel.Error, Keywords.Any); } string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch"); tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any); CacheServerInfo cacheServer = new CacheServerInfo(this.GetRemoteUrl(enlistment), null); tracer.WriteStartEvent( enlistment.EnlistmentRoot, enlistment.RepoUrl, cacheServer.Url, new EventMetadata { { "TargetCommitish", commitish }, { "Checkout", this.Checkout }, }); string error; if (!enlistment.Authentication.TryInitialize(tracer, enlistment, out error)) { tracer.RelatedError(error); Console.WriteLine(error); return(ExitFailure); } RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); BlobPrefetcher prefetcher = this.GetFolderPrefetcher(tracer, enlistment, cacheServer, retryConfig); if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, out error)) { tracer.RelatedError(error); Console.WriteLine(error); return(ExitFailure); } bool isSuccess; try { Func <bool> doPrefetch = () => { try { bool isBranch = this.Commit == null; prefetcher.Prefetch(commitish, isBranch); return(!prefetcher.HasFailures); } catch (BlobPrefetcher.FetchException e) { tracer.RelatedError(e.Message); return(false); } }; if (this.Verbose) { isSuccess = doPrefetch(); } else { isSuccess = ConsoleHelper.ShowStatusWhileRunning( doPrefetch, "Fetching", output: Console.Out, showSpinner: !Console.IsOutputRedirected, gvfsLogEnlistmentRoot: null); Console.WriteLine(); Console.WriteLine("See the full log at " + fastfetchLogFile); } isSuccess &= !prefetcher.HasFailures; } catch (AggregateException e) { isSuccess = false; foreach (Exception ex in e.Flatten().InnerExceptions) { tracer.RelatedError(ex.ToString()); } } catch (Exception e) { isSuccess = false; tracer.RelatedError(e.ToString()); } EventMetadata stopMetadata = new EventMetadata(); stopMetadata.Add("Success", isSuccess); tracer.Stop(stopMetadata); return(isSuccess ? ExitSuccess : ExitFailure); } }
private void PrefetchBlobs( ITracer tracer, GVFSEnlistment enlistment, string headCommitId, List <string> filesList, List <string> foldersList, FileBasedDictionary <string, string> lastPrefetchArgs, GitObjectsHttpRequestor objectRequestor, CacheServerInfo cacheServer) { BlobPrefetcher blobPrefetcher = new BlobPrefetcher( tracer, enlistment, objectRequestor, filesList, foldersList, lastPrefetchArgs, ChunkSize, SearchThreadCount, DownloadThreadCount, IndexThreadCount); if (blobPrefetcher.FolderList.Count == 0 && blobPrefetcher.FileList.Count == 0) { this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files '*'` to confirm."); } if (this.HydrateFiles) { if (!this.CheckIsMounted(verbose: true)) { this.ReportErrorAndExit("You can only specify --hydrate if the repo is mounted. Run 'gvfs mount' and try again."); } } int matchedBlobCount = 0; int downloadedBlobCount = 0; int hydratedFileCount = 0; Func <bool> doPrefetch = () => { try { blobPrefetcher.PrefetchWithStats( headCommitId, isBranch: false, hydrateFilesAfterDownload: this.HydrateFiles, matchedBlobCount: out matchedBlobCount, downloadedBlobCount: out downloadedBlobCount, hydratedFileCount: out hydratedFileCount); return(!blobPrefetcher.HasFailures); } catch (BlobPrefetcher.FetchException e) { tracer.RelatedError(e.Message); return(false); } }; if (this.Verbose) { doPrefetch(); } else { string message = this.HydrateFiles ? "Fetching blobs and hydrating files " : "Fetching blobs "; this.ShowStatusWhileRunning(doPrefetch, message + this.GetCacheServerDisplay(cacheServer, enlistment.RepoUrl)); } if (blobPrefetcher.HasFailures) { Environment.ExitCode = 1; } else { Console.WriteLine(); Console.WriteLine("Stats:"); Console.WriteLine(" Matched blobs: " + matchedBlobCount); Console.WriteLine(" Already cached: " + (matchedBlobCount - downloadedBlobCount)); Console.WriteLine(" Downloaded: " + downloadedBlobCount); if (this.HydrateFiles) { Console.WriteLine(" Hydrated files: " + hydratedFileCount); } } }
protected override void Execute(GVFSEnlistment enlistment) { using (JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "Prefetch")) { if (this.Verbose) { tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any); } string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment); tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Prefetch), EventLevel.Informational, Keywords.Any); tracer.WriteStartEvent( enlistment.EnlistmentRoot, enlistment.RepoUrl, cacheServerUrl); try { EventMetadata metadata = new EventMetadata(); metadata.Add("Commits", this.Commits); metadata.Add("Files", this.Files); metadata.Add("Folders", this.Folders); metadata.Add("FileListFile", this.FilesListFile); metadata.Add("FoldersListFile", this.FoldersListFile); metadata.Add("FilesFromStdIn", this.FilesFromStdIn); metadata.Add("FoldersFromStdIn", this.FoldersFromStdIn); metadata.Add("HydrateFiles", this.HydrateFiles); tracer.RelatedEvent(EventLevel.Informational, "PerformPrefetch", metadata); if (this.Commits) { if (!string.IsNullOrWhiteSpace(this.Files) || !string.IsNullOrWhiteSpace(this.Folders) || !string.IsNullOrWhiteSpace(this.FoldersListFile) || !string.IsNullOrWhiteSpace(this.FilesListFile) || this.FilesFromStdIn || this.FoldersFromStdIn) { this.ReportErrorAndExit(tracer, "You cannot prefetch commits and blobs at the same time."); } if (this.HydrateFiles) { this.ReportErrorAndExit(tracer, "You can only specify --hydrate with --files or --folders"); } GitObjectsHttpRequestor objectRequestor; CacheServerInfo cacheServer; this.InitializeServerConnection( tracer, enlistment, cacheServerUrl, out objectRequestor, out cacheServer); this.PrefetchCommits(tracer, enlistment, objectRequestor, cacheServer); } else { string headCommitId; List <string> filesList; List <string> foldersList; FileBasedDictionary <string, string> lastPrefetchArgs; this.LoadBlobPrefetchArgs(tracer, enlistment, out headCommitId, out filesList, out foldersList, out lastPrefetchArgs); if (BlobPrefetcher.IsNoopPrefetch(tracer, lastPrefetchArgs, headCommitId, filesList, foldersList, this.HydrateFiles)) { Console.WriteLine("All requested files are already available. Nothing new to prefetch."); } else { GitObjectsHttpRequestor objectRequestor; CacheServerInfo cacheServer; this.InitializeServerConnection( tracer, enlistment, cacheServerUrl, out objectRequestor, out cacheServer); this.PrefetchBlobs(tracer, enlistment, headCommitId, filesList, foldersList, lastPrefetchArgs, objectRequestor, cacheServer); } } } catch (VerbAbortedException) { throw; } catch (AggregateException aggregateException) { this.Output.WriteLine( "Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot), enlistment.EnlistmentRoot); foreach (Exception innerException in aggregateException.Flatten().InnerExceptions) { tracer.RelatedError( new EventMetadata { { "Verb", typeof(PrefetchVerb).Name }, { "Exception", innerException.ToString() } }, $"Unhandled {innerException.GetType().Name}: {innerException.Message}"); } Environment.ExitCode = (int)ReturnCode.GenericError; } catch (Exception e) { this.Output.WriteLine( "Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot), enlistment.EnlistmentRoot); tracer.RelatedError( new EventMetadata { { "Verb", typeof(PrefetchVerb).Name }, { "Exception", e.ToString() } }, $"Unhandled {e.GetType().Name}: {e.Message}"); Environment.ExitCode = (int)ReturnCode.GenericError; } } }
private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor blobRequestor, CacheServerInfo cacheServer) { BlobPrefetcher blobPrefetcher = new BlobPrefetcher( tracer, enlistment, blobRequestor, ChunkSize, SearchThreadCount, DownloadThreadCount, IndexThreadCount); string error; if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, blobPrefetcher.FolderList, out error)) { this.ReportErrorAndExit(tracer, error); } if (!BlobPrefetcher.TryLoadFileList(enlistment, this.Files, blobPrefetcher.FileList, out error)) { this.ReportErrorAndExit(tracer, error); } if (blobPrefetcher.FolderList.Count == 0 && blobPrefetcher.FileList.Count == 0) { this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files *` to confirm."); } if (this.HydrateFiles) { if (!this.CheckIsMounted(verbose: true)) { this.ReportErrorAndExit("You can only specify --hydrate if the repo is mounted. Run 'gvfs mount' and try again."); } } GitProcess gitProcess = new GitProcess(enlistment); GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName); if (result.HasErrors) { tracer.RelatedError(result.Errors); this.Output.WriteLine(result.Errors); Environment.ExitCode = (int)ReturnCode.GenericError; return; } int matchedBlobCount = 0; int downloadedBlobCount = 0; int readFileCount = 0; string headCommitId = result.Output; Func <bool> doPrefetch = () => { try { blobPrefetcher.PrefetchWithStats( headCommitId.Trim(), isBranch: false, readFilesAfterDownload: this.HydrateFiles, matchedBlobCount: out matchedBlobCount, downloadedBlobCount: out downloadedBlobCount, readFileCount: out readFileCount); return(!blobPrefetcher.HasFailures); } catch (BlobPrefetcher.FetchException e) { tracer.RelatedError(e.Message); return(false); } }; if (this.Verbose) { doPrefetch(); } else { string message = this.HydrateFiles ? "Fetching blobs and hydrating files " : "Fetching blobs "; this.ShowStatusWhileRunning(doPrefetch, message + this.GetCacheServerDisplay(cacheServer, enlistment.RepoUrl)); } if (blobPrefetcher.HasFailures) { Environment.ExitCode = 1; } else { Console.WriteLine(); Console.WriteLine("Stats:"); Console.WriteLine(" Matched blobs: " + matchedBlobCount); Console.WriteLine(" Already cached: " + (matchedBlobCount - downloadedBlobCount)); Console.WriteLine(" Downloaded: " + downloadedBlobCount); if (this.HydrateFiles) { Console.WriteLine(" Hydrated files: " + readFileCount); } } }