private void LoadBlobPrefetchArgs(
ITracer tracer,
GVFSEnlistment enlistment,
out string headCommitId,
out List <string> filesList,
out List <string> foldersList,
out FileBasedDictionary <string, string> lastPrefetchArgs)
{
string error;
if (!FileBasedDictionary <string, string> .TryCreate(
tracer,
Path.Combine(enlistment.DotGVFSRoot, "LastBlobPrefetch.dat"),
new PhysicalFileSystem(),
out lastPrefetchArgs,
out error))
{
tracer.RelatedWarning("Unable to load last prefetch args: " + error);
}
filesList = new List <string>();
foldersList = new List <string>();
if (!BlobPrefetcher.TryLoadFileList(enlistment, this.Files, this.FilesListFile, filesList, readListFromStdIn: this.FilesFromStdIn, error: out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, foldersList, readListFromStdIn: this.FoldersFromStdIn, error: out error))
{
this.ReportErrorAndExit(tracer, error);
}
GitProcess gitProcess = new GitProcess(enlistment);
GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName);
if (result.ExitCodeIsFailure)
{
this.ReportErrorAndExit(tracer, result.Errors);
}
headCommitId = result.Output.Trim();
}
public void AppendToNewlineSeparatedFileTests()
{
MockFileSystem fileSystem = new MockFileSystem(new MockDirectory(Path.Combine("mock:", "GVFS", "UnitTests", "Repo"), null, null));
// Validate can write to a file that doesn't exist.
string testFileName = Path.Combine("mock:", "GVFS", "UnitTests", "Repo", "appendTests");
BlobPrefetcher.AppendToNewlineSeparatedFile(fileSystem, testFileName, "expected content line 1");
fileSystem.ReadAllText(testFileName).ShouldEqual("expected content line 1\n");
// Validate that if the file doesn't end in a newline it gets a newline added.
fileSystem.WriteAllText(testFileName, "existing content");
BlobPrefetcher.AppendToNewlineSeparatedFile(fileSystem, testFileName, "expected line 2");
fileSystem.ReadAllText(testFileName).ShouldEqual("existing content\nexpected line 2\n");
// Validate that if the file ends in a newline, we don't end up with two newlines
fileSystem.WriteAllText(testFileName, "existing content\n");
BlobPrefetcher.AppendToNewlineSeparatedFile(fileSystem, testFileName, "expected line 2");
fileSystem.ReadAllText(testFileName).ShouldEqual("existing content\nexpected line 2\n");
}
private int ExecuteWithExitCode()
{
// CmdParser doesn't strip quotes, and Path.Combine will throw
this.GitBinPath = this.GitBinPath.Replace("\"", string.Empty);
if (!GVFSPlatform.Instance.GitInstallation.GitExists(this.GitBinPath))
{
Console.WriteLine(
"Could not find git.exe {0}",
!string.IsNullOrWhiteSpace(this.GitBinPath) ? "at " + this.GitBinPath : "on %PATH%");
return(ExitFailure);
}
if (this.Commit != null && this.Branch != null)
{
Console.WriteLine("Cannot specify both a commit sha and a branch name.");
return(ExitFailure);
}
if (this.ForceCheckout && !this.Checkout)
{
Console.WriteLine("Cannot use --force-checkout option without --checkout option.");
return(ExitFailure);
}
this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount;
this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads);
this.IndexThreadCount = this.IndexThreadCount > 0 ? this.IndexThreadCount : Environment.ProcessorCount;
this.CheckoutThreadCount = this.CheckoutThreadCount > 0 ? this.CheckoutThreadCount : Environment.ProcessorCount;
this.GitBinPath = !string.IsNullOrWhiteSpace(this.GitBinPath) ? this.GitBinPath : GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath();
GitEnlistment enlistment = GitEnlistment.CreateFromCurrentDirectory(this.GitBinPath);
if (enlistment == null)
{
Console.WriteLine("Must be run within a git repo");
return(ExitFailure);
}
string commitish = this.Commit ?? this.Branch;
if (string.IsNullOrWhiteSpace(commitish))
{
GitProcess.Result result = new GitProcess(enlistment).GetCurrentBranchName();
if (result.ExitCodeIsFailure || string.IsNullOrWhiteSpace(result.Output))
{
Console.WriteLine("Could not retrieve current branch name: " + result.Errors);
return(ExitFailure);
}
commitish = result.Output.Trim();
}
Guid parentActivityId = Guid.Empty;
if (!string.IsNullOrWhiteSpace(this.ParentActivityId) && !Guid.TryParse(this.ParentActivityId, out parentActivityId))
{
Console.WriteLine("The ParentActivityId provided (" + this.ParentActivityId + ") is not a valid GUID.");
}
using (JsonTracer tracer = new JsonTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch", enlistmentId: null, mountId: null, disableTelemetry: true))
{
if (this.Verbose)
{
tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
else
{
tracer.AddPrettyConsoleEventListener(EventLevel.Error, Keywords.Any);
}
string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch");
tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any);
CacheServerInfo cacheServer = new CacheServerInfo(this.GetRemoteUrl(enlistment), null);
tracer.WriteStartEvent(
enlistment.EnlistmentRoot,
enlistment.RepoUrl,
cacheServer.Url,
new EventMetadata
{
{ "TargetCommitish", commitish },
{ "Checkout", this.Checkout },
});
string error;
if (!enlistment.Authentication.TryInitialize(tracer, enlistment, out error))
{
tracer.RelatedError(error);
Console.WriteLine(error);
return(ExitFailure);
}
RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
BlobPrefetcher prefetcher = this.GetFolderPrefetcher(tracer, enlistment, cacheServer, retryConfig);
if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, out error))
{
tracer.RelatedError(error);
Console.WriteLine(error);
return(ExitFailure);
}
bool isSuccess;
try
{
Func <bool> doPrefetch =
() =>
{
try
{
bool isBranch = this.Commit == null;
prefetcher.Prefetch(commitish, isBranch);
return(!prefetcher.HasFailures);
}
catch (BlobPrefetcher.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
isSuccess = doPrefetch();
}
else
{
isSuccess = ConsoleHelper.ShowStatusWhileRunning(
doPrefetch,
"Fetching",
output: Console.Out,
showSpinner: !Console.IsOutputRedirected,
gvfsLogEnlistmentRoot: null);
Console.WriteLine();
Console.WriteLine("See the full log at " + fastfetchLogFile);
}
isSuccess &= !prefetcher.HasFailures;
}
catch (AggregateException e)
{
isSuccess = false;
foreach (Exception ex in e.Flatten().InnerExceptions)
{
tracer.RelatedError(ex.ToString());
}
}
catch (Exception e)
{
isSuccess = false;
tracer.RelatedError(e.ToString());
}
EventMetadata stopMetadata = new EventMetadata();
stopMetadata.Add("Success", isSuccess);
tracer.Stop(stopMetadata);
return(isSuccess ? ExitSuccess : ExitFailure);
}
}
private void PrefetchBlobs(
ITracer tracer,
GVFSEnlistment enlistment,
string headCommitId,
List <string> filesList,
List <string> foldersList,
FileBasedDictionary <string, string> lastPrefetchArgs,
GitObjectsHttpRequestor objectRequestor,
CacheServerInfo cacheServer)
{
BlobPrefetcher blobPrefetcher = new BlobPrefetcher(
tracer,
enlistment,
objectRequestor,
filesList,
foldersList,
lastPrefetchArgs,
ChunkSize,
SearchThreadCount,
DownloadThreadCount,
IndexThreadCount);
if (blobPrefetcher.FolderList.Count == 0 &&
blobPrefetcher.FileList.Count == 0)
{
this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files '*'` to confirm.");
}
if (this.HydrateFiles)
{
if (!this.CheckIsMounted(verbose: true))
{
this.ReportErrorAndExit("You can only specify --hydrate if the repo is mounted. Run 'gvfs mount' and try again.");
}
}
int matchedBlobCount = 0;
int downloadedBlobCount = 0;
int hydratedFileCount = 0;
Func <bool> doPrefetch =
() =>
{
try
{
blobPrefetcher.PrefetchWithStats(
headCommitId,
isBranch: false,
hydrateFilesAfterDownload: this.HydrateFiles,
matchedBlobCount: out matchedBlobCount,
downloadedBlobCount: out downloadedBlobCount,
hydratedFileCount: out hydratedFileCount);
return(!blobPrefetcher.HasFailures);
}
catch (BlobPrefetcher.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
doPrefetch();
}
else
{
string message =
this.HydrateFiles
? "Fetching blobs and hydrating files "
: "Fetching blobs ";
this.ShowStatusWhileRunning(doPrefetch, message + this.GetCacheServerDisplay(cacheServer, enlistment.RepoUrl));
}
if (blobPrefetcher.HasFailures)
{
Environment.ExitCode = 1;
}
else
{
Console.WriteLine();
Console.WriteLine("Stats:");
Console.WriteLine(" Matched blobs: " + matchedBlobCount);
Console.WriteLine(" Already cached: " + (matchedBlobCount - downloadedBlobCount));
Console.WriteLine(" Downloaded: " + downloadedBlobCount);
if (this.HydrateFiles)
{
Console.WriteLine(" Hydrated files: " + hydratedFileCount);
}
}
}
protected override void Execute(GVFSEnlistment enlistment)
{
using (JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "Prefetch"))
{
if (this.Verbose)
{
tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment);
tracer.AddLogFileEventListener(
GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Prefetch),
EventLevel.Informational,
Keywords.Any);
tracer.WriteStartEvent(
enlistment.EnlistmentRoot,
enlistment.RepoUrl,
cacheServerUrl);
try
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Commits", this.Commits);
metadata.Add("Files", this.Files);
metadata.Add("Folders", this.Folders);
metadata.Add("FileListFile", this.FilesListFile);
metadata.Add("FoldersListFile", this.FoldersListFile);
metadata.Add("FilesFromStdIn", this.FilesFromStdIn);
metadata.Add("FoldersFromStdIn", this.FoldersFromStdIn);
metadata.Add("HydrateFiles", this.HydrateFiles);
tracer.RelatedEvent(EventLevel.Informational, "PerformPrefetch", metadata);
if (this.Commits)
{
if (!string.IsNullOrWhiteSpace(this.Files) ||
!string.IsNullOrWhiteSpace(this.Folders) ||
!string.IsNullOrWhiteSpace(this.FoldersListFile) ||
!string.IsNullOrWhiteSpace(this.FilesListFile) ||
this.FilesFromStdIn ||
this.FoldersFromStdIn)
{
this.ReportErrorAndExit(tracer, "You cannot prefetch commits and blobs at the same time.");
}
if (this.HydrateFiles)
{
this.ReportErrorAndExit(tracer, "You can only specify --hydrate with --files or --folders");
}
GitObjectsHttpRequestor objectRequestor;
CacheServerInfo cacheServer;
this.InitializeServerConnection(
tracer,
enlistment,
cacheServerUrl,
out objectRequestor,
out cacheServer);
this.PrefetchCommits(tracer, enlistment, objectRequestor, cacheServer);
}
else
{
string headCommitId;
List <string> filesList;
List <string> foldersList;
FileBasedDictionary <string, string> lastPrefetchArgs;
this.LoadBlobPrefetchArgs(tracer, enlistment, out headCommitId, out filesList, out foldersList, out lastPrefetchArgs);
if (BlobPrefetcher.IsNoopPrefetch(tracer, lastPrefetchArgs, headCommitId, filesList, foldersList, this.HydrateFiles))
{
Console.WriteLine("All requested files are already available. Nothing new to prefetch.");
}
else
{
GitObjectsHttpRequestor objectRequestor;
CacheServerInfo cacheServer;
this.InitializeServerConnection(
tracer,
enlistment,
cacheServerUrl,
out objectRequestor,
out cacheServer);
this.PrefetchBlobs(tracer, enlistment, headCommitId, filesList, foldersList, lastPrefetchArgs, objectRequestor, cacheServer);
}
}
}
catch (VerbAbortedException)
{
throw;
}
catch (AggregateException aggregateException)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
foreach (Exception innerException in aggregateException.Flatten().InnerExceptions)
{
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", innerException.ToString() }
},
$"Unhandled {innerException.GetType().Name}: {innerException.Message}");
}
Environment.ExitCode = (int)ReturnCode.GenericError;
}
catch (Exception e)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", e.ToString() }
},
$"Unhandled {e.GetType().Name}: {e.Message}");
Environment.ExitCode = (int)ReturnCode.GenericError;
}
}
}
private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor blobRequestor, CacheServerInfo cacheServer)
{
BlobPrefetcher blobPrefetcher = new BlobPrefetcher(
tracer,
enlistment,
blobRequestor,
ChunkSize,
SearchThreadCount,
DownloadThreadCount,
IndexThreadCount);
string error;
if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, blobPrefetcher.FolderList, out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (!BlobPrefetcher.TryLoadFileList(enlistment, this.Files, blobPrefetcher.FileList, out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (blobPrefetcher.FolderList.Count == 0 &&
blobPrefetcher.FileList.Count == 0)
{
this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files *` to confirm.");
}
if (this.HydrateFiles)
{
if (!this.CheckIsMounted(verbose: true))
{
this.ReportErrorAndExit("You can only specify --hydrate if the repo is mounted. Run 'gvfs mount' and try again.");
}
}
GitProcess gitProcess = new GitProcess(enlistment);
GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName);
if (result.HasErrors)
{
tracer.RelatedError(result.Errors);
this.Output.WriteLine(result.Errors);
Environment.ExitCode = (int)ReturnCode.GenericError;
return;
}
int matchedBlobCount = 0;
int downloadedBlobCount = 0;
int readFileCount = 0;
string headCommitId = result.Output;
Func <bool> doPrefetch =
() =>
{
try
{
blobPrefetcher.PrefetchWithStats(
headCommitId.Trim(),
isBranch: false,
readFilesAfterDownload: this.HydrateFiles,
matchedBlobCount: out matchedBlobCount,
downloadedBlobCount: out downloadedBlobCount,
readFileCount: out readFileCount);
return(!blobPrefetcher.HasFailures);
}
catch (BlobPrefetcher.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
doPrefetch();
}
else
{
string message =
this.HydrateFiles
? "Fetching blobs and hydrating files "
: "Fetching blobs ";
this.ShowStatusWhileRunning(doPrefetch, message + this.GetCacheServerDisplay(cacheServer, enlistment.RepoUrl));
}
if (blobPrefetcher.HasFailures)
{
Environment.ExitCode = 1;
}
else
{
Console.WriteLine();
Console.WriteLine("Stats:");
Console.WriteLine(" Matched blobs: " + matchedBlobCount);
Console.WriteLine(" Already cached: " + (matchedBlobCount - downloadedBlobCount));
Console.WriteLine(" Downloaded: " + downloadedBlobCount);
if (this.HydrateFiles)
{
Console.WriteLine(" Hydrated files: " + readFileCount);
}
}
}
