private void PostFetchJob(List <string> packIndexes)
{
try
{
using (FileBasedLock postFetchFileLock = new FileBasedLock(
this.context.FileSystem,
this.context.Tracer,
Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock),
this.context.Enlistment.EnlistmentRoot,
overwriteExistingLock: true))
{
if (!postFetchFileLock.TryAcquireLockAndDeleteOnClose())
{
this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping post-fetch work since another process holds the lock");
return;
}
if (!this.gitObjects.TryWriteMultiPackIndex(this.context.Tracer, this.context.Enlistment, this.context.FileSystem))
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Failed to generate midx for new packfiles",
keywords: Keywords.Telemetry);
}
if (packIndexes == null || packIndexes.Count == 0)
{
this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping commit-graph write due to no new packfiles");
return;
}
using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null))
{
GitProcess process = new GitProcess(this.context.Enlistment);
GitProcess.Result result = process.WriteCommitGraph(this.context.Enlistment.GitObjectsRoot, packIndexes);
if (result.HasErrors)
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Failed to generate commit-graph for new packfiles:" + result.Errors,
keywords: Keywords.Telemetry);
return;
}
}
}
}
catch (ThreadAbortException)
{
this.context.Tracer.RelatedInfo("Aborting post-fetch job due to ThreadAbortException");
}
catch (Exception e)
{
this.context.Tracer.RelatedError(
metadata: null,
message: PostFetchTelemetryKey + ": Exception while running post-fetch job: " + e.Message,
keywords: Keywords.Telemetry);
Environment.Exit((int)ReturnCode.GenericError);
}
}
public void Initialize()
{
this.gitIndexLock = new FileBasedLock(
new PhysicalFileSystem(),
this.tracer,
this.lockPath,
"GVFS",
FileBasedLock.ExistingLockCleanup.DeleteExisting);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
if (this.gitIndexLock != null)
{
this.gitIndexLock.Dispose();
this.gitIndexLock = null;
}
}
}
public void CreateLockWhenDirectoryMissing()
{
string parentPath = Path.Combine("mock:", "path", "to");
string lockPath = Path.Combine(parentPath, "lock");
MockTracer tracer = new MockTracer();
FileBasedLockFileSystem fs = new FileBasedLockFileSystem();
FileBasedLock fileBasedLock = new FileBasedLock(fs, tracer, lockPath, "signature", overwriteExistingLock: true);
fileBasedLock.TryAcquireLockAndDeleteOnClose().ShouldBeTrue();
fs.CreateDirectoryPath.ShouldNotBeNull();
fs.CreateDirectoryPath.ShouldEqual(parentPath);
}
public void Execute()
{
try
{
if (this.RequireObjectCacheLock)
{
using (FileBasedLock cacheLock = GVFSPlatform.Instance.CreateFileBasedLock(
this.Context.FileSystem,
this.Context.Tracer,
Path.Combine(this.Context.Enlistment.GitObjectsRoot, ObjectCacheLock)))
{
if (!cacheLock.TryAcquireLock())
{
this.Context.Tracer.RelatedInfo(this.Area + ": Skipping work since another process holds the lock");
return;
}
this.CreateProcessAndRun();
}
}
else
{
this.CreateProcessAndRun();
}
}
catch (IOException e)
{
this.Context.Tracer.RelatedWarning(
metadata: this.CreateEventMetadata(e),
message: "IOException while running action: " + e.Message,
keywords: Keywords.Telemetry);
}
catch (Exception e)
{
if (this.EnlistmentRootReady())
{
this.Context.Tracer.RelatedError(
metadata: this.CreateEventMetadata(e),
message: "Exception while running action: " + e.Message,
keywords: Keywords.Telemetry);
}
else
{
this.Context.Tracer.RelatedWarning(
metadata: this.CreateEventMetadata(e),
message: "Exception while running action inside a repo that's not ready: " + e.Message);
}
Environment.Exit((int)ReturnCode.GenericError);
}
}
private bool TryFetchUsingGvfsProtocol(GitProcess gitProcess, out string error)
{
if (!this.TryGetMaxGoodPrefetchPackTimestamp(out long last, out error))
{
this.Context.Tracer.RelatedError(error);
return(false);
}
TimeSpan timeBetween = this.GitObjects.IsUsingCacheServer()
? this.timeBetweenFetches
: this.timeBetweenFetchesNoCacheServer;
DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last);
DateTime now = DateTime.UtcNow;
if (!this.forceRun && now <= lastDateTime + timeBetween)
{
this.Context.Tracer.RelatedInfo(this.Area + ": Skipping fetch since most-recent fetch ({0}) is too close to now ({1})", lastDateTime, now);
error = null;
return(true);
}
// We take our own lock here to keep background and foreground fetches
// (i.e. a user running 'scalar run fetch')
// from running at the same time.
using (FileBasedLock fetchLock = ScalarPlatform.Instance.CreateFileBasedLock(
this.Context.FileSystem,
this.Context.Tracer,
Path.Combine(this.Context.Enlistment.GitPackRoot, FetchCommitsAndTreesLock)))
{
WaitUntilLockIsAcquired(this.Context.Tracer, fetchLock);
this.GitObjects.DeleteStaleTempPrefetchPackAndIdxs();
this.GitObjects.DeleteTemporaryFiles();
GitProcess.Result result = gitProcess.GvfsHelperPrefetch();
if (result.ExitCodeIsFailure)
{
error = result.Errors;
return(false);
}
this.UpdateKeepPacks();
}
error = null;
return(true);
}
private void WaitUntilLockIsAcquired(ITracer tracer, FileBasedLock fileBasedLock)
{
int attempt = 0;
while (!fileBasedLock.TryAcquireLockAndDeleteOnClose())
{
Thread.Sleep(LockWaitTimeMs);
++attempt;
if (attempt == WaitingOnLockLogThreshold)
{
attempt = 0;
tracer.RelatedInfo("WaitUntilLockIsAcquired: Waiting to acquire prefetch lock");
}
}
}
public bool TryPrefetchCommitsAndTrees(out string error, GitProcess gitProcess = null)
{
if (gitProcess == null)
{
gitProcess = new GitProcess(this.Context.Enlistment);
}
List <string> packIndexes;
// We take our own lock here to keep background and foreground prefetches
// from running at the same time.
using (FileBasedLock prefetchLock = GSDPlatform.Instance.CreateFileBasedLock(
this.Context.FileSystem,
this.Context.Tracer,
Path.Combine(this.Context.Enlistment.GitPackRoot, PrefetchCommitsAndTreesLock)))
{
WaitUntilLockIsAcquired(this.Context.Tracer, prefetchLock);
long maxGoodTimeStamp;
this.GitObjects.DeleteStaleTempPrefetchPackAndIdxs();
this.GitObjects.DeleteTemporaryFiles();
if (!this.TryGetMaxGoodPrefetchTimestamp(out maxGoodTimeStamp, out error))
{
return(false);
}
if (!this.GitObjects.TryDownloadPrefetchPacks(gitProcess, maxGoodTimeStamp, out packIndexes))
{
error = "Failed to download prefetch packs";
return(false);
}
this.UpdateKeepPacks();
}
this.SchedulePostFetchJob(packIndexes);
return(true);
}
public static bool TryPrefetchCommitsAndTrees(
ITracer tracer,
GVFSEnlistment enlistment,
PhysicalFileSystem fileSystem,
GitObjects gitObjects,
out string error)
{
List <string> packIndexes;
using (FileBasedLock prefetchLock = GVFSPlatform.Instance.CreateFileBasedLock(
fileSystem,
tracer,
Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock)))
{
WaitUntilLockIsAcquired(tracer, prefetchLock);
long maxGoodTimeStamp;
gitObjects.DeleteStaleTempPrefetchPackAndIdxs();
if (!TryGetMaxGoodPrefetchTimestamp(tracer, enlistment, fileSystem, gitObjects, out maxGoodTimeStamp, out error))
{
return(false);
}
if (!gitObjects.TryDownloadPrefetchPacks(maxGoodTimeStamp, out packIndexes))
{
error = "Failed to download prefetch packs";
return(false);
}
}
if (packIndexes?.Count > 0)
{
TrySchedulePostFetchJob(tracer, enlistment.NamedPipeName, packIndexes);
}
return(true);
}
protected override void Execute(GVFSEnlistment enlistment)
{
using (JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "Prefetch"))
{
if (this.Verbose)
{
tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment);
tracer.AddLogFileEventListener(
GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Prefetch),
EventLevel.Informational,
Keywords.Any);
tracer.WriteStartEvent(
enlistment.EnlistmentRoot,
enlistment.RepoUrl,
cacheServerUrl);
RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
CacheServerInfo cacheServer = this.ResolvedCacheServer;
GVFSConfig gvfsConfig = this.GVFSConfig;
if (!this.SkipVersionCheck)
{
string authErrorMessage;
if (!this.ShowStatusWhileRunning(
() => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage),
"Authenticating"))
{
this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed");
}
if (gvfsConfig == null)
{
gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig);
}
if (cacheServer == null)
{
CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment);
cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig);
}
this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: false);
this.Output.WriteLine("Configured cache server: " + cacheServer);
}
this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig, cacheServer);
try
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Commits", this.Commits);
metadata.Add("Files", this.Files);
metadata.Add("Folders", this.Folders);
metadata.Add("FoldersListFile", this.FoldersListFile);
metadata.Add("HydrateFiles", this.HydrateFiles);
tracer.RelatedEvent(EventLevel.Informational, "PerformPrefetch", metadata);
GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig);
if (this.Commits)
{
if (!string.IsNullOrWhiteSpace(this.Files) ||
!string.IsNullOrWhiteSpace(this.Folders) ||
!string.IsNullOrWhiteSpace(this.FoldersListFile))
{
this.ReportErrorAndExit(tracer, "You cannot prefetch commits and blobs at the same time.");
}
if (this.HydrateFiles)
{
this.ReportErrorAndExit(tracer, "You can only specify --hydrate with --files or --folders");
}
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
using (FileBasedLock prefetchLock = new FileBasedLock(
fileSystem,
tracer,
Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock),
enlistment.EnlistmentRoot,
overwriteExistingLock: true))
{
this.WaitUntilLockIsAcquired(tracer, prefetchLock);
this.PrefetchCommits(tracer, enlistment, objectRequestor, cacheServer);
}
}
else
{
this.PrefetchBlobs(tracer, enlistment, objectRequestor, cacheServer);
}
}
catch (VerbAbortedException)
{
throw;
}
catch (AggregateException aggregateException)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
foreach (Exception innerException in aggregateException.Flatten().InnerExceptions)
{
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", innerException.ToString() }
},
$"Unhandled {innerException.GetType().Name}: {innerException.Message}");
}
Environment.ExitCode = (int)ReturnCode.GenericError;
}
catch (Exception e)
{
this.Output.WriteLine(
"Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot),
enlistment.EnlistmentRoot);
tracer.RelatedError(
new EventMetadata
{
{ "Verb", typeof(PrefetchVerb).Name },
{ "Exception", e.ToString() }
},
$"Unhandled {e.GetType().Name}: {e.Message}");
Environment.ExitCode = (int)ReturnCode.GenericError;
}
}
}
private void PostFetchJob(List <string> packIndexes)
{
try
{
using (FileBasedLock postFetchFileLock = GVFSPlatform.Instance.CreateFileBasedLock(
this.context.FileSystem,
this.context.Tracer,
Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock)))
{
if (!postFetchFileLock.TryAcquireLock())
{
this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping post-fetch work since another process holds the lock");
return;
}
this.postFetchGitProcess = new GitProcess(this.context.Enlistment);
using (ITracer activity = this.context.Tracer.StartActivity("TryWriteMultiPackIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null))
{
string midxLockFile = Path.Combine(this.context.Enlistment.GitPackRoot, MultiPackIndexLock);
this.context.FileSystem.TryDeleteFile(midxLockFile);
if (this.stopping)
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Not launching 'git multi-pack-index' because the mount is stopping",
keywords: Keywords.Telemetry);
return;
}
GitProcess.Result result = this.postFetchGitProcess.WriteMultiPackIndex(this.context.Enlistment.GitObjectsRoot);
if (!this.stopping && result.ExitCodeIsFailure)
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Failed to generate multi-pack-index for new packfiles:" + result.Errors,
keywords: Keywords.Telemetry);
return;
}
}
if (packIndexes == null || packIndexes.Count == 0)
{
this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping commit-graph write due to no new packfiles");
return;
}
using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null))
{
string graphLockFile = Path.Combine(this.context.Enlistment.GitObjectsRoot, "info", CommitGraphLock);
this.context.FileSystem.TryDeleteFile(graphLockFile);
if (this.stopping)
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Not launching 'git commit-graph' because the mount is stopping",
keywords: Keywords.Telemetry);
return;
}
GitProcess.Result result = this.postFetchGitProcess.WriteCommitGraph(this.context.Enlistment.GitObjectsRoot, packIndexes);
if (!this.stopping && result.ExitCodeIsFailure)
{
this.context.Tracer.RelatedWarning(
metadata: null,
message: PostFetchTelemetryKey + ": Failed to generate commit-graph for new packfiles:" + result.Errors,
keywords: Keywords.Telemetry);
return;
}
}
}
}
catch (IOException e)
{
this.context.Tracer.RelatedWarning(
metadata: this.CreateEventMetadata(null, e),
message: PostFetchTelemetryKey + ": IOException while running post-fetch job: " + e.Message,
keywords: Keywords.Telemetry);
}
catch (Exception e)
{
this.context.Tracer.RelatedError(
metadata: this.CreateEventMetadata(null, e),
message: PostFetchTelemetryKey + ": Exception while running post-fetch job: " + e.Message,
keywords: Keywords.Telemetry);
Environment.Exit((int)ReturnCode.GenericError);
}
}
public static bool TryPrefetchCommitsAndTrees(
ITracer tracer,
GVFSEnlistment enlistment,
PhysicalFileSystem fileSystem,
GitObjects gitObjects,
out string error)
{
List <string> packIndexes;
using (FileBasedLock prefetchLock = new FileBasedLock(
fileSystem,
tracer,
Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock),
enlistment.EnlistmentRoot,
overwriteExistingLock: true))
{
WaitUntilLockIsAcquired(tracer, prefetchLock);
long maxGoodTimeStamp;
gitObjects.DeleteStaleTempPrefetchPackAndIdxs();
if (!TryGetMaxGoodPrefetchTimestamp(tracer, enlistment, fileSystem, gitObjects, out maxGoodTimeStamp, out error))
{
return(false);
}
if (!gitObjects.TryDownloadPrefetchPacks(maxGoodTimeStamp, out packIndexes))
{
error = "Failed to download prefetch packs";
return(false);
}
}
if (packIndexes == null || packIndexes.Count == 0)
{
return(true);
}
// We make a best-effort request to run MIDX and commit-graph writes
using (NamedPipeClient pipeClient = new NamedPipeClient(enlistment.NamedPipeName))
{
if (!pipeClient.Connect())
{
tracer.RelatedWarning(
metadata: null,
message: "Failed to connect to GVFS. Skipping post-fetch job request.",
keywords: Keywords.Telemetry);
return(true);
}
NamedPipeMessages.RunPostFetchJob.Request request = new NamedPipeMessages.RunPostFetchJob.Request(packIndexes);
if (pipeClient.TrySendRequest(request.CreateMessage()))
{
NamedPipeMessages.Message response;
if (pipeClient.TryReadResponse(out response))
{
tracer.RelatedInfo("Requested post-fetch job with resonse '{0}'", response.Header);
return(true);
}
else
{
tracer.RelatedWarning(
metadata: null,
message: "Requested post-fetch job failed to respond",
keywords: Keywords.Telemetry);
}
}
else
{
tracer.RelatedWarning(
metadata: null,
message: "Message to named pipe failed to send, skipping post-fetch job request.",
keywords: Keywords.Telemetry);
}
}
return(false);
}
