private void PostFetchJob(List <string> packIndexes) { try { using (FileBasedLock postFetchFileLock = new FileBasedLock( this.context.FileSystem, this.context.Tracer, Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock), this.context.Enlistment.EnlistmentRoot, overwriteExistingLock: true)) { if (!postFetchFileLock.TryAcquireLockAndDeleteOnClose()) { this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping post-fetch work since another process holds the lock"); return; } if (!this.gitObjects.TryWriteMultiPackIndex(this.context.Tracer, this.context.Enlistment, this.context.FileSystem)) { this.context.Tracer.RelatedWarning( metadata: null, message: PostFetchTelemetryKey + ": Failed to generate midx for new packfiles", keywords: Keywords.Telemetry); } if (packIndexes == null || packIndexes.Count == 0) { this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping commit-graph write due to no new packfiles"); return; } using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { GitProcess process = new GitProcess(this.context.Enlistment); GitProcess.Result result = process.WriteCommitGraph(this.context.Enlistment.GitObjectsRoot, packIndexes); if (result.HasErrors) { this.context.Tracer.RelatedWarning( metadata: null, message: PostFetchTelemetryKey + ": Failed to generate commit-graph for new packfiles:" + result.Errors, keywords: Keywords.Telemetry); return; } } } } catch (ThreadAbortException) { this.context.Tracer.RelatedInfo("Aborting post-fetch job due to ThreadAbortException"); } catch (Exception e) { this.context.Tracer.RelatedError( metadata: null, message: PostFetchTelemetryKey + ": Exception while running post-fetch job: " + e.Message, keywords: Keywords.Telemetry); Environment.Exit((int)ReturnCode.GenericError); } }
public void Initialize() { this.gitIndexLock = new FileBasedLock( new PhysicalFileSystem(), this.tracer, this.lockPath, "GVFS", FileBasedLock.ExistingLockCleanup.DeleteExisting); }
protected virtual void Dispose(bool disposing) { if (disposing) { if (this.gitIndexLock != null) { this.gitIndexLock.Dispose(); this.gitIndexLock = null; } } }
public void CreateLockWhenDirectoryMissing() { string parentPath = Path.Combine("mock:", "path", "to"); string lockPath = Path.Combine(parentPath, "lock"); MockTracer tracer = new MockTracer(); FileBasedLockFileSystem fs = new FileBasedLockFileSystem(); FileBasedLock fileBasedLock = new FileBasedLock(fs, tracer, lockPath, "signature", overwriteExistingLock: true); fileBasedLock.TryAcquireLockAndDeleteOnClose().ShouldBeTrue(); fs.CreateDirectoryPath.ShouldNotBeNull(); fs.CreateDirectoryPath.ShouldEqual(parentPath); }
public void Execute() { try { if (this.RequireObjectCacheLock) { using (FileBasedLock cacheLock = GVFSPlatform.Instance.CreateFileBasedLock( this.Context.FileSystem, this.Context.Tracer, Path.Combine(this.Context.Enlistment.GitObjectsRoot, ObjectCacheLock))) { if (!cacheLock.TryAcquireLock()) { this.Context.Tracer.RelatedInfo(this.Area + ": Skipping work since another process holds the lock"); return; } this.CreateProcessAndRun(); } } else { this.CreateProcessAndRun(); } } catch (IOException e) { this.Context.Tracer.RelatedWarning( metadata: this.CreateEventMetadata(e), message: "IOException while running action: " + e.Message, keywords: Keywords.Telemetry); } catch (Exception e) { if (this.EnlistmentRootReady()) { this.Context.Tracer.RelatedError( metadata: this.CreateEventMetadata(e), message: "Exception while running action: " + e.Message, keywords: Keywords.Telemetry); } else { this.Context.Tracer.RelatedWarning( metadata: this.CreateEventMetadata(e), message: "Exception while running action inside a repo that's not ready: " + e.Message); } Environment.Exit((int)ReturnCode.GenericError); } }
private bool TryFetchUsingGvfsProtocol(GitProcess gitProcess, out string error) { if (!this.TryGetMaxGoodPrefetchPackTimestamp(out long last, out error)) { this.Context.Tracer.RelatedError(error); return(false); } TimeSpan timeBetween = this.GitObjects.IsUsingCacheServer() ? this.timeBetweenFetches : this.timeBetweenFetchesNoCacheServer; DateTime lastDateTime = EpochConverter.FromUnixEpochSeconds(last); DateTime now = DateTime.UtcNow; if (!this.forceRun && now <= lastDateTime + timeBetween) { this.Context.Tracer.RelatedInfo(this.Area + ": Skipping fetch since most-recent fetch ({0}) is too close to now ({1})", lastDateTime, now); error = null; return(true); } // We take our own lock here to keep background and foreground fetches // (i.e. a user running 'scalar run fetch') // from running at the same time. using (FileBasedLock fetchLock = ScalarPlatform.Instance.CreateFileBasedLock( this.Context.FileSystem, this.Context.Tracer, Path.Combine(this.Context.Enlistment.GitPackRoot, FetchCommitsAndTreesLock))) { WaitUntilLockIsAcquired(this.Context.Tracer, fetchLock); this.GitObjects.DeleteStaleTempPrefetchPackAndIdxs(); this.GitObjects.DeleteTemporaryFiles(); GitProcess.Result result = gitProcess.GvfsHelperPrefetch(); if (result.ExitCodeIsFailure) { error = result.Errors; return(false); } this.UpdateKeepPacks(); } error = null; return(true); }
private void WaitUntilLockIsAcquired(ITracer tracer, FileBasedLock fileBasedLock) { int attempt = 0; while (!fileBasedLock.TryAcquireLockAndDeleteOnClose()) { Thread.Sleep(LockWaitTimeMs); ++attempt; if (attempt == WaitingOnLockLogThreshold) { attempt = 0; tracer.RelatedInfo("WaitUntilLockIsAcquired: Waiting to acquire prefetch lock"); } } }
public bool TryPrefetchCommitsAndTrees(out string error, GitProcess gitProcess = null) { if (gitProcess == null) { gitProcess = new GitProcess(this.Context.Enlistment); } List <string> packIndexes; // We take our own lock here to keep background and foreground prefetches // from running at the same time. using (FileBasedLock prefetchLock = GSDPlatform.Instance.CreateFileBasedLock( this.Context.FileSystem, this.Context.Tracer, Path.Combine(this.Context.Enlistment.GitPackRoot, PrefetchCommitsAndTreesLock))) { WaitUntilLockIsAcquired(this.Context.Tracer, prefetchLock); long maxGoodTimeStamp; this.GitObjects.DeleteStaleTempPrefetchPackAndIdxs(); this.GitObjects.DeleteTemporaryFiles(); if (!this.TryGetMaxGoodPrefetchTimestamp(out maxGoodTimeStamp, out error)) { return(false); } if (!this.GitObjects.TryDownloadPrefetchPacks(gitProcess, maxGoodTimeStamp, out packIndexes)) { error = "Failed to download prefetch packs"; return(false); } this.UpdateKeepPacks(); } this.SchedulePostFetchJob(packIndexes); return(true); }
public static bool TryPrefetchCommitsAndTrees( ITracer tracer, GVFSEnlistment enlistment, PhysicalFileSystem fileSystem, GitObjects gitObjects, out string error) { List <string> packIndexes; using (FileBasedLock prefetchLock = GVFSPlatform.Instance.CreateFileBasedLock( fileSystem, tracer, Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock))) { WaitUntilLockIsAcquired(tracer, prefetchLock); long maxGoodTimeStamp; gitObjects.DeleteStaleTempPrefetchPackAndIdxs(); if (!TryGetMaxGoodPrefetchTimestamp(tracer, enlistment, fileSystem, gitObjects, out maxGoodTimeStamp, out error)) { return(false); } if (!gitObjects.TryDownloadPrefetchPacks(maxGoodTimeStamp, out packIndexes)) { error = "Failed to download prefetch packs"; return(false); } } if (packIndexes?.Count > 0) { TrySchedulePostFetchJob(tracer, enlistment.NamedPipeName, packIndexes); } return(true); }
protected override void Execute(GVFSEnlistment enlistment) { using (JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "Prefetch")) { if (this.Verbose) { tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any); } string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment); tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Prefetch), EventLevel.Informational, Keywords.Any); tracer.WriteStartEvent( enlistment.EnlistmentRoot, enlistment.RepoUrl, cacheServerUrl); RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); CacheServerInfo cacheServer = this.ResolvedCacheServer; GVFSConfig gvfsConfig = this.GVFSConfig; if (!this.SkipVersionCheck) { string authErrorMessage; if (!this.ShowStatusWhileRunning( () => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage), "Authenticating")) { this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed"); } if (gvfsConfig == null) { gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } if (cacheServer == null) { CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig); } this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: false); this.Output.WriteLine("Configured cache server: " + cacheServer); } this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig, cacheServer); try { EventMetadata metadata = new EventMetadata(); metadata.Add("Commits", this.Commits); metadata.Add("Files", this.Files); metadata.Add("Folders", this.Folders); metadata.Add("FoldersListFile", this.FoldersListFile); metadata.Add("HydrateFiles", this.HydrateFiles); tracer.RelatedEvent(EventLevel.Informational, "PerformPrefetch", metadata); GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig); if (this.Commits) { if (!string.IsNullOrWhiteSpace(this.Files) || !string.IsNullOrWhiteSpace(this.Folders) || !string.IsNullOrWhiteSpace(this.FoldersListFile)) { this.ReportErrorAndExit(tracer, "You cannot prefetch commits and blobs at the same time."); } if (this.HydrateFiles) { this.ReportErrorAndExit(tracer, "You can only specify --hydrate with --files or --folders"); } PhysicalFileSystem fileSystem = new PhysicalFileSystem(); using (FileBasedLock prefetchLock = new FileBasedLock( fileSystem, tracer, Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock), enlistment.EnlistmentRoot, overwriteExistingLock: true)) { this.WaitUntilLockIsAcquired(tracer, prefetchLock); this.PrefetchCommits(tracer, enlistment, objectRequestor, cacheServer); } } else { this.PrefetchBlobs(tracer, enlistment, objectRequestor, cacheServer); } } catch (VerbAbortedException) { throw; } catch (AggregateException aggregateException) { this.Output.WriteLine( "Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot), enlistment.EnlistmentRoot); foreach (Exception innerException in aggregateException.Flatten().InnerExceptions) { tracer.RelatedError( new EventMetadata { { "Verb", typeof(PrefetchVerb).Name }, { "Exception", innerException.ToString() } }, $"Unhandled {innerException.GetType().Name}: {innerException.Message}"); } Environment.ExitCode = (int)ReturnCode.GenericError; } catch (Exception e) { this.Output.WriteLine( "Cannot prefetch {0}. " + ConsoleHelper.GetGVFSLogMessage(enlistment.EnlistmentRoot), enlistment.EnlistmentRoot); tracer.RelatedError( new EventMetadata { { "Verb", typeof(PrefetchVerb).Name }, { "Exception", e.ToString() } }, $"Unhandled {e.GetType().Name}: {e.Message}"); Environment.ExitCode = (int)ReturnCode.GenericError; } } }
private void PostFetchJob(List <string> packIndexes) { try { using (FileBasedLock postFetchFileLock = GVFSPlatform.Instance.CreateFileBasedLock( this.context.FileSystem, this.context.Tracer, Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock))) { if (!postFetchFileLock.TryAcquireLock()) { this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping post-fetch work since another process holds the lock"); return; } this.postFetchGitProcess = new GitProcess(this.context.Enlistment); using (ITracer activity = this.context.Tracer.StartActivity("TryWriteMultiPackIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { string midxLockFile = Path.Combine(this.context.Enlistment.GitPackRoot, MultiPackIndexLock); this.context.FileSystem.TryDeleteFile(midxLockFile); if (this.stopping) { this.context.Tracer.RelatedWarning( metadata: null, message: PostFetchTelemetryKey + ": Not launching 'git multi-pack-index' because the mount is stopping", keywords: Keywords.Telemetry); return; } GitProcess.Result result = this.postFetchGitProcess.WriteMultiPackIndex(this.context.Enlistment.GitObjectsRoot); if (!this.stopping && result.ExitCodeIsFailure) { this.context.Tracer.RelatedWarning( metadata: null, message: PostFetchTelemetryKey + ": Failed to generate multi-pack-index for new packfiles:" + result.Errors, keywords: Keywords.Telemetry); return; } } if (packIndexes == null || packIndexes.Count == 0) { this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping commit-graph write due to no new packfiles"); return; } using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { string graphLockFile = Path.Combine(this.context.Enlistment.GitObjectsRoot, "info", CommitGraphLock); this.context.FileSystem.TryDeleteFile(graphLockFile); if (this.stopping) { this.context.Tracer.RelatedWarning( metadata: null, message: PostFetchTelemetryKey + ": Not launching 'git commit-graph' because the mount is stopping", keywords: Keywords.Telemetry); return; } GitProcess.Result result = this.postFetchGitProcess.WriteCommitGraph(this.context.Enlistment.GitObjectsRoot, packIndexes); if (!this.stopping && result.ExitCodeIsFailure) { this.context.Tracer.RelatedWarning( metadata: null, message: PostFetchTelemetryKey + ": Failed to generate commit-graph for new packfiles:" + result.Errors, keywords: Keywords.Telemetry); return; } } } } catch (IOException e) { this.context.Tracer.RelatedWarning( metadata: this.CreateEventMetadata(null, e), message: PostFetchTelemetryKey + ": IOException while running post-fetch job: " + e.Message, keywords: Keywords.Telemetry); } catch (Exception e) { this.context.Tracer.RelatedError( metadata: this.CreateEventMetadata(null, e), message: PostFetchTelemetryKey + ": Exception while running post-fetch job: " + e.Message, keywords: Keywords.Telemetry); Environment.Exit((int)ReturnCode.GenericError); } }
public static bool TryPrefetchCommitsAndTrees( ITracer tracer, GVFSEnlistment enlistment, PhysicalFileSystem fileSystem, GitObjects gitObjects, out string error) { List <string> packIndexes; using (FileBasedLock prefetchLock = new FileBasedLock( fileSystem, tracer, Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock), enlistment.EnlistmentRoot, overwriteExistingLock: true)) { WaitUntilLockIsAcquired(tracer, prefetchLock); long maxGoodTimeStamp; gitObjects.DeleteStaleTempPrefetchPackAndIdxs(); if (!TryGetMaxGoodPrefetchTimestamp(tracer, enlistment, fileSystem, gitObjects, out maxGoodTimeStamp, out error)) { return(false); } if (!gitObjects.TryDownloadPrefetchPacks(maxGoodTimeStamp, out packIndexes)) { error = "Failed to download prefetch packs"; return(false); } } if (packIndexes == null || packIndexes.Count == 0) { return(true); } // We make a best-effort request to run MIDX and commit-graph writes using (NamedPipeClient pipeClient = new NamedPipeClient(enlistment.NamedPipeName)) { if (!pipeClient.Connect()) { tracer.RelatedWarning( metadata: null, message: "Failed to connect to GVFS. Skipping post-fetch job request.", keywords: Keywords.Telemetry); return(true); } NamedPipeMessages.RunPostFetchJob.Request request = new NamedPipeMessages.RunPostFetchJob.Request(packIndexes); if (pipeClient.TrySendRequest(request.CreateMessage())) { NamedPipeMessages.Message response; if (pipeClient.TryReadResponse(out response)) { tracer.RelatedInfo("Requested post-fetch job with resonse '{0}'", response.Header); return(true); } else { tracer.RelatedWarning( metadata: null, message: "Requested post-fetch job failed to respond", keywords: Keywords.Telemetry); } } else { tracer.RelatedWarning( metadata: null, message: "Message to named pipe failed to send, skipping post-fetch job request.", keywords: Keywords.Telemetry); } } return(false); }