public async Task <PlaceFileResult> CreateTempAndPutAsync( OperationContext context, ContentHash contentHash, IContentSession contentSession) { using (var disposableFile = new DisposableFile(context, _fileSystem, AbsolutePath.CreateRandomFileName(_rootPath / "temp"))) { PlaceFileResult placeTempFileResult = await PlaceFileAsync(context, contentHash, disposableFile.Path, FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.HardLink, context.Token); if (!placeTempFileResult.Succeeded) { return(placeTempFileResult); } PutResult putFileResult = await contentSession.PutFileAsync(context, contentHash, disposableFile.Path, FileRealizationMode.Any, context.Token); if (!putFileResult) { return(new PlaceFileResult(putFileResult)); } else { return(new PlaceFileResult(PlaceFileResult.ResultCode.PlacedWithCopy, putFileResult.ContentSize)); } } }
private Task <PlaceFileResponse> PlaceFileAsync(PlaceFileRequest request, CancellationToken token) { return(RunFuncAsync( request.Header, async(context, session) => { PlaceFileResult placeFileResult = await session.PlaceFileAsync( context.OperationContext, request.ContentHash.ToContentHash((HashType)request.HashType), new AbsolutePath(request.Path), (FileAccessMode)request.FileAccessMode, FileReplacementMode.ReplaceExisting, // Hard-coded because the service can't tell if this is a retry (where the previous try may have left a partial file) (FileRealizationMode)request.FileRealizationMode, token); return new PlaceFileResponse { Header = new ResponseHeader( context.StartTime, placeFileResult.Succeeded, (int)placeFileResult.Code, placeFileResult.ErrorMessage, placeFileResult.Diagnostics), ContentSize = placeFileResult.FileSize }; }, (context, errorMessage) => new PlaceFileResponse { Header = ResponseHeader.Failure(context.StartTime, (int)PlaceFileResult.ResultCode.Error, errorMessage) }, token)); }
public static PlaceFileResult ShouldBeSuccess(this PlaceFileResult result) { Assert.NotNull(result); Assert.True(result.Succeeded, $"Place file operation should succeed, but it failed. Error: {result.ErrorMessage}. Diagnostics: {result.Diagnostics}"); Assert.Null(result.ErrorMessage); return(result); }
public override void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, Severity successSeverity) { if (_eventSource.IsEnabled()) { _eventSource.PlaceFileStop(context.TraceId, (int)result.Code, result.ErrorMessage); } base.PlaceFileStop(context, contentHash, result, path, accessMode, replacementMode, realizationMode, successSeverity: DiagnosticLevelSeverity(result.Duration)); }
public virtual void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode) { if (context.IsEnabled) { TracerOperationFinished(context, result, $"{Name}.{PlaceFileCallName}({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode}) stop {result.DurationMs}ms result=[{result}]"); } _placeFileCallCounter.Completed(result.Duration.Ticks); }
public override void PlaceFileStop(Context context, ContentHash input, PlaceFileResult result) { if (_eventSource.IsEnabled()) { _eventSource.PlaceFileStop(context.Id.ToString(), (int)result.Code, result.ErrorMessage); } base.PlaceFileStop(context, input, result); }
public override void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode) { if (_eventSource.IsEnabled()) { _eventSource.PlaceFileStop(context.Id.ToString(), (int)result.Code, result.ErrorMessage); } base.PlaceFileStop(context, contentHash, result, path, accessMode, replacementMode, realizationMode); }
public virtual void PlaceFileStop(Context context, ContentHash input, PlaceFileResult result) { if (context.IsEnabled) { TracerOperationFinished(context, result, $"{Name}.{PlaceFileCallName} stop {result.DurationMs}ms input=[{input}] result=[{result}]"); } _placeFileCallCounter.Completed(result.Duration.Ticks); }
public async Task PlaceFileRequiringNewReplicaCloseToHardLimitDoesNotHang() { var context = new Context(Logger); using (DisposableDirectory testDirectory = new DisposableDirectory(FileSystem)) { #pragma warning disable AsyncFixer04 // A disposable object used in a fire & forget async call Task testTask = TestStore(context, Clock, testDirectory, async store => #pragma warning restore AsyncFixer04 // A disposable object used in a fire & forget async call { // Make a file which will overflow the cache size with just 2 copies. PutResult putResult = await store.PutRandomAsync(context, ContentSizeToStartHardPurging(2)); ResultTestExtensions.ShouldBeSuccess((BoolResult)putResult); ContentHash hash = putResult.ContentHash; // Hardlink the file out 1024 times. Since the limit is 1024 total, and we already have 1 in the CAS, // this will overflow the links and cause the CAS to create a new replica for it. This will cause // the purger to consider that hash for eviction *while making room for that hash*, which is the // trigger for the previous deadlock that this test will now guard against. for (int i = 0; i < 1024; i++) { PlaceFileResult placeResult = await store.PlaceFileAsync( context, hash, testDirectory.Path / $"hardlink{i}.txt", FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.HardLink, null); // The checks below are just to make sure that the calls completed as expected. // The most important part is that they complete *at all*, which is enforced by // racing against the Task.Delay in the outer scope. if (i < 1023 || SucceedsEvenIfFull) { // The first 1023 links should succeed (bringing it up to the limit of 1024) // And *all* of the calls should succeed if the cache takes new content even when overflowed. Assert.True(placeResult.Succeeded); } else { // If the implementation rejects overflowing content, then the last call should fail. Assert.False(placeResult.Succeeded); Assert.Contains("Failed to reserve space", placeResult.ErrorMessage); } } }); // Race between the test and a 2-minute timer. This can be increased if the test ends up flaky. Task firstCompletedTask = await Task.WhenAny(testTask, Task.Delay(TimeSpan.FromMinutes(2))); // The test should finish first, long before a minute passes, but it won't if it deadlocks. Assert.True(firstCompletedTask == testTask); await firstCompletedTask; } }
private Task PlaceFileRecoversAsync( Context context, Func <TestFileSystemContentStoreInternal, DisposableDirectory, Task <byte[]> > corruptFunc, FileRealizationMode fileRealizationMode) { return(TestStore(context, _clock, async store => { using (var tempDirectory = new DisposableDirectory(FileSystem)) { var bytes = await corruptFunc(store, tempDirectory); var contentHash = bytes.CalculateHash(ContentHashType); var placePath = tempDirectory.CreateRandomFileName(); PlaceFileResult result = null; Func <Task> putFunc = async() => { result = await store.PlaceFileAsync( context, contentHash, placePath, FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.HardLink, null); }; putFunc.Should().NotThrow(); Assert.NotNull(result); result.Code.Should() .Be(fileRealizationMode == FileRealizationMode.Copy ? PlaceFileResult.ResultCode.PlacedWithCopy : PlaceFileResult.ResultCode.PlacedWithHardLink); FileSystem.GetHardLinkCount(placePath) .Should() .Be(fileRealizationMode == FileRealizationMode.Copy ? 1 : 2); } })); }
/// <summary> /// Implements a place file request. /// </summary> private async Task <PlaceFileResponse> PlaceFileAsync(PlaceFileRequest request, CancellationToken token) { LogRequestHandling(); DateTime startTime = DateTime.UtcNow; var cacheContext = new Context(new Guid(request.Header.TraceId), _logger); return(await RunFuncAndReportAsync( request.Header.SessionId, async session => { PlaceFileResult placeFileResult = await session.PlaceFileAsync( cacheContext, request.ContentHash.ToContentHash((HashType)request.HashType), new AbsolutePath(request.Path), (FileAccessMode)request.FileAccessMode, FileReplacementMode.ReplaceExisting, // Hard-coded because the service can't tell if this is a retry (where the previous try may have left a partial file) (FileRealizationMode)request.FileRealizationMode, token); return new PlaceFileResponse { Header = new ResponseHeader( startTime, placeFileResult.Succeeded, (int)placeFileResult.Code, placeFileResult.ErrorMessage, placeFileResult.Diagnostics), ContentSize = placeFileResult.FileSize }; }, errorMessage => new PlaceFileResponse { Header = ResponseHeader.Failure(startTime, (int)PlaceFileResult.ResultCode.Error, errorMessage) })); }
private async Task <OpenStreamResult> OpenStreamAsync(OperationContext context, AbsolutePath tempPath, PlaceFileResult placeFileResult) { if (placeFileResult) { try { Stream stream = await FileSystem.OpenReadOnlyAsync(tempPath, FileShare.Delete | FileShare.Read); if (stream == null) { throw new ClientCanRetryException(context, $"Failed to open temp file {tempPath}. The service may have restarted"); } return(new OpenStreamResult(stream)); } catch (Exception ex) when(ex is DirectoryNotFoundException || ex is UnauthorizedAccessException) { throw new ClientCanRetryException(context, $"Failed to open temp file {tempPath}. The service may be restarting", ex); } catch (Exception ex) when(!(ex is ClientCanRetryException)) { // The caller's retry policy needs to see ClientCanRetryExceptions in order to properly retry return(new OpenStreamResult(ex)); } } else if (placeFileResult.Code == PlaceFileResult.ResultCode.NotPlacedContentNotFound) { return(new OpenStreamResult(OpenStreamResult.ResultCode.ContentNotFound, placeFileResult.ErrorMessage)); } else { return(new OpenStreamResult(placeFileResult)); } }