public async Task <PlaceFileResult> CreateTempAndPutAsync(
OperationContext context,
ContentHash contentHash,
IContentSession contentSession)
{
using (var disposableFile = new DisposableFile(context, _fileSystem, AbsolutePath.CreateRandomFileName(_rootPath / "temp")))
{
PlaceFileResult placeTempFileResult = await PlaceFileAsync(context, contentHash, disposableFile.Path, FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.HardLink, context.Token);
if (!placeTempFileResult.Succeeded)
{
return(placeTempFileResult);
}
PutResult putFileResult = await contentSession.PutFileAsync(context, contentHash, disposableFile.Path, FileRealizationMode.Any, context.Token);
if (!putFileResult)
{
return(new PlaceFileResult(putFileResult));
}
else
{
return(new PlaceFileResult(PlaceFileResult.ResultCode.PlacedWithCopy, putFileResult.ContentSize));
}
}
}
private Task <PlaceFileResponse> PlaceFileAsync(PlaceFileRequest request, CancellationToken token)
{
return(RunFuncAsync(
request.Header,
async(context, session) =>
{
PlaceFileResult placeFileResult = await session.PlaceFileAsync(
context.OperationContext,
request.ContentHash.ToContentHash((HashType)request.HashType),
new AbsolutePath(request.Path),
(FileAccessMode)request.FileAccessMode,
FileReplacementMode.ReplaceExisting, // Hard-coded because the service can't tell if this is a retry (where the previous try may have left a partial file)
(FileRealizationMode)request.FileRealizationMode,
token);
return new PlaceFileResponse
{
Header =
new ResponseHeader(
context.StartTime,
placeFileResult.Succeeded,
(int)placeFileResult.Code,
placeFileResult.ErrorMessage,
placeFileResult.Diagnostics),
ContentSize = placeFileResult.FileSize
};
},
(context, errorMessage) => new PlaceFileResponse
{
Header = ResponseHeader.Failure(context.StartTime, (int)PlaceFileResult.ResultCode.Error, errorMessage)
},
token));
}
public static PlaceFileResult ShouldBeSuccess(this PlaceFileResult result)
{
Assert.NotNull(result);
Assert.True(result.Succeeded, $"Place file operation should succeed, but it failed. Error: {result.ErrorMessage}. Diagnostics: {result.Diagnostics}");
Assert.Null(result.ErrorMessage);
return(result);
}
public override void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, Severity successSeverity)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileStop(context.TraceId, (int)result.Code, result.ErrorMessage);
}
base.PlaceFileStop(context, contentHash, result, path, accessMode, replacementMode, realizationMode, successSeverity: DiagnosticLevelSeverity(result.Duration));
}
public virtual void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode)
{
if (context.IsEnabled)
{
TracerOperationFinished(context, result, $"{Name}.{PlaceFileCallName}({contentHash.ToShortString()},{path},{accessMode},{replacementMode},{realizationMode}) stop {result.DurationMs}ms result=[{result}]");
}
_placeFileCallCounter.Completed(result.Duration.Ticks);
}
public override void PlaceFileStop(Context context, ContentHash input, PlaceFileResult result)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileStop(context.Id.ToString(), (int)result.Code, result.ErrorMessage);
}
base.PlaceFileStop(context, input, result);
}
public override void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileStop(context.Id.ToString(), (int)result.Code, result.ErrorMessage);
}
base.PlaceFileStop(context, contentHash, result, path, accessMode, replacementMode, realizationMode);
}
public virtual void PlaceFileStop(Context context, ContentHash input, PlaceFileResult result)
{
if (context.IsEnabled)
{
TracerOperationFinished(context, result, $"{Name}.{PlaceFileCallName} stop {result.DurationMs}ms input=[{input}] result=[{result}]");
}
_placeFileCallCounter.Completed(result.Duration.Ticks);
}
public async Task PlaceFileRequiringNewReplicaCloseToHardLimitDoesNotHang()
{
var context = new Context(Logger);
using (DisposableDirectory testDirectory = new DisposableDirectory(FileSystem))
{
#pragma warning disable AsyncFixer04 // A disposable object used in a fire & forget async call
Task testTask = TestStore(context, Clock, testDirectory, async store =>
#pragma warning restore AsyncFixer04 // A disposable object used in a fire & forget async call
{
// Make a file which will overflow the cache size with just 2 copies.
PutResult putResult = await store.PutRandomAsync(context, ContentSizeToStartHardPurging(2));
ResultTestExtensions.ShouldBeSuccess((BoolResult)putResult);
ContentHash hash = putResult.ContentHash;
// Hardlink the file out 1024 times. Since the limit is 1024 total, and we already have 1 in the CAS,
// this will overflow the links and cause the CAS to create a new replica for it. This will cause
// the purger to consider that hash for eviction *while making room for that hash*, which is the
// trigger for the previous deadlock that this test will now guard against.
for (int i = 0; i < 1024; i++)
{
PlaceFileResult placeResult = await store.PlaceFileAsync(
context,
hash,
testDirectory.Path / $"hardlink{i}.txt",
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.HardLink,
null);
// The checks below are just to make sure that the calls completed as expected.
// The most important part is that they complete *at all*, which is enforced by
// racing against the Task.Delay in the outer scope.
if (i < 1023 || SucceedsEvenIfFull)
{
// The first 1023 links should succeed (bringing it up to the limit of 1024)
// And *all* of the calls should succeed if the cache takes new content even when overflowed.
Assert.True(placeResult.Succeeded);
}
else
{
// If the implementation rejects overflowing content, then the last call should fail.
Assert.False(placeResult.Succeeded);
Assert.Contains("Failed to reserve space", placeResult.ErrorMessage);
}
}
});
// Race between the test and a 2-minute timer. This can be increased if the test ends up flaky.
Task firstCompletedTask = await Task.WhenAny(testTask, Task.Delay(TimeSpan.FromMinutes(2)));
// The test should finish first, long before a minute passes, but it won't if it deadlocks.
Assert.True(firstCompletedTask == testTask);
await firstCompletedTask;
}
}
private Task PlaceFileRecoversAsync(
Context context,
Func <TestFileSystemContentStoreInternal, DisposableDirectory, Task <byte[]> > corruptFunc,
FileRealizationMode fileRealizationMode)
{
return(TestStore(context, _clock, async store =>
{
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
var bytes = await corruptFunc(store, tempDirectory);
var contentHash = bytes.CalculateHash(ContentHashType);
var placePath = tempDirectory.CreateRandomFileName();
PlaceFileResult result = null;
Func <Task> putFunc =
async() =>
{
result = await store.PlaceFileAsync(
context,
contentHash,
placePath,
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.HardLink,
null);
};
putFunc.Should().NotThrow();
Assert.NotNull(result);
result.Code.Should()
.Be(fileRealizationMode == FileRealizationMode.Copy
? PlaceFileResult.ResultCode.PlacedWithCopy
: PlaceFileResult.ResultCode.PlacedWithHardLink);
FileSystem.GetHardLinkCount(placePath)
.Should()
.Be(fileRealizationMode == FileRealizationMode.Copy ? 1 : 2);
}
}));
}
/// <summary>
/// Implements a place file request.
/// </summary>
private async Task <PlaceFileResponse> PlaceFileAsync(PlaceFileRequest request, CancellationToken token)
{
LogRequestHandling();
DateTime startTime = DateTime.UtcNow;
var cacheContext = new Context(new Guid(request.Header.TraceId), _logger);
return(await RunFuncAndReportAsync(
request.Header.SessionId,
async session =>
{
PlaceFileResult placeFileResult = await session.PlaceFileAsync(
cacheContext,
request.ContentHash.ToContentHash((HashType)request.HashType),
new AbsolutePath(request.Path),
(FileAccessMode)request.FileAccessMode,
FileReplacementMode.ReplaceExisting, // Hard-coded because the service can't tell if this is a retry (where the previous try may have left a partial file)
(FileRealizationMode)request.FileRealizationMode,
token);
return new PlaceFileResponse
{
Header =
new ResponseHeader(
startTime,
placeFileResult.Succeeded,
(int)placeFileResult.Code,
placeFileResult.ErrorMessage,
placeFileResult.Diagnostics),
ContentSize = placeFileResult.FileSize
};
},
errorMessage => new PlaceFileResponse
{
Header = ResponseHeader.Failure(startTime, (int)PlaceFileResult.ResultCode.Error, errorMessage)
}));
}
private async Task <OpenStreamResult> OpenStreamAsync(OperationContext context, AbsolutePath tempPath, PlaceFileResult placeFileResult)
{
if (placeFileResult)
{
try
{
Stream stream = await FileSystem.OpenReadOnlyAsync(tempPath, FileShare.Delete | FileShare.Read);
if (stream == null)
{
throw new ClientCanRetryException(context, $"Failed to open temp file {tempPath}. The service may have restarted");
}
return(new OpenStreamResult(stream));
}
catch (Exception ex) when(ex is DirectoryNotFoundException || ex is UnauthorizedAccessException)
{
throw new ClientCanRetryException(context, $"Failed to open temp file {tempPath}. The service may be restarting", ex);
}
catch (Exception ex) when(!(ex is ClientCanRetryException))
{
// The caller's retry policy needs to see ClientCanRetryExceptions in order to properly retry
return(new OpenStreamResult(ex));
}
}
else if (placeFileResult.Code == PlaceFileResult.ResultCode.NotPlacedContentNotFound)
{
return(new OpenStreamResult(OpenStreamResult.ResultCode.ContentNotFound, placeFileResult.ErrorMessage));
}
else
{
return(new OpenStreamResult(placeFileResult));
}
}
