public async Task CopyWithDestinationPathError(int retries)
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), retries);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyAttempts = 0;
var totalCopyAttempts = (retries - 1) * machineLocations.Length + 1;
mockFileCopier.CustomResults = new CopyFileResult[totalCopyAttempts];
mockFileCopier.CustomResults[0] = CopyFileResult.FromResultCode(CopyResultCode.DestinationPathError);
for (int counter = 1; counter < totalCopyAttempts; counter++)
{
mockFileCopier.CustomResults[counter] = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError);
}
;
var destinationResult = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
destinationResult.ShouldBeError();
destinationResult.ErrorMessage.Should().Contain(hash.ToShortString());
mockFileCopier.CopyAttempts.Should().Be(totalCopyAttempts);
}
}
public async Task CopyRetries(int retries)
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero, retries);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var machineLocations = new MachineLocation[] { new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyToAsyncResult = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
result.ShouldBeError();
mockFileCopier.CopyAttempts.Should().Be(retries);
}
}
public async Task CopyRetriesWithRestrictions(int retries)
{
var context = new Context(Logger);
var copyAttemptsWithRestrictedReplicas = 2;
var restrictedCopyReplicaCount = 3;
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(
FileSystem,
directory.Path,
TimeSpan.Zero,
retries,
copyAttemptsWithRestrictedReplicas,
restrictedCopyReplicaCount,
maxRetryCount: retries + 1);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyToAsyncResult = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
result.ShouldBeError();
int copyAttempts = 0;
for (var attemptCount = 0; attemptCount < retries; attemptCount++)
{
var maxReplicaCount = attemptCount < copyAttemptsWithRestrictedReplicas
? restrictedCopyReplicaCount
: int.MaxValue;
copyAttempts += Math.Min(maxReplicaCount, machineLocations.Length);
}
if (copyAttempts < distributedCopier.Settings.MaxRetryCount)
{
mockFileCopier.CopyAttempts.Should().Be(copyAttempts);
result.ErrorMessage.Should().NotContain("Maximum total retries");
}
else
{
mockFileCopier.CopyAttempts.Should().Be(distributedCopier.Settings.MaxRetryCount);
result.ErrorMessage.Should().Contain("Maximum total retries");
}
}
}