public async Task CopyRetries(int retries)
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero, retries);
var machineLocations = new MachineLocation[] { new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyToAsyncResult = new CopyFileResult(CopyFileResult.ResultCode.SourcePathError);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
result.ShouldBeError();
mockFileCopier.CopyAttempts.Should().Be(retries);
}
}
///<summary>
/// Test case for bug <"https://dev.azure.com/mseng/1ES/_boards/board/t/DavidW%20-%20Team/Stories/?workitem=1654106"/>
/// During the first attempt of copying from a list of locations, one of the locations returns a DestinationPathError.
/// Then in subsequent attempts to copy from the list of locations, the previous location that returned DestinationPathError now returns a different error.
/// We should still be able to attempt to copy and return without and out of range exception thrown.
///</summary>
public async Task CopyWithDestinationPathError(int retries)
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.FromMilliseconds((10)), retries);
var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyAttempts = 0;
var totalCopyAttempts = (retries - 1) * machineLocations.Length + 1;
mockFileCopier.CustomResults = new CopyFileResult[totalCopyAttempts];
mockFileCopier.CustomResults[0] = new CopyFileResult(CopyFileResult.ResultCode.DestinationPathError);
for (int counter = 1; counter < totalCopyAttempts; counter++)
{
mockFileCopier.CustomResults[counter] = new CopyFileResult(CopyFileResult.ResultCode.SourcePathError);
}
;
var destinationResult = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
destinationResult.ShouldBeError();
destinationResult.ErrorMessage.Should().Contain(hash.ToShortString());
mockFileCopier.CopyAttempts.Should().Be(totalCopyAttempts);
}
}
public async Task CopyFailsForWrongHash()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero);
var hash = ContentHash.Random();
var wrongHash = VsoHashInfo.Instance.EmptyHash;
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(42);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(wrongHash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain(hash.ToShortString());
result.ErrorMessage.Should().Contain(wrongHash.ToShortString());
}
}
public async Task CopyWithDesignatedLocations()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var machineLocations = new MachineLocation[] { new MachineLocation("Non-designated"), new MachineLocation("Designated") };
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), designatedLocations: new MachineLocation[] { machineLocations[1] });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(99);
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
var destinationResult = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
destinationResult.ShouldBeSuccess();
mockFileCopier.CopyAttempts.Should().Be(1);
distributedCopier.PathTransformer.LastContentLocation.Should().BeEquivalentTo(machineLocations[1].Data);
}
}
public async Task CopyFailsForWrongCopySize()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path);
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyFileAsyncResult = CopyFileResult.SuccessWithSize(41);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain("size");
result.ErrorMessage.Should().Contain("mismatch");
}
}
public async Task CopyFailsForWrongCopySize()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var hash = VsoHashInfo.Instance.EmptyHash;
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(41);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain("size");
result.ErrorMessage.Should().Contain("mismatch");
}
}
public async Task CopyRetriesWithRestrictions(int retries)
{
var context = new Context(Logger);
var copyAttemptsWithRestrictedReplicas = 2;
var restrictedCopyReplicaCount = 3;
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(
FileSystem,
directory.Path,
TimeSpan.Zero,
retries,
copyAttemptsWithRestrictedReplicas,
restrictedCopyReplicaCount,
maxRetryCount: retries + 1);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyToAsyncResult = new CopyFileResult(CopyResultCode.UnknownServerError);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
result.ShouldBeError();
int copyAttempts = 0;
for (var attemptCount = 0; attemptCount < retries; attemptCount++)
{
var maxReplicaCount = attemptCount < copyAttemptsWithRestrictedReplicas
? restrictedCopyReplicaCount
: int.MaxValue;
copyAttempts += Math.Min(maxReplicaCount, machineLocations.Length);
}
if (copyAttempts < distributedCopier.Settings.MaxRetryCount)
{
mockFileCopier.CopyAttempts.Should().Be(copyAttempts);
result.ErrorMessage.Should().NotContain("Maximum total retries");
}
else
{
mockFileCopier.CopyAttempts.Should().Be(distributedCopier.Settings.MaxRetryCount);
result.ErrorMessage.Should().Contain("Maximum total retries");
}
}
}
public GetBulkLocationsResult GetBulkLocations(OperationContext context, IReadOnlyList <ContentHashWithPath> contentHashes)
{
return(context.PerformOperation(Tracer,
operation: () =>
{
var contentHashesInfo = new List <ContentHashWithSizeAndLocations>();
foreach (ContentHashWithPath contentHash in contentHashes)
{
var locations = GetMachineLocations(contentHash.Hash);
var contentHashWithLocations = new ContentHashWithSizeAndLocations(contentHash.Hash, -1, locations);
contentHashesInfo.Add(contentHashWithLocations);
}
return new GetBulkLocationsResult(contentHashesInfo, GetBulkOrigin.ColdStorage);
}));
}
public async Task <PutResult> TryCopyAndPutAsync(
OperationContext operationContext,
IDistributedContentCopierHost host,
ContentHashWithSizeAndLocations hashInfo,
CopyReason reason,
Func <(CopyFileResult copyResult, AbsolutePath tempLocation, int attemptCount), Task <PutResult> > handleCopyAsync)
internal Task <PutResult> TryCopyAndPutAsync(OperationContext operationContext, ContentHashWithSizeAndLocations hashWithLocations, Func <(CopyFileResult copyResult, AbsolutePath tempLocation, int attemptCount), Task <PutResult> > handleCopyAsync)
public record CopyRequest(
IDistributedContentCopierHost Host,
ContentHashWithSizeAndLocations HashInfo,
CopyReason Reason,
Func <(CopyFileResult copyResult, AbsolutePath tempLocation, int attemptCount), Task <PutResult> > HandleCopyAsync,
