public async Task CopyRetries(int retries) { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero, retries); var machineLocations = new MachineLocation[] { new MachineLocation("") }; var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); mockFileCopier.CopyToAsyncResult = new CopyFileResult(CopyFileResult.ResultCode.SourcePathError); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); result.ShouldBeError(); mockFileCopier.CopyAttempts.Should().Be(retries); } }
///<summary> /// Test case for bug <"https://dev.azure.com/mseng/1ES/_boards/board/t/DavidW%20-%20Team/Stories/?workitem=1654106"/> /// During the first attempt of copying from a list of locations, one of the locations returns a DestinationPathError. /// Then in subsequent attempts to copy from the list of locations, the previous location that returned DestinationPathError now returns a different error. /// We should still be able to attempt to copy and return without and out of range exception thrown. ///</summary> public async Task CopyWithDestinationPathError(int retries) { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.FromMilliseconds((10)), retries); var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation("") }; var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); mockFileCopier.CopyAttempts = 0; var totalCopyAttempts = (retries - 1) * machineLocations.Length + 1; mockFileCopier.CustomResults = new CopyFileResult[totalCopyAttempts]; mockFileCopier.CustomResults[0] = new CopyFileResult(CopyFileResult.ResultCode.DestinationPathError); for (int counter = 1; counter < totalCopyAttempts; counter++) { mockFileCopier.CustomResults[counter] = new CopyFileResult(CopyFileResult.ResultCode.SourcePathError); } ; var destinationResult = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); destinationResult.ShouldBeError(); destinationResult.ErrorMessage.Should().Contain(hash.ToShortString()); mockFileCopier.CopyAttempts.Should().Be(totalCopyAttempts); } }
public async Task CopyFailsForWrongHash() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero); var hash = ContentHash.Random(); var wrongHash = VsoHashInfo.Instance.EmptyHash; var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(42); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(wrongHash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain(hash.ToShortString()); result.ErrorMessage.Should().Contain(wrongHash.ToShortString()); } }
public async Task CopyWithDesignatedLocations() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var machineLocations = new MachineLocation[] { new MachineLocation("Non-designated"), new MachineLocation("Designated") }; var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), designatedLocations: new MachineLocation[] { machineLocations[1] }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(99); var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); var destinationResult = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); destinationResult.ShouldBeSuccess(); mockFileCopier.CopyAttempts.Should().Be(1); distributedCopier.PathTransformer.LastContentLocation.Should().BeEquivalentTo(machineLocations[1].Data); } }
public async Task CopyFailsForWrongCopySize() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path); var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyFileAsyncResult = CopyFileResult.SuccessWithSize(41); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain("size"); result.ErrorMessage.Should().Contain("mismatch"); } }
public async Task CopyFailsForWrongCopySize() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero); await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context); var hash = VsoHashInfo.Instance.EmptyHash; var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(41); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain("size"); result.ErrorMessage.Should().Contain("mismatch"); } }
public async Task CopyRetriesWithRestrictions(int retries) { var context = new Context(Logger); var copyAttemptsWithRestrictedReplicas = 2; var restrictedCopyReplicaCount = 3; using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks( FileSystem, directory.Path, TimeSpan.Zero, retries, copyAttemptsWithRestrictedReplicas, restrictedCopyReplicaCount, maxRetryCount: retries + 1); await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context); var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation("") }; var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); mockFileCopier.CopyToAsyncResult = new CopyFileResult(CopyResultCode.UnknownServerError); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); result.ShouldBeError(); int copyAttempts = 0; for (var attemptCount = 0; attemptCount < retries; attemptCount++) { var maxReplicaCount = attemptCount < copyAttemptsWithRestrictedReplicas ? restrictedCopyReplicaCount : int.MaxValue; copyAttempts += Math.Min(maxReplicaCount, machineLocations.Length); } if (copyAttempts < distributedCopier.Settings.MaxRetryCount) { mockFileCopier.CopyAttempts.Should().Be(copyAttempts); result.ErrorMessage.Should().NotContain("Maximum total retries"); } else { mockFileCopier.CopyAttempts.Should().Be(distributedCopier.Settings.MaxRetryCount); result.ErrorMessage.Should().Contain("Maximum total retries"); } } }
public GetBulkLocationsResult GetBulkLocations(OperationContext context, IReadOnlyList <ContentHashWithPath> contentHashes) { return(context.PerformOperation(Tracer, operation: () => { var contentHashesInfo = new List <ContentHashWithSizeAndLocations>(); foreach (ContentHashWithPath contentHash in contentHashes) { var locations = GetMachineLocations(contentHash.Hash); var contentHashWithLocations = new ContentHashWithSizeAndLocations(contentHash.Hash, -1, locations); contentHashesInfo.Add(contentHashWithLocations); } return new GetBulkLocationsResult(contentHashesInfo, GetBulkOrigin.ColdStorage); })); }
public async Task <PutResult> TryCopyAndPutAsync( OperationContext operationContext, IDistributedContentCopierHost host, ContentHashWithSizeAndLocations hashInfo, CopyReason reason, Func <(CopyFileResult copyResult, AbsolutePath tempLocation, int attemptCount), Task <PutResult> > handleCopyAsync)
internal Task <PutResult> TryCopyAndPutAsync(OperationContext operationContext, ContentHashWithSizeAndLocations hashWithLocations, Func <(CopyFileResult copyResult, AbsolutePath tempLocation, int attemptCount), Task <PutResult> > handleCopyAsync)
public record CopyRequest( IDistributedContentCopierHost Host, ContentHashWithSizeAndLocations HashInfo, CopyReason Reason, Func <(CopyFileResult copyResult, AbsolutePath tempLocation, int attemptCount), Task <PutResult> > HandleCopyAsync,