public async Task BandwidthCheckTimesOutOnSlowCopyByUsingCopyToOptions() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Making sure the actual bandwidth checking options more permissive. var totalBytes = actualBandwidthBytesPerSec * 2; var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null); var checker = new BandwidthChecker(checkerConfig); using (var stream = new MemoryStream()) { var bandwidthConfiguration = new BandwidthConfiguration() { Interval = checkInterval, RequiredBytes = actualBandwidthBytesPerSec * 2, }; var options = new CopyOptions(bandwidthConfiguration); var result = await checker.CheckBandwidthAtIntervalAsync( _context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options), options, getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics)); Assert.Equal(CopyResultCode.CopyBandwidthTimeoutError, result.Code); } }
/// <inheritdoc /> public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options) { // The bandwidth checker needs to have an options instance, because it is used for tracking the copy progress as well. options ??= new CopyToOptions(); return(_checker.CheckBandwidthAtIntervalAsync( context, // NOTE: We need to pass through the token from bandwidth checker to ensure copy cancellation for insufficient bandwidth gets triggered. token => _inner.CopyToAsync(context.WithCancellationToken(token), sourcePath, destinationStream, expectedContentSize, options), options)); }
public async Task BandwidthCheckDoesNotAffectGoodCopies() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Lower limit is half actual bandwidth var totalBytes = actualBandwidthBytesPerSec * 2; var checker = new BandwidthChecker(new ConstantBandwidthLimit(bandwidthLimit), checkInterval); using (var stream = new MemoryStream()) { await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream); } }
public async Task BandwidthCheckTimesOutOnSlowCopy() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec * 2); // Lower limit is twice actual bandwidth var totalBytes = actualBandwidthBytesPerSec * 2; var checker = new BandwidthChecker(new ConstantBandwidthLimit(bandwidthLimit), checkInterval); using (var stream = new MemoryStream()) { await Assert.ThrowsAsync( typeof(TimeoutException), async() => await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream)); } }
public async Task BandwidthCheckTimesOutOnSlowCopy() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec * 2); // Lower limit is twice actual bandwidth var totalBytes = actualBandwidthBytesPerSec * 2; var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null); var checker = new BandwidthChecker(checkerConfig); using (var stream = new MemoryStream()) { var result = await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream); Assert.Equal(CopyFileResult.ResultCode.CopyBandwidthTimeoutError, result.Code); } }
public async Task BandwidthCheckDoesNotAffectGoodCopies() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Lower limit is half actual bandwidth var totalBytes = actualBandwidthBytesPerSec * 2; var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null); var checker = new BandwidthChecker(checkerConfig); using (var stream = new MemoryStream()) { var result = await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream); Assert.True(result.Succeeded); } }
public async Task BandwidthCheckDoesNotAffectGoodCopies() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 10); // Lower limit significantly to actual bandwidth var totalBytes = actualBandwidthBytesPerSec * 2; var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null); var checker = new BandwidthChecker(checkerConfig); using (var stream = new MemoryStream()) { var options = new CopyOptions(bandwidthConfiguration: null); var result = await checker.CheckBandwidthAtIntervalAsync( _context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options), options, getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics)); result.ShouldBeSuccess(); } }
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, CopyOptions options, Func <Stream> streamFactory, bool closeStream) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token); var token = cts.Token; bool exceptionThrown = false; TimeSpan? headerResponseTime = null; CopyFileResult?result = null; try { CopyFileRequest request = new CopyFileRequest() { TraceId = context.TracingContext.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString(), Offset = 0, Compression = _configuration.UseGzipCompression ? CopyCompression.Gzip : CopyCompression.None, FailFastIfBusy = options.BandwidthConfiguration?.FailFastIfServerIsBusy ?? false, }; using AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request, options: GetDefaultGrpcOptions(token)); Metadata headers; var stopwatch = StopwatchSlim.Start(); try { var connectionTimeout = GetResponseHeadersTimeout(options); headers = await response.ResponseHeadersAsync.WithTimeoutAsync(connectionTimeout, token); headerResponseTime = stopwatch.Elapsed; } catch (TimeoutException t) { // Trying to cancel the back end operation as well. cts.Cancel(); result = new CopyFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t); return(result); } // If the remote machine couldn't be contacted, GRPC returns an empty // header collection. GRPC would throw an RpcException when we tried // to stream response, but by that time we would have created target // stream. To avoid that, exit early instead. if (headers.Count == 0) { result = new CopyFileResult( CopyResultCode.ServerUnavailable, $"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}."); return(result); } // Parse header collection. string? exception = null; string? message = null; CopyCompression compression = CopyCompression.None; foreach (Metadata.Entry header in headers) { switch (header.Key) { case "exception": exception = header.Value; break; case "message": message = header.Value; break; case "compression": Enum.TryParse(header.Value, out compression); break; } } // Process reported server-side errors. if (exception != null) { Contract.Assert(message != null); switch (exception) { case "ContentNotFound": result = new CopyFileResult(CopyResultCode.FileNotFoundError, message); return(result); default: result = new CopyFileResult(CopyResultCode.UnknownServerError, message); return(result); } } // We got headers back with no errors, so create the target stream. Stream targetStream; try { targetStream = streamFactory(); } catch (Exception targetException) { result = new CopyFileResult(CopyResultCode.DestinationPathError, targetException); return(result); } result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync( context, innerToken => copyToCoreImplementation(response, compression, targetStream, innerToken), options, getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics)); return(result); } catch (RpcException r) { result = CreateResultFromException(r); return(result); } catch (Exception) { exceptionThrown = true; throw; } finally { // Even though we don't expect exceptions in this method, we can't assume they won't happen. // So asserting that the result is not null only when the method completes successfully or with a known errors. Contract.Assert(exceptionThrown || result != null); if (result != null) { result.HeaderResponseTime = headerResponseTime; } } async Task <CopyFileResult> copyToCoreImplementation(AsyncServerStreamingCall <CopyFileResponse> response, CopyCompression compression, Stream targetStream, CancellationToken token) { // Copy the content to the target stream. try { switch (compression) { case CopyCompression.None: await StreamContentAsync(response.ResponseStream, targetStream, options, token); break; case CopyCompression.Gzip: await StreamContentWithCompressionAsync(response.ResponseStream, targetStream, options, token); break; default: throw new NotSupportedException($"CopyCompression {compression} is not supported."); } } finally { if (closeStream) { #pragma warning disable AsyncFixer02 // A disposable object used in a fire & forget async call targetStream.Dispose(); #pragma warning restore AsyncFixer02 // A disposable object used in a fire & forget async call } } return(CopyFileResult.Success); } }
public async Task CancellationShouldNotCauseTaskUnobservedException() { // This test checks that the bandwidth checker won't cause task unobserved exception // for the task provided via 'copyTaskFactory' if the CheckBandwidthAtIntervalAsync // is called and the operation is immediately cancelled. var checker = new BandwidthChecker(GetConfiguration()); using var cts = new CancellationTokenSource(); OperationContext context = new OperationContext(new Context(TestGlobal.Logger), cts.Token); int numberOfUnobservedExceptions = 0; EventHandler <UnobservedTaskExceptionEventArgs> taskSchedulerOnUnobservedTaskException = (o, args) => { numberOfUnobservedExceptions++; }; try { TaskScheduler.UnobservedTaskException += taskSchedulerOnUnobservedTaskException; // Cancelling the operation even before starting it. cts.Cancel(); // Using task completion source as an event to force the task completion in a specific time. var tcs = new TaskCompletionSource <object>(); using var stream = new MemoryStream(); var resultTask = checker.CheckBandwidthAtIntervalAsync( context, copyTaskFactory: token => Task.Run <CopyFileResult>( async() => { await tcs.Task; throw new Exception("1"); }), destinationStream: stream); await Task.Delay(10); try { (await resultTask).IgnoreFailure(); } catch (OperationCanceledException) { } // Triggering a failure tcs.SetResult(null); // Forcing a full GC cycle that will call all the finalizers. // This is important because the finalizer thread will detect tasks with unobserved exceptions. GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); } finally { // It is important to unsubscribe from the global event to prevent a memory leak when a test instance will stay // in memory indefinitely. TaskScheduler.UnobservedTaskException -= taskSchedulerOnUnobservedTaskException; } // This test is not 100% bullet proof, and it is possible that the test will pass even when the issue is still in the code. // But the original issue was very consistent and the test was failing even from the IDE in Debug mode all the time. numberOfUnobservedExceptions.Should().Be(0); }
/// <inheritdoc /> public Task <CopyFileResult> CopyToAsync(T sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { var context = new OperationContext(new Context(_logger), cancellationToken); return(_checker.CheckBandwidthAtIntervalAsync(context, token => _inner.CopyToAsync(sourcePath, destinationStream, expectedContentSize, token), destinationStream)); }
/// <summary> /// Copies content from the server to the stream returned by the factory. /// </summary> private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, Func <Stream> streamFactory) { try { CopyFileRequest request = new CopyFileRequest() { TraceId = context.TracingContext.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString(), Offset = 0, Compression = Key.UseCompression ? CopyCompression.Gzip : CopyCompression.None }; AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request); Metadata headers = await response.ResponseHeadersAsync; // If the remote machine couldn't be contacted, GRPC returns an empty // header collection. GRPC would throw an RpcException when we tried // to stream response, but by that time we would have created target // stream. To avoid that, exit early instead. if (headers.Count == 0) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}.")); } // Parse header collection. string exception = null; string message = null; CopyCompression compression = CopyCompression.None; foreach (Metadata.Entry header in headers) { switch (header.Key) { case "exception": exception = header.Value; break; case "message": message = header.Value; break; case "compression": Enum.TryParse(header.Value, out compression); break; } } // Process reported server-side errors. if (exception != null) { Contract.Assert(message != null); switch (exception) { case "ContentNotFound": return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, message)); default: return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, message)); } } // We got headers back with no errors, so create the target stream. Stream targetStream; try { targetStream = streamFactory(); } catch (Exception targetException) { return(new CopyFileResult(CopyFileResult.ResultCode.DestinationPathError, targetException)); } // Copy the content to the target stream. using (targetStream) { await _bandwidthChecker.CheckBandwidthAtIntervalAsync( context, copyTaskFactory : token => compression switch { CopyCompression.None => StreamContentAsync(targetStream, response.ResponseStream, token), CopyCompression.Gzip => StreamContentWithCompressionAsync(targetStream, response.ResponseStream, token), _ => throw new NotSupportedException($"CopyCompression {compression} is not supported.") }, destinationStream : targetStream); }