/// <summary> /// Checks that a copy has a minimum bandwidth, and cancells it otherwise. /// </summary> /// <param name="context">The context of the operation.</param> /// <param name="copyTaskFactory">Function that will trigger the copy.</param> /// <param name="destinationStream">Stream into which the copy is being made. Used to meassure bandwidth.</param> public async Task CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task> copyTaskFactory, Stream destinationStream) { if (_historicalBandwidthLimitSource != null) { var startPosition = destinationStream.Position; var timer = Stopwatch.StartNew(); await impl(); timer.Stop(); var endPosition = destinationStream.Position; // Bandwidth checker expects speed in MiB/s, so convert it. var bytesCopied = endPosition - startPosition; var speed = bytesCopied / timer.Elapsed.TotalSeconds / (1024 * 1024); _historicalBandwidthLimitSource.AddBandwidthRecord(speed); } else { await impl(); } async Task impl() { // This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash // (given that the app is configured to fail on unobserved task exceptions). var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec(); long previousPosition = 0; var copyCompleted = false; using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token); var copyTask = copyTaskFactory(copyCancellation.Token); try { while (!copyCompleted) { // Wait some time for bytes to be copied var firstCompletedTask = await Task.WhenAny(copyTask, Task.Delay(_checkInterval, context.Token)); copyCompleted = firstCompletedTask == copyTask; if (copyCompleted) { await copyTask; return; } else if (context.Token.IsCancellationRequested) { context.Token.ThrowIfCancellationRequested(); return; } // Copy is not completed and operation has not been canceled, perform // bandwidth check try { var position = destinationStream.Position; var receivedMiB = (position - previousPosition) / BytesInMb; var currentSpeed = receivedMiB / _checkInterval.TotalSeconds; if (currentSpeed < minimumSpeedInMbPerSec) { throw new TimeoutException($"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {position} copied]"); } previousPosition = position; } catch (ObjectDisposedException) { // If the check task races with the copy completing, it might attempt to check the position of a disposed stream. // Don't bother logging because the copy completed successfully. } catch (Exception ex) { var errorMessage = $"Exception thrown while checking bandwidth: {ex}"; // Erring on the side of caution; if something went wrong with the copy, return to avoid spin-logging the same exception. // Converting TaskCanceledException to TimeoutException because the clients should know that the operation was cancelled due to timeout. throw new TimeoutException(errorMessage, ex); } } } finally { if (!copyCompleted) { // Ensure that we signal the copy to cancel copyCancellation.Cancel(); copyTask.FireAndForget(context); } } } }
/// <summary> /// Checks that a copy has a minimum bandwidth, and cancels it otherwise. /// </summary> /// <param name="context">The context of the operation.</param> /// <param name="copyTaskFactory">Function that will trigger the copy.</param> /// <param name="destinationStream">Stream into which the copy is being made. Used to measure bandwidth.</param> public async Task <CopyFileResult> CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task <CopyFileResult> > copyTaskFactory, Stream destinationStream) { if (_historicalBandwidthLimitSource != null) { var timer = Stopwatch.StartNew(); var(result, bytesCopied) = await impl(); timer.Stop(); // Bandwidth checker expects speed in MiB/s, so convert it. var speed = bytesCopied / timer.Elapsed.TotalSeconds / BytesInMb; _historicalBandwidthLimitSource.AddBandwidthRecord(speed); return(result); } else { return((await impl()).result); } async Task <(CopyFileResult result, long bytesCopied)> impl() { // This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash // (given that the app is configured to fail on unobserved task exceptions). var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec() * _config.BandwidthLimitMultiplier; minimumSpeedInMbPerSec = Math.Min(minimumSpeedInMbPerSec, _config.MaxBandwidthLimit); var startPosition = tryGetPosition(destinationStream, out var pos) ? pos : 0; long previousPosition = startPosition; var copyCompleted = false; using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token); var copyTask = copyTaskFactory(copyCancellation.Token); while (!copyCompleted) { // Wait some time for bytes to be copied var firstCompletedTask = await Task.WhenAny(copyTask, Task.Delay(_config.BandwidthCheckInterval, context.Token)); copyCompleted = firstCompletedTask == copyTask; if (copyCompleted) { var result = await copyTask; var bytesCopied = result.Size ?? (previousPosition - startPosition); return(result, bytesCopied); } else if (context.Token.IsCancellationRequested) { context.Token.ThrowIfCancellationRequested(); } // Copy is not completed and operation has not been canceled, perform // bandwidth check if (tryGetPosition(destinationStream, out var position)) { var receivedMiB = (position - previousPosition) / BytesInMb; var currentSpeed = receivedMiB / _config.BandwidthCheckInterval.TotalSeconds; if (currentSpeed == 0 || currentSpeed < minimumSpeedInMbPerSec) { // Ensure that we signal the copy to cancel copyCancellation.Cancel(); traceCopyTaskFailures(copyTask); var bytesCopied = position - startPosition; var result = new CopyFileResult(CopyFileResult.ResultCode.CopyBandwidthTimeoutError, $"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {bytesCopied} bytes copied"); return(result, bytesCopied); } previousPosition = position; } } return(await copyTask, previousPosition - startPosition); void traceCopyTaskFailures(Task task) { // When the operation is cancelled, it is possible for the copy operation to fail. // In this case we still want to trace the failure (but just with the debug severity and not with the error), // but we should exclude ObjectDisposedException completely. // That's why we don't use task.FireAndForget but tracing inside the task's continuation. copyTask.ContinueWith(t => { if (t.IsFaulted) { if (!(t.Exception?.InnerException is ObjectDisposedException)) { context.TraceDebug($"Checked copy failed. {t.Exception}"); } } }); } }