/// <summary>
/// Checks that a copy has a minimum bandwidth, and cancells it otherwise.
/// </summary>
/// <param name="context">The context of the operation.</param>
/// <param name="copyTaskFactory">Function that will trigger the copy.</param>
/// <param name="destinationStream">Stream into which the copy is being made. Used to meassure bandwidth.</param>
public async Task CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task> copyTaskFactory, Stream destinationStream)
{
if (_historicalBandwidthLimitSource != null)
{
var startPosition = destinationStream.Position;
var timer = Stopwatch.StartNew();
await impl();
timer.Stop();
var endPosition = destinationStream.Position;
// Bandwidth checker expects speed in MiB/s, so convert it.
var bytesCopied = endPosition - startPosition;
var speed = bytesCopied / timer.Elapsed.TotalSeconds / (1024 * 1024);
_historicalBandwidthLimitSource.AddBandwidthRecord(speed);
}
else
{
await impl();
}
async Task impl()
{
// This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash
// (given that the app is configured to fail on unobserved task exceptions).
var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec();
long previousPosition = 0;
var copyCompleted = false;
using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
var copyTask = copyTaskFactory(copyCancellation.Token);
try
{
while (!copyCompleted)
{
// Wait some time for bytes to be copied
var firstCompletedTask = await Task.WhenAny(copyTask,
Task.Delay(_checkInterval, context.Token));
copyCompleted = firstCompletedTask == copyTask;
if (copyCompleted)
{
await copyTask;
return;
}
else if (context.Token.IsCancellationRequested)
{
context.Token.ThrowIfCancellationRequested();
return;
}
// Copy is not completed and operation has not been canceled, perform
// bandwidth check
try
{
var position = destinationStream.Position;
var receivedMiB = (position - previousPosition) / BytesInMb;
var currentSpeed = receivedMiB / _checkInterval.TotalSeconds;
if (currentSpeed < minimumSpeedInMbPerSec)
{
throw new TimeoutException($"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {position} copied]");
}
previousPosition = position;
}
catch (ObjectDisposedException)
{
// If the check task races with the copy completing, it might attempt to check the position of a disposed stream.
// Don't bother logging because the copy completed successfully.
}
catch (Exception ex)
{
var errorMessage = $"Exception thrown while checking bandwidth: {ex}";
// Erring on the side of caution; if something went wrong with the copy, return to avoid spin-logging the same exception.
// Converting TaskCanceledException to TimeoutException because the clients should know that the operation was cancelled due to timeout.
throw new TimeoutException(errorMessage, ex);
}
}
}
finally
{
if (!copyCompleted)
{
// Ensure that we signal the copy to cancel
copyCancellation.Cancel();
copyTask.FireAndForget(context);
}
}
}
}
/// <summary>
/// Checks that a copy has a minimum bandwidth, and cancels it otherwise.
/// </summary>
/// <param name="context">The context of the operation.</param>
/// <param name="copyTaskFactory">Function that will trigger the copy.</param>
/// <param name="destinationStream">Stream into which the copy is being made. Used to measure bandwidth.</param>
public async Task <CopyFileResult> CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task <CopyFileResult> > copyTaskFactory, Stream destinationStream)
{
if (_historicalBandwidthLimitSource != null)
{
var timer = Stopwatch.StartNew();
var(result, bytesCopied) = await impl();
timer.Stop();
// Bandwidth checker expects speed in MiB/s, so convert it.
var speed = bytesCopied / timer.Elapsed.TotalSeconds / BytesInMb;
_historicalBandwidthLimitSource.AddBandwidthRecord(speed);
return(result);
}
else
{
return((await impl()).result);
}
async Task <(CopyFileResult result, long bytesCopied)> impl()
{
// This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash
// (given that the app is configured to fail on unobserved task exceptions).
var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec() * _config.BandwidthLimitMultiplier;
minimumSpeedInMbPerSec = Math.Min(minimumSpeedInMbPerSec, _config.MaxBandwidthLimit);
var startPosition = tryGetPosition(destinationStream, out var pos) ? pos : 0;
long previousPosition = startPosition;
var copyCompleted = false;
using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
var copyTask = copyTaskFactory(copyCancellation.Token);
while (!copyCompleted)
{
// Wait some time for bytes to be copied
var firstCompletedTask = await Task.WhenAny(copyTask,
Task.Delay(_config.BandwidthCheckInterval, context.Token));
copyCompleted = firstCompletedTask == copyTask;
if (copyCompleted)
{
var result = await copyTask;
var bytesCopied = result.Size ?? (previousPosition - startPosition);
return(result, bytesCopied);
}
else if (context.Token.IsCancellationRequested)
{
context.Token.ThrowIfCancellationRequested();
}
// Copy is not completed and operation has not been canceled, perform
// bandwidth check
if (tryGetPosition(destinationStream, out var position))
{
var receivedMiB = (position - previousPosition) / BytesInMb;
var currentSpeed = receivedMiB / _config.BandwidthCheckInterval.TotalSeconds;
if (currentSpeed == 0 || currentSpeed < minimumSpeedInMbPerSec)
{
// Ensure that we signal the copy to cancel
copyCancellation.Cancel();
traceCopyTaskFailures(copyTask);
var bytesCopied = position - startPosition;
var result = new CopyFileResult(CopyFileResult.ResultCode.CopyBandwidthTimeoutError, $"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {bytesCopied} bytes copied");
return(result, bytesCopied);
}
previousPosition = position;
}
}
return(await copyTask, previousPosition - startPosition);
void traceCopyTaskFailures(Task task)
{
// When the operation is cancelled, it is possible for the copy operation to fail.
// In this case we still want to trace the failure (but just with the debug severity and not with the error),
// but we should exclude ObjectDisposedException completely.
// That's why we don't use task.FireAndForget but tracing inside the task's continuation.
copyTask.ContinueWith(t =>
{
if (t.IsFaulted)
{
if (!(t.Exception?.InnerException is ObjectDisposedException))
{
context.TraceDebug($"Checked copy failed. {t.Exception}");
}
}
});
}
}
