public async Task BandwidthCheckTimesOutOnSlowCopyByUsingCopyToOptions()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Making sure the actual bandwidth checking options more permissive.
var totalBytes = actualBandwidthBytesPerSec * 2;
var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null);
var checker = new BandwidthChecker(checkerConfig);
using (var stream = new MemoryStream())
{
var bandwidthConfiguration = new BandwidthConfiguration()
{
Interval = checkInterval,
RequiredBytes = actualBandwidthBytesPerSec * 2,
};
var options = new CopyOptions(bandwidthConfiguration);
var result = await checker.CheckBandwidthAtIntervalAsync(
_context,
token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options),
options,
getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics));
Assert.Equal(CopyResultCode.CopyBandwidthTimeoutError, result.Code);
}
}
/// <inheritdoc />
public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options)
{
// The bandwidth checker needs to have an options instance, because it is used for tracking the copy progress as well.
options ??= new CopyToOptions();
return(_checker.CheckBandwidthAtIntervalAsync(
context,
// NOTE: We need to pass through the token from bandwidth checker to ensure copy cancellation for insufficient bandwidth gets triggered.
token => _inner.CopyToAsync(context.WithCancellationToken(token), sourcePath, destinationStream, expectedContentSize, options),
options));
}
public async Task BandwidthCheckDoesNotAffectGoodCopies()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Lower limit is half actual bandwidth
var totalBytes = actualBandwidthBytesPerSec * 2;
var checker = new BandwidthChecker(new ConstantBandwidthLimit(bandwidthLimit), checkInterval);
using (var stream = new MemoryStream())
{
await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream);
}
}
public async Task BandwidthCheckTimesOutOnSlowCopy()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec * 2); // Lower limit is twice actual bandwidth
var totalBytes = actualBandwidthBytesPerSec * 2;
var checker = new BandwidthChecker(new ConstantBandwidthLimit(bandwidthLimit), checkInterval);
using (var stream = new MemoryStream())
{
await Assert.ThrowsAsync(
typeof(TimeoutException),
async() => await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream));
}
}
public async Task BandwidthCheckTimesOutOnSlowCopy()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec * 2); // Lower limit is twice actual bandwidth
var totalBytes = actualBandwidthBytesPerSec * 2;
var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null);
var checker = new BandwidthChecker(checkerConfig);
using (var stream = new MemoryStream())
{
var result = await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream);
Assert.Equal(CopyFileResult.ResultCode.CopyBandwidthTimeoutError, result.Code);
}
}
public async Task BandwidthCheckDoesNotAffectGoodCopies()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Lower limit is half actual bandwidth
var totalBytes = actualBandwidthBytesPerSec * 2;
var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null);
var checker = new BandwidthChecker(checkerConfig);
using (var stream = new MemoryStream())
{
var result = await checker.CheckBandwidthAtIntervalAsync(_context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth), stream);
Assert.True(result.Succeeded);
}
}
public async Task BandwidthCheckDoesNotAffectGoodCopies()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 10); // Lower limit significantly to actual bandwidth
var totalBytes = actualBandwidthBytesPerSec * 2;
var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null);
var checker = new BandwidthChecker(checkerConfig);
using (var stream = new MemoryStream())
{
var options = new CopyOptions(bandwidthConfiguration: null);
var result = await checker.CheckBandwidthAtIntervalAsync(
_context,
token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options),
options,
getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics));
result.ShouldBeSuccess();
}
}
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, CopyOptions options, Func <Stream> streamFactory, bool closeStream)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
var token = cts.Token;
bool exceptionThrown = false;
TimeSpan? headerResponseTime = null;
CopyFileResult?result = null;
try
{
CopyFileRequest request = new CopyFileRequest()
{
TraceId = context.TracingContext.Id.ToString(),
HashType = (int)hash.HashType,
ContentHash = hash.ToByteString(),
Offset = 0,
Compression = _configuration.UseGzipCompression ? CopyCompression.Gzip : CopyCompression.None,
FailFastIfBusy = options.BandwidthConfiguration?.FailFastIfServerIsBusy ?? false,
};
using AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request, options: GetDefaultGrpcOptions(token));
Metadata headers;
var stopwatch = StopwatchSlim.Start();
try
{
var connectionTimeout = GetResponseHeadersTimeout(options);
headers = await response.ResponseHeadersAsync.WithTimeoutAsync(connectionTimeout, token);
headerResponseTime = stopwatch.Elapsed;
}
catch (TimeoutException t)
{
// Trying to cancel the back end operation as well.
cts.Cancel();
result = new CopyFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t);
return(result);
}
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. GRPC would throw an RpcException when we tried
// to stream response, but by that time we would have created target
// stream. To avoid that, exit early instead.
if (headers.Count == 0)
{
result = new CopyFileResult(
CopyResultCode.ServerUnavailable,
$"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}.");
return(result);
}
// Parse header collection.
string? exception = null;
string? message = null;
CopyCompression compression = CopyCompression.None;
foreach (Metadata.Entry header in headers)
{
switch (header.Key)
{
case "exception":
exception = header.Value;
break;
case "message":
message = header.Value;
break;
case "compression":
Enum.TryParse(header.Value, out compression);
break;
}
}
// Process reported server-side errors.
if (exception != null)
{
Contract.Assert(message != null);
switch (exception)
{
case "ContentNotFound":
result = new CopyFileResult(CopyResultCode.FileNotFoundError, message);
return(result);
default:
result = new CopyFileResult(CopyResultCode.UnknownServerError, message);
return(result);
}
}
// We got headers back with no errors, so create the target stream.
Stream targetStream;
try
{
targetStream = streamFactory();
}
catch (Exception targetException)
{
result = new CopyFileResult(CopyResultCode.DestinationPathError, targetException);
return(result);
}
result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync(
context,
innerToken => copyToCoreImplementation(response, compression, targetStream, innerToken),
options,
getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics));
return(result);
}
catch (RpcException r)
{
result = CreateResultFromException(r);
return(result);
}
catch (Exception)
{
exceptionThrown = true;
throw;
}
finally
{
// Even though we don't expect exceptions in this method, we can't assume they won't happen.
// So asserting that the result is not null only when the method completes successfully or with a known errors.
Contract.Assert(exceptionThrown || result != null);
if (result != null)
{
result.HeaderResponseTime = headerResponseTime;
}
}
async Task <CopyFileResult> copyToCoreImplementation(AsyncServerStreamingCall <CopyFileResponse> response, CopyCompression compression, Stream targetStream, CancellationToken token)
{
// Copy the content to the target stream.
try
{
switch (compression)
{
case CopyCompression.None:
await StreamContentAsync(response.ResponseStream, targetStream, options, token);
break;
case CopyCompression.Gzip:
await StreamContentWithCompressionAsync(response.ResponseStream, targetStream, options, token);
break;
default:
throw new NotSupportedException($"CopyCompression {compression} is not supported.");
}
}
finally
{
if (closeStream)
{
#pragma warning disable AsyncFixer02 // A disposable object used in a fire & forget async call
targetStream.Dispose();
#pragma warning restore AsyncFixer02 // A disposable object used in a fire & forget async call
}
}
return(CopyFileResult.Success);
}
}
public async Task CancellationShouldNotCauseTaskUnobservedException()
{
// This test checks that the bandwidth checker won't cause task unobserved exception
// for the task provided via 'copyTaskFactory' if the CheckBandwidthAtIntervalAsync
// is called and the operation is immediately cancelled.
var checker = new BandwidthChecker(GetConfiguration());
using var cts = new CancellationTokenSource();
OperationContext context = new OperationContext(new Context(TestGlobal.Logger), cts.Token);
int numberOfUnobservedExceptions = 0;
EventHandler <UnobservedTaskExceptionEventArgs> taskSchedulerOnUnobservedTaskException = (o, args) => { numberOfUnobservedExceptions++; };
try
{
TaskScheduler.UnobservedTaskException += taskSchedulerOnUnobservedTaskException;
// Cancelling the operation even before starting it.
cts.Cancel();
// Using task completion source as an event to force the task completion in a specific time.
var tcs = new TaskCompletionSource <object>();
using var stream = new MemoryStream();
var resultTask = checker.CheckBandwidthAtIntervalAsync(
context,
copyTaskFactory: token => Task.Run <CopyFileResult>(
async() =>
{
await tcs.Task;
throw new Exception("1");
}),
destinationStream: stream);
await Task.Delay(10);
try
{
(await resultTask).IgnoreFailure();
}
catch (OperationCanceledException)
{
}
// Triggering a failure
tcs.SetResult(null);
// Forcing a full GC cycle that will call all the finalizers.
// This is important because the finalizer thread will detect tasks with unobserved exceptions.
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
}
finally
{
// It is important to unsubscribe from the global event to prevent a memory leak when a test instance will stay
// in memory indefinitely.
TaskScheduler.UnobservedTaskException -= taskSchedulerOnUnobservedTaskException;
}
// This test is not 100% bullet proof, and it is possible that the test will pass even when the issue is still in the code.
// But the original issue was very consistent and the test was failing even from the IDE in Debug mode all the time.
numberOfUnobservedExceptions.Should().Be(0);
}
/// <inheritdoc />
public Task <CopyFileResult> CopyToAsync(T sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
var context = new OperationContext(new Context(_logger), cancellationToken);
return(_checker.CheckBandwidthAtIntervalAsync(context, token => _inner.CopyToAsync(sourcePath, destinationStream, expectedContentSize, token), destinationStream));
}
/// <summary>
/// Copies content from the server to the stream returned by the factory.
/// </summary>
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, Func <Stream> streamFactory)
{
try
{
CopyFileRequest request = new CopyFileRequest()
{
TraceId = context.TracingContext.Id.ToString(),
HashType = (int)hash.HashType,
ContentHash = hash.ToByteString(),
Offset = 0,
Compression = Key.UseCompression ? CopyCompression.Gzip : CopyCompression.None
};
AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request);
Metadata headers = await response.ResponseHeadersAsync;
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. GRPC would throw an RpcException when we tried
// to stream response, but by that time we would have created target
// stream. To avoid that, exit early instead.
if (headers.Count == 0)
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}."));
}
// Parse header collection.
string exception = null;
string message = null;
CopyCompression compression = CopyCompression.None;
foreach (Metadata.Entry header in headers)
{
switch (header.Key)
{
case "exception":
exception = header.Value;
break;
case "message":
message = header.Value;
break;
case "compression":
Enum.TryParse(header.Value, out compression);
break;
}
}
// Process reported server-side errors.
if (exception != null)
{
Contract.Assert(message != null);
switch (exception)
{
case "ContentNotFound":
return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, message));
default:
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, message));
}
}
// We got headers back with no errors, so create the target stream.
Stream targetStream;
try
{
targetStream = streamFactory();
}
catch (Exception targetException)
{
return(new CopyFileResult(CopyFileResult.ResultCode.DestinationPathError, targetException));
}
// Copy the content to the target stream.
using (targetStream)
{
await _bandwidthChecker.CheckBandwidthAtIntervalAsync(
context,
copyTaskFactory : token =>
compression switch
{
CopyCompression.None => StreamContentAsync(targetStream, response.ResponseStream, token),
CopyCompression.Gzip => StreamContentWithCompressionAsync(targetStream, response.ResponseStream, token),
_ => throw new NotSupportedException($"CopyCompression {compression} is not supported.")
},
destinationStream : targetStream);
}
