/// <summary>
/// Pushes content to another machine.
/// </summary>
public async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Result <Stream> > > source)
{
try
{
var pushRequest = new PushRequest(hash, context.TracingContext.Id);
var headers = pushRequest.GetMetadata();
var call = _client.PushFile(headers, cancellationToken: context.Token);
var requestStream = call.RequestStream;
var responseHeaders = await call.ResponseHeadersAsync;
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. To avoid an exception, exit early instead.
if (responseHeaders.Count == 0)
{
return(PushFileResult.ServerUnavailable());
}
var pushResponse = PushResponse.FromMetadata(responseHeaders);
if (!pushResponse.ShouldCopy)
{
context.TraceDebug($"{nameof(PushFileAsync)}: copy of {hash.ToShortString()} was skipped.");
return(PushFileResult.Rejected());
}
var streamResult = await source();
if (!streamResult)
{
await requestStream.CompleteAsync();
return(new PushFileResult(streamResult, "Failed to retrieve source stream."));
}
using (var stream = streamResult.Value)
{
await StreamContentAsync(stream, new byte[_bufferSize], requestStream, context.Token);
}
await requestStream.CompleteAsync();
var responseStream = call.ResponseStream;
await responseStream.MoveNext(context.Token);
var response = responseStream.Current;
return(response.Header.Succeeded
? PushFileResult.PushSucceeded()
: new PushFileResult(response.Header.ErrorMessage));
}
catch (RpcException r)
{
return(new PushFileResult(r));
}
}
public async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Stream stream, CopyOptions options)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
var token = cts.Token;
bool exceptionThrown = false;
TimeSpan? headerResponseTime = null;
PushFileResult?result = null;
try
{
var startingPosition = stream.Position;
var pushRequest = new PushRequest(hash, traceId: context.TracingContext.Id);
var headers = pushRequest.GetMetadata();
using var call = _client.PushFile(options: GetDefaultGrpcOptions(headers, token));
var requestStream = call.RequestStream;
Metadata responseHeaders;
var stopwatch = StopwatchSlim.Start();
try
{
var timeout = GetResponseHeadersTimeout(options);
responseHeaders = await call.ResponseHeadersAsync.WithTimeoutAsync(timeout, token);
headerResponseTime = stopwatch.Elapsed;
}
catch (TimeoutException t)
{
cts.Cancel();
result = new PushFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t);
return(result);
}
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. To avoid an exception, exit early instead.
if (responseHeaders.Count == 0)
{
result = PushFileResult.ServerUnavailable();
return(result);
}
var pushResponse = PushResponse.FromMetadata(responseHeaders);
if (!pushResponse.ShouldCopy)
{
result = PushFileResult.Rejected(pushResponse.Rejection);
return(result);
}
// If we get a response before we finish streaming, it must be that the server cancelled the operation.
var responseStream = call.ResponseStream;
var responseMoveNext = responseStream.MoveNext(token);
var responseCompletedTask = responseMoveNext.ContinueWith(
t =>
{
// It is possible that the next operation in this method will fail
// causing stack unwinding that will dispose serverIsDoneSource.
//
// Then when responseMoveNext is done serverIsDoneSource is already disposed and
// serverIsDoneSource.Cancel will throw ObjectDisposedException.
// This exception is not observed because the stack could've been unwound before
// the result of this method is awaited.
IgnoreObjectDisposedException(() => cts.Cancel());
});
result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync(
context,
innerToken => pushFileImplementation(stream, options, startingPosition, requestStream, responseStream, responseMoveNext, responseCompletedTask, innerToken),
options,
getErrorResult : diagnostics => PushFileResult.BandwidthTimeout(diagnostics));
return(result);
}
catch (RpcException r)
{
result = new PushFileResult(r);
return(result);
}
catch (Exception)
{
exceptionThrown = true;
throw;
}
finally
{
// Even though we don't expect exceptions in this method, we can't assume they won't happen.
// So asserting that the result is not null only when the method completes successfully or with a known errors.
Contract.Assert(exceptionThrown || result != null);
if (result != null)
{
result.HeaderResponseTime = headerResponseTime;
}
}
async Task <PushFileResult> pushFileImplementation(Stream stream, CopyOptions options, long startingPosition, IClientStreamWriter <PushFileRequest> requestStream, IAsyncStreamReader <PushFileResponse> responseStream, Task <bool> responseMoveNext, Task responseCompletedTask, CancellationToken token)
{
using (var primaryBufferHandle = _pool.Get())
using (var secondaryBufferHandle = _pool.Get())
{
await StreamContentAsync(stream, primaryBufferHandle.Value, secondaryBufferHandle.Value, requestStream, options, token);
}
token.ThrowIfCancellationRequested();
await requestStream.CompleteAsync();
await responseCompletedTask;
// Make sure that we only attempt to read response when it is available.
var responseIsAvailable = await responseMoveNext;
if (!responseIsAvailable)
{
return(new PushFileResult("Failed to get final response."));
}
var response = responseStream.Current;
var size = stream.Position - startingPosition;
return(response.Header.Succeeded
? PushFileResult.PushSucceeded(size)
: new PushFileResult(response.Header.ErrorMessage));
}
}
public static PushFileResult RpcError(Exception e)
=> e.Message.Contains("StatusCode=\"DeadlineExceeded\"") ? PushFileResult.TimedOut("Deadline exceeded") : new PushFileResult(CopyResultCode.RpcError, e);
public async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Stream stream)
{
try
{
var pushRequest = new PushRequest(hash, context.TracingContext.Id);
var headers = pushRequest.GetMetadata();
using var call = _client.PushFile(headers, cancellationToken: context.Token);
var requestStream = call.RequestStream;
Metadata responseHeaders;
try
{
responseHeaders = await call.ResponseHeadersAsync.WithTimeoutAsync(_copyConnectionTimeout);
}
catch (TimeoutException t)
{
return(new PushFileResult(CopyResultCode.ConnectionTimeoutError, t));
}
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. To avoid an exception, exit early instead.
if (responseHeaders.Count == 0)
{
return(PushFileResult.ServerUnavailable());
}
var pushResponse = PushResponse.FromMetadata(responseHeaders);
if (!pushResponse.ShouldCopy)
{
return(PushFileResult.Rejected(pushResponse.Rejection));
}
// If we get a response before we finish streaming, it must be that the server cancelled the operation.
using var serverIsDoneSource = new CancellationTokenSource();
var pushCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(serverIsDoneSource.Token, context.Token).Token;
var responseStream = call.ResponseStream;
var responseMoveNext = responseStream.MoveNext(context.Token);
var responseCompletedTask = responseMoveNext.ContinueWith(
t =>
{
// It is possible that the next operation in this method will fail
// causing stack unwinding that will dispose serverIsDoneSource.
//
// Then when responseMoveNext is done serverIsDoneSource is already disposed and
// serverIsDoneSource.Cancel will throw ObjectDisposedException.
// This exception is not observed because the stack could've been unwound before
// the result of this method is awaited.
IgnoreObjectDisposedException(() => serverIsDoneSource.Cancel());
});
await StreamContentAsync(stream, new byte[_bufferSize], requestStream, pushCancellationToken);
context.Token.ThrowIfCancellationRequested();
await requestStream.CompleteAsync();
await responseCompletedTask;
// Make sure that we only attempt to read response when it is available.
var responseIsAvailable = await responseMoveNext;
if (!responseIsAvailable)
{
return(new PushFileResult("Failed to get final response."));
}
var response = responseStream.Current;
return(response.Header.Succeeded
? PushFileResult.PushSucceeded()
: new PushFileResult(response.Header.ErrorMessage));
}
catch (RpcException r)
{
return(new PushFileResult(r));
}
}