/// <summary>
/// Copies file to stream
/// </summary>
public async Task <CopyFileResult> CopyToAsync(Context context, ContentHash contentHash, Stream stream, CancellationToken cancellationToken, long fileSize = -1, bool enableCompression = false)
{
// TODO: Pipe through flag for compression type
CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None;
long bytesReceived = 0L;
try
{
AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest
{
TraceId = context.Id.ToString(),
HashType = (int)contentHash.HashType,
ContentHash = contentHash.ToByteString(),
// TODO: If `Drive` is expected to be the drive of the file on the source machine, then this should have nothing to do with the destination's drive
Drive = "B",
Offset = 0,
Compression = compression
});
bytesReceived = await StreamContentAsync(stream, response.ResponseStream);
}
catch (RpcException r) when(r.StatusCode == StatusCode.Unavailable && r.Message.Contains("Connect Failed"))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r, $"Failed to connect to server {_host} at port {_grpcPort}"));
}
if (bytesReceived == 0)
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Received {bytesReceived} bytes for {contentHash}. Source file does not exist"));
}
else if (fileSize >= 0 && bytesReceived != fileSize)
{
return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}"));
}
return(CopyFileResult.SuccessWithSize(bytesReceived));
}
/// <summary>
/// Copies file to stream
/// </summary>
public async Task <CopyFileResult> CopyFileAsync(Context context, ContentHash contentHash, AbsolutePath destinationPath, long fileSize = -1, bool enableCompression = false)
{
// TODO: Pipe through flag for compression type
CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None;
AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest()
{
TraceId = context.Id.ToString(),
HashType = (int)contentHash.HashType,
ContentHash = contentHash.ToByteString(),
Drive = destinationPath.DriveLetter.ToString(),
Offset = 0,
Compression = compression
});
IAsyncStreamReader <CopyFileResponse> replyStream = response.ResponseStream;
long bytesReceived = 0L;
using (var stream = new
FileStream(destinationPath.Path, FileMode.Create, FileAccess.Write, FileShare.None, ContentStore.Grpc.Utils.DefaultBufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
bytesReceived = await StreamContentAsync(stream, response.ResponseStream);
while (await replyStream.MoveNext(CancellationToken.None))
{
CopyFileResponse oneOfManyReply = replyStream.Current;
bytesReceived += oneOfManyReply.Content.Length;
oneOfManyReply.Content.WriteTo(stream);
}
}
if (fileSize >= 0 && bytesReceived != fileSize)
{
return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}"));
}
return(CopyFileResult.Success);
}
/// <summary>
/// Copies content from the server to the stream returned by the factory.
/// </summary>
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, Func <Stream> streamFactory)
{
try
{
CopyFileRequest request = new CopyFileRequest()
{
TraceId = context.TracingContext.Id.ToString(),
HashType = (int)hash.HashType,
ContentHash = hash.ToByteString(),
Offset = 0,
Compression = Key.UseCompression ? CopyCompression.Gzip : CopyCompression.None
};
AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request);
Metadata headers = await response.ResponseHeadersAsync;
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. GRPC would throw an RpcException when we tried
// to stream response, but by that time we would have created target
// stream. To avoid that, exit early instead.
if (headers.Count == 0)
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}."));
}
// Parse header collection.
string exception = null;
string message = null;
CopyCompression compression = CopyCompression.None;
foreach (Metadata.Entry header in headers)
{
switch (header.Key)
{
case "exception":
exception = header.Value;
break;
case "message":
message = header.Value;
break;
case "compression":
Enum.TryParse(header.Value, out compression);
break;
}
}
// Process reported server-side errors.
if (exception != null)
{
Contract.Assert(message != null);
switch (exception)
{
case "ContentNotFound":
return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, message));
default:
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, message));
}
}
// We got headers back with no errors, so create the target stream.
Stream targetStream;
try
{
targetStream = streamFactory();
}
catch (Exception targetException)
{
return(new CopyFileResult(CopyFileResult.ResultCode.DestinationPathError, targetException));
}
// Copy the content to the target stream.
using (targetStream)
{
switch (compression)
{
case CopyCompression.None:
await StreamContentAsync(targetStream, response.ResponseStream, context.Token);
break;
case CopyCompression.Gzip:
await StreamContentWithCompressionAsync(targetStream, response.ResponseStream, context.Token);
break;
default:
throw new NotSupportedException($"CopyCompression {compression} is not supported.");
}
}
return(CopyFileResult.Success);
}
catch (RpcException r)
{
if (r.StatusCode == StatusCode.Unavailable)
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r));
}
else
{
return(new CopyFileResult(CopyFileResult.ResultCode.Unknown, r));
}
}
}
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, CopyOptions options, Func <Stream> streamFactory, bool closeStream)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
var token = cts.Token;
bool exceptionThrown = false;
TimeSpan? headerResponseTime = null;
CopyFileResult?result = null;
try
{
CopyFileRequest request = new CopyFileRequest()
{
TraceId = context.TracingContext.Id.ToString(),
HashType = (int)hash.HashType,
ContentHash = hash.ToByteString(),
Offset = 0,
Compression = _configuration.UseGzipCompression ? CopyCompression.Gzip : CopyCompression.None,
FailFastIfBusy = options.BandwidthConfiguration?.FailFastIfServerIsBusy ?? false,
};
using AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request, options: GetDefaultGrpcOptions(token));
Metadata headers;
var stopwatch = StopwatchSlim.Start();
try
{
var connectionTimeout = GetResponseHeadersTimeout(options);
headers = await response.ResponseHeadersAsync.WithTimeoutAsync(connectionTimeout, token);
headerResponseTime = stopwatch.Elapsed;
}
catch (TimeoutException t)
{
// Trying to cancel the back end operation as well.
cts.Cancel();
result = new CopyFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t);
return(result);
}
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. GRPC would throw an RpcException when we tried
// to stream response, but by that time we would have created target
// stream. To avoid that, exit early instead.
if (headers.Count == 0)
{
result = new CopyFileResult(
CopyResultCode.ServerUnavailable,
$"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}.");
return(result);
}
// Parse header collection.
string? exception = null;
string? message = null;
CopyCompression compression = CopyCompression.None;
foreach (Metadata.Entry header in headers)
{
switch (header.Key)
{
case "exception":
exception = header.Value;
break;
case "message":
message = header.Value;
break;
case "compression":
Enum.TryParse(header.Value, out compression);
break;
}
}
// Process reported server-side errors.
if (exception != null)
{
Contract.Assert(message != null);
switch (exception)
{
case "ContentNotFound":
result = new CopyFileResult(CopyResultCode.FileNotFoundError, message);
return(result);
default:
result = new CopyFileResult(CopyResultCode.UnknownServerError, message);
return(result);
}
}
// We got headers back with no errors, so create the target stream.
Stream targetStream;
try
{
targetStream = streamFactory();
}
catch (Exception targetException)
{
result = new CopyFileResult(CopyResultCode.DestinationPathError, targetException);
return(result);
}
result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync(
context,
innerToken => copyToCoreImplementation(response, compression, targetStream, innerToken),
options,
getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics));
return(result);
}
catch (RpcException r)
{
result = CreateResultFromException(r);
return(result);
}
catch (Exception)
{
exceptionThrown = true;
throw;
}
finally
{
// Even though we don't expect exceptions in this method, we can't assume they won't happen.
// So asserting that the result is not null only when the method completes successfully or with a known errors.
Contract.Assert(exceptionThrown || result != null);
if (result != null)
{
result.HeaderResponseTime = headerResponseTime;
}
}
async Task <CopyFileResult> copyToCoreImplementation(AsyncServerStreamingCall <CopyFileResponse> response, CopyCompression compression, Stream targetStream, CancellationToken token)
{
// Copy the content to the target stream.
try
{
switch (compression)
{
case CopyCompression.None:
await StreamContentAsync(response.ResponseStream, targetStream, options, token);
break;
case CopyCompression.Gzip:
await StreamContentWithCompressionAsync(response.ResponseStream, targetStream, options, token);
break;
default:
throw new NotSupportedException($"CopyCompression {compression} is not supported.");
}
}
finally
{
if (closeStream)
{
#pragma warning disable AsyncFixer02 // A disposable object used in a fire & forget async call
targetStream.Dispose();
#pragma warning restore AsyncFixer02 // A disposable object used in a fire & forget async call
}
}
return(CopyFileResult.Success);
}
}
/// <summary>
/// Implements a copy file request.
/// </summary>
private async Task CopyFileAsync(CopyFileRequest request, IServerStreamWriter <CopyFileResponse> responseStream, ServerCallContext context)
{
try
{
LogRequestHandling();
// Get the content stream.
Context cacheContext = new Context(new Guid(request.TraceId), _logger);
HashType type = (HashType)request.HashType;
ContentHash hash = request.ContentHash.ToContentHash((HashType)request.HashType);
OpenStreamResult result = await GetFileStreamAsync(cacheContext, hash);
using (result.Stream)
{
// Figure out response headers.
CopyCompression compression = CopyCompression.None;
Metadata headers = new Metadata();
switch (result.Code)
{
case OpenStreamResult.ResultCode.ContentNotFound:
headers.Add("Exception", "ContentNotFound");
headers.Add("Message", $"Requested content at {hash} not found.");
break;
case OpenStreamResult.ResultCode.Error:
Debug.Assert(result.Exception != null);
headers.Add("Exception", result.Exception.GetType().Name);
headers.Add("Message", result.Exception.Message);
break;
case OpenStreamResult.ResultCode.Success:
Debug.Assert(result.Stream != null);
long size = result.Stream.Length;
headers.Add("FileSize", size.ToString());
if ((request.Compression == CopyCompression.Gzip) && (size > _bufferSize))
{
compression = CopyCompression.Gzip;
}
headers.Add("Compression", compression.ToString());
headers.Add("ChunkSize", _bufferSize.ToString());
break;
default:
throw new NotImplementedException();
}
// Send the response headers.
await context.WriteResponseHeadersAsync(headers);
// Send the content.
if (result.Succeeded)
{
_logger.Debug($"Streaming file through GRPC with GZip {(compression == CopyCompression.Gzip ? "on" : "off")}");
byte[] buffer = new byte[_bufferSize];
switch (compression)
{
case CopyCompression.None:
await StreamContentAsync(result.Stream, buffer, responseStream, context.CancellationToken);
break;
case CopyCompression.Gzip:
await StreamContentWithCompressionAsync(result.Stream, buffer, responseStream, context.CancellationToken);
break;
}
}
}
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Implements a copy file request.
/// </summary>
private async Task CopyFileAsync(CopyFileRequest request, IServerStreamWriter <CopyFileResponse> responseStream, ServerCallContext context)
{
OperationStarted();
// Get the content stream.
Context cacheContext = new Context(new Guid(request.TraceId), Logger);
ContentHash hash = request.GetContentHash();
OpenStreamResult result = await GetFileStreamAsync(cacheContext, hash);
// If result is unsuccessful, then result.Stream is null, but using(null) is just a no op.
using (result.Stream)
{
// Figure out response headers.
CopyCompression compression = CopyCompression.None;
Metadata headers = new Metadata();
switch (result.Code)
{
case OpenStreamResult.ResultCode.ContentNotFound:
headers.Add("Exception", "ContentNotFound");
headers.Add("Message", $"Requested content at {hash} not found.");
break;
case OpenStreamResult.ResultCode.Error:
Contract.Assert(result.Exception != null);
headers.Add("Exception", result.Exception.GetType().Name);
headers.Add("Message", result.Exception.Message);
break;
case OpenStreamResult.ResultCode.Success:
Contract.Assert(result.Stream != null);
long size = result.Stream.Length;
headers.Add("FileSize", size.ToString());
if ((request.Compression == CopyCompression.Gzip) && (size > _gzipSizeBarrier))
{
compression = CopyCompression.Gzip;
}
headers.Add("Compression", compression.ToString());
headers.Add("ChunkSize", _bufferSize.ToString());
break;
default:
throw new NotImplementedException($"Unknown result.Code '{result.Code}'.");
}
// Send the response headers.
await context.WriteResponseHeadersAsync(headers);
// Send the content.
if (result.Succeeded)
{
var operationContext = new OperationContext(cacheContext, context.CancellationToken);
using (var arrayHandle = _pool.Get())
{
StreamContentDelegate streamContent = compression == CopyCompression.None ? (StreamContentDelegate)StreamContentAsync : StreamContentWithCompressionAsync;
byte[] buffer = arrayHandle.Value;
await operationContext.PerformOperationAsync(
_tracer,
() => streamContent(result.Stream, buffer, responseStream, context.CancellationToken),
traceOperationStarted : false, // Tracing only stop messages
extraEndMessage : r => $"Hash={hash.ToShortString()}, GZip={(compression == CopyCompression.Gzip ? "on" : "off")}.")
.IgnoreFailure(); // The error was already logged.
}
}
}
}
// Providing stream factory instead of stream allows stream
// to be created only when content is actually available.
public async Task Copy(string name, Func <Stream> streamFactory)
{
CopyRequest request = new CopyRequest()
{
Name = name,
Offset = 0,
Compression = CopyCompression.Gzip
};
bool success = true;
long headerSize = -1;
CopyCompression compression = CopyCompression.None;
Stopwatch responseTimer = Stopwatch.StartNew();
AsyncServerStreamingCall <CopyReply> reply = client.Copy(request);
Metadata data = await reply.ResponseHeadersAsync.ConfigureAwait(false);
responseTimer.Stop();
foreach (Metadata.Entry header in data)
{
Console.WriteLine($"{header.Key}={header.Value}");
if (String.Compare(header.Key, "exception", StringComparison.InvariantCultureIgnoreCase) == 0)
{
success = false;
}
if (String.Compare(header.Key, "fileSize", StringComparison.InvariantCultureIgnoreCase) == 0)
{
headerSize = Int64.Parse(header.Value);
}
if (String.Compare(header.Key, "compression", StringComparison.InvariantCultureIgnoreCase) == 0)
{
compression = Enum.Parse <CopyCompression>(header.Value);
}
}
if (!success)
{
return;
}
long chunks = 0L;
long bytes = 0L;
Stopwatch streamTimer;
long measuredSize = 0L;
using (Stream writeStream = streamFactory())
{
streamTimer = Stopwatch.StartNew();
switch (compression)
{
case CopyCompression.None:
(chunks, bytes) = await StreamContent(writeStream, reply.ResponseStream).ConfigureAwait(false);
break;
case CopyCompression.Gzip:
(chunks, bytes) = await StreamContentWithCompression(writeStream, reply.ResponseStream).ConfigureAwait(false);
break;
}
streamTimer.Stop();
measuredSize = writeStream.Length;
}
Console.WriteLine($"responseTime = {responseTimer.ElapsedMilliseconds} chunks = {chunks} bytes = {bytes} headerSize = {headerSize} measuredSize = {measuredSize} streamTime = {streamTimer.ElapsedMilliseconds}");
}