/// <summary>
/// Gets the serialized form of <paramref name="contentHash"/> but returns a handled to a pooled byte array instead of allocating a new one.
/// </summary>
public static ByteArrayPool.PoolHandle ToPooledByteArray(this in ShortHash contentHash)
{
var handle = ShortHashBytesArrayPool.Get();
contentHash.Serialize(handle.Value);
return(handle);
}
internal PoolingPayloadTokenStream(TestPayloads enclosingInstance, ByteArrayPool pool)
{
InitBlock(enclosingInstance);
this.pool = pool;
payload = pool.Get();
Enclosing_Instance.GenerateRandomData(payload);
term = pool.BytesToString(payload);
first = true;
payloadAtt = AddAttribute <IPayloadAttribute>();
termAtt = AddAttribute <ITermAttribute>();
}
internal PoolingPayloadTokenStream(TestPayloads outerInstance, ByteArrayPool pool)
{
this.outerInstance = outerInstance;
this.pool = pool;
payload = pool.Get();
this.outerInstance.GenerateRandomData(payload);
term = Encoding.UTF8.GetString(payload);
first = true;
payloadAtt = AddAttribute <IPayloadAttribute>();
termAtt = AddAttribute <ICharTermAttribute>();
}
internal PoolingPayloadTokenStream(TestPayloads outerInstance, ByteArrayPool pool)
{
this.OuterInstance = outerInstance;
this.Pool = pool;
Payload = pool.Get();
OuterInstance.GenerateRandomData(Payload);
Term = Encoding.UTF8.GetString((byte[])(Array)Payload);
First = true;
PayloadAtt = AddAttribute <IPayloadAttribute>();
TermAtt = AddAttribute <ICharTermAttribute>();
}
public async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Stream stream, CopyOptions options)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
var token = cts.Token;
bool exceptionThrown = false;
TimeSpan? headerResponseTime = null;
PushFileResult?result = null;
try
{
var startingPosition = stream.Position;
var pushRequest = new PushRequest(hash, traceId: context.TracingContext.Id);
var headers = pushRequest.GetMetadata();
using var call = _client.PushFile(options: GetDefaultGrpcOptions(headers, token));
var requestStream = call.RequestStream;
Metadata responseHeaders;
var stopwatch = StopwatchSlim.Start();
try
{
var timeout = GetResponseHeadersTimeout(options);
responseHeaders = await call.ResponseHeadersAsync.WithTimeoutAsync(timeout, token);
headerResponseTime = stopwatch.Elapsed;
}
catch (TimeoutException t)
{
cts.Cancel();
result = new PushFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t);
return(result);
}
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. To avoid an exception, exit early instead.
if (responseHeaders.Count == 0)
{
result = PushFileResult.ServerUnavailable();
return(result);
}
var pushResponse = PushResponse.FromMetadata(responseHeaders);
if (!pushResponse.ShouldCopy)
{
result = PushFileResult.Rejected(pushResponse.Rejection);
return(result);
}
// If we get a response before we finish streaming, it must be that the server cancelled the operation.
var responseStream = call.ResponseStream;
var responseMoveNext = responseStream.MoveNext(token);
var responseCompletedTask = responseMoveNext.ContinueWith(
t =>
{
// It is possible that the next operation in this method will fail
// causing stack unwinding that will dispose serverIsDoneSource.
//
// Then when responseMoveNext is done serverIsDoneSource is already disposed and
// serverIsDoneSource.Cancel will throw ObjectDisposedException.
// This exception is not observed because the stack could've been unwound before
// the result of this method is awaited.
IgnoreObjectDisposedException(() => cts.Cancel());
});
result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync(
context,
innerToken => pushFileImplementation(stream, options, startingPosition, requestStream, responseStream, responseMoveNext, responseCompletedTask, innerToken),
options,
getErrorResult : diagnostics => PushFileResult.BandwidthTimeout(diagnostics));
return(result);
}
catch (RpcException r)
{
result = new PushFileResult(r);
return(result);
}
catch (Exception)
{
exceptionThrown = true;
throw;
}
finally
{
// Even though we don't expect exceptions in this method, we can't assume they won't happen.
// So asserting that the result is not null only when the method completes successfully or with a known errors.
Contract.Assert(exceptionThrown || result != null);
if (result != null)
{
result.HeaderResponseTime = headerResponseTime;
}
}
async Task <PushFileResult> pushFileImplementation(Stream stream, CopyOptions options, long startingPosition, IClientStreamWriter <PushFileRequest> requestStream, IAsyncStreamReader <PushFileResponse> responseStream, Task <bool> responseMoveNext, Task responseCompletedTask, CancellationToken token)
{
using (var primaryBufferHandle = _pool.Get())
using (var secondaryBufferHandle = _pool.Get())
{
await StreamContentAsync(stream, primaryBufferHandle.Value, secondaryBufferHandle.Value, requestStream, options, token);
}
token.ThrowIfCancellationRequested();
await requestStream.CompleteAsync();
await responseCompletedTask;
// Make sure that we only attempt to read response when it is available.
var responseIsAvailable = await responseMoveNext;
if (!responseIsAvailable)
{
return(new PushFileResult("Failed to get final response."));
}
var response = responseStream.Current;
var size = stream.Position - startingPosition;
return(response.Header.Succeeded
? PushFileResult.PushSucceeded(size)
: new PushFileResult(response.Header.ErrorMessage));
}
}
internal PoolingPayloadTokenStream(TestPayloads enclosingInstance, ByteArrayPool pool)
{
InitBlock(enclosingInstance);
this.pool = pool;
payload = pool.Get();
Enclosing_Instance.GenerateRandomData(payload);
term = pool.BytesToString(payload);
first = true;
payloadAtt = (PayloadAttribute) AddAttribute(typeof(PayloadAttribute));
termAtt = (TermAttribute) AddAttribute(typeof(TermAttribute));
}
private async Task CopyFileAsync(CopyFileRequest request, IServerStreamWriter <CopyFileResponse> responseStream, ServerCallContext context)
{
// Get the content stream.
Context cacheContext = new Context(new Guid(request.TraceId), Logger);
ContentHash hash = request.GetContentHash();
long size = -1;
OpenStreamResult result = await GetFileStreamAsync(cacheContext, hash);
// If result is unsuccessful, then result.Stream is null, but using(null) is just a no op.
using (result.Stream)
{
// Figure out response headers.
CopyCompression compression = CopyCompression.None;
Metadata headers = new Metadata();
switch (result.Code)
{
case OpenStreamResult.ResultCode.ContentNotFound:
headers.Add("Exception", "ContentNotFound");
headers.Add("Message", $"Requested content at {hash} not found.");
break;
case OpenStreamResult.ResultCode.Error:
Contract.AssertNotNull(result.Exception);
headers.Add("Exception", result.Exception.GetType().Name);
headers.Add("Message", result.Exception.Message);
break;
case OpenStreamResult.ResultCode.Success:
Contract.AssertNotNull(result.Stream);
size = result.Stream.Length;
headers.Add("FileSize", size.ToString());
if ((request.Compression == CopyCompression.Gzip) && (size > _gzipSizeBarrier))
{
compression = CopyCompression.Gzip;
}
headers.Add("Compression", compression.ToString());
headers.Add("ChunkSize", _bufferSize.ToString());
break;
default:
throw new NotImplementedException($"Unknown result.Code '{result.Code}'.");
}
// Send the response headers.
await context.WriteResponseHeadersAsync(headers);
// Send the content.
if (result.Succeeded)
{
// Need to cancel the operation when the instance is shut down.
using var shutdownTracker = TrackShutdown(cacheContext, context.CancellationToken);
var operationContext = shutdownTracker.Context;
using var arrayHandle = _pool.Get();
using var arraySecondaryHandle = _pool.Get();
StreamContentDelegate streamContent = compression == CopyCompression.None ? (StreamContentDelegate)StreamContentAsync : StreamContentWithCompressionAsync;
byte[] buffer = arrayHandle.Value;
byte[] secondaryBuffer = arraySecondaryHandle.Value;
await operationContext.PerformOperationAsync(
_tracer,
() => streamContent(result.Stream !, buffer, secondaryBuffer, responseStream, operationContext.Token),
traceOperationStarted : false, // Tracing only stop messages
extraEndMessage : r => $"Hash={hash.ToShortString()}, GZip={(compression == CopyCompression.Gzip ? "on" : "off")}, Size=[{size}], Sender=[{context.Host}].")
.IgnoreFailure(); // The error was already logged.
}
}
}
internal PoolingPayloadTokenStream(ByteArrayPool pool)
{
this.pool = pool;
payload = pool.Get();
Lucene.Net.Index.TestPayloads.GenerateRandomData(payload);
first = true;
}
internal PoolingPayloadTokenStream(TestPayloads outerInstance, ByteArrayPool pool)
{
this.OuterInstance = outerInstance;
this.Pool = pool;
Payload = pool.Get();
OuterInstance.GenerateRandomData(Payload);
Term = Encoding.UTF8.GetString((byte[])(Array)Payload);
First = true;
PayloadAtt = AddAttribute<IPayloadAttribute>();
TermAtt = AddAttribute<ICharTermAttribute>();
}