public int TryReadFromCache(
int position,
byte[] buffer,
int bufferOffset,
int count,
bool readFully
)
{
maxRequestedByte = Math.Max(maxRequestedByte, position + count);
if (error != null)
{
throw error;
}
if (size.HasValue && position == size.Value)
{
return(0);
}
var avail = Interlocked.Read(ref availableBytes);
if (position >= avail)
{
return(-1);
}
var slot = (int)(position / SlotSize);
var offset = (int)(position % SlotSize);
var slotFirstMissing = (int)(avail / SlotSize);
var offsetFirstMissing = (int)(avail % SlotSize);
var actualBytes = Math.Min(count, SlotSize - offset);
if (slot == slotFirstMissing)
{
actualBytes = Math.Min(actualBytes, offsetFirstMissing - offset);
}
if (readFully && actualBytes != count && !completed)
{
return(-1);
}
var slotData = data[slot];
if (slotData == null)
{
throw new Exception("The requested data interval has been discarded and is no longer available.");
}
Buffer.BlockCopy(slotData, offset, buffer, bufferOffset, actualBytes);
Sanity.Assert(actualBytes >= 0);
return(actualBytes);
}
internal void NotifyConsumerRemoved(int id, bool linger)
{
lock (syncObj)
{
var newCount = consumers--;
Sanity.Assert(newCount >= 0);
mediaStreams[id] = null;
if (newCount == 0)
{
disposalCancellation = new CancellationTokenSource();
DelayedDisposalAsync(disposalCancellation.Token, false, linger);
}
}
}
private async Task <HttpResponseMessage> SendAsyncInternal(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (disposed)
{
throw new ObjectDisposedException(nameof(CurlWarcHandler));
}
if (request.Properties.TryGetValue("ShamanURL", out var shamanUrlObj) && shamanUrlObj is LazyUri shamanUrl)
{
shamanUrl.RemoveFragmentParameter("$assume-text");
request.Properties["ShamanURL"] = shamanUrl;
}
if (TryGetCached != null)
{
var cached = TryGetCached(request);
if (cached != null)
{
return(cached);
}
else
{
}
}
CurlEasy easy = null;
MemoryStream requestMs = null;
MemoryStream responseMs = null;
lock (lockObj)
{
easy = BorrowPooled(pooledEasyHandles);
requestMs = BorrowPooled(pooledRequestMemoryStreams);
responseMs = BorrowPooled(pooledResponseMemoryStreams);
}
Sanity.Assert(requestMs != null);
var response = new HttpResponseMessage();
var(httpCode, curlCode, warcItem) = await WebsiteScraper.ScrapeAsync(easy, request, request.RequestUri.AbsoluteUri, requestMs, responseMs, ea =>
{
return(GetDestinationWarc(request.RequestUri, easy, requestMs, responseMs));
}, syncObj, cancellationToken);
if (curlCode != CurlCode.Ok)
{
Release(easy, requestMs, responseMs);
throw new WebException("Curl: " + curlCode, (WebExceptionStatus)(800 + curlCode));
}
responseMs.Seek(0, SeekOrigin.Begin);
var httpResponse = new Utf8StreamReader(responseMs);
response.RequestMessage = request;
response.StatusCode = httpCode;
using (var scratchpad = new Scratchpad())
{
var stream = WarcItem.OpenHttp(httpResponse, scratchpad, request.RequestUri, responseMs.Length, out long payloadLength, out var _, out var _, out var contentType, out var _, (key, val) =>
{
response.Headers.TryAddWithoutValidation(key.ToString(), val.ToString());
});
response.Content = new System.Net.Http.StreamContent(new DisposeCallbackStream(stream, () =>
{
Release(easy, requestMs, responseMs);
}));
}
OnResponseReceived?.Invoke(response, easy, requestMs, responseMs);
return(response);
}