private async Task <HttpResponseMessage> SendAsyncInternal(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (disposed)
{
throw new ObjectDisposedException(nameof(CurlWarcHandler));
}
if (request.Properties.TryGetValue("ShamanURL", out var shamanUrlObj) && shamanUrlObj is LazyUri shamanUrl)
{
shamanUrl.RemoveFragmentParameter("$assume-text");
request.Properties["ShamanURL"] = shamanUrl;
}
if (TryGetCached != null)
{
var cached = TryGetCached(request);
if (cached != null)
{
return(cached);
}
else
{
}
}
CurlEasy easy = null;
MemoryStream requestMs = null;
MemoryStream responseMs = null;
lock (lockObj)
{
easy = BorrowPooled(pooledEasyHandles);
requestMs = BorrowPooled(pooledRequestMemoryStreams);
responseMs = BorrowPooled(pooledResponseMemoryStreams);
}
Sanity.Assert(requestMs != null);
var response = new HttpResponseMessage();
var(httpCode, curlCode, warcItem) = await WebsiteScraper.ScrapeAsync(easy, request, request.RequestUri.AbsoluteUri, requestMs, responseMs, ea =>
{
return(GetDestinationWarc(request.RequestUri, easy, requestMs, responseMs));
}, syncObj, cancellationToken);
if (curlCode != CurlCode.Ok)
{
Release(easy, requestMs, responseMs);
throw new WebException("Curl: " + curlCode, (WebExceptionStatus)(800 + curlCode));
}
responseMs.Seek(0, SeekOrigin.Begin);
var httpResponse = new Utf8StreamReader(responseMs);
response.RequestMessage = request;
response.StatusCode = httpCode;
using (var scratchpad = new Scratchpad())
{
var stream = WarcItem.OpenHttp(httpResponse, scratchpad, request.RequestUri, responseMs.Length, out long payloadLength, out var _, out var _, out var contentType, out var _, (key, val) =>
{
response.Headers.TryAddWithoutValidation(key.ToString(), val.ToString());
});
response.Content = new System.Net.Http.StreamContent(new DisposeCallbackStream(stream, () =>
{
Release(easy, requestMs, responseMs);
}));
}
OnResponseReceived?.Invoke(response, easy, requestMs, responseMs);
return(response);
}
public CsvReader(Utf8StreamReader reader)
{
this.reader = reader;
}
public SqlDumpReader(Utf8StreamReader reader)
{
this.reader = reader;
this.scratchpad = new Scratchpad(8192);
}
