private async Task <HttpResponseMessage> SendAsyncInternal(HttpRequestMessage request, CancellationToken cancellationToken) { if (disposed) { throw new ObjectDisposedException(nameof(CurlWarcHandler)); } if (request.Properties.TryGetValue("ShamanURL", out var shamanUrlObj) && shamanUrlObj is LazyUri shamanUrl) { shamanUrl.RemoveFragmentParameter("$assume-text"); request.Properties["ShamanURL"] = shamanUrl; } if (TryGetCached != null) { var cached = TryGetCached(request); if (cached != null) { return(cached); } else { } } CurlEasy easy = null; MemoryStream requestMs = null; MemoryStream responseMs = null; lock (lockObj) { easy = BorrowPooled(pooledEasyHandles); requestMs = BorrowPooled(pooledRequestMemoryStreams); responseMs = BorrowPooled(pooledResponseMemoryStreams); } Sanity.Assert(requestMs != null); var response = new HttpResponseMessage(); var(httpCode, curlCode, warcItem) = await WebsiteScraper.ScrapeAsync(easy, request, request.RequestUri.AbsoluteUri, requestMs, responseMs, ea => { return(GetDestinationWarc(request.RequestUri, easy, requestMs, responseMs)); }, syncObj, cancellationToken); if (curlCode != CurlCode.Ok) { Release(easy, requestMs, responseMs); throw new WebException("Curl: " + curlCode, (WebExceptionStatus)(800 + curlCode)); } responseMs.Seek(0, SeekOrigin.Begin); var httpResponse = new Utf8StreamReader(responseMs); response.RequestMessage = request; response.StatusCode = httpCode; using (var scratchpad = new Scratchpad()) { var stream = WarcItem.OpenHttp(httpResponse, scratchpad, request.RequestUri, responseMs.Length, out long payloadLength, out var _, out var _, out var contentType, out var _, (key, val) => { response.Headers.TryAddWithoutValidation(key.ToString(), val.ToString()); }); response.Content = new System.Net.Http.StreamContent(new DisposeCallbackStream(stream, () => { Release(easy, requestMs, responseMs); })); } OnResponseReceived?.Invoke(response, easy, requestMs, responseMs); return(response); }
public CsvReader(Utf8StreamReader reader) { this.reader = reader; }
public SqlDumpReader(Utf8StreamReader reader) { this.reader = reader; this.scratchpad = new Scratchpad(8192); }