public int TryReadFromCache( int position, byte[] buffer, int bufferOffset, int count, bool readFully ) { maxRequestedByte = Math.Max(maxRequestedByte, position + count); if (error != null) { throw error; } if (size.HasValue && position == size.Value) { return(0); } var avail = Interlocked.Read(ref availableBytes); if (position >= avail) { return(-1); } var slot = (int)(position / SlotSize); var offset = (int)(position % SlotSize); var slotFirstMissing = (int)(avail / SlotSize); var offsetFirstMissing = (int)(avail % SlotSize); var actualBytes = Math.Min(count, SlotSize - offset); if (slot == slotFirstMissing) { actualBytes = Math.Min(actualBytes, offsetFirstMissing - offset); } if (readFully && actualBytes != count && !completed) { return(-1); } var slotData = data[slot]; if (slotData == null) { throw new Exception("The requested data interval has been discarded and is no longer available."); } Buffer.BlockCopy(slotData, offset, buffer, bufferOffset, actualBytes); Sanity.Assert(actualBytes >= 0); return(actualBytes); }
internal void NotifyConsumerRemoved(int id, bool linger) { lock (syncObj) { var newCount = consumers--; Sanity.Assert(newCount >= 0); mediaStreams[id] = null; if (newCount == 0) { disposalCancellation = new CancellationTokenSource(); DelayedDisposalAsync(disposalCancellation.Token, false, linger); } } }
private async Task <HttpResponseMessage> SendAsyncInternal(HttpRequestMessage request, CancellationToken cancellationToken) { if (disposed) { throw new ObjectDisposedException(nameof(CurlWarcHandler)); } if (request.Properties.TryGetValue("ShamanURL", out var shamanUrlObj) && shamanUrlObj is LazyUri shamanUrl) { shamanUrl.RemoveFragmentParameter("$assume-text"); request.Properties["ShamanURL"] = shamanUrl; } if (TryGetCached != null) { var cached = TryGetCached(request); if (cached != null) { return(cached); } else { } } CurlEasy easy = null; MemoryStream requestMs = null; MemoryStream responseMs = null; lock (lockObj) { easy = BorrowPooled(pooledEasyHandles); requestMs = BorrowPooled(pooledRequestMemoryStreams); responseMs = BorrowPooled(pooledResponseMemoryStreams); } Sanity.Assert(requestMs != null); var response = new HttpResponseMessage(); var(httpCode, curlCode, warcItem) = await WebsiteScraper.ScrapeAsync(easy, request, request.RequestUri.AbsoluteUri, requestMs, responseMs, ea => { return(GetDestinationWarc(request.RequestUri, easy, requestMs, responseMs)); }, syncObj, cancellationToken); if (curlCode != CurlCode.Ok) { Release(easy, requestMs, responseMs); throw new WebException("Curl: " + curlCode, (WebExceptionStatus)(800 + curlCode)); } responseMs.Seek(0, SeekOrigin.Begin); var httpResponse = new Utf8StreamReader(responseMs); response.RequestMessage = request; response.StatusCode = httpCode; using (var scratchpad = new Scratchpad()) { var stream = WarcItem.OpenHttp(httpResponse, scratchpad, request.RequestUri, responseMs.Length, out long payloadLength, out var _, out var _, out var contentType, out var _, (key, val) => { response.Headers.TryAddWithoutValidation(key.ToString(), val.ToString()); }); response.Content = new System.Net.Http.StreamContent(new DisposeCallbackStream(stream, () => { Release(easy, requestMs, responseMs); })); } OnResponseReceived?.Invoke(response, easy, requestMs, responseMs); return(response); }