internal void AddHeaders(List <KeyValuePair <string, string> > headers) { this.ExpectedContentLength = -1; for (int i = 0; i < headers.Count; ++i) { KeyValuePair <string, string> header = headers[i]; if (header.Key.Equals(":status", StringComparison.Ordinal)) { base.StatusCode = int.Parse(header.Value); base.Message = string.Empty; } else { if (!this.IsCompressed && header.Key.Equals("content-encoding", StringComparison.OrdinalIgnoreCase)) { this.IsCompressed = true; } else if (base.baseRequest.OnDownloadProgress != null && header.Key.Equals("content-length", StringComparison.OrdinalIgnoreCase)) { this.ExpectedContentLength = int.Parse(header.Value); } base.AddHeader(header.Key, header.Value); } } if (this.ExpectedContentLength == -1 && base.baseRequest.OnDownloadProgress != null) { HTTPManager.Logger.Information("HTTP2Response", "AddHeaders - No Content-Length header found!"); } RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.Headers)); }
protected void AddStreamedFragment(byte[] buffer, int bufferLength) { #if !BESTHTTP_DISABLE_CACHING if (!IsCacheOnly) #endif { if (this.baseRequest.OnStreamingData != null && buffer != null && bufferLength > 0) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, buffer, bufferLength)); Interlocked.Increment(ref this.UnprocessedFragments); } } if (HTTPManager.Logger.Level == Logger.Loglevels.All && buffer != null) { VerboseLogging(string.Format("AddStreamedFragment buffer length: {0:N0} UnprocessedFragments: {1:N0}", bufferLength, Interlocked.Read(ref this.UnprocessedFragments))); } #if !BESTHTTP_DISABLE_CACHING if (cacheStream != null) { cacheStream.Write(buffer, 0, bufferLength); allFragmentSize += bufferLength; } #endif }
public void Abort(string msg) { if (this.AssignedRequest.State != HTTPRequestStates.Processing) { // do nothing, its state is already set. } else if (this.AssignedRequest.IsCancellationRequested) { this.AssignedRequest.Response = null; this.AssignedRequest.State = this.AssignedRequest.IsTimedOut ? HTTPRequestStates.TimedOut : HTTPRequestStates.Aborted; this.State = HTTP2StreamStates.Closed; } else if (this.AssignedRequest.Retries >= this.AssignedRequest.MaxRetries) { this.AssignedRequest.Response = null; this.AssignedRequest.Exception = new Exception(msg); this.AssignedRequest.State = HTTPRequestStates.Error; this.State = HTTP2StreamStates.Closed; } else { this.AssignedRequest.Retries++; RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.AssignedRequest, RequestEvents.Resend)); } // After receiving a RST_STREAM on a stream, the receiver MUST NOT send additional frames for that stream, with the exception of PRIORITY. this.outgoing.Clear(); }
private static void FinishRequest(HTTPRequest req, HTTP2Response resp, FramesAsStreamView dataStream) { if (dataStream != null) { resp.AddData(dataStream); dataStream.Close(); } bool resendRequest; HTTPConnectionStates proposedConnectionStates; KeepAliveHeader keepAliveHeader = null; ConnectionHelper.HandleResponse("HTTP2Stream", req, out resendRequest, out proposedConnectionStates, ref keepAliveHeader); if (resendRequest && !req.IsCancellationRequested) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(req, RequestEvents.Resend)); } else if (req.State == HTTPRequestStates.Processing && !req.IsCancellationRequested) { req.State = HTTPRequestStates.Finished; } else { if (req.State == HTTPRequestStates.Processing && req.IsCancellationRequested) { req.State = req.IsTimedOut ? HTTPRequestStates.TimedOut : HTTPRequestStates.Aborted; } } }
private void OnSendMessagesFinished(HTTPRequest req, HTTPResponse resp) { /* * The HTTP POST request is made to the URL [endpoint-base]. The mandatory id query string value is used to identify the connection to send to. * If there is no id query string value, a 400 Bad Request response is returned. Upon receipt of the entire payload, * the server will process the payload and responds with 200 OK if the payload was successfully processed. * If a client makes another request to / while an existing request is outstanding, the new request is immediately terminated by the server with the 409 Conflict status code. * * If a client receives a 409 Conflict request, the connection remains open. * Any other response indicates that the connection has been terminated due to an error. * * If the relevant connection has been terminated, a 404 Not Found status code is returned. * If there is an error instantiating an EndPoint or dispatching the message, a 500 Server Error status code is returned. * */ switch (req.State) { // The request finished without any problem. case HTTPRequestStates.Finished: switch (resp.StatusCode) { // Upon receipt of the entire payload, the server will process the payload and responds with 200 OK if the payload was successfully processed. case 200: Interlocked.Exchange(ref this.sendingInProgress, 0); SendMessages(); break; // Any other response indicates that the connection has been terminated due to an error. default: this.ErrorReason = string.Format("Send Request finished Successfully, but the server sent an error. Status Code: {0}-{1} Message: {2}", resp.StatusCode, resp.Message, resp.DataAsText); break; } break; default: int retryCount = (int)req.Tag; if (retryCount < MaxRetries) { req.Tag = retryCount + 1; RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(req, RequestEvents.Resend)); } else { this.ErrorReason = string.Format("Send message reached max retry count ({0})!", MaxRetries); } break; } if (!string.IsNullOrEmpty(this.ErrorReason)) { this.State = TransportStates.Failed; } }
public static HTTPRequest SendRequest(HTTPRequest request) { if (request.IsCancellationRequested || IsQuitting) { return(request); } if (!IsSetupCalled) { Setup(); } RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(request, RequestEvents.Resend)); return(request); }
public static HTTPRequest SendRequest(HTTPRequest request) { if (!IsSetupCalled) { Setup(); } if (request.IsCancellationRequested || IsQuitting) { return(request); } #if !BESTHTTP_DISABLE_CACHING // If possible load the full response from cache. if (Caching.HTTPCacheService.IsCachedEntityExpiresInTheFuture(request)) { DateTime started = DateTime.Now; PlatformSupport.Threading.ThreadedRunner.RunShortLiving <HTTPRequest>((req) => { if (Connections.ConnectionHelper.TryLoadAllFromCache("HTTPManager", req, req.Context)) { req.Timing.Add("Full Cache Load", DateTime.Now - started); req.State = HTTPRequestStates.Finished; } else { // If for some reason it couldn't load we place back the request to the queue. request.State = HTTPRequestStates.Queued; RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(request, RequestEvents.Resend)); } }, request); } else #endif { request.State = HTTPRequestStates.Queued; RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(request, RequestEvents.Resend)); } return(request); }
private static void FinishRequest(HTTP2Stream stream, FramesAsStreamView dataStream) { if (dataStream != null) { try { stream.response.AddData(dataStream); } finally { dataStream.Close(); } } stream.AssignedRequest.Timing.Add(TimingEventNames.Response_Received); bool resendRequest; HTTPConnectionStates proposedConnectionStates; // ignored KeepAliveHeader keepAliveHeader = null; // ignored ConnectionHelper.HandleResponse("HTTP2Stream", stream.AssignedRequest, out resendRequest, out proposedConnectionStates, ref keepAliveHeader, stream.Context, stream.AssignedRequest.Context); if (resendRequest && !stream.AssignedRequest.IsCancellationRequested) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(stream.AssignedRequest, RequestEvents.Resend)); } else if (stream.AssignedRequest.State == HTTPRequestStates.Processing && !stream.AssignedRequest.IsCancellationRequested) { stream.AssignedRequest.State = HTTPRequestStates.Finished; } else { if (stream.AssignedRequest.State == HTTPRequestStates.Processing && stream.AssignedRequest.IsCancellationRequested) { stream.AssignedRequest.State = stream.AssignedRequest.IsTimedOut ? HTTPRequestStates.TimedOut : HTTPRequestStates.Aborted; } } }
protected void ReadHeaders(Stream stream) { string headerName = ReadTo(stream, (byte)':', LF) /*.Trim()*/; while (headerName != string.Empty) { string value = ReadTo(stream, LF); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("Header - '{0}': '{1}'", headerName, value)); } AddHeader(headerName, value); headerName = ReadTo(stream, (byte)':', LF); } if (this.baseRequest.OnHeadersReceived != null) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.Headers)); } }
private static void FinishRequest(HTTPRequest req, HTTP2Response resp, FramesAsStreamView dataStream) { if (dataStream != null) { resp.AddData(dataStream); dataStream.Close(); } bool resendRequest; HTTPConnectionStates proposedConnectionStates; KeepAliveHeader keepAliveHeader = null; ConnectionHelper.HandleResponse("HTTP2Stream", req, out resendRequest, out proposedConnectionStates, ref keepAliveHeader); if (resendRequest) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(req, RequestEvents.Resend)); } else { req.State = HTTPRequestStates.Finished; } }
private void ProcessState(List <HTTP2FrameHeaderAndPayload> outgoingFrames) { switch (this.State) { case HTTP2StreamStates.Idle: UInt32 initiatedInitialWindowSize = this.settings.InitiatedMySettings[HTTP2Settings.INITIAL_WINDOW_SIZE]; this.localWindow = initiatedInitialWindowSize; // window update with a zero increment would be an error (https://httpwg.org/specs/rfc7540.html#WINDOW_UPDATE) //if (HTTP2Connection.MaxValueFor31Bits > initiatedInitialWindowSize) // this.outgoing.Enqueue(HTTP2FrameHelper.CreateWindowUpdateFrame(this.Id, HTTP2Connection.MaxValueFor31Bits - initiatedInitialWindowSize)); //this.localWindow = HTTP2Connection.MaxValueFor31Bits; #if !BESTHTTP_DISABLE_CACHING // Setup cache control headers before we send out the request if (!this.AssignedRequest.DisableCache) { HTTPCacheService.SetHeaders(this.AssignedRequest); } #endif // hpack encode the request's header this.encoder.Encode(this, this.AssignedRequest, this.outgoing, this.Id); // HTTP/2 uses DATA frames to carry message payloads. // The chunked transfer encoding defined in Section 4.1 of [RFC7230] MUST NOT be used in HTTP/2. this.uploadStreamInfo = this.AssignedRequest.GetUpStream(); //this.State = HTTP2StreamStates.Open; if (this.uploadStreamInfo.Stream == null) { this.State = HTTP2StreamStates.HalfClosedLocal; } else { this.State = HTTP2StreamStates.Open; } break; case HTTP2StreamStates.Open: // remote Window can be negative! See https://httpwg.org/specs/rfc7540.html#InitialWindowSize if (this.remoteWindow <= 0) { HTTPManager.Logger.Warning("HTTP2Stream", string.Format("[{0}] Skipping data sending as remote Window is {1}!", this.Id, this.remoteWindow)); return; } // This step will send one frame per OpenState call. Int64 maxFrameSize = Math.Min(this.remoteWindow, this.settings.RemoteSettings[HTTP2Settings.MAX_FRAME_SIZE]); HTTP2FrameHeaderAndPayload frame = new HTTP2FrameHeaderAndPayload(); frame.Type = HTTP2FrameTypes.DATA; frame.StreamId = this.Id; frame.Payload = BufferPool.Get(maxFrameSize, true); int readCount = this.uploadStreamInfo.Stream.Read(frame.Payload, 0, (int)Math.Min(maxFrameSize, int.MaxValue)); if (readCount <= 0) { BufferPool.Release(frame.Payload); frame.Payload = null; frame.PayloadLength = 0; } else { frame.PayloadLength = (UInt32)readCount; } frame.PayloadOffset = 0; frame.DontUseMemPool = false; if (readCount <= 0) { this.uploadStreamInfo.Stream.Dispose(); this.uploadStreamInfo = new HTTPRequest.UploadStreamInfo(); frame.Flags = (byte)(HTTP2DataFlags.END_STREAM); this.State = HTTP2StreamStates.HalfClosedLocal; } this.outgoing.Enqueue(frame); this.remoteWindow -= frame.PayloadLength; this.sentData += frame.PayloadLength; if (this.AssignedRequest.OnUploadProgress != null) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.AssignedRequest, RequestEvents.UploadProgress, this.sentData, this.uploadStreamInfo.Length)); } //HTTPManager.Logger.Information("HTTP2Stream", string.Format("[{0}] New DATA frame created! remoteWindow: {1:N0}", this.Id, this.remoteWindow)); break; case HTTP2StreamStates.HalfClosedLocal: if (this.TimeSpentInCurrentState >= CloseStreamAfter && !this.isStreamedDownload) { this.State = HTTP2StreamStates.Closed; } break; case HTTP2StreamStates.HalfClosedRemote: break; case HTTP2StreamStates.Closed: break; } }
public virtual bool Receive(int forceReadRawContentLength = -1, bool readPayloadData = true, bool sendUpgradedEvent = true) { if (this.baseRequest.IsCancellationRequested) { return(false); } string statusLine = string.Empty; if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("Receive. forceReadRawContentLength: '{0:N0}', readPayloadData: '{1:N0}'", forceReadRawContentLength, readPayloadData)); } // On WP platform we aren't able to determined sure enough whether the tcp connection is closed or not. // So if we get an exception here, we need to recreate the connection. try { // Read out 'HTTP/1.1' from the "HTTP/1.1 {StatusCode} {Message}" statusLine = ReadTo(Stream, (byte)' '); } catch { if (baseRequest.Retries >= baseRequest.MaxRetries) { HTTPManager.Logger.Warning("HTTPResponse", string.Format("{0} - Failed to read Status Line! Retry is enabled, returning with false.", this.baseRequest.CurrentUri.ToString())); return(false); } HTTPManager.Logger.Warning("HTTPResponse", string.Format("{0} - Failed to read Status Line! Retry is disabled, re-throwing exception.", this.baseRequest.CurrentUri.ToString())); throw; } if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("Status Line: '{0}'", statusLine)); } if (string.IsNullOrEmpty(statusLine)) { if (baseRequest.Retries >= baseRequest.MaxRetries) { return(false); } throw new Exception("Remote server closed the connection before sending response header!"); } string[] versions = statusLine.Split(new char[] { '/', '.' }); this.VersionMajor = int.Parse(versions[1]); this.VersionMinor = int.Parse(versions[2]); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("HTTP Version: '{0}.{1}'", this.VersionMajor.ToString(), this.VersionMinor.ToString())); } int statusCode; string statusCodeStr = NoTrimReadTo(Stream, (byte)' ', LF); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("Status Code: '{0}'", statusCodeStr)); } if (baseRequest.Retries >= baseRequest.MaxRetries) { statusCode = int.Parse(statusCodeStr); } else if (!int.TryParse(statusCodeStr, out statusCode)) { return(false); } this.StatusCode = statusCode; if (statusCodeStr.Length > 0 && (byte)statusCodeStr[statusCodeStr.Length - 1] != LF && (byte)statusCodeStr[statusCodeStr.Length - 1] != CR) { this.Message = ReadTo(Stream, LF); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("Status Message: '{0}'", this.Message)); } } else { HTTPManager.Logger.Warning("HTTPResponse", string.Format("{0} - Skipping Status Message reading!", this.baseRequest.CurrentUri.ToString())); this.Message = string.Empty; } //Read Headers ReadHeaders(Stream); IsUpgraded = StatusCode == 101 && (HasHeaderWithValue("connection", "upgrade") || HasHeader("upgrade")); if (IsUpgraded) { if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging("Request Upgraded!"); } RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.Upgraded)); } if (!readPayloadData) { return(true); } return(ReadPayload(forceReadRawContentLength)); }
void OnDownloadProgress(int down, int total) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.CurrentRequest, RequestEvents.DownloadProgress, down, total)); }
protected void ReadUnknownSize(Stream stream) { // Progress report: long Downloaded = 0; long DownloadLength = 0; bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess #if !BESTHTTP_DISABLE_CACHING || this.IsFromCache #endif ); string encoding = #if !BESTHTTP_DISABLE_CACHING IsFromCache ? null : #endif GetFirstHeaderValue("content-encoding"); bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip"; Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null; using (var output = new BufferPoolMemoryStream()) { byte[] buffer = BufferPool.Get(Math.Max(baseRequest.StreamFragmentSize, MinBufferSize), false); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("ReadUnknownSize - buffer size: {0:N0}", buffer.Length)); } int readBytes = 0; int bytes = 0; do { readBytes = 0; do { if (this.baseRequest.IsCancellationRequested) { return; } bytes = 0; #if !NETFX_CORE || UNITY_EDITOR NetworkStream networkStream = stream as NetworkStream; // If we have the good-old NetworkStream, than we can use the DataAvailable property. On WP8 platforms, these are omitted... :/ if (networkStream != null && baseRequest.EnableSafeReadOnUnknownContentLength) { for (int i = readBytes; i < buffer.Length && networkStream.DataAvailable; ++i) { int read = stream.ReadByte(); if (read >= 0) { buffer[i] = (byte)read; bytes++; } else { break; } } } else // This will be good anyway, but a little slower. #endif { bytes = stream.Read(buffer, readBytes, buffer.Length - readBytes); } readBytes += bytes; // Progress report: Downloaded += bytes; DownloadLength = Downloaded; if (sendProgressChanged) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength)); } } while (readBytes < buffer.Length && bytes > 0); if (baseRequest.UseStreaming) { if (gzipped) { var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true); if (decompressed.Data != null) { FeedStreamFragment(decompressed.Data, 0, decompressed.Length); } } else { FeedStreamFragment(buffer, 0, readBytes); } } else { output.Write(buffer, 0, readBytes); } } while (bytes > 0); BufferPool.Release(buffer); if (baseRequest.UseStreaming) { if (gzipped) { var decompressed = decompressor.Decompress(null, 0, 0, true, true); if (decompressed.Data != null) { FeedStreamFragment(decompressed.Data, 0, decompressed.Length); } } FlushRemainingFragmentBuffer(); } if (!baseRequest.UseStreaming) { this.Data = DecodeStream(output); } } if (decompressor != null) { decompressor.Dispose(); } }
public void RunHandler() { HTTPManager.Logger.Information("HTTP1Handler", string.Format("[{0}] started processing request '{1}'", this, this.conn.CurrentRequest.CurrentUri.ToString()), this.Context, this.conn.CurrentRequest.Context); System.Threading.Thread.CurrentThread.Name = "BestHTTP.HTTP1 R&W"; HTTPConnectionStates proposedConnectionState = HTTPConnectionStates.Processing; bool resendRequest = false; try { if (this.conn.CurrentRequest.IsCancellationRequested) { return; } #if !BESTHTTP_DISABLE_CACHING // Setup cache control headers before we send out the request if (!this.conn.CurrentRequest.DisableCache) { HTTPCacheService.SetHeaders(this.conn.CurrentRequest); } #endif // Write the request to the stream this.conn.CurrentRequest.QueuedAt = DateTime.MinValue; this.conn.CurrentRequest.ProcessingStarted = DateTime.UtcNow; this.conn.CurrentRequest.SendOutTo(this.conn.connector.Stream); this.conn.CurrentRequest.Timing.Add(TimingEventNames.Request_Sent); if (this.conn.CurrentRequest.IsCancellationRequested) { return; } this.conn.CurrentRequest.OnCancellationRequested += OnCancellationRequested; // Receive response from the server bool received = Receive(this.conn.CurrentRequest); this.conn.CurrentRequest.Timing.Add(TimingEventNames.Response_Received); if (this.conn.CurrentRequest.IsCancellationRequested) { return; } if (!received && this.conn.CurrentRequest.Retries < this.conn.CurrentRequest.MaxRetries) { proposedConnectionState = HTTPConnectionStates.Closed; this.conn.CurrentRequest.Retries++; resendRequest = true; return; } ConnectionHelper.HandleResponse(this.conn.ToString(), this.conn.CurrentRequest, out resendRequest, out proposedConnectionState, ref this._keepAlive, this.conn.Context, this.conn.CurrentRequest.Context); } catch (TimeoutException e) { this.conn.CurrentRequest.Response = null; // Do nothing here if Abort() got called on the request, its State is already set. if (!this.conn.CurrentRequest.IsTimedOut) { // We will try again only once if (this.conn.CurrentRequest.Retries < this.conn.CurrentRequest.MaxRetries) { this.conn.CurrentRequest.Retries++; resendRequest = true; } else { this.conn.CurrentRequest.Exception = e; this.conn.CurrentRequest.State = HTTPRequestStates.ConnectionTimedOut; } } proposedConnectionState = HTTPConnectionStates.Closed; } catch (Exception e) { if (this.ShutdownType == ShutdownTypes.Immediate) { return; } string exceptionMessage = string.Empty; if (e == null) { exceptionMessage = "null"; } else { System.Text.StringBuilder sb = new System.Text.StringBuilder(); Exception exception = e; int counter = 1; while (exception != null) { sb.AppendFormat("{0}: {1} {2}", counter++.ToString(), exception.Message, exception.StackTrace); exception = exception.InnerException; if (exception != null) { sb.AppendLine(); } } exceptionMessage = sb.ToString(); } HTTPManager.Logger.Verbose("HTTP1Handler", exceptionMessage, this.Context, this.conn.CurrentRequest.Context); #if !BESTHTTP_DISABLE_CACHING if (this.conn.CurrentRequest.UseStreaming) { HTTPCacheService.DeleteEntity(this.conn.CurrentRequest.CurrentUri); } #endif // Something gone bad, Response must be null! this.conn.CurrentRequest.Response = null; // Do nothing here if Abort() got called on the request, its State is already set. if (!this.conn.CurrentRequest.IsCancellationRequested) { this.conn.CurrentRequest.Exception = e; this.conn.CurrentRequest.State = HTTPRequestStates.Error; } proposedConnectionState = HTTPConnectionStates.Closed; } finally { this.conn.CurrentRequest.OnCancellationRequested -= OnCancellationRequested; // Exit ASAP if (this.ShutdownType != ShutdownTypes.Immediate) { if (this.conn.CurrentRequest.IsCancellationRequested) { // we don't know what stage the request is canceled, we can't safely reuse the tcp channel. proposedConnectionState = HTTPConnectionStates.Closed; this.conn.CurrentRequest.Response = null; // The request's State already set, or going to be set soon in RequestEvents.cs. //this.conn.CurrentRequest.State = this.conn.CurrentRequest.IsTimedOut ? HTTPRequestStates.TimedOut : HTTPRequestStates.Aborted; } else if (resendRequest) { // Here introducing a ClosedResendRequest connection state, where we have to process the connection's state change to Closed // than we have to resend the request. // If we would send the Resend request here, than a few lines below the Closed connection state change, // request events are processed before connection events (just switching the EnqueueRequestEvent and EnqueueConnectionEvent wouldn't work // see order of ProcessQueues in HTTPManager.OnUpdate!) and it would pick this very same closing/closed connection! if (proposedConnectionState == HTTPConnectionStates.Closed) { ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this.conn, this.conn.CurrentRequest)); } else { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.conn.CurrentRequest, RequestEvents.Resend)); } } else if (this.conn.CurrentRequest.Response != null && this.conn.CurrentRequest.Response.IsUpgraded) { proposedConnectionState = HTTPConnectionStates.WaitForProtocolShutdown; } else if (this.conn.CurrentRequest.State == HTTPRequestStates.Processing) { if (this.conn.CurrentRequest.Response != null) { this.conn.CurrentRequest.State = HTTPRequestStates.Finished; } else { this.conn.CurrentRequest.Exception = new Exception(string.Format("[{0}] Remote server closed the connection before sending response header! Previous request state: {1}. Connection state: {2}", this.ToString(), this.conn.CurrentRequest.State.ToString(), this.conn.State.ToString())); this.conn.CurrentRequest.State = HTTPRequestStates.Error; proposedConnectionState = HTTPConnectionStates.Closed; } } this.conn.CurrentRequest = null; if (proposedConnectionState == HTTPConnectionStates.Processing) { proposedConnectionState = HTTPConnectionStates.Recycle; } if (proposedConnectionState != HTTPConnectionStates.ClosedResendRequest) { ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this.conn, proposedConnectionState)); } } } }
void OnUploadProgress(int up, int total) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.CurrentRequest, RequestEvents.UploadProgress, up, total)); }
void OnResponse(int httpStatus, byte[] buffer, int bufferLength) { HTTPConnectionStates proposedConnectionState = HTTPConnectionStates.Processing; bool resendRequest = false; try { if (this.CurrentRequest.IsCancellationRequested) { return; } using (var ms = new BufferPoolMemoryStream()) { Stream = ms; XHR_GetStatusLine(NativeId, OnBufferCallback); XHR_GetResponseHeaders(NativeId, OnBufferCallback); if (buffer != null && bufferLength > 0) { ms.Write(buffer, 0, bufferLength); } ms.Seek(0L, SeekOrigin.Begin); var internalBuffer = ms.GetBuffer(); string tmp = System.Text.Encoding.UTF8.GetString(internalBuffer); HTTPManager.Logger.Information(this.NativeId + " OnResponse - full response ", tmp, this.Context); SupportedProtocols protocol = CurrentRequest.ProtocolHandler == SupportedProtocols.Unknown ? HTTPProtocolFactory.GetProtocolFromUri(CurrentRequest.CurrentUri) : CurrentRequest.ProtocolHandler; CurrentRequest.Response = HTTPProtocolFactory.Get(protocol, CurrentRequest, ms, CurrentRequest.UseStreaming, false); CurrentRequest.Response.Receive(buffer != null && bufferLength > 0 ? (int)bufferLength : -1, true); KeepAliveHeader keepAlive = null; ConnectionHelper.HandleResponse(this.ToString(), this.CurrentRequest, out resendRequest, out proposedConnectionState, ref keepAlive); } } catch (Exception e) { HTTPManager.Logger.Exception(this.NativeId + " WebGLConnection", "OnResponse", e, this.Context); if (this.ShutdownType == ShutdownTypes.Immediate) { return; } #if !BESTHTTP_DISABLE_CACHING if (this.CurrentRequest.UseStreaming) { HTTPCacheService.DeleteEntity(this.CurrentRequest.CurrentUri); } #endif // Something gone bad, Response must be null! this.CurrentRequest.Response = null; if (!this.CurrentRequest.IsCancellationRequested) { this.CurrentRequest.Exception = e; this.CurrentRequest.State = HTTPRequestStates.Error; } proposedConnectionState = HTTPConnectionStates.Closed; } finally { // Exit ASAP if (this.ShutdownType != ShutdownTypes.Immediate) { if (this.CurrentRequest.IsCancellationRequested) { // we don't know what stage the request is cancelled, we can't safely reuse the tcp channel. proposedConnectionState = HTTPConnectionStates.Closed; this.CurrentRequest.Response = null; this.CurrentRequest.State = this.CurrentRequest.IsTimedOut ? HTTPRequestStates.TimedOut : HTTPRequestStates.Aborted; } else if (resendRequest) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.CurrentRequest, RequestEvents.Resend)); } else if (this.CurrentRequest.Response != null && this.CurrentRequest.Response.IsUpgraded) { proposedConnectionState = HTTPConnectionStates.WaitForProtocolShutdown; } else if (this.CurrentRequest.State == HTTPRequestStates.Processing) { if (this.CurrentRequest.Response != null) { this.CurrentRequest.State = HTTPRequestStates.Finished; } else { this.CurrentRequest.Exception = new Exception(string.Format("[{0}] Remote server closed the connection before sending response header! Previous request state: {1}. Connection state: {2}", this.ToString(), this.CurrentRequest.State.ToString(), this.State.ToString())); this.CurrentRequest.State = HTTPRequestStates.Error; proposedConnectionState = HTTPConnectionStates.Closed; } } this.CurrentRequest = null; if (proposedConnectionState == HTTPConnectionStates.Processing) { proposedConnectionState = HTTPConnectionStates.Closed; } ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, proposedConnectionState)); } } }
public void RunHandler() { HTTPManager.Logger.Information("HTTP1Handler", string.Format("[{0}] started processing request '{1}'", this, this.conn.CurrentRequest.CurrentUri.ToString()), this.Context, this.conn.CurrentRequest.Context); HTTPConnectionStates proposedConnectionState = HTTPConnectionStates.Processing; bool resendRequest = false; try { if (this.conn.CurrentRequest.IsCancellationRequested) { return; } #if !BESTHTTP_DISABLE_CACHING // Setup cache control headers before we send out the request if (!this.conn.CurrentRequest.DisableCache) { HTTPCacheService.SetHeaders(this.conn.CurrentRequest); } #endif // Write the request to the stream this.conn.CurrentRequest.SendOutTo(this.conn.connector.Stream); if (this.conn.CurrentRequest.IsCancellationRequested) { return; } this.conn.CurrentRequest.OnCancellationRequested += OnCancellationRequested; // Receive response from the server bool received = Receive(this.conn.CurrentRequest); if (this.conn.CurrentRequest.IsCancellationRequested) { return; } if (!received && this.conn.CurrentRequest.Retries < this.conn.CurrentRequest.MaxRetries) { proposedConnectionState = HTTPConnectionStates.Closed; this.conn.CurrentRequest.Retries++; resendRequest = true; return; } ConnectionHelper.HandleResponse(this.conn.ToString(), this.conn.CurrentRequest, out resendRequest, out proposedConnectionState, ref this._keepAlive, this.conn.Context, this.conn.CurrentRequest.Context); } catch (TimeoutException e) { this.conn.CurrentRequest.Response = null; // We will try again only once if (this.conn.CurrentRequest.Retries < this.conn.CurrentRequest.MaxRetries) { this.conn.CurrentRequest.Retries++; resendRequest = true; } else { this.conn.CurrentRequest.Exception = e; this.conn.CurrentRequest.State = HTTPRequestStates.ConnectionTimedOut; } proposedConnectionState = HTTPConnectionStates.Closed; } catch (Exception e) { if (this.ShutdownType == ShutdownTypes.Immediate) { return; } string exceptionMessage = string.Empty; if (e == null) { exceptionMessage = "null"; } else { System.Text.StringBuilder sb = new System.Text.StringBuilder(); Exception exception = e; int counter = 1; while (exception != null) { sb.AppendFormat("{0}: {1} {2}", counter++.ToString(), exception.Message, exception.StackTrace); exception = exception.InnerException; if (exception != null) { sb.AppendLine(); } } exceptionMessage = sb.ToString(); } HTTPManager.Logger.Verbose("HTTP1Handler", exceptionMessage, this.Context, this.conn.CurrentRequest.Context); #if !BESTHTTP_DISABLE_CACHING if (this.conn.CurrentRequest.UseStreaming) { HTTPCacheService.DeleteEntity(this.conn.CurrentRequest.CurrentUri); } #endif // Something gone bad, Response must be null! this.conn.CurrentRequest.Response = null; if (!this.conn.CurrentRequest.IsCancellationRequested) { this.conn.CurrentRequest.Exception = e; this.conn.CurrentRequest.State = HTTPRequestStates.Error; } proposedConnectionState = HTTPConnectionStates.Closed; } finally { this.conn.CurrentRequest.OnCancellationRequested -= OnCancellationRequested; // Exit ASAP if (this.ShutdownType != ShutdownTypes.Immediate) { if (this.conn.CurrentRequest.IsCancellationRequested) { // we don't know what stage the request is canceled, we can't safely reuse the tcp channel. proposedConnectionState = HTTPConnectionStates.Closed; this.conn.CurrentRequest.Response = null; this.conn.CurrentRequest.State = this.conn.CurrentRequest.IsTimedOut ? HTTPRequestStates.TimedOut : HTTPRequestStates.Aborted; } else if (resendRequest) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.conn.CurrentRequest, RequestEvents.Resend)); } else if (this.conn.CurrentRequest.Response != null && this.conn.CurrentRequest.Response.IsUpgraded) { proposedConnectionState = HTTPConnectionStates.WaitForProtocolShutdown; } else if (this.conn.CurrentRequest.State == HTTPRequestStates.Processing) { if (this.conn.CurrentRequest.Response != null) { this.conn.CurrentRequest.State = HTTPRequestStates.Finished; } else { this.conn.CurrentRequest.Exception = new Exception(string.Format("[{0}] Remote server closed the connection before sending response header! Previous request state: {1}. Connection state: {2}", this.ToString(), this.conn.CurrentRequest.State.ToString(), this.conn.State.ToString())); this.conn.CurrentRequest.State = HTTPRequestStates.Error; proposedConnectionState = HTTPConnectionStates.Closed; } } this.conn.CurrentRequest = null; if (proposedConnectionState == HTTPConnectionStates.Processing) { proposedConnectionState = HTTPConnectionStates.Recycle; } ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this.conn, proposedConnectionState)); } } }
private void ProcessIncomingFrames(List <HTTP2FrameHeaderAndPayload> outgoingFrames) { UInt32 windowUpdate = 0; while (this.incomingFrames.Count > 0) { HTTP2FrameHeaderAndPayload frame = this.incomingFrames.Dequeue(); if (this.isRSTFrameSent) { BufferPool.Release(frame.Payload); continue; } if (/*HTTPManager.Logger.Level == Logger.Loglevels.All && */ frame.Type != HTTP2FrameTypes.DATA && frame.Type != HTTP2FrameTypes.WINDOW_UPDATE) { HTTPManager.Logger.Information("HTTP2Stream", string.Format("[{0}] Process - processing frame: {1}", this.Id, frame.ToString())); } switch (frame.Type) { case HTTP2FrameTypes.HEADERS: case HTTP2FrameTypes.CONTINUATION: if (this.State != HTTP2StreamStates.HalfClosedLocal) { // ERROR! continue; } // payload will be released by the view frame.DontUseMemPool = true; if (this.currentFramesView == null) { this.currentFramesView = new FramesAsStreamView(new HeaderFrameView()); } this.currentFramesView.AddFrame(frame); if ((frame.Flags & (byte)HTTP2HeadersFlags.END_HEADERS) != 0) { List <KeyValuePair <string, string> > headers = new List <KeyValuePair <string, string> >(); try { this.encoder.Decode(this, this.currentFramesView, headers); } catch (Exception ex) { HTTPManager.Logger.Exception("HTTP2Stream", string.Format("[{0}] ProcessIncomingFrames", this.Id), ex); } this.response = new HTTP2Response(this.AssignedRequest, false); this.response.AddHeaders(headers); this.AssignedRequest.Response = this.response; this.currentFramesView.Close(); this.currentFramesView = null; if (frame.Type == HTTP2FrameTypes.HEADERS && (frame.Flags & (byte)HTTP2HeadersFlags.END_STREAM) != 0) { PlatformSupport.Threading.ThreadedRunner.RunShortLiving <HTTPRequest, HTTP2Response, FramesAsStreamView>(FinishRequest, this.AssignedRequest, this.response, this.currentFramesView); if (this.State == HTTP2StreamStates.HalfClosedLocal) { this.State = HTTP2StreamStates.Closed; } else { this.State = HTTP2StreamStates.HalfClosedRemote; } } } break; case HTTP2FrameTypes.DATA: if (this.State != HTTP2StreamStates.HalfClosedLocal) { // ERROR! continue; } this.downloaded += frame.PayloadLength; if (this.isStreamedDownload && frame.Payload != null && frame.PayloadLength > 0) { this.response.ProcessData(frame.Payload, (int)frame.PayloadLength); } // frame's buffer will be released by the frames view frame.DontUseMemPool = true; if (this.currentFramesView == null && !this.isStreamedDownload) { this.currentFramesView = new FramesAsStreamView(new DataFrameView()); } if (!this.isStreamedDownload) { this.currentFramesView.AddFrame(frame); } // Track received data, and if necessary(local window getting too low), send a window update frame if (this.localWindow < frame.PayloadLength) { HTTPManager.Logger.Error("HTTP2Stream", string.Format("[{0}] Frame's PayloadLength ({1:N0}) is larger then local window ({2:N0}). Frame: {3}", this.Id, frame.PayloadLength, this.localWindow, frame)); } else { this.localWindow -= frame.PayloadLength; } bool isFinalDataFrame = (frame.Flags & (byte)HTTP2DataFlags.END_STREAM) != 0; // Window update logic. // 1.) We could use a logic to only send window update(s) after a threshold is reached. // When the initial window size is high enough to contain the whole or most of the result, // sending back two window updates (connection and stream) after every data frame is pointless. // 2.) On the other hand, window updates are cheap and works even when initial window size is low. // ( if (isFinalDataFrame || this.localWindow <= this.windowUpdateThreshold) { windowUpdate += this.settings.MySettings[HTTP2Settings.INITIAL_WINDOW_SIZE] - this.localWindow - windowUpdate; } if (isFinalDataFrame) { if (this.isStreamedDownload) { this.response.FinishProcessData(); } HTTPManager.Logger.Information("HTTP2Stream", string.Format("[{0}] All data arrived, data length: {1:N0}", this.Id, this.downloaded)); // create a short living thread to process the downloaded data: PlatformSupport.Threading.ThreadedRunner.RunShortLiving <HTTPRequest, HTTP2Response, FramesAsStreamView>(FinishRequest, this.AssignedRequest, this.response, this.currentFramesView); this.currentFramesView = null; if (this.State == HTTP2StreamStates.HalfClosedLocal) { this.State = HTTP2StreamStates.Closed; } else { this.State = HTTP2StreamStates.HalfClosedRemote; } } else if (this.AssignedRequest.OnDownloadProgress != null) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.AssignedRequest, RequestEvents.DownloadProgress, downloaded, this.response.ExpectedContentLength)); } break; case HTTP2FrameTypes.WINDOW_UPDATE: HTTP2WindowUpdateFrame windowUpdateFrame = HTTP2FrameHelper.ReadWindowUpdateFrame(frame); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { HTTPManager.Logger.Information("HTTP2Stream", string.Format("[{0}] Received Window Update: {1:N0}, new remoteWindow: {2:N0}, initial remote window: {3:N0}, total data sent: {4:N0}", this.Id, windowUpdateFrame.WindowSizeIncrement, this.remoteWindow + windowUpdateFrame.WindowSizeIncrement, this.settings.RemoteSettings[HTTP2Settings.INITIAL_WINDOW_SIZE], this.sentData)); } this.remoteWindow += windowUpdateFrame.WindowSizeIncrement; break; case HTTP2FrameTypes.RST_STREAM: // https://httpwg.org/specs/rfc7540.html#RST_STREAM var rstStreamFrame = HTTP2FrameHelper.ReadRST_StreamFrame(frame); HTTPManager.Logger.Error("HTTP2Stream", string.Format("[{0}] RST Stream frame ({1}) received in state {2}!", this.Id, rstStreamFrame, this.State)); Abort(string.Format("RST_STREAM frame received! Error code: {0}({1})", rstStreamFrame.Error.ToString(), rstStreamFrame.ErrorCode)); break; default: HTTPManager.Logger.Warning("HTTP2Stream", string.Format("[{0}] Unexpected frame ({1}) in state {2}!", this.Id, frame, this.State)); break; } if (!frame.DontUseMemPool) { BufferPool.Release(frame.Payload); } } if (windowUpdate > 0) { if (HTTPManager.Logger.Level <= Logger.Loglevels.All) { HTTPManager.Logger.Information("HTTP2Stream", string.Format("[{0}] Sending window update: {1:N0}, current window: {2:N0}, initial window size: {3:N0}", this.Id, windowUpdate, this.localWindow, this.settings.MySettings[HTTP2Settings.INITIAL_WINDOW_SIZE])); } this.localWindow += windowUpdate; outgoingFrames.Add(HTTP2FrameHelper.CreateWindowUpdateFrame(this.Id, windowUpdate)); } }
public void RunHandler() { HTTPManager.Logger.Information("HTTP1Handler", string.Format("[{0}] started processing request '{1}'", this, this.conn.CurrentRequest.CurrentUri.ToString())); HTTPConnectionStates proposedConnectionState = HTTPConnectionStates.Processing; bool resendRequest = false; try { if (this.conn.CurrentRequest.IsCancellationRequested) { return; } #if !BESTHTTP_DISABLE_CACHING // Try load the full response from an already saved cache entity. // If the response could be loaded completely, we can skip connecting (if not already) and a full round-trip time to the server. if (HTTPCacheService.IsCachedEntityExpiresInTheFuture(this.conn.CurrentRequest) && ConnectionHelper.TryLoadAllFromCache(this.ToString(), this.conn.CurrentRequest)) { HTTPManager.Logger.Information("HTTP1Handler", string.Format("[{0}] Request could be fully loaded from cache! '{1}'", this, this.conn.CurrentRequest.CurrentUri.ToString())); return; } #endif if (this.conn.CurrentRequest.IsCancellationRequested) { return; } #if !BESTHTTP_DISABLE_CACHING // Setup cache control headers before we send out the request if (!this.conn.CurrentRequest.DisableCache) { HTTPCacheService.SetHeaders(this.conn.CurrentRequest); } #endif // Write the request to the stream this.conn.CurrentRequest.SendOutTo(this.conn.connector.Stream); if (this.conn.CurrentRequest.IsCancellationRequested) { return; } // Receive response from the server bool received = Receive(this.conn.CurrentRequest); if (this.conn.CurrentRequest.IsCancellationRequested) { return; } if (!received && this.conn.CurrentRequest.Retries < this.conn.CurrentRequest.MaxRetries) { proposedConnectionState = HTTPConnectionStates.Closed; this.conn.CurrentRequest.Retries++; resendRequest = true; return; } ConnectionHelper.HandleResponse(this.conn.ToString(), this.conn.CurrentRequest, out resendRequest, out proposedConnectionState, ref this._keepAlive); } catch (TimeoutException e) { this.conn.CurrentRequest.Response = null; // We will try again only once if (this.conn.CurrentRequest.Retries < this.conn.CurrentRequest.MaxRetries) { this.conn.CurrentRequest.Retries++; resendRequest = true; } else { this.conn.CurrentRequest.Exception = e; this.conn.CurrentRequest.State = HTTPRequestStates.ConnectionTimedOut; } proposedConnectionState = HTTPConnectionStates.Closed; } catch (Exception e) { if (this.ShutdownType == ShutdownTypes.Immediate) { return; } #if !BESTHTTP_DISABLE_CACHING if (this.conn.CurrentRequest.UseStreaming) { HTTPCacheService.DeleteEntity(this.conn.CurrentRequest.CurrentUri); } #endif // Something gone bad, Response must be null! this.conn.CurrentRequest.Response = null; if (!this.conn.CurrentRequest.IsCancellationRequested) { this.conn.CurrentRequest.Exception = e; this.conn.CurrentRequest.State = HTTPRequestStates.Error; } proposedConnectionState = HTTPConnectionStates.Closed; } finally { // Exit ASAP if (this.ShutdownType != ShutdownTypes.Immediate) { if (this.conn.CurrentRequest.IsCancellationRequested) { // we don't know what stage the request is cancelled, we can't safely reuse the tcp channel. proposedConnectionState = HTTPConnectionStates.Closed; this.conn.CurrentRequest.Response = null; this.conn.CurrentRequest.State = this.conn.CurrentRequest.IsTimedOut ? HTTPRequestStates.TimedOut : HTTPRequestStates.Aborted; } else if (resendRequest) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.conn.CurrentRequest, RequestEvents.Resend)); } else if (this.conn.CurrentRequest.Response != null && this.conn.CurrentRequest.Response.IsUpgraded) { proposedConnectionState = HTTPConnectionStates.WaitForProtocolShutdown; } else if (this.conn.CurrentRequest.State == HTTPRequestStates.Processing) { if (this.conn.CurrentRequest.Response != null) { this.conn.CurrentRequest.State = HTTPRequestStates.Finished; } else { this.conn.CurrentRequest.Exception = new Exception(string.Format("[{0}] Remote server closed the connection before sending response header! Previous request state: {1}. Connection state: {2}", this.ToString(), this.conn.CurrentRequest.State.ToString(), this.conn.State.ToString())); this.conn.CurrentRequest.State = HTTPRequestStates.Error; proposedConnectionState = HTTPConnectionStates.Closed; } } this.conn.CurrentRequest = null; if (proposedConnectionState == HTTPConnectionStates.Processing) { proposedConnectionState = HTTPConnectionStates.Recycle; } ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this.conn, proposedConnectionState)); } } }
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1 protected void ReadChunked(Stream stream) { BeginReceiveStreamFragments(); string contentLengthHeader = GetFirstHeaderValue("Content-Length"); bool hasContentLengthHeader = !string.IsNullOrEmpty(contentLengthHeader); int realLength = 0; if (hasContentLengthHeader) { hasContentLengthHeader = int.TryParse(contentLengthHeader, out realLength); } if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("ReadChunked - hasContentLengthHeader: {0}, contentLengthHeader: {1} realLength: {2:N0}", hasContentLengthHeader.ToString(), contentLengthHeader, realLength)); } using (var output = new BufferPoolMemoryStream()) { int chunkLength = ReadChunkLength(stream); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("chunkLength: {0:N0}", chunkLength)); } byte[] buffer = BufferPool.Get(Mathf.NextPowerOfTwo(chunkLength), true); int contentLength = 0; // Progress report: long Downloaded = 0; long DownloadLength = hasContentLengthHeader ? realLength : chunkLength; bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess #if !BESTHTTP_DISABLE_CACHING || this.IsFromCache #endif ); if (sendProgressChanged) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength)); } string encoding = #if !BESTHTTP_DISABLE_CACHING IsFromCache ? null : #endif GetFirstHeaderValue("content-encoding"); bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip"; Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null; while (chunkLength != 0) { if (this.baseRequest.IsCancellationRequested) { return; } // To avoid more GC garbage we use only one buffer, and resize only if the next chunk doesn't fit. if (buffer.Length < chunkLength) { BufferPool.Resize(ref buffer, chunkLength, true); } int readBytes = 0; // Fill up the buffer do { int bytes = stream.Read(buffer, readBytes, chunkLength - readBytes); if (bytes <= 0) { throw ExceptionHelper.ServerClosedTCPStream(); } readBytes += bytes; // Progress report: // Placing reporting inside this cycle will report progress much more frequent Downloaded += bytes; if (sendProgressChanged) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength)); } } while (readBytes < chunkLength); if (baseRequest.UseStreaming) { if (gzipped) { var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true); if (decompressed.Data != null) { FeedStreamFragment(decompressed.Data, 0, decompressed.Length); } } else { FeedStreamFragment(buffer, 0, readBytes); } } else { output.Write(buffer, 0, readBytes); } // Every chunk data has a trailing CRLF ReadTo(stream, LF); contentLength += readBytes; // read the next chunk's length chunkLength = ReadChunkLength(stream); if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("chunkLength: {0:N0}", chunkLength)); } if (!hasContentLengthHeader && sendProgressChanged) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength)); } } BufferPool.Release(buffer); if (baseRequest.UseStreaming) { if (gzipped) { var decompressed = decompressor.Decompress(null, 0, 0, true, true); if (decompressed.Data != null) { FeedStreamFragment(decompressed.Data, 0, decompressed.Length); } } FlushRemainingFragmentBuffer(); } // Read the trailing headers or the CRLF ReadHeaders(stream); // HTTP servers sometimes use compression (gzip) or deflate methods to optimize transmission. // How both chunked and gzip encoding interact is dictated by the two-staged encoding of HTTP: // first the content stream is encoded as (Content-Encoding: gzip), after which the resulting byte stream is encoded for transfer using another encoder (Transfer-Encoding: chunked). // This means that in case both compression and chunked encoding are enabled, the chunk encoding itself is not compressed, and the data in each chunk should not be compressed individually. // The remote endpoint can decode the incoming stream by first decoding it with the Transfer-Encoding, followed by the specified Content-Encoding. // It would be a better implementation when the chunk would be decododed on-the-fly. Becouse now the whole stream must be downloaded, and then decoded. It needs more memory. if (!baseRequest.UseStreaming) { this.Data = DecodeStream(output); } if (decompressor != null) { decompressor.Dispose(); } } }
protected override void ThreadFunc() { if (this.CurrentRequest.IsRedirected) { this.CurrentRequest.Timing.Add(TimingEventNames.Queued_For_Redirection); } else { this.CurrentRequest.Timing.Add(TimingEventNames.Queued); } if (this.connector != null && !this.connector.IsConnected) { // this will send the request back to the queue RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(CurrentRequest, RequestEvents.Resend)); ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HTTPConnectionStates.Closed)); return; } if (this.connector == null) { this.connector = new Connections.TCPConnector(); try { this.connector.Connect(this.CurrentRequest); } catch (Exception ex) { if (HTTPManager.Logger.Level == Logger.Loglevels.All) { HTTPManager.Logger.Exception("HTTPConnection", "Connector.Connect", ex, this.Context, this.CurrentRequest.Context); } if (ex is TimeoutException) { this.CurrentRequest.State = HTTPRequestStates.ConnectionTimedOut; } else if (!this.CurrentRequest.IsTimedOut) // Do nothing here if Abort() got called on the request, its State is already set. { this.CurrentRequest.Exception = ex; this.CurrentRequest.State = HTTPRequestStates.Error; } ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HTTPConnectionStates.Closed)); return; } #if !NETFX_CORE // data sending is buffered for all protocols, so when we put data into the socket we want to send them asap this.connector.Client.NoDelay = true; #endif StartTime = DateTime.UtcNow; HTTPManager.Logger.Information("HTTPConnection", "Negotiated protocol through ALPN: '" + this.connector.NegotiatedProtocol + "'", this.Context, this.CurrentRequest.Context); switch (this.connector.NegotiatedProtocol) { case HTTPProtocolFactory.W3C_HTTP1: this.requestHandler = new Connections.HTTP1Handler(this); ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HostProtocolSupport.HTTP1)); break; #if (!UNITY_WEBGL || UNITY_EDITOR) && !BESTHTTP_DISABLE_ALTERNATE_SSL && !BESTHTTP_DISABLE_HTTP2 case HTTPProtocolFactory.W3C_HTTP2: this.requestHandler = new Connections.HTTP2.HTTP2Handler(this); this.CurrentRequest = null; ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HostProtocolSupport.HTTP2)); break; #endif default: HTTPManager.Logger.Error("HTTPConnection", "Unknown negotiated protocol: " + this.connector.NegotiatedProtocol, this.Context, this.CurrentRequest.Context); RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(CurrentRequest, RequestEvents.Resend)); ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HTTPConnectionStates.Closed)); return; } } this.requestHandler.Context.Add("Connection", this.GetHashCode()); this.Context.Add("RequestHandler", this.requestHandler.GetHashCode()); this.requestHandler.RunHandler(); LastProcessTime = DateTime.Now; }
public void RunHandler() { HTTPManager.Logger.Information("HTTP2Handler", "Processing thread up and running!"); Thread.CurrentThread.Name = "HTTP2 Process"; PlatformSupport.Threading.ThreadedRunner.RunLongLiving(ReadThread); try { bool atLeastOneStreamHasAFrameToSend = true; this.HPACKEncoder = new HPACKEncoder(this.settings); // https://httpwg.org/specs/rfc7540.html#InitialWindowSize // The connection flow-control window is also 65,535 octets. this.remoteWindow = this.settings.RemoteSettings[HTTP2Settings.INITIAL_WINDOW_SIZE]; // we want to pack as many data as we can in one tcp segment, but setting the buffer's size too high // we might keep data too long and send them in bursts instead of in a steady stream. // Keeping it too low might result in a full tcp segment and one with very low payload // Is it possible that one full tcp segment sized buffer would be the best, or multiple of it. // It would keep the network busy without any fragments. The ethernet layer has a maximum of 1500 bytes, // but there's two layers of 20 byte headers each, so as a theoretical maximum it's 1500-20-20 bytes. // On the other hand, if the buffer is small (1-2), that means that for larger data, we have to do a lot // of system calls, in that case a larger buffer might be better. Still, if we are not cpu bound, // a well saturated network might serve us better. using (WriteOnlyBufferedStream bufferedStream = new WriteOnlyBufferedStream(this.conn.connector.Stream, 1024 * 1024 /*1500 - 20 - 20*/)) { // The client connection preface starts with a sequence of 24 octets bufferedStream.Write(MAGIC, 0, MAGIC.Length); // This sequence MUST be followed by a SETTINGS frame (Section 6.5), which MAY be empty. // The client sends the client connection preface immediately upon receipt of a // 101 (Switching Protocols) response (indicating a successful upgrade) // or as the first application data octets of a TLS connection // Set streams' initial window size to its maximum. this.settings.InitiatedMySettings[HTTP2Settings.INITIAL_WINDOW_SIZE] = HTTPManager.HTTP2Settings.InitialStreamWindowSize; this.settings.InitiatedMySettings[HTTP2Settings.MAX_CONCURRENT_STREAMS] = HTTPManager.HTTP2Settings.MaxConcurrentStreams; this.settings.SendChanges(this.outgoingFrames); // The default window size for the whole connection is 65535 bytes, // but we want to set it to the maximum possible value. Int64 diff = HTTPManager.HTTP2Settings.InitialConnectionWindowSize - 65535; if (diff > 0) { this.outgoingFrames.Add(HTTP2FrameHelper.CreateWindowUpdateFrame(0, (UInt32)diff)); } this.pingFrequency = HTTPManager.HTTP2Settings.PingFrequency; while (this.isRunning) { if (!atLeastOneStreamHasAFrameToSend) { // buffered stream will call flush automatically if its internal buffer is full. // But we have to make it sure that we flush remaining data before we go to sleep. bufferedStream.Flush(); // Wait until we have to send the next ping, OR a new frame is received on the read thread. // Sent Now Sent+frequency //-----|--------|----|------------ int wait = (int)((this.lastPingSent + this.pingFrequency) - DateTime.UtcNow).TotalMilliseconds; wait = (int)Math.Min(wait, this.MaxGoAwayWaitTime.TotalMilliseconds); if (wait >= 1) { if (HTTPManager.Logger.Level <= Logger.Loglevels.All) { HTTPManager.Logger.Information("HTTP2Handler", string.Format("Sleeping for {0:N0}ms", wait)); } this.newFrameSignal.WaitOne(wait); } } DateTime now = DateTime.UtcNow; if (now - this.lastPingSent >= this.pingFrequency) { this.lastPingSent = now; var frame = HTTP2FrameHelper.CreatePingFrame(HTTP2PingFlags.None); BufferHelper.SetLong(frame.Payload, 0, now.Ticks); this.outgoingFrames.Add(frame); } // Process received frames HTTP2FrameHeaderAndPayload header; while (this.newFrames.TryDequeue(out header)) { if (header.StreamId > 0) { HTTP2Stream http2Stream = FindStreamById(header.StreamId); // Add frame to the stream, so it can process it when its Process function is called if (http2Stream != null) { http2Stream.AddFrame(header, this.outgoingFrames); } else { // Error? It's possible that we closed and removed the stream while the server was in the middle of sending frames //HTTPManager.Logger.Warning("HTTP2Handler", string.Format("No stream found for id: {0}! Can't deliver frame: {1}", header.StreamId, header)); } } else { switch (header.Type) { case HTTP2FrameTypes.SETTINGS: this.settings.Process(header, this.outgoingFrames); break; case HTTP2FrameTypes.PING: var pingFrame = HTTP2FrameHelper.ReadPingFrame(header); // if it wasn't an ack for our ping, we have to send one if ((pingFrame.Flags & HTTP2PingFlags.ACK) == 0) { var frame = HTTP2FrameHelper.CreatePingFrame(HTTP2PingFlags.ACK); Array.Copy(pingFrame.OpaqueData, 0, frame.Payload, 0, pingFrame.OpaqueDataLength); this.outgoingFrames.Add(frame); } break; case HTTP2FrameTypes.WINDOW_UPDATE: var windowUpdateFrame = HTTP2FrameHelper.ReadWindowUpdateFrame(header); this.remoteWindow += windowUpdateFrame.WindowSizeIncrement; break; case HTTP2FrameTypes.GOAWAY: // parse the frame, so we can print out detailed information HTTP2GoAwayFrame goAwayFrame = HTTP2FrameHelper.ReadGoAwayFrame(header); HTTPManager.Logger.Information("HTTP2Handler", "Received GOAWAY frame: " + goAwayFrame.ToString()); string msg = string.Format("Server closing the connection! Error code: {0} ({1})", goAwayFrame.Error, goAwayFrame.ErrorCode); for (int i = 0; i < this.clientInitiatedStreams.Count; ++i) { this.clientInitiatedStreams[i].Abort(msg); } // set the running flag to false, so the thread can exit this.isRunning = false; this.conn.State = HTTPConnectionStates.Closed; break; case HTTP2FrameTypes.ALT_SVC: //HTTP2AltSVCFrame altSvcFrame = HTTP2FrameHelper.ReadAltSvcFrame(header); // Implement //HTTPManager.EnqueuePluginEvent(new PluginEventInfo(PluginEvents.AltSvcHeader, new AltSvcEventInfo(altSvcFrame.Origin, )) break; } } } UInt32 maxConcurrentStreams = Math.Min(HTTPManager.HTTP2Settings.MaxConcurrentStreams, this.settings.RemoteSettings[HTTP2Settings.MAX_CONCURRENT_STREAMS]); // pre-test stream count to lock only when truly needed. if (this.clientInitiatedStreams.Count < maxConcurrentStreams && this.isRunning) { // grab requests from queue HTTPRequest request; while (this.clientInitiatedStreams.Count < maxConcurrentStreams && this.requestQueue.TryDequeue(out request)) { // create a new stream var newStream = new HTTP2Stream((UInt32)Interlocked.Add(ref LastStreamId, 2), this.settings, this.HPACKEncoder); // process the request newStream.Assign(request); this.clientInitiatedStreams.Add(newStream); } } // send any settings changes this.settings.SendChanges(this.outgoingFrames); atLeastOneStreamHasAFrameToSend = false; // process other streams // Room for improvement Streams should be processed by their priority! for (int i = 0; i < this.clientInitiatedStreams.Count; ++i) { var stream = this.clientInitiatedStreams[i]; stream.Process(this.outgoingFrames); // remove closed, empty streams (not enough to check the closed flag, a closed stream still can contain frames to send) if (stream.State == HTTP2StreamStates.Closed && !stream.HasFrameToSend) { this.clientInitiatedStreams.RemoveAt(i--); stream.Removed(); } atLeastOneStreamHasAFrameToSend |= stream.HasFrameToSend; this.lastInteraction = DateTime.UtcNow; } // If we encounter a data frame that too large for the current remote window, we have to stop // sending all data frames as we could send smaller data frames before the large ones. // Room for improvement: An improvement would be here to stop data frame sending per-stream. bool haltDataSending = false; if (this.ShutdownType == ShutdownTypes.Running && now - this.lastInteraction >= HTTPManager.HTTP2Settings.MaxIdleTime) { this.lastInteraction = DateTime.UtcNow; HTTPManager.Logger.Information("HTTP2Handler", "Reached idle time, sending GoAway frame!"); this.outgoingFrames.Add(HTTP2FrameHelper.CreateGoAwayFrame(0, HTTP2ErrorCodes.NO_ERROR)); this.goAwaySentAt = DateTime.UtcNow; } // https://httpwg.org/specs/rfc7540.html#GOAWAY // Endpoints SHOULD always send a GOAWAY frame before closing a connection so that the remote peer can know whether a stream has been partially processed or not. if (this.ShutdownType == ShutdownTypes.Gentle) { HTTPManager.Logger.Information("HTTP2Handler", "Connection abort requested, sending GoAway frame!"); this.outgoingFrames.Clear(); this.outgoingFrames.Add(HTTP2FrameHelper.CreateGoAwayFrame(0, HTTP2ErrorCodes.NO_ERROR)); this.goAwaySentAt = DateTime.UtcNow; } if (this.isRunning && now - goAwaySentAt >= this.MaxGoAwayWaitTime) { HTTPManager.Logger.Information("HTTP2Handler", "No GoAway frame received back. Really quitting now!"); this.isRunning = false; conn.State = HTTPConnectionStates.Closed; } uint streamWindowUpdates = 0; // Go through all the collected frames and send them. for (int i = 0; i < this.outgoingFrames.Count; ++i) { var frame = this.outgoingFrames[i]; if (HTTPManager.Logger.Level <= Logger.Loglevels.All && frame.Type != HTTP2FrameTypes.DATA /*&& frame.Type != HTTP2FrameTypes.PING*/) { HTTPManager.Logger.Information("HTTP2Handler", "Sending frame: " + frame.ToString()); } // post process frames switch (frame.Type) { case HTTP2FrameTypes.DATA: if (haltDataSending) { continue; } // if the tracked remoteWindow is smaller than the frame's payload, we stop sending // data frames until we receive window-update frames if (frame.PayloadLength > this.remoteWindow) { haltDataSending = true; HTTPManager.Logger.Warning("HTTP2Handler", string.Format("Data sending halted for this round. Remote Window: {0:N0}, frame: {1}", this.remoteWindow, frame.ToString())); continue; } break; case HTTP2FrameTypes.WINDOW_UPDATE: if (frame.StreamId > 0) { streamWindowUpdates += BufferHelper.ReadUInt31(frame.Payload, 0); } break; } this.outgoingFrames.RemoveAt(i--); using (var buffer = HTTP2FrameHelper.HeaderAsBinary(frame)) bufferedStream.Write(buffer.Data, 0, buffer.Length); if (frame.PayloadLength > 0) { bufferedStream.Write(frame.Payload, (int)frame.PayloadOffset, (int)frame.PayloadLength); if (!frame.DontUseMemPool) { BufferPool.Release(frame.Payload); } } if (frame.Type == HTTP2FrameTypes.DATA) { this.remoteWindow -= frame.PayloadLength; } } if (streamWindowUpdates > 0) { var frame = HTTP2FrameHelper.CreateWindowUpdateFrame(0, streamWindowUpdates); if (HTTPManager.Logger.Level <= Logger.Loglevels.All) { HTTPManager.Logger.Information("HTTP2Handler", "Sending frame: " + frame.ToString()); } using (var buffer = HTTP2FrameHelper.HeaderAsBinary(frame)) bufferedStream.Write(buffer.Data, 0, buffer.Length); bufferedStream.Write(frame.Payload, (int)frame.PayloadOffset, (int)frame.PayloadLength); } } // while (this.isRunning) bufferedStream.Flush(); } } catch (Exception ex) { // Log out the exception if it's a non-expected one. if (this.ShutdownType == ShutdownTypes.Running && this.goAwaySentAt == DateTime.MaxValue) { HTTPManager.Logger.Exception("HTTP2Handler", "Sender thread", ex); } } finally { // First thread closing notifies the ConnectionEventHelper if (Interlocked.Increment(ref this.threadExitCount) == 1) { ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this.conn, HTTPConnectionStates.Closed)); } HTTPManager.Logger.Information("HTTP2Handler", "Sender thread closing - cleaning up remaining request..."); for (int i = 0; i < this.clientInitiatedStreams.Count; ++i) { this.clientInitiatedStreams[i].Abort("Connection closed unexpectedly"); } this.clientInitiatedStreams.Clear(); HTTPRequest request = null; while (this.requestQueue.TryDequeue(out request)) { HTTPManager.Logger.Information("HTTP2Handler", string.Format("Request '{0}' IsCancellationRequested: {1}", request.CurrentUri.ToString(), request.IsCancellationRequested.ToString())); if (request.IsCancellationRequested) { request.Response = null; request.State = HTTPRequestStates.Aborted; } else { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(request, RequestEvents.Resend)); } } HTTPManager.Logger.Information("HTTP2Handler", "Sender thread closing"); } try { // Works in the new runtime if (this.conn.connector.TopmostStream != null) { using (this.conn.connector.TopmostStream) { } } // Works in the old runtime if (this.conn.connector.Stream != null) { using (this.conn.connector.Stream) { } } } catch { } }
// No transfer-encoding just raw bytes. internal void ReadRaw(Stream stream, long contentLength) { BeginReceiveStreamFragments(); // Progress report: long downloaded = 0; long downloadLength = contentLength; bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess #if !BESTHTTP_DISABLE_CACHING || this.IsFromCache #endif ); if (sendProgressChanged) { RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, downloaded, downloadLength)); } if (HTTPManager.Logger.Level == Logger.Loglevels.All) { VerboseLogging(string.Format("ReadRaw - contentLength: {0:N0}", contentLength)); } string encoding = #if !BESTHTTP_DISABLE_CACHING IsFromCache ? null : #endif GetFirstHeaderValue("content-encoding"); bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip"; Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null; if (!baseRequest.UseStreaming && contentLength > 2147483646) { throw new OverflowException("You have to use STREAMING to download files bigger than 2GB!"); } using (var output = new BufferPoolMemoryStream(baseRequest.UseStreaming ? 0 : (int)contentLength)) { // Because of the last parameter, buffer's size can be larger than the requested but there's no reason to use // an exact sized one if there's an larger one available in the pool. Later we will use the whole buffer. byte[] buffer = BufferPool.Get(Math.Max(baseRequest.StreamFragmentSize, MinBufferSize), true); int readBytes = 0; while (contentLength > 0) { if (this.baseRequest.IsCancellationRequested) { return; } readBytes = 0; do { // tryToReadCount contain how much bytes we want to read in once. We try to read the buffer fully in once, // but with a limit of the remaining contentLength. int tryToReadCount = (int)Math.Min(Math.Min(int.MaxValue, contentLength), buffer.Length - readBytes); int bytes = stream.Read(buffer, readBytes, tryToReadCount); if (bytes <= 0) { throw ExceptionHelper.ServerClosedTCPStream(); } readBytes += bytes; contentLength -= bytes; // Progress report: if (sendProgressChanged) { downloaded += bytes; RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, downloaded, downloadLength)); } } while (readBytes < buffer.Length && contentLength > 0); if (baseRequest.UseStreaming) { if (gzipped) { var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true); if (decompressed.Data != null) { FeedStreamFragment(decompressed.Data, 0, decompressed.Length); } } else { FeedStreamFragment(buffer, 0, readBytes); } } else { output.Write(buffer, 0, readBytes); } } ; BufferPool.Release(buffer); if (baseRequest.UseStreaming) { if (gzipped) { var decompressed = decompressor.Decompress(null, 0, 0, true, true); if (decompressed.Data != null) { FeedStreamFragment(decompressed.Data, 0, decompressed.Length); } } FlushRemainingFragmentBuffer(); } if (!baseRequest.UseStreaming) { this.Data = DecodeStream(output); } } if (decompressor != null) { decompressor.Dispose(); } }
protected override void ThreadFunc() { if (this.connector != null && !this.connector.IsConnected) { // this will semd the request back to the queue RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(CurrentRequest, RequestEvents.Resend)); ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HTTPConnectionStates.Closed)); return; } if (this.connector == null) { this.connector = new Connections.TCPConnector(); try { this.connector.Connect(this.CurrentRequest); } catch (Exception ex) { HTTPManager.Logger.Exception("HTTPConnection", "Connector.Connect", ex); this.CurrentRequest.State = HTTPRequestStates.ConnectionTimedOut; ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HTTPConnectionStates.Closed)); return; } #if !NETFX_CORE // data sending is buffered for all protocols, so when we put data into the socket we want to send them asap this.connector.Client.NoDelay = true; #endif StartTime = DateTime.UtcNow; HTTPManager.Logger.Information("HTTPConnection", "Negotiated protocol through ALPN: '" + this.connector.NegotiatedProtocol + "'"); switch (this.connector.NegotiatedProtocol) { case HTTPProtocolFactory.W3C_HTTP1: this.requestHandler = new Connections.HTTP1Handler(this); ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HostProtocolSupport.HTTP1)); break; #if (!UNITY_WEBGL || UNITY_EDITOR) && !BESTHTTP_DISABLE_ALTERNATE_SSL && !BESTHTTP_DISABLE_HTTP2 case HTTPProtocolFactory.W3C_HTTP2: this.requestHandler = new Connections.HTTP2.HTTP2Handler(this); this.CurrentRequest = null; ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HostProtocolSupport.HTTP2)); break; #endif default: HTTPManager.Logger.Error("HTTPConnection", "Unknown negotiated protocol: " + this.connector.NegotiatedProtocol); RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(CurrentRequest, RequestEvents.Resend)); ConnectionEventHelper.EnqueueConnectionEvent(new ConnectionEventInfo(this, HTTPConnectionStates.Closed)); return; } } this.requestHandler.RunHandler(); LastProcessTime = DateTime.Now; }