/// <summary> /// Performs the HTTP request (synchron). /// </summary> /// <param name="state">The state.</param> private void PerformHttpRequest(RequestState state) { try { // next call returns true if the real request should be cancelled // (e.g. if no internet connection available) if (state.OnRequestBeforeStart()) { // signal this state to the worker class this.RequestStartCancelled(state); return; } } catch (Exception signalException) { Log.Error("Error during event dispatch of StartDownloadCallBack()", signalException); } state.StartTime = DateTime.Now; try { Log.Debug("calling GetResponse for " + state.Request.RequestUri); state.Response = state.Request.GetResponse(); } catch (Exception responseException) { //For some reason the HttpWebResponse class throws an exception on 3xx responses WebException we = responseException as WebException; if ((we != null) && (we.Response != null)) { state.Response = we.Response; } else { Log.Debug("GetResponse exception for " + state.Request.RequestUri, responseException); state.OnRequestException(responseException); this.FinalizeWebRequest(state); return; } } ProcessResponse(state); }
private void Run(object ignored) { lock (_waitingRequests) { // wait for queued items _wait001: try { while ((_waitingRequests.Count == 0) || (_runningRequests >= _maxRequests)) { Monitor.Wait(_waitingRequests); } RequestState state = (RequestState)_waitingRequests.Dequeue(); try { // next call returns true if the real request should be cancelled // (e.g. if no internet connection available) if (state.OnRequestStart()) { // signal this state to the worker class myAsyncWebRequest.RequestStartCancelled(state); goto _wait001; } } catch (Exception signalException) { _log.Error("Error during dispatch of StartDownloadCallBack()", signalException); } state.StartTime = DateTime.Now; _runningRequests++; try { _log.Debug("calling BeginGetResponse for " + state.Request.RequestUri); IAsyncResult result = state.Request.BeginGetResponse(myAsyncWebRequest.ResponseCallback, state); ThreadPool.RegisterWaitForSingleObject(result.AsyncWaitHandle, myAsyncWebRequest.TimeoutCallback, state, state.Request.Timeout, true); } catch (Exception responseException) { _log.Debug("BeginGetResponse exception for " + state.Request.RequestUri, responseException); state.OnRequestException(responseException); myAsyncWebRequest.FinalizeWebRequest(state); } goto _wait001; } catch (Exception ex) { _log.Fatal("Critical exception caught in RequestThread.Run()!", ex); } goto _wait001; } }
/// <summary> /// Callback gets called (recursively) on subsequent response stream read requests /// </summary> /// <param name="result"></param> private void ReadCallback(IAsyncResult result) { RequestState state = null; if (result != null) { state = result.AsyncState as RequestState; } if (state == null) { return; } try { Stream responseStream = state.ResponseStream; int read = responseStream.EndRead(result); // fix at least one of the leaks in CLR 1.1 (and 1.0?) // see also http://dturini.blogspot.com/2004/06/on-past-few-days-im-dealing-with-some.html // and: http://support.microsoft.com/?kbid=831138 if (Common.ClrVersion.Major < 2 && result.AsyncWaitHandle != null) { result.AsyncWaitHandle.Close(); } if (read > 0) { state.BytesTransferred += read; state.RequestData.Write(state.ReadBuffer, 0, read); // write buffer to mem stream, queue next read: responseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE, ReadCallback, state); if (((state.BytesTransferred / RequestState.BUFFER_SIZE) % 10) == 0) { state.OnRequestProgress(state.InitialRequestUri, state.BytesTransferred); } // continue read: return; } // completed (stream yet deflated/unzipped, just reset pos.) state.ResponseStream = state.RequestData; state.ResponseStream.Seek(0, SeekOrigin.Begin); state.OnRequestCompleted(state.InitialRequestUri, state.RequestParams.RequestUri, state.RequestParams.ETag, state.RequestParams.LastModified, RequestResult.OK); // usual cleanup: responseStream.Close(); state.RequestData.Close(); } catch (WebException e) { Log.Error("ReadCallBack WebException raised. Status: " + e.Status, e); state.OnRequestException(state.RequestParams.RequestUri, e); } catch (Exception e) { Log.Error("ReadCallBack Exception raised", e); state.OnRequestException(state.RequestParams.RequestUri, e); } FinalizeWebRequest(state); }
/// <summary> /// WebResponse processing. /// </summary> private void ProcessResponse(RequestState state) { try { HttpWebResponse httpResponse = state.Response as HttpWebResponse; FileWebResponse fileResponse = state.Response as FileWebResponse; NntpWebResponse nntpResponse = state.Response as NntpWebResponse; if (httpResponse != null) { if (httpResponse.ResponseUri != state.RequestUri) { Log.Debug( String.Format("httpResponse.ResponseUri != state.RequestUri: \r\n'{0}'\r\n'{1}'", httpResponse.ResponseUri, state.RequestUri)); } if (HttpStatusCode.OK == httpResponse.StatusCode || HttpExtendedStatusCode.IMUsed == (HttpExtendedStatusCode)httpResponse.StatusCode) { HttpCookieManager.GetCookies(httpResponse); // provide last request Uri and ETag: state.RequestParams.ETag = httpResponse.Headers.Get("ETag"); try { state.RequestParams.LastModified = httpResponse.LastModified; } catch (Exception lmEx) { Log.Debug("httpResponse.LastModified() parse failure: " + lmEx.Message); // Build in header parser failed on provided date format // Try our own parser (last chance) try { state.RequestParams.LastModified = DateTimeExt.ParseRfc2822DateTime(httpResponse.Headers.Get("Last-Modified")); } catch (FormatException) { /* ignore */ } } state.ResponseStream = httpResponse.GetResponseStream(); state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE, ReadCallback, state); // async read started, so we are done here: Log.Debug("ProcessResponse() web response OK: " + state.RequestUri); return; } if (httpResponse.StatusCode == HttpStatusCode.NotModified) { HttpCookieManager.GetCookies(httpResponse); string eTag = httpResponse.Headers.Get("ETag"); // also if it was not modified, we receive a httpResponse.LastModified with current date! // so we did not store it (is is just the same as last-retrived) // provide last request Uri and ETag: state.OnRequestCompleted(state.InitialRequestUri, state.RequestParams.RequestUri, eTag, MinValue, RequestResult.NotModified); // cleanup: FinalizeWebRequest(state); } else if ((httpResponse.StatusCode == HttpStatusCode.MovedPermanently) || (httpResponse.StatusCode == HttpStatusCode.Moved)) { state.RetryCount++; if (state.RetryCount > RequestState.MAX_RETRIES) { // there is no WebExceptionStatus.UnknownError in .NET 1.0 !!! throw new WebException("Repeated HTTP httpResponse: " + httpResponse.StatusCode, null, WebExceptionStatus.RequestCanceled, httpResponse); } string url2 = httpResponse.Headers["Location"]; //Check for any cookies HttpCookieManager.GetCookies(httpResponse); state.MovedPermanently = true; //Remove Url from queue _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri()); Log.Debug("ProcessResponse() Moved: '" + state.InitialRequestUri + " to " + url2); // Enqueue the request with the new Url. // We raise the queue priority a bit to get the retry request closer to the just // finished one. So the user get better feedback, because the whole processing // of one request (including the redirection/moved/... ) is visualized as one update // action. Uri req; //Try absolute first if (!Uri.TryCreate(url2, UriKind.Absolute, out req)) { // Try relative if (!Uri.TryCreate(httpResponse.ResponseUri, url2, out req)) { throw new WebException( string.Format( "Original resource temporary redirected. Request new resource at '{0}{1}' failed: ", httpResponse.ResponseUri, url2)); } } RequestParameter rqp = RequestParameter.Create(req, state.RequestParams); QueueRequestAgain(rqp, state.Priority + 1, state); } else if (IsRedirect(httpResponse.StatusCode)) { state.RetryCount++; if (state.RetryCount > RequestState.MAX_RETRIES) { // there is no WebExceptionStatus.UnknownError in .NET 1.0 !!! throw new WebException("Repeated HTTP httpResponse: " + httpResponse.StatusCode, null, WebExceptionStatus.RequestCanceled, httpResponse); } string url2 = httpResponse.Headers["Location"]; //Check for any cookies HttpCookieManager.GetCookies(httpResponse); //Remove Url from queue _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri()); Log.Debug("ProcessResponse() Redirect: '" + state.InitialRequestUri + " to " + url2); // Enqueue the request with the new Url. // We raise the queue priority a bit to get the retry request closer to the just // finished one. So the user get better feedback, because the whole processing // of one request (including the redirection/moved/... ) is visualized as one update // action. Uri req; //Try absolute first if (!Uri.TryCreate(url2, UriKind.Absolute, out req)) { // Try relative if (!Uri.TryCreate(httpResponse.ResponseUri, url2, out req)) { throw new WebException( string.Format( "Original resource temporary redirected. Request new resource at '{0}{1}' failed: ", httpResponse.ResponseUri, url2)); } } RequestParameter rqp = RequestParameter.Create(req, RebuildCredentials(state.RequestParams.Credentials, url2), state.RequestParams); QueueRequestAgain(rqp, state.Priority + 1, state); } else if (IsUnauthorized(httpResponse.StatusCode)) { if (state.RequestParams.Credentials == null) { // no initial credentials, try with default credentials state.RetryCount++; //Remove Url from queue _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri()); // Enqueue the request with the new Url. // We raise the queue priority a bit to get the retry request closer to the just // finished one. So the user get better feedback, because the whole processing // of one request (including the redirection/moved/... ) is visualized as one update // action. RequestParameter rqp = RequestParameter.Create(CredentialCache.DefaultCredentials, state.RequestParams); QueueRequestAgain(rqp, state.Priority + 1, state); } else { // failed with provided credentials if (state.RequestParams.SetCookies) { // one more request without cookies state.RetryCount++; //Remove Url from queue _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri()); // Enqueue the request with the new Url. // We raise the queue priority a bit to get the retry request closer to the just // finished one. So the user get better feedback, because the whole processing // of one request (including the redirection/moved/... ) is visualized as one update // action. RequestParameter rqp = RequestParameter.Create(false, state.RequestParams); QueueRequestAgain(rqp, state.Priority + 1, state); } else { throw new ResourceAuthorizationException(); } } } else if (IsAccessForbidden(httpResponse.StatusCode) && state.InitialRequestUri.Scheme == "https") { throw new ClientCertificateRequiredException(); } else if (httpResponse.StatusCode == HttpStatusCode.Gone) { throw new ResourceGoneException(); } else { string statusDescription = httpResponse.StatusDescription; if (String.IsNullOrEmpty(statusDescription)) { statusDescription = httpResponse.StatusCode.ToString(); } string htmlStatusMessage = null; try { htmlStatusMessage = new StreamReader(httpResponse.GetResponseStream()).ReadToEnd(); } catch { } if (String.IsNullOrEmpty(htmlStatusMessage)) { throw new WebException("Unexpected HTTP Response: " + statusDescription); } if (htmlStatusMessage.Contains("<")) { throw new WebException(htmlStatusMessage); } throw new WebException( "<html><head><title>Unexpected HTTP Response</title></head><body><h2>Unexpected HTTP Response: " + statusDescription + "</h2><p>" + htmlStatusMessage + "</p></html>"); } } else if (fileResponse != null) { string reqFile = fileResponse.ResponseUri.LocalPath; if (File.Exists(reqFile)) { DateTime lwt = File.GetLastWriteTime(reqFile); state.RequestParams.ETag = lwt.ToString(); state.RequestParams.LastModified = lwt; } state.ResponseStream = fileResponse.GetResponseStream(); state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE, ReadCallback, state); // async read started, so we are done here: Log.Debug("ProcessResponse() file response OK: " + state.RequestUri); return; } else if (nntpResponse != null) { state.RequestParams.LastModified = DateTime.Now; state.ResponseStream = nntpResponse.GetResponseStream(); state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE, ReadCallback, state); // async read started, so we are done here: Log.Debug("ProcessResponse() nntp response OK: " + state.RequestUri); return; } else { Debug.Assert(false, "ProcessResponse(): unhandled WebResponse type: " + state.Response.GetType()); FinalizeWebRequest(state); } } catch (ThreadAbortException) { FinalizeWebRequest(state); // ignore, just return } catch (Exception ex) { Log.Debug("ProcessResponse() exception: " + state.RequestUri + " :" + ex.Message); state.OnRequestException(state.InitialRequestUri, ex); FinalizeWebRequest(state); } }