Ejemplo n.º 1
0
        /// <summary>
        /// WebResponse processing.
        /// </summary>
        private void ProcessResponse(RequestState state)
        {
            try
            {
                HttpWebResponse httpResponse = state.Response as HttpWebResponse;
                FileWebResponse fileResponse = state.Response as FileWebResponse;
                NntpWebResponse nntpResponse = state.Response as NntpWebResponse;

                if (httpResponse != null)
                {
                    if (httpResponse.ResponseUri != state.RequestUri)
                    {
                        Log.Debug(
                            String.Format("httpResponse.ResponseUri != state.RequestUri: \r\n'{0}'\r\n'{1}'",
                                          httpResponse.ResponseUri, state.RequestUri));
                    }

                    if (HttpStatusCode.OK == httpResponse.StatusCode ||
                        HttpExtendedStatusCode.IMUsed == (HttpExtendedStatusCode)httpResponse.StatusCode)
                    {
                        HttpCookieManager.GetCookies(httpResponse);

                        // provide last request Uri and ETag:
                        state.RequestParams.ETag = httpResponse.Headers.Get("ETag");
                        try
                        {
                            state.RequestParams.LastModified = httpResponse.LastModified;
                        }
                        catch (Exception lmEx)
                        {
                            Log.Debug("httpResponse.LastModified() parse failure: " + lmEx.Message);
                            // Build in header parser failed on provided date format
                            // Try our own parser (last chance)
                            try
                            {
                                state.RequestParams.LastModified =
                                    DateTimeExt.ParseRfc2822DateTime(httpResponse.Headers.Get("Last-Modified"));
                            }
                            catch (FormatException)
                            {
                                /* ignore */
                            }
                        }

                        state.ResponseStream = httpResponse.GetResponseStream();
                        state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE,
                                                       ReadCallback, state);
                        // async read started, so we are done here:
                        Log.Debug("ProcessResponse() web response OK: " + state.RequestUri);

                        return;
                    }

                    if (httpResponse.StatusCode == HttpStatusCode.NotModified)
                    {
                        HttpCookieManager.GetCookies(httpResponse);

                        string eTag = httpResponse.Headers.Get("ETag");
                        // also if it was not modified, we receive a httpResponse.LastModified with current date!
                        // so we did not store it (is is just the same as last-retrived)
                        // provide last request Uri and ETag:
                        state.OnRequestCompleted(state.InitialRequestUri, state.RequestParams.RequestUri, eTag, MinValue,
                                                 RequestResult.NotModified);
                        // cleanup:
                        FinalizeWebRequest(state);
                    }
                    else if ((httpResponse.StatusCode == HttpStatusCode.MovedPermanently) ||
                             (httpResponse.StatusCode == HttpStatusCode.Moved))
                    {
                        state.RetryCount++;
                        if (state.RetryCount > RequestState.MAX_RETRIES)
                        {
                            // there is no WebExceptionStatus.UnknownError in .NET 1.0 !!!
                            throw new WebException("Repeated HTTP httpResponse: " + httpResponse.StatusCode,
                                                   null, WebExceptionStatus.RequestCanceled, httpResponse);
                        }

                        string url2 = httpResponse.Headers["Location"];
                        //Check for any cookies
                        HttpCookieManager.GetCookies(httpResponse);

                        state.MovedPermanently = true;
                        //Remove Url from queue
                        _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());

                        Log.Debug("ProcessResponse() Moved: '" + state.InitialRequestUri + " to " + url2);

                        // Enqueue the request with the new Url.
                        // We raise the queue priority a bit to get the retry request closer to the just
                        // finished one. So the user get better feedback, because the whole processing
                        // of one request (including the redirection/moved/... ) is visualized as one update
                        // action.


                        Uri req;
                        //Try absolute first
                        if (!Uri.TryCreate(url2, UriKind.Absolute, out req))
                        {
                            // Try relative
                            if (!Uri.TryCreate(httpResponse.ResponseUri, url2, out req))
                            {
                                throw new WebException(
                                          string.Format(
                                              "Original resource temporary redirected. Request new resource at '{0}{1}' failed: ",
                                              httpResponse.ResponseUri, url2));
                            }
                        }

                        RequestParameter rqp = RequestParameter.Create(req, state.RequestParams);
                        QueueRequestAgain(rqp, state.Priority + 1, state);
                    }
                    else if (IsRedirect(httpResponse.StatusCode))
                    {
                        state.RetryCount++;
                        if (state.RetryCount > RequestState.MAX_RETRIES)
                        {
                            // there is no WebExceptionStatus.UnknownError in .NET 1.0 !!!
                            throw new WebException("Repeated HTTP httpResponse: " + httpResponse.StatusCode,
                                                   null, WebExceptionStatus.RequestCanceled, httpResponse);
                        }

                        string url2 = httpResponse.Headers["Location"];
                        //Check for any cookies
                        HttpCookieManager.GetCookies(httpResponse);

                        //Remove Url from queue
                        _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());

                        Log.Debug("ProcessResponse() Redirect: '" + state.InitialRequestUri + " to " + url2);
                        // Enqueue the request with the new Url.
                        // We raise the queue priority a bit to get the retry request closer to the just
                        // finished one. So the user get better feedback, because the whole processing
                        // of one request (including the redirection/moved/... ) is visualized as one update
                        // action.

                        Uri req;
                        //Try absolute first
                        if (!Uri.TryCreate(url2, UriKind.Absolute, out req))
                        {
                            // Try relative
                            if (!Uri.TryCreate(httpResponse.ResponseUri, url2, out req))
                            {
                                throw new WebException(
                                          string.Format(
                                              "Original resource temporary redirected. Request new resource at '{0}{1}' failed: ",
                                              httpResponse.ResponseUri, url2));
                            }
                        }


                        RequestParameter rqp =
                            RequestParameter.Create(req, RebuildCredentials(state.RequestParams.Credentials, url2),
                                                    state.RequestParams);
                        QueueRequestAgain(rqp, state.Priority + 1, state);
                    }
                    else if (IsUnauthorized(httpResponse.StatusCode))
                    {
                        if (state.RequestParams.Credentials == null)
                        {
                            // no initial credentials, try with default credentials
                            state.RetryCount++;

                            //Remove Url from queue
                            _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());

                            // Enqueue the request with the new Url.
                            // We raise the queue priority a bit to get the retry request closer to the just
                            // finished one. So the user get better feedback, because the whole processing
                            // of one request (including the redirection/moved/... ) is visualized as one update
                            // action.
                            RequestParameter rqp =
                                RequestParameter.Create(CredentialCache.DefaultCredentials, state.RequestParams);
                            QueueRequestAgain(rqp, state.Priority + 1, state);
                        }
                        else
                        {
                            // failed with provided credentials

                            if (state.RequestParams.SetCookies)
                            {
                                // one more request without cookies

                                state.RetryCount++;

                                //Remove Url from queue
                                _queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());

                                // Enqueue the request with the new Url.
                                // We raise the queue priority a bit to get the retry request closer to the just
                                // finished one. So the user get better feedback, because the whole processing
                                // of one request (including the redirection/moved/... ) is visualized as one update
                                // action.
                                RequestParameter rqp = RequestParameter.Create(false, state.RequestParams);
                                QueueRequestAgain(rqp, state.Priority + 1, state);
                            }
                            else
                            {
                                throw new ResourceAuthorizationException();
                            }
                        }
                    }
                    else if (IsAccessForbidden(httpResponse.StatusCode) &&
                             state.InitialRequestUri.Scheme == "https")
                    {
                        throw new ClientCertificateRequiredException();
                    }
                    else if (httpResponse.StatusCode == HttpStatusCode.Gone)
                    {
                        throw new ResourceGoneException();
                    }
                    else
                    {
                        string statusDescription = httpResponse.StatusDescription;
                        if (String.IsNullOrEmpty(statusDescription))
                        {
                            statusDescription = httpResponse.StatusCode.ToString();
                        }

                        string htmlStatusMessage = null;
                        try
                        {
                            htmlStatusMessage = new StreamReader(httpResponse.GetResponseStream()).ReadToEnd();
                        }
                        catch { }

                        if (String.IsNullOrEmpty(htmlStatusMessage))
                        {
                            throw new WebException("Unexpected HTTP Response: " + statusDescription);
                        }

                        if (htmlStatusMessage.Contains("<"))
                        {
                            throw new WebException(htmlStatusMessage);
                        }

                        throw new WebException(
                                  "<html><head><title>Unexpected HTTP Response</title></head><body><h2>Unexpected HTTP Response: " +
                                  statusDescription + "</h2><p>" + htmlStatusMessage + "</p></html>");
                    }
                }
                else if (fileResponse != null)
                {
                    string reqFile = fileResponse.ResponseUri.LocalPath;

                    if (File.Exists(reqFile))
                    {
                        DateTime lwt = File.GetLastWriteTime(reqFile);
                        state.RequestParams.ETag         = lwt.ToString();
                        state.RequestParams.LastModified = lwt;
                    }

                    state.ResponseStream = fileResponse.GetResponseStream();
                    state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE,
                                                   ReadCallback, state);
                    // async read started, so we are done here:
                    Log.Debug("ProcessResponse() file response OK: " + state.RequestUri);

                    return;
                }
                else if (nntpResponse != null)
                {
                    state.RequestParams.LastModified = DateTime.Now;
                    state.ResponseStream             = nntpResponse.GetResponseStream();
                    state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE,
                                                   ReadCallback, state);
                    // async read started, so we are done here:
                    Log.Debug("ProcessResponse() nntp response OK: " + state.RequestUri);

                    return;
                }
                else
                {
                    Debug.Assert(false,
                                 "ProcessResponse(): unhandled WebResponse type: " +
                                 state.Response.GetType());
                    FinalizeWebRequest(state);
                }
            }
            catch (ThreadAbortException)
            {
                FinalizeWebRequest(state);
                // ignore, just return
            }
            catch (Exception ex)
            {
                Log.Debug("ProcessResponse() exception: " + state.RequestUri + " :" + ex.Message);
                state.OnRequestException(state.InitialRequestUri, ex);
                FinalizeWebRequest(state);
            }
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Used to create an HTTP request.
        /// </summary>
        /// <param name="requestParameter">Could be modified for each subsequent request</param>
        internal WebRequest PrepareRequest(RequestParameter requestParameter)
        {
            if (requestParameter == null)
            {
                throw new ArgumentNullException("requestParameter");
            }

            // here are the exceptions caused:
            WebRequest webRequest = WebRequest.Create(requestParameter.RequestUri);

            HttpWebRequest httpRequest = webRequest as HttpWebRequest;
            FileWebRequest fileRequest = webRequest as FileWebRequest;
            NntpWebRequest nntpRequest = webRequest as NntpWebRequest;

            if (httpRequest != null)
            {
                // set extended HttpWebRequest params
                httpRequest.Timeout                = Convert.ToInt32(requestParameter.Timeout.TotalMilliseconds); // default: two minutes timeout
                httpRequest.UserAgent              = FullUserAgent(requestParameter.UserAgent);
                httpRequest.Proxy                  = requestParameter.Proxy;
                httpRequest.AllowAutoRedirect      = false;
                httpRequest.AutomaticDecompression = DecompressionMethods.GZip |
                                                     DecompressionMethods.Deflate;
                if (requestParameter.Headers != null)
                {
                    httpRequest.Headers.Add(requestParameter.Headers);
                }

                // due to the reported bug 893620 some web server fail with a server error 500
                // if we send DateTime.MinValue as IfModifiedSince. Smoe Unix derivates only know
                // about valid lowest DateTime around 1970. So in the case we use the
                // httpRequest class default setting:
                if (requestParameter.LastModified > MinValue)
                {
                    httpRequest.IfModifiedSince = requestParameter.LastModified;
                }

                /* #if DEBUG
                 *                                      // further to investigate: with this setting we don't leak connections
                 *                                      // (try TCPView from http://www.sysinternals.com)
                 *                                      // read:
                 *                                      // * http://support.microsoft.com/default.aspx?scid=kb%3Ben-us%3B819450
                 *                                      // * http://cephas.net/blog/2003/10/29/the_intricacies_of_http.html
                 *                                      // * http://weblogs.asp.net/jan/archive/2004/01/28/63771.aspx
                 *
                 *                                      httpRequest.KeepAlive = false;		// to prevent open HTTP connection leak
                 *                                      httpRequest.ProtocolVersion = HttpVersion.Version10;	// to prevent "Underlying connection closed" exception(s)
                 #endif */

                if (httpRequest.Proxy == null)
                {
                    httpRequest.KeepAlive         = false;
                    httpRequest.Proxy             = WebRequest.DefaultWebProxy;
                    httpRequest.Proxy.Credentials = CredentialCache.DefaultCredentials;
                }

                if (requestParameter.ETag != null)
                {
                    httpRequest.Headers.Add("If-None-Match", requestParameter.ETag);
                    httpRequest.Headers.Add("A-IM", "feed");
                }

                if (requestParameter.Credentials != null)
                {
                    httpRequest.KeepAlive       = true;                  // required for authentication to succeed
                    httpRequest.ProtocolVersion = HttpVersion.Version11; // switch back
                    httpRequest.Credentials     = requestParameter.Credentials;
                }

                if (requestParameter.ClientCertificate != null)
                {
                    httpRequest.ClientCertificates.Add(requestParameter.ClientCertificate);
                    httpRequest.Timeout *= 2;   // double the timeout (SSL && Client Certs used!)
                }

                if (requestParameter.SetCookies)
                {
                    HttpCookieManager.SetCookies(httpRequest);
                }

                if (requestParameter.Cookies != null)
                {
                    httpRequest.CookieContainer = new CookieContainer();
                    httpRequest.CookieContainer.Add(requestParameter.Cookies);
                }

                //this prevents the feed mixup issue that we've been facing. See
                //http://www.davelemen.com/archives/2006/04/rss_bandit_feeds_mix_up.html
                //for a user complaint about the issue.
                httpRequest.Pipelined = false;
            }
            else if (fileRequest != null)
            {
                fileRequest.Timeout = DefaultTimeout;

                if (requestParameter.Credentials != null)
                {
                    fileRequest.Credentials = requestParameter.Credentials;
                }
            }
            else if (nntpRequest != null)
            {
                // ten minutes timeout. Large timeout is needed if this is first time we are fetching news
                //TODO: move the timeout handling to the requestor
                nntpRequest.Timeout = DefaultTimeout * 5;

                if (requestParameter.Credentials != null)
                {
                    nntpRequest.Credentials = requestParameter.Credentials;
                }

                if (requestParameter.LastModified > MinValue)
                {
                    nntpRequest.IfModifiedSince = requestParameter.LastModified;
                }
            }
            else
            {
                throw new NotImplementedException("Unsupported WebRequest type: " + webRequest.GetType());
            }

            return(webRequest);
        }