/// <summary>
/// Called for first web request.
/// </summary>
/// <param name="requestParameter">Could be modified for each subsequent request</param>
/// <param name="webRequestComplete"></param>
/// <param name="webRequestException"></param>
/// <param name="webRequestStart"></param>
/// <param name="webRequestProgress"></param>
/// <param name="priority"></param>
private RequestState DoQueueRequest(RequestParameter requestParameter,
RequestStartCallback webRequestStart,
RequestCompleteCallback webRequestComplete,
RequestExceptionCallback webRequestException,
RequestProgressCallback webRequestProgress,
int priority)
{
if (requestParameter == null)
{
throw new ArgumentNullException("requestParameter");
}
if (_queuedRequests.Contains(requestParameter.RequestUri.CanonicalizedUri()))
{
return(null); // httpRequest already there
}
_queuedRequests.Add(requestParameter.RequestUri.CanonicalizedUri(), null);
var webRequest = PrepareRequest(requestParameter);
RequestState state = new RequestState(webRequest, priority, requestParameter);
state.WebRequestBeforeStart += webRequestStart;
state.WebRequestCompleted += webRequestComplete;
state.WebRequestException += webRequestException;
state.WebRequestProgress += webRequestProgress;
PerformHttpRequestAsync(state, priority);
return(state);
}
/// <summary>
/// Asynchronous download method implementation.
/// </summary>
/// <param name="task">The DownloadTask to process.</param>
public void BeginDownload(DownloadTask task)
{
currentTask = task;
// If we resume way too often, just return
if (CheckForResumeAndProceed(currentTask))
{
return;
}
Uri reqUri = new Uri(task.DownloadItem.Enclosure.Url);
int priority = 10;
RequestParameter reqParam = RequestParameter.Create(reqUri, FeedSource.UserAgentString(String.Empty),
task.DownloadItem.Proxy, task.DownloadItem.Credentials,
DateTime.MinValue, null);
// global cookie handling:
reqParam.SetCookies = FeedSource.SetCookies;
state = BackgroundDownloadManager.AsyncWebRequest.QueueRequest(reqParam,
OnRequestStart,
OnRequestComplete,
OnRequestException,
OnRequestProgress,
priority);
}
/// <summary>
/// Used to a queue an HTTP request for processing
/// </summary>
/// <param name="requestParameter"></param>
/// <param name="webRequestComplete"></param>
/// <param name="webRequestException"></param>
/// <param name="webRequestStart"></param>
/// <param name="priority"></param>
/// <exception cref="NotSupportedException">The request scheme specified in address has not been registered.</exception>
/// <exception cref="ArgumentNullException">The requestParameter is a null reference</exception>
/// <exception cref="System.Security.SecurityException">The caller does not have permission to connect to the requested URI or a URI that the request is redirected to.</exception>
internal RequestState QueueRequest(RequestParameter requestParameter,
RequestStartCallback webRequestStart,
RequestCompleteCallback webRequestComplete,
RequestExceptionCallback webRequestException,
int priority)
{
return
(DoQueueRequest(requestParameter, webRequestStart, webRequestComplete,
webRequestException, null, priority));
}
/// <summary>
/// Called for subsequent requests.
/// </summary>
/// <param name="requestParameter">The request parameter.</param>
/// <param name="priority">The priority.</param>
/// <param name="prevState">State of the prev.</param>
private void QueueRequestAgain(RequestParameter requestParameter,
int priority, RequestState prevState)
{
if (requestParameter == null)
{
throw new ArgumentNullException("requestParameter");
}
if (prevState == null)
{
throw new ArgumentNullException("prevState");
}
var webRequest = PrepareRequest(requestParameter);
RequestState state = prevState;
IDisposable dispResponse = state.Response;
if (dispResponse != null)
{
dispResponse.Dispose();
state.Response = null;
}
if (state.ResponseStream != null)
{
// we don't want to get out of connections
state.ResponseStream.Close();
}
if (state.Request != null)
{
state.Request.Credentials = null;
// prevent NotImplementedExceptions:
if (state.Request is HttpWebRequest)
{
state.Request.Abort();
}
}
state.Request = webRequest;
state.RequestParams = requestParameter;
PerformHttpRequestAsync(state, priority);
}
public RequestState(WebRequest request, int priority, RequestParameter requestParameter) :
this()
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (requestParameter == null)
{
throw new ArgumentNullException("requestParameter");
}
this.Request = request;
this.InitialRequestUri = request.RequestUri;
this.Priority = priority;
this.RequestParams = requestParameter;
}
/// <summary>
/// WebResponse processing.
/// </summary>
private void ProcessResponse(RequestState state)
{
try
{
HttpWebResponse httpResponse = state.Response as HttpWebResponse;
FileWebResponse fileResponse = state.Response as FileWebResponse;
NntpWebResponse nntpResponse = state.Response as NntpWebResponse;
if (httpResponse != null)
{
if (httpResponse.ResponseUri != state.RequestUri)
{
Log.Debug(
String.Format("httpResponse.ResponseUri != state.RequestUri: \r\n'{0}'\r\n'{1}'",
httpResponse.ResponseUri, state.RequestUri));
}
if (HttpStatusCode.OK == httpResponse.StatusCode ||
HttpExtendedStatusCode.IMUsed == (HttpExtendedStatusCode)httpResponse.StatusCode)
{
HttpCookieManager.GetCookies(httpResponse);
// provide last request Uri and ETag:
state.RequestParams.ETag = httpResponse.Headers.Get("ETag");
try
{
state.RequestParams.LastModified = httpResponse.LastModified;
}
catch (Exception lmEx)
{
Log.Debug("httpResponse.LastModified() parse failure: " + lmEx.Message);
// Build in header parser failed on provided date format
// Try our own parser (last chance)
try
{
state.RequestParams.LastModified =
DateTimeExt.ParseRfc2822DateTime(httpResponse.Headers.Get("Last-Modified"));
}
catch (FormatException)
{
/* ignore */
}
}
state.ResponseStream = httpResponse.GetResponseStream();
state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE,
ReadCallback, state);
// async read started, so we are done here:
Log.Debug("ProcessResponse() web response OK: " + state.RequestUri);
return;
}
if (httpResponse.StatusCode == HttpStatusCode.NotModified)
{
HttpCookieManager.GetCookies(httpResponse);
string eTag = httpResponse.Headers.Get("ETag");
// also if it was not modified, we receive a httpResponse.LastModified with current date!
// so we did not store it (is is just the same as last-retrived)
// provide last request Uri and ETag:
state.OnRequestCompleted(state.InitialRequestUri, state.RequestParams.RequestUri, eTag, MinValue,
RequestResult.NotModified);
// cleanup:
FinalizeWebRequest(state);
}
else if ((httpResponse.StatusCode == HttpStatusCode.MovedPermanently) ||
(httpResponse.StatusCode == HttpStatusCode.Moved))
{
state.RetryCount++;
if (state.RetryCount > RequestState.MAX_RETRIES)
{
// there is no WebExceptionStatus.UnknownError in .NET 1.0 !!!
throw new WebException("Repeated HTTP httpResponse: " + httpResponse.StatusCode,
null, WebExceptionStatus.RequestCanceled, httpResponse);
}
string url2 = httpResponse.Headers["Location"];
//Check for any cookies
HttpCookieManager.GetCookies(httpResponse);
state.MovedPermanently = true;
//Remove Url from queue
_queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());
Log.Debug("ProcessResponse() Moved: '" + state.InitialRequestUri + " to " + url2);
// Enqueue the request with the new Url.
// We raise the queue priority a bit to get the retry request closer to the just
// finished one. So the user get better feedback, because the whole processing
// of one request (including the redirection/moved/... ) is visualized as one update
// action.
Uri req;
//Try absolute first
if (!Uri.TryCreate(url2, UriKind.Absolute, out req))
{
// Try relative
if (!Uri.TryCreate(httpResponse.ResponseUri, url2, out req))
{
throw new WebException(
string.Format(
"Original resource temporary redirected. Request new resource at '{0}{1}' failed: ",
httpResponse.ResponseUri, url2));
}
}
RequestParameter rqp = RequestParameter.Create(req, state.RequestParams);
QueueRequestAgain(rqp, state.Priority + 1, state);
}
else if (IsRedirect(httpResponse.StatusCode))
{
state.RetryCount++;
if (state.RetryCount > RequestState.MAX_RETRIES)
{
// there is no WebExceptionStatus.UnknownError in .NET 1.0 !!!
throw new WebException("Repeated HTTP httpResponse: " + httpResponse.StatusCode,
null, WebExceptionStatus.RequestCanceled, httpResponse);
}
string url2 = httpResponse.Headers["Location"];
//Check for any cookies
HttpCookieManager.GetCookies(httpResponse);
//Remove Url from queue
_queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());
Log.Debug("ProcessResponse() Redirect: '" + state.InitialRequestUri + " to " + url2);
// Enqueue the request with the new Url.
// We raise the queue priority a bit to get the retry request closer to the just
// finished one. So the user get better feedback, because the whole processing
// of one request (including the redirection/moved/... ) is visualized as one update
// action.
Uri req;
//Try absolute first
if (!Uri.TryCreate(url2, UriKind.Absolute, out req))
{
// Try relative
if (!Uri.TryCreate(httpResponse.ResponseUri, url2, out req))
{
throw new WebException(
string.Format(
"Original resource temporary redirected. Request new resource at '{0}{1}' failed: ",
httpResponse.ResponseUri, url2));
}
}
RequestParameter rqp =
RequestParameter.Create(req, RebuildCredentials(state.RequestParams.Credentials, url2),
state.RequestParams);
QueueRequestAgain(rqp, state.Priority + 1, state);
}
else if (IsUnauthorized(httpResponse.StatusCode))
{
if (state.RequestParams.Credentials == null)
{
// no initial credentials, try with default credentials
state.RetryCount++;
//Remove Url from queue
_queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());
// Enqueue the request with the new Url.
// We raise the queue priority a bit to get the retry request closer to the just
// finished one. So the user get better feedback, because the whole processing
// of one request (including the redirection/moved/... ) is visualized as one update
// action.
RequestParameter rqp =
RequestParameter.Create(CredentialCache.DefaultCredentials, state.RequestParams);
QueueRequestAgain(rqp, state.Priority + 1, state);
}
else
{
// failed with provided credentials
if (state.RequestParams.SetCookies)
{
// one more request without cookies
state.RetryCount++;
//Remove Url from queue
_queuedRequests.Remove(state.InitialRequestUri.CanonicalizedUri());
// Enqueue the request with the new Url.
// We raise the queue priority a bit to get the retry request closer to the just
// finished one. So the user get better feedback, because the whole processing
// of one request (including the redirection/moved/... ) is visualized as one update
// action.
RequestParameter rqp = RequestParameter.Create(false, state.RequestParams);
QueueRequestAgain(rqp, state.Priority + 1, state);
}
else
{
throw new ResourceAuthorizationException();
}
}
}
else if (IsAccessForbidden(httpResponse.StatusCode) &&
state.InitialRequestUri.Scheme == "https")
{
throw new ClientCertificateRequiredException();
}
else if (httpResponse.StatusCode == HttpStatusCode.Gone)
{
throw new ResourceGoneException();
}
else
{
string statusDescription = httpResponse.StatusDescription;
if (String.IsNullOrEmpty(statusDescription))
{
statusDescription = httpResponse.StatusCode.ToString();
}
string htmlStatusMessage = null;
try
{
htmlStatusMessage = new StreamReader(httpResponse.GetResponseStream()).ReadToEnd();
}
catch { }
if (String.IsNullOrEmpty(htmlStatusMessage))
{
throw new WebException("Unexpected HTTP Response: " + statusDescription);
}
if (htmlStatusMessage.Contains("<"))
{
throw new WebException(htmlStatusMessage);
}
throw new WebException(
"<html><head><title>Unexpected HTTP Response</title></head><body><h2>Unexpected HTTP Response: " +
statusDescription + "</h2><p>" + htmlStatusMessage + "</p></html>");
}
}
else if (fileResponse != null)
{
string reqFile = fileResponse.ResponseUri.LocalPath;
if (File.Exists(reqFile))
{
DateTime lwt = File.GetLastWriteTime(reqFile);
state.RequestParams.ETag = lwt.ToString();
state.RequestParams.LastModified = lwt;
}
state.ResponseStream = fileResponse.GetResponseStream();
state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE,
ReadCallback, state);
// async read started, so we are done here:
Log.Debug("ProcessResponse() file response OK: " + state.RequestUri);
return;
}
else if (nntpResponse != null)
{
state.RequestParams.LastModified = DateTime.Now;
state.ResponseStream = nntpResponse.GetResponseStream();
state.ResponseStream.BeginRead(state.ReadBuffer, 0, RequestState.BUFFER_SIZE,
ReadCallback, state);
// async read started, so we are done here:
Log.Debug("ProcessResponse() nntp response OK: " + state.RequestUri);
return;
}
else
{
Debug.Assert(false,
"ProcessResponse(): unhandled WebResponse type: " +
state.Response.GetType());
FinalizeWebRequest(state);
}
}
catch (ThreadAbortException)
{
FinalizeWebRequest(state);
// ignore, just return
}
catch (Exception ex)
{
Log.Debug("ProcessResponse() exception: " + state.RequestUri + " :" + ex.Message);
state.OnRequestException(state.InitialRequestUri, ex);
FinalizeWebRequest(state);
}
}
/// <summary>
/// Used to create an HTTP request.
/// </summary>
/// <param name="requestParameter">Could be modified for each subsequent request</param>
internal WebRequest PrepareRequest(RequestParameter requestParameter)
{
if (requestParameter == null)
{
throw new ArgumentNullException("requestParameter");
}
// here are the exceptions caused:
WebRequest webRequest = WebRequest.Create(requestParameter.RequestUri);
HttpWebRequest httpRequest = webRequest as HttpWebRequest;
FileWebRequest fileRequest = webRequest as FileWebRequest;
NntpWebRequest nntpRequest = webRequest as NntpWebRequest;
if (httpRequest != null)
{
// set extended HttpWebRequest params
httpRequest.Timeout = Convert.ToInt32(requestParameter.Timeout.TotalMilliseconds); // default: two minutes timeout
httpRequest.UserAgent = FullUserAgent(requestParameter.UserAgent);
httpRequest.Proxy = requestParameter.Proxy;
httpRequest.AllowAutoRedirect = false;
httpRequest.AutomaticDecompression = DecompressionMethods.GZip |
DecompressionMethods.Deflate;
if (requestParameter.Headers != null)
{
httpRequest.Headers.Add(requestParameter.Headers);
}
// due to the reported bug 893620 some web server fail with a server error 500
// if we send DateTime.MinValue as IfModifiedSince. Smoe Unix derivates only know
// about valid lowest DateTime around 1970. So in the case we use the
// httpRequest class default setting:
if (requestParameter.LastModified > MinValue)
{
httpRequest.IfModifiedSince = requestParameter.LastModified;
}
/* #if DEBUG
* // further to investigate: with this setting we don't leak connections
* // (try TCPView from http://www.sysinternals.com)
* // read:
* // * http://support.microsoft.com/default.aspx?scid=kb%3Ben-us%3B819450
* // * http://cephas.net/blog/2003/10/29/the_intricacies_of_http.html
* // * http://weblogs.asp.net/jan/archive/2004/01/28/63771.aspx
*
* httpRequest.KeepAlive = false; // to prevent open HTTP connection leak
* httpRequest.ProtocolVersion = HttpVersion.Version10; // to prevent "Underlying connection closed" exception(s)
#endif */
if (httpRequest.Proxy == null)
{
httpRequest.KeepAlive = false;
httpRequest.Proxy = WebRequest.DefaultWebProxy;
httpRequest.Proxy.Credentials = CredentialCache.DefaultCredentials;
}
if (requestParameter.ETag != null)
{
httpRequest.Headers.Add("If-None-Match", requestParameter.ETag);
httpRequest.Headers.Add("A-IM", "feed");
}
if (requestParameter.Credentials != null)
{
httpRequest.KeepAlive = true; // required for authentication to succeed
httpRequest.ProtocolVersion = HttpVersion.Version11; // switch back
httpRequest.Credentials = requestParameter.Credentials;
}
if (requestParameter.ClientCertificate != null)
{
httpRequest.ClientCertificates.Add(requestParameter.ClientCertificate);
httpRequest.Timeout *= 2; // double the timeout (SSL && Client Certs used!)
}
if (requestParameter.SetCookies)
{
HttpCookieManager.SetCookies(httpRequest);
}
if (requestParameter.Cookies != null)
{
httpRequest.CookieContainer = new CookieContainer();
httpRequest.CookieContainer.Add(requestParameter.Cookies);
}
//this prevents the feed mixup issue that we've been facing. See
//http://www.davelemen.com/archives/2006/04/rss_bandit_feeds_mix_up.html
//for a user complaint about the issue.
httpRequest.Pipelined = false;
}
else if (fileRequest != null)
{
fileRequest.Timeout = DefaultTimeout;
if (requestParameter.Credentials != null)
{
fileRequest.Credentials = requestParameter.Credentials;
}
}
else if (nntpRequest != null)
{
// ten minutes timeout. Large timeout is needed if this is first time we are fetching news
//TODO: move the timeout handling to the requestor
nntpRequest.Timeout = DefaultTimeout * 5;
if (requestParameter.Credentials != null)
{
nntpRequest.Credentials = requestParameter.Credentials;
}
if (requestParameter.LastModified > MinValue)
{
nntpRequest.IfModifiedSince = requestParameter.LastModified;
}
}
else
{
throw new NotImplementedException("Unsupported WebRequest type: " + webRequest.GetType());
}
return(webRequest);
}
/// <summary>
/// Creates a new RequestParameter instance.
/// </summary>
/// <param name="address">request url</param>
/// <param name="credentials">The credentials.</param>
/// <param name="p">The RequestParameter.</param>
/// <returns></returns>
public static RequestParameter Create(Uri address, ICredentials credentials, RequestParameter p)
{
return(new RequestParameter(address, p.UserAgent, p.Proxy, credentials, p.LastModified, p.ETag, p.SetCookies, p.Timeout));
}
/// <summary>
/// Creates a new RequestParameter instance.
/// </summary>
/// <param name="setCookies"></param>
/// <param name="p"></param>
/// <returns></returns>
public static RequestParameter Create(bool setCookies, RequestParameter p)
{
return(new RequestParameter(p.RequestUri, p.UserAgent, p.Proxy, p.Credentials, p.LastModified, p.ETag, setCookies, p.Timeout));
}