private void DownloadAsync <T>(RequestState <T> requestState, Exception exception)
{
if (!exception.IsNull() && RetryWaitDuration.HasValue)
{
Thread.Sleep(RetryWaitDuration.Value);
}
if (requestState.Retry-- > 0)
{
requestState.Clean();
requestState.Request = (HttpWebRequest)WebRequest.Create(requestState.CrawlStep.Uri);
requestState.Request.Method = requestState.Method.ToString();
SetDefaultRequestProperties(requestState.Request);
IAsyncResult asyncResult = requestState.Request.BeginGetResponse(null, requestState);
asyncResult.FromAsync((ia, isTimeout) =>
{
if (isTimeout)
{
DownloadAsync(requestState, new TimeoutException("Connection Timeout"));
}
else
{
ResponseCallback <T>(ia);
}
}, ConnectionTimeout);
}
else
{
requestState.CallComplete(null, exception);
}
}
private async Task <RequestState <T> > DownloadAsync <T>(RequestState <T> requestState)
{
if (requestState.Retry-- <= 0)
{
requestState.CallComplete(null, new TimeoutException("Connection Timeout"));
return(requestState);
}
requestState.Clean();
var clientHandler = new HttpClientHandler();
var client = new HttpClient(clientHandler);
var request = new HttpRequestMessage(requestState.Method, requestState.CrawlStep.Uri);
requestState.Request = request;
SetDefaultRequestProperties(clientHandler, request);
Exception unhandledException = null;
HttpResponseMessage httpResponse = null;
try
{
httpResponse = await client.SendAsync(request).WithTimeout(this.ConnectionTimeout);
var response = httpResponse.Content;
var downloadBufferSize = this.DownloadBufferSize ?? DefaultDownloadBufferSize;
var contentLength = response.Headers.ContentLength ?? throw new InvalidOperationException("Content length not specified");
requestState.ResponseBuffer = new MemoryStreamWithFileBackingStore((int)contentLength,
this.MaximumDownloadSizeInRam ?? int.MaxValue,
(int)downloadBufferSize);
// Read the response into a Stream object.
var responseStream = await response.ReadAsStreamAsync().WithTimeout(this.ReadTimeout);
await responseStream.CopyToAsync(requestState.ResponseBuffer,
(source, dest, exception) =>
{
if (exception.IsNull())
{
CallComplete(requestState, httpResponse);
}
else
{
// Put delay here in case of error.
//DownloadWithRetryAsync(requestState, exception);
}
},
bd =>
{
requestState.DownloadProgress?.Invoke(new DownloadProgressEventArgs
{
Referrer = requestState.Referrer,
Step = requestState.CrawlStep,
BytesReceived = bd,
TotalBytesToReceive = (uint)httpResponse.Content.Headers.ContentLength,
DownloadTime = requestState.StartTime - DateTime.UtcNow,
});
},
downloadBufferSize, this.MaximumContentSize, this.ReadTimeout);
CallComplete(requestState, httpResponse);
}
catch (HttpRequestException)
{
CallComplete(requestState, httpResponse);
}
catch (Exception e)
{
unhandledException = e;
}
if (unhandledException != null)
{
return(await DownloadWithRetryAsync(requestState));
}
return(requestState);
}
