/// <summary>
/// Downloads the data.
/// </summary>
/// <param name = "absoluteUri">The absolute URI.</param>
/// <returns></returns>
public byte[] DownloadHttpData(string absoluteUri, bool isGZipStream, bool isDeflateStream, CookieContainer cookieContainer)
{
byte[] data = null;
try
{
int numberOfBytesRead = -1;
byte[] buffer = new byte[4096];
if (HttpWebResponse != null)
{
DateTime startTimeOriginal = DateTime.Now;
DateTime startTime = DateTime.Now;
using (Stream stream = HttpWebResponse.GetResponseStream())
{
_cookieManager.UpdateCookies(absoluteUri, cookieContainer, HttpWebResponse.Cookies);
//memory/gzip stream used here as StreamReader needs an Encoding, and the Encoding is properly determined after downloading the Stream.
using (MemoryStream memoryStream = new MemoryStream())
{
while (numberOfBytesRead != 0)
{
numberOfBytesRead = stream.Read(buffer, 0, buffer.Length);
memoryStream.Write(buffer, 0, numberOfBytesRead);
if (DateTime.Now.Subtract(startTime).TotalSeconds >= _applicationSettings.CrawlRequestTimeoutInMinutes * 60)
{
_consoleManager.OutputString(_consoleManager.BuildOutputString(" -> DownloadHttpData", absoluteUri, startTimeOriginal, DateTime.Now.Subtract(startTimeOriginal)), ConsoleColor.DarkGreen, ConsoleColor.Gray);
startTime = DateTime.Now;
}
}
data = memoryStream.ToArray();
}
if (isGZipStream)
{
numberOfBytesRead = -1;
using (MemoryStream memoryStream = new MemoryStream(data))
{
using (MemoryStream memoryStream2 = new MemoryStream())
{
using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress))
{
while (numberOfBytesRead != 0)
{
numberOfBytesRead = gZipStream.Read(buffer, 0, buffer.Length);
memoryStream2.Write(buffer, 0, numberOfBytesRead);
if (DateTime.Now.Subtract(startTime).TotalSeconds >= _applicationSettings.CrawlRequestTimeoutInMinutes * 60)
{
_consoleManager.OutputString(_consoleManager.BuildOutputString(" -> DownloadHttpData", absoluteUri, startTimeOriginal, DateTime.Now.Subtract(startTimeOriginal)), ConsoleColor.DarkGreen, ConsoleColor.Gray);
startTime = DateTime.Now;
}
}
data = memoryStream2.ToArray();
}
}
}
}
if (isDeflateStream)
{
numberOfBytesRead = -1;
using (MemoryStream memoryStream = new MemoryStream(data))
{
using (MemoryStream memoryStream2 = new MemoryStream())
{
using (DeflateStream deflateStream = new DeflateStream(memoryStream, CompressionMode.Decompress))
{
while (numberOfBytesRead != 0)
{
numberOfBytesRead = deflateStream.Read(buffer, 0, buffer.Length);
memoryStream2.Write(buffer, 0, numberOfBytesRead);
if (DateTime.Now.Subtract(startTime).TotalSeconds >= _applicationSettings.CrawlRequestTimeoutInMinutes * 60)
{
_consoleManager.OutputString(_consoleManager.BuildOutputString(" -> DownloadHttpData", absoluteUri, startTimeOriginal, DateTime.Now.Subtract(startTimeOriginal)), ConsoleColor.DarkGreen, ConsoleColor.Gray);
startTime = DateTime.Now;
}
}
data = memoryStream2.ToArray();
}
}
}
}
}
}
}
catch (Exception exception)
{
throw new Exception(exception.Message, exception);
}
finally
{
if (HttpWebResponse != null)
{
HttpWebResponse.Close();
HttpWebResponse.GetResponseStream().Close();
HttpWebResponse.GetResponseStream().Dispose();
}
}
if (data == null)
{
data = new byte[0];
}
return(data);
}
