/// <summary> /// Downloads the data. /// </summary> /// <param name = "absoluteUri">The absolute URI.</param> /// <returns></returns> public byte[] DownloadHttpData(string absoluteUri, bool isGZipStream, bool isDeflateStream, CookieContainer cookieContainer) { byte[] data = null; try { int numberOfBytesRead = -1; byte[] buffer = new byte[4096]; if (HttpWebResponse != null) { DateTime startTimeOriginal = DateTime.Now; DateTime startTime = DateTime.Now; using (Stream stream = HttpWebResponse.GetResponseStream()) { _cookieManager.UpdateCookies(absoluteUri, cookieContainer, HttpWebResponse.Cookies); //memory/gzip stream used here as StreamReader needs an Encoding, and the Encoding is properly determined after downloading the Stream. using (MemoryStream memoryStream = new MemoryStream()) { while (numberOfBytesRead != 0) { numberOfBytesRead = stream.Read(buffer, 0, buffer.Length); memoryStream.Write(buffer, 0, numberOfBytesRead); if (DateTime.Now.Subtract(startTime).TotalSeconds >= _applicationSettings.CrawlRequestTimeoutInMinutes * 60) { _consoleManager.OutputString(_consoleManager.BuildOutputString(" -> DownloadHttpData", absoluteUri, startTimeOriginal, DateTime.Now.Subtract(startTimeOriginal)), ConsoleColor.DarkGreen, ConsoleColor.Gray); startTime = DateTime.Now; } } data = memoryStream.ToArray(); } if (isGZipStream) { numberOfBytesRead = -1; using (MemoryStream memoryStream = new MemoryStream(data)) { using (MemoryStream memoryStream2 = new MemoryStream()) { using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) { while (numberOfBytesRead != 0) { numberOfBytesRead = gZipStream.Read(buffer, 0, buffer.Length); memoryStream2.Write(buffer, 0, numberOfBytesRead); if (DateTime.Now.Subtract(startTime).TotalSeconds >= _applicationSettings.CrawlRequestTimeoutInMinutes * 60) { _consoleManager.OutputString(_consoleManager.BuildOutputString(" -> DownloadHttpData", absoluteUri, startTimeOriginal, DateTime.Now.Subtract(startTimeOriginal)), ConsoleColor.DarkGreen, ConsoleColor.Gray); startTime = DateTime.Now; } } data = memoryStream2.ToArray(); } } } } if (isDeflateStream) { numberOfBytesRead = -1; using (MemoryStream memoryStream = new MemoryStream(data)) { using (MemoryStream memoryStream2 = new MemoryStream()) { using (DeflateStream deflateStream = new DeflateStream(memoryStream, CompressionMode.Decompress)) { while (numberOfBytesRead != 0) { numberOfBytesRead = deflateStream.Read(buffer, 0, buffer.Length); memoryStream2.Write(buffer, 0, numberOfBytesRead); if (DateTime.Now.Subtract(startTime).TotalSeconds >= _applicationSettings.CrawlRequestTimeoutInMinutes * 60) { _consoleManager.OutputString(_consoleManager.BuildOutputString(" -> DownloadHttpData", absoluteUri, startTimeOriginal, DateTime.Now.Subtract(startTimeOriginal)), ConsoleColor.DarkGreen, ConsoleColor.Gray); startTime = DateTime.Now; } } data = memoryStream2.ToArray(); } } } } } } } catch (Exception exception) { throw new Exception(exception.Message, exception); } finally { if (HttpWebResponse != null) { HttpWebResponse.Close(); HttpWebResponse.GetResponseStream().Close(); HttpWebResponse.GetResponseStream().Dispose(); } } if (data == null) { data = new byte[0]; } return(data); }