public static byte[] DownloadFile(string uri, object customTraceLog = null, bool detailedLog = true, bool blockAllExceptions = false, byte[] errorCaseResult = null)
{
byte[] r = null;
CBB_LoggingCustomTraceLog log = CBB_LoggingCustomTraceLog.Unbox(customTraceLog);
try
{
CBB_LoggingCustomTraceLogExtensions.AddLineAndIncreaseIdent(log, "Requesting '" + uri + "'...");
//source: http://stackoverflow.com/questions/3615800/download-image-from-the-site-in-net-c
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
// Check that the remote file was found. The ContentType
// check is performed since a request for a non-existent
// image file might be redirected to a 404-page, which would
// yield the StatusCode "OK", even though the image was not
// found.
if ((response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.Redirect)
//&& response.ContentType.StartsWith("image", StringComparison.OrdinalIgnoreCase)
)
{
// if the remote file was found, download it
using (Stream inputStream = response.GetResponseStream())
{
using (MemoryStream outputStream = new MemoryStream())
{
byte[] buffer = new byte[4096];
int bytesRead;
do
{
bytesRead = inputStream.Read(buffer, 0, buffer.Length);
outputStream.Write(buffer, 0, bytesRead);
} while (bytesRead != 0);
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Serializing to byte array...");
}
r = outputStream.ToArray();
}
}
}
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting succeeded. Response length in chars:" + r.Length);
}
}
catch (Exception exception)
{
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting of '" + uri + "' failed. Error details:" + exception.Message);
}
if (!blockAllExceptions)
{
throw;
}
else
{
r = errorCaseResult;
}
}
finally
{
CBB_LoggingCustomTraceLogExtensions.DecreaseIdent(log);
}
return(r);
}
/// <summary>
///
/// </summary>
/// <param name="uri"></param>
/// <param name="username"></param>
/// <param name="password"></param>
/// <param name="domain"></param>
/// <param name="accessToken"></param>
/// <param name="headers"></param>
/// <param name="isPostMethod"></param>
/// <param name="requestUriContentType">0-default; 1-JSON</param>
/// <param name="customTraceLog"></param>
/// <param name="blockAllExceptions"></param>
/// <param name="errorCaseResult"></param>
/// <returns></returns>
public static string RequestUri(string uri, string username = null, string password = null, string domain = null, /*string accessToken = null, */ List <string> headers = null, bool isPostMethod = false, string postData = null, int requestUriContentType = 0, object customTraceLog = null, bool blockAllExceptions = false, string errorCaseResult = null, bool expectCookies = false, List <Cookie> cookies = null, bool detailedLog = true, System.Text.Encoding encoderForParsingResult = null, string fileNameWhereToDumpResponse = null, bool?allowAutoRedirect = null)
{
string r = null;
CBB_LoggingCustomTraceLog log = CBB_LoggingCustomTraceLog.Unbox(customTraceLog);
try
{
CBB_LoggingCustomTraceLogExtensions.AddLineAndIncreaseIdent(log, "Requesting '" + uri + "'...");
// used to build entire input
StringBuilder sb = new StringBuilder();
// used on each read operation
byte[] buf = new byte[8192];
// prepare the web page we will be asking for
WebRequest request = (WebRequest)WebRequest.Create(uri);
if (CBB_CommonExtenderClass.IsNOTNullOrWhiteSpace(username) && CBB_CommonExtenderClass.IsNullOrWhiteSpace(password))
{
throw new Exception("Password must be specified if username is.");
}
else if (CBB_CommonExtenderClass.IsNullOrWhiteSpace(username) && CBB_CommonExtenderClass.IsNOTNullOrWhiteSpace(password))
{
throw new Exception("Username must be specified when password is.");
}
else if (CBB_CommonExtenderClass.IsNOTNullOrWhiteSpace(username) && CBB_CommonExtenderClass.IsNOTNullOrWhiteSpace(password))
{
if (CBB_CommonExtenderClass.IsNullOrWhiteSpace(domain))
{
request.Credentials = new NetworkCredential(username, password);
}
else
{
request.Credentials = new NetworkCredential(username, password, domain);
}
CBB_LoggingCustomTraceLogExtensions.AssignSensitiveString(log, password, 2);
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting with credentials '" + CBB_CommonExtenderClass.AppendIfNotNullOrWhiteSpace(CBB_CommonExtenderClass.ToNonNullString(domain), "\\") + username + " " + CBB_CommonExtenderClass.ToNonNullString(password) + "'...");
}
}
//if (accessToken.IsNOTNullOrWhiteSpace())
//{
// request.Headers.Add("access_token", accessToken);
//}
if (headers != null)
{
foreach (string header in headers)
{
//http://stackoverflow.com/questions/239725/cannot-set-some-http-headers-when-using-system-net-webrequest
if (header.ToLower() == "connection:keep-alive")
{
if (request is HttpWebRequest)
{
(request as HttpWebRequest).KeepAlive = true;
}
else
{
throw new Exception("Can not set http headers when using WebRequest.");
}
}
else if (header.Split(':')[0].ToLower() == "referer")
{
if (request is HttpWebRequest)
{
(request as HttpWebRequest).Referer = header.Substring(header.IndexOf(':') + 1);
}
else
{
throw new Exception("Can not set http headers when using WebRequest.");
}
}
else if (header.Split(':')[0].ToLower() == "user-agent")
{
if (request is HttpWebRequest)
{
(request as HttpWebRequest).UserAgent = header.Substring(header.IndexOf(':') + 1);
}
else
{
throw new Exception("Can not set http headers when using WebRequest.");
}
}
else if (header.Split(':')[0].ToLower() == "accept")
{
if (request is HttpWebRequest)
{
(request as HttpWebRequest).Accept = header.Substring(header.IndexOf(':') + 1);
}
else
{
throw new Exception("Can not set http headers when using WebRequest.");
}
}
else
{
if (request.Headers.AllKeys.Contains(header.Split(':')[0]))
{
request.Headers.Set(header.Split(':')[0], header.Substring(header.IndexOf(':') + 1));
}
else
{
request.Headers.Add(header);
}
}
}
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting with headers '" + CBB_CommonExtenderClass.ToCSV(headers) + "'...");
}
}
if (isPostMethod)
{
request.Method = "POST";
}
switch (requestUriContentType)
{
case (int)RequestUriContentType.Default:
break;
case (int)RequestUriContentType.ApplicationJson:
//var hk = request.Headers.AllKeys.FirstOrDefault(k => k.ToLower() == "content-type");
//if(hk!=null)
//{
// request.Headers.Remove(hk);
//}
//request.Headers.Add("Content-Type", "application/json");
request.ContentType = "application/json";
if (CBB_CommonExtenderClass.IsNOTNullOrWhiteSpace(postData))
{
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting with post data '" + postData + "'...");
}
using (var streamWriter = new StreamWriter(request.GetRequestStream()))
{
//initiate the request
//JavaScriptSerializer serializer = new JavaScriptSerializer();
//var d = serializer.Deserialize<Dictionary<string, object>>(postData);
streamWriter.Write(postData);
streamWriter.Flush();
streamWriter.Close();
}
}
break;
case (int)RequestUriContentType.TextJson:
//var hk = request.Headers.AllKeys.FirstOrDefault(k => k.ToLower() == "content-type");
//if(hk!=null)
//{
// request.Headers.Remove(hk);
//}
//request.Headers.Add("Content-Type", "application/json");
request.ContentType = "text/json";
if (CBB_CommonExtenderClass.IsNOTNullOrWhiteSpace(postData))
{
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting with post data '" + postData + "'...");
}
using (var streamWriter = new StreamWriter(request.GetRequestStream()))
{
//initiate the request
//JavaScriptSerializer serializer = new JavaScriptSerializer();
//var d = serializer.Deserialize<Dictionary<string, object>>(postData);
streamWriter.Write(postData);
streamWriter.Flush();
streamWriter.Close();
}
}
break;
case (int)RequestUriContentType.ImageJpeg:
case (int)RequestUriContentType.ImagePng:
case (int)RequestUriContentType.ImageGif:
switch (requestUriContentType)
{
case (int)RequestUriContentType.ImageJpeg: request.ContentType = "image/jpeg"; break;
case (int)RequestUriContentType.ImagePng: request.ContentType = "image/png"; break;
case (int)RequestUriContentType.ImageGif: request.ContentType = "image/gif"; break;
}
if (CBB_CommonExtenderClass.IsNOTNullOrWhiteSpace(postData))
{
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting with post data '" + postData + "'...");
}
using (var streamWriter = new StreamWriter(request.GetRequestStream()))
{
//initiate the request
//JavaScriptSerializer serializer = new JavaScriptSerializer();
//var d = serializer.Deserialize<Dictionary<string, object>>(postData);
streamWriter.Write(postData);
streamWriter.Flush();
streamWriter.Close();
}
}
break;
case (int)RequestUriContentType.ApplicationXWwwFormUlrEncoded:
request.ContentType = "application/x-www-form-urlencoded";
ASCIIEncoding encoder = new ASCIIEncoding();
byte[] data = encoder.GetBytes(postData); // a json object, or xml, whatever...
request.ContentLength = data.Length;
//request.Expect = "application/json";
request.GetRequestStream().Write(data, 0, data.Length);
break;
}
if (cookies != null)
{
if (request is HttpWebRequest)
{
(request as HttpWebRequest).CookieContainer = new CookieContainer();
foreach (Cookie cookie in cookies)
{
(request as HttpWebRequest).CookieContainer.Add(cookie);
}
}
else
{
throw new Exception("Can not set http headers when using WebRequest.");
}
}
if (request is HttpWebRequest && allowAutoRedirect != null)
{
(request as HttpWebRequest).AllowAutoRedirect = allowAutoRedirect.Value;
}
// execute the request
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
//if (
// requestUriContentType == (int) RequestUriContentType.ImageJpeg ||
// requestUriContentType == (int) RequestUriContentType.ImagePng ||
// requestUriContentType == (int) RequestUriContentType.ImageGif
// )
//{
// // Check that the remote file was found. The ContentType
// // check is performed since a request for a non-existent
// // image file might be redirected to a 404-page, which would
// // yield the StatusCode "OK", even though the image was not
// // found.
// if ((response.StatusCode == HttpStatusCode.OK ||
// response.StatusCode == HttpStatusCode.Moved ||
// response.StatusCode == HttpStatusCode.Redirect) &&
// response.ContentType.StartsWith("image", StringComparison.OrdinalIgnoreCase))
// {
// // if the remote file was found, download it
// //code moved to dump response region below...
// return true;
// }
// else
// return false;
//}
if (fileNameWhereToDumpResponse != null)
{
using (Stream inputStream = response.GetResponseStream())
using (Stream outputStream = File.OpenWrite(fileNameWhereToDumpResponse))
{
byte[] buffer = new byte[4096];
int bytesRead;
do
{
bytesRead = inputStream.Read(buffer, 0, buffer.Length);
outputStream.Write(buffer, 0, bytesRead);
} while (bytesRead != 0);
inputStream.Seek(0, SeekOrigin.Begin);
}
}
if (encoderForParsingResult == null)
{
r = DecodeWebResponse(response, log);
}
else
{
// we will read data via the response stream
Stream resStream = response.GetResponseStream();
string tempString = null;
int count = 0;
encoderForParsingResult = encoderForParsingResult ?? Encoding.ASCII;
do
{
// fill the buffer with data
count = resStream.Read(buf, 0, buf.Length);
// make sure we read some data
if (count != 0)
{
// translate from bytes to ASCII text
tempString = encoderForParsingResult.GetString(buf, 0, count);
// continue building the string
sb.Append(tempString);
}
} while (count > 0); // any more data to read?
r = sb.ToString();
}
if (expectCookies && response.Cookies != null && response.Cookies.Count > 0)
{
cookies.Clear();
for (int i = 0; i < response.Cookies.Count; i++)
{
cookies.Add(response.Cookies[i]);
}
}
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting succeeded. Response length in chars:" + r.Length);
}
}
catch (Exception exception)
{
if (detailedLog)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Requesting of '" + uri + "' failed. Error details:" + exception.Message);
}
if (!blockAllExceptions)
{
throw;
}
else
{
r = errorCaseResult;
}
}
finally {
CBB_LoggingCustomTraceLogExtensions.DecreaseIdent(log);
}
return(r);
}
/// <summary>
/// Decodes the web response encoding.
/// Source: https://blogs.msdn.microsoft.com/feroze_daud/2004/03/30/downloading-content-from-the-web-using-different-encodings/
/// </summary>
/// <param name="w">The w.</param>
/// <param name="log">The log.</param>
/// <returns></returns>
private static String DecodeWebResponse(WebResponse w, CBB_LoggingCustomTraceLog log)
{
//
// first see if content length header has charset = calue
//
String charset = null;
String ctype = w.Headers["content - type"];
if (ctype != null)
{
int ind = CBB_CommonExtenderClass.IndexAfterWords(ctype, true, "charset", CBB_CommonExtenderClass.OPTIONAL_SPACES, "=");
if (ind != -1)
{
charset = ctype.Substring(ind + 8);
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "CT: charset =" + charset);
}
}
// save data to a memorystream
MemoryStream rawdata = new MemoryStream();
byte[] buffer = new byte[1024];
Stream rs = w.GetResponseStream();
int read = rs.Read(buffer, 0, buffer.Length);
while (read > 0)
{
rawdata.Write(buffer, 0, read);
read = rs.Read(buffer, 0, buffer.Length);
}
rs.Close();
//
// if ContentType is null, or did not contain charset, we search in body
//
if (charset == null)
{
MemoryStream ms = rawdata;
ms.Seek(0, SeekOrigin.Begin);
StreamReader srr = new StreamReader(ms, Encoding.ASCII);
String meta = srr.ReadToEnd();
if (meta != null)
{
int start_ind = CBB_CommonExtenderClass.IndexAfterWords(meta, true, "charset", CBB_CommonExtenderClass.OPTIONAL_SPACES, "=", CBB_CommonExtenderClass.OPTIONAL_SPACES, "\"");
int end_ind = -1;
if (start_ind != -1)
{
end_ind = meta.IndexOf("\"", start_ind);
if (end_ind != -1)
{
charset = meta.Substring(start_ind, end_ind - start_ind);
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "META: charset=" + charset);
}
}
}
}
Encoding e = null;
if (charset == null)
{
e = Encoding.ASCII; //default encoding
}
else
{
try
{
e = Encoding.GetEncoding(charset);
}
catch (Exception ee)
{
CBB_LoggingCustomTraceLogExtensions.AddLine(log, "Exception: GetEncoding: " + charset);
CBB_LoggingCustomTraceLogExtensions.AddLine(log, ee.ToString());
e = Encoding.ASCII;
}
}
rawdata.Seek(0, SeekOrigin.Begin);
StreamReader sr = new StreamReader(rawdata, e);
String s = sr.ReadToEnd();
return(s);
}
