public String getUrlContents(String urlToFetch, string cookies, bool verbose)
{
try
{
if (verbose)
{
PublicDI.log.info("Fetching url: {0}", urlToFetch);
}
var webRequest = WebRequest.Create(urlToFetch) as HttpWebRequest;
if (webRequest.isNull())
{
return(null);
}
webRequest.Timeout = Web.DefaultHttpWebRequestTimeout;
webRequest.ReadWriteTimeout = Web.DefaultHttpWebRequestTimeout;
//setup headers (& cookies)
if (cookies != null && cookies.valid())
{
Headers_Request.add("Cookie", cookies);
}
foreach (var header in Headers_Request)
{
webRequest.Headers.Add(header.Key, header.Value);
}
var rResponse = webRequest.GetResponse();
updateHeadersResponse(rResponse.Headers); // store response headers
Stream sStream = rResponse.GetResponseStream();
if (sStream.isNull())
{
return(null);
}
var srStreamReader = new StreamReader(sStream);
string sHtml = srStreamReader.ReadToEnd();
sStream.Close();
srStreamReader.Close();
rResponse.Close();
return(sHtml);
}
catch (Exception ex)
{
PublicDI.log.error("Error in getUrlContents: {0}", ex.Message);
return("");
}
}
public Dictionary <string, string> add_Header(string name, string value)
{
Headers_Request.add(name, value);
return(Headers_Request);
}
public Dictionary <string, string> set_Cookies(string value)
{
Headers_Request.add("Cookie", value);
return(Headers_Request);
}
public String getUrlContents_POST(String urlToFetch, string contentType, string cookies, byte[] postData)
{
try
{
// the Timeout and GC calls below were introduced due to GetResponseStream() hangs
var webRequest = (HttpWebRequest)WebRequest.Create(urlToFetch);
webRequest.Timeout = DefaultHttpWebRequestTimeout;
webRequest.ReadWriteTimeout = Web.DefaultHttpWebRequestTimeout;
//setup headers (& cookies)
if (cookies != null && cookies.valid())
{
Headers_Request.add("Cookie", cookies);
}
foreach (var header in Headers_Request)
{
webRequest.Headers.Add(header.Key, header.Value);
}
// setup POST details:
webRequest.Method = "POST";
webRequest.ContentLength = postData.Length;
webRequest.ContentType = contentType;
using (Stream dataStream = webRequest.GetRequestStream())
{
dataStream.Write(postData, 0, postData.Length);
dataStream.Close();
System.GC.Collect();
}
using (WebResponse rResponse = webRequest.GetResponse())
{
updateHeadersResponse(rResponse.Headers); // store response headers
using (Stream sStream = rResponse.GetResponseStream())
{
if (sStream.isNull())
{
return(null);
}
var srStreamReader = new StreamReader(sStream);
string sHtml = srStreamReader.ReadToEnd();
sStream.Close();
srStreamReader.Close();
rResponse.Close();
return(sHtml);
}
}
}
catch (WebException webEx)
{
if (webEx.Response.notNull())
{
updateHeadersResponse(webEx.Response.Headers); // store response headers
}
}
catch (Exception ex)
{
PublicDI.log.error("Error in getUrlContents: {0}", ex.Message);
}
return("");
// });
//thread.
}