/*
* public static CommunicationResponse GetPageCustom(HttpWebRequest custom, byte[] postDatToLog = null, List<string> paramsToMask = null)
* {
* CheckAndSleep();
*
* CommunicationResponse communication = new CommunicationResponse(custom.RequestUri.ToString());
*
* HttpWebRequest request = custom;
*
* HttpWebResponse response = null;
* StreamReader response_stream = null;
* CookieContainer cookies = custom.CookieContainer;
* if (cookies == null)
* {
* cookies = new CookieContainer();
* }
* try
* {
* if (ApplicationConstants.LogVerbosity == LoggingLevel.Verbose)
* { LogRequest(request, postDatToLog, paramsToMask); }
* }
* catch (Exception e) { Responder.LogException(e.Message, e.ToString()); }
* try
* {
* response = (HttpWebResponse)request.GetResponse();
*
* if (response.StatusCode == HttpStatusCode.OK)
* {
*
* if (string.IsNullOrEmpty(Responder.SourceCharsetEncoding))
* {
* response_stream = new StreamReader(response.GetResponseStream());
* communication.Html = response_stream.ReadToEnd();
* }
* else
* {
* response_stream = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(Responder.SourceCharsetEncoding));
* communication.Html = SournceEncodingToUTF8(response_stream.ReadToEnd());
* }
* }
*
* if (response != null)
* {
* communication.StatusCode = response.StatusCode;
* if (response.StatusCode != HttpStatusCode.OK) //if OK, already set above
* {
* communication.Html = response_stream.ReadToEnd();
* }
* communication.Cookies = response.Cookies;
* communication.Uri = response.ResponseUri;
* cookies.Add(communication.Cookies);
* Responder.COOKIES = cookies;
* }
* }
* catch (Exception e)
* {
* Responder.LogException(e.Message, e.ToString());
* communication.Clear();
* }
* finally
* {
* if (response_stream != null)
* response_stream.Close();
*
* if (response != null)
* response.Close();
*
* try
* {
* if (ApplicationConstants.LogVerbosity == LoggingLevel.Verbose)
* {
* LogResponse(communication, request.Method.Trim() == "POST" ? true : false);
* }
* }
* catch (Exception e) { Responder.LogException(e.Message, e.ToString()); }
* }
*
* return communication;
* }
*/
public static bool IsHTTP200AtPage(string url, USER_AGENT userAgent = USER_AGENT.Windows)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = userAgents[userAgent.ToSafeString()];
request.Method = "GET";
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
return(true);
}
}
catch
{
}
finally
{
if (response != null)
{
response.Close();
}
}
return(false);
}
/*
public static CommunicationResponse GetPageCustom(HttpWebRequest custom, byte[] postDatToLog = null, List<string> paramsToMask = null)
{
CheckAndSleep();
CommunicationResponse communication = new CommunicationResponse(custom.RequestUri.ToString());
HttpWebRequest request = custom;
HttpWebResponse response = null;
StreamReader response_stream = null;
CookieContainer cookies = custom.CookieContainer;
if (cookies == null)
{
cookies = new CookieContainer();
}
try
{
if (ApplicationConstants.LogVerbosity == LoggingLevel.Verbose)
{ LogRequest(request, postDatToLog, paramsToMask); }
}
catch (Exception e) { Responder.LogException(e.Message, e.ToString()); }
try
{
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
if (string.IsNullOrEmpty(Responder.SourceCharsetEncoding))
{
response_stream = new StreamReader(response.GetResponseStream());
communication.Html = response_stream.ReadToEnd();
}
else
{
response_stream = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(Responder.SourceCharsetEncoding));
communication.Html = SournceEncodingToUTF8(response_stream.ReadToEnd());
}
}
if (response != null)
{
communication.StatusCode = response.StatusCode;
if (response.StatusCode != HttpStatusCode.OK) //if OK, already set above
{
communication.Html = response_stream.ReadToEnd();
}
communication.Cookies = response.Cookies;
communication.Uri = response.ResponseUri;
cookies.Add(communication.Cookies);
Responder.COOKIES = cookies;
}
}
catch (Exception e)
{
Responder.LogException(e.Message, e.ToString());
communication.Clear();
}
finally
{
if (response_stream != null)
response_stream.Close();
if (response != null)
response.Close();
try
{
if (ApplicationConstants.LogVerbosity == LoggingLevel.Verbose)
{
LogResponse(communication, request.Method.Trim() == "POST" ? true : false);
}
}
catch (Exception e) { Responder.LogException(e.Message, e.ToString()); }
}
return communication;
}
*/
public static bool IsHTTP200AtPage(string url, USER_AGENT userAgent = USER_AGENT.Windows)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = userAgents[userAgent.ToSafeString()];
request.Method = "GET";
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
return true;
}
catch
{
}
finally
{
if (response != null)
response.Close();
}
return false;
}
public static CommunicationResponse GetPage(string url, string referer = "", bool resetCookies = false, USER_AGENT userAgent = USER_AGENT.Windows, CallBack cb = null, int retryCount = 0)
{
CheckAndSleep();
CommunicationResponse communication = new CommunicationResponse(url);
HttpWebRequest request = null;
CookieContainer cookies = null;
HttpWebResponse response = null;
StreamReader response_stream = null;
if (resetCookies)
{
}
else
{
cookies = null; //-Responder.COOKIES; CRITICAL. in GET request. always clearing the cooking
}
if (cookies == null)
{
cookies = new CookieContainer();
}
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = userAgents[userAgent.ToSafeString()];
request.Method = "GET";
request.AllowAutoRedirect = true;
request.CookieContainer = cookies;
if (string.IsNullOrEmpty(referer) == false)
request.Referer = referer;
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
if (string.IsNullOrEmpty(SourceCharsetEncoding))
{
response_stream = new StreamReader(response.GetResponseStream());
communication.Html = response_stream.ReadToEnd();
}
else
{
response_stream = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(SourceCharsetEncoding));
communication.Html = SournceEncodingToUTF8(response_stream.ReadToEnd());
}
}
#region Do this whatever the status is returned
if (response != null)
{
communication.StatusCode = response.StatusCode;
if (response.StatusCode != HttpStatusCode.OK) //if OK, already set above
{
communication.Html = response_stream.ReadToEnd();
}
communication.Cookies = response.Cookies;
communication.Uri = response.ResponseUri;
cookies.Add(communication.Cookies);
COOKIES = cookies;
}
#endregion
}
catch (WebException e)
{
#region Sleep if blocked
if (retryCount < 3)
{
GetPage(url, referer, resetCookies, userAgent, cb, ++retryCount);
}
#endregion
using (WebResponse Exresponse = e.Response)
{
HttpWebResponse httpResponse = (HttpWebResponse)Exresponse;
if (httpResponse != null)
{
communication.StatusCode = httpResponse.StatusCode;
communication.Cookies = httpResponse.Cookies;
communication.Uri = httpResponse.ResponseUri;
try
{
if (string.IsNullOrEmpty(SourceCharsetEncoding))
{
response_stream = new StreamReader(httpResponse.GetResponseStream());
communication.Html = response_stream.ReadToEnd();
}
else
{
response_stream = new StreamReader(httpResponse.GetResponseStream(), Encoding.GetEncoding(SourceCharsetEncoding));
communication.Html = SournceEncodingToUTF8(response_stream.ReadToEnd());
}
}
catch { }
cookies.Add(communication.Cookies);
COOKIES = cookies;
}
}
}
catch (Exception e)
{
#region Sleep if blocked
if (retryCount < 3)
{
GetPage(url, referer, resetCookies, userAgent, cb, ++retryCount);
}
else
{
communication.Clear();
}
#endregion
}
finally
{
if (response_stream != null)
response_stream.Close();
if (response != null)
response.Close();
if (cb != null)
{
communication = cb(communication);
}
}
return communication;
}
public static CommunicationResponse GetPage(string url, string referer = "", bool resetCookies = false, USER_AGENT userAgent = USER_AGENT.Windows, CallBack cb = null, int retryCount = 0)
{
CheckAndSleep();
CommunicationResponse communication = new CommunicationResponse(url);
HttpWebRequest request = null;
CookieContainer cookies = null;
HttpWebResponse response = null;
StreamReader response_stream = null;
if (resetCookies)
{
}
else
{
cookies = null; //-Responder.COOKIES; CRITICAL. in GET request. always clearing the cooking
}
if (cookies == null)
{
cookies = new CookieContainer();
}
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = userAgents[userAgent.ToSafeString()];
request.Method = "GET";
request.AllowAutoRedirect = true;
request.CookieContainer = cookies;
if (string.IsNullOrEmpty(referer) == false)
{
request.Referer = referer;
}
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
if (string.IsNullOrEmpty(SourceCharsetEncoding))
{
response_stream = new StreamReader(response.GetResponseStream());
communication.Html = response_stream.ReadToEnd();
}
else
{
response_stream = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(SourceCharsetEncoding));
communication.Html = SournceEncodingToUTF8(response_stream.ReadToEnd());
}
}
#region Do this whatever the status is returned
if (response != null)
{
communication.StatusCode = response.StatusCode;
if (response.StatusCode != HttpStatusCode.OK) //if OK, already set above
{
communication.Html = response_stream.ReadToEnd();
}
communication.Cookies = response.Cookies;
communication.Uri = response.ResponseUri;
cookies.Add(communication.Cookies);
COOKIES = cookies;
}
#endregion
}
catch (WebException e)
{
#region Sleep if blocked
if (retryCount < 3)
{
GetPage(url, referer, resetCookies, userAgent, cb, ++retryCount);
}
#endregion
using (WebResponse Exresponse = e.Response)
{
HttpWebResponse httpResponse = (HttpWebResponse)Exresponse;
if (httpResponse != null)
{
communication.StatusCode = httpResponse.StatusCode;
communication.Cookies = httpResponse.Cookies;
communication.Uri = httpResponse.ResponseUri;
try
{
if (string.IsNullOrEmpty(SourceCharsetEncoding))
{
response_stream = new StreamReader(httpResponse.GetResponseStream());
communication.Html = response_stream.ReadToEnd();
}
else
{
response_stream = new StreamReader(httpResponse.GetResponseStream(), Encoding.GetEncoding(SourceCharsetEncoding));
communication.Html = SournceEncodingToUTF8(response_stream.ReadToEnd());
}
}
catch { }
cookies.Add(communication.Cookies);
COOKIES = cookies;
}
}
}
catch (Exception e)
{
#region Sleep if blocked
if (retryCount < 3)
{
GetPage(url, referer, resetCookies, userAgent, cb, ++retryCount);
}
else
{
communication.Clear();
}
#endregion
}
finally
{
if (response_stream != null)
{
response_stream.Close();
}
if (response != null)
{
response.Close();
}
if (cb != null)
{
communication = cb(communication);
}
}
return(communication);
}
