public HtmlDocument GetWebRequest()
{
HtmlDocument document = new HtmlDocument();
byte[] responseBytes;
if (!cache.IsCachedUrl(Url))
{
responseBytes = httpClient.DownloadData(Url);
CacheDb.SaveCache(Url, responseBytes);
}
else
{
responseBytes = cache.GetCachedUrl(Url);
}
MemoryStream mStream = new MemoryStream(responseBytes);
document.Load(mStream);
return(document);
}
public HtmlDocument GetWebRequest(String Url)
{
for (int RetryCount = 1; true; RetryCount++)
{
try
{
HttpClient httpClient = new HttpClient();
HtmlDocument document = new HtmlDocument();
document.OptionAutoCloseOnEnd = true;
document.OptionCheckSyntax = true;
document.OptionFixNestedTags = true;
//document.OptionWriteEmptyNodes = true;
byte[] responseBytes;
if (EnableCaching && !cache.IsCachedUrl(Url))
{
responseBytes = httpClient.DownloadData(Url);
CacheDb.SaveCache(Url, responseBytes);
}
else if (EnableCaching)
{
responseBytes = cache.GetCachedUrl(Url);
}
else
{
responseBytes = httpClient.DownloadData(Url);
}
MemoryStream mStream = new MemoryStream(responseBytes);
document.Load(mStream);
return(document);
}
catch (Exception ex)
{
if (RetryCount == MaxRetryCount)
{
throw ex;
}
}
}
}
public HtmlDocument PostRequest(String Url, WebHeaderCollection Header, NameValueCollection formData)
{
for (int RetryCount = 1; true; RetryCount++)
{
try
{
HttpClient httpClient = new HttpClient();
Crawler crawler = new Crawler();
byte[] responseBytes;
crawler.Url = Url;
//WebClient httpClient = new WebClient();
if (!ReferenceEquals(Header, null))
{
httpClient.Headers = Header;
}
if (!ReferenceEquals(formData, null))
{
httpClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
responseBytes = httpClient.UploadValues(Url, "POST", formData);
}
else
{
responseBytes = httpClient.DownloadData(Url);
}
string resultAuthTicket = Encoding.UTF8.GetString(responseBytes);
httpClient.Dispose();
MemoryStream mStream = new MemoryStream(responseBytes);
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.Load(mStream);
return(document);
}
catch (Exception ex)
{
if (RetryCount == MaxRetryCount)
{
throw ex;
}
}
}
}
public HtmlDocument PostRequest(String Url, WebHeaderCollection Header, NameValueCollection formData)
{
for (int RetryCount = 1 ; true ; RetryCount++)
{
try
{
HttpClient httpClient = new HttpClient();
Crawler crawler = new Crawler();
byte[] responseBytes;
crawler.Url = Url;
//WebClient httpClient = new WebClient();
if (!ReferenceEquals(Header, null))
{
httpClient.Headers = Header;
}
if (!ReferenceEquals(formData, null))
{
httpClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
responseBytes = httpClient.UploadValues(Url, "POST", formData);
}
else
{
responseBytes = httpClient.DownloadData(Url);
}
string resultAuthTicket = Encoding.UTF8.GetString(responseBytes);
httpClient.Dispose();
MemoryStream mStream = new MemoryStream(responseBytes);
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.Load(mStream);
return document;
}
catch (Exception ex)
{
if (RetryCount == MaxRetryCount)
throw ex;
}
}
}
public HtmlDocument GetWebRequest(String Url)
{
for (int RetryCount = 1; true; RetryCount++)
{
try
{
HttpClient httpClient = new HttpClient();
HtmlDocument document = new HtmlDocument();
document.OptionAutoCloseOnEnd = true;
document.OptionCheckSyntax = true;
document.OptionFixNestedTags = true;
//document.OptionWriteEmptyNodes = true;
byte[] responseBytes;
if (EnableCaching && !cache.IsCachedUrl(Url))
{
responseBytes = httpClient.DownloadData(Url);
CacheDb.SaveCache(Url, responseBytes);
}
else if (EnableCaching)
responseBytes = cache.GetCachedUrl(Url);
else
responseBytes = httpClient.DownloadData(Url);
MemoryStream mStream = new MemoryStream(responseBytes);
document.Load(mStream);
return document;
}
catch (Exception ex)
{
if (RetryCount == MaxRetryCount)
throw ex;
}
}
}