/// <summary>
/// Add URL to cache
/// </summary>
/// <param name="Url"></param>
/// <returns></returns>
//private static HtmlAgilityPack.HtmlDocument AddUrl(String Url)
//{
// if (!IsCachedUrl(Url))
// {
// Crawler crawler = new Crawler();
// crawler.Url = Url;
// HtmlAgilityPack.HtmlDocument document = crawler.GetDocument();
// String FileName = GenerateFileName();
// document.Save(GetFullFileName(FileName));
// DataRow dRow = CacheDB.Tables[_dtName].NewRow();
// dRow["Url"] = Url;
// dRow["FileName"] = FileName;
// dRow["LastUpdated"] = DateTime.Now;
// dRow["ValidTill"] = DateTime.Now.AddMinutes(_cacheValidity);
// CacheDB.Tables[_dtName].Rows.Add(dRow);
// SaveCache();
// return document;
// }
// return new HtmlAgilityPack.HtmlDocument();
//}
public byte[] AddUrl(String Url, NameValueCollection formData = null )
{
Crawler crawler = new Crawler();
byte[] responseBytes;
crawler.Url = Url;
WebClient webClient = new WebClient();
if (!ReferenceEquals(formData, null))
{
webClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
responseBytes = webClient.UploadValues(Url, "POST", formData);
}
else
{
responseBytes = webClient.DownloadData(Url);
}
string resultAuthTicket = Encoding.UTF8.GetString(responseBytes);
webClient.Dispose();
//String FileName = GenerateFileName();
//File.WriteAllBytes(FileName, responseBytes);
//DataRow dRow = CacheDB.Tables[_dtName].NewRow();
//dRow["Url"] = Url;
//dRow["FileName"] = FileName;
//dRow["LastUpdated"] = DateTime.Now;
//dRow["ValidTill"] = DateTime.Now.AddMinutes(_cacheValidity);
//CacheDB.Tables[_dtName].Rows.Add(dRow);
//SaveCache();
SaveCache(Url, responseBytes);
return responseBytes;
}
public static HtmlAgilityPack.HtmlDocument PostUrl(String Url, WebHeaderCollection Header, NameValueCollection formData)
{
Crawler crawler = new Crawler();
byte[] responseBytes;
crawler.Url = Url;
WebClient webClient = new WebClient();
if (!ReferenceEquals(Header, null))
{
webClient.Headers = Header;
}
if (!ReferenceEquals(formData, null))
{
webClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
responseBytes = webClient.UploadValues(Url, "POST", formData);
}
else
{
responseBytes = webClient.DownloadData(Url);
}
string resultAuthTicket = Encoding.UTF8.GetString(responseBytes);
webClient.Dispose();
MemoryStream mStream = new MemoryStream(responseBytes);
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.Load(mStream);
return document;
}
public HtmlDocument PostRequest(String Url, WebHeaderCollection Header, NameValueCollection formData)
{
for (int RetryCount = 1 ; true ; RetryCount++)
{
try
{
HttpClient httpClient = new HttpClient();
Crawler crawler = new Crawler();
byte[] responseBytes;
crawler.Url = Url;
//WebClient httpClient = new WebClient();
if (!ReferenceEquals(Header, null))
{
httpClient.Headers = Header;
}
if (!ReferenceEquals(formData, null))
{
httpClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
responseBytes = httpClient.UploadValues(Url, "POST", formData);
}
else
{
responseBytes = httpClient.DownloadData(Url);
}
string resultAuthTicket = Encoding.UTF8.GetString(responseBytes);
httpClient.Dispose();
MemoryStream mStream = new MemoryStream(responseBytes);
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.Load(mStream);
return document;
}
catch (Exception ex)
{
if (RetryCount == MaxRetryCount)
throw ex;
}
}
}
