// gets string content from a web URL
private static string RetrieveUrl(string url)
{
string pageContents = "";
// Try to grab the document
try
{
WebGrabber grabber = new WebGrabber(url);
grabber.Request.Accept = "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
//grabber.UserAgent = parsedUserAgent;
grabber.MaxRetries = 10;
grabber.Timeout = 5000;
grabber.TimeoutIncrement = 1000;
//grabber.Encoding = Encoding.UTF8;
//grabber.Encoding = encoding;
//grabber.AllowUnsafeHeader = true;
//grabber.CookieHeader = cookies;
//grabber.Debug = ScriptSettings.DebugMode;
// Retrieve the document
if (grabber.GetResponse())
{
pageContents = grabber.GetString();
}
}
catch (Exception e)
{
if (e is ThreadAbortException)
throw e;
//logger.Warn("Could not connect to " + parsedUrl + ". " + e.Message);
}
return pageContents;
}