public static SimpleWebResponse ExecuteGet(string domain)
{
var ret = new SimpleWebResponse();
var uri = new Uri("http://" + domain + "/robots.txt");
var req = (HttpWebRequest)WebRequest.Create(uri);
req.Method = "GET";
req.KeepAlive = false;
req.UserAgent = "Mozilla/5.0 (compatible; RTSbot/1.0; +http://adamcaudill.com/rtsbot/)";
var resp = (HttpWebResponse)req.GetResponse();
var stream = new StreamReader(resp.GetResponseStream(), Encoding.GetEncoding("ISO-8859-1"));
ret.Body = stream.ReadToEnd();
stream.Close();
var headers = string.Empty;
foreach (string header in resp.Headers)
{
headers += string.Format("{0}: {1}\r\n", header, resp.Headers[header]);
}
ret.Headers = headers;
return ret;
}
public static SimpleWebResponse ExecuteGet(string domain)
{
var ret = new SimpleWebResponse();
var uri = new Uri("http://" + domain + "/robots.txt");
var req = (HttpWebRequest)WebRequest.Create(uri);
req.Method = "GET";
req.KeepAlive = false;
req.UserAgent = "Mozilla/5.0 (compatible; RTSbot/1.0; +http://adamcaudill.com/rtsbot/)";
var resp = (HttpWebResponse)req.GetResponse();
var stream = new StreamReader(resp.GetResponseStream(), Encoding.GetEncoding("ISO-8859-1"));
ret.Body = stream.ReadToEnd();
stream.Close();
var headers = string.Empty;
foreach (string header in resp.Headers)
{
headers += string.Format("{0}: {1}\r\n", header, resp.Headers[header]);
}
ret.Headers = headers;
return(ret);
}
