public List<String> Request(String URL) {
Console.WriteLine("Load URL: " + URL);
PageParser pp = new PageParser();
String pageContent = String.Empty;
try {
using (MyClient client = new MyClient()) {
client.HeadOnly = true;
byte[] body = client.DownloadData(URL); // note should be 0-length
string type = client.ResponseHeaders["content-type"];
client.HeadOnly = false;
// check 'tis not binary... we'll use text/, but could
// check for text/html
if (type.StartsWith(@"text/")) {
pageContent = client.DownloadString(URL);
}
}
} catch (Exception) { }
if (pageContent != String.Empty) {
return pp.Parse(pageContent);
} else {
return new List<String>();
}
}