public List <string> ParseUrlsFromSitemap(string sitemapUrl)
{
var result = new List <string>();
var request = HttpWebRequest.Create(sitemapUrl) as HttpWebRequest;
var response = request.GetResponse() as HttpWebResponse;
if (response.ContentType.StartsWith("text/xml") || response.ContentType.StartsWith("application/xml"))
{
result = UrlMethods.ReadSitemapXml(sitemapUrl);
}
else if (response.ContentType.StartsWith("text/plain"))
{
result = UrlMethods.ReadSitemapTxt(sitemapUrl);
}
return(result);
}
public string LocateSitemap(string url)
{
string sitemapUrl = url.EndsWith('/') ? url + "sitemap.xml" : url + "/sitemap.xml";
if (UrlMethods.UrlIsValid(sitemapUrl))
{
return(sitemapUrl);
}
string robotsUrl = url.EndsWith('/') ? url + "robots.txt" : url + "/robots.txt";
if (UrlMethods.UrlIsValid(robotsUrl))
{
string resultUrl = UrlMethods.ReadRobots(robotsUrl);
return(resultUrl);
}
return(null);
}
public long MeasureResponseTime(string url)
{
long result = -1;
if (!UrlMethods.UrlIsValid(url))
{
return(result);
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
var timer = new Stopwatch();
timer.Start();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
timer.Stop();
result = timer.ElapsedMilliseconds;
return(result);
}
