/// <summary>
/// Adds the URLs found on the page to the queue.
/// </summary>
/// <param name="urls"></param>
/// <param name="referrer"></param>
private void AddUrls(string[] urls, string referrer)
{
foreach (string nextTarget in urls)
{
SpiderUrl nextUrl = new SpiderUrl(nextTarget, referrer);
_urls.Enqueue(nextUrl);
}
}
private void AddUrls(string[] urls, string referrer)
{
foreach (string nextTarget in urls)
{
SpiderUrl spiderUrl = new SpiderUrl(nextTarget, referrer);
if (!spiderUrl.IsJavascript && !spiderUrl.IsMailto && !spiderUrl.Target.Contains("#"))
{
_urls.Enqueue(spiderUrl);
}
}
}
private Program()
{
_startTime = DateTime.Now;
string startingUrl = String.Concat(ConfigurationManager.AppSettings["startingUrl"], ConfigurationManager.AppSettings["path"]);
_urls = new UrlCollection();
_urls.Enqueue(new SpiderUrl(startingUrl, startingUrl));
_visitedUrls = new List <string>();
Logger.InitializeLogFile("Id, Start Time, Message, Target, Referrer, Title, Time");
}
private Program()
{
CheckConfiguration();
InitializeMimeTypesToIgnore();
InitializeContentTypesToInclude();
IntializeLinkHrefPatternsToIgnore();
_startTime = DateTime.Now;
string startingUrl = String.Concat(ConfigurationManager.AppSettings["startingUrl"], ConfigurationManager.AppSettings["path"]);
_urls = new UrlCollection();
_urls.Enqueue(new SpiderUrl(startingUrl, startingUrl));
_visitedUrls = new List <string>();
_onlyFollowUniques = bool.Parse(ConfigurationManager.AppSettings["onlyFollowUniques"]);
_website = new Regex(String.Format("^(http|https){{1}}://({0}){{1}}", ConfigurationManager.AppSettings["website"]), RegexOptions.Compiled | RegexOptions.CultureInvariant);
Logger.InitializeLogFile("\"Id\",\"Checked\",\"Start Time\",\"Message\",\"Target\",\"Referrer\",\"Title\",\"Time\",\"Size\",\"Content Type\",\"MIME Type\"");
}