static void CrawlLinks()
        {
            //takes the next page from the stack and extracts all of the links, adds non visted & allowed pages to the stack
            Link page = toVist.Pop(); //get the next page to view

            //page may have been viewed already (duplicate links on the same page), if not crawl if allowed
            if(!LinkVisted(page) && CanVisit(page))
            {
                Console.WriteLine(page.linkString);
                visited.Add(page);
                try
                {
                    String html = new WebClient().DownloadString(page.linkString); //download the HTML
                    MatchCollection regexLinks = Regex.Matches(html, @"((<a.*?>.*?</a>)|(<A.*?>.*?</A))", RegexOptions.Singleline); //get the ahrefs

                    //put all the unvisited links onto the stack
                    for(int i = 0; i < regexLinks.Count; i++)
                    {
                        Link link = new Link(ReturnPageLink(regexLinks[i].Value), page.path, page.linkString);
                        if (link.linkString != null) { toVist.Push(link); }
                    }
                }
                catch(WebException ex)
                {
                    //maybe not found or server error, no need to fail if so
                }

            }//if the link hasn't already been visited
        }
        static List<Link> visited = new List<Link>(); //the links that have been visited

        #endregion Fields

        #region Methods

        static Boolean CanVisit(Link link)
        {
            //takes a link and determines whether we can visit it, based on the same domain rule and by obeying robots.txt

            if(link.linkString.Contains(seed) && !isDisallowed(link)) //check whether this link is from the correct domain
            {
                List<String> robots = new List<string>();

                //get a list of all the disallowed links from robots.txt, if it hasn't been visited
                if (!RobotsChecked(link.robots))
                {
                    try
                    {
                        robots = Regex.Split(new WebClient().DownloadString(link.robots), "\r\n").ToList();
                        robotsDownloaded.Add(link.robots);
                    }
                    catch(System.Net.WebException ex) //may not be found
                    {
                        return true; //if no robots.txt then assume we can visit
                    }

                    disallowed.AddRange((from line in robots
                                      where line.Contains("Disallow")
                                      select link.path + @"/" + line.Replace(@"Disallow:","").Replace(" ","").Substring(1)).ToList());
                }

                //check whether this passed in link is in the disallowed list
                var exists = (from lnk in disallowed
                              where link.linkString.Contains(lnk)
                              select lnk).Any();

                return !exists;
            }
            else
            {
                return false; //a different domain or is disallowed
            }
        }
 static Boolean isDisallowed(Link link)
 {
     return (from lnk in disallowed
             where lnk == link.path
             select lnk).Any();
 }
 static Boolean LinkVisted(Link link)
 {
     return (from lnk in visited
                   where link.linkString == lnk.linkString
                   select lnk).Any();
 }