public static void AddPendingWebsite(string url)
 {
     if (!PendingWebsites.Keys.Contains(url))
     {
         PendingWebsites.Add(url, true);
     }
 }
 public static void AddPendingWebsites(List <string> urls)
 {
     foreach (string url in urls)
     {
         PendingWebsites.Add(url, true);
     }
 }
        public static void MarkAsVisited(string url)
        {
            visitedCount++;

            if (!VisitedWebsites.Keys.Contains(url))
            {
                VisitedWebsites.Add(url, true);
            }

            if (PendingWebsites.Keys.Contains(url))
            {
                PendingWebsites.Remove(url);
            }
        }