static void CheckNextLink(WebPageClass _currentWebPage) { bool ContinueRunning = true; string _currentLink = String.Empty; lock (_objectLockWebPageInternalLinksList) { _currentLink = WebPagesLinksToCheck[_currentWebPage.WebPageName].FirstOrDefault(); //Get link to check fromweb page WebPagesLinksToCheck[_currentWebPage.WebPageName].Remove(_currentLink); } if (String.IsNullOrEmpty(_currentLink)) { ContinueRunning = false; } while (ContinueRunning) //While web page has unchecked links { //Update web page counters after getting result of links checks WebUtils webUtils = new WebUtils(); switch (webUtils.GetLinkRequestCode(_currentLink)) { case 0: lock (_objectLockWebPageClass) { _currentWebPage.WorkinkLinks++; } break; case 1: lock (_objectLockWebPageClass) { _currentWebPage.BrokenLinks++; } break; default: lock (_objectLockWebPageClass) { _currentWebPage.TimeoutLinks++; } break; } lock (_objectLockWebPagesDictionary) { _currentLink = WebPagesLinksToCheck[_currentWebPage.WebPageName].FirstOrDefault(); //Get link to check fromweb page WebPagesLinksToCheck[_currentWebPage.WebPageName].Remove(_currentLink); } //Stop when web page has its all links checked if (String.IsNullOrEmpty(_currentLink)) { ContinueRunning = false; } } }
static void CheckWebPageLinks() { string _currentWebPage = String.Empty; while (true) //Never stop (for virtualizing IIS socket) { GeneralUtils generalUtils = new GeneralUtils(); if (String.IsNullOrEmpty(_currentWebPage)) { lock (_objectLockWebPagesList) { _currentWebPage = WebPagesToCheck.FirstOrDefault(); //Get web page from IIS queue WebPagesToCheck.Remove(_currentWebPage); } } if ((_currentWebPage != null) && (_currentWebPage != "")) { WebPageClass CurrentWebPage = new WebPageClass(_currentWebPage); Settings settingsValues = new Settings(); SQLRequests SQLReq = new SQLRequests(); CurrentWebPage = SQLReq.GetWebPageCheckResults(CurrentWebPage.WebPageName, settingsValues.TimePeriodForValidCheckResultsInMinutes); if (CurrentWebPage.AllLinks > 0) //Results from last x minutes exist in the DB and will return to the request { Console.WriteLine("Results from DB - " + CurrentWebPage.WebPageName + Environment.NewLine + "Number of links = " + CurrentWebPage.AllLinks + " Working links = " + CurrentWebPage.WorkinkLinks + " Broken links = " + CurrentWebPage.BrokenLinks + " Time out links = " + CurrentWebPage.TimeoutLinks + " Total check time = " + generalUtils.ConvertSecondsToMinutes(CurrentWebPage.TotalCheckTime) + Environment.NewLine); _currentWebPage = ""; } else //No results exist in the DB { bool addToCheckWebList = false; lock (_objectLockWebPagesDictionary) { if ((WebPagesLinksToCheck.ContainsKey(CurrentWebPage.WebPageName)) && (WebPagesLinksToCheck[CurrentWebPage.WebPageName].Count > 0)) //Web page links currently being checked { addToCheckWebList = false; Thread.Sleep(60000); //Sleep 1 minutes to wait for the check results } else //Add a check web page to dictionary { addToCheckWebList = true; if (!WebPagesLinksToCheck.ContainsKey(CurrentWebPage.WebPageName)) { WebPagesLinksToCheck.Add(CurrentWebPage.WebPageName, new List <string>()); } } } if (addToCheckWebList) //Check web page links { DateTime StartTime = new DateTime(); DateTime EndTime = new DateTime(); StartTime = DateTime.Now; //Get all links from http WebUtils webUtils = new WebUtils(); List <string> LinksFromWebPage = webUtils.GetAllLinksFromWebPage(CurrentWebPage.WebPageName); lock (_objectLockWebPagesDictionary) { foreach (string item in LinksFromWebPage) { WebPagesLinksToCheck[CurrentWebPage.WebPageName].Add(item); } CurrentWebPage.AllLinks = WebPagesLinksToCheck[CurrentWebPage.WebPageName].Count; } if (CurrentWebPage.AllLinks > 0) //Address is valid and internet connection works { int numberOfLinksCheckTasks = settingsValues.DeterminNumberOfSockets(CurrentWebPage.AllLinks); Task[] AllTasks = new Task[numberOfLinksCheckTasks]; for (int i = 0; i < numberOfLinksCheckTasks; i++) { AllTasks[i] = Task.Factory.StartNew(() => CheckNextLink(CurrentWebPage)); } //All unchecked links after timeout time will considered as timeout links if (!Task.WaitAll(AllTasks, settingsValues.WebPageCheckTimeoutInMilliseconds)) { int allUncheckLinks = CurrentWebPage.AllLinks - CurrentWebPage.WorkinkLinks - CurrentWebPage.BrokenLinks - CurrentWebPage.TimeoutLinks; lock (_objectLockWebPageClass) { CurrentWebPage.TimeoutLinks = CurrentWebPage.TimeoutLinks + allUncheckLinks; } } EndTime = DateTime.Now; TimeSpan ts = EndTime - StartTime; CurrentWebPage.TotalCheckTime = (int)ts.TotalSeconds; //Add result to the DB and print a message if (SQLReq.AddWebCheckResult(CurrentWebPage.WebPageName, CurrentWebPage.AllLinks, CurrentWebPage.WorkinkLinks, CurrentWebPage.BrokenLinks, CurrentWebPage.TimeoutLinks, CurrentWebPage.TotalCheckTime)) { Console.WriteLine("INSERTED - " + CurrentWebPage.WebPageName + Environment.NewLine + "Number of links = " + CurrentWebPage.AllLinks + " Working links = " + CurrentWebPage.WorkinkLinks + " Broken links = " + CurrentWebPage.BrokenLinks + " Time out links = " + CurrentWebPage.TimeoutLinks + " Total check time = " + generalUtils.ConvertSecondsToMinutes(CurrentWebPage.TotalCheckTime) + Environment.NewLine); } else { Console.WriteLine("NOT INSERTED - " + CurrentWebPage.WebPageName + Environment.NewLine); } _currentWebPage = String.Empty; } else //Address is not valid or no internet connection - set web page as timeout page { EndTime = DateTime.Now; TimeSpan ts = EndTime - StartTime; CurrentWebPage.TotalCheckTime = (int)ts.TotalSeconds; //Add result to the DB (name and all counts are 0) and print a message if (SQLReq.AddWebCheckResult(CurrentWebPage.WebPageName, 0, 0, 0, 0, CurrentWebPage.TotalCheckTime)) { Console.WriteLine("ERROR - Error reading web page links " + Environment.NewLine + "INSERTED - " + CurrentWebPage.WebPageName + Environment.NewLine + "Number of links = " + CurrentWebPage.AllLinks + " Working links = " + CurrentWebPage.WorkinkLinks + " Broken links = " + CurrentWebPage.BrokenLinks + " Time out links = " + CurrentWebPage.TimeoutLinks + " Total check time = " + generalUtils.ConvertSecondsToMinutes(CurrentWebPage.TotalCheckTime) + Environment.NewLine); } else { Console.WriteLine("ERROR - Error reading web page links and DB insertion failed " + Environment.NewLine + "NOT INSERTED - " + CurrentWebPage.WebPageName + Environment.NewLine); } _currentWebPage = String.Empty; } } } } Thread.Sleep(3000); //Sleep 3 seconds for reducing actions while queue is empty } }