public async void Go(PageRegistrationInfo pageRegistrationInfo) // int count, PageService pageService { int count = pageRegistrationInfo.count; int num = pageRegistrationInfo.num; string url = pageRegistrationInfo.url; PageService pageService = pageRegistrationInfo.pageService; while (count < num) // when number of calls of Go method became more than number of depth { try { await Task.Run(() => // initialize and run anonymous task { List <PageEntity> listOfPages = new List <PageEntity>(); List <string> listOfLinks = new List <string>(); string fileinfo = String.Empty; try { listOfLinks = pageService.GetNewBanchOfLinks(count); count++; //} foreach (string linkOfSecondOrder in listOfLinks) { if (linkOfSecondOrder != "") { PageEntity entity = new PageEntity() // creating a PageEntity object to put in the data base { Link = linkOfSecondOrder, IterationId = count // this parameter shows the layer of depth }; pageService.Add(entity); PageRegistrationInfo tempInfo = new PageRegistrationInfo(); tempInfo.num = num; tempInfo.count = count; tempInfo.url = linkOfSecondOrder; tempInfo.pageService = pageService; Console.WriteLine(linkOfSecondOrder + " " + count); Go(tempInfo); // parsing each link from that webpage } else { Logger.Log.Info(DateTime.Now + $"the link from {url} was empty"); } Console.WriteLine(linkOfSecondOrder + " " + count); } ; } catch { Logger.Log.Info(DateTime.Now + $" downloading on {url} was failed"); } }); } catch { Logger.Log.Info(DateTime.Now + $"Failed to run new task for {url}"); } } }
static void Main(string[] args) { Logger.InitLogger(); //logger initialization Logger.Log.Info(DateTime.Now + " The application has started "); var container = new Container(new MultithreadAppRegistry()); var pageService = container.GetInstance <PageService>(); using (var dbContext = container.GetInstance <MultithreadAppDbContext>()) { int linksInDbCount = dbContext.Links.Count(); // just calling dbContext here, for Db initialization PageRegistrationInfo pageInfo = new PageRegistrationInfo(); //dto ParseWebProcess parseWebProcess = new ParseWebProcess(); // main logic methods (int, string)tempTuple = parseWebProcess.Info; pageInfo.num = tempTuple.Item1; pageInfo.url = tempTuple.Item2; pageInfo.pageService = pageService; var count = 0; try { //one thread string fileinfo = pageService.DownLoadPage(pageInfo.url); //downloads a webpage to a file List <string> listOfLinks = pageService.ExtractHtmlTags(fileinfo); // extract all links on the webpage to a list foreach (string link in listOfLinks) { pageInfo.url = link; PageEntity entity = new PageEntity() // creating a PageEntity object to put in the data base { Link = link, IterationId = count // this parameter shows the layer of depth }; pageService.Add(entity); pageInfo.url = link; pageInfo.count = 0; } //multithreading parseWebProcess.Go(pageInfo); //recursive method that parse webpages and put links in the database } catch (Exception e) { Console.WriteLine(e.Message); Logger.Log.Info(DateTime.Now + " process has failed for link pageInfo.url :" + e.Message); } } Console.WriteLine("Wait..."); Console.ReadLine(); Task.WaitAll(); Console.WriteLine("Asyncronous task finished!"); }