public async void Go(PageRegistrationInfo pageRegistrationInfo) // int count, PageService pageService
{
int count = pageRegistrationInfo.count;
int num = pageRegistrationInfo.num;
string url = pageRegistrationInfo.url;
PageService pageService = pageRegistrationInfo.pageService;
while (count < num) // when number of calls of Go method became more than number of depth
{
try
{
await Task.Run(() => // initialize and run anonymous task
{
List <PageEntity> listOfPages = new List <PageEntity>();
List <string> listOfLinks = new List <string>();
string fileinfo = String.Empty;
try
{
listOfLinks = pageService.GetNewBanchOfLinks(count);
count++;
//}
foreach (string linkOfSecondOrder in listOfLinks)
{
if (linkOfSecondOrder != "")
{
PageEntity entity = new PageEntity() // creating a PageEntity object to put in the data base
{
Link = linkOfSecondOrder,
IterationId = count // this parameter shows the layer of depth
};
pageService.Add(entity);
PageRegistrationInfo tempInfo = new PageRegistrationInfo();
tempInfo.num = num;
tempInfo.count = count;
tempInfo.url = linkOfSecondOrder;
tempInfo.pageService = pageService;
Console.WriteLine(linkOfSecondOrder + " " + count);
Go(tempInfo); // parsing each link from that webpage
}
else
{
Logger.Log.Info(DateTime.Now + $"the link from {url} was empty");
}
Console.WriteLine(linkOfSecondOrder + " " + count);
}
;
}
catch
{
Logger.Log.Info(DateTime.Now + $" downloading on {url} was failed");
}
});
}
catch
{
Logger.Log.Info(DateTime.Now + $"Failed to run new task for {url}");
}
}
}
static void Main(string[] args)
{
Logger.InitLogger(); //logger initialization
Logger.Log.Info(DateTime.Now + " The application has started ");
var container = new Container(new MultithreadAppRegistry());
var pageService = container.GetInstance <PageService>();
using (var dbContext = container.GetInstance <MultithreadAppDbContext>())
{
int linksInDbCount = dbContext.Links.Count(); // just calling dbContext here, for Db initialization
PageRegistrationInfo pageInfo = new PageRegistrationInfo(); //dto
ParseWebProcess parseWebProcess = new ParseWebProcess(); // main logic methods
(int, string)tempTuple = parseWebProcess.Info;
pageInfo.num = tempTuple.Item1;
pageInfo.url = tempTuple.Item2;
pageInfo.pageService = pageService;
var count = 0;
try
{ //one thread
string fileinfo = pageService.DownLoadPage(pageInfo.url); //downloads a webpage to a file
List <string> listOfLinks = pageService.ExtractHtmlTags(fileinfo); // extract all links on the webpage to a list
foreach (string link in listOfLinks)
{
pageInfo.url = link;
PageEntity entity = new PageEntity() // creating a PageEntity object to put in the data base
{
Link = link,
IterationId = count // this parameter shows the layer of depth
};
pageService.Add(entity);
pageInfo.url = link;
pageInfo.count = 0;
}
//multithreading
parseWebProcess.Go(pageInfo); //recursive method that parse webpages and put links in the database
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Logger.Log.Info(DateTime.Now + " process has failed for link pageInfo.url :" + e.Message);
}
}
Console.WriteLine("Wait...");
Console.ReadLine();
Task.WaitAll();
Console.WriteLine("Asyncronous task finished!");
}
