private async Task recurrentUrlsExtracting(string address, int iteration, int maxIteration)
{
var pageFile = await pageService.DownloadPageIntoFile(address);
var page = pageService.ReadPageFile(pageFile);
var result = urlParsingService.ExtractAllUrlsFromPage(page, iteration);
if (iteration <= maxIteration)
{
foreach (var url in result)
{
if (!urlsDictionary.ContainsKey(url.URL))
{
if (urlsDictionary.TryAdd(url.URL, iteration))
{
int nextIteration = iteration + 1;
if (nextIteration > maxIteration)
{
continue;
}
lock (locker)
{
tasks.Add(Task.Run(() => recurrentUrlsExtracting(baseAddress + url.URL, nextIteration, maxIteration)));
}
}
}
}
}
}
public void Initialize()
{
List <UrlEntity> urls1 = new List <UrlEntity>()
{
new UrlEntity()
{
URL = "/wiki/Dominic_Serventy"
},
new UrlEntity()
{
URL = "/wiki/World_War_I"
},
new UrlEntity()
{
URL = "/wiki/United_Kingdom"
}
};
List <UrlEntity> urls2 = new List <UrlEntity>()
{
new UrlEntity()
{
URL = "/wiki/George_Pearce"
},
new UrlEntity()
{
URL = "/wiki/Movietone_News"
},
new UrlEntity()
{
URL = "/wiki/World_War_I"
}
};
pageDownloadingService = Substitute.For <IPageService>();
urlParsingService = Substitute.For <IUrlParsingService>();
urlParsingService.ExtractAllUrlsFromPage(Arg.Any <string>(), 1).Returns(urls1);
urlParsingService.ExtractAllUrlsFromPage(Arg.Any <string>(), 2).Returns(urls2);
}