public async Task <HashSet <string> > ParseUrlsAsync()
{
var lockThread = new object();
var doc = await loader.GetSourceAsync(Settings.BaseUrl);
var temp = new HashSet <string>();
temp.UnionWith(Parser.Parse(doc, Settings));
Urls.UnionWith(temp);
int total = 1;
int count = Urls.Count;
//tasks for parallel url parsing
var tasks = new List <Task>();
for (int i = 0; i < total; i++)
{
//creates tasks for new urls to parse
foreach (var tempUrl in temp.ToList())
{
tasks.Add(Task.Run(async() =>
{
doc = await loader.GetSourceAsync(tempUrl);
// only one access at a time
lock (lockThread)
{
if (doc != null)
{
temp.UnionWith(Parser.Parse(doc, Settings));
}
}
}));
}
await Task.WhenAll(tasks);
tasks.Clear();
Urls.UnionWith(temp);
// new urls to parse on next iteration
temp = new HashSet <string>(Urls.Skip(count).Take(Urls.Count - count));
if (temp.Count > 0 && temp != null)
{
total++;
count = Urls.Count;
}
if (Urls.Count > 125)
{
return(Urls);
}
}
return(Urls);
}
