public async Task <HashSet <string> > ParseUrlsAsync() { var lockThread = new object(); var doc = await loader.GetSourceAsync(Settings.BaseUrl); var temp = new HashSet <string>(); temp.UnionWith(Parser.Parse(doc, Settings)); Urls.UnionWith(temp); int total = 1; int count = Urls.Count; //tasks for parallel url parsing var tasks = new List <Task>(); for (int i = 0; i < total; i++) { //creates tasks for new urls to parse foreach (var tempUrl in temp.ToList()) { tasks.Add(Task.Run(async() => { doc = await loader.GetSourceAsync(tempUrl); // only one access at a time lock (lockThread) { if (doc != null) { temp.UnionWith(Parser.Parse(doc, Settings)); } } })); } await Task.WhenAll(tasks); tasks.Clear(); Urls.UnionWith(temp); // new urls to parse on next iteration temp = new HashSet <string>(Urls.Skip(count).Take(Urls.Count - count)); if (temp.Count > 0 && temp != null) { total++; count = Urls.Count; } if (Urls.Count > 125) { return(Urls); } } return(Urls); }