public String ScrapeUrl() { string[] lines = File.ReadAllLines("urlResults.txt"); list = new List <string>(lines); Parallel.ForEach(list, new ParallelOptions { MaxDegreeOfParallelism = 4 }, l => { LevelTwo sc = new LevelTwo(l); }); return(""); }
public void method() { List <string> links = new List <string>(); Dictionary <string, string> dateDict; UrlScrapping scrapping = new UrlScrapping(); UrlScrapping.LevelOne(); List <Package> listofPackages; LevelTwo two; string champ_id; string country_id; using (StreamReader reader = new StreamReader("urlResults.txt")) { var link = reader.ReadLine(); links.Add(link); } Parallel.ForEach(links, new ParallelOptions { MaxDegreeOfParallelism = 2 }, champ => //foreach(var champ in links) { two = new LevelTwo(champ); country_id = two.writeToCountry(); champ_id = two.getChampionshipId(); dateDict = scrapping.LevelTwo(links[0], champ_id); foreach (var date in dateDict) { listofPackages = scrapping.getResults(date.Key, date.Value, champ_id); if (listofPackages != null) { foreach (var pack in listofPackages) { if (pack != null) { var a = scrapping.sendForProcessingPackage(pack, champ_id); var count = a.Count; Parallel.ForEach(a, new ParallelOptions { MaxDegreeOfParallelism = 2 }, aa => { try { Task t2 = Task.Factory.StartNew(() => runDriver(aa)); Task.WaitAll(t2); } catch (Exception e) { } }); } } } } }); }