//Make a list of all available Links For Results // <table class="table-main" id ="tournamentTable"> // /// <summary> /// To be called from The Calling Class to Start the Full Scraping /// Call This Class and Sit Back and Relax /// ///Exception Handling to be done there as well ///return 0; ===>Error Do that again ///return 1;===>Completed /// </summary> /// <returns></returns> //public int startChainReaction() //{ // writer = new DbWriter(); // if (writeToCountry() == Utils.ERROR_WRITING_TO_COUNTRY_CHAMPIONSHIP) // return Utils.ERROR_WRITING_TO_COUNTRY_CHAMPIONSHIP; // //Now Time to Go deep // Dictionary<string, string> choices = scrapper.LevelTwo(title, championship_id); // if (deepMining(choices) == 0) // return Utils.ERROR_WRITING_TO_DEEP_MINING; // return Utils.SUCCESS_TOTAL; //} //Go deep private int deepMining(Dictionary <string, string> choices) { try { int len = choices.Count; foreach (KeyValuePair <string, string> a in choices) { scrapper.getResults(a.Key, a.Value, championship_id); } } catch (Exception e) { return(0); } return(1); }
public void method() { List <string> links = new List <string>(); Dictionary <string, string> dateDict; UrlScrapping scrapping = new UrlScrapping(); UrlScrapping.LevelOne(); List <Package> listofPackages; LevelTwo two; string champ_id; string country_id; using (StreamReader reader = new StreamReader("urlResults.txt")) { var link = reader.ReadLine(); links.Add(link); } Parallel.ForEach(links, new ParallelOptions { MaxDegreeOfParallelism = 2 }, champ => //foreach(var champ in links) { two = new LevelTwo(champ); country_id = two.writeToCountry(); champ_id = two.getChampionshipId(); dateDict = scrapping.LevelTwo(links[0], champ_id); foreach (var date in dateDict) { listofPackages = scrapping.getResults(date.Key, date.Value, champ_id); if (listofPackages != null) { foreach (var pack in listofPackages) { if (pack != null) { var a = scrapping.sendForProcessingPackage(pack, champ_id); var count = a.Count; Parallel.ForEach(a, new ParallelOptions { MaxDegreeOfParallelism = 2 }, aa => { try { Task t2 = Task.Factory.StartNew(() => runDriver(aa)); Task.WaitAll(t2); } catch (Exception e) { } }); } } } } }); }