public void CrawlWorldMarkets() { try { TraceService("Crawling Started:-------------- World Market:--------------------------"); string htmlcontent = Helper.GetWebSiteContent("http://www.oddschecker.com/football/world"); HtmlAgilityPack.HtmlDocument doc = Helper.LoadHtml(htmlcontent); List<BettingMarket> bettinglist = new List<BettingMarket>(); List<Leagues> leaguelist = new List<Leagues>(); HashSet<string> sections = new HashSet<string>(); XmlDocument xmldoc = new XmlDocument(); CrawlFirstPageData crawldata = new CrawlFirstPageData(); string links = ""; var row11 = doc.DocumentNode.SelectSingleNode("//div[@id='mc']"); var rows2 = row11.SelectNodes(".//div[@class='module']"); for (int j = 0; j < rows2.Count; j++) { TraceService("Crawling Started For Section:-------------- World Market:--------------------------"); var heading = rows2[j].SelectNodes(".//h2"); var rows = rows2[j].SelectNodes(".//ul//li[@class='fourth-level-li']"); if (rows != null) { Task[] tasks = new Task[rows.Count]; for (int i = 0; i < rows.Count; i++) { //tasks[i] = Task.Factory.StartNew(() => //{ string section = rows[i].InnerText; var linkrow = rows[i].SelectNodes("./a"); string link = "http://www.oddschecker.com/" + linkrow[0].Attributes["href"].Value; TraceService("Crawling Started For Section:"+section+" URL"+link+"-------------- World Market:--------------------------"); //links+=li+" "+link+" "; if (heading != null) { if (heading[0].InnerText.Equals("World Leagues")) { if (sections.Add(section)) { tasks[i] = Task.Factory.StartNew(() => { CrawlLeagues(section, link, leaguelist); }, TaskCreationOptions.LongRunning); } } else { //if(sections.Add(heading[0].InnerText)) tasks[i] = Task.Factory.StartNew(() => { leaguelist.Add(new Leagues() { League = section, Link = link, Section = heading[0].InnerText }); }); } } else { if (sections.Add(section)) { tasks[i] = Task.Factory.StartNew(() => { CrawlLeagues(section, link, leaguelist); }, TaskCreationOptions.LongRunning); } } } Task.WaitAll(tasks); } //var rows = doc.DocumentNode.SelectNodes("//ul//li[@class='fourth-level-li']"); //if (rows != null) //{ // for (int i = 0; i < rows.Count; i++) // { // string section = rows[i].InnerText; // var linkrow = rows[i].SelectNodes("./a"); // string link = "http://www.oddschecker.com" + linkrow[0].Attributes["href"].Value; // //links+=li+" "+link+" "; // if (sections.Add(section)) // { // CrawlLeagues(section, link, leaguelist); // } // } //} xmldoc = GenerateXmlForLeagues(leaguelist); crawldata.InsertLeagues(xmldoc); TraceService("Data Inserted:-------------- World Market:--------------------------"); } } catch (Exception ex) { TraceService("Error : --------League--------SportID: Footbal , Error:" + ex.ToString() + "\n"); } }
//public string CrawlMyPage(string url,int sportid,long leagueid) //{ // try // { // TraceService("Crawling Started: League ID:" + leagueid + " ,SportID:" + sportid + " , URL:" + url + "\n"); // System.IO.StreamReader rader; // string shtml = Helper.GetWebSiteContent(url); // // shtml = shtml.Replace("<!doctype HTML>", ""); // DataSet ds = new DataSet(); // // HtmlAgilityPack.HtmlDocument doc = Helper.LoadHtml(html); // CrawlFirstPageData crawldata = new CrawlFirstPageData(); // Matches match = new Matches(); // HtmlDocument doc = Helper.LoadHtml(shtml); // TextReader tr = new StringReader(shtml); // //xmldoc.Load(tr); // XmlDocument xmldoc = new XmlDocument(); // List<Matches> matchlist = new List<Matches>(); // var row11 =doc.DocumentNode.SelectSingleNode("//div[@id='fixtures']"); // string title = row11.SelectNodes(".//h2")[0].InnerText; // var rows = row11.SelectNodes("//table//tr"); // if (rows != null) // { // string enddatetime = String.Empty; // for (int ii = 1; ii < rows.Count; ii = ii + 1) // { // var dr = rows[ii].InnerText.Trim(); // var cols = rows[ii].SelectNodes("./td[@class='day']"); // if (cols != null) // { // string t = cols[0].InnerText.Trim(); // match.date = t; // string[] matchdate = t.Split(' '); // enddatetime = matchdate[1].Substring(0, (matchdate[1].Length - 2)) + " " + matchdate[2].Substring(0, 3) + " " + matchdate[3]; // enddatetime = DateTime.Parse(enddatetime).ToString("yyyy-MM-dd"); // } // else // { // var colnew = rows[ii].SelectNodes("./td"); // if (colnew != null && colnew.Count > 4) // { // match.time = colnew[0].InnerText.Trim(); // var Dlink = colnew[4].InnerText.Trim(); // var home = colnew[1].SelectNodes(".//span[@class='fixtures-bet-name']"); // var draw = colnew[2].SelectNodes(".//span[@class='fixtures-bet-name']"); // var away = colnew[3].SelectNodes(".//span[@class='fixtures-bet-name']"); // match.home = home[0].InnerText.Trim(); // match.draw = draw[0].InnerText.Trim(); // match.away = away[0].InnerText.Trim(); // match.createddate = DateTime.Now; // //var link = colnew[4].SelectNodes("./a[contains(@href, '/gaelic-games/gaelic-football/')]"); // var link = colnew[4].SelectNodes("./a"); // match.bettinglink = "http://www.oddschecker.com/" + link[0].Attributes["href"].Value.Replace("/winner", "/betting-markets"); // match.resultlink = "http://www.oddschecker.com/" + link[0].Attributes["href"].Value.Replace("/winner", "/winner"); // match.Displayenddatetime = DateTime.Parse(enddatetime + " " + match.time); // match.league = leagueid.ToString(); // //match.bettinglink = "http://www.oddschecker.com/"+link[0].Attributes["href"].Value; // matchlist.Add(new Matches() { date = match.date, time = match.time, home = match.home, draw = match.draw, away = match.away, bettinglink = match.bettinglink, Displayenddatetime = match.Displayenddatetime,resultlink=match.resultlink }); // crawldata.InsertMatchinfoDev(match,sportid); // } // } // } // xmldoc = GenerateXml(matchlist); // // crawldata.InsertMatchInfo(xmldoc,sportid,leagueid); // TraceService("Data Inserted: League ID:" + leagueid + " ,SportID:" + sportid + " , URL:" + url + "\n"); // } // //ds = crawldata.NewRecords(xmldoc); // //return ds; // return "Command completed successfully"; // } // catch (Exception ex) // { // TraceService("Error:" + leagueid + " ,SportID:" + sportid + " , URL:" + url + "\n"); // return ex.Message; // } //} public void CrawlLeagues(object threadparam) { ThreadParameters p = threadparam as ThreadParameters; string url = p.URL; try { TraceService("Crawling Started: --------Lague--------SportID: Footbal , URL:" + url + "\n"); string html = Helper.GetWebSiteContent(url); DataSet ds = new DataSet(); HtmlAgilityPack.HtmlDocument doc = Helper.LoadHtml(html); CrawlFirstPageData crawldata = new CrawlFirstPageData(); Matches match = new Matches(); XmlDocument xmldoc = new XmlDocument(); List<Leagues> leaguelist = new List<Leagues>(); var list = doc.DocumentNode.SelectSingleNode("//ul[@id='sport-nav']//ul"); int check = 0; var nodes = list.SelectNodes("./li"); if (nodes != null) { for (int i = 0; i < nodes.Count; i++) { //if (check > 0) //{ // var node = nodes[i].SelectNodes("./a"); // if (node != null) // { // string league = node[0].InnerText; // string link = "http://www.oddschecker.com" + node[0].Attributes["href"].Value; // leaguelist.Add(new Leagues() { League = league, Link = link }); // } //} //if (nodes[i].InnerText.Trim().Equals("All Events")) //{ // //i=nodes.GetNodeIndex(nodes[i]); // check++; //} if (nodes[i].InnerText.Trim().Contains("English")) { if (nodes[i].InnerText.Trim().Substring(0, 7).Equals("English")) { var node = nodes[i].SelectNodes(".//a"); if (node != null) { for (int j = 0; j < node.Count; j++) { string league = node[j].InnerText; string link = "http://www.oddschecker.com" + node[j].Attributes["href"].Value; leaguelist.Add(new Leagues() { League = league, Link = link, Section = "English" }); } } } } if (nodes[i].InnerText.Trim().Contains("Scottish")) { if (nodes[i].InnerText.Trim().Substring(0, 8).Equals("Scottish")) { var node = nodes[i].SelectNodes(".//a"); if (node != null) { for (int j = 0; j < node.Count; j++) { string league = node[j].InnerText; string link = "http://www.oddschecker.com" + node[j].Attributes["href"].Value; leaguelist.Add(new Leagues() { League = league, Link = link, Section = "Scottish" }); } } } } if (nodes[i].InnerText.Trim().Contains("European")) { if (nodes[i].InnerText.Trim().Substring(0, 8).Equals("European")) { var node = nodes[i].SelectNodes(".//a"); if (node != null) { for (int j = 0; j < node.Count; j++) { string league = node[j].InnerText; string link = "http://www.oddschecker.com" + node[j].Attributes["href"].Value; leaguelist.Add(new Leagues() { League = league, Link = link, Section = "European" }); } } } } if (nodes[i].InnerText.Trim().Contains("Champions League")) { if (nodes[i].InnerText.Trim().Substring(0, 16).Equals("Champions League")) { var node = nodes[i].SelectNodes(".//a"); if (node != null) { for (int j = 0; j < node.Count; j++) { string league = node[j].InnerText; string link = "http://www.oddschecker.com" + node[j].Attributes["href"].Value; leaguelist.Add(new Leagues() { League = league, Link = link, Section = "Champions League" }); } } } } if (nodes[i].InnerText.Trim().Contains("Europa League")) { if (nodes[i].InnerText.Trim().Substring(0, 13).Equals("Europa League")) { var node = nodes[i].SelectNodes(".//a"); if (node != null) { for (int j = 0; j < node.Count; j++) { string league = node[j].InnerText; string link = "http://www.oddschecker.com" + node[j].Attributes["href"].Value; leaguelist.Add(new Leagues() { League = league, Link = link, Section = "Europa League" }); } } } } } xmldoc = GenerateXmlForLeagues(leaguelist); crawldata.InsertLeagues(xmldoc); TraceService("Data Inserted : --------Lague--------SportID: Footbal , URL:" + url + "\n"); } } catch (Exception ex) { TraceService("Error : --------Lague--------SportID: Footbal , URL:" + url + "\n"); //return ex.Message; } }