public string CrawlBettingLinks() { TraceService("Crawl BettingLinks Started: "); try { CrawlFirstPageData crawldata = new CrawlFirstPageData(); DataTable dt = crawldata.GetMatchInfo(); //Task[] tasks = new Task[dt.Rows.Count]; // for (int i = 0; i < dt.Rows.Count; i++) // { // string url = dt.Rows[i]["BettingLink"].ToString(); // int id = Convert.ToInt32(dt.Rows[i]["id"].ToString()); // tasks[i] = Task.Factory.StartNew(() => // { // CrawlBettingMarkets(url, id); // },TaskCreationOptions.LongRunning); // } // Task.WaitAll(tasks); for (int i = 0; i < dt.Rows.Count; i++) { string url = dt.Rows[i]["BettingLink"].ToString(); int id = Convert.ToInt32(dt.Rows[i]["id"].ToString()); ThreadParameters tp = new ThreadParameters(); tp.URL = url; tp.MatchID = id; ThreadPool.QueueUserWorkItem(new WaitCallback(CrawlBettingMarkets), tp); } return("Command completed successfully"); } catch (Exception ex) { ErrorLog("CrawlBettingMarket ---- Error:" + ex.ToString()); return(ex.Message); } }
public string CrawlBettingLinksByCoupon(bool archive) { TraceService("Crawl BettingLinks Started: "); try { CrawlFirstPageData crawldata = new CrawlFirstPageData(); DataTable dt = crawldata.GetMatchByCoupon(archive); // Task[] tasks = new Task[dt.Rows.Count]; for (int i = 0; i < dt.Rows.Count; i++) { string url = dt.Rows[i]["BettingLink"].ToString(); int id = Convert.ToInt32(dt.Rows[i]["id"].ToString()); ThreadParameters tp = new ThreadParameters(); tp.URL = url; tp.MatchID = id; ThreadPool.QueueUserWorkItem(new WaitCallback(CrawlBettingMarkets), tp); } // Task.WaitAll(tasks); return("Command completed successfully"); } catch (Exception ex) { ErrorLog("CrawlBettingMarket ---- Error:" + ex.ToString()); return(ex.Message); } }
public string CrawlUrls(string link) { try { CrawlFirstPageData crawldata = new CrawlFirstPageData(); UrlModel tp = new UrlModel { FullUrl = link, FullUrl2 = link }; CrawlAllLinks(tp); //ThreadPool.QueueUserWorkItem(new WaitCallback(CrawlAllLinks), tp); return("Command completed successfully"); } catch (Exception ex) { return(ex.Message); } }
public static string GetMatchName(long matchid) { CrawlFirstPageData crawl = new CrawlFirstPageData(); return(crawl.GetMatchName(matchid)); }
public string CrawlGolfTurnament(string url, string sportid) { try { TraceService("Crawling Started: Golf:0 ,SportID:" + sportid + " , URL:" + url + "\n"); string html = Helper.GetWebSiteContent(url); DataSet ds = new DataSet(); HtmlAgilityPack.HtmlDocument doc = Helper.LoadHtml(html); CrawlFirstPageData crawldata = new CrawlFirstPageData(); Matches match = new Matches(); XmlDocument xmldoc = new XmlDocument(); List <GolfTurnament> turnamentlist = new List <GolfTurnament>(); var row11 = doc.DocumentNode.SelectSingleNode("//div[@class='containerHeight']"); var row = row11.SelectNodes(".//a[@class='stats_link']"); var rows = row11.SelectNodes("//div[@class='stats']"); if (rows != null) { string enddatetime = String.Empty; for (int ii = 0; ii < rows.Count; ii = ii + 0) { GolfTurnament golf = new GolfTurnament(); int i = ii; try { var n = rows[ii].ParentNode; if (n.Name == "a") { if (n == null) { golf.Link = ""; } else { golf.Link = "http://wikiform.com.au/oddschecker/" + n.Attributes["href"].Value; } } else { golf.Link = ""; } if (rows[ii].Attributes["style"].Value == "clear:both;width:100px;") { golf.Duration = rows[ii].InnerText.Trim(); ii += 1; } else { golf.Duration = ""; } if (rows[ii].Attributes["style"].Value == "width:317px;text-decoration:underline;" || rows[ii].Attributes["style"].Value == "width:317px;") { golf.Turnament = rows[ii].InnerText.Trim(); ii += 1; } else { golf.Turnament = ""; } if (rows[ii].Attributes["style"].Value == "width:242px;") { golf.Course = rows[ii].InnerText.Trim(); ii += 1; } else { golf.Course = ""; } if (rows[ii].Attributes["style"].Value == "width:122px;border-right:0;") { golf.Champion = rows[ii].InnerText.Trim(); ii += 1; } else { golf.Champion = ""; } InsertGoldTurnament(golf); } catch (Exception ex) { } if (i == ii) { ii += 1; } turnamentlist.Add(golf); } xmldoc = GenerateXmlGolfTurnament(turnamentlist); // InsertGoldTurnament(xmldoc, sportid); TraceService("Data Inserted: Golf:0 ,SportID:" + sportid + " , URL:" + url + "\n"); } //ds = crawldata.NewRecords(xmldoc); //return ds; return("Command completed successfully"); } catch (Exception ex) { TraceService("Error:0 ,SportID:" + sportid + " , URL:" + url + "\n"); return(ex.Message); } }