public static void ScrapePedigree() { rand = new Random(); db_ppdb = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); //var cmd = "UPDATE PQ_Horse SET PQOutcome = 'Rework' WHERE SireId IS NULL OR DamId IS NULL"; //db_ppdb.ExecuteCommand(cmd); foreach ( PQ_Horse scrape in db_ppdb.PQ_Horses.Where(x => x.MergeMatchBasis == null && (x.SireId == null || x.DamId == null)) .OrderBy(x => x.Name)) { Logger.WriteLog("Scrape PQ for " + scrape.Name + " " + scrape.Id); string outcome = ""; string pqid = ""; string status = ""; // get the page string page = ""; GetHorse(scrape.Id, ref status, ref page); if (status == "Complete") { if (page.Contains("can't be found in the database")) { outcome = "not found"; Logger.WriteLog(scrape.Name + " not found\r\n"); } else { ScrapePage(db_ppdb, page, ref outcome, ref pqid); db_ppdb.SubmitChanges(); } } else { outcome = status; Logger.WriteLog(scrape.Name + " " + status + "\r\n"); } scrape.PQOutcome = "Retrieved " + outcome; db_ppdb.SubmitChanges(); } }
public static void ScrapeHorses() { Common.rand = new Random(); db_rph = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); int count = db_rph.Horses.Count(x => x.DetailProcessed == 0 && x.PriorityProcess); while (db_rph.Horses.Any(x => x.DetailProcessed == 0 && x.PriorityProcess)) { foreach (Horse scrape in db_rph.Horses.Where(x => x.DetailProcessed == 0 && x.PriorityProcess).Take(500)) { Logger.WriteLog((count--).ToString() + " - " + scrape.Name); string status = ""; string page = ""; GetHorse((int)scrape.RPId, ref status, ref page); scrape.DetailRaw = "Error"; scrape.DetailProcessed = -1; if (status == "Complete") { var regex_name = new Regex(@"<h1>\s+([^(<]+)(?:\(([A-Z]+)\) ){0,1}</h1>"); Match match_name = regex_name.Match(page); if (match_name.Success) { string country = match_name.Groups[2].ToString(); if (country == "") { country = "GB"; } scrape.Country = country; } var regex_header = new Regex("<ul id=\"detailedInfo\">(.*?)</ul>", RegexOptions.Singleline); Match match_header = regex_header.Match(page); if (match_header.Success) { scrape.DetailRaw = match_header.Groups[1].ToString(); ProcessDetail(scrape); scrape.DetailProcessed = 1; } else { Logger.WriteLog("Detail not found: " + scrape.Name); } } else { Logger.WriteLog("Horse retrieval error " + status + " - " + scrape.Name); } db_rph.SubmitChanges(); Common.Wait(); } db_rph = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); } }
public static void ScrapeUnmatched() { rand = new Random(); db_ppdb = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); while (db_ppdb.Horses.Any(x => x.PPMatchBasis == "Failed RP_PP" && x.PriorityProcess)) { foreach (Horse scrape in db_ppdb.Horses.Where(x => x.PPMatchBasis == "Failed RP_PP" && x.PriorityProcess).Take(500)) { Logger.WriteLog("Scrape PQ for " + scrape.Name + " " + scrape.Id); scrape.PPMatchBasis = "Pending RP_PQ"; string outcome = ""; string pqid = ""; string status = ""; string page = ""; SearchHorse(scrape.Name, ref status, ref page); if (status == "Complete" && page.Contains("can't be found in the database")) { var regex = new Regex("(.*) [ivxIVX]+$"); Match match = regex.Match(scrape.Name); if (match.Success) { status = ""; SearchHorse(match.Groups[1].ToString(), ref status, ref page); } } if (status == "Complete") { if (page.Contains("can't be found in the database")) { outcome = "not found"; Logger.WriteLog(scrape.Name + " not found\r\n"); } else if (page.Contains("more than \none horse named ")) { List <string> pqids = ProcessMultiple(scrape, page); foreach (string multi_pqid in pqids) { if (multi_pqid == "skiing" || multi_pqid == "generator") { continue; } GetHorse(multi_pqid, ref status, ref page); if (status == "Complete") { if (page.Contains("can't be found in the database")) { Logger.WriteLog(scrape.Name + " multiple not found\r\n"); } else { ScrapePage(db_ppdb, page, ref outcome, ref pqid); db_ppdb.SubmitChanges(); } } else { Logger.WriteLog(multi_pqid + " Already retrieved\r\n"); } } } else { ScrapePage(db_ppdb, page, ref outcome, ref pqid); db_ppdb.SubmitChanges(); } Logger.WriteLog(" "); } scrape.PQOutcome = "Retrieved " + outcome; db_ppdb.SubmitChanges(); } db_ppdb = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); } }
private static int?Merge_RP_PQ_Merge(PQ_Horse pq_horse, int gen, int rh_id) { int?merge_sire_id = null; int?merge_dam_id = null; PQ_Horse pq_sire = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.SireId).FirstOrDefault(); if (pq_sire == null) { return(null); } else { if (pq_sire.MergeId != null) { merge_sire_id = pq_sire.MergeId; } else { merge_sire_id = Merge_RP_PQ_Merge(pq_sire, gen + 1, 0); if (merge_sire_id == null) { return(null); } } } PQ_Horse pq_dam = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.DamId).FirstOrDefault(); if (pq_dam == null) { return(null); } else { if (pq_dam.MergeId != null) { merge_dam_id = pq_dam.MergeId; } else { merge_dam_id = Merge_RP_PQ_Merge(pq_dam, gen + 1, 0); if (merge_dam_id == null) { return(null); } } } var merge_horse = new Horse_Merged(); db_racing_update.Horse_Mergeds.InsertOnSubmit(merge_horse); if (rh_id != 0) { merge_horse.RHId = rh_id; } CultureInfo cult_info = Thread.CurrentThread.CurrentCulture; TextInfo text_info = cult_info.TextInfo; merge_horse.Name = text_info.ToTitleCase(pq_horse.Name.ToLower()); merge_horse.Country = pq_horse.Country; if (pq_horse.FoalYear != null) { merge_horse.FoalDate = new DateTime((int)pq_horse.FoalYear, 1, 1); merge_horse.FoalYear = pq_horse.FoalYear; } merge_horse.Colour = pq_horse.Colour; merge_horse.Sex = pq_horse.Sex; merge_horse.SireId = merge_sire_id; merge_horse.DamId = merge_dam_id; merge_horse.Haplo = db_racing_read.Horse_Mergeds.Where(x => x.Id == merge_dam_id).Select(x => x.Haplo).FirstOrDefault(); merge_horse.MergeBasis = "RP-PQ Added"; db_racing_update.SubmitChanges(); pq_horse.MergeId = merge_horse.Id; pq_horse.MergeMatchBasis = "Added"; return(merge_horse.Id); }
public static void Merge_RP_PQ_PP() { db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); db_racing_read = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); while (db_racing_update.Horses.Where(x => x.PPMatchBasis == "Pending RP_PQ").Any()) { foreach ( Horse rp_horse in db_racing_update.Horses.Where(x => x.PPMatchBasis == "Pending RP_PQ") .OrderBy(x => x.Id) .ThenBy(x => x.Name) .Take(500)) { rp_horse.PPId = null; rp_horse.PPMatchBasis = "Failed RP_PQ_PP"; Logger.WriteLog("Merging RP/PQ to PP " + rp_horse.Name + " " + rp_horse.Id); PQ_Horse pq_horse = db_racing_update.PQ_Horses.Where(x => x.Id == rp_horse.PQId).FirstOrDefault(); if (pq_horse != null) { if (pq_horse.MergeId != null) { Horse_Merged merge_horse = db_racing_update.Horse_Mergeds.Where(x => x.PPId == pq_horse.MergeId).FirstOrDefault(); if (merge_horse != null) { merge_horse.RHId = rp_horse.Id; merge_horse.Name = rp_horse.Name; if (merge_horse.Country == null) { merge_horse.Country = rp_horse.Country; } if (merge_horse.FoalDate == null) { merge_horse.FoalDate = rp_horse.FoalDate; } if (merge_horse.FoalYear == null) { merge_horse.FoalYear = rp_horse.FoalYear; } merge_horse.MergeBasis = "RP-PQ Matched"; } rp_horse.PPId = pq_horse.MergeId; rp_horse.PPMatchBasis = "PQ Direct"; } else if (Merge_RP_PQ_Merge(pq_horse, 1, rp_horse.Id) != null) { rp_horse.PPId = pq_horse.MergeId; rp_horse.PPMatchBasis = "PQ Added"; } } db_racing_update.SubmitChanges(); } db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); } }
public static void Match_PQ_PP() { db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); db_racing_read = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); pp_horses = db_racing_read.PP_Horse_Selects.ToDictionary(x => x.Id, x => new PPHorse { HorseName = x.HorseName, Id = x.Id, SireId = x.SireId, SireName = x.SireName, DamName = x.DamName, Country = x.Country, FoalYear = x.FoalYear, OrigName = x.OrigName }); pp_horse_lookup = pp_horses.Select(x => x.Value).ToLookup(x => x.HorseName, x => x.Id); pp_sire_lookup = pp_horses.Where(x => x.Value.SireId != null) .Select(x => x.Value) .ToLookup(x => (int)x.SireId, x => x.Id); while (db_racing_update.PQ_Horses.Where(x => x.MergeMatchBasis == null).Any()) { foreach (PQ_Horse pq_horse in db_racing_update.PQ_Horses.Where(x => x.MergeMatchBasis == null).Take(500) ) { pq_horse.MergeMatchBasis = "Failed"; Logger.WriteLog("Matching PQ to PP " + pq_horse.Name + " " + pq_horse.Id); string horse_name = FlattenName(pq_horse.Name); string sire_name = null; string dam_name = null; int? year = null; if (pq_horse.FoalYear != null) { year = pq_horse.FoalYear; } string country = pq_horse.Country == null ? "" : pq_horse.Country; PQ_Horse pq_sire = null; if (pq_horse.SireId != null) { pq_sire = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.SireId).FirstOrDefault(); if (pq_sire != null) { sire_name = FlattenName(pq_sire.Name); } } PQ_Horse pq_dam = null; if (pq_horse.DamId != null) { pq_dam = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.DamId).FirstOrDefault(); if (pq_dam != null) { dam_name = FlattenName(pq_dam.Name); } } int? ppid = null; string match_basis = null; if (MatchToPPByName(horse_name, sire_name, dam_name, year, country, ref ppid, ref match_basis)) { pq_horse.MergeId = (int)ppid; pq_horse.MergeMatchBasis = match_basis; } if (ppid == null && pq_sire != null) { int?sire_merge_id = null; if (pq_sire.MergeId != null) { sire_merge_id = pq_sire.MergeId; } match_basis = "Sire"; if (MatchToPPBySire(sire_merge_id, horse_name, sire_name, dam_name, year, country, ref ppid, ref match_basis)) { pq_horse.MergeId = (int)ppid; pq_horse.MergeMatchBasis = match_basis; } } } db_racing_update.SubmitChanges(); } db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); }
public static void Match_RP_PQ() { db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); db_racing_read = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); pq_horses = db_racing_read.PQ_Horse_Selects.ToDictionary(x => x.Id, x => new PQHorse { HorseName = x.FlatName, Id = x.Id, SireId = x.SireId, SireName = x.SireName, DamName = x.DamName, Country = x.Country, FoalYear = x.FoalYear }); pq_horse_lookup = pq_horses.Select(x => x.Value).ToLookup(x => x.HorseName, x => x.Id); pq_sire_lookup = pq_horses.Where(x => x.Value.SireId != null) .Select(x => x.Value) .ToLookup(x => x.SireId, x => x.Id); while (db_racing_update.Horses.Where(x => x.PQMatchBasis == null).Any()) { foreach (Horse rp_horse in db_racing_update.Horses.Where(x => x.PQMatchBasis == null).Take(500)) { rp_horse.PQMatchBasis = "Failed"; Logger.WriteLog("Match RP to PQ " + rp_horse.Name + " " + rp_horse.Id); string horse_name = FlattenName(rp_horse.Name); string sire_name = null; string dam_name = null; int? year = null; if (rp_horse.FoalYear != null) { year = rp_horse.FoalYear; } string country = rp_horse.Country == null ? "" : rp_horse.Country; Horse rp_sire = null; if (rp_horse.SireId != null) { rp_sire = db_racing_update.Horses.Where(x => x.Id == rp_horse.SireId).FirstOrDefault(); if (rp_sire != null) { sire_name = FlattenName(rp_sire.Name); } } Horse rp_dam = null; if (rp_horse.DamId != null) { rp_dam = db_racing_update.Horses.Where(x => x.Id == rp_horse.DamId).FirstOrDefault(); if (rp_dam != null) { dam_name = FlattenName(rp_dam.Name); } } string pqid = null; string match_basis = null; if (MatchToPQByName(horse_name, sire_name, dam_name, year, country, ref pqid, ref match_basis)) { rp_horse.PQId = pqid; rp_horse.PQMatchBasis = match_basis; } if (pqid == null && rp_sire != null) { string sire_pqid = null; if (rp_sire.PQId != null) { sire_pqid = rp_sire.PQId; } match_basis = "Sire"; if (MatchToPQBySire(sire_pqid, horse_name, sire_name, dam_name, year, country, ref pqid, ref match_basis)) { rp_horse.PQId = pqid; rp_horse.PQMatchBasis = match_basis; } } // this option for dams for which we have minimal info if (pqid == null && rp_sire != null) { int offspring_year_min = 9999; int offspring_year_max = 0; foreach (Horse offspring in db_racing_read.Horses.Where(x => x.DamId == rp_horse.Id)) { if (offspring.FoalYear != null) { if (offspring.FoalYear < offspring_year_min) { offspring_year_min = (int)offspring.FoalYear; } if (offspring.FoalYear > offspring_year_max) { offspring_year_max = (int)offspring.FoalYear; } } } if (offspring_year_min != 9999) { if (MatchToPQDamSpecial(horse_name, sire_name, offspring_year_min, offspring_year_max, ref pqid, ref match_basis)) { rp_horse.PQId = pqid; rp_horse.PQMatchBasis = match_basis; } } } db_racing_update.SubmitChanges(); } db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); } }
public static void Add_RP_Merge() { db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); db_racing_read = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); db_racing_read.CommandTimeout = 180; while (db_racing_read.RP_Match_By_Parents.Any()) { List <RP_Match_By_Parent> view_recs = db_racing_read.RP_Match_By_Parents.Take(500).ToList(); foreach (RP_Match_By_Parent view_rec in view_recs) { Logger.WriteLog("Attempting RP Merge for " + view_rec.Name + " " + view_rec.Id); Horse rp_horse = db_racing_update.Horses.Where(x => x.Id == view_rec.Id).FirstOrDefault(); if (rp_horse == null) { continue; } rp_horse.PPMatchBasis = "Failed"; if (rp_horse.FoalYear == null || view_rec.SireFoalYear == null || view_rec.DamFoalYear == null) { db_racing_update.SubmitChanges(); continue; } int sire_diff = (int)rp_horse.FoalYear - (int)view_rec.SireFoalYear; if (sire_diff < 3 || sire_diff > 25) { db_racing_update.SubmitChanges(); continue; } int dam_diff = (int)rp_horse.FoalYear - (int)view_rec.DamFoalYear; if (dam_diff < 3 || dam_diff > 25) { db_racing_update.SubmitChanges(); continue; } var merge_horse = new Horse_Merged(); db_racing_update.Horse_Mergeds.InsertOnSubmit(merge_horse); merge_horse.RHId = rp_horse.Id; merge_horse.Name = rp_horse.Name; merge_horse.Country = rp_horse.Country; merge_horse.FoalDate = rp_horse.FoalDate; merge_horse.FoalYear = rp_horse.FoalYear; merge_horse.Colour = rp_horse.Colour; if (rp_horse.Sex != null) { switch (rp_horse.Sex) { case "f": case "m": merge_horse.Sex = "f"; break; case "c": case "h": merge_horse.Sex = "c"; break; case "g": merge_horse.Sex = "g"; break; case "r": merge_horse.Sex = "r"; break; } } merge_horse.SireId = view_rec.SireId; merge_horse.DamId = view_rec.DamId; merge_horse.Haplo = db_racing_read.Horse_Mergeds.Where(x => x.Id == merge_horse.DamId) .Select(x => x.Haplo) .FirstOrDefault(); merge_horse.MergeBasis = "RP Added"; db_racing_update.SubmitChanges(); rp_horse.PPId = merge_horse.Id; rp_horse.PPMatchBasis = "RP Added"; db_racing_update.SubmitChanges(); } db_racing_update = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); } string cmd = "UPDATE Horse SET PPMatchBasis = 'Failed' WHERE PPMatchBasis = 'Failed RP_PQ_PP'"; db_racing_update.ExecuteCommand(cmd); }
private static void ScrapeDatePage(string page, DateTime date) { string meeting_head = ""; string meeting_tail = ""; string races = ""; string re1 = "(<)"; // Any Single Character 1 string re2 = "(div)"; // Word 1 string re3 = "( )"; // Any Single Character 2 string re4 = "(class)"; // Word 2 string re5 = "(=)"; // Any Single Character 3 string re6 = "(\"rp-resultsWrapper__content\")"; // Double Quote String 1 string re7 = ".*?"; // Non-greedy match on filler string re8 = "()"; // Tag 1 string re9 = ".*?"; // Non-greedy match on filler string re10 = "(<\\/h1>)"; // Tag 2 string re11 = "(<\\/div>)"; // Tag 3 Regex r = new Regex(re1 + re2 + re3 + re4 + re5 + re6 + re7 + re8 + re9 + re10 + re11, RegexOptions.IgnoreCase | RegexOptions.Singleline); //var regex_meeting = // new Regex(@"(<)(div)( )(class)(=)(")(rp)(-)(resultsWrapper__content)(")(>)(<[^>]+>).*?<.*?<.*?(<)(\/)(h1)(>)(<)(\/div)(>)", // RegexOptions.Singleline); Match match_meeting = r.Match(page); while (match_meeting.Success) { meeting_head = match_meeting.Groups[1].ToString(); meeting_tail = match_meeting.Groups[2].ToString(); races = match_meeting.Groups[3].ToString(); var regex_course = new Regex( @"<a href=""http://www.racingpost.com/horses/course_home.sd\?crs_id=(\d+)[^>]+>([^<]+)</a>"); Match match_course = regex_course.Match(meeting_head); if (match_course.Success) { int course_id = Convert.ToInt32(match_course.Groups[1].ToString()); string course_whole = match_course.Groups[2].ToString().Replace("(AW)", ":AW:"); string course = ""; string country = ""; var regex_course2 = new Regex(@"([^(]+)(\([^(]+\)){0,1}"); Match match_course2 = regex_course2.Match(course_whole); if (match_course2.Success) { course = match_course2.Groups[1].ToString().Replace(":AW:", "(AW)").Trim(); country = match_course2.Groups[2].ToString().Replace("(", "").Replace(")", "").Trim(); } string going = ""; var regex_going = new Regex(@"<strong>GOING:</strong>([^<\n]+)", RegexOptions.Singleline); Match match_going = regex_going.Match(meeting_tail); if (match_going.Success) { going = match_going.Groups[1].ToString().Trim(); } string weather = ""; var regex_weather = new Regex(@"<strong>Weather conditions:</strong>([^<\n]+)", RegexOptions.Singleline); Match match_weather = regex_weather.Match(meeting_tail); if (match_weather.Success) { weather = match_weather.Groups[1].ToString().Trim(); } string stalls = ""; var regex_stalls = new Regex(@"<strong>STALLS:</strong>([^<\n]+)", RegexOptions.Singleline); Match match_stalls = regex_stalls.Match(meeting_tail); if (match_stalls.Success) { stalls = match_stalls.Groups[1].ToString().Trim(); } Course course_rec = db_rph.Courses.Where(x => x.Id == course_id).FirstOrDefault(); if (course_rec == null) { course_rec = new Course(); course_rec.Id = course_id; db_rph.Courses.InsertOnSubmit(course_rec); } course_rec.Name = course; course_rec.Country = country; db_rph.SubmitChanges(); Meeting meeting_rec = db_rph.Meetings.Where(x => x.CourseId == course_id && x.DateOfMeeting == date).FirstOrDefault(); if (meeting_rec != null) { string cmd = String.Format("DELETE FROM Meeting WHERE Id = {0}", meeting_rec.Id); db_rph.ExecuteCommand(cmd); cmd = String.Format("DELETE FROM ScrapeRace WHERE MeetingId = {0}", meeting_rec.Id); db_rph.ExecuteCommand(cmd); } meeting_rec = new Meeting(); db_rph.Meetings.InsertOnSubmit(meeting_rec); meeting_rec.CourseId = course_id; meeting_rec.DateOfMeeting = date; meeting_rec.Going = going; meeting_rec.Weather = weather; meeting_rec.Stalls = stalls; db_rph.SubmitChanges(); var regex_race = new Regex(@"<td(.*?)</td>", RegexOptions.Singleline); Match match_race = regex_race.Match(races); while (match_race.Success) { string race = match_race.Groups[1].ToString(); string race_link = ""; int race_id = 0; var regex_race_link = new Regex(@"<a href=""(/horses/result_home\.sd\?race_id=(\d+)[^""]+)""", RegexOptions.Singleline); Match match_race_link = regex_race_link.Match(race); if (match_race_link.Success) { race_link = match_race_link.Groups[1].ToString().Trim(); race_id = Convert.ToInt32(match_race_link.Groups[2].ToString().Trim()); var race_rec = new ScrapeRace(); db_rph.ScrapeRaces.InsertOnSubmit(race_rec); race_rec.MeetingId = meeting_rec.Id; race_rec.Link = "http://www.racingpost.com" + race_link; race_rec.Scraped = false; race_rec.RaceId = race_id; race_rec.RaceDate = date; db_rph.SubmitChanges(); } match_race = match_race.NextMatch(); } } match_meeting = match_meeting.NextMatch(); } }
public static void ScrapeDates(DateTime to_date) { try { Common.rand = new Random(); db_rph = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); //todo change LastDateScraped on 240 server //todo new scraperace structure on 240 server //todo amend ScrapeRaceView DateTime date_scrape = db_rph.ScrapeCourses.Select(x => x.LastDateScraped).FirstOrDefault(); if (date_scrape == null) { Logger.WriteLog("Last date scraped missing"); Environment.Exit(-1); } else { date_scrape = date_scrape.AddDays(1); } while (date_scrape <= to_date) { Logger.WriteLog("Scraping " + date_scrape.ToShortDateString()); string status = ""; string page = ""; GetDate(date_scrape, ref status, ref page); if (status == "Complete") { ScrapeDatePage(page, date_scrape); RaceScraper.ScrapeRaces(); HorseScraper.ScrapeHorses(); //Matcher.Match_RP_PP(); //PQScraper.ScrapeUnmatched(); //PQScraper.ScrapePedigree(); //Matcher.Match_RP_PQ(); //Matcher.Match_PQ_PP(); //Matcher.Merge_RP_PQ_PP(); //Matcher.Add_RP_Merge(); } else { Logger.WriteLog("Page retrieval failed: " + status); Environment.Exit(-1); } ScrapeCourse scr_rec = db_rph.ScrapeCourses.FirstOrDefault(); if (scr_rec == null) { scr_rec = new ScrapeCourse(); db_rph.ScrapeCourses.InsertOnSubmit(scr_rec); } scr_rec.LastDateScraped = date_scrape; db_rph.SubmitChanges(); Common.Wait(); date_scrape = date_scrape.AddDays(1); db_rph = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString()); } } catch (Exception exception) { throw exception; } }
private static void ProcessDetail(Horse rec) { var regex_main = new Regex("<li>(.*?)</li>", RegexOptions.Singleline); Match match_main = regex_main.Match(rec.DetailRaw); int ii = 0; while (match_main.Success) { ii++; string item = match_main.Groups[1].ToString().Trim(); if (ii == 1) { var regex_dob_colour_sex = new Regex(@"\((?:(\d{2})([A-Za-z]{3})(\d{2})){0,1}(?: ([a-z/]+)){0,1}(?: ([a-z/]+)){0,1}\s*\)"); Match match_dob_colour_sex = regex_dob_colour_sex.Match(item); if (match_dob_colour_sex.Success) { if (match_dob_colour_sex.Groups[1].ToString() != "") { int dd = Convert.ToInt32(match_dob_colour_sex.Groups[1].ToString()); string mmm = match_dob_colour_sex.Groups[2].ToString(); int mm = 0; switch (mmm) { case "Jan": mm = 1; break; case "Feb": mm = 2; break; case "Mar": mm = 3; break; case "Apr": mm = 4; break; case "May": mm = 5; break; case "Jun": mm = 6; break; case "Jul": mm = 7; break; case "Aug": mm = 8; break; case "Sep": mm = 9; break; case "Oct": mm = 10; break; case "Nov": mm = 11; break; case "Dec": mm = 12; break; } int yy = Convert.ToInt32(match_dob_colour_sex.Groups[3].ToString()); int currentyear2Digit = DateTime.Now.Year % 2000; if (yy <= currentyear2Digit) { yy += 2000; } else { yy += 1900; } if (mm != 0) { rec.FoalDate = new DateTime(yy, mm, dd); rec.FoalYear = yy; } } string field4 = match_dob_colour_sex.Groups[4].ToString().Trim(); string field5 = match_dob_colour_sex.Groups[5].ToString().Trim(); if (field4 != "" && field5 != "") { rec.Colour = field4; rec.Sex = field5; } else if (field4 != "") { rec.Sex = field4; } } } else if (ii == 2) { int? sire_id = null; string sire_name = null; string sire_country = null; int? dam_id = null; string dam_name = null; string dam_country = null; int? dam_sire_id = null; string dam_sire_name = null; string dam_sire_country = null; var regex_sire = new Regex( @"stallionbook/stallion\.sd\?horse_id=(\d+).*?STALLION"">([^<(]+)(?:\(([A-Z]+)\)){0,1}</a>"); Match match_sire = regex_sire.Match(item); ii = 0; while (match_sire.Success) { ii++; if (ii == 1) { sire_id = Convert.ToInt32(match_sire.Groups[1].ToString()); sire_name = match_sire.Groups[2].ToString().Trim(); sire_country = match_sire.Groups[3].ToString().Trim(); } else if (ii == 2) { dam_sire_id = Convert.ToInt32(match_sire.Groups[1].ToString()); dam_sire_name = match_sire.Groups[2].ToString().Trim(); dam_sire_country = match_sire.Groups[3].ToString().Trim(); } match_sire = match_sire.NextMatch(); } var regex_dam = new Regex(@"dam_home\.sd\?horse_id=(\d+).*?DAM "">([^<(]+)(?:\(([A-Z]+)\)){0,1}</a>"); Match match_dam = regex_dam.Match(item); if (match_dam.Success) { dam_id = Convert.ToInt32(match_dam.Groups[1].ToString()); dam_name = match_dam.Groups[2].ToString().Trim(); dam_country = match_dam.Groups[3].ToString().Trim(); } if (sire_id != null && sire_id != 0) { Horse sire_rec = db_rph.Horses.FirstOrDefault(x => x.RPId == sire_id); if (sire_rec == null) { sire_rec = new Horse(); db_rph.Horses.InsertOnSubmit(sire_rec); sire_rec.RPId = (int)sire_id; sire_rec.PedigreeProcessed = 1; sire_rec.DetailProcessed = 0; } if (sire_rec.Name == null) { sire_rec.Name = Common.ProcessName(sire_name); sire_rec.FlatName = Common.FlattenName(sire_name); } if (sire_rec.Country == null) { sire_rec.Country = sire_country; } db_rph.SubmitChanges(); if (rec.SireId == null) { rec.SireId = sire_rec.Id; } } if (dam_id != null && dam_id != 0) { Horse dam_rec = db_rph.Horses.FirstOrDefault(x => x.RPId == dam_id); if (dam_rec == null) { dam_rec = new Horse(); db_rph.Horses.InsertOnSubmit(dam_rec); dam_rec.RPId = (int)dam_id; dam_rec.PedigreeProcessed = 1; dam_rec.DetailProcessed = 0; } if (dam_rec.Name == null) { dam_rec.Name = Common.ProcessName(dam_name); dam_rec.FlatName = Common.FlattenName(dam_name); } if (dam_rec.Country == null) { dam_rec.Country = dam_country; } db_rph.SubmitChanges(); if (rec.DamId == null) { rec.DamId = dam_rec.Id; } if (dam_sire_id != null && dam_sire_id != 0) { Horse dam_sire_rec = db_rph.Horses.FirstOrDefault(x => x.RPId == dam_sire_id); if (dam_sire_rec == null) { dam_sire_rec = new Horse(); db_rph.Horses.InsertOnSubmit(dam_sire_rec); dam_sire_rec.RPId = (int)dam_sire_id; dam_sire_rec.PedigreeProcessed = 1; dam_sire_rec.DetailProcessed = 0; } if (dam_sire_rec.Name == null) { dam_sire_rec.Name = Common.ProcessName(dam_sire_name); dam_sire_rec.FlatName = Common.FlattenName(dam_sire_name); } if (dam_sire_rec.Country == null) { dam_sire_rec.Country = dam_sire_country; } db_rph.SubmitChanges(); if (dam_rec.SireId == null) { dam_rec.SireId = dam_sire_rec.Id; } } } } match_main = match_main.NextMatch(); } }