private static void ScrapePage(RacingPostRacesDataContext db_ppdb, string page, ref string outcome,
ref string pqid)
{
List <PedigreeHorse> horses = PedigreeExtract.Extract(page);
var added = new List <string>();
foreach (PedigreeHorse horse in horses.OrderBy(x => x.Generation).ThenByDescending(x => x.Pedigree))
{
if (added.Contains(horse.PQId))
{
continue;
}
PQ_Horse pq_horse = db_ppdb.PQ_Horses.FirstOrDefault(x => x.Id == horse.PQId);
if (pq_horse == null)
{
pq_horse = new PQ_Horse();
db_ppdb.PQ_Horses.InsertOnSubmit(pq_horse);
pq_horse.Id = horse.PQId;
pq_horse.Name = horse.Name;
pq_horse.FlatName = FlattenName(horse.Name, false);
added.Add(horse.PQId);
}
if (pq_horse.Country == null)
{
pq_horse.Country = horse.Country;
}
if (pq_horse.FoalYear == null)
{
pq_horse.FoalYear = horse.FoalYear;
}
if (pq_horse.Colour == null)
{
pq_horse.Colour = horse.Colour;
}
if (pq_horse.Sex == null)
{
pq_horse.Sex = horse.Sex;
}
if (pq_horse.SireId == null)
{
PedigreeHorse sire1 = horses.FirstOrDefault(x => x.Pedigree == horse.Pedigree + "S");
if (sire1 != null)
{
pq_horse.SireId = sire1.PQId;
}
}
if (pq_horse.DamId == null)
{
PedigreeHorse dam1 = horses.FirstOrDefault(x => x.Pedigree == horse.Pedigree + "D");
if (dam1 != null)
{
pq_horse.DamId = dam1.PQId;
}
}
if (horse.Generation == 0)
{
pq_horse.Starts = horse.Starts;
pq_horse.Wins = horse.Wins;
pq_horse.Places = horse.Places;
pq_horse.Earnings = horse.Earnings;
pq_horse.Owner = horse.Owner;
pq_horse.Breeder = horse.Breeder;
}
Logger.WriteLog(horse.Pedigree + " " + horse.PQId + " " + horse.Name + " " + horse.Country + " " +
horse.FoalYear + " " + horse.Colour
+ " " + horse.Starts + " " + horse.Wins + " " + horse.Places + " " + horse.Earnings +
" " + horse.Owner + " " + horse.Breeder);
}
}
private static void GetHorse(string pqid, ref string status, ref string page)
{
PQ_Horse pq_horse = db_ppdb.PQ_Horses.FirstOrDefault(x => x.Id == pqid);
if (pq_horse != null && pq_horse.Sex != null)
{
status = "Already retrieved";
return;
}
var uri = new Uri(String.Format("http://www.pedigreequery.com/{0}", pqid));
var request = (HttpWebRequest)WebRequest.Create(uri);
request.UserAgent =
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Referer = "";
request.Timeout = 30000;
request.Method = "GET";
try
{
var response = (HttpWebResponse)request.GetResponse();
Stream stream = response.GetResponseStream();
Encoding enc = Encoding.GetEncoding(28591); // iso-8859-1
var reader = new StreamReader(stream, enc);
page = reader.ReadToEnd();
reader.Close();
stream.Close();
status = "Complete";
}
catch (WebException webexcpt)
{
switch (webexcpt.Status)
{
case WebExceptionStatus.NameResolutionFailure:
status = "Host name not resolved";
break;
case WebExceptionStatus.Timeout:
status = "Timeout";
break;
case WebExceptionStatus.ProtocolError:
if (webexcpt.Message.IndexOf("404") != 0)
{
status = "Page not found";
}
else
{
status = "Other error" + webexcpt.Message;
}
break;
default:
status = "Other error " + webexcpt.Message;
break;
}
}
Wait();
}
public static void Merge_RP_PQ_PP()
{
db_racing_update =
new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString());
db_racing_read = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString());
while (db_racing_update.Horses.Where(x => x.PPMatchBasis == "Pending RP_PQ").Any())
{
foreach (
Horse rp_horse in
db_racing_update.Horses.Where(x => x.PPMatchBasis == "Pending RP_PQ")
.OrderBy(x => x.Id)
.ThenBy(x => x.Name)
.Take(500))
{
rp_horse.PPId = null;
rp_horse.PPMatchBasis = "Failed RP_PQ_PP";
Logger.WriteLog("Merging RP/PQ to PP " + rp_horse.Name + " " + rp_horse.Id);
PQ_Horse pq_horse = db_racing_update.PQ_Horses.Where(x => x.Id == rp_horse.PQId).FirstOrDefault();
if (pq_horse != null)
{
if (pq_horse.MergeId != null)
{
Horse_Merged merge_horse =
db_racing_update.Horse_Mergeds.Where(x => x.PPId == pq_horse.MergeId).FirstOrDefault();
if (merge_horse != null)
{
merge_horse.RHId = rp_horse.Id;
merge_horse.Name = rp_horse.Name;
if (merge_horse.Country == null)
{
merge_horse.Country = rp_horse.Country;
}
if (merge_horse.FoalDate == null)
{
merge_horse.FoalDate = rp_horse.FoalDate;
}
if (merge_horse.FoalYear == null)
{
merge_horse.FoalYear = rp_horse.FoalYear;
}
merge_horse.MergeBasis = "RP-PQ Matched";
}
rp_horse.PPId = pq_horse.MergeId;
rp_horse.PPMatchBasis = "PQ Direct";
}
else if (Merge_RP_PQ_Merge(pq_horse, 1, rp_horse.Id) != null)
{
rp_horse.PPId = pq_horse.MergeId;
rp_horse.PPMatchBasis = "PQ Added";
}
}
db_racing_update.SubmitChanges();
}
db_racing_update =
new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString());
}
}
private static int?Merge_RP_PQ_Merge(PQ_Horse pq_horse, int gen, int rh_id)
{
int?merge_sire_id = null;
int?merge_dam_id = null;
PQ_Horse pq_sire = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.SireId).FirstOrDefault();
if (pq_sire == null)
{
return(null);
}
else
{
if (pq_sire.MergeId != null)
{
merge_sire_id = pq_sire.MergeId;
}
else
{
merge_sire_id = Merge_RP_PQ_Merge(pq_sire, gen + 1, 0);
if (merge_sire_id == null)
{
return(null);
}
}
}
PQ_Horse pq_dam = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.DamId).FirstOrDefault();
if (pq_dam == null)
{
return(null);
}
else
{
if (pq_dam.MergeId != null)
{
merge_dam_id = pq_dam.MergeId;
}
else
{
merge_dam_id = Merge_RP_PQ_Merge(pq_dam, gen + 1, 0);
if (merge_dam_id == null)
{
return(null);
}
}
}
var merge_horse = new Horse_Merged();
db_racing_update.Horse_Mergeds.InsertOnSubmit(merge_horse);
if (rh_id != 0)
{
merge_horse.RHId = rh_id;
}
CultureInfo cult_info = Thread.CurrentThread.CurrentCulture;
TextInfo text_info = cult_info.TextInfo;
merge_horse.Name = text_info.ToTitleCase(pq_horse.Name.ToLower());
merge_horse.Country = pq_horse.Country;
if (pq_horse.FoalYear != null)
{
merge_horse.FoalDate = new DateTime((int)pq_horse.FoalYear, 1, 1);
merge_horse.FoalYear = pq_horse.FoalYear;
}
merge_horse.Colour = pq_horse.Colour;
merge_horse.Sex = pq_horse.Sex;
merge_horse.SireId = merge_sire_id;
merge_horse.DamId = merge_dam_id;
merge_horse.Haplo =
db_racing_read.Horse_Mergeds.Where(x => x.Id == merge_dam_id).Select(x => x.Haplo).FirstOrDefault();
merge_horse.MergeBasis = "RP-PQ Added";
db_racing_update.SubmitChanges();
pq_horse.MergeId = merge_horse.Id;
pq_horse.MergeMatchBasis = "Added";
return(merge_horse.Id);
}
public static void Match_PQ_PP()
{
db_racing_update =
new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString());
db_racing_read = new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString());
pp_horses = db_racing_read.PP_Horse_Selects.ToDictionary(x => x.Id, x => new PPHorse
{
HorseName = x.HorseName,
Id = x.Id,
SireId = x.SireId,
SireName = x.SireName,
DamName = x.DamName,
Country = x.Country,
FoalYear = x.FoalYear,
OrigName = x.OrigName
});
pp_horse_lookup = pp_horses.Select(x => x.Value).ToLookup(x => x.HorseName, x => x.Id);
pp_sire_lookup =
pp_horses.Where(x => x.Value.SireId != null)
.Select(x => x.Value)
.ToLookup(x => (int)x.SireId, x => x.Id);
while (db_racing_update.PQ_Horses.Where(x => x.MergeMatchBasis == null).Any())
{
foreach (PQ_Horse pq_horse in db_racing_update.PQ_Horses.Where(x => x.MergeMatchBasis == null).Take(500)
)
{
pq_horse.MergeMatchBasis = "Failed";
Logger.WriteLog("Matching PQ to PP " + pq_horse.Name + " " + pq_horse.Id);
string horse_name = FlattenName(pq_horse.Name);
string sire_name = null;
string dam_name = null;
int? year = null;
if (pq_horse.FoalYear != null)
{
year = pq_horse.FoalYear;
}
string country = pq_horse.Country == null ? "" : pq_horse.Country;
PQ_Horse pq_sire = null;
if (pq_horse.SireId != null)
{
pq_sire = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.SireId).FirstOrDefault();
if (pq_sire != null)
{
sire_name = FlattenName(pq_sire.Name);
}
}
PQ_Horse pq_dam = null;
if (pq_horse.DamId != null)
{
pq_dam = db_racing_read.PQ_Horses.Where(x => x.Id == pq_horse.DamId).FirstOrDefault();
if (pq_dam != null)
{
dam_name = FlattenName(pq_dam.Name);
}
}
int? ppid = null;
string match_basis = null;
if (MatchToPPByName(horse_name, sire_name, dam_name, year, country, ref ppid, ref match_basis))
{
pq_horse.MergeId = (int)ppid;
pq_horse.MergeMatchBasis = match_basis;
}
if (ppid == null && pq_sire != null)
{
int?sire_merge_id = null;
if (pq_sire.MergeId != null)
{
sire_merge_id = pq_sire.MergeId;
}
match_basis = "Sire";
if (MatchToPPBySire(sire_merge_id, horse_name, sire_name, dam_name, year, country, ref ppid,
ref match_basis))
{
pq_horse.MergeId = (int)ppid;
pq_horse.MergeMatchBasis = match_basis;
}
}
}
db_racing_update.SubmitChanges();
}
db_racing_update =
new RacingPostRacesDataContext(ConfigurationManager.ConnectionStrings["Racing"].ToString());
}
