/// <summary> /// Scan's a day's schedule and fetches pips data from /programmes /// DEPRECATED and replaced by the newer code in MainWindow.GetPipsForNewReduxData /// </summary> /// <param name="date"></param> /// <param name="thumbnail"></param> private void ScanWholeDay(DateTime date, Thumbnail thumbnail) { reduxItems = new ReduxEntities(); Dispatcher.Invoke((MyDelegate) delegate { dateScanning.Content = date.ToString("dd/MM/yyyy"); }); Services services = new Services(); DateTime dayStart = date.Date; DateTime dayEnd = dayStart.AddDays(1); //if (reduxItems.Scanned.Any(p => p.DateScanned == dayStart) == false) //{ Dispatcher.Invoke((MyDelegate) delegate { working.Content = "Working..."; }); List <XElement> programmeElements = new List <XElement>(); //var dayItems = from item in reduxItems.Items // where item.Aired >= dayStart && item.Aired < dayEnd // select item; foreach (var sid in services.Select(s => s.Id)) { Dispatcher.Invoke((MyDelegate) delegate { working.Content = "Working... " + services[sid].Name; }); string url = services.ProgrammesUrl(sid, dayStart); XElement schedule = XElement.Load(url); var items = from element in schedule.Element("day").Element("broadcasts").Elements("broadcast") select new XElement("programme", new XAttribute("serviceid", sid), element); XElement previous = null; foreach (var item in items) { if (previous != null) { var st = previous.Element("broadcast").GetElementDate("end"); var en = item.Element("broadcast").GetElementDate("start"); if (st < en) { if ((sid == 1 || sid == 6) && en.Hour == 19 && en.Minute == 0) { } else { } } } previous = item; } programmeElements.AddRange(items); } var programmes = from element in programmeElements let broadcast = element.Element("broadcast") let episode = broadcast.Elements("programme").First(e => e.Attribute("type").Value == "episode") select new pips_programmes() { service_id = element.GetAttributeInt("serviceid"), service_name = services[element.GetAttributeInt("serviceid")].Name, start_gmt = broadcast.GetElementDate("start"), end_gmt = broadcast.GetElementDate("end"), start_time = broadcast.GetElementDate("start").ToLocalTime(), end_time = broadcast.GetElementDate("end").ToLocalTime(), duration = broadcast.GetElementInt("duration"), programme_name = episode.Element("title").Value, display_title = episode.Element("display_titles").Element("title").Value, display_subtitle = episode.Element("display_titles").Element("subtitle").Value, pid = episode.Element("pid").Value, description = episode.Element("short_synopsis").Value, rawdata = element.ToString() }; //foreach (var item in dayItems) //{ // var matched = (from programme in programmeElements // where programme.GetAttributeInt("serviceid") == item.ServiceId // && programme.Element("broadcast").GetElementDate("start").ToUniversalTime() == item.Aired // select programme).FirstOrDefault(); // if (matched != null) // { // var broadcast = matched.Element("broadcast"); // item.IsStartTimeMatched = true; // int duration = broadcast.GetElementInt("duration"); // item.IsDurationMatched = duration == item.DurationSecs; // var episode = (from ep in broadcast.Elements("programme") // where ep.Attribute("type").Value == "episode" // select ep).First(); // //item.PipsName = episode.Element("display_titles").Element("title").Value; // //item.TitleMatch = item.PipsName == item.ProgrammeName; // //item.Pid = episode.Element("pid").Value; // //item.PipsDurationSecs = broadcast.GetElementInt("duration"); // } // item.IsChecked = true; //} DateTime timerstart = DateTime.Now; var already = (from p in reduxItems.pips_programmes where p.start_time >= dayStart && p.start_time < dayEnd select p).ToList(); List <pips_programmes> newProgrammes = new List <pips_programmes>(); foreach (var prog in programmes) { if (already.Any(p => p.service_id == prog.service_id && p.start_time == prog.start_time) == false) { reduxItems.pips_programmes.AddObject(prog); newProgrammes.Add(prog); } } //reduxItems.Scanned.Add(new Scanned() { DateScanned = dayStart }); reduxItems.SaveChanges(); DateTime timerend = DateTime.Now; Dispatcher.Invoke((MyDelegate) delegate { working.Content = "took " + (timerend - timerstart).ToString(); }); //} }
private async Task FetchItemGenreAndIonAsync(scan_pips_contributors item, Thumbnail thumbnail, IProgress progress, bool isTags, bool isIonContributors, bool isCategories) { if (progress.IsCancelled) { return; } thumbnail.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + item.pid + "_314_176.jpg"); try { if (isTags || isIonContributors) { WebClient client = new WebClient(); string result = await client.DownloadStringTaskAsync("http://www.bbc.co.uk/iplayer/ion/episodedetail/episode/" + item.pid + "/format/xml"); XElement episode = XElement.Parse(result); XNamespace ion = "http://bbc.co.uk/2008/iplayer/ion"; if (isIonContributors) { await TaskEx.Run(() => { var contributors = episode.Elements(ion + "blocklist") .Elements(ion + "episode_detail") .Elements(ion + "contributors") .Elements(ion + "contributor"); var data = new ReduxEntities(); foreach (var contributor in contributors) { var ct = new contributor { character_name = contributor.Element(ion + "character_name").Value, family_name = contributor.Element(ion + "family_name").Value, given_name = contributor.Element(ion + "given_name").Value, role = contributor.Element(ion + "role").Value, role_name = contributor.Element(ion + "role_name").Value, type = contributor.Element(ion + "type").Value, contributor_id = Convert.ToInt32(contributor.Element(ion + "id").Value), pid = item.pid }; data.AddObject("contributors", ct); } data.SaveChanges(); }); } if (isTags) { await TaskEx.Run(() => { var tags = episode.Elements(ion + "blocklist") .Elements(ion + "episode_detail") .Elements(ion + "tag_schemes") .Elements(ion + "tag_scheme") .Elements(ion + "tags") .Elements(ion + "tag"); var data = new ReduxEntities(); foreach (var tag in tags) { var tg = new tag { tag_id = tag.Element(ion + "id").Value, name = tag.Element(ion + "name").Value, value = tag.Element(ion + "value").Value, pid = item.pid }; data.AddObject("tags", tg); } data.SaveChanges(); }); } } if (isCategories) { WebClient catClient = new WebClient(); var catresult = await catClient.DownloadStringTaskAsync("http://www.bbc.co.uk/programmes/" + item.pid + ".xml"); var root = XElement.Parse(catresult); await TaskEx.Run(() => { var cats = from cat in root.XPathSelectElements("categories/category[@type != 'genre']") select new category() { pid = item.pid, type = cat.Attribute("type").Value, catkey = cat.Attribute("key").Value, title = cat.Element("title").Value }; var db = new ReduxEntities(); foreach (var c in cats) { db.AddObject("categories", c); } db.SaveChanges(); }); } item.scanned = true; } catch (WebException wex) { MessageBox.Show("Can't find programme " + item.pid); //throw new Exception("Couldn't find programme " + item.pid, wex); } }
private async void RunMatcherClick(object sender, RoutedEventArgs e) { Thumbnail thumbnails = new Thumbnail(); thumbnails.Show(); Progress progress = new Progress(); progress.Show(); ReduxEntities ctx = new ReduxEntities(); //DateTime cutoff = new DateTime(2010, 5, 1); var unmatched = (from r in ctx.redux_items from rp in ctx.redux_to_pips where r.id == rp.redux_id && rp.pips_id == 0 //&& r.aired >= cutoff select r).ToList();; int count = 0; foreach (var unmatcheditem in unmatched) { if (progress.IsCancelled) { break; } var item = new UnmatchedItem(unmatcheditem, ctx.redux_to_pips.FirstOrDefault(rp => rp.redux_id == unmatcheditem.id)); if (item.rp == null || item.rp.pips_id > 0) { continue; } var rangestart = item.r.aired.AddMinutes(-5); var rangeend = item.r.aired.AddMinutes(5); var pipsmatches = await TaskEx.Run <List <pips_programmes> >(() => { return((from p in ctx.pips_programmes where p.start_gmt == item.r.aired && p.service_id == item.r.service_id select p).ToList()); }); string matchedpid = null; bool isMatchMade = false; if (pipsmatches.Count == 0) { //progress.WriteLine("Unmatched: {0} {1} {2}", item.r.disk_reference, item.r.programme_name, item.r.aired); } else if (pipsmatches.Count == 1) { var p = pipsmatches.First(); if (PartialMatch.GetSimpleWeighting(item.r, p) > 0) { MakeMatch(ctx, item, p); matchedpid = p.pid; isMatchMade = true; //progress.WriteLine("Matched: {0} {1} {2} -> {3} {4} {5}", item.r.disk_reference, item.r.programme_name, item.r.aired, p.pid, p.display_title, p.start_gmt); } } else { //progress.WriteLine("Matched: {0} {1} {2} ->", item.r.disk_reference, item.r.programme_name, item.r.aired); //foreach (var pipsmatch in pipsmatches) //{ // var p = pipsmatch; // progress.WriteLine(" -> {0} {1} {2}", p.pid, p.display_title, p.start_gmt); //} //matchedpid = pipsmatches.First().pid; } if (isMatchMade == false) { pipsmatches = await TaskEx.Run <List <pips_programmes> >(() => { DateTime fiveminutesbefore = item.r.aired.AddMinutes(-5); DateTime fiveminutesafter = item.r.aired.AddMinutes(5); return((from p in ctx.pips_programmes join rp in ctx.redux_to_pips on p.id equals rp.pips_id into joined from j in joined.DefaultIfEmpty() where j == null && p.start_gmt >= fiveminutesbefore && p.start_gmt <= fiveminutesafter && p.service_id == item.r.service_id select p).ToList()); }); var suitable = pipsmatches.Where(p => PartialMatch.GetSimpleWeighting(item.r, p) > 0) .OrderByDescending(p => PartialMatch.GetSimpleWeighting(item.r, p)) .OrderBy(p => Math.Abs((p.start_gmt - item.r.aired).TotalSeconds)) .OrderBy(p => Math.Abs(p.duration - item.r.duration)) .ToList(); if (suitable.Count > 0) { pips_programmes matchedpips = suitable.First(); MakeMatch(ctx, item, matchedpips); isMatchMade = true; matchedpid = matchedpips.pid; } } if (matchedpid != null) { thumbnails.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + matchedpid + "_314_176.jpg"); } if (isMatchMade == false) { progress.WriteLine("Failed to match {0} {1} {2}", item.r.programme_name, item.r.disk_reference, item.r.aired); } } MessageBox.Show("Finished matching unmatched items"); }
private async void GetDataAsync(object sender, RoutedEventArgs e) { Progress progress = new Progress(); progress.Show(); Thumbnail thumbnail = new Thumbnail(); thumbnail.Show(); var context = new ReduxEntities(); Stopwatch watch = new Stopwatch(); watch.Start(); do { watch.Restart(); int firstID = 0; if (await TaskEx.Run <bool>(() => { return(context.genres.Count() > 0); })) { firstID = (from g in context.genres select g.pips_id).Max(); //firstID = await TaskEx.Run<int>(() => //{ // return (from g in context.genres // select g.pips_id).Max(); //}); } var pids = (from p in context.pips_programmes where p.id > firstID select new PidItem { Pid = p.pid, Id = p.id }).Take(50).ToList(); //var pids = await TaskEx.Run<List<PidItem>>(() => //{ // return (from p in context.pips_programmes // where p.id > firstID // select new PidItem { Pid = p.pid, Id = p.id }).Take(50).ToList(); //}); if (pids.Count == 0) { break; } //foreach (var pid in pids) //{ // thumbnail.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + pid.Pid + "_314_176.jpg"); //} await TaskEx.WhenAll(from pid in pids select AddGenresAsync(pid.Pid, pid.Id, progress, thumbnail)); //foreach (var pid in pids) //{ // AddGenresAsync(pid.Pid, pid.Id, progress); // if (progress.IsCancelled) // { // break; // } //} } while (progress.IsCancelled == false); progress.WriteLine("Fetching has been cancelled"); }
private async Task AddGenresAsync(string pid, int pips_id, IProgress progress, Thumbnail thumbnail) { //progress.WriteLine("Start Fetching for {0}", pid); if (progress.IsCancelled) { return; } try { var waitIon = LoadXmlAsync("http://www.bbc.co.uk/iplayer/ion/episodedetail/episode/" + pid + "/format/xml"); var waitPips = LoadXmlAsync("http://www.bbc.co.uk/programmes/" + pid + ".xml"); var pages = await TaskEx.WhenAll(waitIon, waitPips); var progpage = pages[1]; var ionPage = pages[0]; //var progpage = await LoadXmlAsync("http://www.bbc.co.uk/programmes/" + pid + ".xml"); var genres = (from cat in progpage.XPathSelectElements("categories//category[ancestor-or-self::category[@type='genre']]") let cats = from c in cat.XPathSelectElements("./descendant-or-self::category") select c.Attribute("key").Value select new { pips_id = pips_id, title = cat.Element("title").Value, name = cat.Attribute("key").Value, //Cats = cats, path = string.Join("/", cats.Reverse().ToArray()) } ).Distinct(); //progress.WriteLine(""); //progress.WriteLine("Fetching genres for {0} ({1})", pid, pips_id); thumbnail.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + pid + "_314_176.jpg"); var data = new ReduxEntities(); foreach (var g in genres) { //progress.WriteLine("{0}, {1}, {2}", g.title, g.name, g.path); data.genres.AddObject(new genre { pips_id = g.pips_id, name = g.name, title = g.title, path = g.path }); //await TaskEx.Run(() => //{ // data.genres.AddObject(new genre // { // pips_id = g.pips_id, // name = g.name, // title = g.title, // path = g.path // }); //}); } FetchContributors(ionPage, data, pid); FetchTags(ionPage, data, pid); FetchCategories(ionPage, data, pid); data.SaveChanges(); //await TaskEx.Run(() => { data.SaveChanges(); }); } catch (WebException) { } }