private async void FetchContributors(XElement episode, ReduxEntities data, string pid)
{
XNamespace ion = "http://bbc.co.uk/2008/iplayer/ion";
var contributors = episode.Elements(ion + "blocklist")
.Elements(ion + "episode_detail")
.Elements(ion + "contributors")
.Elements(ion + "contributor");
var ctx = new ReduxEntities();
foreach (var contributor in contributors)
{
var ct = new contributor
{
character_name = contributor.Element(ion + "character_name").Value,
family_name = contributor.Element(ion + "family_name").Value,
given_name = contributor.Element(ion + "given_name").Value,
role = contributor.Element(ion + "role").Value,
role_name = contributor.Element(ion + "role_name").Value,
type = contributor.Element(ion + "type").Value,
contributor_id = Convert.ToInt32(contributor.Element(ion + "id").Value),
pid = pid
};
Debug.WriteLine("{0} {1} {2} {3} {4} {5} {6}", ct.character_name, ct.contributor_id, ct.family_name, ct.given_name, ct.role, ct.role_name, ct.type);
ctx.contributors.AddObject(ct);
}
ctx.SaveChanges();
//data.SaveChanges();
}
async private void RunAsyncFetches()
{
Progress progress = new Progress();
progress.Show();
int genrecount = await TaskEx.Run <int>(() =>
{
var ctx = new ReduxEntities();
return(ctx.genres.Count());
});
progress.WriteLine("We have {0} genres", genrecount);
WebClient client = new WebClient();
progress.WriteLine("Fetching daily schedule");
var result = await client.DownloadStringTaskAsync("http://www.bbc.co.uk/bbcone/programmes/schedules/london/today.xml");
progress.WriteLine("Got Schedule");
var doc = XElement.Parse(result);
var pids = from pid in doc.XPathSelectElements("./day/broadcasts/broadcast/programme[@type='episode']/pid")
select pid.Value;
await TaskEx.WhenAll((from p in pids select FetchPidDataAsync(p, progress)));
progress.WriteLine("Finished");
}
private void FetchContributors(XElement episode, ReduxEntities data, string pid)
{
XNamespace ion = "http://bbc.co.uk/2008/iplayer/ion";
var contributors = episode.Elements(ion + "blocklist")
.Elements(ion + "episode_detail")
.Elements(ion + "contributors")
.Elements(ion + "contributor");
foreach (var contributor in contributors)
{
var ct = new contributor
{
character_name = contributor.Element(ion + "character_name").Value,
family_name = contributor.Element(ion + "family_name").Value,
given_name = contributor.Element(ion + "given_name").Value,
role = contributor.Element(ion + "role").Value,
role_name = contributor.Element(ion + "role_name").Value,
type = contributor.Element(ion + "type").Value,
contributor_id = Convert.ToInt32(contributor.Element(ion + "id").Value),
pid = pid
};
data.AddObject("contributors", ct);
}
//data.SaveChanges();
}
private async void FetchTags(XElement episode, ReduxEntities data, string pid)
{
XNamespace ion = "http://bbc.co.uk/2008/iplayer/ion";
var tags = episode.Elements(ion + "blocklist")
.Elements(ion + "episode_detail")
.Elements(ion + "tag_schemes")
.Elements(ion + "tag_scheme")
.Elements(ion + "tags")
.Elements(ion + "tag");
var ctx = new ReduxEntities();
foreach (var tag in tags)
{
var tg = new tag
{
tag_id = tag.Element(ion + "id").Value,
name = tag.Element(ion + "name").Value,
value = tag.Element(ion + "value").Value,
pid = pid
};
Debug.WriteLine("{0} {1} {2} {3}", tg.name, tg.pid, tg.tag_id, tg.value);
ctx.tags.AddObject(tg);
}
ctx.SaveChanges();
}
private async void GetPipsForNewReduxData(object sender, RoutedEventArgs e)
{
Thumbnail thumbnail = new Thumbnail();
thumbnail.Show();
Progress progress = new Progress();
progress.Show();
var itemStore = new ReduxEntities();
var notmatched = (from r in itemStore.redux_items
from rp in itemStore.redux_to_pips
where r.id == rp.redux_id && rp.pips_id == 0
select r.aired).ToList().Select(d => d.Date).Distinct().OrderBy(d => d).ToList();
Fetcher fetcher = new Fetcher();
notmatched = notmatched.Where(d => d >= new DateTime(2011, 02, 04)).ToList();
var earliest = notmatched.First(); //itemStore.redux_items.Min(r => r.aired).Date;
var latest = notmatched.Last(); //itemStore.redux_items.Max(r => r.aired).Date;
foreach (DateTime dt in notmatched)
{
await fetcher.GetDayScheduleAsync(dt, progress, thumbnail);
if (progress.IsCancelled)
{
break;
}
}
}
private async void StartFetching()
{
var ctx = new ReduxEntities();
var thumbnail = new Thumbnail();
thumbnail.Show();
while (IsCancelled == false)
{
var nextSixteen = (from item in ctx.scan_pips_contributors
where item.scanned == false
select item).Take(16).ToList();
if (nextSixteen.Count == 0)
{
IsCancelled = true;
}
else
{
await TaskEx.WhenAll(from item in nextSixteen select FetchItemGenreAndIonAsync(item, thumbnail, this, IsTags, IsIonContributors, IsCategories));
ctx.SaveChanges();
}
}
IsRunning = false;
IsCancelled = false;
}
void Chart_Loaded(object sender, RoutedEventArgs e)
{
var ctx = new ReduxEntities();
//var genres = (from item in ctx.genres
// group item by item.path into grouped
// select new { grouped.Key, Count = grouped.Count() }).ToList();
//paths = (from item in genres
// let path = item.Key
// select new GenrePath
// {
// Path = path,
// Count = item.Count,
// Parent = path.Contains("/") ?
// path.Substring(0, path.LastIndexOf('/'))
// :
// "",
// Level = path.Count(c=>c == '/')
// }).ToList();
var genres = (from item in ctx.genres
group item by item.path into grouped
select new
{
Path = grouped.Key,
Count = grouped.Count(),
Pid =
(from ge in ctx.genre_pids
where ge.path == grouped.Key
select ge.pid).FirstOrDefault(),
TotalDur =
(from ge in ctx.genres
from p in ctx.pips_programmes
where ge.pips_id == p.id && ge.path == grouped.Key
select p.duration).Sum()
}).ToList();
paths = (from item in genres
select new GenrePath
{
Path = item.Path,
Count = item.TotalDur,
Parent = item.Path.Contains("/") ?
item.Path.Substring(0, item.Path.LastIndexOf('/'))
:
"",
Level = item.Path.Count(c => c == '/'),
Pid = item.Pid
}).ToList();
foreach (var gp in paths)
{
gp.Children = (from p in paths where p.Parent == gp.Path select p).ToList();
}
treeMap.ItemsSource = (from p in paths where p.Parent == "" select p).ToList();
}
public async Task AddGenresAsync(string pid, int pips_id, IProgress progress, IThumbnail thumbnail)
{
//progress.WriteLine("Start Fetching for {0}", pid);
if (progress.IsCancelled)
{
return;
}
try
{
thumbnail.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + pid + "_314_176.jpg");
var waitIon = LoadXmlAsync("http://www.bbc.co.uk/iplayer/ion/episodedetail/episode/" + pid + "/format/xml");
var waitPips = LoadXmlAsync("http://www.bbc.co.uk/programmes/" + pid + ".xml");
var pages = await TaskEx.WhenAll(waitIon, waitPips);
var progpage = pages[1];
var ionPage = pages[0];
//var progpage = await LoadXmlAsync("http://www.bbc.co.uk/programmes/" + pid + ".xml");
var genres = (from cat in progpage.XPathSelectElements("categories//category[ancestor-or-self::category[@type='genre']]")
let cats = from c in cat.XPathSelectElements("./descendant-or-self::category") select c.Attribute("key").Value
select new
{
pips_id = pips_id,
title = cat.Element("title").Value,
name = cat.Attribute("key").Value,
//Cats = cats,
path = string.Join("/", cats.Reverse().ToArray())
}
).Distinct();
var data = new ReduxEntities();
foreach (var g in genres)
{
data.genres.AddObject(new genre
{
pips_id = g.pips_id,
name = g.name,
title = g.title,
path = g.path
});
}
try
{
data.SaveChanges();
}
catch (Exception exp)
{
var msg4 = exp.Message;
throw;
}
FetchContributors(ionPage, data, pid);
FetchTags(ionPage, data, pid);
FetchCategories(ionPage, data, pid);
}
catch (WebException)
{
}
}
private void ListBox_SelectionChanged(object sender, SelectionChangedEventArgs e)
{
var g = ((ListBox)sender).SelectedItem as GenrePid;
var ctx = new ReduxEntities();
var path = ctx.genre_pids.First(ge => ge.path == GenrePath.Path);
path.pid = g.Pid;
ctx.SaveChanges();
GenrePath.Pid = g.Pid;
}
private void TestThumbnails(object sender, RoutedEventArgs e)
{
Thumbnail thumbnail = new Thumbnail();
thumbnail.Show();
ReduxEntities ctx = new ReduxEntities();
var pids = from p in ctx.pips_programmes.OrderByDescending(p => p.start_gmt).Take(50)
select p.pid;
foreach (var pid in pids)
{
thumbnail.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + pid + "_314_176.jpg");
}
}
private void FetchCategories(XElement root, ReduxEntities data, string pid)
{
var cats = from cat in root.XPathSelectElements("categories/category[@type != 'genre']")
select new category()
{
pid = pid,
type = cat.Attribute("type").Value,
catkey = cat.Attribute("key").Value,
title = cat.Element("title").Value
};
foreach (var c in cats)
{
data.categories.AddObject(c);
}
}
private void Eliminate_Click(object sender, RoutedEventArgs e)
{
using (var data = new ReduxEntities())
{
var gaps = data.gaps.ToList();
gaps = (from gap in gaps
where ((gap.service_id == 5 && gap.gapend.ToLocalTime().TimeOfDay == new TimeSpan(07, 00, 00)) ||
(gap.service_id == 4 && gap.gapend.ToLocalTime().TimeOfDay == new TimeSpan(06, 00, 00)) ||
(gap.service_id == 1 && gap.gapend.ToLocalTime().TimeOfDay == new TimeSpan(19, 00, 00)) ||
(gap.service_id == 2 && gap.gapend.ToLocalTime().TimeOfDay == new TimeSpan(06, 00, 00)) ||
(gap.service_id == 3 && gap.gapend.ToLocalTime().TimeOfDay == new TimeSpan(06, 00, 00)) ||
(gap.service_id == 6 && gap.gapend.ToLocalTime().TimeOfDay == new TimeSpan(19, 00, 00))
) == false
select gap).ToList();
gapGrid.ItemsSource = gaps;
}
}
private async void InitializePidList(string genrePath)
{
var ctx = new ReduxEntities();
List <GenrePid> pids = await TaskEx.Run <List <GenrePid> >(() =>
{
return((from p in ctx.pips_programmes
from g in ctx.genres
where p.id == g.pips_id &&
g.path == genrePath
select new GenrePid
{
Pid = p.pid
}).ToList());
});
GenrePids = pids;
}
private static void MakeMatch(ReduxEntities ctx, UnmatchedItem item, pips_programmes p)
{
var rp = new redux_to_pips
{
duration_match = item.rp.duration_match,
ischecked = item.rp.ischecked,
partial_match = (item.r.programme_name == p.display_title) == false,
pips_id = p.id,
redux_id = item.rp.redux_id,
start_match = true,
title_match = (item.r.programme_name == p.display_title)
};
ctx.redux_to_pips.DeleteObject(item.rp);
ctx.redux_to_pips.AddObject(rp);
item.rp = rp;
ctx.SaveChanges();
}
// This is a quick(ish) way to scan all the programmes in pips_programmes
// and find all gaps in the schedule.
// Note that this process will find all actual gaps where
// the channel is not transmitting
private void ScanGaps_Click(object sender, RoutedEventArgs e)
{
List <Gap> gaps = new List <Gap>();
Task.Factory.StartNew(() =>
{
StringBuilder sql = new StringBuilder();
sql.AppendLine("insert into gaps (programme_id, service_id, gapstart,gapend)"); // VALUES(1234,1,"2007-06-28 03:00","2007-06-28 06:00")
using (var data = new ReduxEntities())
{
var programmes = (from prog in data.pips_programmes
//orderby prog.ServiceId, prog.StartTime
select prog).ToList();
programmes = (from prog in programmes
orderby prog.service_id, prog.start_gmt
select prog).ToList();
pips_programmes prev = null;
using (var newdata = new ReduxEntities())
{
foreach (var prog in programmes)
{
if (prev != null && prev.service_id == prog.service_id && prev.end_gmt < prog.start_gmt)
{
gap gap = new gap
{
programme_id = prog.id,
service_id = prog.service_id,
gapstart = prev.end_gmt,
gapend = prog.start_gmt
};
newdata.gaps.AddObject(gap);
Dispatcher.Invoke((MyDelegate) delegate { gapLabel.Content = string.Format("{0}", gap.gapstart); });
}
if (prev == null || prog.service_id != prev.service_id || prog.end_gmt > prev.end_gmt)
{
prev = prog;
}
}
newdata.SaveChanges();
}
Dispatcher.Invoke((MyDelegate) delegate { gapGrid.ItemsSource = data.gaps; });
}
});
}
private async void MissingGenres(object sender, RoutedEventArgs e)
{
Thumbnail thumbnail = new Thumbnail();
thumbnail.Show();
Progress progress = new Progress();
progress.Show();
var itemStore = new ReduxEntities();
var nogenres = from p in itemStore.pips_programmes
join g in itemStore.genres on p.id equals g.pips_id into joined
from j in joined.DefaultIfEmpty()
where j == null
select p;
var ng = nogenres.ToList();
Random rnd = new Random();
for (int i = ng.Count; i == 0; i--)
{
int pos = rnd.Next(i);
pips_programmes tmp = ng[i - 1];
ng[i - 1] = ng[pos];
ng[pos] = tmp;
}
List <Task> tasks = new List <Task>();
Fetcher fetcher = new Fetcher();
foreach (var prog in ng)
{
tasks.Add(fetcher.AddGenresAsync(prog.pid, prog.id, progress, thumbnail));
if (tasks.Count >= 16)
{
await TaskEx.WhenAll(tasks);
tasks.Clear();
if (progress.IsCancelled)
{
break;
}
}
}
}
private async Task FetchCategories(XElement root, ReduxEntities data, string pid)
{
var cats = from cat in root.XPathSelectElements("categories/category[@type != 'genre']")
select new category()
{
pid = pid,
type = cat.Attribute("type").Value,
catkey = cat.Attribute("key").Value,
title = cat.Element("title").Value
};
var ctx = new ReduxEntities();
foreach (var c in cats)
{
Debug.WriteLine("{0} {1} {2} {3}", c.catkey, c.pid, c.type, c.title);
//data.categories.AddObject(c);
ctx.categories.AddObject(c);
}
ctx.SaveChanges();
}
private void GetData(object sender, RoutedEventArgs e)
{
//programmesEntities entities = new programmesEntities();
//var result = (from item in entities.redux_items
// where item.programme_name == "Doctor Who"
// select item);
//dataGrid.ItemsSource = result;
//using (var items = new ReduxItems())
//{
// var top = items.PipsProgrammes.Take(100);
// dataGrid.ItemsSource = top;
//}
using (var items = new ReduxEntities())
{
var top = items.redux_items.Take(100);
dataGrid.ItemsSource = top;
}
}
private void FetchCurrent_Click(object sender, RoutedEventArgs e)
{
if (datePicker.SelectedDate != null)
{
DateTime date = StartDate.Date;
DateTime end = EndDate.Date.AddDays(1);
using (var data = new ReduxEntities())
{
var programmes = (from prog in data.pips_programmes
where prog.start_gmt >= date &&
prog.start_gmt < end &&
prog.service_id == ServiceId
select new { DisplayTitle = prog.display_title,
Description = prog.description,
StartGmt = prog.start_gmt,
EndGmt = prog.end_gmt,
ServiceName = prog.service_name }).ToList();
gapGrid.ItemsSource = programmes;
}
}
}
private void FetchTags(XElement episode, ReduxEntities data, string pid)
{
XNamespace ion = "http://bbc.co.uk/2008/iplayer/ion";
var tags = episode.Elements(ion + "blocklist")
.Elements(ion + "episode_detail")
.Elements(ion + "tag_schemes")
.Elements(ion + "tag_scheme")
.Elements(ion + "tags")
.Elements(ion + "tag");
foreach (var tag in tags)
{
var tg = new tag
{
tag_id = tag.Element(ion + "id").Value,
name = tag.Element(ion + "name").Value,
value = tag.Element(ion + "value").Value,
pid = pid
};
data.AddObject("tags", tg);
}
}
private void IngestNewReduxData(object sender, RoutedEventArgs e)
{
Services services = new Services();
var context = new ReduxEntities();
XElement doc = XElement.Load(@"D:\Docs\prog.txt");
var diskrefs = (from item in context.redux_items select item.disk_reference).ToArray();
var remapped = from row in doc.Elements("row")
select new Dictionary <string, string>(
(from field in row.Elements("field")
select new { Key = field.Attribute("name").Value, Value = field.Value }).ToDictionary(vp => vp.Key, vp => vp.Value));
var joined = from row in doc.Elements("row")
join dr in diskrefs on row.Elements("field").First(f => f.Attribute("name").Value == "disk_reference").Value equals dr into j
from j2 in j.DefaultIfEmpty()
where j2 == null
select row;
CultureInfo provider = CultureInfo.InvariantCulture;
var items = from row in joined
select new redux_items
{
aired = DateTime.Parse(row.Elements("field").First(f => f.Attribute("name").Value == "start").Value, provider, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal),
disk_reference = row.Elements("field").First(f => f.Attribute("name").Value == "disk_reference").Value,
programme_crid = row.Elements("field").First(f => f.Attribute("name").Value == "pcrid").Value,
series_crid = row.Elements("field").First(f => f.Attribute("name").Value == "scrid").Value,
programme_name = row.Elements("field").First(f => f.Attribute("name").Value == "title").Value,
short_description = row.Elements("field").First(f => f.Attribute("name").Value == "description").Value,
duration = Convert.ToInt32(row.Elements("field").First(f => f.Attribute("name").Value == "duration").Value),
service_id = services.GetServiceByReduxId(row.Elements("field").First(f => f.Attribute("name").Value == "channel_id").Value).Id
};
foreach (var item in items)
{
context.redux_items.AddObject(item);
}
context.SaveChanges();
}
//private Task<int> GetMaxIdScanned()
//{
// TaskCompletionSource<int> tcs = new TaskCompletionSource<int>();
// TaskEx.Run(() =>
// {
// var ctx = new ReduxEntities();
// tcs.SetResult(ctx.genres.Max(g => g.pips_id));
// });
// return tcs.Task;
//}
//private Task<bool> CountGenres()
//{
// TaskCompletionSource<bool> tcs = new TaskCompletionSource<bool>();
// TaskEx.Ru
//}
private void AddGenres(string pid, int pips_id, Progress progress)
{
try
{
var progpage = XElement.Load("http://www.bbc.co.uk/programmes/" + pid + ".xml");
var genres = (from cat in progpage.XPathSelectElements("categories//category[ancestor-or-self::category[@type='genre']]")
let cats = from c in cat.XPathSelectElements("./descendant-or-self::category") select c.Attribute("key").Value
select new
{
pips_id = pips_id,
title = cat.Element("title").Value,
name = cat.Attribute("key").Value,
//Cats = cats,
path = string.Join("/", cats.Reverse().ToArray())
}
).Distinct();
progress.WriteLine("");
progress.WriteLine("Fetching genres for {0} ({1})", pid, pips_id);
var data = new ReduxEntities();
foreach (var g in genres)
{
progress.WriteLine("{0}, {1}, {2}", g.title, g.name, g.path);
data.genres.AddObject(new genre
{
pips_id = g.pips_id,
name = g.name,
title = g.title,
path = g.path
});
}
data.SaveChanges();
}
catch (WebException)
{
}
}
/// <summary>
/// Scan's a day's schedule and fetches pips data from /programmes
/// DEPRECATED and replaced by the newer code in MainWindow.GetPipsForNewReduxData
/// </summary>
/// <param name="date"></param>
/// <param name="thumbnail"></param>
private void ScanWholeDay(DateTime date, Thumbnail thumbnail)
{
reduxItems = new ReduxEntities();
Dispatcher.Invoke((MyDelegate) delegate { dateScanning.Content = date.ToString("dd/MM/yyyy"); });
Services services = new Services();
DateTime dayStart = date.Date;
DateTime dayEnd = dayStart.AddDays(1);
//if (reduxItems.Scanned.Any(p => p.DateScanned == dayStart) == false)
//{
Dispatcher.Invoke((MyDelegate) delegate { working.Content = "Working..."; });
List <XElement> programmeElements = new List <XElement>();
//var dayItems = from item in reduxItems.Items
// where item.Aired >= dayStart && item.Aired < dayEnd
// select item;
foreach (var sid in services.Select(s => s.Id))
{
Dispatcher.Invoke((MyDelegate) delegate { working.Content = "Working... " + services[sid].Name; });
string url = services.ProgrammesUrl(sid, dayStart);
XElement schedule = XElement.Load(url);
var items = from element in schedule.Element("day").Element("broadcasts").Elements("broadcast")
select new XElement("programme", new XAttribute("serviceid", sid), element);
XElement previous = null;
foreach (var item in items)
{
if (previous != null)
{
var st = previous.Element("broadcast").GetElementDate("end");
var en = item.Element("broadcast").GetElementDate("start");
if (st < en)
{
if ((sid == 1 || sid == 6) && en.Hour == 19 && en.Minute == 0)
{
}
else
{
}
}
}
previous = item;
}
programmeElements.AddRange(items);
}
var programmes = from element in programmeElements
let broadcast = element.Element("broadcast")
let episode = broadcast.Elements("programme").First(e => e.Attribute("type").Value == "episode")
select new pips_programmes()
{
service_id = element.GetAttributeInt("serviceid"),
service_name = services[element.GetAttributeInt("serviceid")].Name,
start_gmt = broadcast.GetElementDate("start"),
end_gmt = broadcast.GetElementDate("end"),
start_time = broadcast.GetElementDate("start").ToLocalTime(),
end_time = broadcast.GetElementDate("end").ToLocalTime(),
duration = broadcast.GetElementInt("duration"),
programme_name = episode.Element("title").Value,
display_title = episode.Element("display_titles").Element("title").Value,
display_subtitle = episode.Element("display_titles").Element("subtitle").Value,
pid = episode.Element("pid").Value,
description = episode.Element("short_synopsis").Value,
rawdata = element.ToString()
};
//foreach (var item in dayItems)
//{
// var matched = (from programme in programmeElements
// where programme.GetAttributeInt("serviceid") == item.ServiceId
// && programme.Element("broadcast").GetElementDate("start").ToUniversalTime() == item.Aired
// select programme).FirstOrDefault();
// if (matched != null)
// {
// var broadcast = matched.Element("broadcast");
// item.IsStartTimeMatched = true;
// int duration = broadcast.GetElementInt("duration");
// item.IsDurationMatched = duration == item.DurationSecs;
// var episode = (from ep in broadcast.Elements("programme")
// where ep.Attribute("type").Value == "episode"
// select ep).First();
// //item.PipsName = episode.Element("display_titles").Element("title").Value;
// //item.TitleMatch = item.PipsName == item.ProgrammeName;
// //item.Pid = episode.Element("pid").Value;
// //item.PipsDurationSecs = broadcast.GetElementInt("duration");
// }
// item.IsChecked = true;
//}
DateTime timerstart = DateTime.Now;
var already = (from p in reduxItems.pips_programmes
where p.start_time >= dayStart && p.start_time < dayEnd
select p).ToList();
List <pips_programmes> newProgrammes = new List <pips_programmes>();
foreach (var prog in programmes)
{
if (already.Any(p => p.service_id == prog.service_id && p.start_time == prog.start_time) == false)
{
reduxItems.pips_programmes.AddObject(prog);
newProgrammes.Add(prog);
}
}
//reduxItems.Scanned.Add(new Scanned() { DateScanned = dayStart });
reduxItems.SaveChanges();
DateTime timerend = DateTime.Now;
Dispatcher.Invoke((MyDelegate) delegate { working.Content = "took " + (timerend - timerstart).ToString(); });
//}
}
private void TestDbConnection(object sender, RoutedEventArgs e)
{
var ctx = new ReduxEntities();
var titles = ctx.redux_items.Take(50).ToList();
}
private void ReadNewReduxDataStream(object sender, RoutedEventArgs e)
{
Services services = new Services();
redux_items item = null;
var context = new ReduxEntities();
int count = 0;
XmlReader reader = XmlReader.Create(@"D:\Docs\prog.txt");
while (reader.Read())
{
if (reader.IsStartElement() && reader.Name == "row")
{
if (item != null)
{
if (context.redux_items.Any(i => i.disk_reference == item.disk_reference) == false)
{
//context.redux_items.AddObject(item);
//count++;
//if (count > 100)
//{
// context.SaveChanges();
// context = new ReduxEntities();
// count = 0;
//}
}
}
item = new redux_items();
}
/*
* channel_id
* disk_reference
* title
* description
* duration
* start
* pcrid
* scrid
* end
*/
CultureInfo provider = CultureInfo.InvariantCulture;
if (reader.IsStartElement() && reader.Name == "field")
{
var fieldname = reader.GetAttribute("name");
switch (fieldname)
{
case "channel_id":
Service service = services.GetServiceByReduxId(reader.ReadElementContentAsString());
item.service_id = service.Id;
break;
case "disk_reference":
item.disk_reference = reader.ReadElementContentAsString();
break;
case "title":
item.programme_name = reader.ReadElementContentAsString();
break;
case "description":
item.short_description = reader.ReadElementContentAsString();
break;
case "duration":
item.duration = reader.ReadElementContentAsInt();
break;
case "start":
string dt = reader.ReadElementContentAsString();
item.aired = DateTime.Parse(dt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
break;
case "pcrid":
item.programme_crid = reader.ReadElementContentAsString();
break;
case "scrid":
item.series_crid = reader.ReadElementContentAsString();
break;
case "end":
break;
default:
break;
}
}
}
if (item != null)
{
//context.redux_items.AddObject(item);
//context.SaveChanges();
}
}
private async void RunMatcherClick(object sender, RoutedEventArgs e)
{
Thumbnail thumbnails = new Thumbnail();
thumbnails.Show();
Progress progress = new Progress();
progress.Show();
ReduxEntities ctx = new ReduxEntities();
//DateTime cutoff = new DateTime(2010, 5, 1);
var unmatched = (from r in ctx.redux_items
from rp in ctx.redux_to_pips
where r.id == rp.redux_id && rp.pips_id == 0
//&& r.aired >= cutoff
select r).ToList();;
int count = 0;
foreach (var unmatcheditem in unmatched)
{
if (progress.IsCancelled)
{
break;
}
var item = new UnmatchedItem(unmatcheditem, ctx.redux_to_pips.FirstOrDefault(rp => rp.redux_id == unmatcheditem.id));
if (item.rp == null || item.rp.pips_id > 0)
{
continue;
}
var rangestart = item.r.aired.AddMinutes(-5);
var rangeend = item.r.aired.AddMinutes(5);
var pipsmatches = await TaskEx.Run <List <pips_programmes> >(() =>
{
return((from p in ctx.pips_programmes
where p.start_gmt == item.r.aired &&
p.service_id == item.r.service_id
select p).ToList());
});
string matchedpid = null;
bool isMatchMade = false;
if (pipsmatches.Count == 0)
{
//progress.WriteLine("Unmatched: {0} {1} {2}", item.r.disk_reference, item.r.programme_name, item.r.aired);
}
else if (pipsmatches.Count == 1)
{
var p = pipsmatches.First();
if (PartialMatch.GetSimpleWeighting(item.r, p) > 0)
{
MakeMatch(ctx, item, p);
matchedpid = p.pid;
isMatchMade = true;
//progress.WriteLine("Matched: {0} {1} {2} -> {3} {4} {5}", item.r.disk_reference, item.r.programme_name, item.r.aired, p.pid, p.display_title, p.start_gmt);
}
}
else
{
//progress.WriteLine("Matched: {0} {1} {2} ->", item.r.disk_reference, item.r.programme_name, item.r.aired);
//foreach (var pipsmatch in pipsmatches)
//{
// var p = pipsmatch;
// progress.WriteLine(" -> {0} {1} {2}", p.pid, p.display_title, p.start_gmt);
//}
//matchedpid = pipsmatches.First().pid;
}
if (isMatchMade == false)
{
pipsmatches = await TaskEx.Run <List <pips_programmes> >(() =>
{
DateTime fiveminutesbefore = item.r.aired.AddMinutes(-5);
DateTime fiveminutesafter = item.r.aired.AddMinutes(5);
return((from p in ctx.pips_programmes
join rp in ctx.redux_to_pips on p.id equals rp.pips_id into joined
from j in joined.DefaultIfEmpty()
where j == null && p.start_gmt >= fiveminutesbefore &&
p.start_gmt <= fiveminutesafter &&
p.service_id == item.r.service_id
select p).ToList());
});
var suitable = pipsmatches.Where(p => PartialMatch.GetSimpleWeighting(item.r, p) > 0)
.OrderByDescending(p => PartialMatch.GetSimpleWeighting(item.r, p))
.OrderBy(p => Math.Abs((p.start_gmt - item.r.aired).TotalSeconds))
.OrderBy(p => Math.Abs(p.duration - item.r.duration))
.ToList();
if (suitable.Count > 0)
{
pips_programmes matchedpips = suitable.First();
MakeMatch(ctx, item, matchedpips);
isMatchMade = true;
matchedpid = matchedpips.pid;
}
}
if (matchedpid != null)
{
thumbnails.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + matchedpid + "_314_176.jpg");
}
if (isMatchMade == false)
{
progress.WriteLine("Failed to match {0} {1} {2}", item.r.programme_name, item.r.disk_reference, item.r.aired);
}
}
MessageBox.Show("Finished matching unmatched items");
}
private async void GetDataAsync(object sender, RoutedEventArgs e)
{
Progress progress = new Progress();
progress.Show();
Thumbnail thumbnail = new Thumbnail();
thumbnail.Show();
var context = new ReduxEntities();
Stopwatch watch = new Stopwatch();
watch.Start();
do
{
watch.Restart();
int firstID = 0;
if (await TaskEx.Run <bool>(() => { return(context.genres.Count() > 0); }))
{
firstID = (from g in context.genres
select g.pips_id).Max();
//firstID = await TaskEx.Run<int>(() =>
//{
// return (from g in context.genres
// select g.pips_id).Max();
//});
}
var pids = (from p in context.pips_programmes
where p.id > firstID
select new PidItem {
Pid = p.pid, Id = p.id
}).Take(50).ToList();
//var pids = await TaskEx.Run<List<PidItem>>(() =>
//{
// return (from p in context.pips_programmes
// where p.id > firstID
// select new PidItem { Pid = p.pid, Id = p.id }).Take(50).ToList();
//});
if (pids.Count == 0)
{
break;
}
//foreach (var pid in pids)
//{
// thumbnail.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + pid.Pid + "_314_176.jpg");
//}
await TaskEx.WhenAll(from pid in pids select AddGenresAsync(pid.Pid, pid.Id, progress, thumbnail));
//foreach (var pid in pids)
//{
// AddGenresAsync(pid.Pid, pid.Id, progress);
// if (progress.IsCancelled)
// {
// break;
// }
//}
} while (progress.IsCancelled == false);
progress.WriteLine("Fetching has been cancelled");
}
private void FillIn()
{
Services services = new Services();
DateTime date = StartDate.Date;
DateTime end = EndDate.Date.AddDays(1);
repository = new ReduxEntities();
var data = repository;
//var programmes = (from prog in data.PipsProgrammes
// where prog.StartGmt >= date
// && prog.StartGmt < end
// && prog.ServiceId == ServiceId
// select prog).ToList();
//gapGrid.ItemsSource = programmes;
List <pips_programmes> programmes = new List <pips_programmes>();
allSchedule = new List <pips_programmes>();
for (var sdate = date; sdate < end; sdate = sdate.AddDays(1))
{
foreach (var url in services.RegionalUrls(ServiceId, sdate))
{
try
{
XElement schedule = XElement.Load(url);
var items = from element in schedule.Element("day").Element("broadcasts").Elements("broadcast")
select new XElement("programme", new XAttribute("serviceid", ServiceId), element);
var progs = from element in items
let broadcast = element.Element("broadcast")
let episode = broadcast.Elements("programme").First(e => e.Attribute("type").Value == "episode")
select new pips_programmes()
{
service_id = element.GetAttributeInt("serviceid"),
service_name = services[element.GetAttributeInt("serviceid")].Name,
start_time = broadcast.GetElementDate("start").ToLocalTime(),
end_time = broadcast.GetElementDate("end").ToLocalTime(),
start_gmt = broadcast.GetElementDate("start"),
end_gmt = broadcast.GetElementDate("end"),
duration = broadcast.GetElementInt("duration"),
programme_name = episode.Element("title").Value,
display_title = episode.Element("display_titles").Element("title").Value,
display_subtitle = episode.Element("display_titles").Element("subtitle").Value,
pid = episode.Element("pid").Value,
description = episode.Element("short_synopsis").Value,
rawdata = element.ToString()
};
allSchedule.AddRange(progs);
// filter any that are already there or overlap
progs = from prog in progs
where programmes.Any(c => c.start_gmt >= prog.start_gmt && c.start_gmt < prog.end_gmt) == false
select prog;
programmes.AddRange(progs);
}
catch (WebException)
{
}
}
}
filteredSchedule = new List <pips_programmes>(programmes);
programmes = programmes.Distinct(new CompareProgrammes()).ToList();
distinctSchedule = programmes;
programmes = programmes.Where(p => p.start_gmt >= date && p.start_gmt < end).ToList();
DateTime prevday = date.AddDays(-1);
var current = (from prog in data.pips_programmes
where prog.start_gmt <end &&
prog.start_gmt> prevday &&
prog.end_gmt > date &&
prog.service_id == ServiceId
select prog).ToList();
currentSchedule = current;
programmes = (from prog in programmes
where current.Any(c => !(c.end_gmt <= prog.start_gmt || c.start_gmt > prog.end_gmt)) == false
select prog).ToList();
var gridprog = (from prog in programmes
select new
{
New = true,
DisplayTitle = prog.display_title,
Description = prog.description,
StartGmt = prog.start_gmt,
EndGmt = prog.end_gmt,
ServiceName = prog.service_name
}).Union(from prog in current
select new
{
New = false,
DisplayTitle = prog.display_title,
Description = prog.description,
StartGmt = prog.start_gmt,
EndGmt = prog.end_gmt,
ServiceName = prog.service_name
}).OrderBy(x => x.StartGmt).ToList();
gapGrid.ItemsSource = gridprog;
programmesToSave = programmes;
}
public async Task GetDayScheduleAsync(DateTime date, IProgress progress, IThumbnail thumbnail)
{
if (progress.IsCancelled)
{
return;
}
var reduxItems = new ReduxEntities();
Services services = new Services();
DateTime dayStart = date.Date;
DateTime dayEnd = dayStart.AddDays(1);
//if (reduxItems.Scanned.Any(p => p.DateScanned == dayStart) == false)
//{
List <XElement> programmeElements = new List <XElement>();
//var dayItems = from item in reduxItems.Items
// where item.Aired >= dayStart && item.Aired < dayEnd
// select item;
var sids = services.Select(s => s.Id).ToArray();
var tasks = (from srv in sids
select LoadXmlAsync(services.ProgrammesUrl(srv, dayStart))).ToArray();
var pages = await TaskEx.WhenAll(tasks);
for (int i = 0; i < sids.Length; i++)
{
var sid = sids[i];
string url = services.ProgrammesUrl(sid, dayStart);
XElement schedule = pages[i];
var items = from element in schedule.Element("day").Element("broadcasts").Elements("broadcast")
select new XElement("programme", new XAttribute("serviceid", sid), element);
XElement previous = null;
//foreach (var item in items)
//{
// if (previous != null)
// {
// var st = previous.Element("broadcast").GetElementDate("end");
// var en = item.Element("broadcast").GetElementDate("start");
// if (st < en)
// {
// if ((sid == 1 || sid == 6) && en.Hour == 19 && en.Minute == 0)
// {
// }
// else
// {
// }
// }
// }
// previous = item;
//}
programmeElements.AddRange(items);
}
var programmes = from element in programmeElements
let broadcast = element.Element("broadcast")
let episode = broadcast.Elements("programme").First(e => e.Attribute("type").Value == "episode")
select new pips_programmes()
{
service_id = element.GetAttributeInt("serviceid"),
service_name = services[element.GetAttributeInt("serviceid")].Name,
start_time = broadcast.GetElementDate("start").ToLocalTime(),
end_time = broadcast.GetElementDate("end").ToLocalTime(),
start_gmt = broadcast.GetElementDate("start"),
end_gmt = broadcast.GetElementDate("end"),
duration = broadcast.GetElementInt("duration"),
programme_name = episode.Element("title").Value,
display_title = episode.Element("display_titles").Element("title").Value,
display_subtitle = episode.Element("display_titles").Element("subtitle").Value,
pid = episode.Element("pid").Value,
description = episode.Element("short_synopsis").Value
};
DateTime timerstart = DateTime.Now;
var start_pad = dayStart.AddDays(-1);
var end_pad = dayEnd.AddDays(1);
var already = (from p in reduxItems.pips_programmes
where p.start_time >= start_pad && p.start_time < end_pad
orderby p.start_gmt
select p).ToList();
List <pips_programmes> newProgrammes = new List <pips_programmes>();
List <Task> pendingTasks = new List <Task>();
foreach (var prog in programmes)
{
if (progress.IsCancelled)
{
break;
}
if (already.Any(p => p.service_id == prog.service_id && p.start_gmt == prog.start_gmt) == false)
{
reduxItems.pips_programmes.AddObject(prog);
reduxItems.SaveChanges();
newProgrammes.Add(prog);
progress.WriteLine("Updating {0} on {1}", prog.display_title, prog.start_gmt);
pendingTasks.Add(AddGenresAsync(prog.pid, prog.id, progress, thumbnail));
if (pendingTasks.Count >= 12)
{
await TaskEx.WhenAll(pendingTasks);
pendingTasks.Clear();
}
}
}
if (pendingTasks.Count > 0)
{
await TaskEx.WhenAll(pendingTasks);
}
//reduxItems.Scanned.Add(new Scanned() { DateScanned = dayStart });
//}
}
private async Task FetchItemGenreAndIonAsync(scan_pips_contributors item, Thumbnail thumbnail, IProgress progress, bool isTags, bool isIonContributors, bool isCategories)
{
if (progress.IsCancelled)
{
return;
}
thumbnail.ShowImage("http://node2.bbcimg.co.uk/iplayer/images/episode/" + item.pid + "_314_176.jpg");
try
{
if (isTags || isIonContributors)
{
WebClient client = new WebClient();
string result = await client.DownloadStringTaskAsync("http://www.bbc.co.uk/iplayer/ion/episodedetail/episode/" + item.pid + "/format/xml");
XElement episode = XElement.Parse(result);
XNamespace ion = "http://bbc.co.uk/2008/iplayer/ion";
if (isIonContributors)
{
await TaskEx.Run(() =>
{
var contributors = episode.Elements(ion + "blocklist")
.Elements(ion + "episode_detail")
.Elements(ion + "contributors")
.Elements(ion + "contributor");
var data = new ReduxEntities();
foreach (var contributor in contributors)
{
var ct = new contributor
{
character_name = contributor.Element(ion + "character_name").Value,
family_name = contributor.Element(ion + "family_name").Value,
given_name = contributor.Element(ion + "given_name").Value,
role = contributor.Element(ion + "role").Value,
role_name = contributor.Element(ion + "role_name").Value,
type = contributor.Element(ion + "type").Value,
contributor_id = Convert.ToInt32(contributor.Element(ion + "id").Value),
pid = item.pid
};
data.AddObject("contributors", ct);
}
data.SaveChanges();
});
}
if (isTags)
{
await TaskEx.Run(() =>
{
var tags = episode.Elements(ion + "blocklist")
.Elements(ion + "episode_detail")
.Elements(ion + "tag_schemes")
.Elements(ion + "tag_scheme")
.Elements(ion + "tags")
.Elements(ion + "tag");
var data = new ReduxEntities();
foreach (var tag in tags)
{
var tg = new tag
{
tag_id = tag.Element(ion + "id").Value,
name = tag.Element(ion + "name").Value,
value = tag.Element(ion + "value").Value,
pid = item.pid
};
data.AddObject("tags", tg);
}
data.SaveChanges();
});
}
}
if (isCategories)
{
WebClient catClient = new WebClient();
var catresult = await catClient.DownloadStringTaskAsync("http://www.bbc.co.uk/programmes/" + item.pid + ".xml");
var root = XElement.Parse(catresult);
await TaskEx.Run(() =>
{
var cats = from cat in root.XPathSelectElements("categories/category[@type != 'genre']")
select new category()
{
pid = item.pid,
type = cat.Attribute("type").Value,
catkey = cat.Attribute("key").Value,
title = cat.Element("title").Value
};
var db = new ReduxEntities();
foreach (var c in cats)
{
db.AddObject("categories", c);
}
db.SaveChanges();
});
}
item.scanned = true;
}
catch (WebException wex)
{
MessageBox.Show("Can't find programme " + item.pid);
//throw new Exception("Couldn't find programme " + item.pid, wex);
}
}