List<Show> Grab(GrabParametersBase p)
{
var pp = (GrabParameters)p;
var url = string.Format(URL, pp.FromDate.ToString("dd/MM/yyyy"), pp.ToDate.ToString("dd/MM/yyyy"));
var wr = WebRequest.Create(url);
_logger.WriteEntry("Grabbing BBCW", LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
using (var sr = new StreamReader(res.GetResponseStream()))
{
var lst = new List<Show>();
sr.ReadLine(); // first line
while (!sr.EndOfStream)
{
var line = sr.ReadLine();
if (!string.IsNullOrEmpty(line) && line.Length > 10)
{
var show = new Show();
show.Channel = "BBC World News";
var tokens = line.Split('\t');
show.StartTime = DateTime.SpecifyKind(Convert.ToDateTime(tokens[0]) + Convert.ToDateTime(tokens[1]).TimeOfDay, DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
show.Title = tokens[2];
//show.Episode = string.IsNullOrEmpty(tokens[3]) ? null : (int?)Convert.ToInt32(tokens[3]); // not number
show.Description = tokens[4];
lst.Add(show);
}
}
return lst;
}
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
logger.WriteEntry("Grabbing jn1 schedule", LogType.Info);
var wr = WebRequest.Create(URL);
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(res.GetResponseStream());
var lst = new List<Show>();
var ul = doc.DocumentNode.Descendants("ul").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "program_list");
foreach (var li in ul.Descendants("li"))
{
var show = new Show();
show.Channel = "Jewish News One";
var startTime = li.Descendants("span").First().InnerText.Replace("::", ":");
show.StartTime = DateTime.SpecifyKind(DateTime.Now.Date + Convert.ToDateTime(startTime).TimeOfDay, DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
show.Title = li.Descendants("span").ToList()[1].InnerText.Trim().ToLower();
show.Title = show.Title.First().ToString().ToUpper() + String.Join("", show.Title.Skip(1));
show.Description = li.Descendants("span").ToList()[2].InnerText.Trim();
lst.Add(show);
}
var secondList = new List<Show>();
foreach (var show in lst)
{
var s = show.Clone(); // has same daily schedule every day , so just duplicate entries with tommorow date
s.StartTime = show.StartTime.AddDays(1);
secondList.Add(s);
}
lst.AddRange(secondList);
FixShowsEndTimeByStartTime(lst);
return lst;
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
var shows = new List<Show>();
var doc = XDocument.Parse(xmlParameters);
var sdElement = doc.Descendants("StartDate").FirstOrDefault();
var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
var edElement = doc.Descendants("EndDate").FirstOrDefault();
var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;
foreach (Channel c in Enum.GetValues(typeof(Channel)))
{
var wr = WebRequest.Create(string.Format(url, (int)c, DateTime.Now.Date.AddDays(startDateDiff).ToString(DateFormat), DateTime.Now.Date.AddDays(endDateDays).ToString(DateFormat)));
wr.Timeout = 30000;
logger.WriteEntry(string.Format("Grabbing hot.net.il channel {0} ", c.ToString()), LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
var html = new HtmlAgilityPack.HtmlDocument();
html.Load(res.GetResponseStream(), Encoding.UTF8);
foreach (var tr in html.DocumentNode.Descendants("tr").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "redtr_off"))
{
var tds = tr.Descendants("td").ToList();
var show = new Show();
show.Title = tds[2].InnerText;
show.StartTime = DateTime.SpecifyKind(Convert.ToDateTime(tds[4].InnerText),DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.Local, TimeZoneInfo.Utc);
show.EndTime = show.StartTime.Add(Convert.ToDateTime(tds[5].InnerText).TimeOfDay);
show.Channel = c.ToString();
shows.Add(show);
}
}
return shows;
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
var shows = new List<Show>();
var doc = XDocument.Parse(xmlParameters);
var sdElement = doc.Descendants("StartDate").FirstOrDefault();
var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
var edElement = doc.Descendants("EndDate").FirstOrDefault();
var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;
for (int i = startDateDiff; i <= endDateDays; i++)
{
var date = DateTime.Now.Date.AddDays(i);
var wr = WebRequest.Create(string.Format(URL, date.ToString(DateFormat)));
logger.WriteEntry(string.Format("Grabbing bein sports date {0} ...", date.ToString(DateFormat)), LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
var data = XDocument.Load(res.GetResponseStream());
var html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(data.Descendants("body").First().Value);
var sections = html.DocumentNode.Descendants("section").ToList();
int divCounter = 0;
foreach (var section in sections)
{
if (sections.Count > 0)
{
foreach (var li in sections[divCounter].Descendants("li"))
{
var time = li.Descendants("time").FirstOrDefault();
if (time == null)
continue;
var show = new Show();
var a = li.Descendants("a").FirstOrDefault();
if (a == null)
continue;
show.Channel = "ch-" + a.Attributes["href"].Value.Split('/')[2];
show.StartTime = Convert.ToDateTime(time.Attributes["datetime"].Value);
show.StartTime = DateTime.SpecifyKind(show.StartTime, DateTimeKind.Utc);
var dvText = li.Descendants("div").First();
show.Title = dvText.ChildNodes[0].InnerText.Trim();
var spans = dvText.Descendants("span").ToList();
if (spans.Count > 0)
show.Description = spans[0].InnerText.Trim();
shows.Add(show);
}
}
divCounter++;
}
}
var lst = new List<Show>();
foreach (var chan in shows.Select(x => x.Channel).Distinct())
{
var channelShows = shows.Where(x => x.Channel == chan).OrderBy(x => x.StartTime).ToList();
FixShowsEndTimeByStartTime(channelShows);
lst.AddRange(channelShows);
}
return lst;
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
var shows = new List<Show>();
var doc = XDocument.Parse(xmlParameters);
var sdElement = doc.Descendants("StartDate").FirstOrDefault();
var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
var edElement = doc.Descendants("EndDate").FirstOrDefault();
var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;
for (int i = startDateDiff; i <= endDateDays; i++)
{
var date = DateTime.Now.Date.AddDays(i);
var wr = WebRequest.Create(string.Format(url, date.ToString(DateFormat)));
logger.WriteEntry(string.Format("Grabbing rtd date {0} ...", date.ToString(DateFormat)), LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
var html = new HtmlAgilityPack.HtmlDocument();
html.Load(res.GetResponseStream());
var divs = html.DocumentNode.Descendants("div").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.EndsWith("b-lenta-film_one"));
foreach (var div in divs)
{
var show = new Show();
var titleA = div.Descendants("a").First(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "b-film_heading");
show.Title = titleA.InnerText;
show.Channel = "RT DOC";
var groupA = div.Descendants("div").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "b-film_group");
if (groupA != null)
{
var groupText = groupA.Descendants("a").First().InnerText;
show.Description = show.Title;
show.Title = groupText;
}
show.StartTime = DateTime.SpecifyKind(date.AddMinutes(Convert.ToInt32(div.Attributes["class"].Value.Split(' ')[1].Substring(5))), DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Russian Standard Time"), TimeZoneInfo.Utc);
shows.Add(show);
}
}
FixShowsEndTimeByStartTime(shows);
return shows;
}
public override List<Show> Grab(string xmlParameters, ILogger logger)
{
var shows = new List<Show>();
foreach (Channel c in Enum.GetValues(typeof(Channel)))
{
var d = DateTime.Now.Date;
var wr = WebRequest.Create(string.Format(urlFormat, d.ToString("dd/MM/yyyy"), (int)c));
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(res.GetResponseStream(),Encoding.UTF8);
var tbl = doc.DocumentNode.Descendants("table").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "omega_mt1 table1");
var lst = new List<Show>();
DateTime? lastStartTime = null;
foreach (var tr in tbl.Descendants("tr").Where(x => x.Attributes["class"].Value == "bg3" || x.Attributes["class"].Value == "bg5"))
{
var show = new Show();
var gmtTime = Tools.CleanupText(tr.Descendants("td").ToList()[2].InnerText);
var startTime = Convert.ToDateTime(gmtTime);
show.StartTime = DateTime.SpecifyKind(d.Date + Convert.ToDateTime(startTime).TimeOfDay, DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("GMT Standard Time"), TimeZoneInfo.Utc);
if (lastStartTime.HasValue && lastStartTime.Value > show.StartTime)
{
d = d.AddDays(1);
show.StartTime = show.StartTime.AddDays(1);
}
lastStartTime = show.StartTime;
show.Title = Tools.CleanupText(tr.Descendants("td").ToList()[3].ChildNodes[1].InnerText);
show.Description = Tools.CleanupText(tr.Descendants("td").ToList()[3].ChildNodes[2].InnerText);
show.Channel = c.ToString();
lst.Add(show);
}
FixShowsEndTimeByStartTime(lst);
shows.AddRange(lst);
}
return shows;
}
private void FillShows(XDocument xml, List<Show> shows)
{
var body = xml.Element("span").Element("html").Element("body");
var table = body.Element("table");
bool firstRow = true;
foreach (var row in table.Elements("tr"))
{
if (firstRow)
{
firstRow = false;
continue;
}
var show = new Show();
var cells = row.Elements("td").ToList();
show.Title = cells[0].Value;
show.Description = cells[1].Value;
show.StartTime = Convert.ToDateTime(cells[5].Value);
show.StartTime += Convert.ToDateTime(cells[3].Value).TimeOfDay;
show.StartTime = DateTime.SpecifyKind(show.StartTime, DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Arabic Standard Time"), TimeZoneInfo.Utc);
shows.Add(show);
}
}
public List<Show> Grab(GrabParametersBase p, ILogger logger)
{
currentNow = TimeZoneInfo.ConvertTime(DateTime.Now, TimeZoneInfo.Local, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"));
var pp = (GrabParameters)p;
logger.WriteEntry("grabbing CyfrowyPolsat.pl channel : " + pp.Channel, LogType.Info);
var wr = (HttpWebRequest)WebRequest.Create(string.Format(URL, pp.Channel.ToString().Replace("_", "-")));
wr.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0";
wr.Timeout = 5000;
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(res.GetResponseStream());
var lst = new List<Show>();
var div = doc.DocumentNode.Descendants("div").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "main col");
var times = div.Descendants("span").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "time").ToList();
var names = div.Descendants("a").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "name").ToList();
var metas = div.Descendants("div").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "meta").ToList();
for (int i = 0; i < times.Count; i++)
{
var s = new Show();
s.Channel = pp.Channel.ToString();
var startTime = Convert.ToDateTime(times[i].InnerText);
s.StartTime = DateTime.SpecifyKind(currentNow.Date + Convert.ToDateTime(startTime).TimeOfDay, DateTimeKind.Unspecified);
s.StartTime = TimeZoneInfo.ConvertTime(s.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
if (lst.Count > 0 && s.StartTime <= lst[lst.Count - 1].StartTime)
{
s.StartTime = s.StartTime.AddDays(1);
currentNow = currentNow.AddDays(1);
}
s.Title = HttpUtility.HtmlDecode(names[i].InnerText);
s.Description = HttpUtility.HtmlDecode(metas[i * 2].InnerText + "\n" + metas[(i * 2) + 1].InnerText).Trim();
lst.Add(s);
}
return lst;
}
List<Show> Grab(GrabParametersBase p,ILogger logger)
{
var pp = (CyfraPlus.GrabParameters)p;
var shows = new List<Show>();
var wr = WebRequest.Create(string.Format(urlFormat, pp.Date.ToString(DateFormat)));
logger.WriteEntry(string.Format("Grabbing Cyfra+ date {0} ...", pp.Date.ToString(DateFormat)), LogType.Info);
var res = (HttpWebResponse)wr.GetResponse();
const int ChannelDepth = 2;
using (var sr = new StreamReader(res.GetResponseStream()))
{
var startDownloadTime = DateTime.Now;
var data = new StringBuilder();
int blockSize = 16384;
while (!sr.EndOfStream)
{
var buf = new char[blockSize];
var totalRead = sr.ReadBlock(buf, 0, blockSize);
data.Append(buf);
if (DateTime.Now - startDownloadTime > TimeSpan.FromSeconds(1))
{
startDownloadTime = DateTime.Now;
logger.WriteEntry(string.Format("Downloaded {0:#,##0} bytes so far", data.Length), LogType.Info);
}
}
var r = new Newtonsoft.Json.JsonTextReader(new StringReader(data.ToString()));
while (r.Read())
{
r.Read();
var channelNumber = r.ReadAsInt32();
var channelName = r.ReadAsString();
r.Read();
r.Read();
while (r.Depth > ChannelDepth)
{
var show = new Show();
show.Channel = channelName.Trim();
var programId = r.ReadAsInt32();
show.Title = Tools.CleanupText(r.ReadAsString());
show.StartTime = new DateTime(1970, 1, 1).Add(TimeSpan.FromSeconds(r.ReadAsInt32().Value));
show.EndTime = show.StartTime.Add(TimeSpan.FromSeconds(Convert.ToDouble(r.ReadAsInt32())));
var num = r.ReadAsInt32();
shows.Add(show);
var depth = r.Depth;
while (r.Depth == depth)
r.Read();
r.Read();
}
}
}
return shows;
}
List<Show> Grab(GrabParametersBase p, ILogger logger)
{
var pp = (GrabParameters)p;
var url = string.Format(URL, pp.Channel.ToString().Replace("_", "-").Replace("AANNDD", "%26").Replace("PPLLUUSS", "%2B"));
var wr = WebRequest.Create(url);
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
logger.WriteEntry(string.Format("Grabbing Channel {0}", pp.Channel), LogType.Info);
doc.Load(res.GetResponseStream());
var shows = new List<Show>();
foreach (Day d in Enum.GetValues(typeof(Day)))
{
var dayOfWeek = (DayOfWeek)d;
var div = doc.DocumentNode.Descendants("div").FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == d.ToString());
if (div != null)
{
var date = NextDateOfDayOfWeek(dayOfWeek);
foreach (var ul in div.Descendants("ul"))
{
foreach (var li in ul.Descendants("li"))
{
var par = li.Descendants("p").First();
var a = li.Descendants("a").First();
var show = new Show();
show.Channel = pp.Channel.ToString();
show.Title = a.InnerText.Trim();
show.StartTime = DateTime.SpecifyKind(date + Convert.ToDateTime(par.InnerText.Trim()).TimeOfDay, DateTimeKind.Unspecified);
show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
shows.Add(show);
}
}
}
}
return shows;
}
List<Show> Grab(GrabParametersBase p)
{
var pp = (GrabParameters)p;
var url = GetUrl(pp);
_logger.WriteEntry(string.Format("Grabbing rashut2 {0} for date {1}", pp.Channel, pp.Date.ToString("d")), LogType.Info);
var wr = WebRequest.Create(string.Format(url, pp.Date.ToString(DateFormat)));
var res = (HttpWebResponse)wr.GetResponse();
var doc = new HtmlAgilityPack.HtmlDocument();
doc.Load(res.GetResponseStream());
var nodes = doc.DocumentNode.SelectNodes("//comment()");
if (nodes != null)
{
foreach (HtmlAgilityPack.HtmlNode comment in nodes)
{
if (!comment.InnerText.StartsWith("DOCTYPE"))
comment.ParentNode.RemoveChild(comment);
}
}
int cellDelta = 0;
var tbl = doc.GetElementbyId("table5");
if (pp.Channel == Channel.Channel10)
{
tbl = doc.GetElementbyId("table1");
cellDelta = 1;
}
const int childStart = 9;
var shows = new List<Show>();
for (int i = childStart; i < tbl.ChildNodes.Count; i += 2)
{
var show = new Show();
var d = Convert.ToDateTime(Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[1].InnerText));
show.StartTime = pp.Date.AddHours(d.Hour).AddMinutes(d.Minute);
if (d.Hour < 6) // data is shown from 6 (AM) till next day 6 (AM) so after midnight we need to increase the date
show.StartTime = show.StartTime.AddDays(1);
try
{
show.StartTime = TimeZoneInfo.ConvertTimeToUtc(show.StartTime, TimeZoneInfo.Local);
}
catch (Exception ex) // error on the verge of daylight saving start
{
_logger.WriteEntry(ex.Message, LogType.Error);
continue;
}
show.Title = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[3 + cellDelta].InnerText);
var episodeName = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[5 + cellDelta].InnerText);
var episodeNumber = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[7 + cellDelta].InnerText);
var genere = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[9 + cellDelta].InnerText);
show.Description = string.Empty;
if (!string.IsNullOrEmpty(episodeName))
show.Description += string.Format("שם הפרק : {0}\n", episodeName);
if (!string.IsNullOrEmpty(episodeNumber))
{
show.Description += string.Format("מספר הפרק : {0}\n", episodeNumber);
int num;
if (int.TryParse(episodeNumber, out num))
show.Episode = num;
}
if (!string.IsNullOrEmpty(genere))
show.Description += string.Format("סוג תכנית : {0}\n", genere);
shows.Add(show);
}
for (int i = shows.Count - 1; i >= 0; i--)
{
var show = shows[i];
if (show.Description != null)
show.Description = show.Description.Trim();
show.Channel = pp.Channel.ToString();
if (i == shows.Count - 1)
show.EndTime = show.StartTime.AddHours(1);
else
show.EndTime = shows[i + 1].StartTime;
}
return shows;
}
