Пример #1
0
        List<Show> Grab(GrabParametersBase p)
        {
            var pp = (GrabParameters)p;

            var url = string.Format(URL, pp.FromDate.ToString("dd/MM/yyyy"), pp.ToDate.ToString("dd/MM/yyyy"));

            var wr = WebRequest.Create(url);
            _logger.WriteEntry("Grabbing BBCW", LogType.Info);
            var res = (HttpWebResponse)wr.GetResponse();
            using (var sr = new StreamReader(res.GetResponseStream()))
            {
                var lst = new List<Show>();
                sr.ReadLine(); // first line
                while (!sr.EndOfStream)
                {
                    var line = sr.ReadLine();
                    if (!string.IsNullOrEmpty(line) && line.Length > 10)
                    {
                        var show = new Show();
                        show.Channel = "BBC World News";
                        var tokens = line.Split('\t');
                        show.StartTime = DateTime.SpecifyKind(Convert.ToDateTime(tokens[0]) + Convert.ToDateTime(tokens[1]).TimeOfDay, DateTimeKind.Unspecified);
                        show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
                        show.Title = tokens[2];
                        //show.Episode = string.IsNullOrEmpty(tokens[3]) ? null : (int?)Convert.ToInt32(tokens[3]); // not number
                        show.Description = tokens[4];
                        lst.Add(show);
                    }
                }
                return lst;
            }
        }
Пример #2
0
        public override List<Show> Grab(string xmlParameters, ILogger logger)
        {
            logger.WriteEntry("Grabbing jn1 schedule", LogType.Info);
            var wr = WebRequest.Create(URL);
            var res = (HttpWebResponse)wr.GetResponse();
            var doc = new HtmlAgilityPack.HtmlDocument();
            doc.Load(res.GetResponseStream());
            var lst = new List<Show>();
            var ul = doc.DocumentNode.Descendants("ul").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "program_list");
            foreach (var li in ul.Descendants("li"))
            {
                var show = new Show();
                show.Channel = "Jewish News One";
                var startTime = li.Descendants("span").First().InnerText.Replace("::", ":");
                show.StartTime = DateTime.SpecifyKind(DateTime.Now.Date + Convert.ToDateTime(startTime).TimeOfDay, DateTimeKind.Unspecified);
                show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
                show.Title = li.Descendants("span").ToList()[1].InnerText.Trim().ToLower();
                show.Title = show.Title.First().ToString().ToUpper() + String.Join("", show.Title.Skip(1));
                show.Description = li.Descendants("span").ToList()[2].InnerText.Trim();

                lst.Add(show);
            }
            var secondList = new List<Show>();
            foreach (var show in lst)
            {
                var s = show.Clone(); // has same daily schedule every day , so just duplicate entries with tommorow date
                s.StartTime = show.StartTime.AddDays(1);
                secondList.Add(s);
            }
            lst.AddRange(secondList);
            FixShowsEndTimeByStartTime(lst);
            return lst;
        }
Пример #3
0
        public override List<Show> Grab(string xmlParameters, ILogger logger)
        {
            var shows = new List<Show>();
            var doc = XDocument.Parse(xmlParameters);
            var sdElement = doc.Descendants("StartDate").FirstOrDefault();
            var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
            var edElement = doc.Descendants("EndDate").FirstOrDefault();
            var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;

            foreach (Channel c in Enum.GetValues(typeof(Channel)))
            {
                var wr = WebRequest.Create(string.Format(url, (int)c, DateTime.Now.Date.AddDays(startDateDiff).ToString(DateFormat), DateTime.Now.Date.AddDays(endDateDays).ToString(DateFormat)));
                wr.Timeout = 30000;
                logger.WriteEntry(string.Format("Grabbing hot.net.il channel {0} ", c.ToString()), LogType.Info);
                var res = (HttpWebResponse)wr.GetResponse();

                var html = new HtmlAgilityPack.HtmlDocument();
                html.Load(res.GetResponseStream(), Encoding.UTF8);

                foreach (var tr in html.DocumentNode.Descendants("tr").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "redtr_off"))
                {
                    var tds = tr.Descendants("td").ToList();
                    var show = new Show();
                    show.Title = tds[2].InnerText;
                    show.StartTime = DateTime.SpecifyKind(Convert.ToDateTime(tds[4].InnerText),DateTimeKind.Unspecified);
                    show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.Local, TimeZoneInfo.Utc);
                    show.EndTime = show.StartTime.Add(Convert.ToDateTime(tds[5].InnerText).TimeOfDay);
                    show.Channel = c.ToString();
                    shows.Add(show);
                }
            }
            return shows;
        }
Пример #4
0
        public override List<Show> Grab(string xmlParameters, ILogger logger)
        {
            var shows = new List<Show>();

            var doc = XDocument.Parse(xmlParameters);
            var sdElement = doc.Descendants("StartDate").FirstOrDefault();
            var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
            var edElement = doc.Descendants("EndDate").FirstOrDefault();
            var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;
            for (int i = startDateDiff; i <= endDateDays; i++)
            {
                var date = DateTime.Now.Date.AddDays(i);

                var wr = WebRequest.Create(string.Format(URL, date.ToString(DateFormat)));
                logger.WriteEntry(string.Format("Grabbing bein sports date {0} ...", date.ToString(DateFormat)), LogType.Info);
                var res = (HttpWebResponse)wr.GetResponse();

                var data = XDocument.Load(res.GetResponseStream());
                var html = new HtmlAgilityPack.HtmlDocument();
                html.LoadHtml(data.Descendants("body").First().Value);

                var sections = html.DocumentNode.Descendants("section").ToList();
                int divCounter = 0;
                foreach (var section in sections)
                {
                    if (sections.Count > 0)
                    {
                        foreach (var li in sections[divCounter].Descendants("li"))
                        {
                            var time = li.Descendants("time").FirstOrDefault();
                            if (time == null)
                                continue;
                            var show = new Show();
                            var a = li.Descendants("a").FirstOrDefault();
                            if (a == null)
                                continue;
                            show.Channel = "ch-" + a.Attributes["href"].Value.Split('/')[2];
                            show.StartTime = Convert.ToDateTime(time.Attributes["datetime"].Value);
                            show.StartTime = DateTime.SpecifyKind(show.StartTime, DateTimeKind.Utc);
                            var dvText = li.Descendants("div").First();
                            show.Title = dvText.ChildNodes[0].InnerText.Trim();
                            var spans = dvText.Descendants("span").ToList();
                            if (spans.Count > 0)
                                show.Description = spans[0].InnerText.Trim();
                            shows.Add(show);
                        }
                    }
                    divCounter++;
                }
            }
            var lst = new List<Show>();
            foreach (var chan in shows.Select(x => x.Channel).Distinct())
            {
                var channelShows = shows.Where(x => x.Channel == chan).OrderBy(x => x.StartTime).ToList();
                FixShowsEndTimeByStartTime(channelShows);
                lst.AddRange(channelShows);
            }
            return lst;
        }
Пример #5
0
        public override List<Show> Grab(string xmlParameters, ILogger logger)
        {
            var shows = new List<Show>();

            var doc = XDocument.Parse(xmlParameters);
            var sdElement = doc.Descendants("StartDate").FirstOrDefault();
            var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
            var edElement = doc.Descendants("EndDate").FirstOrDefault();
            var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;
            for (int i = startDateDiff; i <= endDateDays; i++)
            {
                var date = DateTime.Now.Date.AddDays(i);
                var wr = WebRequest.Create(string.Format(url, date.ToString(DateFormat)));
                logger.WriteEntry(string.Format("Grabbing rtd date {0} ...", date.ToString(DateFormat)), LogType.Info);
                var res = (HttpWebResponse)wr.GetResponse();

                var html = new HtmlAgilityPack.HtmlDocument();
                html.Load(res.GetResponseStream());

                var divs = html.DocumentNode.Descendants("div").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value.EndsWith("b-lenta-film_one"));
                foreach (var div in divs)
                {
                    var show = new Show();
                    var titleA = div.Descendants("a").First(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "b-film_heading");
                    show.Title = titleA.InnerText;
                    show.Channel = "RT DOC";

                    var groupA = div.Descendants("div").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "b-film_group");
                    if (groupA != null)
                    {
                        var groupText = groupA.Descendants("a").First().InnerText;
                        show.Description = show.Title;
                        show.Title = groupText;
                    }
                    show.StartTime = DateTime.SpecifyKind(date.AddMinutes(Convert.ToInt32(div.Attributes["class"].Value.Split(' ')[1].Substring(5))), DateTimeKind.Unspecified);
                    show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Russian Standard Time"), TimeZoneInfo.Utc);
                    shows.Add(show);
                }
            }
            FixShowsEndTimeByStartTime(shows);
            return shows;
        }
Пример #6
0
        public override List<Show> Grab(string xmlParameters, ILogger logger)
        {
            var shows = new List<Show>();

            foreach (Channel c in Enum.GetValues(typeof(Channel)))
            {
                var d = DateTime.Now.Date;
                var wr = WebRequest.Create(string.Format(urlFormat, d.ToString("dd/MM/yyyy"), (int)c));
                var res = (HttpWebResponse)wr.GetResponse();
                var doc = new HtmlAgilityPack.HtmlDocument();
                doc.Load(res.GetResponseStream(),Encoding.UTF8);

                var tbl = doc.DocumentNode.Descendants("table").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "omega_mt1 table1");
                var lst = new List<Show>();
                DateTime? lastStartTime = null;
                foreach (var tr in tbl.Descendants("tr").Where(x => x.Attributes["class"].Value == "bg3" || x.Attributes["class"].Value == "bg5"))
                {
                    var show = new Show();
                    var gmtTime = Tools.CleanupText(tr.Descendants("td").ToList()[2].InnerText);
                    var startTime = Convert.ToDateTime(gmtTime);
                    show.StartTime = DateTime.SpecifyKind(d.Date + Convert.ToDateTime(startTime).TimeOfDay, DateTimeKind.Unspecified);
                    show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("GMT Standard Time"), TimeZoneInfo.Utc);
                    if (lastStartTime.HasValue && lastStartTime.Value > show.StartTime)
                    {
                        d = d.AddDays(1);
                        show.StartTime = show.StartTime.AddDays(1);
                    }
                    lastStartTime = show.StartTime;
                    show.Title = Tools.CleanupText(tr.Descendants("td").ToList()[3].ChildNodes[1].InnerText);
                    show.Description = Tools.CleanupText(tr.Descendants("td").ToList()[3].ChildNodes[2].InnerText);
                    show.Channel = c.ToString();
                    lst.Add(show);
                }
                FixShowsEndTimeByStartTime(lst);
                shows.AddRange(lst);
            }
            return shows;
        }
Пример #7
0
 private void FillShows(XDocument xml, List<Show> shows)
 {
     var body = xml.Element("span").Element("html").Element("body");
     var table = body.Element("table");
     bool firstRow = true;
     foreach (var row in table.Elements("tr"))
     {
         if (firstRow)
         {
             firstRow = false;
             continue;
         }
         var show = new Show();
         var cells = row.Elements("td").ToList();
         show.Title = cells[0].Value;
         show.Description = cells[1].Value;
         show.StartTime = Convert.ToDateTime(cells[5].Value);
         show.StartTime += Convert.ToDateTime(cells[3].Value).TimeOfDay;
         show.StartTime = DateTime.SpecifyKind(show.StartTime, DateTimeKind.Unspecified);
         show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Arabic Standard Time"), TimeZoneInfo.Utc);
         shows.Add(show);
     }
 }
Пример #8
0
        public List<Show> Grab(GrabParametersBase p, ILogger logger)
        {
            currentNow = TimeZoneInfo.ConvertTime(DateTime.Now, TimeZoneInfo.Local, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"));

            var pp = (GrabParameters)p;
            logger.WriteEntry("grabbing CyfrowyPolsat.pl channel : " + pp.Channel, LogType.Info);
            var wr = (HttpWebRequest)WebRequest.Create(string.Format(URL, pp.Channel.ToString().Replace("_", "-")));
            wr.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0";
            wr.Timeout = 5000;
            var res = (HttpWebResponse)wr.GetResponse();
            var doc = new HtmlAgilityPack.HtmlDocument();
            doc.Load(res.GetResponseStream());
            var lst = new List<Show>();
            var div = doc.DocumentNode.Descendants("div").FirstOrDefault(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "main col");
            var times = div.Descendants("span").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "time").ToList();
            var names = div.Descendants("a").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "name").ToList();
            var metas = div.Descendants("div").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "meta").ToList();

            for (int i = 0; i < times.Count; i++)
            {
                var s = new Show();
                s.Channel = pp.Channel.ToString();
                var startTime = Convert.ToDateTime(times[i].InnerText);
                s.StartTime = DateTime.SpecifyKind(currentNow.Date + Convert.ToDateTime(startTime).TimeOfDay, DateTimeKind.Unspecified);
                s.StartTime = TimeZoneInfo.ConvertTime(s.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
                if (lst.Count > 0 && s.StartTime <= lst[lst.Count - 1].StartTime)
                {
                    s.StartTime = s.StartTime.AddDays(1);
                    currentNow = currentNow.AddDays(1);
                }
                s.Title = HttpUtility.HtmlDecode(names[i].InnerText);
                s.Description = HttpUtility.HtmlDecode(metas[i * 2].InnerText + "\n" + metas[(i * 2) + 1].InnerText).Trim();
                lst.Add(s);
            }
            return lst;
        }
Пример #9
0
        List<Show> Grab(GrabParametersBase p,ILogger logger)
        {
            var pp = (CyfraPlus.GrabParameters)p;
            var shows = new List<Show>();
            var wr = WebRequest.Create(string.Format(urlFormat, pp.Date.ToString(DateFormat)));
            logger.WriteEntry(string.Format("Grabbing Cyfra+ date {0} ...", pp.Date.ToString(DateFormat)), LogType.Info);
            var res = (HttpWebResponse)wr.GetResponse();
            const int ChannelDepth = 2;

            using (var sr = new StreamReader(res.GetResponseStream()))
            {
                var startDownloadTime = DateTime.Now;
                var data = new StringBuilder();
                int blockSize = 16384;
                while (!sr.EndOfStream)
                {
                    var buf = new char[blockSize];
                    var totalRead = sr.ReadBlock(buf, 0, blockSize);
                    data.Append(buf);
                    if (DateTime.Now - startDownloadTime > TimeSpan.FromSeconds(1))
                    {
                        startDownloadTime = DateTime.Now;
                        logger.WriteEntry(string.Format("Downloaded {0:#,##0} bytes so far", data.Length), LogType.Info);
                    }
                }

                var r = new Newtonsoft.Json.JsonTextReader(new StringReader(data.ToString()));

                while (r.Read())
                {
                    r.Read();
                    var channelNumber = r.ReadAsInt32();
                    var channelName = r.ReadAsString();
                    r.Read();
                    r.Read();
                    while (r.Depth > ChannelDepth)
                    {
                        var show = new Show();
                        show.Channel = channelName.Trim();
                        var programId = r.ReadAsInt32();
                        show.Title = Tools.CleanupText(r.ReadAsString());

                        show.StartTime = new DateTime(1970, 1, 1).Add(TimeSpan.FromSeconds(r.ReadAsInt32().Value));
                        show.EndTime = show.StartTime.Add(TimeSpan.FromSeconds(Convert.ToDouble(r.ReadAsInt32())));
                        var num = r.ReadAsInt32();
                        shows.Add(show);
                        var depth = r.Depth;
                        while (r.Depth == depth)
                            r.Read();
                        r.Read();
                    }
                }

            }
            return shows;
        }
Пример #10
0
 List<Show> Grab(GrabParametersBase p, ILogger logger)
 {
     var pp = (GrabParameters)p;
     var url = string.Format(URL, pp.Channel.ToString().Replace("_", "-").Replace("AANNDD", "%26").Replace("PPLLUUSS", "%2B"));
     var wr = WebRequest.Create(url);
     var res = (HttpWebResponse)wr.GetResponse();
     var doc = new HtmlAgilityPack.HtmlDocument();
     logger.WriteEntry(string.Format("Grabbing Channel {0}", pp.Channel), LogType.Info);
     doc.Load(res.GetResponseStream());
     var shows = new List<Show>();
     foreach (Day d in Enum.GetValues(typeof(Day)))
     {
         var dayOfWeek = (DayOfWeek)d;
         var div = doc.DocumentNode.Descendants("div").FirstOrDefault(x => x.Attributes.Contains("id") && x.Attributes["id"].Value == d.ToString());
         if (div != null)
         {
             var date = NextDateOfDayOfWeek(dayOfWeek);
             foreach (var ul in div.Descendants("ul"))
             {
                 foreach (var li in ul.Descendants("li"))
                 {
                     var par = li.Descendants("p").First();
                     var a = li.Descendants("a").First();
                     var show = new Show();
                     show.Channel = pp.Channel.ToString();
                     show.Title = a.InnerText.Trim();
                     show.StartTime = DateTime.SpecifyKind(date + Convert.ToDateTime(par.InnerText.Trim()).TimeOfDay, DateTimeKind.Unspecified);
                     show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"), TimeZoneInfo.Utc);
                     shows.Add(show);
                 }
             }
         }
     }
     return shows;
 }
Пример #11
0
        List<Show> Grab(GrabParametersBase p)
        {
            var pp = (GrabParameters)p;

            var url = GetUrl(pp);
            _logger.WriteEntry(string.Format("Grabbing rashut2 {0} for date {1}", pp.Channel, pp.Date.ToString("d")), LogType.Info);
            var wr = WebRequest.Create(string.Format(url, pp.Date.ToString(DateFormat)));
            var res = (HttpWebResponse)wr.GetResponse();
            var doc = new HtmlAgilityPack.HtmlDocument();
            doc.Load(res.GetResponseStream());

            var nodes = doc.DocumentNode.SelectNodes("//comment()");
            if (nodes != null)
            {
                foreach (HtmlAgilityPack.HtmlNode comment in nodes)
                {
                    if (!comment.InnerText.StartsWith("DOCTYPE"))
                        comment.ParentNode.RemoveChild(comment);
                }
            }

            int cellDelta = 0;
            var tbl = doc.GetElementbyId("table5");
            if (pp.Channel == Channel.Channel10)
            {
                tbl = doc.GetElementbyId("table1");
                cellDelta = 1;
            }

            const int childStart = 9;
            var shows = new List<Show>();
            for (int i = childStart; i < tbl.ChildNodes.Count; i += 2)
            {
                var show = new Show();
                var d = Convert.ToDateTime(Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[1].InnerText));
                show.StartTime = pp.Date.AddHours(d.Hour).AddMinutes(d.Minute);
                if (d.Hour < 6) // data is shown from 6 (AM) till next day 6 (AM) so after midnight we need to increase the date
                    show.StartTime = show.StartTime.AddDays(1);
                try
                {
                    show.StartTime = TimeZoneInfo.ConvertTimeToUtc(show.StartTime, TimeZoneInfo.Local);
                }
                catch (Exception ex) // error on the verge of daylight saving start
                {
                    _logger.WriteEntry(ex.Message, LogType.Error);
                    continue;
                }

                show.Title = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[3 + cellDelta].InnerText);
                var episodeName = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[5 + cellDelta].InnerText);
                var episodeNumber = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[7 + cellDelta].InnerText);
                var genere = Tools.CleanupText(tbl.ChildNodes[i].ChildNodes[9 + cellDelta].InnerText);
                show.Description = string.Empty;
                if (!string.IsNullOrEmpty(episodeName))
                    show.Description += string.Format("שם הפרק : {0}\n", episodeName);
                if (!string.IsNullOrEmpty(episodeNumber))
                {
                    show.Description += string.Format("מספר הפרק : {0}\n", episodeNumber);
                    int num;
                    if (int.TryParse(episodeNumber, out num))
                        show.Episode = num;
                }
                if (!string.IsNullOrEmpty(genere))
                    show.Description += string.Format("סוג תכנית : {0}\n", genere);
                shows.Add(show);
            }

            for (int i = shows.Count - 1; i >= 0; i--)
            {
                var show = shows[i];
                if (show.Description != null)
                    show.Description = show.Description.Trim();
                show.Channel = pp.Channel.ToString();
                if (i == shows.Count - 1)
                    show.EndTime = show.StartTime.AddHours(1);
                else
                    show.EndTime = shows[i + 1].StartTime;
            }

            return shows;
        }