예제 #1
0
        public void ScrapeRss(Feed specificFeed)
        {
            FeedList feeds;

            if (specificFeed == null)
            {
                feeds = FeedList.LoadByFeedType("RSS").Where(f => f.IsActive).ToList();
            }
            else
            {
                feeds = new FeedList();
                feeds.Add(specificFeed);
            }

            var runResult    = "";
            var xmlString    = "";
            var feedIsActive = true;

            foreach (var feed in feeds)
            {
                try {
                    string url = feed.FeedUrl;
                    xmlString = Http.Get(url);
                    //Web.WriteLine(xmlString);
                    //string profileImageUrl = "https://pbs.twimg.com/profile_images/2461203629/2kvvznykuqzo46kj6kec_normal.jpeg";
                    //string currentUserToken = "52e0eaf6e4b07e1b6390a5dc";  //mikes

                    var xml = bwbXml.Parse(xmlString);
                    foreach (var node in xml.Descendants("item"))
                    {
                        var uniqueStringValue = "rss:" + Crypto.CreateHash(node.ElementValue("guid") ?? node.ElementValue("link"));
                        var story             = FeedStory.LoadByUniqueString(uniqueStringValue);

                        if (story == null)
                        {
                            string link = node.ElementValue("link");

                            if (String.IsNullOrEmpty(link))
                            {
                                link = ExtractUrlFromHTML(node.ElementValue("description"));
                            }

                            var shortUrl = "";

                            if (!String.IsNullOrEmpty(link))
                            {
                                shortUrl = UrlShortenerController.UrlShortener.Create(link, "RSS Feed", true);
                            }

                            //string message = Fmt.TruncHTML(node.ElementValue("title").StripTags() + ". " + node.ElementValue("description").StripTags(), 130) + " " + shortUrl;

                            string message = node.ElementValue("title").StripTags() + ". " + node.ElementValue("description").StripTags();
                            message = (message.Length > 130 ? Fmt.TruncHTML(message, 130) + "..." : message) + " " + shortUrl;

                            //var imageUrl = node.ElementValue("enclosure");
                            story = new FeedStory();
                            story.UniqueString    = uniqueStringValue;
                            story.Title           = node.ElementValue("title");
                            story.Message         = message;
                            story.OriginalLinkUrl = node.ElementValue("link");
                            story.DateAdded       = node.ElementValue("pub-date").ConvertToDate(DateTime.Now);
                            story.Status          = "new";
                            story.AuthorName      = feed.Author.Name + (feed.Author.Via.IsNotBlank() ? " via " + feed.Author.Via : "");
                            story.FeedID          = feed.FeedID;
                            story.ProfileImageUrl = feed.Author.ProfilePicUrl;
                            story.Priority        = feed.DefaultPriority;
                            story.Save();
                            story.ApplyFilters();

                            // If it didn't match any filter and it's still null, get from the feed
                            if (story.Latitude == null || story.Longitude == null)
                            {
                                story.Latitude  = feed.Latitude;
                                story.Longitude = feed.Longitude;
                                story.Save();
                            }

                            Web.Write("<p><strong>Added new item as feed story (RSS Feed ID: " + feed.FeedID + ")</strong></p><p>" + message + "</p><hr/>");
                        }
                    }

                    runResult = "Success";
                } catch (Exception ex) {
                    Web.Write("<p><strong style='color:red'>Error! RSS Feed ID: " + feed.FeedID + "</strong></p><p>" + ex.Message + "</p><hr/>");
                    runResult = ex.Message;
                    //SendFeedErrorEmail(feed, ex);
                    //feedIsActive = false;
                    //SendEMail.SimpleSendHtmlEmail("*****@*****.**", "Herepin Feed error", "RSS Feed ID: " + feed.FeedID + "<br>"+ex.Message+"<br><br>" + xmlString);
                } finally {
                    feed.LastResult      = runResult;
                    feed.LastTimeRunning = DateTime.Now;
                    feed.IsActive        = feedIsActive;
                    feed.Save();
                }
            }
        }
예제 #2
0
        public void ScrapeEventFinder(Feed specificFeed)
        {
            FeedList feeds;

            if (specificFeed == null)
            {
                feeds = FeedList.LoadByFeedType("Eventfinder").ToList();
            }
            else
            {
                feeds = new FeedList();
                feeds.Add(specificFeed);
            }

            var runResult    = "";
            var feedIsActive = true;

            foreach (var feed in feeds)
            {
                try {
                    string    username = "******";
                    string    password = "******";
                    WebClient wc       = new WebClient();
                    wc.Credentials = new NetworkCredential(username, password);
                    string url = "http://api.eventfinder.co.nz/v2/events.json?rows=20";
                    url += "&fields=event:(id,url,name,description~200,datetime_summary,point,location_summary,address,username,images),images:(image),image:(transforms),transforms:(transform),transform:(url,width)";
                    //url += "&order=popularity";
                    //url += "&created_since=" + Fmt.DateISO(DateTime.UtcNow.Date);
                    //url += "&end_date=" + Fmt.DateISO(DateTime.UtcNow.Date.AddDays(14));
                    //url += "&point=-36.796581,174.777978&radius=100";
                    url += "&point=" + feed.Latitude + "," + feed.Longitude + "&radius=" + feed.Radius;
                    //url += "&free=1";
                    var jsonString = wc.DownloadString(url);
                    //Web.WriteLine(jsonString);
                    //string currentUserToken = "52e0eaf6e4b07e1b6390a5dc";  //mikes

                    dynamic json = JsonHelper.Parse(jsonString);

                    foreach (var obj in json.events)
                    {
                        var uniqueStringValue = "eventfinda:" + obj.id;
                        var story             = FeedStory.LoadByUniqueString(uniqueStringValue);

                        if (story == null)
                        {
                            string currentUserName = obj.username + " via Eventfinder";
                            var    shortUrl        = UrlShortenerController.UrlShortener.Create(obj.url, currentUserName, true);

                            string message = ((String)obj.name).StripTags() + " " + obj.datetime_summary + ". " + ((String)obj.description).StripTags();
                            message = (message.Length > 130 ? Fmt.TruncHTML(message, 130) + "..." : message) + " #events " + shortUrl;
                            //var imageUrl = node.ElementValue("enclosure");
                            story = new FeedStory();
                            story.UniqueString = uniqueStringValue;
                            //story.Title = node.ElementValue("title");
                            story.ImageUrl        = GetImage(obj);
                            story.Message         = message;
                            story.OriginalLinkUrl = obj.url;
                            //story.DateAdded = node.ElementValue("pub-date").ConvertToDate(DateTime.Now);
                            story.Status          = "new";
                            story.AuthorName      = currentUserName;
                            story.FeedID          = feed.FeedID;
                            story.ProfileImageUrl = "https://pbs.twimg.com/profile_images/2461203629/2kvvznykuqzo46kj6kec_normal.jpeg";
                            story.Latitude        = (decimal)obj.point.lat;
                            story.Longitude       = (decimal)obj.point.lng;
                            story.Priority        = feed.DefaultPriority;
                            story.Save();
                            story.ApplyFilters();

                            Web.Write("<p><strong>Added new event as feed story (Feed ID: " + feed.FeedID + ")</strong></p><p>" + message + "</p><hr/>");
                        }
                    }

                    runResult = "Success";
                } catch (Exception ex) {
                    Web.Write("<p><strong style='color:red'>Error! Event Finder Feed ID: " + feed.FeedID + "</strong></p><p>" + ex.Message + "</p><hr/>");
                    runResult = ex.Message;
                    //SendFeedErrorEmail(feed, ex);
                    //feedIsActive = false;
                } finally {
                    feed.LastResult      = runResult;
                    feed.LastTimeRunning = DateTime.Now;
                    feed.IsActive        = feedIsActive;
                    feed.Save();
                }
            }
        }