Пример #1
0
        public void ScrapeTwitter(Feed specificFeed)
        {
            var consumerKey    = Util.GetSetting("TwitterOAuthConsumerKey", "");
            var consumerSecret = Util.GetSetting("TwitterOAuthConsumerSecret", "");

            FeedList feeds;

            if (specificFeed == null)
            {
                feeds = FeedList.LoadByFeedType("Twitter").Where(f => f.IsActive).ToList();
            }
            else
            {
                feeds = new FeedList();
                feeds.Add(specificFeed);
            }

            var    twitter      = new TwitterAPI(consumerKey, consumerSecret);
            var    runResult    = "";
            var    feedIsActive = true;
            string jsonString   = "";

            foreach (var feed in feeds)
            {
                try {
                    jsonString = twitter.GetJson(feed.FeedUrl);
                    Web.Write("<br><br><br>" + jsonString);
                    dynamic json = JsonHelper.Parse(jsonString);
                    dynamic tweets;
                    if (jsonString.Trim().StartsWith("["))
                    {
                        // json format is an array of tweets
                        tweets = json;
                    }
                    else
                    {
                        // json format is an object containing a list of tweets as 'statuses' property
                        tweets = json.statuses;
                    }

                    const string twitterDateTimeFormat = "ddd MMM dd HH:mm:ss zzzz yyyy";

                    foreach (var tweet in tweets)
                    {
                        var createdAt = DateTime.ParseExact(tweet.created_at, twitterDateTimeFormat, CultureInfo.InvariantCulture);
                        if (createdAt < DateTime.Now.AddDays(-1))
                        {
                            continue;
                        }

                        var uniqueStringValue = "twitter:" + tweet.id_str;

                        var story = FeedStory.LoadByUniqueString(uniqueStringValue);

                        if (story == null)
                        {
                            story = new FeedStory();
                            story.UniqueString    = uniqueStringValue;
                            story.AuthorName      = tweet.user.name + " via Twitter";
                            story.ProfileImageUrl = tweet.user.profile_image_url;
                            story.ImageUrl        = tweet.entities.media != null ? tweet.entities.media[0].media_url : null;
                            story.Message         = tweet.text;
                            story.FeedID          = feed.FeedID;
                            story.Status          = "new";
                            story.Priority        = feed.DefaultPriority;

                            // get location from tweet if supplied in "geo"
                            // note: weirdly it seems that these checks on dynamic objects cannot put in a single if statement together, maybe the short cut logic does not work the same
                            if (tweet.geo != null)
                            {
                                if (tweet.geo.type == "Point")
                                {
                                    if (tweet.geo.coordinates != null)
                                    {
                                        if (tweet.geo.coordinates.Length == 2)
                                        {
                                            story.Latitude  = tweet.geo.coordinates[0];
                                            story.Longitude = tweet.geo.coordinates[1];
                                        }
                                    }
                                }
                            }

                            story.Save();
                            story.ApplyFilters();

                            // If it didn't match any filter and it's still null, get from the feed
                            if (story.Latitude == null || story.Longitude == null)
                            {
                                story.Latitude  = feed.Latitude;
                                story.Longitude = feed.Longitude;
                                story.Save();
                            }

                            Web.Write("<p><strong>Added new tweet as feed story (Twitter Feed ID: " + feed.FeedID + ")</strong></p><p>" + story.Message + "</p><hr/>");
                        }
                    }

                    runResult = "Success";
                } catch (Exception ex) {
                    Web.Write("<p><strong style='color:red'>Error! Twitter Feed ID: " + feed.FeedID + "</strong></p><p>" + ex.Message + "</p><hr/>");
                    Web.Write(jsonString);
                    runResult = ex.Message;
                    //SendFeedErrorEmail(feed, ex);
                    //feedIsActive = false;
                } finally {
                    feed.LastResult      = runResult;
                    feed.LastTimeRunning = DateTime.Now;
                    feed.IsActive        = feedIsActive;
                    feed.Save();
                }
            }
        }
Пример #2
0
        public void ScrapeRss(Feed specificFeed)
        {
            FeedList feeds;

            if (specificFeed == null)
            {
                feeds = FeedList.LoadByFeedType("RSS").Where(f => f.IsActive).ToList();
            }
            else
            {
                feeds = new FeedList();
                feeds.Add(specificFeed);
            }

            var runResult    = "";
            var xmlString    = "";
            var feedIsActive = true;

            foreach (var feed in feeds)
            {
                try {
                    string url = feed.FeedUrl;
                    xmlString = Http.Get(url);
                    //Web.WriteLine(xmlString);
                    //string profileImageUrl = "https://pbs.twimg.com/profile_images/2461203629/2kvvznykuqzo46kj6kec_normal.jpeg";
                    //string currentUserToken = "52e0eaf6e4b07e1b6390a5dc";  //mikes

                    var xml = bwbXml.Parse(xmlString);
                    foreach (var node in xml.Descendants("item"))
                    {
                        var uniqueStringValue = "rss:" + Crypto.CreateHash(node.ElementValue("guid") ?? node.ElementValue("link"));
                        var story             = FeedStory.LoadByUniqueString(uniqueStringValue);

                        if (story == null)
                        {
                            string link = node.ElementValue("link");

                            if (String.IsNullOrEmpty(link))
                            {
                                link = ExtractUrlFromHTML(node.ElementValue("description"));
                            }

                            var shortUrl = "";

                            if (!String.IsNullOrEmpty(link))
                            {
                                shortUrl = UrlShortenerController.UrlShortener.Create(link, "RSS Feed", true);
                            }

                            //string message = Fmt.TruncHTML(node.ElementValue("title").StripTags() + ". " + node.ElementValue("description").StripTags(), 130) + " " + shortUrl;

                            string message = node.ElementValue("title").StripTags() + ". " + node.ElementValue("description").StripTags();
                            message = (message.Length > 130 ? Fmt.TruncHTML(message, 130) + "..." : message) + " " + shortUrl;

                            //var imageUrl = node.ElementValue("enclosure");
                            story = new FeedStory();
                            story.UniqueString    = uniqueStringValue;
                            story.Title           = node.ElementValue("title");
                            story.Message         = message;
                            story.OriginalLinkUrl = node.ElementValue("link");
                            story.DateAdded       = node.ElementValue("pub-date").ConvertToDate(DateTime.Now);
                            story.Status          = "new";
                            story.AuthorName      = feed.Author.Name + (feed.Author.Via.IsNotBlank() ? " via " + feed.Author.Via : "");
                            story.FeedID          = feed.FeedID;
                            story.ProfileImageUrl = feed.Author.ProfilePicUrl;
                            story.Priority        = feed.DefaultPriority;
                            story.Save();
                            story.ApplyFilters();

                            // If it didn't match any filter and it's still null, get from the feed
                            if (story.Latitude == null || story.Longitude == null)
                            {
                                story.Latitude  = feed.Latitude;
                                story.Longitude = feed.Longitude;
                                story.Save();
                            }

                            Web.Write("<p><strong>Added new item as feed story (RSS Feed ID: " + feed.FeedID + ")</strong></p><p>" + message + "</p><hr/>");
                        }
                    }

                    runResult = "Success";
                } catch (Exception ex) {
                    Web.Write("<p><strong style='color:red'>Error! RSS Feed ID: " + feed.FeedID + "</strong></p><p>" + ex.Message + "</p><hr/>");
                    runResult = ex.Message;
                    //SendFeedErrorEmail(feed, ex);
                    //feedIsActive = false;
                    //SendEMail.SimpleSendHtmlEmail("*****@*****.**", "Herepin Feed error", "RSS Feed ID: " + feed.FeedID + "<br>"+ex.Message+"<br><br>" + xmlString);
                } finally {
                    feed.LastResult      = runResult;
                    feed.LastTimeRunning = DateTime.Now;
                    feed.IsActive        = feedIsActive;
                    feed.Save();
                }
            }
        }
Пример #3
0
        public void ScrapeEventFinder(Feed specificFeed)
        {
            FeedList feeds;

            if (specificFeed == null)
            {
                feeds = FeedList.LoadByFeedType("Eventfinder").ToList();
            }
            else
            {
                feeds = new FeedList();
                feeds.Add(specificFeed);
            }

            var runResult    = "";
            var feedIsActive = true;

            foreach (var feed in feeds)
            {
                try {
                    string    username = "******";
                    string    password = "******";
                    WebClient wc       = new WebClient();
                    wc.Credentials = new NetworkCredential(username, password);
                    string url = "http://api.eventfinder.co.nz/v2/events.json?rows=20";
                    url += "&fields=event:(id,url,name,description~200,datetime_summary,point,location_summary,address,username,images),images:(image),image:(transforms),transforms:(transform),transform:(url,width)";
                    //url += "&order=popularity";
                    //url += "&created_since=" + Fmt.DateISO(DateTime.UtcNow.Date);
                    //url += "&end_date=" + Fmt.DateISO(DateTime.UtcNow.Date.AddDays(14));
                    //url += "&point=-36.796581,174.777978&radius=100";
                    url += "&point=" + feed.Latitude + "," + feed.Longitude + "&radius=" + feed.Radius;
                    //url += "&free=1";
                    var jsonString = wc.DownloadString(url);
                    //Web.WriteLine(jsonString);
                    //string currentUserToken = "52e0eaf6e4b07e1b6390a5dc";  //mikes

                    dynamic json = JsonHelper.Parse(jsonString);

                    foreach (var obj in json.events)
                    {
                        var uniqueStringValue = "eventfinda:" + obj.id;
                        var story             = FeedStory.LoadByUniqueString(uniqueStringValue);

                        if (story == null)
                        {
                            string currentUserName = obj.username + " via Eventfinder";
                            var    shortUrl        = UrlShortenerController.UrlShortener.Create(obj.url, currentUserName, true);

                            string message = ((String)obj.name).StripTags() + " " + obj.datetime_summary + ". " + ((String)obj.description).StripTags();
                            message = (message.Length > 130 ? Fmt.TruncHTML(message, 130) + "..." : message) + " #events " + shortUrl;
                            //var imageUrl = node.ElementValue("enclosure");
                            story = new FeedStory();
                            story.UniqueString = uniqueStringValue;
                            //story.Title = node.ElementValue("title");
                            story.ImageUrl        = GetImage(obj);
                            story.Message         = message;
                            story.OriginalLinkUrl = obj.url;
                            //story.DateAdded = node.ElementValue("pub-date").ConvertToDate(DateTime.Now);
                            story.Status          = "new";
                            story.AuthorName      = currentUserName;
                            story.FeedID          = feed.FeedID;
                            story.ProfileImageUrl = "https://pbs.twimg.com/profile_images/2461203629/2kvvznykuqzo46kj6kec_normal.jpeg";
                            story.Latitude        = (decimal)obj.point.lat;
                            story.Longitude       = (decimal)obj.point.lng;
                            story.Priority        = feed.DefaultPriority;
                            story.Save();
                            story.ApplyFilters();

                            Web.Write("<p><strong>Added new event as feed story (Feed ID: " + feed.FeedID + ")</strong></p><p>" + message + "</p><hr/>");
                        }
                    }

                    runResult = "Success";
                } catch (Exception ex) {
                    Web.Write("<p><strong style='color:red'>Error! Event Finder Feed ID: " + feed.FeedID + "</strong></p><p>" + ex.Message + "</p><hr/>");
                    runResult = ex.Message;
                    //SendFeedErrorEmail(feed, ex);
                    //feedIsActive = false;
                } finally {
                    feed.LastResult      = runResult;
                    feed.LastTimeRunning = DateTime.Now;
                    feed.IsActive        = feedIsActive;
                    feed.Save();
                }
            }
        }