Ejemplo n.º 1
0
        public override void Publish(IEnumerator <Post> posts, int count)
        {
            // git all hashtags from all posts to build recent/popular lists
            List <string>            recentHashtags  = null;
            Dictionary <string, int> popularHashtags = null;

            if (Config["hashtags"] == "true")
            {
                recentHashtags  = new List <string>();
                popularHashtags = new Dictionary <string, int>();
                while (posts.MoveNext())
                {
                    var matches = _hashtags.Matches(posts.Current.Content);
                    foreach (Match match in matches)
                    {
                        var hashtag = match.Value.ToLower();
                        if (!recentHashtags.Contains(hashtag))
                        {
                            recentHashtags.Add(hashtag);
                        }
                        if (!popularHashtags.ContainsKey(hashtag))
                        {
                            popularHashtags.Add(hashtag, 0);
                        }
                        popularHashtags[hashtag] += 1;
                    }
                }
            }

            posts.Reset();

            var invalidations = new List <string> {
                "/"
            };

            var pagePosts = new List <ExtendedPost>();
            var hashtags  = new Dictionary <string, List <ExtendedPost> >();
            var data      = new HandlebarsPageData
            {
                Config      = Config,
                TotalPosts  = count,
                MaxPage     = (int)Math.Ceiling((float)count / _postsPerPage) - 1,
                PageNum     = 0,
                Prefix      = "index",
                RecentTags  = recentHashtags.Take(10),
                PopularTags = popularHashtags != null
                    ? popularHashtags.OrderByDescending(t => t.Value).Take(10).Select(t => t.Key).ToArray()
                    : null
            };

            while (posts.MoveNext())
            {
                var clone = new ExtendedPost((Post)posts.Current.Clone())
                {
                    Config = Config
                };
                if (Config["hashtags"] == "true")
                {
                    Hashtagify(hashtags, clone);
                }

                data.Post = clone;
                invalidations.Add(SavePost(data));
                pagePosts.Add(clone);
                if (pagePosts.Count == _postsPerPage)
                {
                    data.Posts = pagePosts.ToArray();
                    invalidations.Add(SaveIndex(data));
                    if (data.PageNum == 0)
                    {
                        invalidations.Add(SaveFeed(data));
                    }
                    data.PageNum += 1;
                    pagePosts.Clear();
                }
            }
            if (pagePosts.Count > 0)
            {
                data.Posts = pagePosts.ToArray();
                invalidations.Add(SaveIndex(data));
                if (data.PageNum == 0)
                {
                    invalidations.Add(SaveFeed(data));
                }
            }

            foreach (var hashtag in hashtags.Keys)
            {
                var hashData = new HandlebarsPageData
                {
                    Config      = Config,
                    TotalPosts  = hashtags[hashtag].Count,
                    MaxPage     = (int)Math.Ceiling((float)hashtags[hashtag].Count / _postsPerPage) - 1,
                    PageNum     = 0,
                    Prefix      = hashtag,
                    RecentTags  = recentHashtags.Take(10),
                    PopularTags = popularHashtags != null
                        ? popularHashtags.OrderByDescending(t => t.Value).Take(10).Select(t => t.Key).ToArray()
                        : null
                };
                pagePosts.Clear();
                foreach (var post in hashtags[hashtag])
                {
                    pagePosts.Add(post);
                    if (pagePosts.Count == _postsPerPage)
                    {
                        hashData.Posts = pagePosts.ToArray();
                        invalidations.Add(SaveIndex(hashData));
                        hashData.PageNum += 1;
                        pagePosts.Clear();
                    }
                }
                if (pagePosts.Count > 0)
                {
                    hashData.Posts = pagePosts.ToArray();
                    invalidations.Add(SaveIndex(hashData));
                }
            }

            var invalidPaths = invalidations.Where(i => !string.IsNullOrEmpty(i)).ToList();

            invalidPaths.AddRange(UploadDirectory(_assetPath, "assets"));

            Log.Debug("S3 upload complete");
            if (invalidPaths.Count > 0)
            {
                var distId = Config["cloudfrontDistId"];
                if (distId != null)
                {
                    _client.InvalidateCloudFrontDistribution(distId, invalidPaths);
                }
            }
        }
Ejemplo n.º 2
0
        public override void Process(Post post)
        {
            // download linked url, look for og:image meta tag
            var urls = PostUtils.UrlRegex.Matches(post.Content);

            foreach (Match murl in urls)
            {
                var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(murl.Value));

                var cachedImgUrl = App.CacheGet(url);
                if (cachedImgUrl != null)
                {
                    if (!cachedImgUrl.Equals(":none"))
                    {
                        Log.Debug(string.Format("Found cached image at {0}", cachedImgUrl));
                        post.Attach("image", url, cachedImgUrl);
                    }
                    break;
                }

                Log.Debug(string.Format("Checking {0} for opengraph image", url));

                var properties = new Dictionary <string, string>();
                var content    = PostUtils.GetPageContent(Log, url);
                var metas      = _ogimage.Matches(content);
                foreach (Match meta in metas)
                {
                    string prop = null;
                    string val  = null;

                    var mprop = _metaname.Match(meta.Value);
                    if (mprop.Success)
                    {
                        prop = WebUtility.HtmlDecode(mprop.Groups["property"].Value.ToLower());
                    }
                    var mcont = _metacontent.Match(meta.Value);
                    if (mcont.Success)
                    {
                        val = WebUtility.HtmlDecode(mcont.Groups["content"].Value);
                    }

                    if (!string.IsNullOrEmpty(prop) && !string.IsNullOrEmpty(val) && !properties.ContainsKey(prop))
                    {
                        properties.Add(prop, val);
                    }
                }

                if (properties.Count == 0 || !properties.ContainsKey("og:image"))
                {
                    App.CacheSet(url, ":none");
                    continue;
                }

                string imgurl = null;
                Log.Debug("Got image meta properties", properties);
                // if it's twitter we only want user generated images
                if (!_twitter.IsMatch(url) || (properties.ContainsKey("og:image:user_generated") && properties["og:image:user_generated"] == "true"))
                {
                    imgurl = properties["og:image"];
                }

                if (string.IsNullOrWhiteSpace(imgurl))
                {
                    App.CacheSet(url, ":none");
                    continue;
                }

                try
                {
                    var uri   = new Uri(imgurl);
                    var fname = Path.GetFileName(uri.AbsolutePath);

                    var wc    = new WebClient();
                    var tpath = Path.Combine(Config["tempDir"], fname);

                    Log.Debug(string.Format("Downloading image {0} to {1}", imgurl, tpath));

                    wc.DownloadFile(imgurl, tpath);
                    using (var bitmap = Image.FromFile(tpath))
                    {
                        if (bitmap.Width < _minWidth ||
                            bitmap.Height < _minHeight)
                        {
                            Log.Debug(string.Format("Ignoring image {0}, too small ({1}x{2})", fname, bitmap.Width, bitmap.Height));
                            App.CacheSet(url, ":none");
                            File.Delete(tpath);
                            continue;
                        }

                        var type = bitmap.RawFormat;
                        if (!type.Equals(ImageFormat.Jpeg) &&
                            !type.Equals(ImageFormat.Gif) &&
                            !type.Equals(ImageFormat.Png))
                        {
                            Log.Debug(string.Format("Ignoring image {0}, unsupported type ({1})", fname, bitmap.RawFormat));
                            App.CacheSet(url, ":none");
                            File.Delete(tpath);
                            continue;
                        }

                        if (bitmap.Width > _maxWidth)
                        {
                            using (var resized = ResizeImage(bitmap, _maxWidth, (int)Math.Round(bitmap.Height / (float)bitmap.Width * _maxWidth)))
                                resized.Save(tpath, type);
                        }

                        string extension;
                        if (type.Equals(ImageFormat.Jpeg))
                        {
                            extension = "jpg";
                        }
                        else if (type.Equals(ImageFormat.Png))
                        {
                            extension = "png";
                        }
                        else
                        {
                            extension = "gif";
                        }

                        var uploadName = string.Format("{0}/{1}.{2}", Config["s3path"], post.Id, extension);

                        if (_client.UploadFile(Config["bucket"], tpath, uploadName) && Config["cloudfrontDistId"] != null)
                        {
                            _client.InvalidateCloudFrontDistribution(Config["cloudfrontDistId"], new List <string> {
                                string.Format("/{0}", uploadName)
                            });
                        }
                        // don't bother getting more than one image per post
                        var newImgUrl = string.Format("{0}/{1}", Config["siteUrl"], uploadName);
                        App.CacheSet(url, newImgUrl);
                        post.Attach("image", url, newImgUrl);
                        File.Delete(tpath);
                    }
                }
                catch (Exception ex)
                {
                    Log.Error(string.Format("Error processing image {0}", imgurl), ex);
                }
            }
        }