public override void Publish(IEnumerator <Post> posts, int count) { // git all hashtags from all posts to build recent/popular lists List <string> recentHashtags = null; Dictionary <string, int> popularHashtags = null; if (Config["hashtags"] == "true") { recentHashtags = new List <string>(); popularHashtags = new Dictionary <string, int>(); while (posts.MoveNext()) { var matches = _hashtags.Matches(posts.Current.Content); foreach (Match match in matches) { var hashtag = match.Value.ToLower(); if (!recentHashtags.Contains(hashtag)) { recentHashtags.Add(hashtag); } if (!popularHashtags.ContainsKey(hashtag)) { popularHashtags.Add(hashtag, 0); } popularHashtags[hashtag] += 1; } } } posts.Reset(); var invalidations = new List <string> { "/" }; var pagePosts = new List <ExtendedPost>(); var hashtags = new Dictionary <string, List <ExtendedPost> >(); var data = new HandlebarsPageData { Config = Config, TotalPosts = count, MaxPage = (int)Math.Ceiling((float)count / _postsPerPage) - 1, PageNum = 0, Prefix = "index", RecentTags = recentHashtags.Take(10), PopularTags = popularHashtags != null ? popularHashtags.OrderByDescending(t => t.Value).Take(10).Select(t => t.Key).ToArray() : null }; while (posts.MoveNext()) { var clone = new ExtendedPost((Post)posts.Current.Clone()) { Config = Config }; if (Config["hashtags"] == "true") { Hashtagify(hashtags, clone); } data.Post = clone; invalidations.Add(SavePost(data)); pagePosts.Add(clone); if (pagePosts.Count == _postsPerPage) { data.Posts = pagePosts.ToArray(); invalidations.Add(SaveIndex(data)); if (data.PageNum == 0) { invalidations.Add(SaveFeed(data)); } data.PageNum += 1; pagePosts.Clear(); } } if (pagePosts.Count > 0) { data.Posts = pagePosts.ToArray(); invalidations.Add(SaveIndex(data)); if (data.PageNum == 0) { invalidations.Add(SaveFeed(data)); } } foreach (var hashtag in hashtags.Keys) { var hashData = new HandlebarsPageData { Config = Config, TotalPosts = hashtags[hashtag].Count, MaxPage = (int)Math.Ceiling((float)hashtags[hashtag].Count / _postsPerPage) - 1, PageNum = 0, Prefix = hashtag, RecentTags = recentHashtags.Take(10), PopularTags = popularHashtags != null ? popularHashtags.OrderByDescending(t => t.Value).Take(10).Select(t => t.Key).ToArray() : null }; pagePosts.Clear(); foreach (var post in hashtags[hashtag]) { pagePosts.Add(post); if (pagePosts.Count == _postsPerPage) { hashData.Posts = pagePosts.ToArray(); invalidations.Add(SaveIndex(hashData)); hashData.PageNum += 1; pagePosts.Clear(); } } if (pagePosts.Count > 0) { hashData.Posts = pagePosts.ToArray(); invalidations.Add(SaveIndex(hashData)); } } var invalidPaths = invalidations.Where(i => !string.IsNullOrEmpty(i)).ToList(); invalidPaths.AddRange(UploadDirectory(_assetPath, "assets")); Log.Debug("S3 upload complete"); if (invalidPaths.Count > 0) { var distId = Config["cloudfrontDistId"]; if (distId != null) { _client.InvalidateCloudFrontDistribution(distId, invalidPaths); } } }
public override void Process(Post post) { // download linked url, look for og:image meta tag var urls = PostUtils.UrlRegex.Matches(post.Content); foreach (Match murl in urls) { var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(murl.Value)); var cachedImgUrl = App.CacheGet(url); if (cachedImgUrl != null) { if (!cachedImgUrl.Equals(":none")) { Log.Debug(string.Format("Found cached image at {0}", cachedImgUrl)); post.Attach("image", url, cachedImgUrl); } break; } Log.Debug(string.Format("Checking {0} for opengraph image", url)); var properties = new Dictionary <string, string>(); var content = PostUtils.GetPageContent(Log, url); var metas = _ogimage.Matches(content); foreach (Match meta in metas) { string prop = null; string val = null; var mprop = _metaname.Match(meta.Value); if (mprop.Success) { prop = WebUtility.HtmlDecode(mprop.Groups["property"].Value.ToLower()); } var mcont = _metacontent.Match(meta.Value); if (mcont.Success) { val = WebUtility.HtmlDecode(mcont.Groups["content"].Value); } if (!string.IsNullOrEmpty(prop) && !string.IsNullOrEmpty(val) && !properties.ContainsKey(prop)) { properties.Add(prop, val); } } if (properties.Count == 0 || !properties.ContainsKey("og:image")) { App.CacheSet(url, ":none"); continue; } string imgurl = null; Log.Debug("Got image meta properties", properties); // if it's twitter we only want user generated images if (!_twitter.IsMatch(url) || (properties.ContainsKey("og:image:user_generated") && properties["og:image:user_generated"] == "true")) { imgurl = properties["og:image"]; } if (string.IsNullOrWhiteSpace(imgurl)) { App.CacheSet(url, ":none"); continue; } try { var uri = new Uri(imgurl); var fname = Path.GetFileName(uri.AbsolutePath); var wc = new WebClient(); var tpath = Path.Combine(Config["tempDir"], fname); Log.Debug(string.Format("Downloading image {0} to {1}", imgurl, tpath)); wc.DownloadFile(imgurl, tpath); using (var bitmap = Image.FromFile(tpath)) { if (bitmap.Width < _minWidth || bitmap.Height < _minHeight) { Log.Debug(string.Format("Ignoring image {0}, too small ({1}x{2})", fname, bitmap.Width, bitmap.Height)); App.CacheSet(url, ":none"); File.Delete(tpath); continue; } var type = bitmap.RawFormat; if (!type.Equals(ImageFormat.Jpeg) && !type.Equals(ImageFormat.Gif) && !type.Equals(ImageFormat.Png)) { Log.Debug(string.Format("Ignoring image {0}, unsupported type ({1})", fname, bitmap.RawFormat)); App.CacheSet(url, ":none"); File.Delete(tpath); continue; } if (bitmap.Width > _maxWidth) { using (var resized = ResizeImage(bitmap, _maxWidth, (int)Math.Round(bitmap.Height / (float)bitmap.Width * _maxWidth))) resized.Save(tpath, type); } string extension; if (type.Equals(ImageFormat.Jpeg)) { extension = "jpg"; } else if (type.Equals(ImageFormat.Png)) { extension = "png"; } else { extension = "gif"; } var uploadName = string.Format("{0}/{1}.{2}", Config["s3path"], post.Id, extension); if (_client.UploadFile(Config["bucket"], tpath, uploadName) && Config["cloudfrontDistId"] != null) { _client.InvalidateCloudFrontDistribution(Config["cloudfrontDistId"], new List <string> { string.Format("/{0}", uploadName) }); } // don't bother getting more than one image per post var newImgUrl = string.Format("{0}/{1}", Config["siteUrl"], uploadName); App.CacheSet(url, newImgUrl); post.Attach("image", url, newImgUrl); File.Delete(tpath); } } catch (Exception ex) { Log.Error(string.Format("Error processing image {0}", imgurl), ex); } } }