public override void Publish(IEnumerator <Post> posts, int count)
{
// git all hashtags from all posts to build recent/popular lists
List <string> recentHashtags = null;
Dictionary <string, int> popularHashtags = null;
if (Config["hashtags"] == "true")
{
recentHashtags = new List <string>();
popularHashtags = new Dictionary <string, int>();
while (posts.MoveNext())
{
var matches = _hashtags.Matches(posts.Current.Content);
foreach (Match match in matches)
{
var hashtag = match.Value.ToLower();
if (!recentHashtags.Contains(hashtag))
{
recentHashtags.Add(hashtag);
}
if (!popularHashtags.ContainsKey(hashtag))
{
popularHashtags.Add(hashtag, 0);
}
popularHashtags[hashtag] += 1;
}
}
}
posts.Reset();
var invalidations = new List <string> {
"/"
};
var pagePosts = new List <ExtendedPost>();
var hashtags = new Dictionary <string, List <ExtendedPost> >();
var data = new HandlebarsPageData
{
Config = Config,
TotalPosts = count,
MaxPage = (int)Math.Ceiling((float)count / _postsPerPage) - 1,
PageNum = 0,
Prefix = "index",
RecentTags = recentHashtags.Take(10),
PopularTags = popularHashtags != null
? popularHashtags.OrderByDescending(t => t.Value).Take(10).Select(t => t.Key).ToArray()
: null
};
while (posts.MoveNext())
{
var clone = new ExtendedPost((Post)posts.Current.Clone())
{
Config = Config
};
if (Config["hashtags"] == "true")
{
Hashtagify(hashtags, clone);
}
data.Post = clone;
invalidations.Add(SavePost(data));
pagePosts.Add(clone);
if (pagePosts.Count == _postsPerPage)
{
data.Posts = pagePosts.ToArray();
invalidations.Add(SaveIndex(data));
if (data.PageNum == 0)
{
invalidations.Add(SaveFeed(data));
}
data.PageNum += 1;
pagePosts.Clear();
}
}
if (pagePosts.Count > 0)
{
data.Posts = pagePosts.ToArray();
invalidations.Add(SaveIndex(data));
if (data.PageNum == 0)
{
invalidations.Add(SaveFeed(data));
}
}
foreach (var hashtag in hashtags.Keys)
{
var hashData = new HandlebarsPageData
{
Config = Config,
TotalPosts = hashtags[hashtag].Count,
MaxPage = (int)Math.Ceiling((float)hashtags[hashtag].Count / _postsPerPage) - 1,
PageNum = 0,
Prefix = hashtag,
RecentTags = recentHashtags.Take(10),
PopularTags = popularHashtags != null
? popularHashtags.OrderByDescending(t => t.Value).Take(10).Select(t => t.Key).ToArray()
: null
};
pagePosts.Clear();
foreach (var post in hashtags[hashtag])
{
pagePosts.Add(post);
if (pagePosts.Count == _postsPerPage)
{
hashData.Posts = pagePosts.ToArray();
invalidations.Add(SaveIndex(hashData));
hashData.PageNum += 1;
pagePosts.Clear();
}
}
if (pagePosts.Count > 0)
{
hashData.Posts = pagePosts.ToArray();
invalidations.Add(SaveIndex(hashData));
}
}
var invalidPaths = invalidations.Where(i => !string.IsNullOrEmpty(i)).ToList();
invalidPaths.AddRange(UploadDirectory(_assetPath, "assets"));
Log.Debug("S3 upload complete");
if (invalidPaths.Count > 0)
{
var distId = Config["cloudfrontDistId"];
if (distId != null)
{
_client.InvalidateCloudFrontDistribution(distId, invalidPaths);
}
}
}
public override void Process(Post post)
{
// download linked url, look for og:image meta tag
var urls = PostUtils.UrlRegex.Matches(post.Content);
foreach (Match murl in urls)
{
var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(murl.Value));
var cachedImgUrl = App.CacheGet(url);
if (cachedImgUrl != null)
{
if (!cachedImgUrl.Equals(":none"))
{
Log.Debug(string.Format("Found cached image at {0}", cachedImgUrl));
post.Attach("image", url, cachedImgUrl);
}
break;
}
Log.Debug(string.Format("Checking {0} for opengraph image", url));
var properties = new Dictionary <string, string>();
var content = PostUtils.GetPageContent(Log, url);
var metas = _ogimage.Matches(content);
foreach (Match meta in metas)
{
string prop = null;
string val = null;
var mprop = _metaname.Match(meta.Value);
if (mprop.Success)
{
prop = WebUtility.HtmlDecode(mprop.Groups["property"].Value.ToLower());
}
var mcont = _metacontent.Match(meta.Value);
if (mcont.Success)
{
val = WebUtility.HtmlDecode(mcont.Groups["content"].Value);
}
if (!string.IsNullOrEmpty(prop) && !string.IsNullOrEmpty(val) && !properties.ContainsKey(prop))
{
properties.Add(prop, val);
}
}
if (properties.Count == 0 || !properties.ContainsKey("og:image"))
{
App.CacheSet(url, ":none");
continue;
}
string imgurl = null;
Log.Debug("Got image meta properties", properties);
// if it's twitter we only want user generated images
if (!_twitter.IsMatch(url) || (properties.ContainsKey("og:image:user_generated") && properties["og:image:user_generated"] == "true"))
{
imgurl = properties["og:image"];
}
if (string.IsNullOrWhiteSpace(imgurl))
{
App.CacheSet(url, ":none");
continue;
}
try
{
var uri = new Uri(imgurl);
var fname = Path.GetFileName(uri.AbsolutePath);
var wc = new WebClient();
var tpath = Path.Combine(Config["tempDir"], fname);
Log.Debug(string.Format("Downloading image {0} to {1}", imgurl, tpath));
wc.DownloadFile(imgurl, tpath);
using (var bitmap = Image.FromFile(tpath))
{
if (bitmap.Width < _minWidth ||
bitmap.Height < _minHeight)
{
Log.Debug(string.Format("Ignoring image {0}, too small ({1}x{2})", fname, bitmap.Width, bitmap.Height));
App.CacheSet(url, ":none");
File.Delete(tpath);
continue;
}
var type = bitmap.RawFormat;
if (!type.Equals(ImageFormat.Jpeg) &&
!type.Equals(ImageFormat.Gif) &&
!type.Equals(ImageFormat.Png))
{
Log.Debug(string.Format("Ignoring image {0}, unsupported type ({1})", fname, bitmap.RawFormat));
App.CacheSet(url, ":none");
File.Delete(tpath);
continue;
}
if (bitmap.Width > _maxWidth)
{
using (var resized = ResizeImage(bitmap, _maxWidth, (int)Math.Round(bitmap.Height / (float)bitmap.Width * _maxWidth)))
resized.Save(tpath, type);
}
string extension;
if (type.Equals(ImageFormat.Jpeg))
{
extension = "jpg";
}
else if (type.Equals(ImageFormat.Png))
{
extension = "png";
}
else
{
extension = "gif";
}
var uploadName = string.Format("{0}/{1}.{2}", Config["s3path"], post.Id, extension);
if (_client.UploadFile(Config["bucket"], tpath, uploadName) && Config["cloudfrontDistId"] != null)
{
_client.InvalidateCloudFrontDistribution(Config["cloudfrontDistId"], new List <string> {
string.Format("/{0}", uploadName)
});
}
// don't bother getting more than one image per post
var newImgUrl = string.Format("{0}/{1}", Config["siteUrl"], uploadName);
App.CacheSet(url, newImgUrl);
post.Attach("image", url, newImgUrl);
File.Delete(tpath);
}
}
catch (Exception ex)
{
Log.Error(string.Format("Error processing image {0}", imgurl), ex);
}
}
}