private string Process(string content)
{
string processed = content;
var urls = PostUtils.UrlRegex.Matches(content);
var linked = new List <string>();
foreach (Match match in urls)
{
var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(match.Value));
if (!linked.Contains(url))
{
var noproto = PostUtils.SanitizeUrl(match.Value);
processed = processed.Replace(match.Value,
string.Format("<a href='{0}'>{1}</a>", url, noproto));
linked.Add(url);
}
}
// line breaks
if (Config["allowNewlines"] == "true")
{
processed = _newlines.Replace(processed, "<br>");
}
return(processed);
}
public override void Process(Post post)
{
// look for twitter links
var urls = _twitter.Matches(post.Content);
foreach (Match match in urls)
{
var url = match.Value.Replace("//mobile.", "//");
var user = match.Groups["user"].Value;
var id = match.Groups["id"].Value;
Log.Debug(string.Format("Found twitter url {0}", url));
var tweet = App.CacheGet(id);
if (string.IsNullOrEmpty(tweet))
{
var content = PostUtils.GetPageContent(Log, url);
var ogdata = new Dictionary <string, string>();
var ogs = _ogdata.Matches(content);
foreach (Match og in ogs)
{
var ogfield = og.Groups["field"].Value;
var ogcontent = og.Groups["content"].Value;
if (!ogdata.ContainsKey(ogfield))
{
ogdata.Add(ogfield, ogcontent);
}
}
Log.Debug("Extracted tweet data", ogdata);
if (ogdata.ContainsKey("type") &&
ogdata["type"] == "article" &&
ogdata.ContainsKey("description"))
{
tweet = string.Format("{0} - @{1}",
Regex.Unescape(ogdata["description"]), user);
App.CacheSet(id, tweet);
}
}
if (!string.IsNullOrEmpty(tweet))
{
post.Attach("tweet", url, tweet, true);
}
}
}
private string Process(string content)
{
var processed = content;
var urls = PostUtils.UrlRegex.Matches(content);
foreach (Match match in urls)
{
var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(match.Value));
var newurl = App.CacheGet(url);
if (string.IsNullOrEmpty(newurl))
{
var req = (HttpWebRequest)WebRequest.Create(Config["url"]);
req.AllowWriteStreamBuffering = false;
req.Method = "POST";
req.ContentType = "application/x-www-form-urlencoded";
var data = HttpUtility.ParseQueryString(string.Empty);
data.Add("longurl", url);
var postData = Encoding.UTF8.GetBytes(data.ToString());
req.ContentLength = postData.Length;
using (var stream = req.GetRequestStream())
{
stream.Write(postData, 0, postData.Length);
stream.Flush();
stream.Close();
}
Log.Debug(string.Format("Getting lilUrl for {0}", url));
using (var resp = (HttpWebResponse)req.GetResponse())
{
var sr = new StreamReader(resp.GetResponseStream());
var lilcontent = sr.ReadToEnd();
var lilmatch = _lilok.Match(lilcontent);
if (lilmatch.Success)
{
newurl = lilmatch.Groups["lilurl"].Value;
App.CacheSet(url, newurl);
Log.Debug(string.Format("Got new lilurl {0}", newurl));
}
else
{
lilmatch = _lilerr.Match(lilcontent);
if (lilmatch.Success)
{
var msg = lilmatch.Groups["msg"].Value;
Log.Error(string.Format("Lilurl error for {0}: {1}", url, msg));
}
else
{
Log.Error(string.Format("Unexpected lilurl response for {0}: {1}", url, resp.StatusCode));
}
}
}
}
if (!string.IsNullOrEmpty(newurl))
{
processed = processed.Replace(match.Value, newurl);
}
}
return(processed);
}
public override void Process(Post post)
{
// download linked url, look for og:image meta tag
var urls = PostUtils.UrlRegex.Matches(post.Content);
foreach (Match murl in urls)
{
var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(murl.Value));
var cachedImgUrl = App.CacheGet(url);
if (cachedImgUrl != null)
{
if (!cachedImgUrl.Equals(":none"))
{
Log.Debug(string.Format("Found cached image at {0}", cachedImgUrl));
post.Attach("image", url, cachedImgUrl);
}
break;
}
Log.Debug(string.Format("Checking {0} for opengraph image", url));
var properties = new Dictionary <string, string>();
var content = PostUtils.GetPageContent(Log, url);
var metas = _ogimage.Matches(content);
foreach (Match meta in metas)
{
string prop = null;
string val = null;
var mprop = _metaname.Match(meta.Value);
if (mprop.Success)
{
prop = WebUtility.HtmlDecode(mprop.Groups["property"].Value.ToLower());
}
var mcont = _metacontent.Match(meta.Value);
if (mcont.Success)
{
val = WebUtility.HtmlDecode(mcont.Groups["content"].Value);
}
if (!string.IsNullOrEmpty(prop) && !string.IsNullOrEmpty(val) && !properties.ContainsKey(prop))
{
properties.Add(prop, val);
}
}
if (properties.Count == 0 || !properties.ContainsKey("og:image"))
{
App.CacheSet(url, ":none");
continue;
}
string imgurl = null;
Log.Debug("Got image meta properties", properties);
// if it's twitter we only want user generated images
if (!_twitter.IsMatch(url) || (properties.ContainsKey("og:image:user_generated") && properties["og:image:user_generated"] == "true"))
{
imgurl = properties["og:image"];
}
if (string.IsNullOrWhiteSpace(imgurl))
{
App.CacheSet(url, ":none");
continue;
}
try
{
var uri = new Uri(imgurl);
var fname = Path.GetFileName(uri.AbsolutePath);
var wc = new WebClient();
var tpath = Path.Combine(Config["tempDir"], fname);
Log.Debug(string.Format("Downloading image {0} to {1}", imgurl, tpath));
wc.DownloadFile(imgurl, tpath);
using (var bitmap = Image.FromFile(tpath))
{
if (bitmap.Width < _minWidth ||
bitmap.Height < _minHeight)
{
Log.Debug(string.Format("Ignoring image {0}, too small ({1}x{2})", fname, bitmap.Width, bitmap.Height));
App.CacheSet(url, ":none");
File.Delete(tpath);
continue;
}
var type = bitmap.RawFormat;
if (!type.Equals(ImageFormat.Jpeg) &&
!type.Equals(ImageFormat.Gif) &&
!type.Equals(ImageFormat.Png))
{
Log.Debug(string.Format("Ignoring image {0}, unsupported type ({1})", fname, bitmap.RawFormat));
App.CacheSet(url, ":none");
File.Delete(tpath);
continue;
}
if (bitmap.Width > _maxWidth)
{
using (var resized = ResizeImage(bitmap, _maxWidth, (int)Math.Round(bitmap.Height / (float)bitmap.Width * _maxWidth)))
resized.Save(tpath, type);
}
string extension;
if (type.Equals(ImageFormat.Jpeg))
{
extension = "jpg";
}
else if (type.Equals(ImageFormat.Png))
{
extension = "png";
}
else
{
extension = "gif";
}
var uploadName = string.Format("{0}/{1}.{2}", Config["s3path"], post.Id, extension);
if (_client.UploadFile(Config["bucket"], tpath, uploadName) && Config["cloudfrontDistId"] != null)
{
_client.InvalidateCloudFrontDistribution(Config["cloudfrontDistId"], new List <string> {
string.Format("/{0}", uploadName)
});
}
// don't bother getting more than one image per post
var newImgUrl = string.Format("{0}/{1}", Config["siteUrl"], uploadName);
App.CacheSet(url, newImgUrl);
post.Attach("image", url, newImgUrl);
File.Delete(tpath);
}
}
catch (Exception ex)
{
Log.Error(string.Format("Error processing image {0}", imgurl), ex);
}
}
}