private string Process(string content) { string processed = content; var urls = PostUtils.UrlRegex.Matches(content); var linked = new List <string>(); foreach (Match match in urls) { var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(match.Value)); if (!linked.Contains(url)) { var noproto = PostUtils.SanitizeUrl(match.Value); processed = processed.Replace(match.Value, string.Format("<a href='{0}'>{1}</a>", url, noproto)); linked.Add(url); } } // line breaks if (Config["allowNewlines"] == "true") { processed = _newlines.Replace(processed, "<br>"); } return(processed); }
public override void Process(Post post) { // look for twitter links var urls = _twitter.Matches(post.Content); foreach (Match match in urls) { var url = match.Value.Replace("//mobile.", "//"); var user = match.Groups["user"].Value; var id = match.Groups["id"].Value; Log.Debug(string.Format("Found twitter url {0}", url)); var tweet = App.CacheGet(id); if (string.IsNullOrEmpty(tweet)) { var content = PostUtils.GetPageContent(Log, url); var ogdata = new Dictionary <string, string>(); var ogs = _ogdata.Matches(content); foreach (Match og in ogs) { var ogfield = og.Groups["field"].Value; var ogcontent = og.Groups["content"].Value; if (!ogdata.ContainsKey(ogfield)) { ogdata.Add(ogfield, ogcontent); } } Log.Debug("Extracted tweet data", ogdata); if (ogdata.ContainsKey("type") && ogdata["type"] == "article" && ogdata.ContainsKey("description")) { tweet = string.Format("{0} - @{1}", Regex.Unescape(ogdata["description"]), user); App.CacheSet(id, tweet); } } if (!string.IsNullOrEmpty(tweet)) { post.Attach("tweet", url, tweet, true); } } }
private string Process(string content) { var processed = content; var urls = PostUtils.UrlRegex.Matches(content); foreach (Match match in urls) { var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(match.Value)); var newurl = App.CacheGet(url); if (string.IsNullOrEmpty(newurl)) { var req = (HttpWebRequest)WebRequest.Create(Config["url"]); req.AllowWriteStreamBuffering = false; req.Method = "POST"; req.ContentType = "application/x-www-form-urlencoded"; var data = HttpUtility.ParseQueryString(string.Empty); data.Add("longurl", url); var postData = Encoding.UTF8.GetBytes(data.ToString()); req.ContentLength = postData.Length; using (var stream = req.GetRequestStream()) { stream.Write(postData, 0, postData.Length); stream.Flush(); stream.Close(); } Log.Debug(string.Format("Getting lilUrl for {0}", url)); using (var resp = (HttpWebResponse)req.GetResponse()) { var sr = new StreamReader(resp.GetResponseStream()); var lilcontent = sr.ReadToEnd(); var lilmatch = _lilok.Match(lilcontent); if (lilmatch.Success) { newurl = lilmatch.Groups["lilurl"].Value; App.CacheSet(url, newurl); Log.Debug(string.Format("Got new lilurl {0}", newurl)); } else { lilmatch = _lilerr.Match(lilcontent); if (lilmatch.Success) { var msg = lilmatch.Groups["msg"].Value; Log.Error(string.Format("Lilurl error for {0}: {1}", url, msg)); } else { Log.Error(string.Format("Unexpected lilurl response for {0}: {1}", url, resp.StatusCode)); } } } } if (!string.IsNullOrEmpty(newurl)) { processed = processed.Replace(match.Value, newurl); } } return(processed); }
public override void Process(Post post) { // download linked url, look for og:image meta tag var urls = PostUtils.UrlRegex.Matches(post.Content); foreach (Match murl in urls) { var url = PostUtils.DemobilizeUrl(PostUtils.NormalizeUrl(murl.Value)); var cachedImgUrl = App.CacheGet(url); if (cachedImgUrl != null) { if (!cachedImgUrl.Equals(":none")) { Log.Debug(string.Format("Found cached image at {0}", cachedImgUrl)); post.Attach("image", url, cachedImgUrl); } break; } Log.Debug(string.Format("Checking {0} for opengraph image", url)); var properties = new Dictionary <string, string>(); var content = PostUtils.GetPageContent(Log, url); var metas = _ogimage.Matches(content); foreach (Match meta in metas) { string prop = null; string val = null; var mprop = _metaname.Match(meta.Value); if (mprop.Success) { prop = WebUtility.HtmlDecode(mprop.Groups["property"].Value.ToLower()); } var mcont = _metacontent.Match(meta.Value); if (mcont.Success) { val = WebUtility.HtmlDecode(mcont.Groups["content"].Value); } if (!string.IsNullOrEmpty(prop) && !string.IsNullOrEmpty(val) && !properties.ContainsKey(prop)) { properties.Add(prop, val); } } if (properties.Count == 0 || !properties.ContainsKey("og:image")) { App.CacheSet(url, ":none"); continue; } string imgurl = null; Log.Debug("Got image meta properties", properties); // if it's twitter we only want user generated images if (!_twitter.IsMatch(url) || (properties.ContainsKey("og:image:user_generated") && properties["og:image:user_generated"] == "true")) { imgurl = properties["og:image"]; } if (string.IsNullOrWhiteSpace(imgurl)) { App.CacheSet(url, ":none"); continue; } try { var uri = new Uri(imgurl); var fname = Path.GetFileName(uri.AbsolutePath); var wc = new WebClient(); var tpath = Path.Combine(Config["tempDir"], fname); Log.Debug(string.Format("Downloading image {0} to {1}", imgurl, tpath)); wc.DownloadFile(imgurl, tpath); using (var bitmap = Image.FromFile(tpath)) { if (bitmap.Width < _minWidth || bitmap.Height < _minHeight) { Log.Debug(string.Format("Ignoring image {0}, too small ({1}x{2})", fname, bitmap.Width, bitmap.Height)); App.CacheSet(url, ":none"); File.Delete(tpath); continue; } var type = bitmap.RawFormat; if (!type.Equals(ImageFormat.Jpeg) && !type.Equals(ImageFormat.Gif) && !type.Equals(ImageFormat.Png)) { Log.Debug(string.Format("Ignoring image {0}, unsupported type ({1})", fname, bitmap.RawFormat)); App.CacheSet(url, ":none"); File.Delete(tpath); continue; } if (bitmap.Width > _maxWidth) { using (var resized = ResizeImage(bitmap, _maxWidth, (int)Math.Round(bitmap.Height / (float)bitmap.Width * _maxWidth))) resized.Save(tpath, type); } string extension; if (type.Equals(ImageFormat.Jpeg)) { extension = "jpg"; } else if (type.Equals(ImageFormat.Png)) { extension = "png"; } else { extension = "gif"; } var uploadName = string.Format("{0}/{1}.{2}", Config["s3path"], post.Id, extension); if (_client.UploadFile(Config["bucket"], tpath, uploadName) && Config["cloudfrontDistId"] != null) { _client.InvalidateCloudFrontDistribution(Config["cloudfrontDistId"], new List <string> { string.Format("/{0}", uploadName) }); } // don't bother getting more than one image per post var newImgUrl = string.Format("{0}/{1}", Config["siteUrl"], uploadName); App.CacheSet(url, newImgUrl); post.Attach("image", url, newImgUrl); File.Delete(tpath); } } catch (Exception ex) { Log.Error(string.Format("Error processing image {0}", imgurl), ex); } } }