private async Task <OpenGraphResult> GetGraphResult(HttpRequest req, dynamic singleLinkItem, ILogger log) { string url = singleLinkItem.url, id = singleLinkItem.id; if (!string.IsNullOrEmpty(url) && !string.IsNullOrEmpty(id)) { /** * This check is a hack in support of adding our own URLs to lists. Rendered URLs return no Open Graph * metadata and deep links return HTTP 404s when hosting in Blob storage. So we skip the HTTP request. */ if (!req.Host.HasValue || !url.Contains(req.Host.Host)) { try { OpenGraph graph = await OpenGraph.ParseUrlAsync(url, "Urlist"); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(graph.OriginalHtml); var descriptionMetaTag = doc.DocumentNode.SelectSingleNode("//meta[@name='description']"); var titleTag = doc.DocumentNode.SelectSingleNode("//head/title"); return(new OpenGraphResult(id, graph, descriptionMetaTag, titleTag)); } catch (Exception ex) { log.LogError(ex, "Processing URL {URL} failed. {Message}", url, ex.Message); } } } return(new OpenGraphResult { Id = id }); }
public void UpdateUi() { this.hyperlinkManager = new HyperlinkManager(); if (MessageItem != null) { PreviewFrame.Visibility = Visibility.Collapsed; if (hyperlinkManager.LinkClicked != null) { hyperlinkManager.LinkClicked -= MediaPreview_Clicked; } if (MessageItem.Type == MessageType.Info || MessageItem.Type == MessageType.JoinPart) { MessageBox.Style = (Style)Application.Current.Resources["InfoTextRichStyle"]; } else if (MessageItem.Type == MessageType.Action) { MessageBox.FontStyle = Windows.UI.Text.FontStyle.Italic; } if (MessageItem.Type == MessageType.MOTD) { this.FontFamily = new FontFamily("Consolas"); } else { this.FontFamily = new FontFamily(Config.GetString(Config.FontFamily, "Segoe UI")); this.FontSize = Config.GetInt(Config.FontSize, 14); } hyperlinkManager.SetText(MessageParagraph, MessageItem.Text); hyperlinkManager.LinkClicked += MediaPreview_Clicked; } try { if (!hyperlinkManager.InlineLink && hyperlinkManager.FirstLink != null && Config.GetBoolean(Config.ShowMetadata, true)) { Task.Run(async() => { var graph = await OpenGraph.ParseUrlAsync(hyperlinkManager.FirstLink); if (graph.Values.Count > 0 && graph.Title != "" && graph["description"] != "") { await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { PreviewFrame.Visibility = Visibility.Visible; PreviewFrame.Navigate(typeof(LinkView), graph, new SuppressNavigationTransitionInfo()); }); } }); } } catch { } // swallow exceptions this.HasLoaded = true; UpdateLayout(); }
public async Task TestParsingAsyncTest() { OpenGraph graph = await OpenGraph.ParseUrlAsync(SpacedLink); Assert.Contains("<html", graph.OriginalHtml); this.AssertSpaced(graph); }
public override async Task <string> ProcessMessageAsync(DynamoDBEvent dynamoDbEvent) { LogInfo($"# DynamoDB Stream Records Count = {dynamoDbEvent.Records.Count}"); foreach (var record in dynamoDbEvent.Records.Where(r => r.EventName == "INSERT")) { LogInfo($"EventID = {record.EventID}"); var url = new Uri(record.Dynamodb.NewImage["Url"].S); try { var graph = await OpenGraph.ParseUrlAsync(url); var bookmark = new Bookmark { ID = record.Dynamodb.NewImage["ID"].S, Url = url, Title = graph.Title, Description = graph.Metadata["og:description"].FirstOrDefault()?.Value, ImageUrl = graph.Image, Type = graph.Type }; LogInfo($"Updated Bookmark:\n{SerializeJson(bookmark)}"); _table.PutItemAsync(Document.FromJson(SerializeJson(bookmark))).Wait(); } catch (Exception e) { LogError(e); continue; } } return("Ok"); }
private async Task <OpenGraphResult> GetGraphResultAsync(HttpRequest req, OpenGraphRequest openGraphRequest) { string url = openGraphRequest.Url, id = openGraphRequest.Id; if (!string.IsNullOrEmpty(url) && !string.IsNullOrEmpty(id)) { /** * This check is a hack in support of adding our own URLs to lists. Rendered URLs return no Open Graph * metadata and deep links return HTTP 404s when hosting in Blob storage. So we skip the HTTP request. */ if (!req.Host.HasValue || !url.Contains(req.Host.Host)) { try { OpenGraph graph = await OpenGraph.ParseUrlAsync(url, "Urlist"); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(graph.OriginalHtml); var descriptionMetaTag = doc.DocumentNode.SelectSingleNode("//meta[@name='description']"); var titleTag = doc.DocumentNode.SelectSingleNode("//head/title"); return(new OpenGraphResult(id, graph, descriptionMetaTag, titleTag)); } catch (Exception) { // Todo - Add logging } } } return(new OpenGraphResult { Id = id }); }
private async Task AddOpenGraphDataToMessagesAsync(IEnumerable <MessageDto> messages) { var aTagRegex = new Regex("<a href=\"(.*?)\" target=\"_blank\">.*?<\\/a>"); foreach (var m in messages) { if (string.IsNullOrEmpty(m.Content) || !aTagRegex.IsMatch(m.Content)) { continue; } var openGraphTasks = new List <Task <OpenGraph> >(); var matches = aTagRegex.Matches(m.Content); try { foreach (Match match in matches) { // first group is the entire matched string var url = HttpUtility.HtmlDecode(match.Groups[1].Value); openGraphTasks.Add(OpenGraph.ParseUrlAsync(url)); } var openGraphs = (await Task.WhenAll(openGraphTasks.ToArray())).ToList(); AddOpenGraphDataToMessage(openGraphs, m); } catch (Exception) { // ignore errors for invalid open graph objects } } }
public async Task TestParsingAsyncTest() { OpenGraph graph = await OpenGraph.ParseUrlAsync(SpacedLink).ConfigureAwait(false); Assert.Contains("<html", graph.OriginalHtml, StringComparison.InvariantCultureIgnoreCase); this.AssertSpaced(graph); }
public static async Task <OpenGraphResult> GetGraphResult(dynamic singleLinkItem, ILogger log) { string url = singleLinkItem.url, id = singleLinkItem.id; if (!string.IsNullOrEmpty(url) && !string.IsNullOrEmpty(id)) { try { OpenGraph graph = await OpenGraph.ParseUrlAsync(url, "Urlist"); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(graph.OriginalHtml); var descriptionMetaTag = doc.DocumentNode.SelectSingleNode("//meta[@name='description']"); var titleTag = doc.DocumentNode.SelectSingleNode("//head/title"); return(new OpenGraphResult(id, graph, descriptionMetaTag, titleTag)); } catch (Exception ex) { log.LogError(ex, ex.Message); return(new OpenGraphResult { Id = id }); } } return(null); }
public async Task TestParsingUrlAsyncValidateEncodingIsCorrect() { var expectedContent = "Создайте себе горное настроение с нашим первым фан-китом по игре #SteepGame&#33; -&gt; http://ubi.li/u8w9n"; var tags = await OpenGraph.ParseUrlAsync("https://vk.com/wall-41600377_66756").ConfigureAwait(false); Assert.Equal(expectedContent, tags.Metadata["og:description"].First().Value); }
public void FactParsingUrlAsyncValidateEncodingIsCorrect() { var expectedContent = "Создайте себе горное настроение с нашим первым фан-китом по игре #SteepGame&#33; -&gt; http://ubi.li/u8w9n"; var tags = OpenGraph.ParseUrlAsync("https://vk.com/wall-41600377_66756").Result; Assert.True(tags["description"] == expectedContent); }
public async Task TestParsingUrlAsyncValidateEncodingIsCorrect() { var expectedContent = "Создайте себе горное настроение с нашим первым фан-китом по игре #SteepGame&#33; -&gt; http://ubi.li/u8w9n"; var tags = await OpenGraph.ParseUrlAsync("https://vk.com/wall-41600377_66756"); Assert.That(tags["description"], Is.EqualTo(expectedContent)); }
private async void MessageLine_Loaded(object sender, RoutedEventArgs e) { DataContext = MessageItem; UpdateLayout(); if (double.IsNaN(UsernameBox.ActualWidth) || double.IsNaN(TimestampBox.ActualWidth)) { return; } MessageParagraph.TextIndent = UsernameBox.ActualWidth + TimestampBox.ActualWidth; if (MessageBox.ActualHeight > UsernameBox.ActualHeight) { Thickness margin = new Thickness(0, -1, 0, 0); MessageBox.Margin = margin; } if (MessageItem != null) { if (MessageItem.Type == MessageType.Info) { UsernameBox.Style = (Style)Application.Current.Resources["InfoTextBlockStyle"]; MessageBox.Style = (Style)Application.Current.Resources["InfoTextRichStyle"]; } else if (MessageItem.Type == MessageType.Action) { UsernameBox.FontStyle = Windows.UI.Text.FontStyle.Italic; MessageBox.FontStyle = Windows.UI.Text.FontStyle.Italic; } if (MessageItem.Mention) { UsernameBox.Foreground = new SolidColorBrush(Colors.Red); MessageBox.Foreground = new SolidColorBrush(Colors.Red); } hyperlinkManager.SetText(MessageParagraph, MessageItem.Text); hyperlinkManager.LinkClicked += MediaPreview_Clicked; } try { if (!hyperlinkManager.InlineLink && hyperlinkManager.FirstLink != null && Config.GetBoolean(Config.ShowMetadata, true)) { var graph = await OpenGraph.ParseUrlAsync(hyperlinkManager.FirstLink); if (graph.Values.Count > 0 && graph.Title != "" && graph["description"] != "") { PreviewFrame.Visibility = Visibility.Visible; PreviewFrame.Navigate(typeof(LinkView), graph); } } } catch { } // swallow exceptions this.HasLoaded = true; }
public async Task <object> FetchOpenGraph(string url) { try { return(await OpenGraph.ParseUrlAsync(url, timeout : 6000)); } catch { return(null); } }
public async Task TestParsingUrlsWithoutScheme() { var withoutScheme = SpacedLink.Replace("http://", string.Empty); var withHttpsScheme = SpacedLink.Replace("http", "https"); OpenGraph.ParseUrl(withHttpsScheme); await OpenGraph.ParseUrlAsync(withHttpsScheme); OpenGraph.ParseUrl(withoutScheme); await OpenGraph.ParseUrlAsync(withoutScheme); }
public async Task TestIssue14FromGit() { var urls = new List <Task <OpenGraph> > { OpenGraph.ParseUrlAsync("https://www.cntraveler.com/story/how-cruise-lines-are-getting-more-eco-friendly"), OpenGraph.ParseUrlAsync("https://www.nytimes.com/2019/05/22/travel/leaping-caimans-and-tasty-piranha-in-the-wild-amazon.html"), OpenGraph.ParseUrlAsync("https://www.thrillist.com/travel/nation/visiting-hainan-things-to-know"), }; await Task.WhenAll(urls); }
public async Task Test301RedirectUrl() { try { var graph = await OpenGraph.ParseUrlAsync("https://news.google.com/__i/rss/rd/articles/CBMigwFodHRwczovL3d3dy5zdWRpbmZvLmJlL2lkMzEzNzM4L2FydGljbGUvMjAyMS0wMS0yMi9sZS1jb21pdGUtZGUtY29uY2VydGF0aW9uLWRlYnV0ZS1kZS1ub3V2ZWxsZXMtbWVzdXJlcy1wbHVzLXJlc3RyaWN0aXZlcy1xdWlkLWRlc9IBAA?oc=5"); } catch (WebException ex) { Assert.True(false, ex.Message); } }
public async Task TestParsingUrlsWithoutScheme() { var withoutScheme = SpacedLink.Replace("http://", string.Empty, StringComparison.InvariantCultureIgnoreCase); var withHttpsScheme = SpacedLink.Replace("http", "https", StringComparison.InvariantCultureIgnoreCase); OpenGraph.ParseUrl(withHttpsScheme); await OpenGraph.ParseUrlAsync(withHttpsScheme).ConfigureAwait(false); OpenGraph.ParseUrl(withoutScheme); await OpenGraph.ParseUrlAsync(withoutScheme).ConfigureAwait(false); }
public async Task <String> ScrapMeta(string url) { OpenGraph graph = await OpenGraph.ParseUrlAsync(url); HtmlDocument pageDocument = new HtmlDocument(); pageDocument.LoadHtml(graph.ToString()); string newContent = "<html><head><title>Fc lunaar</title>" + pageDocument.DocumentNode.OuterHtml + "</head><body></body></html><script>window.location.replace('" + url + "')</script>"; return(newContent); }
public async Task TestParsingAmazonUrlAsyncTest() { OpenGraph graph = await OpenGraph.ParseUrlAsync("http://www.amazon.com/Spaced-Complete-Simon-Pegg/dp/B0019MFY3Q"); Assert.AreEqual("http://www.amazon.com/dp/B0019MFY3Q/ref=tsm_1_fb_lk", graph.Url.ToString()); Assert.IsTrue(graph.Title.StartsWith("Spaced: The Complete Series")); Assert.IsTrue(graph["description"].Contains("Spaced")); Assert.IsTrue(graph.Image.ToString().Contains("images-amazon")); Assert.AreEqual("movie", graph.Type); Assert.AreEqual("Amazon.com", graph["site_name"]); }
public async Task <UrlLinkPayload> Create(string url) { var openGraph = await OpenGraph.ParseUrlAsync(url); return(new UrlLinkPayload { Url = openGraph.Url.AbsoluteUri, Title = openGraph.Title, ThumbnailUrl = openGraph.Image?.AbsoluteUri, Description = openGraph.Metadata["description"].FirstOrDefault()?.Value }); }
/// <summary> /// Gets Open Graph data for prefilling. /// </summary> async Task GetOpenGraphData(string clipboardText) { try { State = LayoutState.Loading; var ogData = await OpenGraph.ParseUrlAsync(clipboardText); if (ogData == null) { return; } DateTime?dateTime = null; if (ogData.Metadata.ContainsKey("article:published_time") && DateTime.TryParse(ogData.Metadata["article:published_time"].Value(), out var activityDate)) { dateTime = activityDate; } Contribution.Title = new Validation.ValidatableObject <string> { Value = HttpUtility.HtmlDecode(ogData.Title) }; Contribution.ReferenceUrl = new Validation.ValidatableObject <string> { Value = ogData.Url?.AbsoluteUri }; Contribution.Description = ogData.Metadata.ContainsKey("og:description") ? HttpUtility.HtmlDecode(ogData.Metadata["og:description"].Value()) : string.Empty; if (dateTime.HasValue) { Contribution.StartDate = dateTime.Value; } } catch (Exception ex) { // Fail silently. AnalyticsService.Report(ex, new Dictionary <string, string> { { nameof(clipboardText), clipboardText } }); } finally { State = LayoutState.None; } }
public async Task <UrlLink> Create(string url) { if (Logger.IsEnabled(LogLevel.Trace)) { Logger.LogTrace($"create({url})"); } var openGraph = await OpenGraph.ParseUrlAsync(url); return(new UrlLink(new UrlLinkPayload { Description = openGraph.Metadata["description"]?.FirstOrDefault()?.Value, ThumbnailUrl = openGraph.Image?.AbsoluteUri, Title = openGraph.Title, Url = openGraph.Url.AbsoluteUri })); }
public async Task <PreviewModel> GetPreviewFromUrl(string url) { try { OpenGraph openGraph = await OpenGraph.ParseUrlAsync(url); var preview = new PreviewModel() { Title = openGraph.Title, Image = openGraph.Image.AbsoluteUri, Url = openGraph.Url, Type = openGraph.Type }; return(preview); } catch { return(null); } }
public async Task <LinkPreview> ParseUrlAsync(Uri url) { var openGraphData = await OpenGraph.ParseUrlAsync(url); if (openGraphData != null) { var tracks = new List <Track>(); if (url.Host.ToLower().Contains("spotify")) { var tracksToParse = openGraphData.Metadata["music:song"]; var trackCrawlerTasks = new List <Task>(); foreach (var track in tracksToParse) { var songLink = track.Value; var songTask = Task.Run(async() => { var data = await OpenGraph.ParseUrlAsync(songLink); var trackToAdd = new Track { Name = data.Metadata["og:title"].FirstOrDefault()?.Value, PlayUrl = data.Metadata["og:audio"].FirstOrDefault()?.Value }; tracks.Add(trackToAdd); }); trackCrawlerTasks.Add(songTask); } if (trackCrawlerTasks.Any()) { await Task.WhenAll(trackCrawlerTasks); } } return(new LinkPreview { ImageUrl = openGraphData.Image, Title = openGraphData.Title, Html = openGraphData.OriginalHtml, Tracks = tracks.OrderBy(x => x.TrackNumber).ToList() }); } return(null); }
async Task <bool> CheckForClipboardUrl() { var text = string.Empty; try { if (!Clipboard.HasText) { return(false); } text = await Clipboard.GetTextAsync(); if (string.IsNullOrEmpty(text) || (!text.StartsWith("http://") && !text.StartsWith("https://"))) { return(false); } var shouldCreateActivity = await DialogService.ConfirmAsync( "We notice a URL on your clipboard. Do you want us to pre-fill an activity out of that?", "That looks cool!", "Yes", "No" ); if (!shouldCreateActivity) { return(false); } var ogData = await OpenGraph.ParseUrlAsync(text); if (ogData == null) { return(false); } DateTime?dateTime = null; if (ogData.Metadata.ContainsKey("article:published_time") && DateTime.TryParse(ogData.Metadata["article:published_time"].Value(), out var activityDate)) { dateTime = activityDate; } var contrib = new Contribution { Title = HttpUtility.HtmlDecode(ogData.Title), ReferenceUrl = ogData.Url.AbsoluteUri, Description = ogData.Metadata.ContainsKey("og:description") ? HttpUtility.HtmlDecode(ogData.Metadata["og:description"].Value()) : string.Empty, StartDate = dateTime }; await NavigationHelper.OpenModalAsync(nameof(WizardActivityTypePage), contrib, true).ConfigureAwait(false); return(true); } catch (Exception ex) { AnalyticsService.Report(ex, new Dictionary <string, string> { { "clipboard_value", text } }); return(false); } }
public void UpdateUi() { this.hyperlinkManager = new HyperlinkManager(); if (double.IsNaN(UsernameBox.ActualWidth) || double.IsNaN(TimestampBox.ActualWidth)) { return; } if (MessageItem != null) { PreviewFrame.Visibility = Visibility.Collapsed; if (hyperlinkManager.LinkClicked != null) { hyperlinkManager.LinkClicked -= MediaPreview_Clicked; } if (MessageBox.ActualHeight > UsernameBox.ActualHeight) { Thickness margin = new Thickness(0, -1, 0, 0); MessageBox.Margin = margin; } if (MessageItem.Type == MessageType.Info || MessageItem.Type == MessageType.JoinPart) { UsernameBox.Style = (Style)Application.Current.Resources["InfoTextBlockStyle"]; MessageBox.Style = (Style)Application.Current.Resources["InfoTextRichStyle"]; } else if (MessageItem.Type == MessageType.Action) { UsernameBox.FontStyle = Windows.UI.Text.FontStyle.Italic; MessageBox.FontStyle = Windows.UI.Text.FontStyle.Italic; } if (MessageItem.Mention) { UsernameBox.Foreground = new SolidColorBrush(Colors.Red); } if (MessageItem.Type == MessageType.MOTD) { this.FontFamily = new FontFamily("Consolas"); } hyperlinkManager.SetText(MessageParagraph, MessageItem.Text); hyperlinkManager.LinkClicked += MediaPreview_Clicked; } try { if (!hyperlinkManager.InlineLink && hyperlinkManager.FirstLink != null && Config.GetBoolean(Config.ShowMetadata, true)) { Task.Run(async() => { var graph = await OpenGraph.ParseUrlAsync(hyperlinkManager.FirstLink); if (graph.Values.Count > 0 && graph.Title != "" && graph["description"] != "") { await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => { PreviewFrame.Visibility = Visibility.Visible; PreviewFrame.Navigate(typeof(LinkView), graph, new SuppressNavigationTransitionInfo()); }); } }); } } catch { } // swallow exceptions this.HasLoaded = true; UpdateLayout(); }
public async Task TestParsingAsyncTest() { OpenGraph graph = await OpenGraph.ParseUrlAsync(SpacedLink); this.AssertSpaced(graph); }