private async Task <OpenGraphResult> GetGraphResult(HttpRequest req, dynamic singleLinkItem, ILogger log)
{
string url = singleLinkItem.url, id = singleLinkItem.id;
if (!string.IsNullOrEmpty(url) && !string.IsNullOrEmpty(id))
{
/**
* This check is a hack in support of adding our own URLs to lists. Rendered URLs return no Open Graph
* metadata and deep links return HTTP 404s when hosting in Blob storage. So we skip the HTTP request.
*/
if (!req.Host.HasValue || !url.Contains(req.Host.Host))
{
try
{
OpenGraph graph = await OpenGraph.ParseUrlAsync(url, "Urlist");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(graph.OriginalHtml);
var descriptionMetaTag = doc.DocumentNode.SelectSingleNode("//meta[@name='description']");
var titleTag = doc.DocumentNode.SelectSingleNode("//head/title");
return(new OpenGraphResult(id, graph, descriptionMetaTag, titleTag));
}
catch (Exception ex)
{
log.LogError(ex, "Processing URL {URL} failed. {Message}", url, ex.Message);
}
}
}
return(new OpenGraphResult {
Id = id
});
}
public void UpdateUi()
{
this.hyperlinkManager = new HyperlinkManager();
if (MessageItem != null)
{
PreviewFrame.Visibility = Visibility.Collapsed;
if (hyperlinkManager.LinkClicked != null)
{
hyperlinkManager.LinkClicked -= MediaPreview_Clicked;
}
if (MessageItem.Type == MessageType.Info || MessageItem.Type == MessageType.JoinPart)
{
MessageBox.Style = (Style)Application.Current.Resources["InfoTextRichStyle"];
}
else if (MessageItem.Type == MessageType.Action)
{
MessageBox.FontStyle = Windows.UI.Text.FontStyle.Italic;
}
if (MessageItem.Type == MessageType.MOTD)
{
this.FontFamily = new FontFamily("Consolas");
}
else
{
this.FontFamily = new FontFamily(Config.GetString(Config.FontFamily, "Segoe UI"));
this.FontSize = Config.GetInt(Config.FontSize, 14);
}
hyperlinkManager.SetText(MessageParagraph, MessageItem.Text);
hyperlinkManager.LinkClicked += MediaPreview_Clicked;
}
try
{
if (!hyperlinkManager.InlineLink && hyperlinkManager.FirstLink != null && Config.GetBoolean(Config.ShowMetadata, true))
{
Task.Run(async() =>
{
var graph = await OpenGraph.ParseUrlAsync(hyperlinkManager.FirstLink);
if (graph.Values.Count > 0 && graph.Title != "" && graph["description"] != "")
{
await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
PreviewFrame.Visibility = Visibility.Visible;
PreviewFrame.Navigate(typeof(LinkView), graph, new SuppressNavigationTransitionInfo());
});
}
});
}
}
catch { } // swallow exceptions
this.HasLoaded = true;
UpdateLayout();
}
public async Task TestParsingAsyncTest()
{
OpenGraph graph = await OpenGraph.ParseUrlAsync(SpacedLink);
Assert.Contains("<html", graph.OriginalHtml);
this.AssertSpaced(graph);
}
public override async Task <string> ProcessMessageAsync(DynamoDBEvent dynamoDbEvent)
{
LogInfo($"# DynamoDB Stream Records Count = {dynamoDbEvent.Records.Count}");
foreach (var record in dynamoDbEvent.Records.Where(r => r.EventName == "INSERT"))
{
LogInfo($"EventID = {record.EventID}");
var url = new Uri(record.Dynamodb.NewImage["Url"].S);
try {
var graph = await OpenGraph.ParseUrlAsync(url);
var bookmark = new Bookmark {
ID = record.Dynamodb.NewImage["ID"].S,
Url = url,
Title = graph.Title,
Description = graph.Metadata["og:description"].FirstOrDefault()?.Value,
ImageUrl = graph.Image,
Type = graph.Type
};
LogInfo($"Updated Bookmark:\n{SerializeJson(bookmark)}");
_table.PutItemAsync(Document.FromJson(SerializeJson(bookmark))).Wait();
} catch (Exception e) {
LogError(e);
continue;
}
}
return("Ok");
}
private async Task <OpenGraphResult> GetGraphResultAsync(HttpRequest req, OpenGraphRequest openGraphRequest)
{
string url = openGraphRequest.Url, id = openGraphRequest.Id;
if (!string.IsNullOrEmpty(url) && !string.IsNullOrEmpty(id))
{
/**
* This check is a hack in support of adding our own URLs to lists. Rendered URLs return no Open Graph
* metadata and deep links return HTTP 404s when hosting in Blob storage. So we skip the HTTP request.
*/
if (!req.Host.HasValue || !url.Contains(req.Host.Host))
{
try
{
OpenGraph graph = await OpenGraph.ParseUrlAsync(url, "Urlist");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(graph.OriginalHtml);
var descriptionMetaTag = doc.DocumentNode.SelectSingleNode("//meta[@name='description']");
var titleTag = doc.DocumentNode.SelectSingleNode("//head/title");
return(new OpenGraphResult(id, graph, descriptionMetaTag, titleTag));
}
catch (Exception)
{
// Todo - Add logging
}
}
}
return(new OpenGraphResult {
Id = id
});
}
private async Task AddOpenGraphDataToMessagesAsync(IEnumerable <MessageDto> messages)
{
var aTagRegex = new Regex("<a href=\"(.*?)\" target=\"_blank\">.*?<\\/a>");
foreach (var m in messages)
{
if (string.IsNullOrEmpty(m.Content) || !aTagRegex.IsMatch(m.Content))
{
continue;
}
var openGraphTasks = new List <Task <OpenGraph> >();
var matches = aTagRegex.Matches(m.Content);
try {
foreach (Match match in matches)
{
// first group is the entire matched string
var url = HttpUtility.HtmlDecode(match.Groups[1].Value);
openGraphTasks.Add(OpenGraph.ParseUrlAsync(url));
}
var openGraphs = (await Task.WhenAll(openGraphTasks.ToArray())).ToList();
AddOpenGraphDataToMessage(openGraphs, m);
} catch (Exception) {
// ignore errors for invalid open graph objects
}
}
}
public async Task TestParsingAsyncTest()
{
OpenGraph graph = await OpenGraph.ParseUrlAsync(SpacedLink).ConfigureAwait(false);
Assert.Contains("<html", graph.OriginalHtml, StringComparison.InvariantCultureIgnoreCase);
this.AssertSpaced(graph);
}
public static async Task <OpenGraphResult> GetGraphResult(dynamic singleLinkItem, ILogger log)
{
string url = singleLinkItem.url, id = singleLinkItem.id;
if (!string.IsNullOrEmpty(url) && !string.IsNullOrEmpty(id))
{
try
{
OpenGraph graph = await OpenGraph.ParseUrlAsync(url, "Urlist");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(graph.OriginalHtml);
var descriptionMetaTag = doc.DocumentNode.SelectSingleNode("//meta[@name='description']");
var titleTag = doc.DocumentNode.SelectSingleNode("//head/title");
return(new OpenGraphResult(id, graph, descriptionMetaTag, titleTag));
}
catch (Exception ex)
{
log.LogError(ex, ex.Message);
return(new OpenGraphResult {
Id = id
});
}
}
return(null);
}
public async Task TestParsingUrlAsyncValidateEncodingIsCorrect()
{
var expectedContent = "Создайте себе горное настроение с нашим первым фан-китом по игре #SteepGame&#33; -&gt; http://ubi.li/u8w9n";
var tags = await OpenGraph.ParseUrlAsync("https://vk.com/wall-41600377_66756").ConfigureAwait(false);
Assert.Equal(expectedContent, tags.Metadata["og:description"].First().Value);
}
public void FactParsingUrlAsyncValidateEncodingIsCorrect()
{
var expectedContent =
"Создайте себе горное настроение с нашим первым фан-китом по игре #SteepGame&#33; -&gt; http://ubi.li/u8w9n";
var tags = OpenGraph.ParseUrlAsync("https://vk.com/wall-41600377_66756").Result;
Assert.True(tags["description"] == expectedContent);
}
public async Task TestParsingUrlAsyncValidateEncodingIsCorrect()
{
var expectedContent =
"Создайте себе горное настроение с нашим первым фан-китом по игре #SteepGame&#33; -&gt; http://ubi.li/u8w9n";
var tags = await OpenGraph.ParseUrlAsync("https://vk.com/wall-41600377_66756");
Assert.That(tags["description"], Is.EqualTo(expectedContent));
}
private async void MessageLine_Loaded(object sender, RoutedEventArgs e)
{
DataContext = MessageItem;
UpdateLayout();
if (double.IsNaN(UsernameBox.ActualWidth) || double.IsNaN(TimestampBox.ActualWidth))
{
return;
}
MessageParagraph.TextIndent = UsernameBox.ActualWidth + TimestampBox.ActualWidth;
if (MessageBox.ActualHeight > UsernameBox.ActualHeight)
{
Thickness margin = new Thickness(0, -1, 0, 0);
MessageBox.Margin = margin;
}
if (MessageItem != null)
{
if (MessageItem.Type == MessageType.Info)
{
UsernameBox.Style = (Style)Application.Current.Resources["InfoTextBlockStyle"];
MessageBox.Style = (Style)Application.Current.Resources["InfoTextRichStyle"];
}
else if (MessageItem.Type == MessageType.Action)
{
UsernameBox.FontStyle = Windows.UI.Text.FontStyle.Italic;
MessageBox.FontStyle = Windows.UI.Text.FontStyle.Italic;
}
if (MessageItem.Mention)
{
UsernameBox.Foreground = new SolidColorBrush(Colors.Red);
MessageBox.Foreground = new SolidColorBrush(Colors.Red);
}
hyperlinkManager.SetText(MessageParagraph, MessageItem.Text);
hyperlinkManager.LinkClicked += MediaPreview_Clicked;
}
try
{
if (!hyperlinkManager.InlineLink && hyperlinkManager.FirstLink != null && Config.GetBoolean(Config.ShowMetadata, true))
{
var graph = await OpenGraph.ParseUrlAsync(hyperlinkManager.FirstLink);
if (graph.Values.Count > 0 && graph.Title != "" && graph["description"] != "")
{
PreviewFrame.Visibility = Visibility.Visible;
PreviewFrame.Navigate(typeof(LinkView), graph);
}
}
}
catch { } // swallow exceptions
this.HasLoaded = true;
}
public async Task <object> FetchOpenGraph(string url)
{
try {
return(await OpenGraph.ParseUrlAsync(url, timeout : 6000));
}
catch {
return(null);
}
}
public async Task TestParsingUrlsWithoutScheme()
{
var withoutScheme = SpacedLink.Replace("http://", string.Empty);
var withHttpsScheme = SpacedLink.Replace("http", "https");
OpenGraph.ParseUrl(withHttpsScheme);
await OpenGraph.ParseUrlAsync(withHttpsScheme);
OpenGraph.ParseUrl(withoutScheme);
await OpenGraph.ParseUrlAsync(withoutScheme);
}
public async Task TestIssue14FromGit()
{
var urls = new List <Task <OpenGraph> >
{
OpenGraph.ParseUrlAsync("https://www.cntraveler.com/story/how-cruise-lines-are-getting-more-eco-friendly"),
OpenGraph.ParseUrlAsync("https://www.nytimes.com/2019/05/22/travel/leaping-caimans-and-tasty-piranha-in-the-wild-amazon.html"),
OpenGraph.ParseUrlAsync("https://www.thrillist.com/travel/nation/visiting-hainan-things-to-know"),
};
await Task.WhenAll(urls);
}
public async Task Test301RedirectUrl()
{
try
{
var graph = await OpenGraph.ParseUrlAsync("https://news.google.com/__i/rss/rd/articles/CBMigwFodHRwczovL3d3dy5zdWRpbmZvLmJlL2lkMzEzNzM4L2FydGljbGUvMjAyMS0wMS0yMi9sZS1jb21pdGUtZGUtY29uY2VydGF0aW9uLWRlYnV0ZS1kZS1ub3V2ZWxsZXMtbWVzdXJlcy1wbHVzLXJlc3RyaWN0aXZlcy1xdWlkLWRlc9IBAA?oc=5");
}
catch (WebException ex)
{
Assert.True(false, ex.Message);
}
}
public async Task TestParsingUrlsWithoutScheme()
{
var withoutScheme = SpacedLink.Replace("http://", string.Empty, StringComparison.InvariantCultureIgnoreCase);
var withHttpsScheme = SpacedLink.Replace("http", "https", StringComparison.InvariantCultureIgnoreCase);
OpenGraph.ParseUrl(withHttpsScheme);
await OpenGraph.ParseUrlAsync(withHttpsScheme).ConfigureAwait(false);
OpenGraph.ParseUrl(withoutScheme);
await OpenGraph.ParseUrlAsync(withoutScheme).ConfigureAwait(false);
}
public async Task <String> ScrapMeta(string url)
{
OpenGraph graph = await OpenGraph.ParseUrlAsync(url);
HtmlDocument pageDocument = new HtmlDocument();
pageDocument.LoadHtml(graph.ToString());
string newContent = "<html><head><title>Fc lunaar</title>" + pageDocument.DocumentNode.OuterHtml + "</head><body></body></html><script>window.location.replace('" + url + "')</script>";
return(newContent);
}
public async Task TestParsingAmazonUrlAsyncTest()
{
OpenGraph graph = await OpenGraph.ParseUrlAsync("http://www.amazon.com/Spaced-Complete-Simon-Pegg/dp/B0019MFY3Q");
Assert.AreEqual("http://www.amazon.com/dp/B0019MFY3Q/ref=tsm_1_fb_lk", graph.Url.ToString());
Assert.IsTrue(graph.Title.StartsWith("Spaced: The Complete Series"));
Assert.IsTrue(graph["description"].Contains("Spaced"));
Assert.IsTrue(graph.Image.ToString().Contains("images-amazon"));
Assert.AreEqual("movie", graph.Type);
Assert.AreEqual("Amazon.com", graph["site_name"]);
}
public async Task <UrlLinkPayload> Create(string url)
{
var openGraph = await OpenGraph.ParseUrlAsync(url);
return(new UrlLinkPayload
{
Url = openGraph.Url.AbsoluteUri,
Title = openGraph.Title,
ThumbnailUrl = openGraph.Image?.AbsoluteUri,
Description = openGraph.Metadata["description"].FirstOrDefault()?.Value
});
}
/// <summary>
/// Gets Open Graph data for prefilling.
/// </summary>
async Task GetOpenGraphData(string clipboardText)
{
try
{
State = LayoutState.Loading;
var ogData = await OpenGraph.ParseUrlAsync(clipboardText);
if (ogData == null)
{
return;
}
DateTime?dateTime = null;
if (ogData.Metadata.ContainsKey("article:published_time") &&
DateTime.TryParse(ogData.Metadata["article:published_time"].Value(), out var activityDate))
{
dateTime = activityDate;
}
Contribution.Title = new Validation.ValidatableObject <string> {
Value = HttpUtility.HtmlDecode(ogData.Title)
};
Contribution.ReferenceUrl = new Validation.ValidatableObject <string> {
Value = ogData.Url?.AbsoluteUri
};
Contribution.Description = ogData.Metadata.ContainsKey("og:description")
? HttpUtility.HtmlDecode(ogData.Metadata["og:description"].Value())
: string.Empty;
if (dateTime.HasValue)
{
Contribution.StartDate = dateTime.Value;
}
}
catch (Exception ex)
{
// Fail silently.
AnalyticsService.Report(ex, new Dictionary <string, string> {
{ nameof(clipboardText), clipboardText }
});
}
finally
{
State = LayoutState.None;
}
}
public async Task <UrlLink> Create(string url)
{
if (Logger.IsEnabled(LogLevel.Trace))
{
Logger.LogTrace($"create({url})");
}
var openGraph = await OpenGraph.ParseUrlAsync(url);
return(new UrlLink(new UrlLinkPayload
{
Description = openGraph.Metadata["description"]?.FirstOrDefault()?.Value,
ThumbnailUrl = openGraph.Image?.AbsoluteUri,
Title = openGraph.Title,
Url = openGraph.Url.AbsoluteUri
}));
}
public async Task <PreviewModel> GetPreviewFromUrl(string url)
{
try
{
OpenGraph openGraph = await OpenGraph.ParseUrlAsync(url);
var preview = new PreviewModel()
{
Title = openGraph.Title,
Image = openGraph.Image.AbsoluteUri,
Url = openGraph.Url,
Type = openGraph.Type
};
return(preview);
}
catch
{
return(null);
}
}
public async Task <LinkPreview> ParseUrlAsync(Uri url)
{
var openGraphData = await OpenGraph.ParseUrlAsync(url);
if (openGraphData != null)
{
var tracks = new List <Track>();
if (url.Host.ToLower().Contains("spotify"))
{
var tracksToParse = openGraphData.Metadata["music:song"];
var trackCrawlerTasks = new List <Task>();
foreach (var track in tracksToParse)
{
var songLink = track.Value;
var songTask = Task.Run(async() =>
{
var data = await OpenGraph.ParseUrlAsync(songLink);
var trackToAdd = new Track
{
Name = data.Metadata["og:title"].FirstOrDefault()?.Value,
PlayUrl = data.Metadata["og:audio"].FirstOrDefault()?.Value
};
tracks.Add(trackToAdd);
});
trackCrawlerTasks.Add(songTask);
}
if (trackCrawlerTasks.Any())
{
await Task.WhenAll(trackCrawlerTasks);
}
}
return(new LinkPreview
{
ImageUrl = openGraphData.Image,
Title = openGraphData.Title,
Html = openGraphData.OriginalHtml,
Tracks = tracks.OrderBy(x => x.TrackNumber).ToList()
});
}
return(null);
}
async Task <bool> CheckForClipboardUrl()
{
var text = string.Empty;
try
{
if (!Clipboard.HasText)
{
return(false);
}
text = await Clipboard.GetTextAsync();
if (string.IsNullOrEmpty(text) || (!text.StartsWith("http://") && !text.StartsWith("https://")))
{
return(false);
}
var shouldCreateActivity = await DialogService.ConfirmAsync(
"We notice a URL on your clipboard. Do you want us to pre-fill an activity out of that?",
"That looks cool!",
"Yes",
"No"
);
if (!shouldCreateActivity)
{
return(false);
}
var ogData = await OpenGraph.ParseUrlAsync(text);
if (ogData == null)
{
return(false);
}
DateTime?dateTime = null;
if (ogData.Metadata.ContainsKey("article:published_time") &&
DateTime.TryParse(ogData.Metadata["article:published_time"].Value(), out var activityDate))
{
dateTime = activityDate;
}
var contrib = new Contribution
{
Title = HttpUtility.HtmlDecode(ogData.Title),
ReferenceUrl = ogData.Url.AbsoluteUri,
Description = ogData.Metadata.ContainsKey("og:description")
? HttpUtility.HtmlDecode(ogData.Metadata["og:description"].Value())
: string.Empty,
StartDate = dateTime
};
await NavigationHelper.OpenModalAsync(nameof(WizardActivityTypePage), contrib, true).ConfigureAwait(false);
return(true);
}
catch (Exception ex)
{
AnalyticsService.Report(ex, new Dictionary <string, string> {
{ "clipboard_value", text }
});
return(false);
}
}
public void UpdateUi()
{
this.hyperlinkManager = new HyperlinkManager();
if (double.IsNaN(UsernameBox.ActualWidth) || double.IsNaN(TimestampBox.ActualWidth))
{
return;
}
if (MessageItem != null)
{
PreviewFrame.Visibility = Visibility.Collapsed;
if (hyperlinkManager.LinkClicked != null)
{
hyperlinkManager.LinkClicked -= MediaPreview_Clicked;
}
if (MessageBox.ActualHeight > UsernameBox.ActualHeight)
{
Thickness margin = new Thickness(0, -1, 0, 0);
MessageBox.Margin = margin;
}
if (MessageItem.Type == MessageType.Info || MessageItem.Type == MessageType.JoinPart)
{
UsernameBox.Style = (Style)Application.Current.Resources["InfoTextBlockStyle"];
MessageBox.Style = (Style)Application.Current.Resources["InfoTextRichStyle"];
}
else if (MessageItem.Type == MessageType.Action)
{
UsernameBox.FontStyle = Windows.UI.Text.FontStyle.Italic;
MessageBox.FontStyle = Windows.UI.Text.FontStyle.Italic;
}
if (MessageItem.Mention)
{
UsernameBox.Foreground = new SolidColorBrush(Colors.Red);
}
if (MessageItem.Type == MessageType.MOTD)
{
this.FontFamily = new FontFamily("Consolas");
}
hyperlinkManager.SetText(MessageParagraph, MessageItem.Text);
hyperlinkManager.LinkClicked += MediaPreview_Clicked;
}
try
{
if (!hyperlinkManager.InlineLink && hyperlinkManager.FirstLink != null && Config.GetBoolean(Config.ShowMetadata, true))
{
Task.Run(async() =>
{
var graph = await OpenGraph.ParseUrlAsync(hyperlinkManager.FirstLink);
if (graph.Values.Count > 0 && graph.Title != "" && graph["description"] != "")
{
await Windows.ApplicationModel.Core.CoreApplication.MainView.CoreWindow.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
PreviewFrame.Visibility = Visibility.Visible;
PreviewFrame.Navigate(typeof(LinkView), graph, new SuppressNavigationTransitionInfo());
});
}
});
}
}
catch { } // swallow exceptions
this.HasLoaded = true;
UpdateLayout();
}
public async Task TestParsingAsyncTest()
{
OpenGraph graph = await OpenGraph.ParseUrlAsync(SpacedLink);
this.AssertSpaced(graph);
}