static Uri MakeThumbnailUrl(Uri baseUrl, Uri thumbnail)
{
thumbnail = SyndicationUtil.Rebase(thumbnail, baseUrl);
if (!thumbnail.IsAbsoluteUri)
{
return(null);
}
if (thumbnail.Scheme != "http" && thumbnail.Scheme != "https")
{
return(null);
}
for (int i = 0; i < BadThumbnails.Length; i++)
{
if (thumbnail.AbsolutePath.EndsWith(BadThumbnails[i], StringComparison.OrdinalIgnoreCase))
{
return(null);
}
}
for (int i = 0; i < BadThumbnailHosts.Length; i++)
{
if (thumbnail.Host.EndsWith(BadThumbnailHosts[i], StringComparison.OrdinalIgnoreCase))
{
return(null);
}
}
return(thumbnail);
}
async Task <Item> GetItemThumbnailAsync(Uri baseUri, Item item)
{
Uri itemLink = SyndicationUtil.Rebase(item.Link, baseUri);
ThumbnailResponse sourceImage = null;
// We might already have found a thumbnail...
if (item.Thumbnail != null)
{
Uri baseUrl = itemLink ?? baseUri;
Uri thumbnailUrl = MakeThumbnailUrl(baseUrl, item.Thumbnail.Url);
if (thumbnailUrl != null)
{
sourceImage = await FetchThumbnailAsync(
new ImageUrl { Kind = "EmbeddedThumb", Uri = thumbnailUrl },
baseUrl,
skipChecks : true);
if (sourceImage != null)
{
ThumbnailLog.LogThumbnail(baseUrl, thumbnailUrl, "EmbeddedThumb", sourceImage, "Best");
}
}
}
// Look in the item soup; maybe we have it?
string[] soup_ids = new string[] { "content", "description", "summary", };
XElement[] soups = new XElement[] { item.Content, item.Description, item.Summary };
for (int i = 0; i < soups.Length && sourceImage == null; i++)
{
XElement xe = soups[i];
if (xe != null)
{
Uri soupBase = SyndicationUtil.TryParseAbsoluteUrl(xe.BaseUri, baseUri) ?? itemLink ?? baseUri;
sourceImage = await FindThumbnailInSoupAsync(
soup_ids[i],
soupBase,
SoupFromElement(soups[i])
);
}
}
if (sourceImage == null && itemLink != null)
{
sourceImage = await FindImageAsync(itemLink);
}
if (sourceImage == null)
{
return(item);
}
return(item.With(
thumbnail: new Thumbnail(
sourceImage.ThumbnailUrl,
sourceImage.ThumbnailWidth,
sourceImage.ThumbnailHeight
)
));
}
static Uri MakeThumbnailUrl(Uri baseUrl, string src)
{
Uri thumbnail = SyndicationUtil.TryParseAbsoluteUrl(src, baseUrl);
if (thumbnail == null)
{
return(null);
}
return(MakeThumbnailUrl(baseUrl, thumbnail));
}
static void Foo()
{
string title = null;
Uri url = Foo(75);
Uri url0 = SyndicationUtil.TryParseUrl(element.Attribute(XNames.Media.Url)?.Value, null);
Uri url1 = SyndicationUtil.TryParseUrl(element.Attribute(XNames.Media.Url)?.Value, null, element);
Uri url2 = SyndicationUtil.TryParseUrl(element.Attribute(XNames.Media.Url)?.Value, null, element, element, element);
return(item.With(enclosures: item.Enclosures.Add(new Enclosure(length: element.Attribute(XNames.RSS.Length)?.Value, type: element.Attribute(XNames.RSS.Type)?.Value, url: SyndicationUtil.ParseLink(element.Attribute(XNames.RSS.Url)?.Value, element)))));
}
public async Task <string> WriteRiverArchive(River oldRiver)
{
byte[] data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(oldRiver, Policies.SerializerSettings));
string id = SyndicationUtil.HashBytes(data);
using (var memoryStream = new MemoryStream(data))
{
await this.blobStore.PutObject(id, "application/json", memoryStream);
}
return(id);
}
static int DoSubscribe(ParsedOpts args)
{
string user = args["user"].Value;
string feed = args["feed"].Value;
string riverName = args["river"].Value;
// Check feed.
var parser = new RiverLoader();
var feedStore = new RiverFeedStore();
River feedRiver = parser.FetchAndUpdateRiver(new Uri(feed)).Result;
if (feedRiver.Metadata.LastStatus < (HttpStatusCode)200 ||
feedRiver.Metadata.LastStatus >= (HttpStatusCode)400)
{
Console.Error.WriteLine("Could not fetch feed {0}", feed);
return(-1);
}
var subscriptionStore = new UserProfileStore();
UserProfile profile = subscriptionStore.GetProfileFor(user).Result;
RiverDefinition river = profile.Rivers.FirstOrDefault(r => r.Name == riverName);
UserProfile newProfile;
if (river == null)
{
newProfile = profile.With(
rivers: profile.Rivers.Add(
new RiverDefinition(
name: riverName,
id: SyndicationUtil.MakeID(),
feeds: new Uri[] { feedRiver.Metadata.OriginUrl })));
}
else
{
var newRiver = river.With(feeds: river.Feeds.Add(feedRiver.Metadata.OriginUrl));
newProfile = profile.With(rivers: profile.Rivers.Replace(river, newRiver));
}
subscriptionStore.SaveProfileFor(user, newProfile).Wait();
Console.WriteLine("OK");
return(0);
}
protected override string GetObjectID(Uri id) => SyndicationUtil.HashString(id.AbsoluteUri);
protected override string GetObjectID(string id) => SyndicationUtil.HashString(id);
async Task <ThumbnailResponse> FindThumbnailInSoupAsync(string soup, Uri baseUrl, IHtmlDocument document)
{
// These get preferential treatment; if we find them then great otherwise we have to search the whole doc.
// (Note that they also still have to pass the URL filter.)
ImageUrl easyUri = SyndicationUtil.ConcatSequence(
ExtractOpenGraphImageUrls(baseUrl, document),
ExtractTwitterImageUrls(baseUrl, document),
ExtractLinkRelImageUrls(baseUrl, document),
ExtractKnownGoodnessImageUrls(baseUrl, document)
).FirstOrDefault();
if (easyUri != null)
{
ThumbnailResponse easyResponse = await FetchThumbnailAsync(easyUri, baseUrl, skipChecks : true);
ThumbnailLog.LogThumbnail(baseUrl, easyUri.Uri, easyUri.Kind, easyResponse, "Best");
return(easyResponse);
}
IEnumerable <Uri> distinctSrc =
(from element in document.GetElementsByTagName("img")
let src = MakeThumbnailUrl(baseUrl, element.Attributes["src"]?.Value)
where src != null
select src).Distinct();
ImageUrl[] imageUrls =
(from src in distinctSrc
select new ImageUrl {
Uri = src, Kind = "ImgTag"
}).ToArray();
Stopwatch loadTimer = Stopwatch.StartNew();
Log.BeginGetThumbsFromSoup(soup, baseUrl, imageUrls.Length);
var potentialThumbnails = new Task <ThumbnailResponse> [imageUrls.Length];
for (int i = 0; i < potentialThumbnails.Length; i++)
{
potentialThumbnails[i] = FetchThumbnailAsync(imageUrls[i], baseUrl);
}
ThumbnailResponse[] images = await Task.WhenAll(potentialThumbnails);
Log.EndGetThumbsFromSoup(soup, baseUrl, imageUrls.Length, loadTimer);
ImageUrl bestImageUrl = null;
ThumbnailResponse bestImage = null;
float bestArea = 0;
for (int i = 0; i < images.Length; i++)
{
ImageUrl imageUrl = imageUrls[i];
ThumbnailResponse image = images[i];
if (image == null)
{
continue;
} // It was invalid.
int width = image.OriginalWidth;
int height = image.OriginalHeight;
float area = width * height;
if (area < 5000)
{
ThumbnailLog.LogThumbnail(baseUrl, imageUrl.Uri, imageUrl.Kind, image, "Small");
CacheError(imageUrl, "Too Small");
continue;
}
float ratio = (float)Math.Max(width, height) / (float)Math.Min(width, height);
if (ratio > 2.25f)
{
ThumbnailLog.LogThumbnail(baseUrl, imageUrl.Uri, imageUrl.Kind, image, "Oblong");
CacheError(imageUrl, "Too Oblong");
continue;
}
if (imageUrl.Uri.AbsolutePath.Contains("sprite"))
{
area /= 10;
} // Penalize images named "sprite"
if (area > bestArea)
{
if (bestImageUrl != null)
{
CacheError(bestImageUrl, "Not the best");
ThumbnailLog.LogThumbnail(baseUrl, bestImageUrl.Uri, bestImageUrl.Kind, bestImage, "NotBest");
}
bestArea = area;
bestImage = image;
bestImageUrl = imageUrls[i];
}
else
{
ThumbnailLog.LogThumbnail(baseUrl, imageUrl.Uri, imageUrl.Kind, image, "NotBest");
}
}
if (bestImage != null)
{
ThumbnailLog.LogThumbnail(baseUrl, bestImageUrl.Uri, bestImageUrl.Kind, bestImage, "Best");
Log.FoundThumbnail(soup, baseUrl, bestImageUrl.Uri, bestImageUrl.Kind);
}
else
{
Log.NoThumbnailFound(soup, baseUrl);
}
return(bestImage);
}
