public async Task ProcessPosts(IEnumerable <Post> posts, TraceWriter log, string likerBlogname = null)
{
foreach (Post post in posts)
{
SanitizePostPhotos(post); // sometimes post.Photos has Alt_sizes with length 0, needs to be sanitized
PostEntity postEntityInTable = postsTableAdapter.GetPost(post.Blog_name, post.Id.ToString());
PostEntity postEntityFromTumblr = new PostEntity(post);
if (!postsTableAdapter.InsertPost(postEntityFromTumblr))
{
break;
}
if (likerBlogname != null && post.Liked_Timestamp.HasValue)
{
likeIndexTableAdapter.InsertLikeIndex(likerBlogname, post.Liked_Timestamp.ToString(), post.Blog_name, post.Id.ToString(), post.Reblog_key);
}
log.Info("Post " + post.Blog_name + "/" + post.Id + " inserted to table");
PhotosToDownload photosToDownloadMessage = null;
if (postEntityFromTumblr.PhotosJson != null)
{
if (postEntityInTable == null || postEntityInTable.PicsDownloadLevel == null ||
postEntityInTable.PicsDownloadLevel < Constants.MaxPicsDownloadLevel)
{
photosToDownloadMessage = new PhotosToDownload(post)
{
Photos = post.Photos
};
}
else
{
log.Info("Photos already downloaded");
}
}
List <VideoUrls> videoUrlsList = new List <VideoUrls>();
if (post.Content != null && post.Content.Length > 0)
{
List <Photo> photos = new List <Photo>(post.Content.Length);
foreach (Content content in post.Content)
{
if (content.Type == "image")
{
Photo photo = ConvertContentToPhoto(content);
photos.Add(photo);
}
else if (content.Type == "video" && content.Url != null && content.Poster != null)
{
VideoUrls videoUrls = new VideoUrls
{
VideoUrl = content.Url,
VideoThumbUrl = content.Poster.OrderBy(x => x.Width).LastOrDefault()?.Url
};
videoUrlsList.Add(videoUrls);
}
}
if (photos.Count > 0)
{
UpdatePhotosToDownloadMessage(ref photosToDownloadMessage, post, photos);
}
}
if (postEntityInTable == null || postEntityInTable.VideosDownloadLevel == null ||
postEntityInTable.VideosDownloadLevel < Constants.MaxVideosDownloadLevel)
{
if (!string.IsNullOrEmpty(post.Video_url))
{
VideoUrls videoUrls = new VideoUrls
{
VideoUrl = post.Video_url,
VideoThumbUrl = post.Thumbnail_url
};
videoUrlsList.Add(videoUrls);
}
if (post.Player != null && post.Player.Length > 0 && post.Video_type.Equals("instagram", StringComparison.OrdinalIgnoreCase))
{
Player largestPlayer = post.Player.OrderBy(x => x.Width).Last();
HtmlDocument playerHtmlDoc = new HtmlDocument();
playerHtmlDoc.LoadHtml(largestPlayer.Embed_code);
HtmlNode blockquoteNode = playerHtmlDoc.DocumentNode.Descendants("blockquote")
.FirstOrDefault(x => !string.IsNullOrEmpty(x.Attributes["data-instgrm-permalink"].Value));
if (blockquoteNode != null)
{
string url = blockquoteNode.Attributes["data-instgrm-permalink"].Value;
VideoUrls videoUrls = await GetInstagramVideo(url);
if (videoUrls != null)
{
videoUrlsList.Add(videoUrls);
}
}
}
}
if (!string.IsNullOrEmpty(post.Body))
{
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(post.Body);
if (postEntityInTable == null || postEntityInTable.PicsDownloadLevel == null ||
postEntityInTable.PicsDownloadLevel < Constants.MaxPicsDownloadLevel)
{
List <Photo> photos = ExctractPhotosFromHtml(htmlDoc);
if (photos.Count > 0)
{
photosToDownloadMessage = UpdatePhotosToDownloadMessage(ref photosToDownloadMessage, post, photos);
}
}
if (postEntityInTable == null || postEntityInTable.VideosDownloadLevel == null ||
postEntityInTable.VideosDownloadLevel < Constants.MaxVideosDownloadLevel)
{
List <VideoUrls> videoUrlsListFromBody = GetVideoUrls(htmlDoc, log);
videoUrlsList.AddRange(videoUrlsListFromBody);
}
}
if (photosToDownloadMessage != null)
{
queueAdapter.SendPhotosToDownload(photosToDownloadMessage);
log.Info("PhotosToDownload message published");
}
if (videoUrlsList.Count > 0)
{
VideosToDownload videosToDownload = new VideosToDownload(post)
{
VideoUrls = videoUrlsList.ToArray()
};
queueAdapter.SendVideosToDownload(videosToDownload);
log.Info("VideosToDownload message published");
}
}
}
private static int InsertReversePosts(string blogname, Dictionary <string, List <Model.Site.Photo> > photosByBlogById, List <PostEntity> postEntities,
ReversePostsTableAdapter reversePostsTableAdapter, PostsTableAdapter postsTableAdapter,
PhotoIndexTableAdapter photoIndexTableAdapter, MediaToDownloadQueueAdapter mediaToDownloadQueueAdapter, TraceWriter log)
{
int index = 0;
List <ReversePostEntity> reverseEntities = new List <ReversePostEntity>(100);
foreach (PostEntity entity in postEntities)
{
ReversePostEntity reversePost =
new ReversePostEntity(entity.PartitionKey, entity.RowKey, entity.Type, entity.Date, entity.ModifiedBody, entity.Title);
if (photosByBlogById.TryGetValue(entity.RowKey, out List <Model.Site.Photo> photos))
{
reversePost.Photos = JsonConvert.SerializeObject(photos, JsonUtils.JsonSerializerSettings);
}
else if (!string.IsNullOrEmpty(entity.VideoBlobUrls) && entity.VideoBlobUrls.StartsWith("[{"))
{
reversePost.Videos = entity.VideoBlobUrls;
}
if (string.IsNullOrEmpty(entity.ModifiedBody) && !string.IsNullOrEmpty(entity.Body))
{
string sourceBlog = string.IsNullOrEmpty(entity.SourceTitle) ? blogname : SanityHelper.SanitizeSourceBlog(entity.SourceTitle);
string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, entity.Body, photoIndexTableAdapter, photos, out List <Photo> extractedPhotos);
if (extractedPhotos != null && extractedPhotos.Count > 0)
{
PhotosToDownload photosToDownload = new PhotosToDownload(entity)
{
Photos = extractedPhotos.ToArray()
};
mediaToDownloadQueueAdapter.SendPhotosToDownload(photosToDownload);
log.Warning("Could not modify body successfully, sending PhotosToDownload to get missing photos");
}
else
{
entity.ModifiedBody = modifiedBody;
postsTableAdapter.InsertPost(entity);
log.Info($"ModifiedBody updated on post {entity.PartitionKey}/{entity.RowKey}");
}
}
if (!string.IsNullOrEmpty(reversePost.Photos) || !string.IsNullOrEmpty(reversePost.Videos) || !string.IsNullOrEmpty(reversePost.Body))
{
reverseEntities.Add(reversePost);
index++;
if (index % 100 == 0)
{
reversePostsTableAdapter.InsertBatch(reverseEntities);
reverseEntities.Clear();
log.Info("Inserted " + index + " reverse posts for " + entity.PartitionKey);
}
}
}
reversePostsTableAdapter.InsertBatch(reverseEntities);
log.Info("Inserted " + index + " reverse posts for " + blogname);
return(index);
}