Ejemplo n.º 1
0
        public async Task ProcessPosts(IEnumerable <Post> posts, TraceWriter log, string likerBlogname = null)
        {
            foreach (Post post in posts)
            {
                SanitizePostPhotos(post); // sometimes post.Photos has Alt_sizes with length 0, needs to be sanitized

                PostEntity postEntityInTable = postsTableAdapter.GetPost(post.Blog_name, post.Id.ToString());

                PostEntity postEntityFromTumblr = new PostEntity(post);

                if (!postsTableAdapter.InsertPost(postEntityFromTumblr))
                {
                    break;
                }

                if (likerBlogname != null && post.Liked_Timestamp.HasValue)
                {
                    likeIndexTableAdapter.InsertLikeIndex(likerBlogname, post.Liked_Timestamp.ToString(), post.Blog_name, post.Id.ToString(), post.Reblog_key);
                }

                log.Info("Post " + post.Blog_name + "/" + post.Id + " inserted to table");

                PhotosToDownload photosToDownloadMessage = null;

                if (postEntityFromTumblr.PhotosJson != null)
                {
                    if (postEntityInTable == null || postEntityInTable.PicsDownloadLevel == null ||
                        postEntityInTable.PicsDownloadLevel < Constants.MaxPicsDownloadLevel)
                    {
                        photosToDownloadMessage = new PhotosToDownload(post)
                        {
                            Photos = post.Photos
                        };
                    }
                    else
                    {
                        log.Info("Photos already downloaded");
                    }
                }

                List <VideoUrls> videoUrlsList = new List <VideoUrls>();

                if (post.Content != null && post.Content.Length > 0)
                {
                    List <Photo> photos = new List <Photo>(post.Content.Length);

                    foreach (Content content in post.Content)
                    {
                        if (content.Type == "image")
                        {
                            Photo photo = ConvertContentToPhoto(content);
                            photos.Add(photo);
                        }
                        else if (content.Type == "video" && content.Url != null && content.Poster != null)
                        {
                            VideoUrls videoUrls = new VideoUrls
                            {
                                VideoUrl      = content.Url,
                                VideoThumbUrl = content.Poster.OrderBy(x => x.Width).LastOrDefault()?.Url
                            };
                            videoUrlsList.Add(videoUrls);
                        }
                    }

                    if (photos.Count > 0)
                    {
                        UpdatePhotosToDownloadMessage(ref photosToDownloadMessage, post, photos);
                    }
                }

                if (postEntityInTable == null || postEntityInTable.VideosDownloadLevel == null ||
                    postEntityInTable.VideosDownloadLevel < Constants.MaxVideosDownloadLevel)
                {
                    if (!string.IsNullOrEmpty(post.Video_url))
                    {
                        VideoUrls videoUrls = new VideoUrls
                        {
                            VideoUrl      = post.Video_url,
                            VideoThumbUrl = post.Thumbnail_url
                        };

                        videoUrlsList.Add(videoUrls);
                    }

                    if (post.Player != null && post.Player.Length > 0 && post.Video_type.Equals("instagram", StringComparison.OrdinalIgnoreCase))
                    {
                        Player largestPlayer = post.Player.OrderBy(x => x.Width).Last();

                        HtmlDocument playerHtmlDoc = new HtmlDocument();
                        playerHtmlDoc.LoadHtml(largestPlayer.Embed_code);
                        HtmlNode blockquoteNode = playerHtmlDoc.DocumentNode.Descendants("blockquote")
                                                  .FirstOrDefault(x => !string.IsNullOrEmpty(x.Attributes["data-instgrm-permalink"].Value));
                        if (blockquoteNode != null)
                        {
                            string    url       = blockquoteNode.Attributes["data-instgrm-permalink"].Value;
                            VideoUrls videoUrls = await GetInstagramVideo(url);

                            if (videoUrls != null)
                            {
                                videoUrlsList.Add(videoUrls);
                            }
                        }
                    }
                }

                if (!string.IsNullOrEmpty(post.Body))
                {
                    HtmlDocument htmlDoc = new HtmlDocument();
                    htmlDoc.LoadHtml(post.Body);
                    if (postEntityInTable == null || postEntityInTable.PicsDownloadLevel == null ||
                        postEntityInTable.PicsDownloadLevel < Constants.MaxPicsDownloadLevel)
                    {
                        List <Photo> photos = ExctractPhotosFromHtml(htmlDoc);

                        if (photos.Count > 0)
                        {
                            photosToDownloadMessage = UpdatePhotosToDownloadMessage(ref photosToDownloadMessage, post, photos);
                        }
                    }

                    if (postEntityInTable == null || postEntityInTable.VideosDownloadLevel == null ||
                        postEntityInTable.VideosDownloadLevel < Constants.MaxVideosDownloadLevel)
                    {
                        List <VideoUrls> videoUrlsListFromBody = GetVideoUrls(htmlDoc, log);
                        videoUrlsList.AddRange(videoUrlsListFromBody);
                    }
                }

                if (photosToDownloadMessage != null)
                {
                    queueAdapter.SendPhotosToDownload(photosToDownloadMessage);
                    log.Info("PhotosToDownload message published");
                }

                if (videoUrlsList.Count > 0)
                {
                    VideosToDownload videosToDownload = new VideosToDownload(post)
                    {
                        VideoUrls = videoUrlsList.ToArray()
                    };

                    queueAdapter.SendVideosToDownload(videosToDownload);
                    log.Info("VideosToDownload message published");
                }
            }
        }
Ejemplo n.º 2
0
        private static int InsertReversePosts(string blogname, Dictionary <string, List <Model.Site.Photo> > photosByBlogById, List <PostEntity> postEntities,
                                              ReversePostsTableAdapter reversePostsTableAdapter, PostsTableAdapter postsTableAdapter,
                                              PhotoIndexTableAdapter photoIndexTableAdapter, MediaToDownloadQueueAdapter mediaToDownloadQueueAdapter, TraceWriter log)
        {
            int index = 0;

            List <ReversePostEntity> reverseEntities = new List <ReversePostEntity>(100);

            foreach (PostEntity entity in postEntities)
            {
                ReversePostEntity reversePost =
                    new ReversePostEntity(entity.PartitionKey, entity.RowKey, entity.Type, entity.Date, entity.ModifiedBody, entity.Title);
                if (photosByBlogById.TryGetValue(entity.RowKey, out List <Model.Site.Photo> photos))
                {
                    reversePost.Photos = JsonConvert.SerializeObject(photos, JsonUtils.JsonSerializerSettings);
                }
                else if (!string.IsNullOrEmpty(entity.VideoBlobUrls) && entity.VideoBlobUrls.StartsWith("[{"))
                {
                    reversePost.Videos = entity.VideoBlobUrls;
                }

                if (string.IsNullOrEmpty(entity.ModifiedBody) && !string.IsNullOrEmpty(entity.Body))
                {
                    string sourceBlog = string.IsNullOrEmpty(entity.SourceTitle) ? blogname : SanityHelper.SanitizeSourceBlog(entity.SourceTitle);

                    string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, entity.Body, photoIndexTableAdapter, photos, out List <Photo> extractedPhotos);
                    if (extractedPhotos != null && extractedPhotos.Count > 0)
                    {
                        PhotosToDownload photosToDownload = new PhotosToDownload(entity)
                        {
                            Photos = extractedPhotos.ToArray()
                        };
                        mediaToDownloadQueueAdapter.SendPhotosToDownload(photosToDownload);
                        log.Warning("Could not modify body successfully, sending PhotosToDownload to get missing photos");
                    }
                    else
                    {
                        entity.ModifiedBody = modifiedBody;

                        postsTableAdapter.InsertPost(entity);
                        log.Info($"ModifiedBody updated on post {entity.PartitionKey}/{entity.RowKey}");
                    }
                }

                if (!string.IsNullOrEmpty(reversePost.Photos) || !string.IsNullOrEmpty(reversePost.Videos) || !string.IsNullOrEmpty(reversePost.Body))
                {
                    reverseEntities.Add(reversePost);

                    index++;
                    if (index % 100 == 0)
                    {
                        reversePostsTableAdapter.InsertBatch(reverseEntities);
                        reverseEntities.Clear();
                        log.Info("Inserted " + index + " reverse posts for " + entity.PartitionKey);
                    }
                }
            }

            reversePostsTableAdapter.InsertBatch(reverseEntities);
            log.Info("Inserted " + index + " reverse posts for " + blogname);

            return(index);
        }