public ReversePostEntity GetPost(string blogName, string id) { TableOperation retrieveOperation = TableOperation.Retrieve <ReversePostEntity>(blogName, id); TableResult result = reversePostsTable.Execute(retrieveOperation); if (result.HttpStatusCode == 200) { ReversePostEntity entity = (ReversePostEntity)result.Result; return(entity); } return(null); }
public bool InsertPost(ReversePostEntity reversePostEntity) { TableOperation insertOrMergeOperation = TableOperation.InsertOrMerge(reversePostEntity); try { reversePostsTable.Execute(insertOrMergeOperation); } catch (StorageException ex) { log.Warning("Saving ReversePostEntity " + reversePostEntity.PartitionKey + "/" + reversePostEntity.RowKey + " failed: " + ex.Message); return(false); } return(true); }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "posts/{blogname}/{id}")] HttpRequestMessage req, string blogname, string id, TraceWriter log) { ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); ReversePostEntity entity = reversePostsTableAdapter.GetPost(blogname, id); Model.Site.Post post = entity != null?entity.GetSitePost() : null; string postJson = JsonConvert.SerializeObject(post, JsonSerializerSettings); HttpResponseMessage response = req.CreateResponse(HttpStatusCode.OK); response.Content = new StringContent(postJson, Encoding.UTF8, "application/json"); return(response); }
private static void UpdateMonthIndex(string blogname, List <PostEntity> postEntities, BlogInfoTableAdapter blogInfoTableAdapter) { Dictionary <string, MonthIndex> indexEntriesByMonth = new Dictionary <string, MonthIndex>(); foreach (PostEntity postEntity in postEntities) { string monthKey = postEntity.Date.ToString(MonthIndex.MonthKeyFormat); if (!indexEntriesByMonth.TryGetValue(monthKey, out MonthIndex indexEntry)) { indexEntry = new MonthIndex(blogname, monthKey); indexEntriesByMonth.Add(monthKey, indexEntry); } indexEntry.MonthsPosts++; long postId = long.Parse(postEntity.RowKey); if (indexEntry.FirstPostId == 0 || postId < indexEntry.FirstPostId) { indexEntry.FirstPostId = ReversePostEntity.GetRowKeyId(postId); } } blogInfoTableAdapter.InsertMonthIndice(indexEntriesByMonth.Values); }
private static int InsertReversePosts(string blogname, Dictionary <string, List <Model.Site.Photo> > photosByBlogById, List <PostEntity> postEntities, ReversePostsTableAdapter reversePostsTableAdapter, PostsTableAdapter postsTableAdapter, PhotoIndexTableAdapter photoIndexTableAdapter, MediaToDownloadQueueAdapter mediaToDownloadQueueAdapter, TraceWriter log) { int index = 0; List <ReversePostEntity> reverseEntities = new List <ReversePostEntity>(100); foreach (PostEntity entity in postEntities) { ReversePostEntity reversePost = new ReversePostEntity(entity.PartitionKey, entity.RowKey, entity.Type, entity.Date, entity.ModifiedBody, entity.Title); if (photosByBlogById.TryGetValue(entity.RowKey, out List <Model.Site.Photo> photos)) { reversePost.Photos = JsonConvert.SerializeObject(photos, JsonUtils.JsonSerializerSettings); } else if (!string.IsNullOrEmpty(entity.VideoBlobUrls) && entity.VideoBlobUrls.StartsWith("[{")) { reversePost.Videos = entity.VideoBlobUrls; } if (string.IsNullOrEmpty(entity.ModifiedBody) && !string.IsNullOrEmpty(entity.Body)) { string sourceBlog = string.IsNullOrEmpty(entity.SourceTitle) ? blogname : SanityHelper.SanitizeSourceBlog(entity.SourceTitle); string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, entity.Body, photoIndexTableAdapter, photos, out List <Photo> extractedPhotos); if (extractedPhotos != null && extractedPhotos.Count > 0) { PhotosToDownload photosToDownload = new PhotosToDownload(entity) { Photos = extractedPhotos.ToArray() }; mediaToDownloadQueueAdapter.SendPhotosToDownload(photosToDownload); log.Warning("Could not modify body successfully, sending PhotosToDownload to get missing photos"); } else { entity.ModifiedBody = modifiedBody; postsTableAdapter.InsertPost(entity); log.Info($"ModifiedBody updated on post {entity.PartitionKey}/{entity.RowKey}"); } } if (!string.IsNullOrEmpty(reversePost.Photos) || !string.IsNullOrEmpty(reversePost.Videos) || !string.IsNullOrEmpty(reversePost.Body)) { reverseEntities.Add(reversePost); index++; if (index % 100 == 0) { reversePostsTableAdapter.InsertBatch(reverseEntities); reverseEntities.Clear(); log.Info("Inserted " + index + " reverse posts for " + entity.PartitionKey); } } } reversePostsTableAdapter.InsertBatch(reverseEntities); log.Info("Inserted " + index + " reverse posts for " + blogname); return(index); }
public static async Task Run([QueueTrigger(Constants.VideosToDownloadQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); log.Info($"C# Queue trigger function processed: {myQueueItem}"); VideosToDownload videosToDownload = JsonConvert.DeserializeObject <VideosToDownload>(myQueueItem); BlobAdapter blobAdapter = new BlobAdapter(); blobAdapter.Init(); VideoIndexTableAdapter videoIndexTableAdapter = new VideoIndexTableAdapter(); videoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); string sourceBlog = string.IsNullOrEmpty(videosToDownload.SourceBlog) ? videosToDownload.IndexInfo.BlogName : videosToDownload.SourceBlog; sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog); using (HttpClient httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("video/*")); httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*")); List <Video> videos = new List <Video>(); string blogname = videosToDownload.IndexInfo.BlogName; string id = videosToDownload.IndexInfo.PostId; DateTime date = videosToDownload.IndexInfo.PostDate; foreach (VideoUrls videoUrls in videosToDownload.VideoUrls) { try { Video blobVideo = await blobAdapter.HandleVideo(videoUrls, videosToDownload.IndexInfo.BlogName, log); videos.Add(blobVideo); videoIndexTableAdapter.InsertVideoIndex(blogname, id, date, blobVideo, videosToDownload.VideoType, blobVideo.Bytes, videosToDownload.Duration); log.Info("Video successfully downloaded: " + videoUrls.VideoUrl); } catch (HttpRequestException ex) { log.Warning("HTTP Error while downloading video " + videoUrls.VideoUrl + " - " + ex.Message); postsTableAdapter.MarkWithVideoDownloadError(blogname, id, ex.Message); } catch (Exception ex) { log.Error("Error while downloading video ", ex); throw; } } if (videos.Count > 0) { postsTableAdapter.MarkVideosAsDownloaded(videosToDownload.IndexInfo.BlogName, videosToDownload.IndexInfo.PostId, videos.ToArray()); ReversePostEntity reversePost = new ReversePostEntity(blogname, id, videosToDownload.PostType, date, videosToDownload.Body, videosToDownload.Title) { Videos = JsonConvert.SerializeObject(videos) }; reversePostsTableAdapter.InsertPost(reversePost); } } }
public static async Task Run([QueueTrigger(Constants.PhotosToDownloadQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); string requestUrl = null; try { PhotosToDownload photosToDownload = JsonConvert.DeserializeObject <PhotosToDownload>(myQueueItem); BlobAdapter blobAdapter = new BlobAdapter(); blobAdapter.Init(); PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter(); photoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); List <Photo> sitePhotos = new List <Photo>(); string blogname = photosToDownload.IndexInfo.BlogName; string id = photosToDownload.IndexInfo.PostId; DateTime date = photosToDownload.IndexInfo.PostDate; string sourceBlog = string.IsNullOrEmpty(photosToDownload.SourceBlog) ? photosToDownload.IndexInfo.BlogName : photosToDownload.SourceBlog; sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog); using (HttpClient httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*")); foreach (Model.Tumblr.Photo photo in photosToDownload.Photos) { bool isOriginal = true; Photo sitePhoto = null; foreach (AltSize altSize in photo.Alt_sizes) { PhotoUrlHelper urlHelper = PhotoUrlHelper.ParseTumblr(altSize.Url); if (isOriginal || urlHelper != null && DownloadSizes.Contains(urlHelper.Size)) { if (sitePhoto == null) { sitePhoto = new Photo { Name = urlHelper.Container + "_" + urlHelper.Name, Extension = urlHelper.Extension, Sizes = new PhotoSize[0] } } ; PhotoUrlIndexEntity urlIndexEntity = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, altSize.Url); if (urlIndexEntity != null) // photo already downloaded { AddSizeToSitePhoto(sitePhoto, urlIndexEntity.BlobUrl, altSize); // need this to produce correct sitePhotos isOriginal = false; } else // photo not downloaded { requestUrl = altSize.Url; byte[] photoBytes = await httpClient.GetByteArrayAsync(altSize.Url); if (photoBytes.Length > 0) { Uri blobUri = await blobAdapter.UploadPhotoBlob(urlHelper, photoBytes, isOriginal); AddSizeToSitePhoto(sitePhoto, blobUri.ToString(), altSize); photoIndexTableAdapter.InsertPhotoIndex(blogname, id, date, SanityHelper.SanitizeSourceBlog(photosToDownload.SourceBlog), blobUri.ToString(), urlHelper.Name, urlHelper.Size, altSize.Width, altSize.Height, altSize.Url); isOriginal = false; log.Info("Downloaded photo from: " + altSize.Url); } } } } if (sitePhoto?.Sizes.Length > 0) { sitePhotos.Add(sitePhoto); } } } string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, photosToDownload.Body, photoIndexTableAdapter, sitePhotos, out List <Model.Tumblr.Photo> extractedPhotos); if (extractedPhotos != null) { log.Warning("Trying to modify body in ProcessPhotosToDownload but some images were not possible to replace"); } postsTableAdapter.MarkPhotosAsDownloaded(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, sitePhotos, modifiedBody); ReversePostEntity reversePost = new ReversePostEntity(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, photosToDownload.PostType, photosToDownload.IndexInfo.PostDate, modifiedBody, photosToDownload.Title) { Photos = JsonConvert.SerializeObject(sitePhotos) }; reversePostsTableAdapter.InsertPost(reversePost); } catch (Exception ex) { if (ex is HttpRequestException httpRequestException && httpRequestException.Message.Contains("403") && httpRequestException.Message.Contains("Forbidden")) { log.Warning("HTTP request was forbidden to URL: " + requestUrl); }