public static async Task Run([QueueTrigger(Constants.BlogToIndexQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); BlogToIndex blogToIndex = JsonConvert.DeserializeObject <BlogToIndex>(myQueueItem); PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter(); photoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); PostToGetQueueAdapter postToGetQueueAdapter = new PostToGetQueueAdapter(); postToGetQueueAdapter.Init(); BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter(); blogInfoTableAdapter.Init(); MediaToDownloadQueueAdapter mediaToDownloadQueueAdapter = new MediaToDownloadQueueAdapter(); mediaToDownloadQueueAdapter.Init(log); List <PhotoIndexEntity> photoIndexEntities = photoIndexTableAdapter.GetAll(blogToIndex.Blogname); log.Info("Loaded " + photoIndexEntities.Count + " photo index entities"); BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToIndex.Blogname); Dictionary <string, List <Model.Site.Photo> > photosByBlogById = CreatePhotosByBlogById(photoIndexEntities); BlogStats blogStats = CreateBlogStatsFromPhotos(photoIndexEntities, blogToIndex.Blogname); blogStats.UpdateFromBlogEntity(blogEntity); List <PostEntity> postEntities = postsTableAdapter.GetAll(blogToIndex.Blogname); UpdateBlogStatsFromPosts(blogStats, postEntities); UpdateMonthIndex(blogToIndex.Blogname, postEntities, blogInfoTableAdapter); log.Info("Loaded " + postEntities.Count + " post entities"); foreach (PostEntity postEntity in postEntities) { if (!string.IsNullOrEmpty(postEntity.PhotoBlobUrls)) { try { Model.Site.Photo[] photos = JsonConvert.DeserializeObject <Model.Site.Photo[]>(postEntity.PhotoBlobUrls); if (photos.Any(x => !x.Name.Contains("_"))) { SendToReprocessing(postEntity.PartitionKey, mediaToDownloadQueueAdapter, log, postEntity); } } catch (Exception e) { log.Error("Error: " + e.Message); throw; } } } blogStats.DisplayablePosts = InsertReversePosts(blogToIndex.Blogname, photosByBlogById, postEntities, reversePostsTableAdapter, postsTableAdapter, photoIndexTableAdapter, mediaToDownloadQueueAdapter, log); blogInfoTableAdapter.InsertBlobStats(blogStats); }
public static async Task Run([QueueTrigger(Constants.PhotosToDownloadQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); string requestUrl = null; try { PhotosToDownload photosToDownload = JsonConvert.DeserializeObject <PhotosToDownload>(myQueueItem); BlobAdapter blobAdapter = new BlobAdapter(); blobAdapter.Init(); PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter(); photoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); List <Photo> sitePhotos = new List <Photo>(); string blogname = photosToDownload.IndexInfo.BlogName; string id = photosToDownload.IndexInfo.PostId; DateTime date = photosToDownload.IndexInfo.PostDate; string sourceBlog = string.IsNullOrEmpty(photosToDownload.SourceBlog) ? photosToDownload.IndexInfo.BlogName : photosToDownload.SourceBlog; sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog); using (HttpClient httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*")); foreach (Model.Tumblr.Photo photo in photosToDownload.Photos) { bool isOriginal = true; Photo sitePhoto = null; foreach (AltSize altSize in photo.Alt_sizes) { PhotoUrlHelper urlHelper = PhotoUrlHelper.ParseTumblr(altSize.Url); if (isOriginal || urlHelper != null && DownloadSizes.Contains(urlHelper.Size)) { if (sitePhoto == null) { sitePhoto = new Photo { Name = urlHelper.Container + "_" + urlHelper.Name, Extension = urlHelper.Extension, Sizes = new PhotoSize[0] } } ; PhotoUrlIndexEntity urlIndexEntity = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, altSize.Url); if (urlIndexEntity != null) // photo already downloaded { AddSizeToSitePhoto(sitePhoto, urlIndexEntity.BlobUrl, altSize); // need this to produce correct sitePhotos isOriginal = false; } else // photo not downloaded { requestUrl = altSize.Url; byte[] photoBytes = await httpClient.GetByteArrayAsync(altSize.Url); if (photoBytes.Length > 0) { Uri blobUri = await blobAdapter.UploadPhotoBlob(urlHelper, photoBytes, isOriginal); AddSizeToSitePhoto(sitePhoto, blobUri.ToString(), altSize); photoIndexTableAdapter.InsertPhotoIndex(blogname, id, date, SanityHelper.SanitizeSourceBlog(photosToDownload.SourceBlog), blobUri.ToString(), urlHelper.Name, urlHelper.Size, altSize.Width, altSize.Height, altSize.Url); isOriginal = false; log.Info("Downloaded photo from: " + altSize.Url); } } } } if (sitePhoto?.Sizes.Length > 0) { sitePhotos.Add(sitePhoto); } } } string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, photosToDownload.Body, photoIndexTableAdapter, sitePhotos, out List <Model.Tumblr.Photo> extractedPhotos); if (extractedPhotos != null) { log.Warning("Trying to modify body in ProcessPhotosToDownload but some images were not possible to replace"); } postsTableAdapter.MarkPhotosAsDownloaded(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, sitePhotos, modifiedBody); ReversePostEntity reversePost = new ReversePostEntity(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, photosToDownload.PostType, photosToDownload.IndexInfo.PostDate, modifiedBody, photosToDownload.Title) { Photos = JsonConvert.SerializeObject(sitePhotos) }; reversePostsTableAdapter.InsertPost(reversePost); } catch (Exception ex) { if (ex is HttpRequestException httpRequestException && httpRequestException.Message.Contains("403") && httpRequestException.Message.Contains("Forbidden")) { log.Warning("HTTP request was forbidden to URL: " + requestUrl); }