public async Task <Uri> UploadPhotoBlob(PhotoUrlHelper urlHelper, byte[] bytes, bool isOriginal) { string prefix = isOriginal ? "orig-" : "thumb-"; CloudBlobContainer serverContainer = cloudBlobClient.GetContainerReference(prefix + urlHelper.Server.ToString()); if (!await serverContainer.ExistsAsync()) { await serverContainer.CreateAsync(); // Set the permissions so the blobs are public. BlobContainerPermissions permissions = new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }; await serverContainer.SetPermissionsAsync(permissions); } string path = GetPhotoPath(urlHelper); CloudBlockBlob blob = serverContainer.GetBlockBlobReference(path); blob.Properties.CacheControl = "max-age=31536000"; blob.Properties.ContentType = GetContentType(path); await blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length); return(blob.Uri); }
private static Model.Site.Photo CreatePhoto(PhotoUrlHelper urlHelper, PhotoIndexEntity entity) { return(new Model.Site.Photo { Name = urlHelper.Name, Extension = urlHelper.Extension, Sizes = new[] { new PhotoSize { Container = urlHelper.Container, Nominal = urlHelper.Size, Heigth = entity.Height, Width = entity.Width } } }); }
private static Photo ConvertContentToPhoto(Content content) { return(new Photo { Alt_sizes = (from media in content.Media let helper = PhotoUrlHelper.ParseTumblr(media.Url) where DownloadSizes.Contains(helper.Size) && (helper.VSize == null || helper.VSize > helper.Size) select new AltSize { Url = media.Url, Width = media.Width, Height = media.Height }).ToArray() }); }
public void TestParseNewSquareUrl() { PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(newSquareUrl); Assert.IsNotNull(helper); Assert.AreEqual(46, helper.Server); Assert.AreEqual(400, helper.Size); Assert.AreEqual(600, helper.VSize); Assert.AreEqual("a239440e3fa7a99a898401d85197aea0", helper.Container); Assert.AreEqual("790fc572931cda8a-78", helper.Name); Assert.AreEqual("ce2a425b67c9e3605f6545c6c308327af5ca71f4", helper.Hash); Assert.AreEqual("gif", helper.Extension); }
public void TestParseNewUrl() { PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(newUrl); Assert.IsNotNull(helper); Assert.AreEqual(66, helper.Server); Assert.AreEqual(1280, helper.Size); Assert.AreEqual(1920, helper.VSize); Assert.AreEqual("2658176ff018529c2dfbacb57a8a36dc", helper.Container); Assert.AreEqual("4edcb8991d3eea7f-00", helper.Name); Assert.AreEqual("7c1f445b8d63b45cb881c7a86eba4459f05d9a96", helper.Hash); Assert.AreEqual("jpg", helper.Extension); }
public void TestParseOldSquareUrl() { PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(oldSquareUrl); Assert.IsNotNull(helper); Assert.AreEqual(66, helper.Server); Assert.AreEqual(75, helper.Size); Assert.AreEqual("f63951217c08428cc3db05cdfa440426", helper.Container); Assert.AreEqual("ptauwmXZAa2t88716", helper.Name); Assert.AreEqual("jpg", helper.Extension); Assert.IsNull(helper.Hash); Assert.IsNull(helper.VSize); }
public void TestParseOldUrl() { PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(oldUrl); Assert.IsNotNull(helper); Assert.AreEqual(66, helper.Server); Assert.AreEqual(500, helper.Size); Assert.AreEqual("2d0b81c6d988de86dcaf9abaaaaab563", helper.Container); Assert.AreEqual("p738u7LON31tum2j0o1", helper.Name); Assert.AreEqual("jpg", helper.Extension); Assert.IsNull(helper.Hash); Assert.IsNull(helper.VSize); }
private static string TryToGetMappedUrl(string origUrl, List <Photo> sitePhotos, string sourceBlog, PhotoIndexTableAdapter photoIndexTableAdapter) { PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(origUrl); if (helper != null) { if (sitePhotos != null) { foreach (Photo sitePhoto in sitePhotos) { int underscoreIndex = sitePhoto.Name.IndexOf("_", StringComparison.Ordinal); if (underscoreIndex >= 0) { string containerPart = sitePhoto.Name.Substring(0, underscoreIndex); string namePart = sitePhoto.Name.Substring(underscoreIndex + 1, sitePhoto.Name.Length - underscoreIndex - 1); if (namePart.Equals(helper.Name) && containerPart.Equals(helper.Container)) { PhotoSize photoSize = sitePhoto.Sizes.OrderBy(x => x.Nominal).Last(); string blobBaseUrl = ConfigurationManager.AppSettings["BlobBaseUrl"]; string newUrl = blobBaseUrl + "/" + photoSize.Container + "/" + sitePhoto.Name + "_" + photoSize.Nominal + "." + sitePhoto.Extension; return(newUrl); } } } } PhotoUrlIndexEntity photoIndex = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, origUrl); if (photoIndex != null) { return(photoIndex.BlobUrl); } string url = "https://" + helper.Server + ".media.tumblr.com/" + (helper.Container != null ? helper.Container + "/" : "") + "tumblr_" + helper.Name + "_" + 640 + "." + helper.Extension; photoIndex = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, url); if (photoIndex != null) { return(photoIndex.BlobUrl); } } return(null); }
private string GetPhotoPath(PhotoUrlHelper urlHelper) { StringBuilder sb = new StringBuilder(); if (urlHelper.Container != null) { sb.Append(urlHelper.Container); sb.Append("_"); } sb.Append(urlHelper.Name); sb.Append("_"); sb.Append(urlHelper.Size); sb.Append("."); sb.Append(urlHelper.Extension); return(sb.ToString()); }
private static Dictionary <string, List <Model.Site.Photo> > CreatePhotosByBlogById(List <PhotoIndexEntity> photoIndexEntities) { Dictionary <string, List <Model.Site.Photo> > currentDictionary = new Dictionary <string, List <Model.Site.Photo> >(); foreach (PhotoIndexEntity entity in photoIndexEntities) { PhotoUrlHelper urlHelper = PhotoUrlHelper.ParsePicai(entity.Uri); if (urlHelper != null) { if (currentDictionary.TryGetValue(entity.PostId, out List <Model.Site.Photo> photosForId)) { Model.Site.Photo photo = photosForId.FirstOrDefault(x => x.Name.Equals(urlHelper.Name)); if (photo == null) { photo = CreatePhoto(urlHelper, entity); photosForId.Add(photo); } else { PhotoSize photoSize = new PhotoSize { Container = urlHelper.Container, Nominal = urlHelper.Size, Heigth = entity.Height, Width = entity.Width }; photo.Sizes = photo.Sizes.Concat(new[] { photoSize }).ToArray(); } } else { photosForId = new List <Model.Site.Photo>(); Model.Site.Photo photo = CreatePhoto(urlHelper, entity); photosForId.Add(photo); currentDictionary.Add(entity.PostId, photosForId); } } else { throw new Exception("Parsing failed: " + entity.Uri); } } return(currentDictionary); }
private static Photo GeneratePhotoFromSrc(string src) { PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(src); if (helper == null) { return(null); } Photo photo = new Photo(); List <AltSize> altSizes = new List <AltSize>(); if (!string.IsNullOrEmpty(helper.Hash)) // new format photo url { altSizes.Add(new AltSize { Url = src }); } else { foreach (int size in DownloadSizes) { AltSize altSize = new AltSize { Url = "https://" + helper.Server + ".media.tumblr.com/" + (helper.Container != null ? helper.Container + "/" : "") + "tumblr_" + helper.Name + "_" + size + "." + helper.Extension, Width = 0, Height = 0 }; altSizes.Add(altSize); } } photo.Alt_sizes = altSizes.ToArray(); return(photo); }
public static async Task Run([QueueTrigger(Constants.PhotosToDownloadQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); string requestUrl = null; try { PhotosToDownload photosToDownload = JsonConvert.DeserializeObject <PhotosToDownload>(myQueueItem); BlobAdapter blobAdapter = new BlobAdapter(); blobAdapter.Init(); PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter(); photoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); List <Photo> sitePhotos = new List <Photo>(); string blogname = photosToDownload.IndexInfo.BlogName; string id = photosToDownload.IndexInfo.PostId; DateTime date = photosToDownload.IndexInfo.PostDate; string sourceBlog = string.IsNullOrEmpty(photosToDownload.SourceBlog) ? photosToDownload.IndexInfo.BlogName : photosToDownload.SourceBlog; sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog); using (HttpClient httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*")); foreach (Model.Tumblr.Photo photo in photosToDownload.Photos) { bool isOriginal = true; Photo sitePhoto = null; foreach (AltSize altSize in photo.Alt_sizes) { PhotoUrlHelper urlHelper = PhotoUrlHelper.ParseTumblr(altSize.Url); if (isOriginal || urlHelper != null && DownloadSizes.Contains(urlHelper.Size)) { if (sitePhoto == null) { sitePhoto = new Photo { Name = urlHelper.Container + "_" + urlHelper.Name, Extension = urlHelper.Extension, Sizes = new PhotoSize[0] } } ; PhotoUrlIndexEntity urlIndexEntity = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, altSize.Url); if (urlIndexEntity != null) // photo already downloaded { AddSizeToSitePhoto(sitePhoto, urlIndexEntity.BlobUrl, altSize); // need this to produce correct sitePhotos isOriginal = false; } else // photo not downloaded { requestUrl = altSize.Url; byte[] photoBytes = await httpClient.GetByteArrayAsync(altSize.Url); if (photoBytes.Length > 0) { Uri blobUri = await blobAdapter.UploadPhotoBlob(urlHelper, photoBytes, isOriginal); AddSizeToSitePhoto(sitePhoto, blobUri.ToString(), altSize); photoIndexTableAdapter.InsertPhotoIndex(blogname, id, date, SanityHelper.SanitizeSourceBlog(photosToDownload.SourceBlog), blobUri.ToString(), urlHelper.Name, urlHelper.Size, altSize.Width, altSize.Height, altSize.Url); isOriginal = false; log.Info("Downloaded photo from: " + altSize.Url); } } } } if (sitePhoto?.Sizes.Length > 0) { sitePhotos.Add(sitePhoto); } } } string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, photosToDownload.Body, photoIndexTableAdapter, sitePhotos, out List <Model.Tumblr.Photo> extractedPhotos); if (extractedPhotos != null) { log.Warning("Trying to modify body in ProcessPhotosToDownload but some images were not possible to replace"); } postsTableAdapter.MarkPhotosAsDownloaded(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, sitePhotos, modifiedBody); ReversePostEntity reversePost = new ReversePostEntity(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, photosToDownload.PostType, photosToDownload.IndexInfo.PostDate, modifiedBody, photosToDownload.Title) { Photos = JsonConvert.SerializeObject(sitePhotos) }; reversePostsTableAdapter.InsertPost(reversePost); } catch (Exception ex) { if (ex is HttpRequestException httpRequestException && httpRequestException.Message.Contains("403") && httpRequestException.Message.Contains("Forbidden")) { log.Warning("HTTP request was forbidden to URL: " + requestUrl); }