private static Photo ConvertContentToPhoto(Content content)
{
return(new Photo
{
Alt_sizes = (from media in content.Media
let helper = PhotoUrlHelper.ParseTumblr(media.Url)
where DownloadSizes.Contains(helper.Size) && (helper.VSize == null || helper.VSize > helper.Size)
select new AltSize {
Url = media.Url, Width = media.Width, Height = media.Height
}).ToArray()
});
}
public void TestParseNewSquareUrl()
{
PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(newSquareUrl);
Assert.IsNotNull(helper);
Assert.AreEqual(46, helper.Server);
Assert.AreEqual(400, helper.Size);
Assert.AreEqual(600, helper.VSize);
Assert.AreEqual("a239440e3fa7a99a898401d85197aea0", helper.Container);
Assert.AreEqual("790fc572931cda8a-78", helper.Name);
Assert.AreEqual("ce2a425b67c9e3605f6545c6c308327af5ca71f4", helper.Hash);
Assert.AreEqual("gif", helper.Extension);
}
public void TestParseNewUrl()
{
PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(newUrl);
Assert.IsNotNull(helper);
Assert.AreEqual(66, helper.Server);
Assert.AreEqual(1280, helper.Size);
Assert.AreEqual(1920, helper.VSize);
Assert.AreEqual("2658176ff018529c2dfbacb57a8a36dc", helper.Container);
Assert.AreEqual("4edcb8991d3eea7f-00", helper.Name);
Assert.AreEqual("7c1f445b8d63b45cb881c7a86eba4459f05d9a96", helper.Hash);
Assert.AreEqual("jpg", helper.Extension);
}
public void TestParseOldSquareUrl()
{
PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(oldSquareUrl);
Assert.IsNotNull(helper);
Assert.AreEqual(66, helper.Server);
Assert.AreEqual(75, helper.Size);
Assert.AreEqual("f63951217c08428cc3db05cdfa440426", helper.Container);
Assert.AreEqual("ptauwmXZAa2t88716", helper.Name);
Assert.AreEqual("jpg", helper.Extension);
Assert.IsNull(helper.Hash);
Assert.IsNull(helper.VSize);
}
public void TestParseOldUrl()
{
PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(oldUrl);
Assert.IsNotNull(helper);
Assert.AreEqual(66, helper.Server);
Assert.AreEqual(500, helper.Size);
Assert.AreEqual("2d0b81c6d988de86dcaf9abaaaaab563", helper.Container);
Assert.AreEqual("p738u7LON31tum2j0o1", helper.Name);
Assert.AreEqual("jpg", helper.Extension);
Assert.IsNull(helper.Hash);
Assert.IsNull(helper.VSize);
}
private static string TryToGetMappedUrl(string origUrl, List <Photo> sitePhotos, string sourceBlog, PhotoIndexTableAdapter photoIndexTableAdapter)
{
PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(origUrl);
if (helper != null)
{
if (sitePhotos != null)
{
foreach (Photo sitePhoto in sitePhotos)
{
int underscoreIndex = sitePhoto.Name.IndexOf("_", StringComparison.Ordinal);
if (underscoreIndex >= 0)
{
string containerPart = sitePhoto.Name.Substring(0, underscoreIndex);
string namePart = sitePhoto.Name.Substring(underscoreIndex + 1, sitePhoto.Name.Length - underscoreIndex - 1);
if (namePart.Equals(helper.Name) && containerPart.Equals(helper.Container))
{
PhotoSize photoSize = sitePhoto.Sizes.OrderBy(x => x.Nominal).Last();
string blobBaseUrl = ConfigurationManager.AppSettings["BlobBaseUrl"];
string newUrl = blobBaseUrl + "/" + photoSize.Container + "/" + sitePhoto.Name + "_" + photoSize.Nominal + "." +
sitePhoto.Extension;
return(newUrl);
}
}
}
}
PhotoUrlIndexEntity photoIndex = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, origUrl);
if (photoIndex != null)
{
return(photoIndex.BlobUrl);
}
string url = "https://" + helper.Server + ".media.tumblr.com/" + (helper.Container != null ? helper.Container + "/" : "") + "tumblr_" +
helper.Name + "_" + 640 + "." + helper.Extension;
photoIndex = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, url);
if (photoIndex != null)
{
return(photoIndex.BlobUrl);
}
}
return(null);
}
private static Photo GeneratePhotoFromSrc(string src)
{
PhotoUrlHelper helper = PhotoUrlHelper.ParseTumblr(src);
if (helper == null)
{
return(null);
}
Photo photo = new Photo();
List <AltSize> altSizes = new List <AltSize>();
if (!string.IsNullOrEmpty(helper.Hash)) // new format photo url
{
altSizes.Add(new AltSize {
Url = src
});
}
else
{
foreach (int size in DownloadSizes)
{
AltSize altSize = new AltSize
{
Url = "https://" + helper.Server + ".media.tumblr.com/" + (helper.Container != null ? helper.Container + "/" : "") + "tumblr_" +
helper.Name + "_" + size + "." + helper.Extension,
Width = 0,
Height = 0
};
altSizes.Add(altSize);
}
}
photo.Alt_sizes = altSizes.ToArray();
return(photo);
}
public static async Task Run([QueueTrigger(Constants.PhotosToDownloadQueueName, Connection = "AzureWebJobsStorage")]
string myQueueItem, TraceWriter log)
{
Startup.Init();
string requestUrl = null;
try
{
PhotosToDownload photosToDownload = JsonConvert.DeserializeObject <PhotosToDownload>(myQueueItem);
BlobAdapter blobAdapter = new BlobAdapter();
blobAdapter.Init();
PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter();
photoIndexTableAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter();
reversePostsTableAdapter.Init(log);
List <Photo> sitePhotos = new List <Photo>();
string blogname = photosToDownload.IndexInfo.BlogName;
string id = photosToDownload.IndexInfo.PostId;
DateTime date = photosToDownload.IndexInfo.PostDate;
string sourceBlog = string.IsNullOrEmpty(photosToDownload.SourceBlog)
? photosToDownload.IndexInfo.BlogName
: photosToDownload.SourceBlog;
sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog);
using (HttpClient httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*"));
foreach (Model.Tumblr.Photo photo in photosToDownload.Photos)
{
bool isOriginal = true;
Photo sitePhoto = null;
foreach (AltSize altSize in photo.Alt_sizes)
{
PhotoUrlHelper urlHelper = PhotoUrlHelper.ParseTumblr(altSize.Url);
if (isOriginal || urlHelper != null && DownloadSizes.Contains(urlHelper.Size))
{
if (sitePhoto == null)
{
sitePhoto = new Photo
{
Name = urlHelper.Container + "_" + urlHelper.Name,
Extension = urlHelper.Extension,
Sizes = new PhotoSize[0]
}
}
;
PhotoUrlIndexEntity urlIndexEntity = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, altSize.Url);
if (urlIndexEntity != null) // photo already downloaded
{
AddSizeToSitePhoto(sitePhoto, urlIndexEntity.BlobUrl, altSize); // need this to produce correct sitePhotos
isOriginal = false;
}
else // photo not downloaded
{
requestUrl = altSize.Url;
byte[] photoBytes = await httpClient.GetByteArrayAsync(altSize.Url);
if (photoBytes.Length > 0)
{
Uri blobUri = await blobAdapter.UploadPhotoBlob(urlHelper, photoBytes, isOriginal);
AddSizeToSitePhoto(sitePhoto, blobUri.ToString(), altSize);
photoIndexTableAdapter.InsertPhotoIndex(blogname, id, date, SanityHelper.SanitizeSourceBlog(photosToDownload.SourceBlog),
blobUri.ToString(), urlHelper.Name, urlHelper.Size,
altSize.Width, altSize.Height, altSize.Url);
isOriginal = false;
log.Info("Downloaded photo from: " + altSize.Url);
}
}
}
}
if (sitePhoto?.Sizes.Length > 0)
{
sitePhotos.Add(sitePhoto);
}
}
}
string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, photosToDownload.Body, photoIndexTableAdapter, sitePhotos, out List <Model.Tumblr.Photo> extractedPhotos);
if (extractedPhotos != null)
{
log.Warning("Trying to modify body in ProcessPhotosToDownload but some images were not possible to replace");
}
postsTableAdapter.MarkPhotosAsDownloaded(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, sitePhotos, modifiedBody);
ReversePostEntity reversePost = new ReversePostEntity(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId,
photosToDownload.PostType, photosToDownload.IndexInfo.PostDate, modifiedBody, photosToDownload.Title)
{
Photos = JsonConvert.SerializeObject(sitePhotos)
};
reversePostsTableAdapter.InsertPost(reversePost);
}
catch (Exception ex)
{
if (ex is HttpRequestException httpRequestException && httpRequestException.Message.Contains("403") && httpRequestException.Message.Contains("Forbidden"))
{
log.Warning("HTTP request was forbidden to URL: " + requestUrl);
}
