public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = "blogs")] HttpRequestMessage req, TraceWriter log)
{
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
List <BlogStats> blogStats = blogInfoTableAdapter.GetBlogStats();
List <BlogInfo> infos = blogStats.Select(x => x.GetSiteBlog()).ToList();
string infosJson = JsonConvert.SerializeObject(infos, JsonSerializerSettings);
HttpResponseMessage response = req.CreateResponse(HttpStatusCode.OK);
response.Content = new StringContent(infosJson, Encoding.UTF8, "application/json");
return(response);
}
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "monthindex/{blogname}")]
HttpRequestMessage req, string blogname)
{
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
List <TableInterface.Entities.MonthIndex> monthIndexEntities = blogInfoTableAdapter.GetMonthIndex(blogname);
IEnumerable <Model.Site.MonthIndex> siteEntities = monthIndexEntities.Select(x => x.GetSiteEntity()).Reverse();
string postsJson = JsonConvert.SerializeObject(siteEntities, JsonSerializerSettings);
HttpResponseMessage response = req.CreateResponse(HttpStatusCode.OK);
response.Content = new StringContent(postsJson, Encoding.UTF8, "application/json");
return(response);
}
private static async Task UpdateBlogInfo(PhotoToAnalyze photoToAnalyze, BlogInfoTableAdapter tableAdapter)
{
BlogEntity info = await tableAdapter.GetBlog(photoToAnalyze.Blog);
bool changed = false;
if (info.AnalyzedStartingFrom == null || info.AnalyzedStartingFrom > photoToAnalyze.PostDate)
{
info.AnalyzedStartingFrom = photoToAnalyze.PostDate;
changed = true;
}
if (info.AnalyzedUntil == null || info.AnalyzedUntil < photoToAnalyze.PostDate)
{
info.AnalyzedUntil = photoToAnalyze.PostDate;
changed = true;
}
if (changed)
{
tableAdapter.InsertBlog(info);
}
}
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "analyzerandomphotos")]
HttpRequestMessage req, TraceWriter log)
{
Startup.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();
imageAnalysisTableAdapter.Init();
PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter();
photoToAnalyzeQueueAdapter.Init();
string blobBaseUrl = ConfigurationManager.AppSettings["BlobBaseUrl"];
int blogsLimit = 50;
int photosInBlogLimit = 10;
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
List <BlogStats> blogStats = blogInfoTableAdapter.GetBlogStats();
log.Info($"Got {blogStats.Count} blogs to index");
Random random = new Random();
blogStats.Shuffle(random);
blogStats = blogStats.Take(blogsLimit).ToList();
int totalCount = 0;
foreach (string blogname in blogStats.Select(x => x.RowKey))
{
int analyzedInBlogCount = 0;
List <PostEntity> noteCounts = postsTableAdapter.GetPostNoteCounts(blogname).OrderByDescending(x => x.NoteCount).ToList();
log.Info($"Got note counts for {noteCounts.Count} posts in blog {blogname}");
foreach (PostEntity noteCountPost in noteCounts)
{
PostEntity postEntity = postsTableAdapter.GetPost(blogname, noteCountPost.RowKey);
if (postEntity == null)
{
log.Warning($"Post {blogname}/{noteCountPost.RowKey} not found, skipping");
continue;
}
if (string.IsNullOrEmpty(postEntity.PhotoBlobUrls))
{
continue;
}
List <Photo> sitePhotos = JsonConvert.DeserializeObject <List <Photo> >(postEntity.PhotoBlobUrls);
foreach (Photo photo in sitePhotos)
{
List <PhotoSize> sortedSizes = photo.Sizes.OrderByDescending(x => x.Nominal).ToList();
PhotoSize original = sortedSizes.FirstOrDefault();
if (original == null)
{
continue;
}
string url = blobBaseUrl + "/" + original.Container + "/" + photo.Name + "_" + original.Nominal + "." + photo.Extension;
if (imageAnalysisTableAdapter.GetImageAnalysis(url) != null)
{
log.Info($"Image {url} already analyzed");
continue;
}
PhotoToAnalyze message = new PhotoToAnalyze
{
Blog = blogname,
PostDate = postEntity.Date,
Url = url
};
photoToAnalyzeQueueAdapter.Send(message);
log.Info($"Published PhotoToAnalyze message with URL {url}");
analyzedInBlogCount++;
totalCount++;
}
if (analyzedInBlogCount >= photosInBlogLimit)
{
break;
}
}
}
return(req.CreateResponse(HttpStatusCode.OK, $"Will analyze {totalCount} new photos"));
}
public async Task <GetPostsResult> GetPosts(TraceWriter log, string blogname, int startingOffset = 0, int maxOffset = Constants.MaxPostsToFetch,
long timeoutSeconds = 270, bool updateNpf = false)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
long totalInBlog = 0;
long totalReceived = 0;
Blog blog = null;
bool success = true;
int offset = startingOffset;
using (HttpClient httpClient = new HttpClient())
{
Stopwatch stopwatch = Stopwatch.StartNew();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
do
{
string url = "https://api.tumblr.com/v2/blog/" + blogname + "/posts?npf=true&offset=" + offset + "&api_key=" + apiKey;
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
TumblrResponse <BlogPosts> tumblrResponse = JsonConvert.DeserializeObject <TumblrResponse <BlogPosts> >(content);
BlogPosts blogPosts = tumblrResponse.Response;
totalInBlog = blogPosts.Blog.Posts;
blog = blogPosts.Blog;
totalReceived += blogPosts.Posts.Count;
offset += 20;
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
if (updateNpf && blogPosts.Posts.Any(x => x.ShouldOpenInLegacy))
{
success = false;
break;
}
}
else
{
success = false;
break;
}
if (stopwatch.ElapsedMilliseconds > timeoutSeconds * 1000)
{
success = false;
break;
}
} while (offset < totalInBlog && offset < maxOffset);
}
if (blog != null)
{
BlogEntity blogEntity = new BlogEntity(blog)
{
FetchedUntilOffset = updateNpf ? (int?)null : offset,
LastFetched = FunctionUtilities.GetUnixTime(DateTime.UtcNow)
};
blogInfoTableAdapter.InsertBlog(blogEntity);
}
return(new GetPostsResult
{
TotalInBlog = totalInBlog,
TotalReceived = totalReceived,
Success = success
});
}
public async Task <GetPostsResult> GetNewerPosts(TraceWriter log, string blogname, long newerThan, long timeoutSeconds = 270)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
long totalInBlog = 0;
long totalReceived = 0;
Blog blog = null;
bool success = true;
using (HttpClient httpClient = new HttpClient())
{
Stopwatch stopwatch = Stopwatch.StartNew();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
string linkUrl = null;
do
{
string url;
if (linkUrl == null)
{
// start from newest posts
url = "https://api.tumblr.com/v2/blog/" + blogname + "/posts?npf=true&before=" + FunctionUtilities.GetUnixTime(DateTime.UtcNow) + "&api_key=" +
apiKey;
}
else
{
url = "https://api.tumblr.com" + linkUrl + "&api_key=" + apiKey;
}
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
TumblrResponse <BlogPosts> tumblrResponse = JsonConvert.DeserializeObject <TumblrResponse <BlogPosts> >(content);
BlogPosts blogPosts = tumblrResponse.Response;
totalInBlog = blogPosts.Blog.Posts;
blog = blogPosts.Blog;
totalReceived += blogPosts.Posts.Count;
if (blogPosts._links?.Next != null)
{
linkUrl = blogPosts._links.Next.Href;
}
else
{
linkUrl = null;
}
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
if (blogPosts.Posts.Any(x => x.Timestamp < newerThan))
{
// have reached the point that was gotten previously
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts.Where(x => x.Timestamp >= newerThan));
break;
}
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
}
else
{
success = false;
break;
}
if (stopwatch.ElapsedMilliseconds > timeoutSeconds * 1000)
{
success = false;
break;
}
} while (linkUrl != null);
}
if (blog != null)
{
BlogEntity blogEntity = new BlogEntity(blog)
{
LastFetched = FunctionUtilities.GetUnixTime(DateTime.UtcNow)
};
blogInfoTableAdapter.InsertBlog(blogEntity);
}
return(new GetPostsResult
{
TotalInBlog = totalInBlog,
TotalReceived = totalReceived,
Success = success
});
}
public static async Task Run([QueueTrigger(Constants.BlogToIndexQueueName, Connection = "AzureWebJobsStorage")]
string myQueueItem, TraceWriter log)
{
Startup.Init();
BlogToIndex blogToIndex = JsonConvert.DeserializeObject <BlogToIndex>(myQueueItem);
PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter();
photoIndexTableAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter();
reversePostsTableAdapter.Init(log);
PostToGetQueueAdapter postToGetQueueAdapter = new PostToGetQueueAdapter();
postToGetQueueAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
MediaToDownloadQueueAdapter mediaToDownloadQueueAdapter = new MediaToDownloadQueueAdapter();
mediaToDownloadQueueAdapter.Init(log);
List <PhotoIndexEntity> photoIndexEntities = photoIndexTableAdapter.GetAll(blogToIndex.Blogname);
log.Info("Loaded " + photoIndexEntities.Count + " photo index entities");
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToIndex.Blogname);
Dictionary <string, List <Model.Site.Photo> > photosByBlogById = CreatePhotosByBlogById(photoIndexEntities);
BlogStats blogStats = CreateBlogStatsFromPhotos(photoIndexEntities, blogToIndex.Blogname);
blogStats.UpdateFromBlogEntity(blogEntity);
List <PostEntity> postEntities = postsTableAdapter.GetAll(blogToIndex.Blogname);
UpdateBlogStatsFromPosts(blogStats, postEntities);
UpdateMonthIndex(blogToIndex.Blogname, postEntities, blogInfoTableAdapter);
log.Info("Loaded " + postEntities.Count + " post entities");
foreach (PostEntity postEntity in postEntities)
{
if (!string.IsNullOrEmpty(postEntity.PhotoBlobUrls))
{
try
{
Model.Site.Photo[] photos = JsonConvert.DeserializeObject <Model.Site.Photo[]>(postEntity.PhotoBlobUrls);
if (photos.Any(x => !x.Name.Contains("_")))
{
SendToReprocessing(postEntity.PartitionKey, mediaToDownloadQueueAdapter, log, postEntity);
}
}
catch (Exception e)
{
log.Error("Error: " + e.Message);
throw;
}
}
}
blogStats.DisplayablePosts = InsertReversePosts(blogToIndex.Blogname, photosByBlogById, postEntities, reversePostsTableAdapter,
postsTableAdapter, photoIndexTableAdapter, mediaToDownloadQueueAdapter, log);
blogInfoTableAdapter.InsertBlobStats(blogStats);
}
private static void UpdateMonthIndex(string blogname, List <PostEntity> postEntities, BlogInfoTableAdapter blogInfoTableAdapter)
{
Dictionary <string, MonthIndex> indexEntriesByMonth = new Dictionary <string, MonthIndex>();
foreach (PostEntity postEntity in postEntities)
{
string monthKey = postEntity.Date.ToString(MonthIndex.MonthKeyFormat);
if (!indexEntriesByMonth.TryGetValue(monthKey, out MonthIndex indexEntry))
{
indexEntry = new MonthIndex(blogname, monthKey);
indexEntriesByMonth.Add(monthKey, indexEntry);
}
indexEntry.MonthsPosts++;
long postId = long.Parse(postEntity.RowKey);
if (indexEntry.FirstPostId == 0 || postId < indexEntry.FirstPostId)
{
indexEntry.FirstPostId = ReversePostEntity.GetRowKeyId(postId);
}
}
blogInfoTableAdapter.InsertMonthIndice(indexEntriesByMonth.Values);
}
public static async Task Run([TimerTrigger("0 25 * * * *")] TimerInfo myTimer, TraceWriter log)
{
Startup.Init();
log.Info($"C# Timer trigger function executed at: {DateTime.Now}");
BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();
blogToFetchQueueAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
PostsGetter postsGetter = new PostsGetter();
Stopwatch stopwatch = Stopwatch.StartNew();
bool success; // basically means that can the message be deleted, or if it needs to be left in queue to be resumed later
do
{
//TODO: error handling, if there is error from e.g. postsGetter
CloudQueueMessage message = await blogToFetchQueueAdapter.GetNextMessage();
if (message == null)
{
return;
}
BlogToFetch blogToFetch = JsonConvert.DeserializeObject <BlogToFetch>(message.AsString);
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToFetch.Blogname);
long timeoutLeft = 270 - stopwatch.ElapsedMilliseconds / 1000;
if (timeoutLeft < 10)
{
return;
}
success = false;
GetPostsResult result = null;
if (blogToFetch.NewerThan.HasValue)
{
result = await postsGetter.GetNewerPosts(log, blogToFetch.Blogname, blogToFetch.NewerThan.Value, timeoutLeft);
if (result.Success)
{
success = true;
}
}
int offset = 0;
if (blogEntity?.FetchedUntilOffset != null && !blogToFetch.UpdateNpf)
{
offset = blogEntity.FetchedUntilOffset.Value;
}
if (result != null)
{
offset += (int)result.TotalReceived;
}
if (blogEntity != null && (!blogEntity.FetchedUntilOffset.HasValue || blogEntity.FetchedUntilOffset.Value < Constants.MaxPostsToFetch))
{
result = await postsGetter.GetPosts(log, blogToFetch.Blogname, offset, timeoutSeconds : timeoutLeft, updateNpf : blogToFetch.UpdateNpf);
if (result.Success)
{
success = true;
}
}
else
{
success = true; // enough fetched already, message can be deleted
}
if (success)
{
await blogToFetchQueueAdapter.DeleteMessage(message);
}
} while (success);
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getblogsfromlikes/{blogname}")] HttpRequestMessage req,
string blogname, TraceWriter log)
{
Startup.Init();
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();
blogToFetchQueueAdapter.Init();
List <LikeIndexEntity> entities = likeIndexTableAdapter.GetAll(blogname);
ILookup <string, LikeIndexEntity> likesByBlog = entities.ToLookup(e => e.LikedBlogName);
List <BlogStatsRow> stats = likesByBlog.Select(gr => new BlogStatsRow {
Blogname = gr.Key, LikedPostCount = gr.Count()
}).OrderByDescending(x => x.LikedPostCount).ToList();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
List <BlogStatsRow> toDownload = new List <BlogStatsRow>();
using (HttpClient httpClient = new HttpClient())
{
foreach (BlogStatsRow blogStatsRow in stats)
{
if (blogStatsRow.LikedPostCount < 10)
{
continue;
}
string url = "https://api.tumblr.com/v2/blog/" + blogStatsRow.Blogname + "/info?api_key=" + apiKey;
//log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse <BlogInfo> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogInfo> >();
Blog blog = tumblrResponse.Response.Blog;
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogStatsRow.Blogname);
blogStatsRow.HadPostCount = postsTableAdapter.GetPostCount(blogStatsRow.Blogname);
blogStatsRow.TotalPostCount = blog.Posts;
long difference = blog.Posts - blogStatsRow.HadPostCount;
bool fetch = false;
long?newerThan = null;
if (blogEntity != null && blogEntity.Updated < blog.Updated)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, has new posts");
fetch = true;
newerThan = blogEntity.Updated;
}
else if (blogStatsRow.HadPostCount > Constants.MaxPostsToFetch)
{
log.Info("Already fetched " + blogStatsRow.HadPostCount + " posts from blog " + blogStatsRow.Blogname);
}
else if (difference > 5)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, missing " + difference + " posts");
fetch = true;
}
else
{
log.Info("Blog " + blogStatsRow.Blogname + " already downloaded (difference " + difference + ")");
}
if (fetch)
{
blogToFetchQueueAdapter.SendBlogToFetch(new BlogToFetch
{
Blogname = blog.Name,
TotalPostCount = blog.Posts,
NewerThan = newerThan
});
toDownload.Add(blogStatsRow);
}
blogEntity = new BlogEntity(blog);
blogInfoTableAdapter.InsertBlog(blogEntity);
}
}
}
return(req.CreateResponse(HttpStatusCode.OK, "Got " + toDownload.Count + " blogs to fetch"));
}
public static async Task Run([TimerTrigger("0 15 * * * *")] TimerInfo myTimer, TraceWriter log)
{
Startup.Init();
PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter();
photoToAnalyzeQueueAdapter.Init();
CloudQueueMessage message = null;
PhotoToAnalyze photoToAnalyze = null;
try
{
ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();
imageAnalysisTableAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
int processedCount = 0;
do
{
message = await photoToAnalyzeQueueAdapter.GetNextMessage();
if (message == null)
{
return;
}
photoToAnalyze = JsonConvert.DeserializeObject <PhotoToAnalyze>(message.AsString);
if (imageAnalysisTableAdapter.GetImageAnalysis(photoToAnalyze.Url) != null)
{
log.Info($"Image {photoToAnalyze.Url} already analyzed, aborting");
await UpdateBlogInfo(photoToAnalyze, blogInfoTableAdapter);
continue;
}
string failure = await ImageAnalyzer.AnalyzePhoto(log, photoToAnalyze, imageAnalysisTableAdapter);
if (string.IsNullOrEmpty(failure))
{
await UpdateBlogInfo(photoToAnalyze, blogInfoTableAdapter);
}
else
{
photoToAnalyze.Error = failure;
await photoToAnalyzeQueueAdapter.SendToPoisonQueue(photoToAnalyze);
log.Info($"Message for {photoToAnalyze.Url} stored to poison queue");
}
processedCount++;
await photoToAnalyzeQueueAdapter.DeleteMessage(message);
} while (processedCount < 10);
}
catch (Exception ex)
{
log.Error("Error in ProcessPhotosToAnalyze: " + ex.Message, ex);
if (message != null && photoToAnalyze != null)
{
photoToAnalyze.Error = ex.Message;
photoToAnalyze.StackTrace = ex.StackTrace.Substring(0, Math.Max(36000, ex.StackTrace.Length));
await photoToAnalyzeQueueAdapter.SendToPoisonQueue(photoToAnalyze);
log.Info($"Message for {photoToAnalyze.Url} stored to poison queue");
await photoToAnalyzeQueueAdapter.DeleteMessage(message);
log.Info($"Failed message deleted successfully from main queue");
message = null;
}
}
}
