public void Init(TraceWriter log)
{
postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
queueAdapter = new MediaToDownloadQueueAdapter();
queueAdapter.Init(log);
}
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "analyze-from-likes/{blogname}")]
HttpRequestMessage req, string blogname, TraceWriter log)
{
Startup.Init();
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();
imageAnalysisTableAdapter.Init();
PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter();
photoToAnalyzeQueueAdapter.Init();
string blobBaseUrl = ConfigurationManager.AppSettings["BlobBaseUrl"];
string afterParam = req.GetQueryNameValuePairs().FirstOrDefault(q => q.Key.Equals("after", StringComparison.OrdinalIgnoreCase)).Value;
List <LikeIndexEntity> likes;
if (!string.IsNullOrEmpty(afterParam) && long.TryParse(afterParam, out long afterTimestamp))
{
log.Info($"Getting likes newer than timestamp {afterTimestamp}");
likes = likeIndexTableAdapter.GetNewerThan(blogname, afterTimestamp);
}
else
{
likes = likeIndexTableAdapter.GetAll(blogname);
}
log.Info($"Loaded {likes.Count} posts");
int messageCount = 0;
foreach (LikeIndexEntity like in likes)
{
if (like.LikedBlogName == null || like.LikedPostId == null)
{
continue;
}
PostEntity postEntity = postsTableAdapter.GetPost(like.LikedBlogName, like.LikedPostId);
if (postEntity == null)
{
log.Warning($"Post {like.LikedBlogName}/{like.LikedPostId} not found, skipping");
continue;
}
if (string.IsNullOrEmpty(postEntity.PhotoBlobUrls))
{
continue;
}
List <Photo> sitePhotos = JsonConvert.DeserializeObject <List <Photo> >(postEntity.PhotoBlobUrls);
foreach (Photo photo in sitePhotos)
{
List <PhotoSize> sortedSizes = photo.Sizes.OrderByDescending(x => x.Nominal).ToList();
PhotoSize original = sortedSizes.FirstOrDefault();
if (original == null)
{
continue;
}
string url = blobBaseUrl + "/" + original.Container + "/" + photo.Name + "_" + original.Nominal + "." + photo.Extension;
if (imageAnalysisTableAdapter.GetImageAnalysis(url) != null)
{
log.Info($"Image {url} already analyzed");
continue;
}
PhotoToAnalyze message = new PhotoToAnalyze
{
Blog = blogname,
PostDate = postEntity.Date,
Url = url
};
photoToAnalyzeQueueAdapter.Send(message);
log.Info($"Published PhotoToAnalyze message with URL {url}");
messageCount++;
}
}
return(req.CreateResponse(HttpStatusCode.OK, $"Processed {likes.Count} posts, sent {messageCount} messages"));
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getblogsfromlikes/{blogname}")] HttpRequestMessage req,
string blogname, TraceWriter log)
{
Startup.Init();
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();
blogToFetchQueueAdapter.Init();
List <LikeIndexEntity> entities = likeIndexTableAdapter.GetAll(blogname);
ILookup <string, LikeIndexEntity> likesByBlog = entities.ToLookup(e => e.LikedBlogName);
List <BlogStatsRow> stats = likesByBlog.Select(gr => new BlogStatsRow {
Blogname = gr.Key, LikedPostCount = gr.Count()
}).OrderByDescending(x => x.LikedPostCount).ToList();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
List <BlogStatsRow> toDownload = new List <BlogStatsRow>();
using (HttpClient httpClient = new HttpClient())
{
foreach (BlogStatsRow blogStatsRow in stats)
{
if (blogStatsRow.LikedPostCount < 10)
{
continue;
}
string url = "https://api.tumblr.com/v2/blog/" + blogStatsRow.Blogname + "/info?api_key=" + apiKey;
//log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse <BlogInfo> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogInfo> >();
Blog blog = tumblrResponse.Response.Blog;
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogStatsRow.Blogname);
blogStatsRow.HadPostCount = postsTableAdapter.GetPostCount(blogStatsRow.Blogname);
blogStatsRow.TotalPostCount = blog.Posts;
long difference = blog.Posts - blogStatsRow.HadPostCount;
bool fetch = false;
long?newerThan = null;
if (blogEntity != null && blogEntity.Updated < blog.Updated)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, has new posts");
fetch = true;
newerThan = blogEntity.Updated;
}
else if (blogStatsRow.HadPostCount > Constants.MaxPostsToFetch)
{
log.Info("Already fetched " + blogStatsRow.HadPostCount + " posts from blog " + blogStatsRow.Blogname);
}
else if (difference > 5)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, missing " + difference + " posts");
fetch = true;
}
else
{
log.Info("Blog " + blogStatsRow.Blogname + " already downloaded (difference " + difference + ")");
}
if (fetch)
{
blogToFetchQueueAdapter.SendBlogToFetch(new BlogToFetch
{
Blogname = blog.Name,
TotalPostCount = blog.Posts,
NewerThan = newerThan
});
toDownload.Add(blogStatsRow);
}
blogEntity = new BlogEntity(blog);
blogInfoTableAdapter.InsertBlog(blogEntity);
}
}
}
return(req.CreateResponse(HttpStatusCode.OK, "Got " + toDownload.Count + " blogs to fetch"));
}
public static async Task<HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getlikes/{blogname}")]
HttpRequestMessage req, string blogname, TraceWriter log)
{
Startup.Init();
blogname = blogname.ToLower().Replace(".tumblr.com", "");
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
long newestLikedTimestamp = likeIndexTableAdapter.GetNewestLikedTimestamp(blogname);
if (newestLikedTimestamp > 0)
{
log.Info($"Getting likes newer than timestamp {newestLikedTimestamp}");
}
Likes likes = null;
long totalCount = 0;
using (HttpClient httpClient = new HttpClient())
{
DateTime beforeTime = DateTime.UtcNow;
do
{
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
string url;
if (likes == null)
{
long timestamp = FunctionUtilities.GetUnixTime(beforeTime);
url = "https://api.tumblr.com/v2/blog/" + blogname + "/likes?before=" + timestamp + "&api_key=" + apiKey;
}
else
{
url = "https://api.tumblr.com" + likes._links.Next.Href + "&api_key=" + apiKey;
}
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse<Likes> tumblrResponse = await response.Content.ReadAsAsync<TumblrResponse<Likes>>();
likes = tumblrResponse.Response;
if (newestLikedTimestamp > 0)
{
List<Post> newerPosts = likes.Liked_posts.Where(x => x.Liked_Timestamp > newestLikedTimestamp).ToList();
if (newerPosts.Count < likes.Liked_posts.Count)
{
log.Info($"Reached Liked_Timestamp of {newestLikedTimestamp} which has already been fetched, finishing");
postsToProcessQueueAdapter.SendPostsToProcess(newerPosts, blogname);
totalCount += newerPosts.Count;
break;
}
}
totalCount += likes.Liked_posts.Count;
postsToProcessQueueAdapter.SendPostsToProcess(likes.Liked_posts, blogname);
}
else
{
log.Info("Got response: " + response.ReasonPhrase + " " + response.StatusCode);
break;
}
} while (likes._links != null && likes._links.Next != null && !string.IsNullOrEmpty(likes._links.Next.Href));
}
log.Info("C# HTTP trigger function processed a request.");
// Fetching the name from the path parameter in the request URL
return req.CreateResponse(HttpStatusCode.OK, "Got " + totalCount + " posts");
}