public static async Task Run([TimerTrigger("0 25 * * * *")] TimerInfo myTimer, TraceWriter log)
{
Startup.Init();
log.Info($"C# Timer trigger function executed at: {DateTime.Now}");
BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();
blogToFetchQueueAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
PostsGetter postsGetter = new PostsGetter();
Stopwatch stopwatch = Stopwatch.StartNew();
bool success; // basically means that can the message be deleted, or if it needs to be left in queue to be resumed later
do
{
//TODO: error handling, if there is error from e.g. postsGetter
CloudQueueMessage message = await blogToFetchQueueAdapter.GetNextMessage();
if (message == null)
{
return;
}
BlogToFetch blogToFetch = JsonConvert.DeserializeObject <BlogToFetch>(message.AsString);
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToFetch.Blogname);
long timeoutLeft = 270 - stopwatch.ElapsedMilliseconds / 1000;
if (timeoutLeft < 10)
{
return;
}
success = false;
GetPostsResult result = null;
if (blogToFetch.NewerThan.HasValue)
{
result = await postsGetter.GetNewerPosts(log, blogToFetch.Blogname, blogToFetch.NewerThan.Value, timeoutLeft);
if (result.Success)
{
success = true;
}
}
int offset = 0;
if (blogEntity?.FetchedUntilOffset != null && !blogToFetch.UpdateNpf)
{
offset = blogEntity.FetchedUntilOffset.Value;
}
if (result != null)
{
offset += (int)result.TotalReceived;
}
if (blogEntity != null && (!blogEntity.FetchedUntilOffset.HasValue || blogEntity.FetchedUntilOffset.Value < Constants.MaxPostsToFetch))
{
result = await postsGetter.GetPosts(log, blogToFetch.Blogname, offset, timeoutSeconds : timeoutLeft, updateNpf : blogToFetch.UpdateNpf);
if (result.Success)
{
success = true;
}
}
else
{
success = true; // enough fetched already, message can be deleted
}
if (success)
{
await blogToFetchQueueAdapter.DeleteMessage(message);
}
} while (success);
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getblogsfromlikes/{blogname}")] HttpRequestMessage req,
string blogname, TraceWriter log)
{
Startup.Init();
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();
blogToFetchQueueAdapter.Init();
List <LikeIndexEntity> entities = likeIndexTableAdapter.GetAll(blogname);
ILookup <string, LikeIndexEntity> likesByBlog = entities.ToLookup(e => e.LikedBlogName);
List <BlogStatsRow> stats = likesByBlog.Select(gr => new BlogStatsRow {
Blogname = gr.Key, LikedPostCount = gr.Count()
}).OrderByDescending(x => x.LikedPostCount).ToList();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
List <BlogStatsRow> toDownload = new List <BlogStatsRow>();
using (HttpClient httpClient = new HttpClient())
{
foreach (BlogStatsRow blogStatsRow in stats)
{
if (blogStatsRow.LikedPostCount < 10)
{
continue;
}
string url = "https://api.tumblr.com/v2/blog/" + blogStatsRow.Blogname + "/info?api_key=" + apiKey;
//log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse <BlogInfo> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogInfo> >();
Blog blog = tumblrResponse.Response.Blog;
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogStatsRow.Blogname);
blogStatsRow.HadPostCount = postsTableAdapter.GetPostCount(blogStatsRow.Blogname);
blogStatsRow.TotalPostCount = blog.Posts;
long difference = blog.Posts - blogStatsRow.HadPostCount;
bool fetch = false;
long?newerThan = null;
if (blogEntity != null && blogEntity.Updated < blog.Updated)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, has new posts");
fetch = true;
newerThan = blogEntity.Updated;
}
else if (blogStatsRow.HadPostCount > Constants.MaxPostsToFetch)
{
log.Info("Already fetched " + blogStatsRow.HadPostCount + " posts from blog " + blogStatsRow.Blogname);
}
else if (difference > 5)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, missing " + difference + " posts");
fetch = true;
}
else
{
log.Info("Blog " + blogStatsRow.Blogname + " already downloaded (difference " + difference + ")");
}
if (fetch)
{
blogToFetchQueueAdapter.SendBlogToFetch(new BlogToFetch
{
Blogname = blog.Name,
TotalPostCount = blog.Posts,
NewerThan = newerThan
});
toDownload.Add(blogStatsRow);
}
blogEntity = new BlogEntity(blog);
blogInfoTableAdapter.InsertBlog(blogEntity);
}
}
}
return(req.CreateResponse(HttpStatusCode.OK, "Got " + toDownload.Count + " blogs to fetch"));
}
