public async Task <GetPostsResult> GetPosts(TraceWriter log, string blogname, int startingOffset = 0, int maxOffset = Constants.MaxPostsToFetch,
long timeoutSeconds = 270, bool updateNpf = false)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
long totalInBlog = 0;
long totalReceived = 0;
Blog blog = null;
bool success = true;
int offset = startingOffset;
using (HttpClient httpClient = new HttpClient())
{
Stopwatch stopwatch = Stopwatch.StartNew();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
do
{
string url = "https://api.tumblr.com/v2/blog/" + blogname + "/posts?npf=true&offset=" + offset + "&api_key=" + apiKey;
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
TumblrResponse <BlogPosts> tumblrResponse = JsonConvert.DeserializeObject <TumblrResponse <BlogPosts> >(content);
BlogPosts blogPosts = tumblrResponse.Response;
totalInBlog = blogPosts.Blog.Posts;
blog = blogPosts.Blog;
totalReceived += blogPosts.Posts.Count;
offset += 20;
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
if (updateNpf && blogPosts.Posts.Any(x => x.ShouldOpenInLegacy))
{
success = false;
break;
}
}
else
{
success = false;
break;
}
if (stopwatch.ElapsedMilliseconds > timeoutSeconds * 1000)
{
success = false;
break;
}
} while (offset < totalInBlog && offset < maxOffset);
}
if (blog != null)
{
BlogEntity blogEntity = new BlogEntity(blog)
{
FetchedUntilOffset = updateNpf ? (int?)null : offset,
LastFetched = FunctionUtilities.GetUnixTime(DateTime.UtcNow)
};
blogInfoTableAdapter.InsertBlog(blogEntity);
}
return(new GetPostsResult
{
TotalInBlog = totalInBlog,
TotalReceived = totalReceived,
Success = success
});
}
public static async Task<HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getlikes/{blogname}")]
HttpRequestMessage req, string blogname, TraceWriter log)
{
Startup.Init();
blogname = blogname.ToLower().Replace(".tumblr.com", "");
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
long newestLikedTimestamp = likeIndexTableAdapter.GetNewestLikedTimestamp(blogname);
if (newestLikedTimestamp > 0)
{
log.Info($"Getting likes newer than timestamp {newestLikedTimestamp}");
}
Likes likes = null;
long totalCount = 0;
using (HttpClient httpClient = new HttpClient())
{
DateTime beforeTime = DateTime.UtcNow;
do
{
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
string url;
if (likes == null)
{
long timestamp = FunctionUtilities.GetUnixTime(beforeTime);
url = "https://api.tumblr.com/v2/blog/" + blogname + "/likes?before=" + timestamp + "&api_key=" + apiKey;
}
else
{
url = "https://api.tumblr.com" + likes._links.Next.Href + "&api_key=" + apiKey;
}
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse<Likes> tumblrResponse = await response.Content.ReadAsAsync<TumblrResponse<Likes>>();
likes = tumblrResponse.Response;
if (newestLikedTimestamp > 0)
{
List<Post> newerPosts = likes.Liked_posts.Where(x => x.Liked_Timestamp > newestLikedTimestamp).ToList();
if (newerPosts.Count < likes.Liked_posts.Count)
{
log.Info($"Reached Liked_Timestamp of {newestLikedTimestamp} which has already been fetched, finishing");
postsToProcessQueueAdapter.SendPostsToProcess(newerPosts, blogname);
totalCount += newerPosts.Count;
break;
}
}
totalCount += likes.Liked_posts.Count;
postsToProcessQueueAdapter.SendPostsToProcess(likes.Liked_posts, blogname);
}
else
{
log.Info("Got response: " + response.ReasonPhrase + " " + response.StatusCode);
break;
}
} while (likes._links != null && likes._links.Next != null && !string.IsNullOrEmpty(likes._links.Next.Href));
}
log.Info("C# HTTP trigger function processed a request.");
// Fetching the name from the path parameter in the request URL
return req.CreateResponse(HttpStatusCode.OK, "Got " + totalCount + " posts");
}
public async Task <GetPostsResult> GetNewerPosts(TraceWriter log, string blogname, long newerThan, long timeoutSeconds = 270)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
long totalInBlog = 0;
long totalReceived = 0;
Blog blog = null;
bool success = true;
using (HttpClient httpClient = new HttpClient())
{
Stopwatch stopwatch = Stopwatch.StartNew();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
string linkUrl = null;
do
{
string url;
if (linkUrl == null)
{
// start from newest posts
url = "https://api.tumblr.com/v2/blog/" + blogname + "/posts?npf=true&before=" + FunctionUtilities.GetUnixTime(DateTime.UtcNow) + "&api_key=" +
apiKey;
}
else
{
url = "https://api.tumblr.com" + linkUrl + "&api_key=" + apiKey;
}
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
TumblrResponse <BlogPosts> tumblrResponse = JsonConvert.DeserializeObject <TumblrResponse <BlogPosts> >(content);
BlogPosts blogPosts = tumblrResponse.Response;
totalInBlog = blogPosts.Blog.Posts;
blog = blogPosts.Blog;
totalReceived += blogPosts.Posts.Count;
if (blogPosts._links?.Next != null)
{
linkUrl = blogPosts._links.Next.Href;
}
else
{
linkUrl = null;
}
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
if (blogPosts.Posts.Any(x => x.Timestamp < newerThan))
{
// have reached the point that was gotten previously
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts.Where(x => x.Timestamp >= newerThan));
break;
}
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
}
else
{
success = false;
break;
}
if (stopwatch.ElapsedMilliseconds > timeoutSeconds * 1000)
{
success = false;
break;
}
} while (linkUrl != null);
}
if (blog != null)
{
BlogEntity blogEntity = new BlogEntity(blog)
{
LastFetched = FunctionUtilities.GetUnixTime(DateTime.UtcNow)
};
blogInfoTableAdapter.InsertBlog(blogEntity);
}
return(new GetPostsResult
{
TotalInBlog = totalInBlog,
TotalReceived = totalReceived,
Success = success
});
}
public static async Task Run([TimerTrigger("0 30 */2 * * *")] TimerInfo myTimer, TraceWriter log)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
PostToGetQueueAdapter postToGetQueueAdapter = new PostToGetQueueAdapter();
postToGetQueueAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
using (HttpClient httpClient = new HttpClient())
{
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
do
{
CloudQueueMessage message = await postToGetQueueAdapter.GetNextMessage();
if (message == null)
{
return;
}
PostToGet postToGet = JsonConvert.DeserializeObject <PostToGet>(message.AsString);
string url = "https://api.tumblr.com/v2/blog/" + postToGet.Blogname + "/posts?id=" + postToGet.Id + "&api_key=" + apiKey;
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse <BlogPosts> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogPosts> >();
BlogPosts blogPosts = tumblrResponse.Response;
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
await postToGetQueueAdapter.DeleteMessage(message);
log.Info("Successfully fetched " + postToGet.Blogname + "/" + postToGet.Id + " and queued for processing");
}
else
{
log.Error("Error getting post " + postToGet.Blogname + "/" + postToGet.Id + ": " + response.ReasonPhrase);
if (response.ReasonPhrase.IndexOf("limit exceeded", StringComparison.OrdinalIgnoreCase) >= 0)
{
log.Error("Limit exceeded, exiting");
return;
}
else if (response.ReasonPhrase.IndexOf("not found", StringComparison.OrdinalIgnoreCase) >= 0)
{
log.Error("Not found, deleting message, marking post as Not Found");
postsTableAdapter.MarkPostNotFound(postToGet.Blogname, postToGet.Id);
await postToGetQueueAdapter.DeleteMessage(message);
}
}
} while (true);
}
}