public TumblrResponse GetPosts()
{
string requestUrl = string.Format(TumblrAPI, this.BlogUrl, this.ConsumerKey);
if (this.TextOnly)
{
requestUrl += "&filter=text";
}
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(requestUrl);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream resStream = response.GetResponseStream();
DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(TumblrResponse));
TumblrResponse tumblrResponse = (TumblrResponse)ser.ReadObject(resStream);
return(tumblrResponse);
}
public static MvcHtmlString TumblrTopPosts(this HtmlHelper html, string blog, string consumerKey, int numberOfPosts, string template)
{
StringBuilder htmlString = new StringBuilder();
TumblrRequestHelper request = new TumblrRequestHelper(blog, consumerKey, true);
TumblrResponse response = request.GetPosts();
var viewResult = ViewEngines.Engines.FindPartialView(html.ViewContext.Controller.ControllerContext, template);
if (!(viewResult.View is RazorView))
{
throw new Exception("Only razor views are supported");
}
var razorView = (RazorView)viewResult.View;
var viewFile = System.IO.File.ReadAllText(html.ViewContext.HttpContext.Server.MapPath(razorView.ViewPath));
response.response.posts.Where(p => p.type != "quote")
.Take(numberOfPosts)
.ToList()
.ForEach(p => htmlString.Append(Razor.Parse(viewFile, p)));
return(MvcHtmlString.Create(htmlString.ToString()));
}
public HomeController()
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://api.tumblr.com/v2/blog/mrkjffrsn.tumblr.com/posts/text?api_key=JdMtfILFiB2pqcf3xRqlSzg7WYKCT4UgcAuBctmu91g6BN91v2");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception(String.Format(
"Server error (HTTP {0}: {1}).",
response.StatusCode,
response.StatusDescription));
}
DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(TumblrResponse));
object objResponse = jsonSerializer.ReadObject(response.GetResponseStream());
TumblrResponse tumblrResponse = (TumblrResponse)objResponse;
TumblrSuccessResponse successResponse = tumblrResponse.jsonResponse;
postCollection = successResponse.posts;
}
catch (Exception)
{
}
}
public async Task <GetPostsResult> GetPosts(TraceWriter log, string blogname, int startingOffset = 0, int maxOffset = Constants.MaxPostsToFetch,
long timeoutSeconds = 270, bool updateNpf = false)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
long totalInBlog = 0;
long totalReceived = 0;
Blog blog = null;
bool success = true;
int offset = startingOffset;
using (HttpClient httpClient = new HttpClient())
{
Stopwatch stopwatch = Stopwatch.StartNew();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
do
{
string url = "https://api.tumblr.com/v2/blog/" + blogname + "/posts?npf=true&offset=" + offset + "&api_key=" + apiKey;
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
TumblrResponse <BlogPosts> tumblrResponse = JsonConvert.DeserializeObject <TumblrResponse <BlogPosts> >(content);
BlogPosts blogPosts = tumblrResponse.Response;
totalInBlog = blogPosts.Blog.Posts;
blog = blogPosts.Blog;
totalReceived += blogPosts.Posts.Count;
offset += 20;
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
if (updateNpf && blogPosts.Posts.Any(x => x.ShouldOpenInLegacy))
{
success = false;
break;
}
}
else
{
success = false;
break;
}
if (stopwatch.ElapsedMilliseconds > timeoutSeconds * 1000)
{
success = false;
break;
}
} while (offset < totalInBlog && offset < maxOffset);
}
if (blog != null)
{
BlogEntity blogEntity = new BlogEntity(blog)
{
FetchedUntilOffset = updateNpf ? (int?)null : offset,
LastFetched = FunctionUtilities.GetUnixTime(DateTime.UtcNow)
};
blogInfoTableAdapter.InsertBlog(blogEntity);
}
return(new GetPostsResult
{
TotalInBlog = totalInBlog,
TotalReceived = totalReceived,
Success = success
});
}
public async Task <GetPostsResult> GetNewerPosts(TraceWriter log, string blogname, long newerThan, long timeoutSeconds = 270)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
long totalInBlog = 0;
long totalReceived = 0;
Blog blog = null;
bool success = true;
using (HttpClient httpClient = new HttpClient())
{
Stopwatch stopwatch = Stopwatch.StartNew();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
string linkUrl = null;
do
{
string url;
if (linkUrl == null)
{
// start from newest posts
url = "https://api.tumblr.com/v2/blog/" + blogname + "/posts?npf=true&before=" + FunctionUtilities.GetUnixTime(DateTime.UtcNow) + "&api_key=" +
apiKey;
}
else
{
url = "https://api.tumblr.com" + linkUrl + "&api_key=" + apiKey;
}
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
TumblrResponse <BlogPosts> tumblrResponse = JsonConvert.DeserializeObject <TumblrResponse <BlogPosts> >(content);
BlogPosts blogPosts = tumblrResponse.Response;
totalInBlog = blogPosts.Blog.Posts;
blog = blogPosts.Blog;
totalReceived += blogPosts.Posts.Count;
if (blogPosts._links?.Next != null)
{
linkUrl = blogPosts._links.Next.Href;
}
else
{
linkUrl = null;
}
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
if (blogPosts.Posts.Any(x => x.Timestamp < newerThan))
{
// have reached the point that was gotten previously
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts.Where(x => x.Timestamp >= newerThan));
break;
}
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
}
else
{
success = false;
break;
}
if (stopwatch.ElapsedMilliseconds > timeoutSeconds * 1000)
{
success = false;
break;
}
} while (linkUrl != null);
}
if (blog != null)
{
BlogEntity blogEntity = new BlogEntity(blog)
{
LastFetched = FunctionUtilities.GetUnixTime(DateTime.UtcNow)
};
blogInfoTableAdapter.InsertBlog(blogEntity);
}
return(new GetPostsResult
{
TotalInBlog = totalInBlog,
TotalReceived = totalReceived,
Success = success
});
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getblogsfromlikes/{blogname}")] HttpRequestMessage req,
string blogname, TraceWriter log)
{
Startup.Init();
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
blogInfoTableAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();
blogToFetchQueueAdapter.Init();
List <LikeIndexEntity> entities = likeIndexTableAdapter.GetAll(blogname);
ILookup <string, LikeIndexEntity> likesByBlog = entities.ToLookup(e => e.LikedBlogName);
List <BlogStatsRow> stats = likesByBlog.Select(gr => new BlogStatsRow {
Blogname = gr.Key, LikedPostCount = gr.Count()
}).OrderByDescending(x => x.LikedPostCount).ToList();
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
List <BlogStatsRow> toDownload = new List <BlogStatsRow>();
using (HttpClient httpClient = new HttpClient())
{
foreach (BlogStatsRow blogStatsRow in stats)
{
if (blogStatsRow.LikedPostCount < 10)
{
continue;
}
string url = "https://api.tumblr.com/v2/blog/" + blogStatsRow.Blogname + "/info?api_key=" + apiKey;
//log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse <BlogInfo> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogInfo> >();
Blog blog = tumblrResponse.Response.Blog;
BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogStatsRow.Blogname);
blogStatsRow.HadPostCount = postsTableAdapter.GetPostCount(blogStatsRow.Blogname);
blogStatsRow.TotalPostCount = blog.Posts;
long difference = blog.Posts - blogStatsRow.HadPostCount;
bool fetch = false;
long?newerThan = null;
if (blogEntity != null && blogEntity.Updated < blog.Updated)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, has new posts");
fetch = true;
newerThan = blogEntity.Updated;
}
else if (blogStatsRow.HadPostCount > Constants.MaxPostsToFetch)
{
log.Info("Already fetched " + blogStatsRow.HadPostCount + " posts from blog " + blogStatsRow.Blogname);
}
else if (difference > 5)
{
log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, missing " + difference + " posts");
fetch = true;
}
else
{
log.Info("Blog " + blogStatsRow.Blogname + " already downloaded (difference " + difference + ")");
}
if (fetch)
{
blogToFetchQueueAdapter.SendBlogToFetch(new BlogToFetch
{
Blogname = blog.Name,
TotalPostCount = blog.Posts,
NewerThan = newerThan
});
toDownload.Add(blogStatsRow);
}
blogEntity = new BlogEntity(blog);
blogInfoTableAdapter.InsertBlog(blogEntity);
}
}
}
return(req.CreateResponse(HttpStatusCode.OK, "Got " + toDownload.Count + " blogs to fetch"));
}
public static async Task<HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getlikes/{blogname}")]
HttpRequestMessage req, string blogname, TraceWriter log)
{
Startup.Init();
blogname = blogname.ToLower().Replace(".tumblr.com", "");
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
likeIndexTableAdapter.Init();
long newestLikedTimestamp = likeIndexTableAdapter.GetNewestLikedTimestamp(blogname);
if (newestLikedTimestamp > 0)
{
log.Info($"Getting likes newer than timestamp {newestLikedTimestamp}");
}
Likes likes = null;
long totalCount = 0;
using (HttpClient httpClient = new HttpClient())
{
DateTime beforeTime = DateTime.UtcNow;
do
{
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
string url;
if (likes == null)
{
long timestamp = FunctionUtilities.GetUnixTime(beforeTime);
url = "https://api.tumblr.com/v2/blog/" + blogname + "/likes?before=" + timestamp + "&api_key=" + apiKey;
}
else
{
url = "https://api.tumblr.com" + likes._links.Next.Href + "&api_key=" + apiKey;
}
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse<Likes> tumblrResponse = await response.Content.ReadAsAsync<TumblrResponse<Likes>>();
likes = tumblrResponse.Response;
if (newestLikedTimestamp > 0)
{
List<Post> newerPosts = likes.Liked_posts.Where(x => x.Liked_Timestamp > newestLikedTimestamp).ToList();
if (newerPosts.Count < likes.Liked_posts.Count)
{
log.Info($"Reached Liked_Timestamp of {newestLikedTimestamp} which has already been fetched, finishing");
postsToProcessQueueAdapter.SendPostsToProcess(newerPosts, blogname);
totalCount += newerPosts.Count;
break;
}
}
totalCount += likes.Liked_posts.Count;
postsToProcessQueueAdapter.SendPostsToProcess(likes.Liked_posts, blogname);
}
else
{
log.Info("Got response: " + response.ReasonPhrase + " " + response.StatusCode);
break;
}
} while (likes._links != null && likes._links.Next != null && !string.IsNullOrEmpty(likes._links.Next.Href));
}
log.Info("C# HTTP trigger function processed a request.");
// Fetching the name from the path parameter in the request URL
return req.CreateResponse(HttpStatusCode.OK, "Got " + totalCount + " posts");
}
public static async Task Run([TimerTrigger("0 30 */2 * * *")] TimerInfo myTimer, TraceWriter log)
{
PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
postsToProcessQueueAdapter.Init(log);
PostToGetQueueAdapter postToGetQueueAdapter = new PostToGetQueueAdapter();
postToGetQueueAdapter.Init();
PostsTableAdapter postsTableAdapter = new PostsTableAdapter();
postsTableAdapter.Init(log);
using (HttpClient httpClient = new HttpClient())
{
string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
do
{
CloudQueueMessage message = await postToGetQueueAdapter.GetNextMessage();
if (message == null)
{
return;
}
PostToGet postToGet = JsonConvert.DeserializeObject <PostToGet>(message.AsString);
string url = "https://api.tumblr.com/v2/blog/" + postToGet.Blogname + "/posts?id=" + postToGet.Id + "&api_key=" + apiKey;
log.Info("Making request to: " + url);
HttpResponseMessage response = await httpClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
TumblrResponse <BlogPosts> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogPosts> >();
BlogPosts blogPosts = tumblrResponse.Response;
if (blogPosts.Posts != null && blogPosts.Posts.Count > 0)
{
postsToProcessQueueAdapter.SendPostsToProcess(blogPosts.Posts);
}
await postToGetQueueAdapter.DeleteMessage(message);
log.Info("Successfully fetched " + postToGet.Blogname + "/" + postToGet.Id + " and queued for processing");
}
else
{
log.Error("Error getting post " + postToGet.Blogname + "/" + postToGet.Id + ": " + response.ReasonPhrase);
if (response.ReasonPhrase.IndexOf("limit exceeded", StringComparison.OrdinalIgnoreCase) >= 0)
{
log.Error("Limit exceeded, exiting");
return;
}
else if (response.ReasonPhrase.IndexOf("not found", StringComparison.OrdinalIgnoreCase) >= 0)
{
log.Error("Not found, deleting message, marking post as Not Found");
postsTableAdapter.MarkPostNotFound(postToGet.Blogname, postToGet.Id);
await postToGetQueueAdapter.DeleteMessage(message);
}
}
} while (true);
}
}
