コード例 #1
0
        public void Init(TraceWriter log)
        {
            postsTableAdapter = new PostsTableAdapter();
            postsTableAdapter.Init(log);

            likeIndexTableAdapter = new LikeIndexTableAdapter();
            likeIndexTableAdapter.Init();

            queueAdapter = new MediaToDownloadQueueAdapter();
            queueAdapter.Init(log);
        }
コード例 #2
0
        public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "analyze-from-likes/{blogname}")]
                                              HttpRequestMessage req, string blogname, TraceWriter log)
        {
            Startup.Init();

            LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();

            likeIndexTableAdapter.Init();

            PostsTableAdapter postsTableAdapter = new PostsTableAdapter();

            postsTableAdapter.Init(log);

            ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();

            imageAnalysisTableAdapter.Init();

            PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter();

            photoToAnalyzeQueueAdapter.Init();

            string blobBaseUrl = ConfigurationManager.AppSettings["BlobBaseUrl"];

            string afterParam = req.GetQueryNameValuePairs().FirstOrDefault(q => q.Key.Equals("after", StringComparison.OrdinalIgnoreCase)).Value;

            List <LikeIndexEntity> likes;

            if (!string.IsNullOrEmpty(afterParam) && long.TryParse(afterParam, out long afterTimestamp))
            {
                log.Info($"Getting likes newer than timestamp {afterTimestamp}");
                likes = likeIndexTableAdapter.GetNewerThan(blogname, afterTimestamp);
            }
            else
            {
                likes = likeIndexTableAdapter.GetAll(blogname);
            }

            log.Info($"Loaded {likes.Count} posts");

            int messageCount = 0;

            foreach (LikeIndexEntity like in likes)
            {
                if (like.LikedBlogName == null || like.LikedPostId == null)
                {
                    continue;
                }

                PostEntity postEntity = postsTableAdapter.GetPost(like.LikedBlogName, like.LikedPostId);

                if (postEntity == null)
                {
                    log.Warning($"Post {like.LikedBlogName}/{like.LikedPostId} not found, skipping");
                    continue;
                }

                if (string.IsNullOrEmpty(postEntity.PhotoBlobUrls))
                {
                    continue;
                }

                List <Photo> sitePhotos = JsonConvert.DeserializeObject <List <Photo> >(postEntity.PhotoBlobUrls);

                foreach (Photo photo in sitePhotos)
                {
                    List <PhotoSize> sortedSizes = photo.Sizes.OrderByDescending(x => x.Nominal).ToList();

                    PhotoSize original = sortedSizes.FirstOrDefault();
                    if (original == null)
                    {
                        continue;
                    }

                    string url = blobBaseUrl + "/" + original.Container + "/" + photo.Name + "_" + original.Nominal + "." + photo.Extension;

                    if (imageAnalysisTableAdapter.GetImageAnalysis(url) != null)
                    {
                        log.Info($"Image {url} already analyzed");
                        continue;
                    }

                    PhotoToAnalyze message = new PhotoToAnalyze
                    {
                        Blog     = blogname,
                        PostDate = postEntity.Date,
                        Url      = url
                    };
                    photoToAnalyzeQueueAdapter.Send(message);
                    log.Info($"Published PhotoToAnalyze message with URL {url}");
                    messageCount++;
                }
            }

            return(req.CreateResponse(HttpStatusCode.OK, $"Processed {likes.Count} posts, sent {messageCount} messages"));
        }
コード例 #3
0
        public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getblogsfromlikes/{blogname}")] HttpRequestMessage req,
                                                           string blogname, TraceWriter log)
        {
            Startup.Init();

            LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();

            likeIndexTableAdapter.Init();

            BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();

            blogInfoTableAdapter.Init();

            PostsTableAdapter postsTableAdapter = new PostsTableAdapter();

            postsTableAdapter.Init(log);

            BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();

            blogToFetchQueueAdapter.Init();

            List <LikeIndexEntity>            entities    = likeIndexTableAdapter.GetAll(blogname);
            ILookup <string, LikeIndexEntity> likesByBlog = entities.ToLookup(e => e.LikedBlogName);
            List <BlogStatsRow> stats = likesByBlog.Select(gr => new BlogStatsRow {
                Blogname = gr.Key, LikedPostCount = gr.Count()
            }).OrderByDescending(x => x.LikedPostCount).ToList();

            string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];

            List <BlogStatsRow> toDownload = new List <BlogStatsRow>();

            using (HttpClient httpClient = new HttpClient())
            {
                foreach (BlogStatsRow blogStatsRow in stats)
                {
                    if (blogStatsRow.LikedPostCount < 10)
                    {
                        continue;
                    }

                    string url = "https://api.tumblr.com/v2/blog/" + blogStatsRow.Blogname + "/info?api_key=" + apiKey;
                    //log.Info("Making request to: " + url);
                    HttpResponseMessage response = await httpClient.GetAsync(url);

                    if (response.IsSuccessStatusCode)
                    {
                        TumblrResponse <BlogInfo> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogInfo> >();

                        Blog       blog       = tumblrResponse.Response.Blog;
                        BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogStatsRow.Blogname);

                        blogStatsRow.HadPostCount   = postsTableAdapter.GetPostCount(blogStatsRow.Blogname);
                        blogStatsRow.TotalPostCount = blog.Posts;
                        long difference = blog.Posts - blogStatsRow.HadPostCount;
                        bool fetch      = false;
                        long?newerThan  = null;

                        if (blogEntity != null && blogEntity.Updated < blog.Updated)
                        {
                            log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, has new posts");
                            fetch     = true;
                            newerThan = blogEntity.Updated;
                        }
                        else if (blogStatsRow.HadPostCount > Constants.MaxPostsToFetch)
                        {
                            log.Info("Already fetched " + blogStatsRow.HadPostCount + " posts from blog " + blogStatsRow.Blogname);
                        }
                        else if (difference > 5)
                        {
                            log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, missing " + difference + " posts");
                            fetch = true;
                        }
                        else
                        {
                            log.Info("Blog " + blogStatsRow.Blogname + " already downloaded (difference " + difference + ")");
                        }

                        if (fetch)
                        {
                            blogToFetchQueueAdapter.SendBlogToFetch(new BlogToFetch
                            {
                                Blogname       = blog.Name,
                                TotalPostCount = blog.Posts,
                                NewerThan      = newerThan
                            });
                            toDownload.Add(blogStatsRow);
                        }

                        blogEntity = new BlogEntity(blog);
                        blogInfoTableAdapter.InsertBlog(blogEntity);
                    }
                }
            }

            return(req.CreateResponse(HttpStatusCode.OK, "Got " + toDownload.Count + " blogs to fetch"));
        }
コード例 #4
0
        public static async Task<HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getlikes/{blogname}")]
            HttpRequestMessage req, string blogname, TraceWriter log)
        {
            Startup.Init();

            blogname = blogname.ToLower().Replace(".tumblr.com", "");

            PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
            postsToProcessQueueAdapter.Init(log);

            LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
            likeIndexTableAdapter.Init();

            long newestLikedTimestamp = likeIndexTableAdapter.GetNewestLikedTimestamp(blogname);

            if (newestLikedTimestamp > 0)
            {
                log.Info($"Getting likes newer than timestamp {newestLikedTimestamp}");
            }

            Likes likes = null;
            long totalCount = 0;
            using (HttpClient httpClient = new HttpClient())
            {
                DateTime beforeTime = DateTime.UtcNow;
                do
                {
                    string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
                    string url;
                    if (likes == null)
                    {
                        long timestamp = FunctionUtilities.GetUnixTime(beforeTime);
                        url = "https://api.tumblr.com/v2/blog/" + blogname + "/likes?before=" + timestamp + "&api_key=" + apiKey;
                    }
                    else
                    {
                        url = "https://api.tumblr.com" + likes._links.Next.Href + "&api_key=" + apiKey;
                    }

                    log.Info("Making request to: " + url);
                    HttpResponseMessage response = await httpClient.GetAsync(url);
                    if (response.IsSuccessStatusCode)
                    {
                        TumblrResponse<Likes> tumblrResponse = await response.Content.ReadAsAsync<TumblrResponse<Likes>>();
                        likes = tumblrResponse.Response;

                        if (newestLikedTimestamp > 0)
                        {
                            List<Post> newerPosts = likes.Liked_posts.Where(x => x.Liked_Timestamp > newestLikedTimestamp).ToList();
                            if (newerPosts.Count < likes.Liked_posts.Count)
                            {
                                log.Info($"Reached Liked_Timestamp of {newestLikedTimestamp} which has already been fetched, finishing");
                                postsToProcessQueueAdapter.SendPostsToProcess(newerPosts, blogname);

                                totalCount += newerPosts.Count;
                                break;
                            }
                        }

                        totalCount += likes.Liked_posts.Count;

                        postsToProcessQueueAdapter.SendPostsToProcess(likes.Liked_posts, blogname);
                    }
                    else
                    {
                        log.Info("Got response: " + response.ReasonPhrase + " " + response.StatusCode);
                        break;
                    }
                } while (likes._links != null && likes._links.Next != null && !string.IsNullOrEmpty(likes._links.Next.Href));
            }

            log.Info("C# HTTP trigger function processed a request.");

            // Fetching the name from the path parameter in the request URL
            return req.CreateResponse(HttpStatusCode.OK, "Got " + totalCount + " posts");
        }