Beispiel #1
0
        public static async Task Run([QueueTrigger(Constants.VideosToDownloadQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log)
        {
            Startup.Init();

            log.Info($"C# Queue trigger function processed: {myQueueItem}");

            VideosToDownload videosToDownload = JsonConvert.DeserializeObject <VideosToDownload>(myQueueItem);

            BlobAdapter blobAdapter = new BlobAdapter();

            blobAdapter.Init();

            VideoIndexTableAdapter videoIndexTableAdapter = new VideoIndexTableAdapter();

            videoIndexTableAdapter.Init();

            PostsTableAdapter postsTableAdapter = new PostsTableAdapter();

            postsTableAdapter.Init(log);

            ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter();

            reversePostsTableAdapter.Init(log);

            string sourceBlog = string.IsNullOrEmpty(videosToDownload.SourceBlog)
                ? videosToDownload.IndexInfo.BlogName
                : videosToDownload.SourceBlog;

            sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog);

            using (HttpClient httpClient = new HttpClient())
            {
                httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("video/*"));
                httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*"));

                List <Video> videos = new List <Video>();

                string   blogname = videosToDownload.IndexInfo.BlogName;
                string   id       = videosToDownload.IndexInfo.PostId;
                DateTime date     = videosToDownload.IndexInfo.PostDate;

                foreach (VideoUrls videoUrls in videosToDownload.VideoUrls)
                {
                    try
                    {
                        Video blobVideo = await blobAdapter.HandleVideo(videoUrls, videosToDownload.IndexInfo.BlogName, log);

                        videos.Add(blobVideo);

                        videoIndexTableAdapter.InsertVideoIndex(blogname, id, date, blobVideo, videosToDownload.VideoType, blobVideo.Bytes, videosToDownload.Duration);

                        log.Info("Video successfully downloaded: " + videoUrls.VideoUrl);
                    }
                    catch (HttpRequestException ex)
                    {
                        log.Warning("HTTP Error while downloading video " + videoUrls.VideoUrl + " - " + ex.Message);
                        postsTableAdapter.MarkWithVideoDownloadError(blogname, id, ex.Message);
                    }
                    catch (Exception ex)
                    {
                        log.Error("Error while downloading video ", ex);
                        throw;
                    }
                }

                if (videos.Count > 0)
                {
                    postsTableAdapter.MarkVideosAsDownloaded(videosToDownload.IndexInfo.BlogName, videosToDownload.IndexInfo.PostId, videos.ToArray());

                    ReversePostEntity reversePost = new ReversePostEntity(blogname, id, videosToDownload.PostType, date, videosToDownload.Body, videosToDownload.Title)
                    {
                        Videos = JsonConvert.SerializeObject(videos)
                    };
                    reversePostsTableAdapter.InsertPost(reversePost);
                }
            }
        }
Beispiel #2
0
        public static async Task Run([TimerTrigger("0 25 * * * *")] TimerInfo myTimer, TraceWriter log)
        {
            Startup.Init();

            log.Info($"C# Timer trigger function executed at: {DateTime.Now}");

            BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();

            blogToFetchQueueAdapter.Init();

            BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();

            blogInfoTableAdapter.Init();

            PostsGetter postsGetter = new PostsGetter();

            Stopwatch stopwatch = Stopwatch.StartNew();

            bool success; // basically means that can the message be deleted, or if it needs to be left in queue to be resumed later

            do
            {
                //TODO: error handling, if there is error from e.g. postsGetter
                CloudQueueMessage message = await blogToFetchQueueAdapter.GetNextMessage();

                if (message == null)
                {
                    return;
                }

                BlogToFetch blogToFetch = JsonConvert.DeserializeObject <BlogToFetch>(message.AsString);

                BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToFetch.Blogname);

                long timeoutLeft = 270 - stopwatch.ElapsedMilliseconds / 1000;
                if (timeoutLeft < 10)
                {
                    return;
                }

                success = false;

                GetPostsResult result = null;
                if (blogToFetch.NewerThan.HasValue)
                {
                    result = await postsGetter.GetNewerPosts(log, blogToFetch.Blogname, blogToFetch.NewerThan.Value, timeoutLeft);

                    if (result.Success)
                    {
                        success = true;
                    }
                }

                int offset = 0;
                if (blogEntity?.FetchedUntilOffset != null && !blogToFetch.UpdateNpf)
                {
                    offset = blogEntity.FetchedUntilOffset.Value;
                }

                if (result != null)
                {
                    offset += (int)result.TotalReceived;
                }

                if (blogEntity != null && (!blogEntity.FetchedUntilOffset.HasValue || blogEntity.FetchedUntilOffset.Value < Constants.MaxPostsToFetch))
                {
                    result = await postsGetter.GetPosts(log, blogToFetch.Blogname, offset, timeoutSeconds : timeoutLeft, updateNpf : blogToFetch.UpdateNpf);

                    if (result.Success)
                    {
                        success = true;
                    }
                }
                else
                {
                    success = true; // enough fetched already, message can be deleted
                }

                if (success)
                {
                    await blogToFetchQueueAdapter.DeleteMessage(message);
                }
            } while (success);
        }
Beispiel #3
0
        public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getblogsfromlikes/{blogname}")] HttpRequestMessage req,
                                                           string blogname, TraceWriter log)
        {
            Startup.Init();

            LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();

            likeIndexTableAdapter.Init();

            BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();

            blogInfoTableAdapter.Init();

            PostsTableAdapter postsTableAdapter = new PostsTableAdapter();

            postsTableAdapter.Init(log);

            BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter();

            blogToFetchQueueAdapter.Init();

            List <LikeIndexEntity>            entities    = likeIndexTableAdapter.GetAll(blogname);
            ILookup <string, LikeIndexEntity> likesByBlog = entities.ToLookup(e => e.LikedBlogName);
            List <BlogStatsRow> stats = likesByBlog.Select(gr => new BlogStatsRow {
                Blogname = gr.Key, LikedPostCount = gr.Count()
            }).OrderByDescending(x => x.LikedPostCount).ToList();

            string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];

            List <BlogStatsRow> toDownload = new List <BlogStatsRow>();

            using (HttpClient httpClient = new HttpClient())
            {
                foreach (BlogStatsRow blogStatsRow in stats)
                {
                    if (blogStatsRow.LikedPostCount < 10)
                    {
                        continue;
                    }

                    string url = "https://api.tumblr.com/v2/blog/" + blogStatsRow.Blogname + "/info?api_key=" + apiKey;
                    //log.Info("Making request to: " + url);
                    HttpResponseMessage response = await httpClient.GetAsync(url);

                    if (response.IsSuccessStatusCode)
                    {
                        TumblrResponse <BlogInfo> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogInfo> >();

                        Blog       blog       = tumblrResponse.Response.Blog;
                        BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogStatsRow.Blogname);

                        blogStatsRow.HadPostCount   = postsTableAdapter.GetPostCount(blogStatsRow.Blogname);
                        blogStatsRow.TotalPostCount = blog.Posts;
                        long difference = blog.Posts - blogStatsRow.HadPostCount;
                        bool fetch      = false;
                        long?newerThan  = null;

                        if (blogEntity != null && blogEntity.Updated < blog.Updated)
                        {
                            log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, has new posts");
                            fetch     = true;
                            newerThan = blogEntity.Updated;
                        }
                        else if (blogStatsRow.HadPostCount > Constants.MaxPostsToFetch)
                        {
                            log.Info("Already fetched " + blogStatsRow.HadPostCount + " posts from blog " + blogStatsRow.Blogname);
                        }
                        else if (difference > 5)
                        {
                            log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, missing " + difference + " posts");
                            fetch = true;
                        }
                        else
                        {
                            log.Info("Blog " + blogStatsRow.Blogname + " already downloaded (difference " + difference + ")");
                        }

                        if (fetch)
                        {
                            blogToFetchQueueAdapter.SendBlogToFetch(new BlogToFetch
                            {
                                Blogname       = blog.Name,
                                TotalPostCount = blog.Posts,
                                NewerThan      = newerThan
                            });
                            toDownload.Add(blogStatsRow);
                        }

                        blogEntity = new BlogEntity(blog);
                        blogInfoTableAdapter.InsertBlog(blogEntity);
                    }
                }
            }

            return(req.CreateResponse(HttpStatusCode.OK, "Got " + toDownload.Count + " blogs to fetch"));
        }
Beispiel #4
0
        public static async Task Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "processimage")]
                                     HttpRequestMessage req, TraceWriter log)
        {
            Startup.Init();

            ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();

            imageAnalysisTableAdapter.Init();

            const string imageUrl = "https://tumblrpics.blob.core.windows.net/orig-66/c48440825ac6eab1af6c4de39bbc59d6_ph8zkhbeVA1tf8706_1280.jpg";

            using (HttpClient httpClient = new HttpClient())
            {
                Stopwatch stopwatch = Stopwatch.StartNew();

                string apiKey = ConfigurationManager.AppSettings["GoogleApiKey"];

                string url = "https://vision.googleapis.com/v1/images:annotate?key=" + apiKey;

                VisionApiRequest request = VisionApiRequest.CreateFromImageUris(imageUrl);

                string requestJson = JsonConvert.SerializeObject(request, JsonUtils.GoogleSerializerSettings);

                StringContent stringContent = new StringContent(requestJson, Encoding.UTF8, "application/json");

                HttpResponseMessage response = await httpClient.PostAsync(url, stringContent);

                HttpContent responseContent            = response.Content;
                string      googleVisionResponseString = await responseContent.ReadAsStringAsync();

                VisionApiResponse visionApiResponse =
                    JsonConvert.DeserializeObject <VisionApiResponse>(googleVisionResponseString, JsonUtils.GoogleSerializerSettings);

                string faceApiKey = ConfigurationManager.AppSettings["FaceApiKey"];
                httpClient.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", faceApiKey);
                stringContent = new StringContent($"{{\"url\":\"{imageUrl}\"}}", Encoding.UTF8, "application/json");
                response      = await httpClient.PostAsync(
                    "https://northeurope.api.cognitive.microsoft.com/face/v1.0/detect?returnFaceId=true&returnFaceLandmarks=false&returnFaceAttributes=" +
                    "age,gender,headPose,smile,facialHair,glasses,emotion,hair,makeup,occlusion,accessories,blur,exposure,noise",
                    stringContent);

                responseContent = response.Content;

                string msDetectResponseString = await responseContent.ReadAsStringAsync();

                List <Face> msFaces = JsonConvert.DeserializeObject <List <Face> >(msDetectResponseString);

                string visionApiKey = ConfigurationManager.AppSettings["ComputerVisionApiKey"];
                httpClient.DefaultRequestHeaders.Remove("Ocp-Apim-Subscription-Key");
                httpClient.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", visionApiKey);
                stringContent = new StringContent($"{{\"url\":\"{imageUrl}\"}}", Encoding.UTF8, "application/json");
                response      = await httpClient.PostAsync(
                    "https://northeurope.api.cognitive.microsoft.com/vision/v2.0/analyze?visualFeatures=Description,ImageType,Adult,Categories,Tags,Objects,Color&language=en",
                    stringContent);

                responseContent = response.Content;

                string msAnalyzeResponseString = await responseContent.ReadAsStringAsync();

                Analysis msAnalysis = JsonConvert.DeserializeObject <Analysis>(msAnalyzeResponseString);

                if (visionApiResponse.Responses.Count == 1 && msFaces.Count > 0 && msAnalysis != null)
                {
                    ImageAnalysis canonicalImageAnalysis = new ImageAnalysis(visionApiResponse.Responses[0], msAnalysis, msFaces);
                }
            }
        }
Beispiel #5
0
        public static async Task<HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getlikes/{blogname}")]
            HttpRequestMessage req, string blogname, TraceWriter log)
        {
            Startup.Init();

            blogname = blogname.ToLower().Replace(".tumblr.com", "");

            PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter();
            postsToProcessQueueAdapter.Init(log);

            LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter();
            likeIndexTableAdapter.Init();

            long newestLikedTimestamp = likeIndexTableAdapter.GetNewestLikedTimestamp(blogname);

            if (newestLikedTimestamp > 0)
            {
                log.Info($"Getting likes newer than timestamp {newestLikedTimestamp}");
            }

            Likes likes = null;
            long totalCount = 0;
            using (HttpClient httpClient = new HttpClient())
            {
                DateTime beforeTime = DateTime.UtcNow;
                do
                {
                    string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"];
                    string url;
                    if (likes == null)
                    {
                        long timestamp = FunctionUtilities.GetUnixTime(beforeTime);
                        url = "https://api.tumblr.com/v2/blog/" + blogname + "/likes?before=" + timestamp + "&api_key=" + apiKey;
                    }
                    else
                    {
                        url = "https://api.tumblr.com" + likes._links.Next.Href + "&api_key=" + apiKey;
                    }

                    log.Info("Making request to: " + url);
                    HttpResponseMessage response = await httpClient.GetAsync(url);
                    if (response.IsSuccessStatusCode)
                    {
                        TumblrResponse<Likes> tumblrResponse = await response.Content.ReadAsAsync<TumblrResponse<Likes>>();
                        likes = tumblrResponse.Response;

                        if (newestLikedTimestamp > 0)
                        {
                            List<Post> newerPosts = likes.Liked_posts.Where(x => x.Liked_Timestamp > newestLikedTimestamp).ToList();
                            if (newerPosts.Count < likes.Liked_posts.Count)
                            {
                                log.Info($"Reached Liked_Timestamp of {newestLikedTimestamp} which has already been fetched, finishing");
                                postsToProcessQueueAdapter.SendPostsToProcess(newerPosts, blogname);

                                totalCount += newerPosts.Count;
                                break;
                            }
                        }

                        totalCount += likes.Liked_posts.Count;

                        postsToProcessQueueAdapter.SendPostsToProcess(likes.Liked_posts, blogname);
                    }
                    else
                    {
                        log.Info("Got response: " + response.ReasonPhrase + " " + response.StatusCode);
                        break;
                    }
                } while (likes._links != null && likes._links.Next != null && !string.IsNullOrEmpty(likes._links.Next.Href));
            }

            log.Info("C# HTTP trigger function processed a request.");

            // Fetching the name from the path parameter in the request URL
            return req.CreateResponse(HttpStatusCode.OK, "Got " + totalCount + " posts");
        }
Beispiel #6
0
        public static async Task Run([TimerTrigger("0 15 * * * *")] TimerInfo myTimer, TraceWriter log)
        {
            Startup.Init();

            PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter();

            photoToAnalyzeQueueAdapter.Init();
            CloudQueueMessage message        = null;
            PhotoToAnalyze    photoToAnalyze = null;

            try
            {
                ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter();
                imageAnalysisTableAdapter.Init();

                BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter();
                blogInfoTableAdapter.Init();

                int processedCount = 0;

                do
                {
                    message = await photoToAnalyzeQueueAdapter.GetNextMessage();

                    if (message == null)
                    {
                        return;
                    }

                    photoToAnalyze = JsonConvert.DeserializeObject <PhotoToAnalyze>(message.AsString);

                    if (imageAnalysisTableAdapter.GetImageAnalysis(photoToAnalyze.Url) != null)
                    {
                        log.Info($"Image {photoToAnalyze.Url} already analyzed, aborting");
                        await UpdateBlogInfo(photoToAnalyze, blogInfoTableAdapter);

                        continue;
                    }

                    string failure = await ImageAnalyzer.AnalyzePhoto(log, photoToAnalyze, imageAnalysisTableAdapter);

                    if (string.IsNullOrEmpty(failure))
                    {
                        await UpdateBlogInfo(photoToAnalyze, blogInfoTableAdapter);
                    }
                    else
                    {
                        photoToAnalyze.Error = failure;
                        await photoToAnalyzeQueueAdapter.SendToPoisonQueue(photoToAnalyze);

                        log.Info($"Message for {photoToAnalyze.Url} stored to poison queue");
                    }

                    processedCount++;

                    await photoToAnalyzeQueueAdapter.DeleteMessage(message);
                } while (processedCount < 10);
            }
            catch (Exception ex)
            {
                log.Error("Error in ProcessPhotosToAnalyze: " + ex.Message, ex);

                if (message != null && photoToAnalyze != null)
                {
                    photoToAnalyze.Error      = ex.Message;
                    photoToAnalyze.StackTrace = ex.StackTrace.Substring(0, Math.Max(36000, ex.StackTrace.Length));
                    await photoToAnalyzeQueueAdapter.SendToPoisonQueue(photoToAnalyze);

                    log.Info($"Message for {photoToAnalyze.Url} stored to poison queue");
                    await photoToAnalyzeQueueAdapter.DeleteMessage(message);

                    log.Info($"Failed message deleted successfully from main queue");
                    message = null;
                }
            }
        }