public static async Task Run([QueueTrigger(Constants.PostsToProcessQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); PostsToProcess postsToProcess = JsonConvert.DeserializeObject <PostsToProcess>(myQueueItem); PostProcessor postProcessor = new PostProcessor(); postProcessor.Init(log); try { await postProcessor.ProcessPosts(postsToProcess.Posts, log, postsToProcess.LikerBlogname); } catch (Exception ex) { log.Error("Error", ex); throw; } }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getfrequencies")] HttpRequestMessage req, TraceWriter log) { Startup.Init(); ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter(); imageAnalysisTableAdapter.Init(); TokenAllocationTableAdapter tokenAllocationTableAdapter = new TokenAllocationTableAdapter(); tokenAllocationTableAdapter.Init(); List <ImageAnalysisEntity> analyses = imageAnalysisTableAdapter.GetAllCanonical(); Dictionary <string, int> digramFrequencies = new Dictionary <string, int>(); Dictionary <string, int> labelFrequencies = new Dictionary <string, int>(); int processedCount = 0; foreach (ImageAnalysisEntity entity in analyses) { ImageAnalysis canonicalAnalysis = JsonConvert.DeserializeObject <ImageAnalysis>(entity.CanonicalJson); UpdateCounts(StringTokenizer.GetDigrams(canonicalAnalysis.TokenizedText), digramFrequencies); UpdateCounts(canonicalAnalysis.Labels.Keys.Distinct(), labelFrequencies); processedCount++; if (processedCount % 100 == 0) { log.Info($"Processed frequencies for {processedCount} image analyses"); } } log.Info($"Inserting {digramFrequencies.Count} digrams"); tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionDigram, digramFrequencies); log.Info($"Inserting {labelFrequencies.Count} labels"); tokenAllocationTableAdapter.InsertFrequencies(TokenAllocationTableAdapter.PartitionLabel, labelFrequencies); return(req.CreateResponse(HttpStatusCode.OK, $"Processed {analyses.Count} analyses")); }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "initindexing")] HttpRequestMessage req, TraceWriter log) { Startup.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); BlogToIndexQueueAdapter blogToIndexQueueAdapter = new BlogToIndexQueueAdapter(); blogToIndexQueueAdapter.Init(); List <string> partitions = postsTableAdapter.GetAllPartitions(); log.Info($"Got {partitions.Count} blogs to index"); //Random random = new Random(); //partitions.Shuffle(random); //partitions = partitions.Take(10).ToList(); int index = 0; foreach (string blogname in partitions) { blogToIndexQueueAdapter.Send(new BlogToIndex { Blogname = blogname }); index++; if (index % 100 == 0) { log.Info($"Queued {index} blogs for indexing"); } } return(req.CreateResponse(HttpStatusCode.OK, "Queued " + partitions.Count + " blogs for indexing")); }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "redo-canonical")] HttpRequestMessage req, TraceWriter log) { Startup.Init(); ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter(); imageAnalysisTableAdapter.Init(); List <ImageAnalysisEntity> analyses = imageAnalysisTableAdapter.GetAll(); foreach (ImageAnalysisEntity entity in analyses) { Response visionResponse = JsonConvert.DeserializeObject <Response>(entity.GoogleVisionApiJson); Analysis msAnalysis = JsonConvert.DeserializeObject <Analysis>(entity.MsAnalysisJson); List <Face> msFaces = JsonConvert.DeserializeObject <List <Face> >(entity.MsCognitiveFaceDetectJson); ImageAnalysis canonicalImageAnalysis = new ImageAnalysis(visionResponse, msAnalysis, msFaces); entity.CanonicalJson = JsonConvert.SerializeObject(canonicalImageAnalysis, JsonUtils.AnalysisSerializerSettings); imageAnalysisTableAdapter.UpdateImageAnalysis(entity); } return(req.CreateResponse(HttpStatusCode.OK, $"Updated {analyses.Count} analyses")); }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "analyzerandomphotos")] HttpRequestMessage req, TraceWriter log) { Startup.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter(); imageAnalysisTableAdapter.Init(); PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter(); photoToAnalyzeQueueAdapter.Init(); string blobBaseUrl = ConfigurationManager.AppSettings["BlobBaseUrl"]; int blogsLimit = 50; int photosInBlogLimit = 10; BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter(); blogInfoTableAdapter.Init(); List <BlogStats> blogStats = blogInfoTableAdapter.GetBlogStats(); log.Info($"Got {blogStats.Count} blogs to index"); Random random = new Random(); blogStats.Shuffle(random); blogStats = blogStats.Take(blogsLimit).ToList(); int totalCount = 0; foreach (string blogname in blogStats.Select(x => x.RowKey)) { int analyzedInBlogCount = 0; List <PostEntity> noteCounts = postsTableAdapter.GetPostNoteCounts(blogname).OrderByDescending(x => x.NoteCount).ToList(); log.Info($"Got note counts for {noteCounts.Count} posts in blog {blogname}"); foreach (PostEntity noteCountPost in noteCounts) { PostEntity postEntity = postsTableAdapter.GetPost(blogname, noteCountPost.RowKey); if (postEntity == null) { log.Warning($"Post {blogname}/{noteCountPost.RowKey} not found, skipping"); continue; } if (string.IsNullOrEmpty(postEntity.PhotoBlobUrls)) { continue; } List <Photo> sitePhotos = JsonConvert.DeserializeObject <List <Photo> >(postEntity.PhotoBlobUrls); foreach (Photo photo in sitePhotos) { List <PhotoSize> sortedSizes = photo.Sizes.OrderByDescending(x => x.Nominal).ToList(); PhotoSize original = sortedSizes.FirstOrDefault(); if (original == null) { continue; } string url = blobBaseUrl + "/" + original.Container + "/" + photo.Name + "_" + original.Nominal + "." + photo.Extension; if (imageAnalysisTableAdapter.GetImageAnalysis(url) != null) { log.Info($"Image {url} already analyzed"); continue; } PhotoToAnalyze message = new PhotoToAnalyze { Blog = blogname, PostDate = postEntity.Date, Url = url }; photoToAnalyzeQueueAdapter.Send(message); log.Info($"Published PhotoToAnalyze message with URL {url}"); analyzedInBlogCount++; totalCount++; } if (analyzedInBlogCount >= photosInBlogLimit) { break; } } } return(req.CreateResponse(HttpStatusCode.OK, $"Will analyze {totalCount} new photos")); }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "analyze-from-likes/{blogname}")] HttpRequestMessage req, string blogname, TraceWriter log) { Startup.Init(); LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter(); likeIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter(); imageAnalysisTableAdapter.Init(); PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter(); photoToAnalyzeQueueAdapter.Init(); string blobBaseUrl = ConfigurationManager.AppSettings["BlobBaseUrl"]; string afterParam = req.GetQueryNameValuePairs().FirstOrDefault(q => q.Key.Equals("after", StringComparison.OrdinalIgnoreCase)).Value; List <LikeIndexEntity> likes; if (!string.IsNullOrEmpty(afterParam) && long.TryParse(afterParam, out long afterTimestamp)) { log.Info($"Getting likes newer than timestamp {afterTimestamp}"); likes = likeIndexTableAdapter.GetNewerThan(blogname, afterTimestamp); } else { likes = likeIndexTableAdapter.GetAll(blogname); } log.Info($"Loaded {likes.Count} posts"); int messageCount = 0; foreach (LikeIndexEntity like in likes) { if (like.LikedBlogName == null || like.LikedPostId == null) { continue; } PostEntity postEntity = postsTableAdapter.GetPost(like.LikedBlogName, like.LikedPostId); if (postEntity == null) { log.Warning($"Post {like.LikedBlogName}/{like.LikedPostId} not found, skipping"); continue; } if (string.IsNullOrEmpty(postEntity.PhotoBlobUrls)) { continue; } List <Photo> sitePhotos = JsonConvert.DeserializeObject <List <Photo> >(postEntity.PhotoBlobUrls); foreach (Photo photo in sitePhotos) { List <PhotoSize> sortedSizes = photo.Sizes.OrderByDescending(x => x.Nominal).ToList(); PhotoSize original = sortedSizes.FirstOrDefault(); if (original == null) { continue; } string url = blobBaseUrl + "/" + original.Container + "/" + photo.Name + "_" + original.Nominal + "." + photo.Extension; if (imageAnalysisTableAdapter.GetImageAnalysis(url) != null) { log.Info($"Image {url} already analyzed"); continue; } PhotoToAnalyze message = new PhotoToAnalyze { Blog = blogname, PostDate = postEntity.Date, Url = url }; photoToAnalyzeQueueAdapter.Send(message); log.Info($"Published PhotoToAnalyze message with URL {url}"); messageCount++; } } return(req.CreateResponse(HttpStatusCode.OK, $"Processed {likes.Count} posts, sent {messageCount} messages")); }
public static async Task Run([QueueTrigger(Constants.BlogToIndexQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); BlogToIndex blogToIndex = JsonConvert.DeserializeObject <BlogToIndex>(myQueueItem); PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter(); photoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); PostToGetQueueAdapter postToGetQueueAdapter = new PostToGetQueueAdapter(); postToGetQueueAdapter.Init(); BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter(); blogInfoTableAdapter.Init(); MediaToDownloadQueueAdapter mediaToDownloadQueueAdapter = new MediaToDownloadQueueAdapter(); mediaToDownloadQueueAdapter.Init(log); List <PhotoIndexEntity> photoIndexEntities = photoIndexTableAdapter.GetAll(blogToIndex.Blogname); log.Info("Loaded " + photoIndexEntities.Count + " photo index entities"); BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToIndex.Blogname); Dictionary <string, List <Model.Site.Photo> > photosByBlogById = CreatePhotosByBlogById(photoIndexEntities); BlogStats blogStats = CreateBlogStatsFromPhotos(photoIndexEntities, blogToIndex.Blogname); blogStats.UpdateFromBlogEntity(blogEntity); List <PostEntity> postEntities = postsTableAdapter.GetAll(blogToIndex.Blogname); UpdateBlogStatsFromPosts(blogStats, postEntities); UpdateMonthIndex(blogToIndex.Blogname, postEntities, blogInfoTableAdapter); log.Info("Loaded " + postEntities.Count + " post entities"); foreach (PostEntity postEntity in postEntities) { if (!string.IsNullOrEmpty(postEntity.PhotoBlobUrls)) { try { Model.Site.Photo[] photos = JsonConvert.DeserializeObject <Model.Site.Photo[]>(postEntity.PhotoBlobUrls); if (photos.Any(x => !x.Name.Contains("_"))) { SendToReprocessing(postEntity.PartitionKey, mediaToDownloadQueueAdapter, log, postEntity); } } catch (Exception e) { log.Error("Error: " + e.Message); throw; } } } blogStats.DisplayablePosts = InsertReversePosts(blogToIndex.Blogname, photosByBlogById, postEntities, reversePostsTableAdapter, postsTableAdapter, photoIndexTableAdapter, mediaToDownloadQueueAdapter, log); blogInfoTableAdapter.InsertBlobStats(blogStats); }
public static async Task Run([QueueTrigger(Constants.VideosToDownloadQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); log.Info($"C# Queue trigger function processed: {myQueueItem}"); VideosToDownload videosToDownload = JsonConvert.DeserializeObject <VideosToDownload>(myQueueItem); BlobAdapter blobAdapter = new BlobAdapter(); blobAdapter.Init(); VideoIndexTableAdapter videoIndexTableAdapter = new VideoIndexTableAdapter(); videoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); string sourceBlog = string.IsNullOrEmpty(videosToDownload.SourceBlog) ? videosToDownload.IndexInfo.BlogName : videosToDownload.SourceBlog; sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog); using (HttpClient httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("video/*")); httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*")); List <Video> videos = new List <Video>(); string blogname = videosToDownload.IndexInfo.BlogName; string id = videosToDownload.IndexInfo.PostId; DateTime date = videosToDownload.IndexInfo.PostDate; foreach (VideoUrls videoUrls in videosToDownload.VideoUrls) { try { Video blobVideo = await blobAdapter.HandleVideo(videoUrls, videosToDownload.IndexInfo.BlogName, log); videos.Add(blobVideo); videoIndexTableAdapter.InsertVideoIndex(blogname, id, date, blobVideo, videosToDownload.VideoType, blobVideo.Bytes, videosToDownload.Duration); log.Info("Video successfully downloaded: " + videoUrls.VideoUrl); } catch (HttpRequestException ex) { log.Warning("HTTP Error while downloading video " + videoUrls.VideoUrl + " - " + ex.Message); postsTableAdapter.MarkWithVideoDownloadError(blogname, id, ex.Message); } catch (Exception ex) { log.Error("Error while downloading video ", ex); throw; } } if (videos.Count > 0) { postsTableAdapter.MarkVideosAsDownloaded(videosToDownload.IndexInfo.BlogName, videosToDownload.IndexInfo.PostId, videos.ToArray()); ReversePostEntity reversePost = new ReversePostEntity(blogname, id, videosToDownload.PostType, date, videosToDownload.Body, videosToDownload.Title) { Videos = JsonConvert.SerializeObject(videos) }; reversePostsTableAdapter.InsertPost(reversePost); } } }
public static async Task Run([TimerTrigger("0 25 * * * *")] TimerInfo myTimer, TraceWriter log) { Startup.Init(); log.Info($"C# Timer trigger function executed at: {DateTime.Now}"); BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter(); blogToFetchQueueAdapter.Init(); BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter(); blogInfoTableAdapter.Init(); PostsGetter postsGetter = new PostsGetter(); Stopwatch stopwatch = Stopwatch.StartNew(); bool success; // basically means that can the message be deleted, or if it needs to be left in queue to be resumed later do { //TODO: error handling, if there is error from e.g. postsGetter CloudQueueMessage message = await blogToFetchQueueAdapter.GetNextMessage(); if (message == null) { return; } BlogToFetch blogToFetch = JsonConvert.DeserializeObject <BlogToFetch>(message.AsString); BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogToFetch.Blogname); long timeoutLeft = 270 - stopwatch.ElapsedMilliseconds / 1000; if (timeoutLeft < 10) { return; } success = false; GetPostsResult result = null; if (blogToFetch.NewerThan.HasValue) { result = await postsGetter.GetNewerPosts(log, blogToFetch.Blogname, blogToFetch.NewerThan.Value, timeoutLeft); if (result.Success) { success = true; } } int offset = 0; if (blogEntity?.FetchedUntilOffset != null && !blogToFetch.UpdateNpf) { offset = blogEntity.FetchedUntilOffset.Value; } if (result != null) { offset += (int)result.TotalReceived; } if (blogEntity != null && (!blogEntity.FetchedUntilOffset.HasValue || blogEntity.FetchedUntilOffset.Value < Constants.MaxPostsToFetch)) { result = await postsGetter.GetPosts(log, blogToFetch.Blogname, offset, timeoutSeconds : timeoutLeft, updateNpf : blogToFetch.UpdateNpf); if (result.Success) { success = true; } } else { success = true; // enough fetched already, message can be deleted } if (success) { await blogToFetchQueueAdapter.DeleteMessage(message); } } while (success); }
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getblogsfromlikes/{blogname}")] HttpRequestMessage req, string blogname, TraceWriter log) { Startup.Init(); LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter(); likeIndexTableAdapter.Init(); BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter(); blogInfoTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); BlogToFetchQueueAdapter blogToFetchQueueAdapter = new BlogToFetchQueueAdapter(); blogToFetchQueueAdapter.Init(); List <LikeIndexEntity> entities = likeIndexTableAdapter.GetAll(blogname); ILookup <string, LikeIndexEntity> likesByBlog = entities.ToLookup(e => e.LikedBlogName); List <BlogStatsRow> stats = likesByBlog.Select(gr => new BlogStatsRow { Blogname = gr.Key, LikedPostCount = gr.Count() }).OrderByDescending(x => x.LikedPostCount).ToList(); string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"]; List <BlogStatsRow> toDownload = new List <BlogStatsRow>(); using (HttpClient httpClient = new HttpClient()) { foreach (BlogStatsRow blogStatsRow in stats) { if (blogStatsRow.LikedPostCount < 10) { continue; } string url = "https://api.tumblr.com/v2/blog/" + blogStatsRow.Blogname + "/info?api_key=" + apiKey; //log.Info("Making request to: " + url); HttpResponseMessage response = await httpClient.GetAsync(url); if (response.IsSuccessStatusCode) { TumblrResponse <BlogInfo> tumblrResponse = await response.Content.ReadAsAsync <TumblrResponse <BlogInfo> >(); Blog blog = tumblrResponse.Response.Blog; BlogEntity blogEntity = await blogInfoTableAdapter.GetBlog(blogStatsRow.Blogname); blogStatsRow.HadPostCount = postsTableAdapter.GetPostCount(blogStatsRow.Blogname); blogStatsRow.TotalPostCount = blog.Posts; long difference = blog.Posts - blogStatsRow.HadPostCount; bool fetch = false; long?newerThan = null; if (blogEntity != null && blogEntity.Updated < blog.Updated) { log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, has new posts"); fetch = true; newerThan = blogEntity.Updated; } else if (blogStatsRow.HadPostCount > Constants.MaxPostsToFetch) { log.Info("Already fetched " + blogStatsRow.HadPostCount + " posts from blog " + blogStatsRow.Blogname); } else if (difference > 5) { log.Info("Blog " + blogStatsRow.Blogname + " to be downloaded, missing " + difference + " posts"); fetch = true; } else { log.Info("Blog " + blogStatsRow.Blogname + " already downloaded (difference " + difference + ")"); } if (fetch) { blogToFetchQueueAdapter.SendBlogToFetch(new BlogToFetch { Blogname = blog.Name, TotalPostCount = blog.Posts, NewerThan = newerThan }); toDownload.Add(blogStatsRow); } blogEntity = new BlogEntity(blog); blogInfoTableAdapter.InsertBlog(blogEntity); } } } return(req.CreateResponse(HttpStatusCode.OK, "Got " + toDownload.Count + " blogs to fetch")); }
public static async Task Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "processimage")] HttpRequestMessage req, TraceWriter log) { Startup.Init(); ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter(); imageAnalysisTableAdapter.Init(); const string imageUrl = "https://tumblrpics.blob.core.windows.net/orig-66/c48440825ac6eab1af6c4de39bbc59d6_ph8zkhbeVA1tf8706_1280.jpg"; using (HttpClient httpClient = new HttpClient()) { Stopwatch stopwatch = Stopwatch.StartNew(); string apiKey = ConfigurationManager.AppSettings["GoogleApiKey"]; string url = "https://vision.googleapis.com/v1/images:annotate?key=" + apiKey; VisionApiRequest request = VisionApiRequest.CreateFromImageUris(imageUrl); string requestJson = JsonConvert.SerializeObject(request, JsonUtils.GoogleSerializerSettings); StringContent stringContent = new StringContent(requestJson, Encoding.UTF8, "application/json"); HttpResponseMessage response = await httpClient.PostAsync(url, stringContent); HttpContent responseContent = response.Content; string googleVisionResponseString = await responseContent.ReadAsStringAsync(); VisionApiResponse visionApiResponse = JsonConvert.DeserializeObject <VisionApiResponse>(googleVisionResponseString, JsonUtils.GoogleSerializerSettings); string faceApiKey = ConfigurationManager.AppSettings["FaceApiKey"]; httpClient.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", faceApiKey); stringContent = new StringContent($"{{\"url\":\"{imageUrl}\"}}", Encoding.UTF8, "application/json"); response = await httpClient.PostAsync( "https://northeurope.api.cognitive.microsoft.com/face/v1.0/detect?returnFaceId=true&returnFaceLandmarks=false&returnFaceAttributes=" + "age,gender,headPose,smile,facialHair,glasses,emotion,hair,makeup,occlusion,accessories,blur,exposure,noise", stringContent); responseContent = response.Content; string msDetectResponseString = await responseContent.ReadAsStringAsync(); List <Face> msFaces = JsonConvert.DeserializeObject <List <Face> >(msDetectResponseString); string visionApiKey = ConfigurationManager.AppSettings["ComputerVisionApiKey"]; httpClient.DefaultRequestHeaders.Remove("Ocp-Apim-Subscription-Key"); httpClient.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", visionApiKey); stringContent = new StringContent($"{{\"url\":\"{imageUrl}\"}}", Encoding.UTF8, "application/json"); response = await httpClient.PostAsync( "https://northeurope.api.cognitive.microsoft.com/vision/v2.0/analyze?visualFeatures=Description,ImageType,Adult,Categories,Tags,Objects,Color&language=en", stringContent); responseContent = response.Content; string msAnalyzeResponseString = await responseContent.ReadAsStringAsync(); Analysis msAnalysis = JsonConvert.DeserializeObject <Analysis>(msAnalyzeResponseString); if (visionApiResponse.Responses.Count == 1 && msFaces.Count > 0 && msAnalysis != null) { ImageAnalysis canonicalImageAnalysis = new ImageAnalysis(visionApiResponse.Responses[0], msAnalysis, msFaces); } } }
public static async Task<HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "getlikes/{blogname}")] HttpRequestMessage req, string blogname, TraceWriter log) { Startup.Init(); blogname = blogname.ToLower().Replace(".tumblr.com", ""); PostsToProcessQueueAdapter postsToProcessQueueAdapter = new PostsToProcessQueueAdapter(); postsToProcessQueueAdapter.Init(log); LikeIndexTableAdapter likeIndexTableAdapter = new LikeIndexTableAdapter(); likeIndexTableAdapter.Init(); long newestLikedTimestamp = likeIndexTableAdapter.GetNewestLikedTimestamp(blogname); if (newestLikedTimestamp > 0) { log.Info($"Getting likes newer than timestamp {newestLikedTimestamp}"); } Likes likes = null; long totalCount = 0; using (HttpClient httpClient = new HttpClient()) { DateTime beforeTime = DateTime.UtcNow; do { string apiKey = ConfigurationManager.AppSettings["TumblrApiKey"]; string url; if (likes == null) { long timestamp = FunctionUtilities.GetUnixTime(beforeTime); url = "https://api.tumblr.com/v2/blog/" + blogname + "/likes?before=" + timestamp + "&api_key=" + apiKey; } else { url = "https://api.tumblr.com" + likes._links.Next.Href + "&api_key=" + apiKey; } log.Info("Making request to: " + url); HttpResponseMessage response = await httpClient.GetAsync(url); if (response.IsSuccessStatusCode) { TumblrResponse<Likes> tumblrResponse = await response.Content.ReadAsAsync<TumblrResponse<Likes>>(); likes = tumblrResponse.Response; if (newestLikedTimestamp > 0) { List<Post> newerPosts = likes.Liked_posts.Where(x => x.Liked_Timestamp > newestLikedTimestamp).ToList(); if (newerPosts.Count < likes.Liked_posts.Count) { log.Info($"Reached Liked_Timestamp of {newestLikedTimestamp} which has already been fetched, finishing"); postsToProcessQueueAdapter.SendPostsToProcess(newerPosts, blogname); totalCount += newerPosts.Count; break; } } totalCount += likes.Liked_posts.Count; postsToProcessQueueAdapter.SendPostsToProcess(likes.Liked_posts, blogname); } else { log.Info("Got response: " + response.ReasonPhrase + " " + response.StatusCode); break; } } while (likes._links != null && likes._links.Next != null && !string.IsNullOrEmpty(likes._links.Next.Href)); } log.Info("C# HTTP trigger function processed a request."); // Fetching the name from the path parameter in the request URL return req.CreateResponse(HttpStatusCode.OK, "Got " + totalCount + " posts"); }
public static async Task Run([TimerTrigger("0 15 * * * *")] TimerInfo myTimer, TraceWriter log) { Startup.Init(); PhotoToAnalyzeQueueAdapter photoToAnalyzeQueueAdapter = new PhotoToAnalyzeQueueAdapter(); photoToAnalyzeQueueAdapter.Init(); CloudQueueMessage message = null; PhotoToAnalyze photoToAnalyze = null; try { ImageAnalysisTableAdapter imageAnalysisTableAdapter = new ImageAnalysisTableAdapter(); imageAnalysisTableAdapter.Init(); BlogInfoTableAdapter blogInfoTableAdapter = new BlogInfoTableAdapter(); blogInfoTableAdapter.Init(); int processedCount = 0; do { message = await photoToAnalyzeQueueAdapter.GetNextMessage(); if (message == null) { return; } photoToAnalyze = JsonConvert.DeserializeObject <PhotoToAnalyze>(message.AsString); if (imageAnalysisTableAdapter.GetImageAnalysis(photoToAnalyze.Url) != null) { log.Info($"Image {photoToAnalyze.Url} already analyzed, aborting"); await UpdateBlogInfo(photoToAnalyze, blogInfoTableAdapter); continue; } string failure = await ImageAnalyzer.AnalyzePhoto(log, photoToAnalyze, imageAnalysisTableAdapter); if (string.IsNullOrEmpty(failure)) { await UpdateBlogInfo(photoToAnalyze, blogInfoTableAdapter); } else { photoToAnalyze.Error = failure; await photoToAnalyzeQueueAdapter.SendToPoisonQueue(photoToAnalyze); log.Info($"Message for {photoToAnalyze.Url} stored to poison queue"); } processedCount++; await photoToAnalyzeQueueAdapter.DeleteMessage(message); } while (processedCount < 10); } catch (Exception ex) { log.Error("Error in ProcessPhotosToAnalyze: " + ex.Message, ex); if (message != null && photoToAnalyze != null) { photoToAnalyze.Error = ex.Message; photoToAnalyze.StackTrace = ex.StackTrace.Substring(0, Math.Max(36000, ex.StackTrace.Length)); await photoToAnalyzeQueueAdapter.SendToPoisonQueue(photoToAnalyze); log.Info($"Message for {photoToAnalyze.Url} stored to poison queue"); await photoToAnalyzeQueueAdapter.DeleteMessage(message); log.Info($"Failed message deleted successfully from main queue"); message = null; } } }
public static async Task Run([QueueTrigger(Constants.PhotosToDownloadQueueName, Connection = "AzureWebJobsStorage")] string myQueueItem, TraceWriter log) { Startup.Init(); string requestUrl = null; try { PhotosToDownload photosToDownload = JsonConvert.DeserializeObject <PhotosToDownload>(myQueueItem); BlobAdapter blobAdapter = new BlobAdapter(); blobAdapter.Init(); PhotoIndexTableAdapter photoIndexTableAdapter = new PhotoIndexTableAdapter(); photoIndexTableAdapter.Init(); PostsTableAdapter postsTableAdapter = new PostsTableAdapter(); postsTableAdapter.Init(log); ReversePostsTableAdapter reversePostsTableAdapter = new ReversePostsTableAdapter(); reversePostsTableAdapter.Init(log); List <Photo> sitePhotos = new List <Photo>(); string blogname = photosToDownload.IndexInfo.BlogName; string id = photosToDownload.IndexInfo.PostId; DateTime date = photosToDownload.IndexInfo.PostDate; string sourceBlog = string.IsNullOrEmpty(photosToDownload.SourceBlog) ? photosToDownload.IndexInfo.BlogName : photosToDownload.SourceBlog; sourceBlog = SanityHelper.SanitizeSourceBlog(sourceBlog); using (HttpClient httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/*")); foreach (Model.Tumblr.Photo photo in photosToDownload.Photos) { bool isOriginal = true; Photo sitePhoto = null; foreach (AltSize altSize in photo.Alt_sizes) { PhotoUrlHelper urlHelper = PhotoUrlHelper.ParseTumblr(altSize.Url); if (isOriginal || urlHelper != null && DownloadSizes.Contains(urlHelper.Size)) { if (sitePhoto == null) { sitePhoto = new Photo { Name = urlHelper.Container + "_" + urlHelper.Name, Extension = urlHelper.Extension, Sizes = new PhotoSize[0] } } ; PhotoUrlIndexEntity urlIndexEntity = photoIndexTableAdapter.GetPhotoUrlndex(sourceBlog, altSize.Url); if (urlIndexEntity != null) // photo already downloaded { AddSizeToSitePhoto(sitePhoto, urlIndexEntity.BlobUrl, altSize); // need this to produce correct sitePhotos isOriginal = false; } else // photo not downloaded { requestUrl = altSize.Url; byte[] photoBytes = await httpClient.GetByteArrayAsync(altSize.Url); if (photoBytes.Length > 0) { Uri blobUri = await blobAdapter.UploadPhotoBlob(urlHelper, photoBytes, isOriginal); AddSizeToSitePhoto(sitePhoto, blobUri.ToString(), altSize); photoIndexTableAdapter.InsertPhotoIndex(blogname, id, date, SanityHelper.SanitizeSourceBlog(photosToDownload.SourceBlog), blobUri.ToString(), urlHelper.Name, urlHelper.Size, altSize.Width, altSize.Height, altSize.Url); isOriginal = false; log.Info("Downloaded photo from: " + altSize.Url); } } } } if (sitePhoto?.Sizes.Length > 0) { sitePhotos.Add(sitePhoto); } } } string modifiedBody = BodyUrlModifier.ModifyUrls(sourceBlog, photosToDownload.Body, photoIndexTableAdapter, sitePhotos, out List <Model.Tumblr.Photo> extractedPhotos); if (extractedPhotos != null) { log.Warning("Trying to modify body in ProcessPhotosToDownload but some images were not possible to replace"); } postsTableAdapter.MarkPhotosAsDownloaded(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, sitePhotos, modifiedBody); ReversePostEntity reversePost = new ReversePostEntity(photosToDownload.IndexInfo.BlogName, photosToDownload.IndexInfo.PostId, photosToDownload.PostType, photosToDownload.IndexInfo.PostDate, modifiedBody, photosToDownload.Title) { Photos = JsonConvert.SerializeObject(sitePhotos) }; reversePostsTableAdapter.InsertPost(reversePost); } catch (Exception ex) { if (ex is HttpRequestException httpRequestException && httpRequestException.Message.Contains("403") && httpRequestException.Message.Contains("Forbidden")) { log.Warning("HTTP request was forbidden to URL: " + requestUrl); }