public async Task Process(SocialGistPostId post) { this.objectLogger.Log(post, "ThreadProcessor"); telemetryClient.TrackEvent(TelemetryNames.ThreadProcessor_ProcessThread); var result = await socialgist.CommentsForThread(post); var filteredPosts = new List <ResponsePost>() { result.Post }.Where(x => !string.IsNullOrEmpty(x.Id) && !string.IsNullOrEmpty(x.Author) && x.CreatedUtc != 0); var filteredComments = result.Comments.Where(x => !string.IsNullOrEmpty(x.Id) && !string.IsNullOrEmpty(x.Author) && x.CreatedUtc != 0); if (filteredComments.Any() || filteredPosts.Any()) { IEnumerable <IDocument> allDocuments = FilterAndWriteDocuments(filteredPosts.ToList().AsReadOnly(), filteredComments.ToList().AsReadOnly()).ToList(); // Perform all actions that need to be run over every post and comment foreach (var processor in this.documentProcessors) { processor.ProcessDocuments(allDocuments); } } else { log.Info($"No thread found with url {post.Url}"); } }
public async Task <ThreadApiResponse> CommentsForThread(SocialGistPostId post) { // Telemetry telemetryClient.TrackEvent(TelemetryNames.SocialGist_Thread, new Dictionary <string, string> { { "url", post.Url } } ); // Do the real work var parameters = GetBaseParameters(); parameters.Add("url", post.Url); parameters.Add("use_compression", true); var baseUrl = "http://redditcomments.socialgist.com/"; var paramMessage = string.Join(",", parameters.Select(kvp => kvp.Key + ": " + kvp.Value.ToString())); log.Info($"Request to: {baseUrl} with initial parameters: {paramMessage}."); var uriBuilder = new UriBuilder(baseUrl); uriBuilder.QueryParamsFromDictionary(parameters); var result = await httpClient.GetJsonAsync <ThreadApiResponse>(uriBuilder.Uri, configuration.SocialGistApiRequestTimeout); await CheckError(result); return(result.Object); }
public async Task <ThreadApiResponse> CommentsForThread(SocialGistPostId post) { // Telemetry telemetryClient.TrackEvent(TelemetryNames.SocialGist_Thread, new Dictionary <string, string> { { "url", post.Url } } ); // Do the real work var parameters = GetBaseParameters(); parameters.Add("url", post.Url); parameters.Add("use_compression", true); var uriBuilder = new UriBuilder("http://redditcomments.socialgist.com/"); uriBuilder.QueryParamsFromDictionary(parameters); var result = await httpClient.GetJsonAsync <ThreadApiResponse>(uriBuilder.Uri, configuration.SocialGistApiRequestTimeout); await CheckError(result); return(result.Object); }
public async Task <SortedSet <SocialGistPostId> > MatchesForQuery( string query, string sortMode, long?startUnixTime = null ) { telemetryClient.TrackEvent(TelemetryNames.SocialGist_Search, new Dictionary <string, string> { { "query", query } }); var parameters = GetBaseParameters(); parameters.Add("query", query); parameters.Add("dn", "reddit.com"); parameters.Add("sort_mode", sortMode); parameters.Add("keep_original", "true"); parameters.Add("group_mode", "thread"); parameters.Add("match_mode", "boolean"); var threadIdSet = new SortedSet <SocialGistPostId>(); var baseUrl = "https://redditapi.socialgist.com/v1/Boards/Search"; var paramMessage = string.Join(",", parameters.Select(kvp => kvp.Key + ": " + kvp.Value.ToString())); log.Info($"Request to: {baseUrl} with initial parameters: {paramMessage}."); var resultList = await paginator.PageThroughCallResults <SearchApiResponse, SearchResponse, SearchMatches, SearchMatch>( baseUrl, parameters, ResultLimitPerPage ); log.Verbose($"Returned Socialgist matches from query. Count: {resultList.Count}"); foreach (var match in resultList) { var postId = new SocialGistPostId() { Url = match.Url }; if (!threadIdSet.Contains(postId)) { log.Verbose($"--Adding {postId} to process queue"); threadIdSet.Add(postId); } else { log.Verbose($"--Ignoring {postId} (already on process queue)"); } } log.Info($"SocialGist query returned {threadIdSet.Count} unique results"); return(threadIdSet); }
public static async Task ProcessRedditPost( [QueueTrigger(QueueConstants.RedditPostQueueName, Connection = QueueConstants.QueueConnectionStringName)] SocialGistPostId socialGistPost, TraceWriter log, ExecutionContext executionContext ) { log.Info($"{FunctionName} Execution begun at {DateTime.Now}"); var config = new WebConfiguration(executionContext); var logger = new FunctionLog(log, executionContext.InvocationId); var objectLogger = (config.UseObjectLogger) ? new BlobObjectLogger(config, logger) : null; using (var kernel = new KernelFactory().GetKernel(logger, config, objectLogger)) { var processor = kernel.Get <IThreadProcessor>(); var socialGist = kernel.Get <ISocialGist>(); socialGist.ResultLimitPerPage = config.ResultLimitPerPage; socialGist.MaximumResultsPerSearch = config.MaximumResultsPerSearch; await processor.Process(socialGistPost); } log.Info($"{FunctionName} completed at {DateTime.Now}"); }