public async Task Process(SocialGistPostId post)
{
this.objectLogger.Log(post, "ThreadProcessor");
telemetryClient.TrackEvent(TelemetryNames.ThreadProcessor_ProcessThread);
var result = await socialgist.CommentsForThread(post);
var filteredPosts = new List <ResponsePost>()
{
result.Post
}.Where(x =>
!string.IsNullOrEmpty(x.Id) && !string.IsNullOrEmpty(x.Author) && x.CreatedUtc != 0);
var filteredComments = result.Comments.Where(x =>
!string.IsNullOrEmpty(x.Id) && !string.IsNullOrEmpty(x.Author) && x.CreatedUtc != 0);
if (filteredComments.Any() || filteredPosts.Any())
{
IEnumerable <IDocument> allDocuments = FilterAndWriteDocuments(filteredPosts.ToList().AsReadOnly(), filteredComments.ToList().AsReadOnly()).ToList();
// Perform all actions that need to be run over every post and comment
foreach (var processor in this.documentProcessors)
{
processor.ProcessDocuments(allDocuments);
}
}
else
{
log.Info($"No thread found with url {post.Url}");
}
}
public async Task <ThreadApiResponse> CommentsForThread(SocialGistPostId post)
{
// Telemetry
telemetryClient.TrackEvent(TelemetryNames.SocialGist_Thread,
new Dictionary <string, string> {
{ "url", post.Url }
}
);
// Do the real work
var parameters = GetBaseParameters();
parameters.Add("url", post.Url);
parameters.Add("use_compression", true);
var baseUrl = "http://redditcomments.socialgist.com/";
var paramMessage = string.Join(",", parameters.Select(kvp => kvp.Key + ": " + kvp.Value.ToString()));
log.Info($"Request to: {baseUrl} with initial parameters: {paramMessage}.");
var uriBuilder = new UriBuilder(baseUrl);
uriBuilder.QueryParamsFromDictionary(parameters);
var result =
await httpClient.GetJsonAsync <ThreadApiResponse>(uriBuilder.Uri,
configuration.SocialGistApiRequestTimeout);
await CheckError(result);
return(result.Object);
}
public async Task <ThreadApiResponse> CommentsForThread(SocialGistPostId post)
{
// Telemetry
telemetryClient.TrackEvent(TelemetryNames.SocialGist_Thread,
new Dictionary <string, string> {
{ "url", post.Url }
}
);
// Do the real work
var parameters = GetBaseParameters();
parameters.Add("url", post.Url);
parameters.Add("use_compression", true);
var uriBuilder = new UriBuilder("http://redditcomments.socialgist.com/");
uriBuilder.QueryParamsFromDictionary(parameters);
var result =
await httpClient.GetJsonAsync <ThreadApiResponse>(uriBuilder.Uri,
configuration.SocialGistApiRequestTimeout);
await CheckError(result);
return(result.Object);
}
public async Task <SortedSet <SocialGistPostId> > MatchesForQuery(
string query,
string sortMode,
long?startUnixTime = null
)
{
telemetryClient.TrackEvent(TelemetryNames.SocialGist_Search, new Dictionary <string, string> {
{ "query", query }
});
var parameters = GetBaseParameters();
parameters.Add("query", query);
parameters.Add("dn", "reddit.com");
parameters.Add("sort_mode", sortMode);
parameters.Add("keep_original", "true");
parameters.Add("group_mode", "thread");
parameters.Add("match_mode", "boolean");
var threadIdSet = new SortedSet <SocialGistPostId>();
var baseUrl = "https://redditapi.socialgist.com/v1/Boards/Search";
var paramMessage = string.Join(",", parameters.Select(kvp => kvp.Key + ": " + kvp.Value.ToString()));
log.Info($"Request to: {baseUrl} with initial parameters: {paramMessage}.");
var resultList = await paginator.PageThroughCallResults <SearchApiResponse, SearchResponse, SearchMatches, SearchMatch>(
baseUrl,
parameters,
ResultLimitPerPage
);
log.Verbose($"Returned Socialgist matches from query. Count: {resultList.Count}");
foreach (var match in resultList)
{
var postId = new SocialGistPostId()
{
Url = match.Url
};
if (!threadIdSet.Contains(postId))
{
log.Verbose($"--Adding {postId} to process queue");
threadIdSet.Add(postId);
}
else
{
log.Verbose($"--Ignoring {postId} (already on process queue)");
}
}
log.Info($"SocialGist query returned {threadIdSet.Count} unique results");
return(threadIdSet);
}
public static async Task ProcessRedditPost(
[QueueTrigger(QueueConstants.RedditPostQueueName, Connection = QueueConstants.QueueConnectionStringName)] SocialGistPostId socialGistPost,
TraceWriter log,
ExecutionContext executionContext
)
{
log.Info($"{FunctionName} Execution begun at {DateTime.Now}");
var config = new WebConfiguration(executionContext);
var logger = new FunctionLog(log, executionContext.InvocationId);
var objectLogger = (config.UseObjectLogger) ? new BlobObjectLogger(config, logger) : null;
using (var kernel = new KernelFactory().GetKernel(logger, config, objectLogger))
{
var processor = kernel.Get <IThreadProcessor>();
var socialGist = kernel.Get <ISocialGist>();
socialGist.ResultLimitPerPage = config.ResultLimitPerPage;
socialGist.MaximumResultsPerSearch = config.MaximumResultsPerSearch;
await processor.Process(socialGistPost);
}
log.Info($"{FunctionName} completed at {DateTime.Now}");
}
