public async Task <IEnumerable <TextSentiment> > DetectPositiveSentiments(IEnumerable <TextEntry> entries)
{
var response = await _amazonComprehendClient.BatchDetectSentimentAsync(new BatchDetectSentimentRequest
{
LanguageCode = LanguageCode.En,
TextList = entries.Select(x => x.Text).ToList()
});
return(entries.Zip(response.ResultList, (entry, result) => new TextSentiment
{
OriginId = entry.OriginId,
Text = entry.Text,
SentimentType = result.Sentiment.Value,
SentimentScore = result.SentimentScore.GetScore(),
SentimentIndex = result.Index
}).ToList());
}
private Amazon.Comprehend.Model.BatchDetectSentimentResponse CallAWSServiceOperation(IAmazonComprehend client, Amazon.Comprehend.Model.BatchDetectSentimentRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Comprehend", "BatchDetectSentiment");
try
{
#if DESKTOP
return(client.BatchDetectSentiment(request));
#elif CORECLR
return(client.BatchDetectSentimentAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
public override async Task ProcessEventAsync(LambdaScheduleEvent request)
{
var lastId = 0L;
// read last_id from table
var document = await _table.GetItemAsync("last");
if (
(document != null) &&
document.TryGetValue("Query", out var queryEntry) &&
(queryEntry.AsString() == _twitterSearchQuery) &&
document.TryGetValue("LastId", out var lastIdEntry)
)
{
lastId = lastIdEntry.AsLong();
}
// query for tweets since last id
LogInfo($"searching for tweets: query='{_twitterSearchQuery}', last_id={lastId}");
var tweets = TwitterSearch.SearchTweets(new SearchTweetsParameters(_twitterSearchQuery)
{
Lang = null,
TweetSearchType = TweetSearchType.OriginalTweetsOnly,
SinceId = lastId
}) ?? Enumerable.Empty <ITweet>();
// check if any tweets were found
LogInfo($"found {tweets.Count():N0} tweets");
if (tweets.Any())
{
var languages = string.Join(",", _twitterLanguageFilter.OrderBy(language => language));
// send all tweets to topic
var tasks = tweets
// convert tweet object back to JSON to extract the ISO language
.Select(tweet => JObject.Parse(tweet.ToJson()))
.Select(json => new {
Json = json,
IsoLanguage = (string)json["metadata"]["iso_language_code"]
})
// only keep tweets that match the ISO language filter if one is set
.Where(item => {
if (_twitterLanguageFilter.Any() && !_twitterLanguageFilter.Contains(item.IsoLanguage))
{
LogInfo($"tweet language '{item.IsoLanguage}' did not match '{languages}'");
return(false);
}
return(true);
})
// group tweets by language for sentiment analysis
.GroupBy(item => item.IsoLanguage, item => item.Json)
.Select(async group => {
if (_twitterSentimentFilter == "SKIP")
{
// skip sentiment analysis
return(group);
}
// loop over all tweets
var remaining = group.ToList();
while (remaining.Any())
{
// batch analyze tweets
var batch = remaining.Take(25).ToArray();
var response = await _comprehendClient.BatchDetectSentimentAsync(new BatchDetectSentimentRequest {
LanguageCode = group.Key,
TextList = batch.Select(item => (string)item["full_text"]).ToList()
});
// set result for successfully analyzed tweets
foreach (var result in response.ResultList)
{
batch[result.Index]["aws_sentiment"] = result.Sentiment.Value;
}
// set 'N/A' for tweets that failed analysis
foreach (var result in response.ErrorList)
{
batch[result.Index]["aws_sentiment"] = "N/A";
}
// skip analyzed tweets
remaining = remaining.Skip(25).ToList();
}
return(group);
})
.SelectMany(group => group.Result)
.Where(jsonTweet => {
// check if message should be filtered based-on sentiment
switch (_twitterSentimentFilter)
{
case "SKIP":
case "ALL":
return(true);
default:
if (((string)jsonTweet["aws_sentiment"]) != _twitterSentimentFilter)
{
LogInfo($"tweet sentiment '{jsonTweet["aws_sentiment"]}' did not match '{_twitterSentimentFilter}'");
return(false);
}
return(true);
}
})
.Select((jsonTweet, index) => {
LogInfo($"sending tweet #{index + 1}: {jsonTweet["full_text"]}\n{jsonTweet}");
return(_snsClient.PublishAsync(new PublishRequest {
TopicArn = _notificationTopic,
Message = jsonTweet.ToString(Formatting.None)
}));
})
.ToList();
// store updated last_id
await _table.PutItemAsync(new Document {
["Id"] = "last",
["LastId"] = tweets.Max(tweet => tweet.Id),
["Query"] = _twitterSearchQuery
});
// wait for all tasks to finish before exiting
LogInfo($"waiting for all tweets to be sent");
await Task.WhenAll(tasks.ToArray());
LogInfo($"all done");
}
}