private async static Task AddConversationsToDatabaseAsync(Provider provider, Category category,
IEnumerable<Conversation> newConversations, TextWriter logger)
{
int conversationsAddedToDatabase = 0;
int conversationsUpdatedInDatabase = 0;
foreach (var newConversation in newConversations)
{
var existingCon = db.Conversations
.Where(c => c.CategoryID == category.CategoryID &&
c.Url == newConversation.Url)
.SingleOrDefault();
if (existingCon != null && existingCon.LastUpdated < newConversation.LastUpdated)
{
existingCon.LastUpdated = newConversation.LastUpdated;
existingCon.DbUpdated = DateTimeOffset.UtcNow;
existingCon.Body = newConversation.Body;
db.Entry(existingCon).State = EntityState.Modified;
conversationsUpdatedInDatabase++;
}
else if (existingCon == null)
{
newConversation.DbUpdated = DateTimeOffset.UtcNow;
db.Conversations.Add(newConversation);
conversationsAddedToDatabase++;
}
}
logger.WriteLine("Added {0} new conversations, updated {1} conversations",
conversationsAddedToDatabase, conversationsUpdatedInDatabase);
await db.SaveChangesAsync();
}
private static async Task GetConversationsAsync(Provider provider, IProviderAPI providerAPI, IEnumerable<Category> categories, DateTime fromDate, int maxResults, TextWriter logger)
{
// Loop through categories to make one provider API call for each category
// that has tags for the current provider.
foreach (Category category in categories)
{
IEnumerable<Tag> tagsForCategoryAndProvider = category.Tags.Where(t => t.ProviderID == provider.ProviderID);
if (tagsForCategoryAndProvider.Count() > 0)
{
var newConversations = await providerAPI.GetConversationsAsync(provider, category, tagsForCategoryAndProvider, fromDate, maxResults, logger);
await AddConversationsToDatabaseAsync(provider, category, newConversations, logger);
}
else
{
throw new Exception("DB not seeded; No tags for " + provider.Name.ToString());
}
}
}
public async Task<IEnumerable<DAL.Conversation>> GetConversationsAsync(Provider provider, Altostratus.DAL.Category category, IEnumerable<Tag> tags, DateTime from, int maxResults, TextWriter logger)
{
var conversations = new List<Altostratus.DAL.Conversation>();
var twitterContext = new TwitterContext(getAuth());
if (maxResults > 100)
{
logger.WriteLine("maxResults {0} > 100 ", maxResults);
throw new Exception("maxResults " + maxResults + " > 100");
}
try
{
// inclusive OR clause
// https://dev.twitter.com/rest/public/search
var tagString = String.Join(" OR ", tags.Select(t => t.Value));
var searchResponse = await
(from search in twitterContext.Search
where search.Type == SearchType.Search &&
search.Query == tagString &&
search.Count == maxResults
select search)
.SingleOrDefaultAsync();
if (searchResponse == null || searchResponse.Statuses == null)
{
string errMsg = searchResponse == null ? "null Response from LinqToTwitter" : "null Statuses from LinqToTwitter";
logger.WriteLine(errMsg);
return conversations;
}
logger.WriteLine("Found {0} tweets for tag {1}", searchResponse.Statuses.Count(), tagString);
// Perform a LINQ to Objects query on the results filtering out retweets.
var nonRetweetedStatuses =
(from tweet in searchResponse.Statuses
where tweet.RetweetedStatus.StatusID == 0
select tweet)
.ToList();
if (nonRetweetedStatuses == null)
{
logger.WriteLine("null Response from No RetweetedStatus on LinqToTwitter ");
return conversations;
}
logger.WriteLine("Found {0} tweets with nonRetweetedStatuses", nonRetweetedStatuses.Count.ToString());
nonRetweetedStatuses.ForEach(tweet =>
conversations.Add(
new Altostratus.DAL.Conversation()
{
Body = ConvertURLsToLinks(tweet.Text),
ProviderID = provider.ProviderID,
Title = "Tweet by " + tweet.User.ScreenNameResponse,
LastUpdated = tweet.CreatedAt,
Url = "http://twitter.com/" + tweet.User.UserIDResponse + "/status/" + tweet.StatusID,
CategoryID = category.CategoryID
}));
}
catch (TwitterQueryException tqe)
{
logger.WriteLine(tqe.Message);
// You can test this by passing in a negative number for search
// GetConversationsAsync(maxResults: -3);
throw tqe;
}
return conversations;
}
public async Task<IEnumerable<Conversation>> GetConversationsAsync(Provider provider, Category category, IEnumerable<DAL.Tag> tags, DateTime fromDate, int maxResults, TextWriter logger)
{
var conversations = new List<Altostratus.DAL.Conversation>();
// Create semi-colon delimited list of tags
StringBuilder sb = new StringBuilder();
foreach (var t in tags)
{
sb.AppendFormat("{0};", t.Value.ToLowerInvariant());
}
string tagString = sb.ToString();
ISearchMethods search = _client;
try
{
var results = await search.GetMatches("stackoverflow",
tagged: tagString,
fromdate: fromDate,
page: 1,
pagesize: maxResults,
filter: "!Ldk(uYFB4LapzC5C.3)FCD"); // Includes answer body and more
if (results == null)
{
logger.WriteLine("StacMan call failed, null response", tagString);
throw new Exception("StacMan call failed, null response");
}
if (results.Error != null)
{
logger.WriteLine("StacMan error: {0}", results.Error.Message);
throw results.Error;
}
if (results.Data.Items.Length == 0)
{
logger.WriteLine("No threads found for tags {0}", tagString);
return conversations;
}
if (results.Data.Backoff > 0)
{
logger.WriteLine("Throttling in effect for {0} seconds", results.Data.Backoff);
// http://api.stackexchange.com/docs/throttle
// In V2 we could stop this provider only and not delay other providers.
await Task.Delay(1000 * (int)results.Data.Backoff);
}
logger.WriteLine("{0} threads found for tags {1}", results.Data.Items.Length, tagString);
foreach (var question in results.Data.Items)
{
var sbq = new StringBuilder();
sbq.Append(question.Body);
var lastUpdate = question.LastActivityDate;
if (question.AnswerCount > 0)
{
foreach (var answer in question.Answers)
{
sbq.Append("<div class='panel panel-primary'><div class='panel-heading'><h3 class='panel-title'>Answer</h3></div><div class='panel-body'>" + answer.Body + "</div></div>");
if (answer.LastActivityDate > lastUpdate)
{
lastUpdate = answer.LastActivityDate;
}
}
}
conversations.Add(new Conversation
{
Title = question.Title,
Body = sbq.ToString(),
Url = question.Link,
LastUpdated = lastUpdate,
ProviderID = provider.ProviderID,
CategoryID = category.CategoryID
});
}
}
catch (Exception e)
{
logger.WriteLine("Exception: {0}\n{1}", e.Message, e.StackTrace);
throw;
}
return conversations;
}
