예제 #1
1
      private async static Task AddConversationsToDatabaseAsync(Provider provider, Category category, 
         IEnumerable<Conversation> newConversations, TextWriter logger)
      {
         int conversationsAddedToDatabase = 0;
         int conversationsUpdatedInDatabase = 0;

         foreach (var newConversation in newConversations)
         {
            var existingCon = db.Conversations
              .Where(c => c.CategoryID == category.CategoryID &&   
                          c.Url == newConversation.Url)            
               .SingleOrDefault();

            if (existingCon != null && existingCon.LastUpdated < newConversation.LastUpdated)
            {
               existingCon.LastUpdated = newConversation.LastUpdated;
               existingCon.DbUpdated = DateTimeOffset.UtcNow;
               existingCon.Body = newConversation.Body;
               db.Entry(existingCon).State = EntityState.Modified;
               conversationsUpdatedInDatabase++;
            }
            else if (existingCon == null)
            {
               newConversation.DbUpdated = DateTimeOffset.UtcNow;
               db.Conversations.Add(newConversation);
               conversationsAddedToDatabase++;
            }
         }
         logger.WriteLine("Added {0} new conversations, updated {1} conversations", 
            conversationsAddedToDatabase, conversationsUpdatedInDatabase);
         await db.SaveChangesAsync();
      }
예제 #2
0
        public async Task <IEnumerable <DAL.Conversation> > GetConversationsAsync(Provider provider, Altostratus.DAL.Category category, IEnumerable <Tag> tags, DateTime from, int maxResults, TextWriter logger)
        {
            var conversations = new List <Altostratus.DAL.Conversation>();

            var twitterContext = new TwitterContext(getAuth());

            if (maxResults > 100)
            {
                logger.WriteLine("maxResults {0} > 100 ", maxResults);
                throw new Exception("maxResults " + maxResults + " > 100");
            }

            try
            {
                // inclusive OR clause
                // https://dev.twitter.com/rest/public/search
                var tagString = String.Join(" OR ", tags.Select(t => t.Value));

                var searchResponse = await
                                         (from search in twitterContext.Search
                                         where search.Type == SearchType.Search &&
                                         search.Query == tagString &&
                                         search.Count == maxResults
                                         select search)
                                     .SingleOrDefaultAsync();

                if (searchResponse == null || searchResponse.Statuses == null)
                {
                    string errMsg = searchResponse == null ? "null Response from LinqToTwitter" : "null Statuses from LinqToTwitter";
                    logger.WriteLine(errMsg);
                    return(conversations);
                }

                logger.WriteLine("Found {0} tweets for tag {1}", searchResponse.Statuses.Count(), tagString);

                // Perform a LINQ to Objects query on the results filtering out retweets.

                var nonRetweetedStatuses =
                    (from tweet in searchResponse.Statuses
                     where tweet.RetweetedStatus.StatusID == 0
                     select tweet)
                    .ToList();

                if (nonRetweetedStatuses == null)
                {
                    logger.WriteLine("null Response from No RetweetedStatus on LinqToTwitter ");
                    return(conversations);
                }

                logger.WriteLine("Found {0} tweets with nonRetweetedStatuses", nonRetweetedStatuses.Count.ToString());

                nonRetweetedStatuses.ForEach(tweet =>
                                             conversations.Add(
                                                 new Altostratus.DAL.Conversation()
                {
                    Body        = ConvertURLsToLinks(tweet.Text),
                    ProviderID  = provider.ProviderID,
                    Title       = "Tweet by " + tweet.User.ScreenNameResponse,
                    LastUpdated = tweet.CreatedAt,
                    Url         = "http://twitter.com/" + tweet.User.UserIDResponse + "/status/" + tweet.StatusID,
                    CategoryID  = category.CategoryID
                }));
            }

            catch (TwitterQueryException tqe)
            {
                logger.WriteLine(tqe.Message);
                // You can test this by  passing in a negative number for search
                // GetConversationsAsync(maxResults: -3);
                throw tqe;
            }
            return(conversations);
        }
      public async Task<IEnumerable<Conversation>> GetConversationsAsync(Provider provider, Category category, IEnumerable<DAL.Tag> tags, DateTime fromDate, int maxResults, TextWriter logger)
      {
         var conversations = new List<Altostratus.DAL.Conversation>();

         // Create semi-colon delimited list of tags
         StringBuilder sb = new StringBuilder();
         foreach (var t in tags)
         {
            sb.AppendFormat("{0};", t.Value.ToLowerInvariant());
         }

         string tagString = sb.ToString();         
         ISearchMethods search = _client;

         try
         {
            var results = await search.GetMatches("stackoverflow",
                tagged: tagString,
                fromdate: fromDate,
                page: 1,
                pagesize: maxResults,
                filter: "!Ldk(uYFB4LapzC5C.3)FCD");  // Includes answer body and more

            if (results == null)
            {
               logger.WriteLine("StacMan call failed, null response", tagString);
               throw new Exception("StacMan call failed, null response");
            }

            if (results.Error != null)
            {
               logger.WriteLine("StacMan error: {0}", results.Error.Message);
               throw results.Error;
            }

            if (results.Data.Items.Length == 0)
            {
               logger.WriteLine("No threads found for tags {0}", tagString);
               return conversations;
            }

            if (results.Data.Backoff > 0)
            {
               logger.WriteLine("Throttling in effect for {0} seconds", results.Data.Backoff);
               // http://api.stackexchange.com/docs/throttle
               // In V2 we could stop this provider only and not delay other providers.
               await Task.Delay(1000 * (int)results.Data.Backoff);
            }

            logger.WriteLine("{0} threads found for tags {1}", results.Data.Items.Length, tagString);
            foreach (var question in results.Data.Items)
            {
               var sbq = new StringBuilder();
               sbq.Append(question.Body);
               var lastUpdate = question.LastActivityDate;

               if (question.AnswerCount > 0)
               {
                  foreach (var answer in question.Answers)
                  {
                      sbq.Append("<div class='panel panel-primary'><div class='panel-heading'><h3 class='panel-title'>Answer</h3></div><div class='panel-body'>" + answer.Body + "</div></div>");
                      if (answer.LastActivityDate > lastUpdate)
                     {
                        lastUpdate = answer.LastActivityDate;
                     }
                  }
               }
               conversations.Add(new Conversation
               {
                  Title = question.Title,
                  Body = sbq.ToString(),
                  Url = question.Link,
                  LastUpdated = lastUpdate,
                  ProviderID = provider.ProviderID,
                  CategoryID = category.CategoryID
               });
            }
         }
         catch (Exception e)
         {
            logger.WriteLine("Exception: {0}\n{1}", e.Message, e.StackTrace);
            throw;
         }
         return conversations;
      }