/// <summary> /// FreshenThread() asks the MessagePuller to refresh the specified thread as soon /// as possible, ignoring its normal algorithm priorities. /// </summary> /// <remarks> /// Since this operation may require multiple REST service calls and is subject to /// Yammer rate limits, it may still take a while to process. The returned /// FreshenThreadRequest object can be used to track the progress. Only one /// FreshenThread() request can be active at a time; any previous requests are /// canceled by a new call to FreshenThread(). Note that no processing will occur /// unless MessagePuller.Enabled=true and MessagePuller.Process() is being called /// at regular intervals. /// </remarks> public FreshenThreadRequest FreshenThread(YamsterThread thread) { FreshenThreadRequest interruptedRequest = freshenThreadRequest; freshenThreadRequest = new FreshenThreadRequest(thread, this); if (interruptedRequest != null) { interruptedRequest.SetError(new Exception("The operation was interrupted by a more recent request")); } return(freshenThreadRequest); }
void OnError(Exception exception) { Debug.WriteLine("ERROR: " + exception.Message); if (freshenThreadRequest != null) { freshenThreadRequest.SetError(exception); freshenThreadRequest = null; } if (Error != null) { Error(this, new MessagePullerErrorEventArgs(exception)); } }
async Task ProcessGappedThreadAsync(DbSyncingThread syncingThread) { this.appContext.RequireForegroundThread(); Debug.WriteLine("MessagePuller: Fetching ThreaId={0} to close gap {1}..{2}", syncingThread.ThreadId, syncingThread.StopMessageId, syncingThread.LastPulledMessageId == null ? "newest" : syncingThread.LastPulledMessageId.ToString()); if (CallingService != null) { CallingService(this, new MessagePullerCallingServiceEventArgs(syncingThread.FeedId, syncingThread.ThreadId)); } DateTime queryUtc = DateTime.UtcNow; JsonMessageEnvelope envelope; try { // Perform a REST query like this: // https://www.yammer.com/example.com/api/v1/messages/in_thread/123.json?older_than=123 envelope = await yamsterApi.GetMessagesInThreadAsync(syncingThread.ThreadId, olderThan : syncingThread.LastPulledMessageId); this.appContext.RequireForegroundThread(); } catch (RateLimitExceededException ex) { yamsterApi.NotifyRateLimitExceeded(); OnError(ex); return; } catch (WebException ex) { HttpWebResponse response = ex.Response as HttpWebResponse; if (response != null && response.StatusCode == HttpStatusCode.NotFound) { using (var transaction = yamsterArchiveDb.BeginTransaction()) { // A 404 error indicates that the thread does not exist, i.e. it was deleted // from Yammer after we started syncing it. We need to skip it, otherwise // we'll get stuck in a loop retrying this request. yamsterCoreDb.SyncingThreads.DeleteRecords("WHERE ThreadId = " + syncingThread.ThreadId); // NOTE: We should also delete the partially synced messages from YamsterCoreDb, // however this is problematic for YamsterCache, which currently isn't able // to flush items (or even to flush everything in a way that wouldn't // break certain views). That's only worth implementing if we wanted to // support deletion in general (either for syncing Yammer deletions, or maybe // for a user command to clean up the Yamster database), but these scenarios are // not currently a priority. Nobody has asked about it. transaction.Commit(); } // This is rare, so for now just report it to the user as an error. OnError(new Exception("Failed to sync thread #" + syncingThread.ThreadId + " because it appears to have been deleted from Yammer. (404 error)")); return; } else { // For all other exception types, keep retrying until successful throw; } } using (var transaction = yamsterArchiveDb.BeginTransaction()) { WriteMetaPropertiesToDb(envelope); WriteReferencesToDb(envelope.References, queryUtc); bool deleteSyncingThread = false; if (envelope.Messages.Length == 0) { // Normally we expect v1/messages/in_thread to return at least one message. // There are two cases where that is not true: // 1. If someone deleted the messages from the thread *after* // v1/messages/in_group reported it to Yamster. In this case, we // can skip this thread (and ideally remove the deleted messages). // 2. On rare occasions (maybe once in every 10,000 requests?) the // Yammer service can return this result for a real thread. // The raw JSON is indistinguishable from case #2, except that the // messages reappear when the same request is retried. // // #2 is actually more common than #1, so we bother handling it ++syncingThread.RetryCount; if (syncingThread.RetryCount <= 3) { // Check for case #2 yamsterApi.BackOff(); OnError(new YamsterEmptyResultException(syncingThread.FeedId, syncingThread.ThreadId, syncingThread.LastPulledMessageId, syncingThread.RetryCount)); } else { // Assume case #1 this.yamsterCoreDb.SyncingThreads.DeleteRecordUsingPrimaryKey(syncingThread); OnError(new YamsterEmptyResultException(syncingThread.FeedId, syncingThread.ThreadId, syncingThread.LastPulledMessageId, -1)); } } else { // Update the gapped thread syncingThread.LastPulledMessageId = envelope.Messages.Min(x => x.Id); foreach (var message in envelope.Messages) { WriteMessageToDb(message, queryUtc); } // Did we close the gap? Normally this happens when we reach the message // that we wanted to stop at. However, there is an edge case where that // message has been deleted, in which case we also need to stop if we // reach the start of the thread (i.e. envelope.Messages comes back empty). if (syncingThread.LastPulledMessageId <= syncingThread.StopMessageId) { // Yes, remove this gapped thread deleteSyncingThread = true; // Does this complete a FreshenThread() request? if (freshenThreadRequest != null) { if (freshenThreadRequest.Thread.ThreadId == syncingThread.ThreadId) { freshenThreadRequest.SetState(FreshenThreadState.Completed); } else { // This should only be possible if an event handler issues a new request // while processing an existing one freshenThreadRequest.SetError(new Exception("State machine processed wrong thread")); } freshenThreadRequest = null; } } } if (deleteSyncingThread) { this.yamsterCoreDb.SyncingThreads.DeleteRecordUsingPrimaryKey(syncingThread); } else { // Save the changes to syncingThread this.yamsterCoreDb.SyncingThreads.InsertRecord(syncingThread, SQLiteConflictResolution.Replace); } transaction.Commit(); if (UpdatedDatabase != null) { UpdatedDatabase(this, EventArgs.Empty); } } }
async Task ProcessAsync() { this.appContext.RequireForegroundThread(); if (!this.enabled) { return; } DateTime nowUtc = DateTime.UtcNow; // Don't exceed the Yammer throttling limit. For a FreshenThread() request, // we increase the priority. if (!yamsterApi.IsSafeToRequest(increasedPriority: freshenThreadRequest != null)) { return; } // Start by assuming we're not up to date, unless proven otherwise UpToDate = false; // 1. Is there a request to freshen a specific thread? if (freshenThreadRequest != null) { var freshenedThread = yamsterCoreDb.SyncingThreads .Query("WHERE ThreadId = " + freshenThreadRequest.Thread.ThreadId) .FirstOrDefault(); if (freshenThreadRequest.State == FreshenThreadState.Queued) { freshenThreadRequest.SetState(FreshenThreadState.Processing); // Is there already an existing gap for this thread? if (freshenedThread != null) { // Yes, simply reopen it freshenedThread.LastPulledMessageId = null; } else { // No, so create a new one freshenedThread = new DbSyncingThread(); freshenedThread.FeedId = freshenThreadRequest.Thread.GroupId; freshenedThread.ThreadId = freshenThreadRequest.Thread.ThreadId; // NOTE: The thread is presumed to be contiguous at this point. long latestMessageInDb = yamsterArchiveDb.Mapper.QueryScalar <long>( "SELECT MAX(Id) FROM " + this.yamsterArchiveDb.ArchiveMessages.TableName + " WHERE ThreadId = " + freshenThreadRequest.Thread.ThreadId.ToString()); freshenedThread.StopMessageId = latestMessageInDb; freshenedThread.LastPulledMessageId = null; yamsterCoreDb.SyncingThreads.InsertRecord(freshenedThread); } } if (freshenThreadRequest.State != FreshenThreadState.Processing) { // This should be impossible freshenThreadRequest.SetError(new Exception("State machine error")); freshenThreadRequest = null; return; } await ProcessGappedThreadAsync(freshenedThread); this.appContext.RequireForegroundThread(); return; } // 2. Are there any syncing threads? We must finish them before processing more spans var syncingThread = yamsterCoreDb.SyncingThreads .QueryAll() .FirstOrDefault(); if (syncingThread != null) { // Start at the top of the list await ProcessGappedThreadAsync(syncingThread); this.appContext.RequireForegroundThread(); return; } // Get the list of subscribed feeds List <DbGroupState> groupsToSync = yamsterCoreDb.GroupStates .Query("WHERE ShouldSync").ToList(); bool forceCheckNew = false; List <JsonSyncingFeed> syncingFeeds = groupsToSync .Select( groupState => { var syncingFeed = yamsterCoreDb.GetJsonSyncingFeed(groupState.GroupId) ?? new JsonSyncingFeed(); syncingFeed.GroupState = groupState; syncingFeed.FeedId = groupState.GroupId; return(syncingFeed); } ).ToList(); if (yamsterCoreDb.Properties.SyncInbox) { // The Inbox is not a real group, so it doesn't have a DbGroupState record. var inboxSyncingFeed = yamsterCoreDb.GetJsonSyncingFeed(YamsterGroup.InboxFeedId) ?? new JsonSyncingFeed(); inboxSyncingFeed.GroupState = null; inboxSyncingFeed.FeedId = YamsterGroup.InboxFeedId; syncingFeeds.Insert(0, inboxSyncingFeed); } JsonSyncingFeed chosenSyncingFeed = null; // 3. Should we interrupt work on the history and instead check for new messages? if (Algorithm == MessagePullerAlgorithm.OptimizeReading) { TimeSpan longCheckNewDuration = TimeSpan.FromMinutes(7); chosenSyncingFeed = syncingFeeds // Choose a feed that wasn't synced recently, but only if it has already // done some work on its history .Where(x => x.SpanCyclesSinceCheckNew >= 2 && (nowUtc - x.LastCheckNewUtc) > longCheckNewDuration) // Pick the feed that was synced least recently .OrderBy(x => x.LastCheckNewUtc) .FirstOrDefault(); if (chosenSyncingFeed != null) { forceCheckNew = true; } } // 4. Are there any incomplete histories? If so, choose the feed who // made the least progress syncing so far if (chosenSyncingFeed == null) { // There are two kinds of feeds that need work: // 1. If it has gaps in the spans // 2. If we did not reach the beginning of the stream yet var nextHistoricalFeed = syncingFeeds .Where(x => x.HasSpanGaps || !x.ReachedEmptyResult) .OrderByDescending(x => x.GetNextOlderThanTime()) .FirstOrDefault(); if (nextHistoricalFeed != null) { var time = nextHistoricalFeed.GetNextOlderThanTime(); if (time != DateTime.MaxValue) // don't show this degenerate value in the UI { HistoryProgress = time; } if (HistoryLimitDays > 0) { // If HistoryLimitDays is enabled, then don't pull threads that are // older than the historyLimit DateTime historyLimit = DateTime.Now.Date.Subtract(TimeSpan.FromDays(HistoryLimitDays)); if (nextHistoricalFeed.GetNextOlderThanTime() >= historyLimit) { chosenSyncingFeed = nextHistoricalFeed; } } else { chosenSyncingFeed = nextHistoricalFeed; } } } // 5. If all the histories are complete, then check for new messages at periodic intervals if (chosenSyncingFeed == null) { TimeSpan shortCheckNewDuration = TimeSpan.FromMinutes(3); chosenSyncingFeed = syncingFeeds // Don't sync more often than shortCheckNewDuration .Where(x => (nowUtc - x.LastCheckNewUtc) > shortCheckNewDuration) // Pick the feed that was synced least recently .OrderBy(x => x.LastCheckNewUtc) .FirstOrDefault(); if (chosenSyncingFeed != null) { forceCheckNew = true; } } UpToDate = chosenSyncingFeed == null; if (!UpToDate) { await ProcessSpanAsync(chosenSyncingFeed, forceCheckNew); this.appContext.RequireForegroundThread(); } else { Debug.WriteLine("Up to date."); } }