/// <summary>
/// FreshenThread() asks the MessagePuller to refresh the specified thread as soon
/// as possible, ignoring its normal algorithm priorities.
/// </summary>
/// <remarks>
/// Since this operation may require multiple REST service calls and is subject to
/// Yammer rate limits, it may still take a while to process. The returned
/// FreshenThreadRequest object can be used to track the progress. Only one
/// FreshenThread() request can be active at a time; any previous requests are
/// canceled by a new call to FreshenThread(). Note that no processing will occur
/// unless MessagePuller.Enabled=true and MessagePuller.Process() is being called
/// at regular intervals.
/// </remarks>
public FreshenThreadRequest FreshenThread(YamsterThread thread)
{
FreshenThreadRequest interruptedRequest = freshenThreadRequest;
freshenThreadRequest = new FreshenThreadRequest(thread, this);
if (interruptedRequest != null)
{
interruptedRequest.SetError(new Exception("The operation was interrupted by a more recent request"));
}
return(freshenThreadRequest);
}
void OnError(Exception exception)
{
Debug.WriteLine("ERROR: " + exception.Message);
if (freshenThreadRequest != null)
{
freshenThreadRequest.SetError(exception);
freshenThreadRequest = null;
}
if (Error != null)
{
Error(this, new MessagePullerErrorEventArgs(exception));
}
}
async Task ProcessGappedThreadAsync(DbSyncingThread syncingThread)
{
this.appContext.RequireForegroundThread();
Debug.WriteLine("MessagePuller: Fetching ThreaId={0} to close gap {1}..{2}",
syncingThread.ThreadId, syncingThread.StopMessageId,
syncingThread.LastPulledMessageId == null ? "newest" : syncingThread.LastPulledMessageId.ToString());
if (CallingService != null)
{
CallingService(this, new MessagePullerCallingServiceEventArgs(syncingThread.FeedId, syncingThread.ThreadId));
}
DateTime queryUtc = DateTime.UtcNow;
JsonMessageEnvelope envelope;
try
{
// Perform a REST query like this:
// https://www.yammer.com/example.com/api/v1/messages/in_thread/123.json?older_than=123
envelope = await yamsterApi.GetMessagesInThreadAsync(syncingThread.ThreadId,
olderThan : syncingThread.LastPulledMessageId);
this.appContext.RequireForegroundThread();
}
catch (RateLimitExceededException ex)
{
yamsterApi.NotifyRateLimitExceeded();
OnError(ex);
return;
}
catch (WebException ex)
{
HttpWebResponse response = ex.Response as HttpWebResponse;
if (response != null && response.StatusCode == HttpStatusCode.NotFound)
{
using (var transaction = yamsterArchiveDb.BeginTransaction())
{
// A 404 error indicates that the thread does not exist, i.e. it was deleted
// from Yammer after we started syncing it. We need to skip it, otherwise
// we'll get stuck in a loop retrying this request.
yamsterCoreDb.SyncingThreads.DeleteRecords("WHERE ThreadId = " + syncingThread.ThreadId);
// NOTE: We should also delete the partially synced messages from YamsterCoreDb,
// however this is problematic for YamsterCache, which currently isn't able
// to flush items (or even to flush everything in a way that wouldn't
// break certain views). That's only worth implementing if we wanted to
// support deletion in general (either for syncing Yammer deletions, or maybe
// for a user command to clean up the Yamster database), but these scenarios are
// not currently a priority. Nobody has asked about it.
transaction.Commit();
}
// This is rare, so for now just report it to the user as an error.
OnError(new Exception("Failed to sync thread #" + syncingThread.ThreadId
+ " because it appears to have been deleted from Yammer. (404 error)"));
return;
}
else
{
// For all other exception types, keep retrying until successful
throw;
}
}
using (var transaction = yamsterArchiveDb.BeginTransaction())
{
WriteMetaPropertiesToDb(envelope);
WriteReferencesToDb(envelope.References, queryUtc);
bool deleteSyncingThread = false;
if (envelope.Messages.Length == 0)
{
// Normally we expect v1/messages/in_thread to return at least one message.
// There are two cases where that is not true:
// 1. If someone deleted the messages from the thread *after*
// v1/messages/in_group reported it to Yamster. In this case, we
// can skip this thread (and ideally remove the deleted messages).
// 2. On rare occasions (maybe once in every 10,000 requests?) the
// Yammer service can return this result for a real thread.
// The raw JSON is indistinguishable from case #2, except that the
// messages reappear when the same request is retried.
//
// #2 is actually more common than #1, so we bother handling it
++syncingThread.RetryCount;
if (syncingThread.RetryCount <= 3)
{
// Check for case #2
yamsterApi.BackOff();
OnError(new YamsterEmptyResultException(syncingThread.FeedId, syncingThread.ThreadId,
syncingThread.LastPulledMessageId, syncingThread.RetryCount));
}
else
{
// Assume case #1
this.yamsterCoreDb.SyncingThreads.DeleteRecordUsingPrimaryKey(syncingThread);
OnError(new YamsterEmptyResultException(syncingThread.FeedId, syncingThread.ThreadId,
syncingThread.LastPulledMessageId, -1));
}
}
else
{
// Update the gapped thread
syncingThread.LastPulledMessageId = envelope.Messages.Min(x => x.Id);
foreach (var message in envelope.Messages)
{
WriteMessageToDb(message, queryUtc);
}
// Did we close the gap? Normally this happens when we reach the message
// that we wanted to stop at. However, there is an edge case where that
// message has been deleted, in which case we also need to stop if we
// reach the start of the thread (i.e. envelope.Messages comes back empty).
if (syncingThread.LastPulledMessageId <= syncingThread.StopMessageId)
{
// Yes, remove this gapped thread
deleteSyncingThread = true;
// Does this complete a FreshenThread() request?
if (freshenThreadRequest != null)
{
if (freshenThreadRequest.Thread.ThreadId == syncingThread.ThreadId)
{
freshenThreadRequest.SetState(FreshenThreadState.Completed);
}
else
{
// This should only be possible if an event handler issues a new request
// while processing an existing one
freshenThreadRequest.SetError(new Exception("State machine processed wrong thread"));
}
freshenThreadRequest = null;
}
}
}
if (deleteSyncingThread)
{
this.yamsterCoreDb.SyncingThreads.DeleteRecordUsingPrimaryKey(syncingThread);
}
else
{
// Save the changes to syncingThread
this.yamsterCoreDb.SyncingThreads.InsertRecord(syncingThread, SQLiteConflictResolution.Replace);
}
transaction.Commit();
if (UpdatedDatabase != null)
{
UpdatedDatabase(this, EventArgs.Empty);
}
}
}
async Task ProcessAsync()
{
this.appContext.RequireForegroundThread();
if (!this.enabled)
{
return;
}
DateTime nowUtc = DateTime.UtcNow;
// Don't exceed the Yammer throttling limit. For a FreshenThread() request,
// we increase the priority.
if (!yamsterApi.IsSafeToRequest(increasedPriority: freshenThreadRequest != null))
{
return;
}
// Start by assuming we're not up to date, unless proven otherwise
UpToDate = false;
// 1. Is there a request to freshen a specific thread?
if (freshenThreadRequest != null)
{
var freshenedThread = yamsterCoreDb.SyncingThreads
.Query("WHERE ThreadId = " + freshenThreadRequest.Thread.ThreadId)
.FirstOrDefault();
if (freshenThreadRequest.State == FreshenThreadState.Queued)
{
freshenThreadRequest.SetState(FreshenThreadState.Processing);
// Is there already an existing gap for this thread?
if (freshenedThread != null)
{
// Yes, simply reopen it
freshenedThread.LastPulledMessageId = null;
}
else
{
// No, so create a new one
freshenedThread = new DbSyncingThread();
freshenedThread.FeedId = freshenThreadRequest.Thread.GroupId;
freshenedThread.ThreadId = freshenThreadRequest.Thread.ThreadId;
// NOTE: The thread is presumed to be contiguous at this point.
long latestMessageInDb = yamsterArchiveDb.Mapper.QueryScalar <long>(
"SELECT MAX(Id) FROM " + this.yamsterArchiveDb.ArchiveMessages.TableName
+ " WHERE ThreadId = " + freshenThreadRequest.Thread.ThreadId.ToString());
freshenedThread.StopMessageId = latestMessageInDb;
freshenedThread.LastPulledMessageId = null;
yamsterCoreDb.SyncingThreads.InsertRecord(freshenedThread);
}
}
if (freshenThreadRequest.State != FreshenThreadState.Processing)
{
// This should be impossible
freshenThreadRequest.SetError(new Exception("State machine error"));
freshenThreadRequest = null;
return;
}
await ProcessGappedThreadAsync(freshenedThread);
this.appContext.RequireForegroundThread();
return;
}
// 2. Are there any syncing threads? We must finish them before processing more spans
var syncingThread = yamsterCoreDb.SyncingThreads
.QueryAll()
.FirstOrDefault();
if (syncingThread != null)
{
// Start at the top of the list
await ProcessGappedThreadAsync(syncingThread);
this.appContext.RequireForegroundThread();
return;
}
// Get the list of subscribed feeds
List <DbGroupState> groupsToSync = yamsterCoreDb.GroupStates
.Query("WHERE ShouldSync").ToList();
bool forceCheckNew = false;
List <JsonSyncingFeed> syncingFeeds = groupsToSync
.Select(
groupState => {
var syncingFeed = yamsterCoreDb.GetJsonSyncingFeed(groupState.GroupId)
?? new JsonSyncingFeed();
syncingFeed.GroupState = groupState;
syncingFeed.FeedId = groupState.GroupId;
return(syncingFeed);
}
).ToList();
if (yamsterCoreDb.Properties.SyncInbox)
{
// The Inbox is not a real group, so it doesn't have a DbGroupState record.
var inboxSyncingFeed = yamsterCoreDb.GetJsonSyncingFeed(YamsterGroup.InboxFeedId)
?? new JsonSyncingFeed();
inboxSyncingFeed.GroupState = null;
inboxSyncingFeed.FeedId = YamsterGroup.InboxFeedId;
syncingFeeds.Insert(0, inboxSyncingFeed);
}
JsonSyncingFeed chosenSyncingFeed = null;
// 3. Should we interrupt work on the history and instead check for new messages?
if (Algorithm == MessagePullerAlgorithm.OptimizeReading)
{
TimeSpan longCheckNewDuration = TimeSpan.FromMinutes(7);
chosenSyncingFeed = syncingFeeds
// Choose a feed that wasn't synced recently, but only if it has already
// done some work on its history
.Where(x => x.SpanCyclesSinceCheckNew >= 2 &&
(nowUtc - x.LastCheckNewUtc) > longCheckNewDuration)
// Pick the feed that was synced least recently
.OrderBy(x => x.LastCheckNewUtc)
.FirstOrDefault();
if (chosenSyncingFeed != null)
{
forceCheckNew = true;
}
}
// 4. Are there any incomplete histories? If so, choose the feed who
// made the least progress syncing so far
if (chosenSyncingFeed == null)
{
// There are two kinds of feeds that need work:
// 1. If it has gaps in the spans
// 2. If we did not reach the beginning of the stream yet
var nextHistoricalFeed = syncingFeeds
.Where(x => x.HasSpanGaps || !x.ReachedEmptyResult)
.OrderByDescending(x => x.GetNextOlderThanTime())
.FirstOrDefault();
if (nextHistoricalFeed != null)
{
var time = nextHistoricalFeed.GetNextOlderThanTime();
if (time != DateTime.MaxValue) // don't show this degenerate value in the UI
{
HistoryProgress = time;
}
if (HistoryLimitDays > 0)
{
// If HistoryLimitDays is enabled, then don't pull threads that are
// older than the historyLimit
DateTime historyLimit = DateTime.Now.Date.Subtract(TimeSpan.FromDays(HistoryLimitDays));
if (nextHistoricalFeed.GetNextOlderThanTime() >= historyLimit)
{
chosenSyncingFeed = nextHistoricalFeed;
}
}
else
{
chosenSyncingFeed = nextHistoricalFeed;
}
}
}
// 5. If all the histories are complete, then check for new messages at periodic intervals
if (chosenSyncingFeed == null)
{
TimeSpan shortCheckNewDuration = TimeSpan.FromMinutes(3);
chosenSyncingFeed = syncingFeeds
// Don't sync more often than shortCheckNewDuration
.Where(x => (nowUtc - x.LastCheckNewUtc) > shortCheckNewDuration)
// Pick the feed that was synced least recently
.OrderBy(x => x.LastCheckNewUtc)
.FirstOrDefault();
if (chosenSyncingFeed != null)
{
forceCheckNew = true;
}
}
UpToDate = chosenSyncingFeed == null;
if (!UpToDate)
{
await ProcessSpanAsync(chosenSyncingFeed, forceCheckNew);
this.appContext.RequireForegroundThread();
}
else
{
Debug.WriteLine("Up to date.");
}
}