Ejemplo n.º 1
0
        async Task ProcessSpanAsync(JsonSyncingFeed syncingFeed, bool forceCheckNew)
        {
            this.appContext.RequireForegroundThread();

            long     feedId   = syncingFeed.FeedId;
            DateTime queryUtc = DateTime.UtcNow;

            bool checkNew = forceCheckNew ||
                            syncingFeed.Spans.Count < 1 ||
                            (syncingFeed.Spans.Count == 1 && syncingFeed.ReachedEmptyResult);

            long?olderThan = null;

            if (checkNew)
            {
                syncingFeed.SpanCyclesSinceCheckNew = 0;
                syncingFeed.LastCheckNewUtc         = queryUtc;
            }
            else
            {
                ++syncingFeed.SpanCyclesSinceCheckNew;

                // Work backwards from the most recent gap
                var lastSpan = syncingFeed.Spans.Last();
                olderThan = lastSpan.StartMessageId;
            }
            syncingFeed.LastUpdateUtc = queryUtc;

            if (CallingService != null)
            {
                CallingService(this, new MessagePullerCallingServiceEventArgs(feedId, null));
            }

            JsonMessageEnvelope envelope;

            try
            {
                // Perform a REST query like this:
                // https://www.yammer.com/example.com/api/v1/messages/in_group/3.json?threaded=extended&older_than=129
                envelope = await yamsterApi.GetMessagesInFeedAsync(feedId, olderThan);

                this.appContext.RequireForegroundThread();
            }
            catch (RateLimitExceededException ex)
            {
                yamsterApi.NotifyRateLimitExceeded();
                OnError(ex);
                return;
            }
            catch (WebException ex)
            {
                var response = ex.Response as HttpWebResponse;
                if (response != null)
                {
                    if (response.StatusCode == HttpStatusCode.NotFound && syncingFeed.GroupState != null)
                    {
                        // The group does not exist; disable further syncing for it and report
                        // a more specific error
                        DbGroupState groupState = syncingFeed.GroupState;
                        groupState.ShouldSync = false;
                        yamsterCoreDb.GroupStates.InsertRecord(groupState, SQLiteConflictResolution.Replace);
                        OnError(new YamsterFailedSyncException(feedId, ex));
                        return;
                    }
                }
                // A general error has occurred
                yamsterApi.BackOff();
                OnError(ex);
                return;
            }

            var newSpan = new JsonMessagePullerSpan();

            newSpan.StartMessageId = long.MaxValue;
            newSpan.StartTimeUtc   = DateTime.MaxValue;
            newSpan.EndMessageId   = long.MinValue;

            using (var transaction = yamsterArchiveDb.BeginTransaction())
            {
                WriteReferencesToDb(envelope.References, queryUtc);

                foreach (var threadStarter in envelope.Messages)
                {
                    // Clean up any corrupted data
                    if (yamsterCoreDb.SyncingThreads
                        .DeleteRecords("WHERE [ThreadId] = " + threadStarter.ThreadId) > 0)
                    {
                        Debug.WriteLine("MessagePuller: WARNING: Removed unexpected sync state for thread ID={0}",
                                        threadStarter.ThreadId);
                    }

                    JsonMessage[] extendedMessages;

                    // Note that ThreadedExtended is indexed by thread ID, not message ID
                    if (!envelope.ThreadedExtended.TryGetValue(threadStarter.ThreadId, out extendedMessages))
                    {
                        extendedMessages = new JsonMessage[0];
                    }

                    // Update the span bounds
                    long     latestMessageIdInThread;
                    DateTime latestMessageTimeInThread;
                    if (extendedMessages.Length > 0)
                    {
                        latestMessageIdInThread   = extendedMessages.Max(x => x.Id);
                        latestMessageTimeInThread = extendedMessages.Max(x => x.Created);
                    }
                    else
                    {
                        latestMessageIdInThread   = threadStarter.Id;
                        latestMessageTimeInThread = threadStarter.Created;
                    }
                    newSpan.StartMessageId = Math.Min(newSpan.StartMessageId, latestMessageIdInThread);
                    if (latestMessageTimeInThread < newSpan.StartTimeUtc)
                    {
                        newSpan.StartTimeUtc = latestMessageTimeInThread;
                    }
                    newSpan.EndMessageId = Math.Max(newSpan.EndMessageId, latestMessageIdInThread);

                    WriteMessageToDb(threadStarter, queryUtc);

                    // NOTE: The thread is presumed to be contiguous at this point.
                    // This is guaranteed to return at least threadStarter.Id written above
                    long latestMessageInDb = yamsterArchiveDb.Mapper.QueryScalar <long>(
                        "SELECT MAX(Id) FROM [" + this.yamsterArchiveDb.ArchiveMessages.TableName
                        + "] WHERE ThreadId = " + threadStarter.ThreadId.ToString());

                    // There are two scenarios where we can prove that there is no gap,
                    // i.e. that we already have all the messages for the thread.
                    // NOTE:  Originally we assumed there was no gap if extendedMessages.Length<2,
                    // but a counterexample was found.
                    bool gapped = true;

// For debugging -- skip pulling most messages to accumulate threads faster
#if false
                    if ((threadStarter.ThreadId & 31) != 0)
                    {
                        gapped = false;
                    }
#endif
                    // Scenario 1: Does the envelope contain the complete thread?
                    var threadReference = envelope.References
                                          .OfType <ThreadReferenceJson>()
                                          .Where(x => x.Id == threadStarter.ThreadId)
                                          .FirstOrDefault();

                    if (threadReference != null)
                    {
                        // (+1 for threadStarter)
                        if (extendedMessages.Length + 1 == threadReference.Stats.MessagesCount)
                        {
// This criteria should work, but I found cases where Yammer's counter is incorrect
#if false
                            // The envelope contains the complete thread
                            gapped = false;
#endif
                        }
                    }
                    else
                    {
                        // This should never happen, but if it does it's okay if we wrongly assume
                        // the thread is gapped
                        Debug.Assert(false);
                    }

                    // Scenario 2: Do the envelope messages overlap with the database's version of the thread?
                    if (gapped && extendedMessages.Length > 0)
                    {
                        long extendedStartId = extendedMessages.Min(x => x.Id);
                        if (latestMessageInDb >= extendedStartId)
                        {
                            // Yes, the messages overlap
                            gapped = false;
                        }
                    }

                    if (gapped)
                    {
                        var gappedThread = new DbSyncingThread();
                        gappedThread.FeedId        = feedId;
                        gappedThread.ThreadId      = threadStarter.ThreadId;
                        gappedThread.StopMessageId = latestMessageInDb;

                        // NOTE: In a static database, it would be most efficient to call
                        // WriteMessageToDb() for the extendedMessages that we already received
                        // and pick up with LastPulledMessageId=extendedStartId.
                        // However, if we assume people are actively posting in Yammer, it's
                        // better to begin processing a gapped thread by querying for the absolute
                        // latest stuff, since a fair amount of time may have elapsed by the
                        // time we get around to doing the query.
                        gappedThread.LastPulledMessageId = null;

                        gappedThread.RetryCount = 0;

                        // A key violation should be impossible here since if there was a conflicting
                        // record, we deleted it above.
                        yamsterCoreDb.SyncingThreads.InsertRecord(gappedThread);
                    }
                    else
                    {
                        foreach (var extendedMessage in extendedMessages)
                        {
                            WriteMessageToDb(extendedMessage, queryUtc);
                        }
                    }
                }

                if (envelope.Messages.Length > 0)
                {
                    if (olderThan.HasValue)
                    {
                        // If the Yammer result includes messages newer than what we asked
                        // for with olderThan, this is most likely a bug.
                        // NOTE: Skip this check for the Inbox feed, which seems to have minor
                        // overlap about 50% of the time.  This issue wasn't observed in the Yammer web page,
                        // but that may be due to the additional filtering there for seen/unarchived.
                        if (feedId != YamsterGroup.InboxFeedId)
                        {
                            Debug.Assert(newSpan.EndMessageId < olderThan);
                        }

                        // If olderThan was specified, then the span actually covers anything
                        // up to that point in the history
                        newSpan.EndMessageId = olderThan.Value - 1;
                    }

                    // Now create a span corresponding to the range of messages we just received.
                    syncingFeed.AddSpan(newSpan);
                }
                else
                {
                    syncingFeed.ReachedEmptyResult = true;
                }

                yamsterCoreDb.UpdateJsonSyncingFeed(feedId, syncingFeed);

                if (feedId == YamsterGroup.InboxFeedId)
                {
                    // For each GroupId in the messages that we wrote, make sure ShowInYamster = 1
                    string showInYamsterSql = string.Format(
                        @"UPDATE [GroupStates] SET [ShowInYamster] = 1"
                        + " WHERE [GroupId] in ({0}) AND [ShowInYamster] <> 1",
                        string.Join(
                            ", ",
                            envelope.Messages.Where(x => x.GroupId != null).Select(x => x.GroupId).Distinct()
                            )
                        );
                    yamsterCoreDb.Mapper.ExecuteNonQuery(showInYamsterSql);

                    // For each ThreadId in the messages that we wrote, mark it as appearing in the inbox
                    string seenInInboxSql = string.Format(
                        @"UPDATE [ThreadStates] SET [SeenInInboxFeed] = 1"
                        + " WHERE [ThreadId] in ({0}) AND [SeenInInboxFeed] <> 1",
                        string.Join(
                            ", ",
                            envelope.Messages.Select(x => x.ThreadId).Distinct()
                            )
                        );
                    yamsterCoreDb.Mapper.ExecuteNonQuery(seenInInboxSql);
                }

                transaction.Commit();

                if (UpdatedDatabase != null)
                {
                    UpdatedDatabase(this, EventArgs.Empty);
                }
            }
        }
Ejemplo n.º 2
0
        void UpdateMessage(DbArchiveMessageRecord archiveMessage)
        {
            JsonMessage message = SQLiteJsonConverter.LoadFromJson <JsonMessage>(archiveMessage.Json);

            DbMessage coreMessage = new DbMessage();

            coreMessage.LastFetchedUtc = archiveMessage.LastFetchedUtc;
            coreMessage.MessageId      = message.Id;

            coreMessage.GroupId            = archiveMessage.GroupId;
            coreMessage.ThreadId           = message.ThreadId;
            coreMessage.ConversationId     = message.ConversationId;
            coreMessage.CreatedDate        = message.Created;
            coreMessage.SenderUserId       = message.SenderId;
            coreMessage.MessageIdRepliedTo = message.RepliedToId ?? 0;

            coreMessage.LikingUserIds.AssignFrom(message.Likes.Users.Select(x => x.UserId));
            foreach (var likingUser in message.Likes.Users)
            {
                // We don't get a proper UserReference for liking users, but we do get
                // some basic information.  Write this to the Users table *only* if there
                // is not already some real data there.
                DbUser coreUser = new DbUser();
                coreUser.LastFetchedUtc = archiveMessage.LastFetchedUtc;
                coreUser.UserId         = likingUser.UserId;
                coreUser.FullName       = likingUser.FullName ?? "";
                coreUser.JobTitle       = "";
                coreUser.WebUrl         = ""; // we could infer this from likingUser.Alias
                coreUser.MugshotUrl     = "";

                // Ignore = only write if there isn't already an existing record
                Users.InsertRecord(coreUser, SQLiteConflictResolution.Ignore);
            }

            coreMessage.LikesCount = message.Likes.Count;
            coreMessage.NotifiedUserIds.AssignFrom(message.NotifiedUserIds ?? new long[0]);
            coreMessage.Body   = message.Body.Plain ?? "";
            coreMessage.WebUrl = message.Permalink ?? "";

            var firstImageAttachment = message.Attachments.Where(x => x.AttachmentType == "image").FirstOrDefault();

            if (firstImageAttachment != null)
            {
                coreMessage.AttachmentFilename          = firstImageAttachment.Name;
                coreMessage.AttachmentWebUrl            = firstImageAttachment.WebUrl;
                coreMessage.AttachmentScaledUrlTemplate = firstImageAttachment.ScaledUrlTemplate;
                coreMessage.AttachmentWidth             = firstImageAttachment.Width ?? 0;
                coreMessage.AttachmentHeight            = firstImageAttachment.Height ?? 0;
            }


            if (!Enum.TryParse(message.MessageType, true, out coreMessage.MessageType))
            {
                coreMessage.MessageType = DbMessageType.Unknown;
            }

            Messages.InsertRecord(coreMessage, SQLiteConflictResolution.Replace);

            DbMessageState messageState = new DbMessageState()
            {
                MessageId = archiveMessage.Id
            };

            MessageStates.InsertRecord(messageState, SQLiteConflictResolution.Ignore);

            // Ensure that every message has a corresponding DbThreadState for its thread
            DbThreadState threadState = new DbThreadState()
            {
                ThreadId = coreMessage.ThreadId
            };

            ThreadStates.InsertRecord(threadState, SQLiteConflictResolution.Ignore);
        }