Exemplo n.º 1
0
        void ReadCsvFiles()
        {
            using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Networks.csv")))
            {
                int col_Id  = csvReader.GetColumnIndex("id");
                int col_Url = csvReader.GetColumnIndex("url");

                while (csvReader.ReadNextLine())
                {
                    csvReader.WrapExceptions(() =>
                    {
                        networkId  = long.Parse(csvReader[col_Id]);
                        networkUrl = csvReader[col_Url];
                    });
                }
            }

            using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Groups.csv")))
            {
                int col_Id          = csvReader.GetColumnIndex("id");
                int col_Name        = csvReader.GetColumnIndex("name");
                int col_Description = csvReader.GetColumnIndex("description");
                int col_Private     = csvReader.GetColumnIndex("private");
                int col_Deleted     = csvReader.GetColumnIndex("deleted");

                while (csvReader.ReadNextLine())
                {
                    DbGroup group     = new DbGroup();
                    bool    isDeleted = false;
                    csvReader.WrapExceptions(() =>
                    {
                        group.GroupId          = long.Parse(csvReader[col_Id]);
                        group.GroupName        = csvReader[col_Name];
                        group.GroupDescription = csvReader[col_Description];

                        // TODO: How to recognize DbGroupPrivacy.Restricted?
                        bool isPrivate = bool.Parse(csvReader[col_Private]);
                        group.Privacy  = isPrivate ? DbGroupPrivacy.Private : DbGroupPrivacy.Public;

                        group.WebUrl = networkUrl + "/#/threads/inGroup?type=in_group&feedId=" + group.GroupId;

                        // TODO: How to obtain MugshotUrl?

                        isDeleted = bool.Parse(csvReader[col_Deleted]);
                    });

                    groupsById.Add(group.GroupId, group);

                    if (isDeleted)
                    {
                        deletedGroups.Add(group.GroupId);
                    }
                }
            }

            using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Users.csv")))
            {
                int col_Id       = csvReader.GetColumnIndex("id");
                int col_Name     = csvReader.GetColumnIndex("name");
                int col_JobTitle = csvReader.GetColumnIndex("job_title");
                int col_Email    = csvReader.GetColumnIndex("email");
                int col_State    = csvReader.GetColumnIndex("state");

                while (csvReader.ReadNextLine())
                {
                    DbUser user      = new DbUser();
                    bool   isDeleted = false;
                    csvReader.WrapExceptions(() =>
                    {
                        user.UserId = long.Parse(csvReader[col_Id]);
                        // TODO: Is there a way to obtain the correct alias?
                        user.Alias    = csvReader[col_Email].Replace('@', '_');
                        user.Email    = csvReader[col_Email];
                        user.FullName = csvReader[col_Name];
                        user.JobTitle = csvReader[col_JobTitle];

                        // TODO: We need the alias to calculate user.WebUrl

                        // TODO: user.MugshotUrl

                        isDeleted = csvReader[col_State].Trim().ToUpper() == "SOFT_DELETE";
                    });

                    usersById.Add(user.UserId, user);

                    if (isDeleted)
                    {
                        deletedUsers.Add(user.UserId);
                    }
                }
            }

            using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Messages.csv")))
            {
                int col_Id             = csvReader.GetColumnIndex("id");
                int col_GroupId        = csvReader.GetColumnIndex("group_id");
                int col_ThreadId       = csvReader.GetColumnIndex("thread_id");
                int col_ConversationId = csvReader.GetColumnIndex("conversation_id");

                int col_InPrivateGroup        = csvReader.GetColumnIndex("in_private_group");
                int col_InPrivateConversation = csvReader.GetColumnIndex("in_private_conversation");
                int col_Participants          = csvReader.GetColumnIndex("participants");

                int col_CreatedAt   = csvReader.GetColumnIndex("created_at");
                int col_SenderId    = csvReader.GetColumnIndex("sender_id");
                int col_RepliedToId = csvReader.GetColumnIndex("replied_to_id");
                int col_Body        = csvReader.GetColumnIndex("body");
                int col_DeletedAt   = csvReader.GetColumnIndex("deleted_at");

                while (csvReader.ReadNextLine())
                {
                    DbMessage message   = new DbMessage();
                    bool      isDeleted = false;
                    csvReader.WrapExceptions(() =>
                    {
                        message.MessageId = long.Parse(csvReader[col_Id]);
                        message.GroupId   = ParseLongWithDefault(csvReader[col_GroupId], YamsterGroup.AllCompanyGroupId);
                        message.ThreadId  = long.Parse(csvReader[col_ThreadId]);

                        bool inPrivateGroup        = bool.Parse(csvReader[col_InPrivateGroup]);
                        bool inPrivateConversation = bool.Parse(csvReader[col_InPrivateConversation]);

                        if (inPrivateConversation)
                        {
                            // This is apparently broken
                            // long conversationId = ParseLongWithDefault(csvReader[col_ConversationId], 0);
                            long conversationId = message.ThreadId;

                            DbConversation conversation;
                            if (!conversationsById.TryGetValue(conversationId, out conversation))
                            {
                                conversation = new DbConversation();
                                conversation.ConversationId = conversationId;
                                ParseParticipants(conversation.ParticipantUserIds, csvReader[col_Participants]);
                                conversationsById.Add(conversationId, conversation);
                            }

                            message.ConversationId = conversationId;
                            message.GroupId        = YamsterGroup.ConversationsGroupId;
                        }

                        message.CreatedDate        = DateTime.Parse(csvReader[col_CreatedAt]);
                        message.SenderUserId       = long.Parse(csvReader[col_SenderId]);
                        message.MessageIdRepliedTo = ParseLongWithDefault(csvReader[col_RepliedToId], 0);

                        // TODO: Likes?

                        // TODO: Parse message.NotifiedUserIds from "CC" line
                        message.Body   = csvReader[col_Body];
                        message.WebUrl = networkUrl + "/messages/" + message.MessageId;

                        // TODO: Attachments?

                        message.MessageType = DbMessageType.Update;

                        isDeleted = !string.IsNullOrWhiteSpace(csvReader[col_DeletedAt]);
                    });

                    messagesById.Add(message.MessageId, message);

                    if (isDeleted)
                    {
                        deletedMessages.Add(message.MessageId);
                    }
                    else
                    {
                        if (message.ConversationId != 0)
                        {
                            notDeletedConversations.Add(message.ConversationId);
                        }
                    }
                }
            }
        }
Exemplo n.º 2
0
        void UpdateMessage(DbArchiveMessageRecord archiveMessage)
        {
            JsonMessage message = SQLiteJsonConverter.LoadFromJson <JsonMessage>(archiveMessage.Json);

            DbMessage coreMessage = new DbMessage();

            coreMessage.LastFetchedUtc = archiveMessage.LastFetchedUtc;
            coreMessage.MessageId      = message.Id;

            coreMessage.GroupId            = archiveMessage.GroupId;
            coreMessage.ThreadId           = message.ThreadId;
            coreMessage.ConversationId     = message.ConversationId;
            coreMessage.CreatedDate        = message.Created;
            coreMessage.SenderUserId       = message.SenderId;
            coreMessage.MessageIdRepliedTo = message.RepliedToId ?? 0;

            coreMessage.LikingUserIds.AssignFrom(message.Likes.Users.Select(x => x.UserId));
            foreach (var likingUser in message.Likes.Users)
            {
                // We don't get a proper UserReference for liking users, but we do get
                // some basic information.  Write this to the Users table *only* if there
                // is not already some real data there.
                DbUser coreUser = new DbUser();
                coreUser.LastFetchedUtc = archiveMessage.LastFetchedUtc;
                coreUser.UserId         = likingUser.UserId;
                coreUser.FullName       = likingUser.FullName ?? "";
                coreUser.JobTitle       = "";
                coreUser.WebUrl         = ""; // we could infer this from likingUser.Alias
                coreUser.MugshotUrl     = "";

                // Ignore = only write if there isn't already an existing record
                Users.InsertRecord(coreUser, SQLiteConflictResolution.Ignore);
            }

            coreMessage.LikesCount = message.Likes.Count;
            coreMessage.NotifiedUserIds.AssignFrom(message.NotifiedUserIds ?? new long[0]);
            coreMessage.Body   = message.Body.Plain ?? "";
            coreMessage.WebUrl = message.Permalink ?? "";

            var firstImageAttachment = message.Attachments.Where(x => x.AttachmentType == "image").FirstOrDefault();

            if (firstImageAttachment != null)
            {
                coreMessage.AttachmentFilename          = firstImageAttachment.Name;
                coreMessage.AttachmentWebUrl            = firstImageAttachment.WebUrl;
                coreMessage.AttachmentScaledUrlTemplate = firstImageAttachment.ScaledUrlTemplate;
                coreMessage.AttachmentWidth             = firstImageAttachment.Width ?? 0;
                coreMessage.AttachmentHeight            = firstImageAttachment.Height ?? 0;
            }


            if (!Enum.TryParse(message.MessageType, true, out coreMessage.MessageType))
            {
                coreMessage.MessageType = DbMessageType.Unknown;
            }

            Messages.InsertRecord(coreMessage, SQLiteConflictResolution.Replace);

            DbMessageState messageState = new DbMessageState()
            {
                MessageId = archiveMessage.Id
            };

            MessageStates.InsertRecord(messageState, SQLiteConflictResolution.Ignore);

            // Ensure that every message has a corresponding DbThreadState for its thread
            DbThreadState threadState = new DbThreadState()
            {
                ThreadId = coreMessage.ThreadId
            };

            ThreadStates.InsertRecord(threadState, SQLiteConflictResolution.Ignore);
        }