void ReadCsvFiles()
{
using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Networks.csv")))
{
int col_Id = csvReader.GetColumnIndex("id");
int col_Url = csvReader.GetColumnIndex("url");
while (csvReader.ReadNextLine())
{
csvReader.WrapExceptions(() =>
{
networkId = long.Parse(csvReader[col_Id]);
networkUrl = csvReader[col_Url];
});
}
}
using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Groups.csv")))
{
int col_Id = csvReader.GetColumnIndex("id");
int col_Name = csvReader.GetColumnIndex("name");
int col_Description = csvReader.GetColumnIndex("description");
int col_Private = csvReader.GetColumnIndex("private");
int col_Deleted = csvReader.GetColumnIndex("deleted");
while (csvReader.ReadNextLine())
{
DbGroup group = new DbGroup();
bool isDeleted = false;
csvReader.WrapExceptions(() =>
{
group.GroupId = long.Parse(csvReader[col_Id]);
group.GroupName = csvReader[col_Name];
group.GroupDescription = csvReader[col_Description];
// TODO: How to recognize DbGroupPrivacy.Restricted?
bool isPrivate = bool.Parse(csvReader[col_Private]);
group.Privacy = isPrivate ? DbGroupPrivacy.Private : DbGroupPrivacy.Public;
group.WebUrl = networkUrl + "/#/threads/inGroup?type=in_group&feedId=" + group.GroupId;
// TODO: How to obtain MugshotUrl?
isDeleted = bool.Parse(csvReader[col_Deleted]);
});
groupsById.Add(group.GroupId, group);
if (isDeleted)
{
deletedGroups.Add(group.GroupId);
}
}
}
using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Users.csv")))
{
int col_Id = csvReader.GetColumnIndex("id");
int col_Name = csvReader.GetColumnIndex("name");
int col_JobTitle = csvReader.GetColumnIndex("job_title");
int col_Email = csvReader.GetColumnIndex("email");
int col_State = csvReader.GetColumnIndex("state");
while (csvReader.ReadNextLine())
{
DbUser user = new DbUser();
bool isDeleted = false;
csvReader.WrapExceptions(() =>
{
user.UserId = long.Parse(csvReader[col_Id]);
// TODO: Is there a way to obtain the correct alias?
user.Alias = csvReader[col_Email].Replace('@', '_');
user.Email = csvReader[col_Email];
user.FullName = csvReader[col_Name];
user.JobTitle = csvReader[col_JobTitle];
// TODO: We need the alias to calculate user.WebUrl
// TODO: user.MugshotUrl
isDeleted = csvReader[col_State].Trim().ToUpper() == "SOFT_DELETE";
});
usersById.Add(user.UserId, user);
if (isDeleted)
{
deletedUsers.Add(user.UserId);
}
}
}
using (var csvReader = new CsvReader(Path.Combine(this.FolderPath, "Messages.csv")))
{
int col_Id = csvReader.GetColumnIndex("id");
int col_GroupId = csvReader.GetColumnIndex("group_id");
int col_ThreadId = csvReader.GetColumnIndex("thread_id");
int col_ConversationId = csvReader.GetColumnIndex("conversation_id");
int col_InPrivateGroup = csvReader.GetColumnIndex("in_private_group");
int col_InPrivateConversation = csvReader.GetColumnIndex("in_private_conversation");
int col_Participants = csvReader.GetColumnIndex("participants");
int col_CreatedAt = csvReader.GetColumnIndex("created_at");
int col_SenderId = csvReader.GetColumnIndex("sender_id");
int col_RepliedToId = csvReader.GetColumnIndex("replied_to_id");
int col_Body = csvReader.GetColumnIndex("body");
int col_DeletedAt = csvReader.GetColumnIndex("deleted_at");
while (csvReader.ReadNextLine())
{
DbMessage message = new DbMessage();
bool isDeleted = false;
csvReader.WrapExceptions(() =>
{
message.MessageId = long.Parse(csvReader[col_Id]);
message.GroupId = ParseLongWithDefault(csvReader[col_GroupId], YamsterGroup.AllCompanyGroupId);
message.ThreadId = long.Parse(csvReader[col_ThreadId]);
bool inPrivateGroup = bool.Parse(csvReader[col_InPrivateGroup]);
bool inPrivateConversation = bool.Parse(csvReader[col_InPrivateConversation]);
if (inPrivateConversation)
{
// This is apparently broken
// long conversationId = ParseLongWithDefault(csvReader[col_ConversationId], 0);
long conversationId = message.ThreadId;
DbConversation conversation;
if (!conversationsById.TryGetValue(conversationId, out conversation))
{
conversation = new DbConversation();
conversation.ConversationId = conversationId;
ParseParticipants(conversation.ParticipantUserIds, csvReader[col_Participants]);
conversationsById.Add(conversationId, conversation);
}
message.ConversationId = conversationId;
message.GroupId = YamsterGroup.ConversationsGroupId;
}
message.CreatedDate = DateTime.Parse(csvReader[col_CreatedAt]);
message.SenderUserId = long.Parse(csvReader[col_SenderId]);
message.MessageIdRepliedTo = ParseLongWithDefault(csvReader[col_RepliedToId], 0);
// TODO: Likes?
// TODO: Parse message.NotifiedUserIds from "CC" line
message.Body = csvReader[col_Body];
message.WebUrl = networkUrl + "/messages/" + message.MessageId;
// TODO: Attachments?
message.MessageType = DbMessageType.Update;
isDeleted = !string.IsNullOrWhiteSpace(csvReader[col_DeletedAt]);
});
messagesById.Add(message.MessageId, message);
if (isDeleted)
{
deletedMessages.Add(message.MessageId);
}
else
{
if (message.ConversationId != 0)
{
notDeletedConversations.Add(message.ConversationId);
}
}
}
}
}
void UpdateMessage(DbArchiveMessageRecord archiveMessage)
{
JsonMessage message = SQLiteJsonConverter.LoadFromJson <JsonMessage>(archiveMessage.Json);
DbMessage coreMessage = new DbMessage();
coreMessage.LastFetchedUtc = archiveMessage.LastFetchedUtc;
coreMessage.MessageId = message.Id;
coreMessage.GroupId = archiveMessage.GroupId;
coreMessage.ThreadId = message.ThreadId;
coreMessage.ConversationId = message.ConversationId;
coreMessage.CreatedDate = message.Created;
coreMessage.SenderUserId = message.SenderId;
coreMessage.MessageIdRepliedTo = message.RepliedToId ?? 0;
coreMessage.LikingUserIds.AssignFrom(message.Likes.Users.Select(x => x.UserId));
foreach (var likingUser in message.Likes.Users)
{
// We don't get a proper UserReference for liking users, but we do get
// some basic information. Write this to the Users table *only* if there
// is not already some real data there.
DbUser coreUser = new DbUser();
coreUser.LastFetchedUtc = archiveMessage.LastFetchedUtc;
coreUser.UserId = likingUser.UserId;
coreUser.FullName = likingUser.FullName ?? "";
coreUser.JobTitle = "";
coreUser.WebUrl = ""; // we could infer this from likingUser.Alias
coreUser.MugshotUrl = "";
// Ignore = only write if there isn't already an existing record
Users.InsertRecord(coreUser, SQLiteConflictResolution.Ignore);
}
coreMessage.LikesCount = message.Likes.Count;
coreMessage.NotifiedUserIds.AssignFrom(message.NotifiedUserIds ?? new long[0]);
coreMessage.Body = message.Body.Plain ?? "";
coreMessage.WebUrl = message.Permalink ?? "";
var firstImageAttachment = message.Attachments.Where(x => x.AttachmentType == "image").FirstOrDefault();
if (firstImageAttachment != null)
{
coreMessage.AttachmentFilename = firstImageAttachment.Name;
coreMessage.AttachmentWebUrl = firstImageAttachment.WebUrl;
coreMessage.AttachmentScaledUrlTemplate = firstImageAttachment.ScaledUrlTemplate;
coreMessage.AttachmentWidth = firstImageAttachment.Width ?? 0;
coreMessage.AttachmentHeight = firstImageAttachment.Height ?? 0;
}
if (!Enum.TryParse(message.MessageType, true, out coreMessage.MessageType))
{
coreMessage.MessageType = DbMessageType.Unknown;
}
Messages.InsertRecord(coreMessage, SQLiteConflictResolution.Replace);
DbMessageState messageState = new DbMessageState()
{
MessageId = archiveMessage.Id
};
MessageStates.InsertRecord(messageState, SQLiteConflictResolution.Ignore);
// Ensure that every message has a corresponding DbThreadState for its thread
DbThreadState threadState = new DbThreadState()
{
ThreadId = coreMessage.ThreadId
};
ThreadStates.InsertRecord(threadState, SQLiteConflictResolution.Ignore);
}
