Пример #1
0
        internal void RebuildData(RebuildTask changes = null)
        {
            var fullRebuild = _dataRevision == 0 || changes == null || changes.IsFullSync;
            var entityKeys  = new Dictionary <string, byte[]>();

            // teams
            keeperTeams.Clear();
            foreach (var team in Storage.Teams.GetAll())
            {
                try
                {
                    var teamKey = CryptoUtils.DecryptAesV1(team.TeamKey.Base64UrlDecode(), ClientKey);
                    var t       = new Team(team, teamKey);
                    keeperTeams.TryAdd(t.TeamUid, t);
                }
                catch (Exception e)
                {
                    Trace.TraceError(e.Message);
                }
            }

            var uids = new HashSet <string>();

            // shared folders
            {
                entityKeys.Clear();
                var sharedFoldersToLoad = new List <ISharedFolder>();
                if (!fullRebuild && (changes.SharedFolders?.Count ?? 0) * 4 > keeperSharedFolders.Count)
                {
                    fullRebuild = true;
                }

                if (fullRebuild)
                {
                    keeperSharedFolders.Clear();
                    sharedFoldersToLoad.AddRange(Storage.SharedFolders.GetAll());
                    foreach (var sfKey in Storage.SharedFolderKeys.GetAllLinks())
                    {
                        if (entityKeys.ContainsKey(sfKey.SharedFolderUid))
                        {
                            continue;
                        }

                        if (DecryptSharedFolderKey(sfKey, out var key))
                        {
                            entityKeys[sfKey.SharedFolderUid] = key;
                        }
                    }
                }
                else
                {
                    if (changes.SharedFolders != null)
                    {
                        foreach (var sharedFolderUid in changes.SharedFolders)
                        {
                            if (keeperSharedFolders.TryRemove(sharedFolderUid, out var sharedFolder))
                            {
                                changes.AddRecords(sharedFolder.RecordPermissions.Select(x => x.RecordUid));
                            }

                            var sf = Storage.SharedFolders.GetEntity(sharedFolderUid);
                            if (sf != null)
                            {
                                sharedFoldersToLoad.Add(sf);
                            }

                            foreach (var sfKey in Storage.SharedFolderKeys.GetLinksForSubject(sharedFolderUid))
                            {
                                if (!DecryptSharedFolderKey(sfKey, out var key))
                                {
                                    continue;
                                }

                                entityKeys[sfKey.SharedFolderUid] = key;
                                break;
                            }
                        }
                    }
                }

                uids.Clear();
                foreach (var sharedFolder in sharedFoldersToLoad)
                {
                    if (sharedFolder == null)
                    {
                        continue;
                    }
                    if (entityKeys.ContainsKey(sharedFolder.SharedFolderUid))
                    {
                        var sfKey = entityKeys[sharedFolder.SharedFolderUid];
                        var sfmds = Storage.RecordKeys.GetLinksForObject(sharedFolder.SharedFolderUid)
                                    .ToArray();
                        var sfus = Storage.SharedFolderPermissions
                                   .GetLinksForSubject(sharedFolder.SharedFolderUid).ToArray();
                        var sf = sharedFolder.Load(sfmds, sfus, sfKey);
                        keeperSharedFolders.TryAdd(sharedFolder.SharedFolderUid, sf);
                    }
                    else
                    {
                        uids.Add(sharedFolder.SharedFolderUid);
                    }
                }

                if (uids.Count > 0)
                {
                    Storage.SharedFolders.DeleteUids(uids);
                    Storage.RecordKeys.DeleteLinksForObjects(uids);
                    Storage.SharedFolderKeys.DeleteLinksForSubjects(uids);
                    Storage.SharedFolderPermissions.DeleteLinksForSubjects(uids);
                }
            }

            // records
            {
                entityKeys.Clear();
                var lostKeys      = new List <IUidLink>();
                var recordsToLoad = new Dictionary <string, IPasswordRecord>();
                if (!fullRebuild && (changes.Records?.Count ?? 0) * 5 > keeperRecords.Count)
                {
                    fullRebuild = true;
                }
                if (fullRebuild)
                {
                    keeperRecords.Clear();
                    foreach (var record in Storage.Records.GetAll())
                    {
                        recordsToLoad[record.RecordUid] = record;
                    }

                    foreach (var rmd in Storage.RecordKeys.GetAllLinks())
                    {
                        if (entityKeys.ContainsKey(rmd.RecordUid))
                        {
                            continue;
                        }
                        if (!recordsToLoad.ContainsKey(rmd.RecordUid))
                        {
                            lostKeys.Add(rmd);
                            continue;
                        }

                        if (DecryptRecordKey(rmd, out var rKey))
                        {
                            entityKeys[rmd.RecordUid] = rKey;
                        }
                        else
                        {
                            lostKeys.Add(rmd);
                        }
                    }
                }
                else
                {
                    if (changes.Records != null)
                    {
                        foreach (var recordUid in changes.Records)
                        {
                            var r = Storage.Records.GetEntity(recordUid);
                            if (r == null)
                            {
                                continue;
                            }
                            recordsToLoad[r.RecordUid] = r;

                            keeperRecords.TryRemove(recordUid, out _);

                            foreach (var rmd in Storage.RecordKeys.GetLinksForSubject(r.RecordUid))
                            {
                                if (DecryptRecordKey(rmd, out var rKey))
                                {
                                    entityKeys[rmd.RecordUid] = rKey;
                                    break;
                                }

                                lostKeys.Add(rmd);
                            }
                        }
                    }
                }

                uids.Clear();
                foreach (var r in recordsToLoad.Values)
                {
                    if (entityKeys.ContainsKey(r.RecordUid))
                    {
                        var rKey   = entityKeys[r.RecordUid];
                        var record = r.Load(rKey);
                        keeperRecords.TryAdd(r.RecordUid, record);
                    }
                    else
                    {
                        uids.Add(r.RecordUid);
                    }
                }

                if (lostKeys.Any())
                {
                    Storage.RecordKeys.DeleteLinks(lostKeys);
                }

                if (uids.Any())
                {
                    Storage.RecordKeys.DeleteLinksForSubjects(uids);
                    Storage.Records.DeleteUids(uids);
                }
            }

            BuildFolders();
            _dataRevision = Storage.Revision;
        }
Пример #2
0
        /// <summary>
        /// Incrementally downloads vault data.
        /// </summary>
        /// <param name="vault">Vault connected to Keeper.</param>
        /// <returns></returns>
        internal static async Task RunSyncDown(this VaultOnline vault)
        {
            var storage   = vault.Storage;
            var context   = vault.Auth.AuthContext;
            var clientKey = vault.ClientKey;

            var command = new SyncDownCommand
            {
                revision   = storage.Revision,
                include    = new[] { "sfheaders", "sfrecords", "sfusers", "teams", "folders" },
                deviceName = vault.Auth.Endpoint.DeviceName,
                deviceId   = vault.Auth.Endpoint.DeviceName
            };

            var rs = await vault.Auth.ExecuteAuthCommand <SyncDownCommand, SyncDownResponse>(command);

            Debug.WriteLine("Sync Down: Enter");
            var isFullSync = rs.fullSync;

            if (isFullSync)
            {
                storage.Clear();
            }

            var result = new RebuildTask(isFullSync);

            if (rs.removedRecords != null)
            {
                storage.RecordKeys.DeleteLinks(
                    rs.removedRecords
                    .Select(recordUid => UidLink.Create(recordUid, storage.PersonalScopeUid)));

                var folderRecords = new List <IUidLink>();
                foreach (var recordUid in rs.removedRecords)
                {
                    result.AddRecord(recordUid);
                    var links = storage.FolderRecords.GetLinksForObject(recordUid).ToArray();
                    foreach (var link in links)
                    {
                        var folderUid = link.FolderUid;
                        if (string.IsNullOrEmpty(folderUid) && folderUid == storage.PersonalScopeUid)
                        {
                            folderRecords.Add(link);
                        }
                        else
                        {
                            var folder = storage.Folders.GetEntity(folderUid);
                            if (folder?.FolderType == "user_folder")
                            {
                                folderRecords.Add(link);
                            }
                        }
                    }
                }
                storage.FolderRecords.DeleteLinks(folderRecords);
            }

            if (rs.removedTeams != null)
            {
                foreach (var teamUid in rs.removedTeams)
                {
                    var sfLinks = storage.SharedFolderKeys.GetLinksForObject(teamUid).ToArray();
                    foreach (var sfLink in sfLinks)
                    {
                        result.AddSharedFolder(sfLink.SharedFolderUid);
                    }

                    storage.SharedFolderKeys.DeleteLinks(sfLinks);
                }
                storage.Teams.DeleteUids(rs.removedTeams);
            }

            if (rs.removedSharedFolders != null)
            {
                foreach (var sharedFolderUid in rs.removedSharedFolders)
                {
                    result.AddSharedFolder(sharedFolderUid);
                    var links = storage.RecordKeys.GetLinksForObject(sharedFolderUid).ToArray();
                    foreach (var recLink in links)
                    {
                        result.AddRecord(recLink.RecordUid);
                    }
                }
                storage.SharedFolderKeys.DeleteLinks(
                    rs.removedSharedFolders
                    .Select(x => UidLink.Create(x, storage.PersonalScopeUid)));
            }

            if (rs.userFoldersRemoved != null)
            {
                storage.FolderRecords.DeleteLinksForSubjects(rs.userFoldersRemoved.Select(x => x.folderUid));
                storage.Folders.DeleteUids(rs.userFoldersRemoved.Select(x => x.folderUid));
            }

            if (rs.sharedFolderFolderRemoved != null)
            {
                var folderUids = rs.sharedFolderFolderRemoved
                                 .Select(x => x.FolderUid ?? x.SharedFolderUid).ToArray();

                storage.FolderRecords.DeleteLinksForSubjects(folderUids);
                storage.Folders.DeleteUids(folderUids);
            }

            if (rs.userFolderSharedFoldersRemoved != null)
            {
                storage.FolderRecords.DeleteLinksForSubjects(rs.userFolderSharedFoldersRemoved
                                                             .Select(x => x.SharedFolderUid));
                storage.Folders.DeleteUids(rs.userFolderSharedFoldersRemoved
                                           .Select(x => x.SharedFolderUid));
            }

            if (rs.userFoldersRemovedRecords != null)
            {
                var links = rs.userFoldersRemovedRecords
                            .Select(x => UidLink.Create(x.folderUid ?? storage.PersonalScopeUid, x.RecordUid))
                            .ToArray();
                storage.FolderRecords.DeleteLinks(links);
            }

            if (rs.sharedFolderFolderRecordsRemoved != null)
            {
                var links = rs.sharedFolderFolderRecordsRemoved
                            .Select(x => UidLink.Create(x.folderUid ?? x.sharedFolderUid, x.recordUid))
                            .ToArray();

                storage.FolderRecords.DeleteLinks(links);
            }

            if (rs.sharedFolders != null)
            {
                // full sync shared folders
                var fullSyncSharedFolders = rs.sharedFolders
                                            .Where(x => x.fullSync == true)
                                            .Select(x => x.SharedFolderUid)
                                            .ToArray();

                storage.RecordKeys.DeleteLinksForObjects(fullSyncSharedFolders);
                storage.SharedFolderKeys.DeleteLinksForSubjects(fullSyncSharedFolders);
                storage.SharedFolderPermissions.DeleteLinksForSubjects(fullSyncSharedFolders);

                // records
                var affectedLinks = rs.sharedFolders
                                    .Where(x => !x.fullSync.HasValue || !x.fullSync.Value)
                                    .Where(x => x.recordsRemoved != null)
                                    .SelectMany(x => x.recordsRemoved,
                                                (x, recordUid) => UidLink.Create(recordUid, x.SharedFolderUid))
                                    .Cast <IUidLink>()
                                    .ToArray();

                if (affectedLinks.Any())
                {
                    storage.RecordKeys.DeleteLinks(affectedLinks);
                    foreach (var x in affectedLinks)
                    {
                        result.AddRecord(x.SubjectUid);
                    }
                }

                // teams
                var affectedTeams = rs.sharedFolders
                                    .Where(x => !x.fullSync.HasValue || !x.fullSync.Value)
                                    .Where(x => x.teamsRemoved != null)
                                    .SelectMany(x => x.teamsRemoved,
                                                (x, teamUid) => UidLink.Create(x.SharedFolderUid, teamUid))
                                    .Cast <IUidLink>()
                                    .ToArray();
                if (affectedTeams.Any())
                {
                    storage.SharedFolderKeys.DeleteLinks(affectedTeams);
                }

                //users
                var affectedUsers = rs.sharedFolders
                                    .Where(x => !x.fullSync.HasValue || !x.fullSync.Value)
                                    .Where(x => x.usersRemoved != null)
                                    .SelectMany(x => x.usersRemoved,
                                                (x, username) => UidLink.Create(x.SharedFolderUid, username))
                                    .Cast <IUidLink>()
                                    .ToArray();


                if (affectedTeams.Any() || affectedLinks.Any())
                {
                    storage.SharedFolderPermissions.DeleteLinks(affectedTeams.Concat(affectedUsers));
                }
            }

            if (rs.nonSharedData != null)
            {
                storage.NonSharedData.PutEntities(rs.nonSharedData
                                                  .Where(x => !string.IsNullOrEmpty(x.data))
                                                  .Select(x =>
                {
                    try
                    {
                        var data = x.data.Base64UrlDecode();
                        data     = CryptoUtils.DecryptAesV1(data, context.DataKey);
                        data     = CryptoUtils.EncryptAesV1(data, clientKey);
                        x.data   = data.Base64UrlEncode();
                        return(x);
                    }
                    catch (Exception e)
                    {
                        Trace.TraceError(e.Message);
                        return(null);
                    }
                })
                                                  .Where(x => x != null));
            }

            var recordOwnership = new Dictionary <string, bool>();

            if (rs.recordMetaData != null)
            {
                foreach (var rmd in rs.recordMetaData)
                {
                    recordOwnership[rmd.RecordUid] = rmd.Owner;
                }
            }

            if (rs.records != null)
            {
                result.AddRecords(rs.records.Select(x => x.RecordUid));

                storage.Records.PutEntities(rs.records
                                            .Select(x =>
                {
                    x.AdjustUdata();
                    if (!recordOwnership.ContainsKey(x.RecordUid))
                    {
                        return(x);
                    }

                    x.Owner = recordOwnership[x.RecordUid];
                    recordOwnership.Remove(x.RecordUid);

                    return(x);
                }));
            }

            if (rs.recordMetaData != null)
            {
                result.AddRecords(rs.recordMetaData.Select(x => x.RecordUid));

                var toUpdate = rs.recordMetaData
                               .Where(x => recordOwnership.ContainsKey(x.RecordUid))
                               .Select(x =>
                {
                    var sr = storage.Records.GetEntity(x.RecordUid);
                    if (sr == null)
                    {
                        return(null);
                    }
                    if (sr.Owner == x.Owner)
                    {
                        return(null);
                    }

                    sr.Owner = x.Owner;
                    return(sr);
                })
                               .Where(x => x != null)
                               .ToArray();
                if (toUpdate.Any())
                {
                    storage.Records.PutEntities(toUpdate);
                }

                var rmds = rs.recordMetaData
                           .Select(rmd =>
                {
                    try
                    {
                        byte[] key;
                        switch (rmd.RecordKeyType)
                        {
                        case 0:
                            key = context.DataKey;
                            break;

                        case 1:
                            key = CryptoUtils.DecryptAesV1(rmd.RecordKey.Base64UrlDecode(), context.DataKey);
                            break;

                        case 2:
                            key = CryptoUtils.DecryptRsa(rmd.RecordKey.Base64UrlDecode(), context.PrivateKey);
                            break;

                        default:
                            throw new Exception(
                                $"Record metadata UID {rmd.RecordUid}: unsupported key type {rmd.RecordKeyType}");
                        }

                        if (key != null)
                        {
                            rmd.RecordKey       = CryptoUtils.EncryptAesV1(key, context.ClientKey).Base64UrlEncode();
                            rmd.RecordKeyType   = (int)KeyType.DataKey;
                            rmd.SharedFolderUid = storage.PersonalScopeUid;
                            return(rmd);
                        }
                    }
                    catch (Exception e)
                    {
                        Trace.TraceError(e.Message);
                    }

                    return(null);
                })
                           .ToArray();

                storage.RecordKeys.PutLinks(rmds);
            }

            if (rs.teams != null)
            {
                var removedSharedFolderLinks = rs.teams
                                               .Where(x => x.removedSharedFolders != null)
                                               .SelectMany(x => x.removedSharedFolders,
                                                           (team, sharedFolderUid) => UidLink.Create(sharedFolderUid, team.TeamUid))
                                               .Cast <IUidLink>()
                                               .ToArray();
                if (removedSharedFolderLinks.Any())
                {
                    result.AddSharedFolders(removedSharedFolderLinks.Select(x => x.SubjectUid));
                    storage.SharedFolderKeys.DeleteLinks(removedSharedFolderLinks);
                }

                var teams = rs.teams
                            .Select(x =>
                {
                    try
                    {
                        byte[] teamKey;
                        switch (x.KeyType)
                        {
                        case (int)KeyType.DataKey:
                            teamKey = CryptoUtils.DecryptAesV1(x.TeamKey.Base64UrlDecode(), context.DataKey);
                            break;

                        case (int)KeyType.PrivateKey:
                            teamKey = CryptoUtils.DecryptRsa(x.TeamKey.Base64UrlDecode(), context.PrivateKey);
                            break;

                        default:
                            throw new Exception($"Team UID {x.TeamUid}: unsupported key type {x.KeyType}");
                        }

                        x.TeamKey = CryptoUtils.EncryptAesV1(teamKey, clientKey).Base64UrlEncode();
                        x.KeyType = (int)KeyType.DataKey;
                        return(x);
                    }
                    catch (Exception e)
                    {
                        Trace.TraceError(e.Message);
                        return(null);
                    }
                })
                            .Where(x => x != null)
                            .ToArray();
                storage.Teams.PutEntities(teams);

                var sharedFolderKeys = rs.teams
                                       .Where(x => x.sharedFolderKeys != null)
                                       .SelectMany(x => x.sharedFolderKeys,
                                                   (team, sharedFolderKey) =>
                {
                    sharedFolderKey.TeamUid = team.TeamUid;
                    sharedFolderKey.KeyType = sharedFolderKey.KeyType == 2 ? (int)KeyType.TeamPrivateKey : (int)KeyType.TeamKey;
                    return(sharedFolderKey);
                })
                                       .ToArray();
                storage.SharedFolderKeys.PutLinks(sharedFolderKeys);
            }

            if (rs.sharedFolders != null)
            {
                result.AddSharedFolders(rs.sharedFolders.Select(x => x.SharedFolderUid));

                // shared folders
                storage.SharedFolders.PutEntities(rs.sharedFolders);

                // shared folder keys
                var sharedFolderKeys = rs.sharedFolders
                                       .Where(x => !string.IsNullOrEmpty(x.SharedFolderKey))
                                       .Select(x =>
                {
                    try
                    {
                        var sharedFolderKey = x.SharedFolderKey.Base64UrlDecode();
                        switch (x.KeyType)
                        {
                        case 1:
                            sharedFolderKey = CryptoUtils.DecryptAesV1(sharedFolderKey, context.DataKey);
                            break;

                        case 2:
                            sharedFolderKey = CryptoUtils.DecryptRsa(sharedFolderKey, context.PrivateKey);
                            break;

                        default:
                            throw new Exception(
                                $"Shared Folder UID {x.SharedFolderUid}: unsupported key type {x.KeyType}");
                        }

                        return(new SyncDownSharedFolderKey
                        {
                            SharedFolderUid = x.SharedFolderUid,
                            TeamUid = storage.PersonalScopeUid,
                            SharedFolderKey = CryptoUtils.EncryptAesV1(sharedFolderKey, clientKey)
                                              .Base64UrlEncode(),
                            KeyType = (int)KeyType.DataKey
                        });
                    }
                    catch (Exception e)
                    {
                        Trace.TraceError(e.Message);
                        return(null);
                    }
                })
                                       .ToArray();
                if (sharedFolderKeys.Any())
                {
                    storage.SharedFolderKeys.PutLinks(sharedFolderKeys);
                }

                result.AddRecords(rs.sharedFolders
                                  .Where(x => x.records != null)
                                  .SelectMany(x => x.records,
                                              (sf, r) => r.RecordUid));

                // Records
                var sharedFolderRecords = rs.sharedFolders
                                          .Where(x => x.records != null)
                                          .SelectMany(x => x.records,
                                                      (sf, sfr) => new SyncDownRecordMetaData
                {
                    SharedFolderUid = sf.SharedFolderUid,
                    RecordUid       = sfr.RecordUid,
                    RecordKey       = sfr.RecordKey,
                    RecordKeyType   = (int)KeyType.SharedFolderKey,
                    CanEdit         = sfr.CanEdit,
                    CanShare        = sfr.CanShare
                })
                                          .ToArray();
                if (sharedFolderRecords.Any())
                {
                    storage.RecordKeys.PutLinks(sharedFolderRecords);
                }

                // Teams
                var teams = rs.sharedFolders
                            .Where(x => x.teams != null)
                            .SelectMany(x => x.teams,
                                        (sf, sft) =>
                {
                    sft.SharedFolderUid = sf.SharedFolderUid;
                    return(sft);
                })
                            .Cast <ISharedFolderPermission>()
                            .ToArray();
                // Users
                var users = rs.sharedFolders
                            .Where(x => x.users != null)
                            .SelectMany(x => x.users,
                                        (sf, sfu) =>
                {
                    sfu.SharedFolderUid = sf.SharedFolderUid;
                    return(sfu);
                })
                            .Cast <ISharedFolderPermission>()
                            .ToArray();

                if (teams.Any() || users.Any())
                {
                    storage.SharedFolderPermissions.PutLinks(teams.Concat(users));
                }
            }

            if (rs.userFolders != null)
            {
                var userFolders = rs.userFolders
                                  .Select(uf =>
                {
                    try
                    {
                        var folderKey = uf.FolderKey.Base64UrlDecode();
                        switch (uf.keyType)
                        {
                        case (int)KeyType.DataKey:
                            folderKey = CryptoUtils.DecryptAesV1(folderKey, context.DataKey);
                            break;

                        case (int)KeyType.PrivateKey:
                            folderKey = CryptoUtils.DecryptRsa(folderKey, context.PrivateKey);
                            break;

                        default:
                            throw new Exception($"User Folder UID {uf.FolderUid}: unsupported key type {uf.keyType}");
                        }

                        uf.FolderKey = CryptoUtils.EncryptAesV1(folderKey, clientKey).Base64UrlEncode();
                        uf.keyType   = (int)KeyType.DataKey;
                        return(uf);
                    }
                    catch (Exception e)
                    {
                        Trace.TraceError(e.Message);
                        return(null);
                    }
                })
                                  .ToArray();

                storage.Folders.PutEntities(userFolders);
            }

            if (rs.sharedFolderFolders != null)
            {
                storage.Folders.PutEntities(rs.sharedFolderFolders);
            }

            if (rs.userFolderSharedFolders != null)
            {
                storage.Folders.PutEntities(rs.userFolderSharedFolders);
            }

            if (rs.userFolderRecords != null)
            {
                storage.FolderRecords.PutLinks(rs.userFolderRecords
                                               .Select(ufr =>
                {
                    ufr.FolderUid = string.IsNullOrEmpty(ufr.FolderUid) ? storage.PersonalScopeUid : ufr.FolderUid;
                    return(ufr);
                }));
            }

            if (rs.sharedFolderFolderRecords != null)
            {
                storage.FolderRecords.PutLinks(rs.sharedFolderFolderRecords);
            }

            storage.Revision = rs.revision;
            Debug.WriteLine("Sync Down: Leave");

            Debug.WriteLine("Rebuild Data: Enter");
            vault.RebuildData(result);
            Debug.WriteLine("Rebuild Data: Leave");
        }