internal void RebuildData(RebuildTask changes = null) { var fullRebuild = _dataRevision == 0 || changes == null || changes.IsFullSync; var entityKeys = new Dictionary <string, byte[]>(); // teams keeperTeams.Clear(); foreach (var team in Storage.Teams.GetAll()) { try { var teamKey = CryptoUtils.DecryptAesV1(team.TeamKey.Base64UrlDecode(), ClientKey); var t = new Team(team, teamKey); keeperTeams.TryAdd(t.TeamUid, t); } catch (Exception e) { Trace.TraceError(e.Message); } } var uids = new HashSet <string>(); // shared folders { entityKeys.Clear(); var sharedFoldersToLoad = new List <ISharedFolder>(); if (!fullRebuild && (changes.SharedFolders?.Count ?? 0) * 4 > keeperSharedFolders.Count) { fullRebuild = true; } if (fullRebuild) { keeperSharedFolders.Clear(); sharedFoldersToLoad.AddRange(Storage.SharedFolders.GetAll()); foreach (var sfKey in Storage.SharedFolderKeys.GetAllLinks()) { if (entityKeys.ContainsKey(sfKey.SharedFolderUid)) { continue; } if (DecryptSharedFolderKey(sfKey, out var key)) { entityKeys[sfKey.SharedFolderUid] = key; } } } else { if (changes.SharedFolders != null) { foreach (var sharedFolderUid in changes.SharedFolders) { if (keeperSharedFolders.TryRemove(sharedFolderUid, out var sharedFolder)) { changes.AddRecords(sharedFolder.RecordPermissions.Select(x => x.RecordUid)); } var sf = Storage.SharedFolders.GetEntity(sharedFolderUid); if (sf != null) { sharedFoldersToLoad.Add(sf); } foreach (var sfKey in Storage.SharedFolderKeys.GetLinksForSubject(sharedFolderUid)) { if (!DecryptSharedFolderKey(sfKey, out var key)) { continue; } entityKeys[sfKey.SharedFolderUid] = key; break; } } } } uids.Clear(); foreach (var sharedFolder in sharedFoldersToLoad) { if (sharedFolder == null) { continue; } if (entityKeys.ContainsKey(sharedFolder.SharedFolderUid)) { var sfKey = entityKeys[sharedFolder.SharedFolderUid]; var sfmds = Storage.RecordKeys.GetLinksForObject(sharedFolder.SharedFolderUid) .ToArray(); var sfus = Storage.SharedFolderPermissions .GetLinksForSubject(sharedFolder.SharedFolderUid).ToArray(); var sf = sharedFolder.Load(sfmds, sfus, sfKey); keeperSharedFolders.TryAdd(sharedFolder.SharedFolderUid, sf); } else { uids.Add(sharedFolder.SharedFolderUid); } } if (uids.Count > 0) { Storage.SharedFolders.DeleteUids(uids); Storage.RecordKeys.DeleteLinksForObjects(uids); Storage.SharedFolderKeys.DeleteLinksForSubjects(uids); Storage.SharedFolderPermissions.DeleteLinksForSubjects(uids); } } // records { entityKeys.Clear(); var lostKeys = new List <IUidLink>(); var recordsToLoad = new Dictionary <string, IPasswordRecord>(); if (!fullRebuild && (changes.Records?.Count ?? 0) * 5 > keeperRecords.Count) { fullRebuild = true; } if (fullRebuild) { keeperRecords.Clear(); foreach (var record in Storage.Records.GetAll()) { recordsToLoad[record.RecordUid] = record; } foreach (var rmd in Storage.RecordKeys.GetAllLinks()) { if (entityKeys.ContainsKey(rmd.RecordUid)) { continue; } if (!recordsToLoad.ContainsKey(rmd.RecordUid)) { lostKeys.Add(rmd); continue; } if (DecryptRecordKey(rmd, out var rKey)) { entityKeys[rmd.RecordUid] = rKey; } else { lostKeys.Add(rmd); } } } else { if (changes.Records != null) { foreach (var recordUid in changes.Records) { var r = Storage.Records.GetEntity(recordUid); if (r == null) { continue; } recordsToLoad[r.RecordUid] = r; keeperRecords.TryRemove(recordUid, out _); foreach (var rmd in Storage.RecordKeys.GetLinksForSubject(r.RecordUid)) { if (DecryptRecordKey(rmd, out var rKey)) { entityKeys[rmd.RecordUid] = rKey; break; } lostKeys.Add(rmd); } } } } uids.Clear(); foreach (var r in recordsToLoad.Values) { if (entityKeys.ContainsKey(r.RecordUid)) { var rKey = entityKeys[r.RecordUid]; var record = r.Load(rKey); keeperRecords.TryAdd(r.RecordUid, record); } else { uids.Add(r.RecordUid); } } if (lostKeys.Any()) { Storage.RecordKeys.DeleteLinks(lostKeys); } if (uids.Any()) { Storage.RecordKeys.DeleteLinksForSubjects(uids); Storage.Records.DeleteUids(uids); } } BuildFolders(); _dataRevision = Storage.Revision; }
/// <summary> /// Incrementally downloads vault data. /// </summary> /// <param name="vault">Vault connected to Keeper.</param> /// <returns></returns> internal static async Task RunSyncDown(this VaultOnline vault) { var storage = vault.Storage; var context = vault.Auth.AuthContext; var clientKey = vault.ClientKey; var command = new SyncDownCommand { revision = storage.Revision, include = new[] { "sfheaders", "sfrecords", "sfusers", "teams", "folders" }, deviceName = vault.Auth.Endpoint.DeviceName, deviceId = vault.Auth.Endpoint.DeviceName }; var rs = await vault.Auth.ExecuteAuthCommand <SyncDownCommand, SyncDownResponse>(command); Debug.WriteLine("Sync Down: Enter"); var isFullSync = rs.fullSync; if (isFullSync) { storage.Clear(); } var result = new RebuildTask(isFullSync); if (rs.removedRecords != null) { storage.RecordKeys.DeleteLinks( rs.removedRecords .Select(recordUid => UidLink.Create(recordUid, storage.PersonalScopeUid))); var folderRecords = new List <IUidLink>(); foreach (var recordUid in rs.removedRecords) { result.AddRecord(recordUid); var links = storage.FolderRecords.GetLinksForObject(recordUid).ToArray(); foreach (var link in links) { var folderUid = link.FolderUid; if (string.IsNullOrEmpty(folderUid) && folderUid == storage.PersonalScopeUid) { folderRecords.Add(link); } else { var folder = storage.Folders.GetEntity(folderUid); if (folder?.FolderType == "user_folder") { folderRecords.Add(link); } } } } storage.FolderRecords.DeleteLinks(folderRecords); } if (rs.removedTeams != null) { foreach (var teamUid in rs.removedTeams) { var sfLinks = storage.SharedFolderKeys.GetLinksForObject(teamUid).ToArray(); foreach (var sfLink in sfLinks) { result.AddSharedFolder(sfLink.SharedFolderUid); } storage.SharedFolderKeys.DeleteLinks(sfLinks); } storage.Teams.DeleteUids(rs.removedTeams); } if (rs.removedSharedFolders != null) { foreach (var sharedFolderUid in rs.removedSharedFolders) { result.AddSharedFolder(sharedFolderUid); var links = storage.RecordKeys.GetLinksForObject(sharedFolderUid).ToArray(); foreach (var recLink in links) { result.AddRecord(recLink.RecordUid); } } storage.SharedFolderKeys.DeleteLinks( rs.removedSharedFolders .Select(x => UidLink.Create(x, storage.PersonalScopeUid))); } if (rs.userFoldersRemoved != null) { storage.FolderRecords.DeleteLinksForSubjects(rs.userFoldersRemoved.Select(x => x.folderUid)); storage.Folders.DeleteUids(rs.userFoldersRemoved.Select(x => x.folderUid)); } if (rs.sharedFolderFolderRemoved != null) { var folderUids = rs.sharedFolderFolderRemoved .Select(x => x.FolderUid ?? x.SharedFolderUid).ToArray(); storage.FolderRecords.DeleteLinksForSubjects(folderUids); storage.Folders.DeleteUids(folderUids); } if (rs.userFolderSharedFoldersRemoved != null) { storage.FolderRecords.DeleteLinksForSubjects(rs.userFolderSharedFoldersRemoved .Select(x => x.SharedFolderUid)); storage.Folders.DeleteUids(rs.userFolderSharedFoldersRemoved .Select(x => x.SharedFolderUid)); } if (rs.userFoldersRemovedRecords != null) { var links = rs.userFoldersRemovedRecords .Select(x => UidLink.Create(x.folderUid ?? storage.PersonalScopeUid, x.RecordUid)) .ToArray(); storage.FolderRecords.DeleteLinks(links); } if (rs.sharedFolderFolderRecordsRemoved != null) { var links = rs.sharedFolderFolderRecordsRemoved .Select(x => UidLink.Create(x.folderUid ?? x.sharedFolderUid, x.recordUid)) .ToArray(); storage.FolderRecords.DeleteLinks(links); } if (rs.sharedFolders != null) { // full sync shared folders var fullSyncSharedFolders = rs.sharedFolders .Where(x => x.fullSync == true) .Select(x => x.SharedFolderUid) .ToArray(); storage.RecordKeys.DeleteLinksForObjects(fullSyncSharedFolders); storage.SharedFolderKeys.DeleteLinksForSubjects(fullSyncSharedFolders); storage.SharedFolderPermissions.DeleteLinksForSubjects(fullSyncSharedFolders); // records var affectedLinks = rs.sharedFolders .Where(x => !x.fullSync.HasValue || !x.fullSync.Value) .Where(x => x.recordsRemoved != null) .SelectMany(x => x.recordsRemoved, (x, recordUid) => UidLink.Create(recordUid, x.SharedFolderUid)) .Cast <IUidLink>() .ToArray(); if (affectedLinks.Any()) { storage.RecordKeys.DeleteLinks(affectedLinks); foreach (var x in affectedLinks) { result.AddRecord(x.SubjectUid); } } // teams var affectedTeams = rs.sharedFolders .Where(x => !x.fullSync.HasValue || !x.fullSync.Value) .Where(x => x.teamsRemoved != null) .SelectMany(x => x.teamsRemoved, (x, teamUid) => UidLink.Create(x.SharedFolderUid, teamUid)) .Cast <IUidLink>() .ToArray(); if (affectedTeams.Any()) { storage.SharedFolderKeys.DeleteLinks(affectedTeams); } //users var affectedUsers = rs.sharedFolders .Where(x => !x.fullSync.HasValue || !x.fullSync.Value) .Where(x => x.usersRemoved != null) .SelectMany(x => x.usersRemoved, (x, username) => UidLink.Create(x.SharedFolderUid, username)) .Cast <IUidLink>() .ToArray(); if (affectedTeams.Any() || affectedLinks.Any()) { storage.SharedFolderPermissions.DeleteLinks(affectedTeams.Concat(affectedUsers)); } } if (rs.nonSharedData != null) { storage.NonSharedData.PutEntities(rs.nonSharedData .Where(x => !string.IsNullOrEmpty(x.data)) .Select(x => { try { var data = x.data.Base64UrlDecode(); data = CryptoUtils.DecryptAesV1(data, context.DataKey); data = CryptoUtils.EncryptAesV1(data, clientKey); x.data = data.Base64UrlEncode(); return(x); } catch (Exception e) { Trace.TraceError(e.Message); return(null); } }) .Where(x => x != null)); } var recordOwnership = new Dictionary <string, bool>(); if (rs.recordMetaData != null) { foreach (var rmd in rs.recordMetaData) { recordOwnership[rmd.RecordUid] = rmd.Owner; } } if (rs.records != null) { result.AddRecords(rs.records.Select(x => x.RecordUid)); storage.Records.PutEntities(rs.records .Select(x => { x.AdjustUdata(); if (!recordOwnership.ContainsKey(x.RecordUid)) { return(x); } x.Owner = recordOwnership[x.RecordUid]; recordOwnership.Remove(x.RecordUid); return(x); })); } if (rs.recordMetaData != null) { result.AddRecords(rs.recordMetaData.Select(x => x.RecordUid)); var toUpdate = rs.recordMetaData .Where(x => recordOwnership.ContainsKey(x.RecordUid)) .Select(x => { var sr = storage.Records.GetEntity(x.RecordUid); if (sr == null) { return(null); } if (sr.Owner == x.Owner) { return(null); } sr.Owner = x.Owner; return(sr); }) .Where(x => x != null) .ToArray(); if (toUpdate.Any()) { storage.Records.PutEntities(toUpdate); } var rmds = rs.recordMetaData .Select(rmd => { try { byte[] key; switch (rmd.RecordKeyType) { case 0: key = context.DataKey; break; case 1: key = CryptoUtils.DecryptAesV1(rmd.RecordKey.Base64UrlDecode(), context.DataKey); break; case 2: key = CryptoUtils.DecryptRsa(rmd.RecordKey.Base64UrlDecode(), context.PrivateKey); break; default: throw new Exception( $"Record metadata UID {rmd.RecordUid}: unsupported key type {rmd.RecordKeyType}"); } if (key != null) { rmd.RecordKey = CryptoUtils.EncryptAesV1(key, context.ClientKey).Base64UrlEncode(); rmd.RecordKeyType = (int)KeyType.DataKey; rmd.SharedFolderUid = storage.PersonalScopeUid; return(rmd); } } catch (Exception e) { Trace.TraceError(e.Message); } return(null); }) .ToArray(); storage.RecordKeys.PutLinks(rmds); } if (rs.teams != null) { var removedSharedFolderLinks = rs.teams .Where(x => x.removedSharedFolders != null) .SelectMany(x => x.removedSharedFolders, (team, sharedFolderUid) => UidLink.Create(sharedFolderUid, team.TeamUid)) .Cast <IUidLink>() .ToArray(); if (removedSharedFolderLinks.Any()) { result.AddSharedFolders(removedSharedFolderLinks.Select(x => x.SubjectUid)); storage.SharedFolderKeys.DeleteLinks(removedSharedFolderLinks); } var teams = rs.teams .Select(x => { try { byte[] teamKey; switch (x.KeyType) { case (int)KeyType.DataKey: teamKey = CryptoUtils.DecryptAesV1(x.TeamKey.Base64UrlDecode(), context.DataKey); break; case (int)KeyType.PrivateKey: teamKey = CryptoUtils.DecryptRsa(x.TeamKey.Base64UrlDecode(), context.PrivateKey); break; default: throw new Exception($"Team UID {x.TeamUid}: unsupported key type {x.KeyType}"); } x.TeamKey = CryptoUtils.EncryptAesV1(teamKey, clientKey).Base64UrlEncode(); x.KeyType = (int)KeyType.DataKey; return(x); } catch (Exception e) { Trace.TraceError(e.Message); return(null); } }) .Where(x => x != null) .ToArray(); storage.Teams.PutEntities(teams); var sharedFolderKeys = rs.teams .Where(x => x.sharedFolderKeys != null) .SelectMany(x => x.sharedFolderKeys, (team, sharedFolderKey) => { sharedFolderKey.TeamUid = team.TeamUid; sharedFolderKey.KeyType = sharedFolderKey.KeyType == 2 ? (int)KeyType.TeamPrivateKey : (int)KeyType.TeamKey; return(sharedFolderKey); }) .ToArray(); storage.SharedFolderKeys.PutLinks(sharedFolderKeys); } if (rs.sharedFolders != null) { result.AddSharedFolders(rs.sharedFolders.Select(x => x.SharedFolderUid)); // shared folders storage.SharedFolders.PutEntities(rs.sharedFolders); // shared folder keys var sharedFolderKeys = rs.sharedFolders .Where(x => !string.IsNullOrEmpty(x.SharedFolderKey)) .Select(x => { try { var sharedFolderKey = x.SharedFolderKey.Base64UrlDecode(); switch (x.KeyType) { case 1: sharedFolderKey = CryptoUtils.DecryptAesV1(sharedFolderKey, context.DataKey); break; case 2: sharedFolderKey = CryptoUtils.DecryptRsa(sharedFolderKey, context.PrivateKey); break; default: throw new Exception( $"Shared Folder UID {x.SharedFolderUid}: unsupported key type {x.KeyType}"); } return(new SyncDownSharedFolderKey { SharedFolderUid = x.SharedFolderUid, TeamUid = storage.PersonalScopeUid, SharedFolderKey = CryptoUtils.EncryptAesV1(sharedFolderKey, clientKey) .Base64UrlEncode(), KeyType = (int)KeyType.DataKey }); } catch (Exception e) { Trace.TraceError(e.Message); return(null); } }) .ToArray(); if (sharedFolderKeys.Any()) { storage.SharedFolderKeys.PutLinks(sharedFolderKeys); } result.AddRecords(rs.sharedFolders .Where(x => x.records != null) .SelectMany(x => x.records, (sf, r) => r.RecordUid)); // Records var sharedFolderRecords = rs.sharedFolders .Where(x => x.records != null) .SelectMany(x => x.records, (sf, sfr) => new SyncDownRecordMetaData { SharedFolderUid = sf.SharedFolderUid, RecordUid = sfr.RecordUid, RecordKey = sfr.RecordKey, RecordKeyType = (int)KeyType.SharedFolderKey, CanEdit = sfr.CanEdit, CanShare = sfr.CanShare }) .ToArray(); if (sharedFolderRecords.Any()) { storage.RecordKeys.PutLinks(sharedFolderRecords); } // Teams var teams = rs.sharedFolders .Where(x => x.teams != null) .SelectMany(x => x.teams, (sf, sft) => { sft.SharedFolderUid = sf.SharedFolderUid; return(sft); }) .Cast <ISharedFolderPermission>() .ToArray(); // Users var users = rs.sharedFolders .Where(x => x.users != null) .SelectMany(x => x.users, (sf, sfu) => { sfu.SharedFolderUid = sf.SharedFolderUid; return(sfu); }) .Cast <ISharedFolderPermission>() .ToArray(); if (teams.Any() || users.Any()) { storage.SharedFolderPermissions.PutLinks(teams.Concat(users)); } } if (rs.userFolders != null) { var userFolders = rs.userFolders .Select(uf => { try { var folderKey = uf.FolderKey.Base64UrlDecode(); switch (uf.keyType) { case (int)KeyType.DataKey: folderKey = CryptoUtils.DecryptAesV1(folderKey, context.DataKey); break; case (int)KeyType.PrivateKey: folderKey = CryptoUtils.DecryptRsa(folderKey, context.PrivateKey); break; default: throw new Exception($"User Folder UID {uf.FolderUid}: unsupported key type {uf.keyType}"); } uf.FolderKey = CryptoUtils.EncryptAesV1(folderKey, clientKey).Base64UrlEncode(); uf.keyType = (int)KeyType.DataKey; return(uf); } catch (Exception e) { Trace.TraceError(e.Message); return(null); } }) .ToArray(); storage.Folders.PutEntities(userFolders); } if (rs.sharedFolderFolders != null) { storage.Folders.PutEntities(rs.sharedFolderFolders); } if (rs.userFolderSharedFolders != null) { storage.Folders.PutEntities(rs.userFolderSharedFolders); } if (rs.userFolderRecords != null) { storage.FolderRecords.PutLinks(rs.userFolderRecords .Select(ufr => { ufr.FolderUid = string.IsNullOrEmpty(ufr.FolderUid) ? storage.PersonalScopeUid : ufr.FolderUid; return(ufr); })); } if (rs.sharedFolderFolderRecords != null) { storage.FolderRecords.PutLinks(rs.sharedFolderFolderRecords); } storage.Revision = rs.revision; Debug.WriteLine("Sync Down: Leave"); Debug.WriteLine("Rebuild Data: Enter"); vault.RebuildData(result); Debug.WriteLine("Rebuild Data: Leave"); }