public override RemoteUploadResult UploadNoteToRemote(ref INote inote, out INote conflict, ConflictResolutionStrategy strategy) { var note = (StandardFileNote)inote; if (_syncResult.saved_notes.Any(n => n.ID == note.ID)) { note.ApplyUpdatedData(_syncResult.saved_notes.First(n => n.ID == note.ID)); conflict = null; return(RemoteUploadResult.Uploaded); } if (_syncResult.retrieved_notes.Any(n => n.ID == note.ID)) { _logger.Warn(StandardNotePlugin.Name, "Uploaded note found in retrieved notes ... upload failed ?"); note.ApplyUpdatedData(_syncResult.retrieved_notes.First(n => n.ID == note.ID)); conflict = null; return(RemoteUploadResult.Merged); } if (_syncResult.error_notes.Any(n => n.ID == note.ID)) { throw new Exception("Could not upload note - server returned note in {unsaved_notes}"); } if (_syncResult.conflict_notes.Any(n => n.ID == note.ID)) { conflict = _syncResult.conflict_notes.First(n => n.ID == note.ID); return(RemoteUploadResult.Conflict); } conflict = null; return(RemoteUploadResult.UpToDate); }
public static RepoLock Lock(AlephLogger logger, string pathLocalFolder) { var filename = Path.Combine(pathLocalFolder, ".lock"); var content = $"[[AlephNote::RepoLock::Lockfile]]\n" + $"{{\n" + $" ProcessID := {Process.GetCurrentProcess().Id}\n" + $" ProcessHandle := {Process.GetCurrentProcess().Handle.ToInt64()}\n" + $" StartTime := {Process.GetCurrentProcess().StartTime:u}\n" + $" FileName := {Path.GetFileName(Process.GetCurrentProcess().MainModule?.FileName)}\n" + $" FilePath := {Process.GetCurrentProcess().MainModule?.FileName}\n" + $" ModuleName := {Process.GetCurrentProcess().MainModule?.ModuleName}\n" + $" ProcessName := {Process.GetCurrentProcess().ProcessName}\n" + $"}}\n"; try { if (File.Exists(filename)) { try { var oldcontent = File.ReadAllText(filename); logger.Warn("RepoLock", "Old Lockfile found", $"File := {filename}\n\nContent:\n{oldcontent}"); } catch (Exception) { logger.Warn("RepoLock", "Old Lockfile found (but could not read)", $"File := {filename}"); } } logger.Debug("RepoLock", "Trying to acquire lock", $"File := {filename}\n\nContent:\n{content}"); var fs = File.Open(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read); var bytes = Encoding.UTF8.GetBytes(content); fs.SetLength(0); fs.Write(bytes, 0, bytes.Length); fs.Flush(); logger.Debug("RepoLock", "Lock acquired successfully", $"File := {filename}\n\nContent:\n{content}"); return(new RepoLock(logger, filename, fs)); } catch (Exception e1) { logger.Error("RepoLock", "Acquiring lock failed", $"File := {filename}\n\nException:\n{e1}"); try { var oldcontent = File.ReadAllText(filename); logger.Info("RepoLock", "Old lock file read", $"File := Content:\n{oldcontent}"); } catch (Exception e2) { logger.Error("RepoLock", "Cannot read existing lockfile", e2.ToString()); } throw new RepoLockedException($"Could not open Repository '{pathLocalFolder}'.\nThe repository is currently locked by a different process.", e1); } }
private Tuple <IRemoteStorageSyncPersistance, List <INote>, int, int> DoSync(RemoteStorageAccount acc, AlephLogger log) { var data = SelectedProvider.CreateEmptyRemoteSyncData(); var conn = acc.Plugin.CreateRemoteStorageConnection(PluginManagerSingleton.Inst.GetProxyFactory().Build(), acc.Config, new HierarchyEmulationConfig(false, "\\", '\\')); var resultNotes = new ConcurrentQueue <INote>(); var resultErrors = new ConcurrentQueue <string>(); int fullCount; Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Connect to remote"; }); conn.StartSync(data, new List <INote>(), new List <INote>()); { Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "List notes from remote"; }); var missing = conn.ListMissingNotes(new List <INote>()); fullCount = missing.Count; Application.Current.Dispatcher.Invoke(() => { CanAbort = !acc.Plugin.SupportsNewDownloadMultithreading || missing.Count < AppSettings.DEFAULT_INITIALDOWNLOAD_PARALLELISM_THRESHOLD; }); SynchronizationThread.ExecuteInParallel( log, "InitialDownloadNewNotes", acc.Plugin.SupportsNewDownloadMultithreading, missing, AppSettings.DEFAULT_INITIALDOWNLOAD_PARALLELISM_LEVEL, AppSettings.DEFAULT_INITIALDOWNLOAD_PARALLELISM_THRESHOLD, (e, xnoteid) => { resultErrors.Enqueue(xnoteid); return(true); }, xnoteid => { var msg = $"Download Note {resultNotes.Count}/{missing.Count}"; Application.Current.Dispatcher.Invoke(() => { SyncInfoText = msg; }); var note = conn.DownloadNote(xnoteid, out var isnewnote); if (isnewnote) { note.SetLocalDirty("Set Note LocalDirty=true after download in Startupmode"); note.ResetRemoteDirty("Set Note RemoteDirty=false after download in Startupmode"); resultNotes.Enqueue(note); } else { log.Warn("Sync_FirstStart", $"Download new note {{id:'{xnoteid}'}} returned false"); } }); } Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Finish synchronization"; }); conn.FinishSync(out var _); return(Tuple.Create(data, resultNotes.ToList(), resultErrors.Count, fullCount)); }
private Tuple <IRemoteStorageSyncPersistance, List <INote> > DoSync(RemoteStorageAccount acc, AlephLogger log) { var data = SelectedProvider.CreateEmptyRemoteSyncData(); var conn = acc.Plugin.CreateRemoteStorageConnection(PluginManagerSingleton.Inst.GetProxyFactory().Build(), acc.Config, new HierachyEmulationConfig(false, "\\", '\\')); var resultNotes = new List <INote>(); Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Connect to remote"; }); conn.StartSync(data, new List <INote>(), new List <INote>()); { Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "List notes from remote"; }); var missing = conn.ListMissingNotes(new List <INote>()); int idx = 0; foreach (var xnoteid in missing) { var noteid = xnoteid; idx++; try { string msg = $"Download Note {idx}/{missing.Count}"; Application.Current.Dispatcher.Invoke(() => { SyncInfoText = msg; }); var note = conn.DownloadNote(noteid, out var isnewnote); if (isnewnote) { note.SetLocalDirty("Set Note LocalDirty=true after download in Startupmode"); note.ResetRemoteDirty("Set Note RemoteDirty=false after download in Startupmode"); resultNotes.Add(note); } else { log.Warn("Sync_FirstStart", string.Format("Download new note {{id:'{0}'}} returned false", noteid)); } } catch (ThreadAbortException) { throw; } catch (Exception e) { throw new Exception(string.Format("Could not download new note '{0}' on remote cause of {1}", noteid, e.Message)); } } } Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Finish synchronization"; }); conn.FinishSync(); return(Tuple.Create(data, resultNotes)); }
private static DateTimeOffset ConvertFromEpochDate(double seconds) { const double DTO_MAX = 10.0 * 1000 * 1000 * 1000; if (seconds <= 0) { Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (<=0)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN); } if (double.IsNaN(seconds)) { Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (IsNaN)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN); } if (double.IsInfinity(seconds)) { Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (IsInfinity)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN); } if (seconds > DTO_MAX) { Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (>Max)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN); } return(TIMESTAMP_ORIGIN.AddSeconds(seconds)); }
private void UploadNotes(IReadOnlyCollection <Tuple <INote, INote> > notesToUpload, IEnumerable <Tuple <INote, INote> > notesToResetRemoteDirty, ICollection <Tuple <string, Exception> > errors) { ExecuteInParallel( _log, "UploadNotes", _noteUploadEnableMultithreading, notesToUpload, _noteUploadParallelismLevel, _noteUploadParallelismThreshold, (e, notetuple) => { var message = string.Format("Could not upload note '{2}' ({0}) cause of {1}", notetuple.Item2.UniqueName, e.Message, notetuple.Item2.Title); _log.Error("Sync", message, e); errors.Add(Tuple.Create(message, e)); return(true); }, notetuple => { var realnote = notetuple.Item1; var clonenote = notetuple.Item2; _log.Info("Sync", string.Format("Upload note {0}", clonenote.UniqueName)); if (!realnote.IsLocalSaved) { _dispatcher.Invoke(() => { if (!realnote.IsLocalSaved) { _repo.SaveNote(realnote); } }); } var result = _repo.Connection.UploadNoteToRemote(ref clonenote, out var conflictnote, _conflictStrategy); switch (result) { case RemoteUploadResult.UpToDate: case RemoteUploadResult.Uploaded: _dispatcher.Invoke(() => { if (realnote.IsLocalSaved) { realnote.OnAfterUpload(clonenote); realnote.ResetRemoteDirty("Note was successfully uploaded (no conflict)"); _repo.SaveNote(realnote); } }); break; case RemoteUploadResult.Merged: _dispatcher.Invoke(() => { realnote.ApplyUpdatedData(clonenote); realnote.TriggerOnChanged(true); realnote.SetLocalDirty("Note was uploaded and a merge has happened"); realnote.ResetRemoteDirty("Note was successfully uploaded (auto-merge)"); }); break; case RemoteUploadResult.Conflict: _log.Warn("Sync", "Uploading note " + clonenote.UniqueName + " resulted in conflict"); ResolveUploadConflict(realnote, clonenote, conflictnote); break; default: throw new ArgumentOutOfRangeException(); } });
private void DoSync(bool isImmediateResync) { _log.Info("Sync", $"Starting remote synchronization (immediate={isImmediateResync})"); List <Tuple <string, Exception> > errors = new List <Tuple <string, Exception> >(); _dispatcher.BeginInvoke(() => { foreach (var l in _listener) { l.StartSync(); } }); bool doImmediateResync = false; try { var data = _repo.GetSyncData(); List <Tuple <INote, INote> > allNotes = new List <Tuple <INote, INote> >(); // <real, clone> List <INote> notesToDelete = new List <INote>(); _dispatcher.Invoke(() => { allNotes = _repo.Notes.Select(p => Tuple.Create(p, p.Clone())).ToList(); notesToDelete = _repo.LocalDeletedNotes.ToList(); }); _log.Info( "Sync", $"Found {allNotes.Count} alive notes and {notesToDelete.Count} deleted notes", $"Alive:\n{string.Join("\n", allNotes.Select(n => $"{n.Item2.UniqueName} {n.Item2.Title}"))}\n\n\n" + $"Deleted:\n{string.Join("\n", notesToDelete.Select(n => $"{n.UniqueName} {n.Title}"))}"); _repo.Connection.StartSync(data, allNotes.Select(p => p.Item2).ToList(), notesToDelete); { // plugin says 'upload' var notesToUpload = allNotes.Where(p => _repo.Connection.NeedsUpload(p.Item2)).ToList(); // plugin says 'download' var notesToDownload = allNotes.Where(p => _repo.Connection.NeedsDownload(p.Item2)).ToList(); // we think 'upload', but provider doesn't say so var notesToResetDirty = allNotes.Where(p => !p.Item2.IsRemoteSaved && !notesToUpload.Contains(p)).ToList(); _log.Info( "Sync", $"Found {notesToUpload.Count} notes for upload and {notesToDownload.Count} notes for download", $"Upload:\n{string.Join("\n", notesToUpload.Select(n => $"{n.Item2.UniqueName} {n.Item2.Title}"))}\n\n\n" + $"Download:\n{string.Join("\n", notesToDownload.Select(n => $"{n.Item2.UniqueName} {n.Item2.Title}"))}"); UploadNotes(notesToUpload, notesToResetDirty, errors); DownloadNotes(notesToDownload, errors); DeleteNotes(notesToDelete, errors); DownloadNewNotes(allNotes, errors); } _repo.Connection.FinishSync(out doImmediateResync); _repo.WriteSyncData(data); } catch (RestException e) { errors.Add(Tuple.Create <string, Exception>("Execption while syncing notes: " + e.ShortMessage, e)); } catch (Exception e) { errors.Add(Tuple.Create("Execption while syncing notes: " + e.Message, e)); } if (errors.Any()) { _dispatcher.BeginInvoke(() => { foreach (var l in _listener) { l.SyncError(errors); } }); } else { _dispatcher.BeginInvoke(() => { foreach (var l in _listener) { l.SyncSuccess(DateTimeOffset.Now); } }); } _log.Info("Sync", "Finished remote synchronization"); if (doImmediateResync) { if (isImmediateResync) { _log.Warn("Sync", "Sync triggered an immediate resync. This was preventeed because past sync event was already immediate (prevent endless sync)"); } else { _log.Info("Sync", "Sync triggered an immediate resync. Next sync will be executed now."); DoSync(true); } } }
public override void StartSync(IRemoteStorageSyncPersistance idata, List <INote> ilocalnotes, List <INote> localdeletednotes) { StandardNoteAPI.Logger = _logger; _immediateResync = false; var data = (StandardNoteData)idata; try { using (var web = CreateAuthenticatedClient(data)) { var localnotes = ilocalnotes.Cast <StandardFileNote>().ToList(); var upNotes = localnotes.Where(p => NeedsUploadReal(p, idata)).ToList(); var delNotes = localdeletednotes.Cast <StandardFileNote>().ToList(); var delTags = data.GetUnusedTags(localnotes.ToList()); if (int.Parse(data.SessionData.Version) >= 4 && !data.ItemsKeys.Any(p => p.Version == data.SessionData.Version)) { _logger.Warn(StandardNotePlugin.Name, "Repository doesn't have any matching items_key's - trying to get one"); StandardNoteAPI.SyncToEnsureItemsKeys(web, this, _config, data); } _syncResult = StandardNoteAPI.Sync(web, this, _config, data, localnotes, upNotes, delNotes, delTags); _lastUploadBatch = upNotes.Select(p => p.ID).ToList(); _logger.Debug(StandardNotePlugin.Name, "StandardFile sync finished.", $"upload:\n" + $"[\n" + $" notes = {upNotes.Count}\n" + $" deleted = {delNotes.Count}\n" + $"]\n" + $"\n" + $"download:\n" + $"[\n" + $" note:\n" + $" [\n" + $" retrieved = {_syncResult.retrieved_notes.Count}\n" + $" deleted = {_syncResult.deleted_notes.Count}\n" + $" saved = {_syncResult.saved_notes.Count}\n" + $" sync_conflicts = {_syncResult.syncconflict_notes.Count}\n" + $" uuid_conflicts = {_syncResult.uuidconflict_notes.Count}\n" + $" ]\n" + $" tags:\n" + $" [\n" + $" retrieved = {_syncResult.retrieved_tags.Count}\n" + $" deleted = {_syncResult.deleted_tags.Count}\n" + $" saved = {_syncResult.saved_tags.Count}\n" + $" sync_conflicts = {_syncResult.syncconflict_tags.Count}\n" + $" uuid_conflicts = {_syncResult.uuidconflict_tags.Count}\n" + $" ]\n" + $" items_keys:\n" + $" [\n" + $" retrieved = {_syncResult.retrieved_keys.Count}\n" + $" deleted = {_syncResult.deleted_keys.Count}\n" + $" saved = {_syncResult.saved_keys.Count}\n" + $" sync_conflicts = {_syncResult.syncconflict_keys.Count}\n" + $" uuid_conflicts = {_syncResult.uuidconflict_keys.Count}\n" + $" ]\n" + $"]"); } } catch (RestStatuscodeException e) { if (e.StatusCode == 401) { _logger.Warn(StandardNotePlugin.Name, "Reset Token (401 statuscode)", $"Exception := {e.GetType()}\n" + $"Message := {e.Message}\n" + $"Source := {e.Source}\n" + $"StatusCode := {e.StatusCode}\n" + $"StatusPhrase := {e.StatusPhrase}\n" + $"StackTrace := \n{e.StackTrace}\n" + $"HTTPContent := \n{e.HTTPContent}\n"); data.SessionData = null; } throw; } }
private static IEnumerable <StandardFileNote> GetMissingNoteUpdates(IEnumerable <SyncResultTag> syncedTags, List <StandardFileTag> allTags, List <StandardFileNote> _oldNotes, List <StandardFileNote> newNotes) { // update tag ref changes in notes List <StandardFileNote> result = new List <StandardFileNote>(); List <StandardFileNote> oldNotes = _oldNotes.Select(n => n.Clone()).Cast <StandardFileNote>().ToList(); // [1] added tags foreach (var tag in syncedTags) { foreach (var tagRef in tag.references) { var oldnote = oldNotes.FirstOrDefault(n => n.ID == tagRef.uuid); var newnote = newNotes.FirstOrDefault(n => n.ID == tagRef.uuid); if (newnote != null) // note is already in changed-list { if (newnote.ContainsTag(tag.uuid)) { // tag is in new one - all ok } else { // tag was missing, we add it // tag was probably added on remote side newnote.AddTag(new StandardFileTagRef(tag.uuid, tag.title)); } } else if (oldnote != null) // another note was changed that is not in the changed-list { if (oldnote.ContainsTag(tag.uuid)) { // tag is in it - all ok } else { // Add tag to note and add note to changed-list oldnote.AddTag(new StandardFileTagRef(tag.uuid, tag.title)); result.Add(oldnote); } } else // tag references non-existant note ??? { Logger.Warn(StandardNotePlugin.Name, $"Reference from tag {tag.uuid} to missing note {tagRef.uuid}"); } } } // [2] removed tags from notes-in-bucket foreach (var note in newNotes) { foreach (var noteTag in note.InternalTags.ToList()) { if (noteTag.UUID == null) { continue; // unsynced } var fullTag = allTags.FirstOrDefault(t => t.UUID == noteTag.UUID); if (fullTag == null) { // Note references tag that doesn't even exist -- remove it from note note.RemoveTag(noteTag); } else { if (fullTag.References.Any(r => r == note.ID)) { // all ok, tag is in note and in reference-list of tag } else { // tag is _NOT_ in reference list of tag -- remove it from note note.RemoveTag(noteTag); } } } } // [3] removed tags from old-bucket foreach (var note in oldNotes.Where(n1 => newNotes.All(n2 => n1.ID != n2.ID))) { foreach (var noteTag in note.InternalTags.ToList()) { if (noteTag.UUID == null) { continue; // unsynced } var fullTag = allTags.FirstOrDefault(t => t.UUID == noteTag.UUID); if (fullTag == null) { // Note references tag that doesn't even exist -- remove it from note note.RemoveTag(noteTag); result.Add(note); } else { if (fullTag.References.Any(r => r == note.ID)) { // all ok, tag is in note and in reference-list of tag } else { // tag is _NOT_ in reference list of tag -- remove it from note note.RemoveTag(noteTag); result.Add(note); } } } } return(result.DistinctBy(n => n.ID)); }
private void UploadNotes(List <Tuple <INote, INote> > notesToUpload, List <Tuple <INote, INote> > notesToResetRemoteDirty, ref List <Tuple <string, Exception> > errors) { foreach (var notetuple in notesToUpload) { var realnote = notetuple.Item1; var clonenote = notetuple.Item2; _log.Info("Sync", string.Format("Upload note {0}", clonenote.UniqueName)); try { if (!realnote.IsLocalSaved) { dispatcher.Invoke(() => { if (!realnote.IsLocalSaved) { repo.SaveNote(realnote); } }); } var result = repo.Connection.UploadNoteToRemote(ref clonenote, out var conflictnote, conflictStrategy); switch (result) { case RemoteUploadResult.UpToDate: case RemoteUploadResult.Uploaded: dispatcher.Invoke(() => { if (realnote.IsLocalSaved) { realnote.OnAfterUpload(clonenote); realnote.ResetRemoteDirty("Note was successfully uploaded (no conflict)"); repo.SaveNote(realnote); } }); break; case RemoteUploadResult.Merged: dispatcher.Invoke(() => { realnote.ApplyUpdatedData(clonenote); realnote.TriggerOnChanged(true); realnote.SetLocalDirty("Note was uploaded and a merge has happened"); realnote.ResetRemoteDirty("Note was successfully uploaded (auto-merge)"); }); break; case RemoteUploadResult.Conflict: _log.Warn("Sync", "Uploading note " + clonenote.UniqueName + " resulted in conflict"); ResolveUploadConflict(realnote, clonenote, conflictnote); break; default: throw new ArgumentOutOfRangeException(); } } catch (Exception e) { var message = string.Format("Could not upload note '{2}' ({0}) cause of {1}", clonenote.UniqueName, e.Message, clonenote.Title); _log.Error("Sync", message, e); errors.Add(Tuple.Create(message, e)); } } foreach (var notetuple in notesToResetRemoteDirty) { var realnote = notetuple.Item1; var clonenote = notetuple.Item2; _log.Info("Sync", string.Format("Reset remote dirty of note {0} (no upload needed)", clonenote.UniqueName)); try { if (!realnote.IsLocalSaved) { dispatcher.Invoke(() => { if (!realnote.IsLocalSaved) { repo.SaveNote(realnote); } }); } dispatcher.Invoke(() => { if (realnote.IsLocalSaved) { realnote.ResetRemoteDirty("Reset remote dirty - was marked for upload but plugin says no upload is needed"); repo.SaveNote(realnote); } }); } catch (Exception e) { var message = string.Format("Could not reset remote dirty note '{2}' ({0}) cause of {1}", clonenote.UniqueName, e.Message, clonenote.Title); _log.Error("Sync", message, e); errors.Add(Tuple.Create(message, e)); } } }