public override RemoteUploadResult UploadNoteToRemote(ref INote inote, out INote conflict, ConflictResolutionStrategy strategy)
{
var note = (StandardFileNote)inote;
if (_syncResult.saved_notes.Any(n => n.ID == note.ID))
{
note.ApplyUpdatedData(_syncResult.saved_notes.First(n => n.ID == note.ID));
conflict = null;
return(RemoteUploadResult.Uploaded);
}
if (_syncResult.retrieved_notes.Any(n => n.ID == note.ID))
{
_logger.Warn(StandardNotePlugin.Name, "Uploaded note found in retrieved notes ... upload failed ?");
note.ApplyUpdatedData(_syncResult.retrieved_notes.First(n => n.ID == note.ID));
conflict = null;
return(RemoteUploadResult.Merged);
}
if (_syncResult.error_notes.Any(n => n.ID == note.ID))
{
throw new Exception("Could not upload note - server returned note in {unsaved_notes}");
}
if (_syncResult.conflict_notes.Any(n => n.ID == note.ID))
{
conflict = _syncResult.conflict_notes.First(n => n.ID == note.ID);
return(RemoteUploadResult.Conflict);
}
conflict = null;
return(RemoteUploadResult.UpToDate);
}
public static RepoLock Lock(AlephLogger logger, string pathLocalFolder)
{
var filename = Path.Combine(pathLocalFolder, ".lock");
var content =
$"[[AlephNote::RepoLock::Lockfile]]\n" +
$"{{\n" +
$" ProcessID := {Process.GetCurrentProcess().Id}\n" +
$" ProcessHandle := {Process.GetCurrentProcess().Handle.ToInt64()}\n" +
$" StartTime := {Process.GetCurrentProcess().StartTime:u}\n" +
$" FileName := {Path.GetFileName(Process.GetCurrentProcess().MainModule?.FileName)}\n" +
$" FilePath := {Process.GetCurrentProcess().MainModule?.FileName}\n" +
$" ModuleName := {Process.GetCurrentProcess().MainModule?.ModuleName}\n" +
$" ProcessName := {Process.GetCurrentProcess().ProcessName}\n" +
$"}}\n";
try
{
if (File.Exists(filename))
{
try
{
var oldcontent = File.ReadAllText(filename);
logger.Warn("RepoLock", "Old Lockfile found", $"File := {filename}\n\nContent:\n{oldcontent}");
}
catch (Exception)
{
logger.Warn("RepoLock", "Old Lockfile found (but could not read)", $"File := {filename}");
}
}
logger.Debug("RepoLock", "Trying to acquire lock", $"File := {filename}\n\nContent:\n{content}");
var fs = File.Open(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read);
var bytes = Encoding.UTF8.GetBytes(content);
fs.SetLength(0);
fs.Write(bytes, 0, bytes.Length);
fs.Flush();
logger.Debug("RepoLock", "Lock acquired successfully", $"File := {filename}\n\nContent:\n{content}");
return(new RepoLock(logger, filename, fs));
}
catch (Exception e1)
{
logger.Error("RepoLock", "Acquiring lock failed", $"File := {filename}\n\nException:\n{e1}");
try
{
var oldcontent = File.ReadAllText(filename);
logger.Info("RepoLock", "Old lock file read", $"File := Content:\n{oldcontent}");
}
catch (Exception e2)
{
logger.Error("RepoLock", "Cannot read existing lockfile", e2.ToString());
}
throw new RepoLockedException($"Could not open Repository '{pathLocalFolder}'.\nThe repository is currently locked by a different process.", e1);
}
}
private Tuple <IRemoteStorageSyncPersistance, List <INote>, int, int> DoSync(RemoteStorageAccount acc, AlephLogger log)
{
var data = SelectedProvider.CreateEmptyRemoteSyncData();
var conn = acc.Plugin.CreateRemoteStorageConnection(PluginManagerSingleton.Inst.GetProxyFactory().Build(), acc.Config, new HierarchyEmulationConfig(false, "\\", '\\'));
var resultNotes = new ConcurrentQueue <INote>();
var resultErrors = new ConcurrentQueue <string>();
int fullCount;
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Connect to remote"; });
conn.StartSync(data, new List <INote>(), new List <INote>());
{
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "List notes from remote"; });
var missing = conn.ListMissingNotes(new List <INote>());
fullCount = missing.Count;
Application.Current.Dispatcher.Invoke(() =>
{
CanAbort = !acc.Plugin.SupportsNewDownloadMultithreading || missing.Count < AppSettings.DEFAULT_INITIALDOWNLOAD_PARALLELISM_THRESHOLD;
});
SynchronizationThread.ExecuteInParallel(
log,
"InitialDownloadNewNotes",
acc.Plugin.SupportsNewDownloadMultithreading,
missing,
AppSettings.DEFAULT_INITIALDOWNLOAD_PARALLELISM_LEVEL,
AppSettings.DEFAULT_INITIALDOWNLOAD_PARALLELISM_THRESHOLD,
(e, xnoteid) =>
{
resultErrors.Enqueue(xnoteid);
return(true);
},
xnoteid =>
{
var msg = $"Download Note {resultNotes.Count}/{missing.Count}";
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = msg; });
var note = conn.DownloadNote(xnoteid, out var isnewnote);
if (isnewnote)
{
note.SetLocalDirty("Set Note LocalDirty=true after download in Startupmode");
note.ResetRemoteDirty("Set Note RemoteDirty=false after download in Startupmode");
resultNotes.Enqueue(note);
}
else
{
log.Warn("Sync_FirstStart", $"Download new note {{id:'{xnoteid}'}} returned false");
}
});
}
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Finish synchronization"; });
conn.FinishSync(out var _);
return(Tuple.Create(data, resultNotes.ToList(), resultErrors.Count, fullCount));
}
private Tuple <IRemoteStorageSyncPersistance, List <INote> > DoSync(RemoteStorageAccount acc, AlephLogger log)
{
var data = SelectedProvider.CreateEmptyRemoteSyncData();
var conn = acc.Plugin.CreateRemoteStorageConnection(PluginManagerSingleton.Inst.GetProxyFactory().Build(), acc.Config, new HierachyEmulationConfig(false, "\\", '\\'));
var resultNotes = new List <INote>();
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Connect to remote"; });
conn.StartSync(data, new List <INote>(), new List <INote>());
{
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "List notes from remote"; });
var missing = conn.ListMissingNotes(new List <INote>());
int idx = 0;
foreach (var xnoteid in missing)
{
var noteid = xnoteid;
idx++;
try
{
string msg = $"Download Note {idx}/{missing.Count}";
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = msg; });
var note = conn.DownloadNote(noteid, out var isnewnote);
if (isnewnote)
{
note.SetLocalDirty("Set Note LocalDirty=true after download in Startupmode");
note.ResetRemoteDirty("Set Note RemoteDirty=false after download in Startupmode");
resultNotes.Add(note);
}
else
{
log.Warn("Sync_FirstStart", string.Format("Download new note {{id:'{0}'}} returned false", noteid));
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception e)
{
throw new Exception(string.Format("Could not download new note '{0}' on remote cause of {1}", noteid, e.Message));
}
}
}
Application.Current.Dispatcher.Invoke(() => { SyncInfoText = "Finish synchronization"; });
conn.FinishSync();
return(Tuple.Create(data, resultNotes));
}
private static DateTimeOffset ConvertFromEpochDate(double seconds)
{
const double DTO_MAX = 10.0 * 1000 * 1000 * 1000;
if (seconds <= 0)
{
Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (<=0)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN);
}
if (double.IsNaN(seconds))
{
Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (IsNaN)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN);
}
if (double.IsInfinity(seconds))
{
Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (IsInfinity)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN);
}
if (seconds > DTO_MAX)
{
Logger.Warn(SimpleNotePlugin.Name, "ConvertFromEpochDate with invalid value (>Max)", $"seconds: {seconds}"); return(TIMESTAMP_ORIGIN);
}
return(TIMESTAMP_ORIGIN.AddSeconds(seconds));
}
private void UploadNotes(IReadOnlyCollection <Tuple <INote, INote> > notesToUpload, IEnumerable <Tuple <INote, INote> > notesToResetRemoteDirty, ICollection <Tuple <string, Exception> > errors)
{
ExecuteInParallel(
_log,
"UploadNotes",
_noteUploadEnableMultithreading,
notesToUpload,
_noteUploadParallelismLevel,
_noteUploadParallelismThreshold,
(e, notetuple) =>
{
var message = string.Format("Could not upload note '{2}' ({0}) cause of {1}", notetuple.Item2.UniqueName, e.Message, notetuple.Item2.Title);
_log.Error("Sync", message, e);
errors.Add(Tuple.Create(message, e));
return(true);
},
notetuple =>
{
var realnote = notetuple.Item1;
var clonenote = notetuple.Item2;
_log.Info("Sync", string.Format("Upload note {0}", clonenote.UniqueName));
if (!realnote.IsLocalSaved)
{
_dispatcher.Invoke(() =>
{
if (!realnote.IsLocalSaved)
{
_repo.SaveNote(realnote);
}
});
}
var result = _repo.Connection.UploadNoteToRemote(ref clonenote, out var conflictnote, _conflictStrategy);
switch (result)
{
case RemoteUploadResult.UpToDate:
case RemoteUploadResult.Uploaded:
_dispatcher.Invoke(() =>
{
if (realnote.IsLocalSaved)
{
realnote.OnAfterUpload(clonenote);
realnote.ResetRemoteDirty("Note was successfully uploaded (no conflict)");
_repo.SaveNote(realnote);
}
});
break;
case RemoteUploadResult.Merged:
_dispatcher.Invoke(() =>
{
realnote.ApplyUpdatedData(clonenote);
realnote.TriggerOnChanged(true);
realnote.SetLocalDirty("Note was uploaded and a merge has happened");
realnote.ResetRemoteDirty("Note was successfully uploaded (auto-merge)");
});
break;
case RemoteUploadResult.Conflict:
_log.Warn("Sync", "Uploading note " + clonenote.UniqueName + " resulted in conflict");
ResolveUploadConflict(realnote, clonenote, conflictnote);
break;
default:
throw new ArgumentOutOfRangeException();
}
});
private void DoSync(bool isImmediateResync)
{
_log.Info("Sync", $"Starting remote synchronization (immediate={isImmediateResync})");
List <Tuple <string, Exception> > errors = new List <Tuple <string, Exception> >();
_dispatcher.BeginInvoke(() => { foreach (var l in _listener)
{
l.StartSync();
}
});
bool doImmediateResync = false;
try
{
var data = _repo.GetSyncData();
List <Tuple <INote, INote> > allNotes = new List <Tuple <INote, INote> >(); // <real, clone>
List <INote> notesToDelete = new List <INote>();
_dispatcher.Invoke(() =>
{
allNotes = _repo.Notes.Select(p => Tuple.Create(p, p.Clone())).ToList();
notesToDelete = _repo.LocalDeletedNotes.ToList();
});
_log.Info(
"Sync",
$"Found {allNotes.Count} alive notes and {notesToDelete.Count} deleted notes",
$"Alive:\n{string.Join("\n", allNotes.Select(n => $"{n.Item2.UniqueName} {n.Item2.Title}"))}\n\n\n" +
$"Deleted:\n{string.Join("\n", notesToDelete.Select(n => $"{n.UniqueName} {n.Title}"))}");
_repo.Connection.StartSync(data, allNotes.Select(p => p.Item2).ToList(), notesToDelete);
{
// plugin says 'upload'
var notesToUpload = allNotes.Where(p => _repo.Connection.NeedsUpload(p.Item2)).ToList();
// plugin says 'download'
var notesToDownload = allNotes.Where(p => _repo.Connection.NeedsDownload(p.Item2)).ToList();
// we think 'upload', but provider doesn't say so
var notesToResetDirty = allNotes.Where(p => !p.Item2.IsRemoteSaved && !notesToUpload.Contains(p)).ToList();
_log.Info(
"Sync",
$"Found {notesToUpload.Count} notes for upload and {notesToDownload.Count} notes for download",
$"Upload:\n{string.Join("\n", notesToUpload.Select(n => $"{n.Item2.UniqueName} {n.Item2.Title}"))}\n\n\n" +
$"Download:\n{string.Join("\n", notesToDownload.Select(n => $"{n.Item2.UniqueName} {n.Item2.Title}"))}");
UploadNotes(notesToUpload, notesToResetDirty, errors);
DownloadNotes(notesToDownload, errors);
DeleteNotes(notesToDelete, errors);
DownloadNewNotes(allNotes, errors);
}
_repo.Connection.FinishSync(out doImmediateResync);
_repo.WriteSyncData(data);
}
catch (RestException e)
{
errors.Add(Tuple.Create <string, Exception>("Execption while syncing notes: " + e.ShortMessage, e));
}
catch (Exception e)
{
errors.Add(Tuple.Create("Execption while syncing notes: " + e.Message, e));
}
if (errors.Any())
{
_dispatcher.BeginInvoke(() => { foreach (var l in _listener)
{
l.SyncError(errors);
}
});
}
else
{
_dispatcher.BeginInvoke(() => { foreach (var l in _listener)
{
l.SyncSuccess(DateTimeOffset.Now);
}
});
}
_log.Info("Sync", "Finished remote synchronization");
if (doImmediateResync)
{
if (isImmediateResync)
{
_log.Warn("Sync", "Sync triggered an immediate resync. This was preventeed because past sync event was already immediate (prevent endless sync)");
}
else
{
_log.Info("Sync", "Sync triggered an immediate resync. Next sync will be executed now.");
DoSync(true);
}
}
}
public override void StartSync(IRemoteStorageSyncPersistance idata, List <INote> ilocalnotes, List <INote> localdeletednotes)
{
StandardNoteAPI.Logger = _logger;
_immediateResync = false;
var data = (StandardNoteData)idata;
try
{
using (var web = CreateAuthenticatedClient(data))
{
var localnotes = ilocalnotes.Cast <StandardFileNote>().ToList();
var upNotes = localnotes.Where(p => NeedsUploadReal(p, idata)).ToList();
var delNotes = localdeletednotes.Cast <StandardFileNote>().ToList();
var delTags = data.GetUnusedTags(localnotes.ToList());
if (int.Parse(data.SessionData.Version) >= 4 && !data.ItemsKeys.Any(p => p.Version == data.SessionData.Version))
{
_logger.Warn(StandardNotePlugin.Name, "Repository doesn't have any matching items_key's - trying to get one");
StandardNoteAPI.SyncToEnsureItemsKeys(web, this, _config, data);
}
_syncResult = StandardNoteAPI.Sync(web, this, _config, data, localnotes, upNotes, delNotes, delTags);
_lastUploadBatch = upNotes.Select(p => p.ID).ToList();
_logger.Debug(StandardNotePlugin.Name, "StandardFile sync finished.",
$"upload:\n" +
$"[\n" +
$" notes = {upNotes.Count}\n" +
$" deleted = {delNotes.Count}\n" +
$"]\n" +
$"\n" +
$"download:\n" +
$"[\n" +
$" note:\n" +
$" [\n" +
$" retrieved = {_syncResult.retrieved_notes.Count}\n" +
$" deleted = {_syncResult.deleted_notes.Count}\n" +
$" saved = {_syncResult.saved_notes.Count}\n" +
$" sync_conflicts = {_syncResult.syncconflict_notes.Count}\n" +
$" uuid_conflicts = {_syncResult.uuidconflict_notes.Count}\n" +
$" ]\n" +
$" tags:\n" +
$" [\n" +
$" retrieved = {_syncResult.retrieved_tags.Count}\n" +
$" deleted = {_syncResult.deleted_tags.Count}\n" +
$" saved = {_syncResult.saved_tags.Count}\n" +
$" sync_conflicts = {_syncResult.syncconflict_tags.Count}\n" +
$" uuid_conflicts = {_syncResult.uuidconflict_tags.Count}\n" +
$" ]\n" +
$" items_keys:\n" +
$" [\n" +
$" retrieved = {_syncResult.retrieved_keys.Count}\n" +
$" deleted = {_syncResult.deleted_keys.Count}\n" +
$" saved = {_syncResult.saved_keys.Count}\n" +
$" sync_conflicts = {_syncResult.syncconflict_keys.Count}\n" +
$" uuid_conflicts = {_syncResult.uuidconflict_keys.Count}\n" +
$" ]\n" +
$"]");
}
}
catch (RestStatuscodeException e)
{
if (e.StatusCode == 401)
{
_logger.Warn(StandardNotePlugin.Name,
"Reset Token (401 statuscode)",
$"Exception := {e.GetType()}\n" +
$"Message := {e.Message}\n" +
$"Source := {e.Source}\n" +
$"StatusCode := {e.StatusCode}\n" +
$"StatusPhrase := {e.StatusPhrase}\n" +
$"StackTrace := \n{e.StackTrace}\n" +
$"HTTPContent := \n{e.HTTPContent}\n");
data.SessionData = null;
}
throw;
}
}
private static IEnumerable <StandardFileNote> GetMissingNoteUpdates(IEnumerable <SyncResultTag> syncedTags, List <StandardFileTag> allTags, List <StandardFileNote> _oldNotes, List <StandardFileNote> newNotes)
{
// update tag ref changes in notes
List <StandardFileNote> result = new List <StandardFileNote>();
List <StandardFileNote> oldNotes = _oldNotes.Select(n => n.Clone()).Cast <StandardFileNote>().ToList();
// [1] added tags
foreach (var tag in syncedTags)
{
foreach (var tagRef in tag.references)
{
var oldnote = oldNotes.FirstOrDefault(n => n.ID == tagRef.uuid);
var newnote = newNotes.FirstOrDefault(n => n.ID == tagRef.uuid);
if (newnote != null) // note is already in changed-list
{
if (newnote.ContainsTag(tag.uuid))
{
// tag is in new one - all ok
}
else
{
// tag was missing, we add it
// tag was probably added on remote side
newnote.AddTag(new StandardFileTagRef(tag.uuid, tag.title));
}
}
else if (oldnote != null) // another note was changed that is not in the changed-list
{
if (oldnote.ContainsTag(tag.uuid))
{
// tag is in it - all ok
}
else
{
// Add tag to note and add note to changed-list
oldnote.AddTag(new StandardFileTagRef(tag.uuid, tag.title));
result.Add(oldnote);
}
}
else // tag references non-existant note ???
{
Logger.Warn(StandardNotePlugin.Name, $"Reference from tag {tag.uuid} to missing note {tagRef.uuid}");
}
}
}
// [2] removed tags from notes-in-bucket
foreach (var note in newNotes)
{
foreach (var noteTag in note.InternalTags.ToList())
{
if (noteTag.UUID == null)
{
continue; // unsynced
}
var fullTag = allTags.FirstOrDefault(t => t.UUID == noteTag.UUID);
if (fullTag == null)
{
// Note references tag that doesn't even exist -- remove it from note
note.RemoveTag(noteTag);
}
else
{
if (fullTag.References.Any(r => r == note.ID))
{
// all ok, tag is in note and in reference-list of tag
}
else
{
// tag is _NOT_ in reference list of tag -- remove it from note
note.RemoveTag(noteTag);
}
}
}
}
// [3] removed tags from old-bucket
foreach (var note in oldNotes.Where(n1 => newNotes.All(n2 => n1.ID != n2.ID)))
{
foreach (var noteTag in note.InternalTags.ToList())
{
if (noteTag.UUID == null)
{
continue; // unsynced
}
var fullTag = allTags.FirstOrDefault(t => t.UUID == noteTag.UUID);
if (fullTag == null)
{
// Note references tag that doesn't even exist -- remove it from note
note.RemoveTag(noteTag);
result.Add(note);
}
else
{
if (fullTag.References.Any(r => r == note.ID))
{
// all ok, tag is in note and in reference-list of tag
}
else
{
// tag is _NOT_ in reference list of tag -- remove it from note
note.RemoveTag(noteTag);
result.Add(note);
}
}
}
}
return(result.DistinctBy(n => n.ID));
}
private void UploadNotes(List <Tuple <INote, INote> > notesToUpload, List <Tuple <INote, INote> > notesToResetRemoteDirty, ref List <Tuple <string, Exception> > errors)
{
foreach (var notetuple in notesToUpload)
{
var realnote = notetuple.Item1;
var clonenote = notetuple.Item2;
_log.Info("Sync", string.Format("Upload note {0}", clonenote.UniqueName));
try
{
if (!realnote.IsLocalSaved)
{
dispatcher.Invoke(() =>
{
if (!realnote.IsLocalSaved)
{
repo.SaveNote(realnote);
}
});
}
var result = repo.Connection.UploadNoteToRemote(ref clonenote, out var conflictnote, conflictStrategy);
switch (result)
{
case RemoteUploadResult.UpToDate:
case RemoteUploadResult.Uploaded:
dispatcher.Invoke(() =>
{
if (realnote.IsLocalSaved)
{
realnote.OnAfterUpload(clonenote);
realnote.ResetRemoteDirty("Note was successfully uploaded (no conflict)");
repo.SaveNote(realnote);
}
});
break;
case RemoteUploadResult.Merged:
dispatcher.Invoke(() =>
{
realnote.ApplyUpdatedData(clonenote);
realnote.TriggerOnChanged(true);
realnote.SetLocalDirty("Note was uploaded and a merge has happened");
realnote.ResetRemoteDirty("Note was successfully uploaded (auto-merge)");
});
break;
case RemoteUploadResult.Conflict:
_log.Warn("Sync", "Uploading note " + clonenote.UniqueName + " resulted in conflict");
ResolveUploadConflict(realnote, clonenote, conflictnote);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception e)
{
var message = string.Format("Could not upload note '{2}' ({0}) cause of {1}", clonenote.UniqueName, e.Message, clonenote.Title);
_log.Error("Sync", message, e);
errors.Add(Tuple.Create(message, e));
}
}
foreach (var notetuple in notesToResetRemoteDirty)
{
var realnote = notetuple.Item1;
var clonenote = notetuple.Item2;
_log.Info("Sync", string.Format("Reset remote dirty of note {0} (no upload needed)", clonenote.UniqueName));
try
{
if (!realnote.IsLocalSaved)
{
dispatcher.Invoke(() =>
{
if (!realnote.IsLocalSaved)
{
repo.SaveNote(realnote);
}
});
}
dispatcher.Invoke(() =>
{
if (realnote.IsLocalSaved)
{
realnote.ResetRemoteDirty("Reset remote dirty - was marked for upload but plugin says no upload is needed");
repo.SaveNote(realnote);
}
});
}
catch (Exception e)
{
var message = string.Format("Could not reset remote dirty note '{2}' ({0}) cause of {1}", clonenote.UniqueName, e.Message, clonenote.Title);
_log.Error("Sync", message, e);
errors.Add(Tuple.Create(message, e));
}
}
}
