public static SynchronizationDestination ToSynchronizationDestination(this IAsyncFilesCommands self) { var selfImpl = (IAsyncFilesCommandsImpl)self; var result = new SynchronizationDestination { FileSystem = self.FileSystemName, ServerUrl = selfImpl.ServerUrl, }; if (self.PrimaryCredentials != null) { var networkCredential = self.PrimaryCredentials.Credentials as NetworkCredential; if (networkCredential != null) { result.Username = networkCredential.UserName; result.Password = networkCredential.Password; result.Domain = networkCredential.Domain; } else { throw new InvalidOperationException("Expected NetworkCredential object while get: " + self.PrimaryCredentials.Credentials); } result.ApiKey = self.PrimaryCredentials.ApiKey; } return result; }
private IEnumerable<SynchronizationDetails> GetSyncingConfigurations(SynchronizationDestination destination) { var configObjects = new List<SynchronizationDetails>(); try { storage.Batch( accessor => { configObjects = accessor.GetConfigsStartWithPrefix(RavenFileNameHelper.SyncNameForFile(string.Empty, destination.Url), 0, 100) .Select(config => config.JsonDeserialization<SynchronizationDetails>()) .ToList(); }); } catch (Exception e) { Log.WarnException(string.Format("Could not get syncing configurations for a destination {0}", destination), e); } return configObjects; }
private async Task<IEnumerable<Task<SynchronizationReport>>> SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingAll) { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var destinationSyncClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, destination.ApiKey, credentials); bool repeat; do { var lastETag = await destinationSyncClient.GetLastSynchronizationFromAsync(storage.Id).ConfigureAwait(false); var activeTasks = synchronizationQueue.Active; var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList(); var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationSyncClient).ConfigureAwait(false); var needSyncingAgain = new List<FileHeader>(); foreach (var confirmation in confirmations) { if (confirmation.Status == FileStatus.Safe) { Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName); RemoveSyncingConfiguration(confirmation.FileName, destination.Url); } else { storage.Batch(accessor => { var fileHeader = accessor.ReadFile(confirmation.FileName); if (fileHeader != null) { needSyncingAgain.Add(fileHeader); Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status); } }); } } if (synchronizationQueue.NumberOfPendingSynchronizationsFor(destination.Url) < AvailableSynchronizationRequestsTo(destination.Url)) { repeat = await EnqueueMissingUpdatesAsync(destinationSyncClient, lastETag, needSyncingAgain).ConfigureAwait(false) == false; } else repeat = false; } while (repeat); return SynchronizePendingFilesAsync(destinationSyncClient, forceSyncingAll); }
public async Task<SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination) { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var conventions = new FilesConvention(); if (string.IsNullOrEmpty(destination.AuthenticationScheme) == false) conventions.AuthenticationScheme = destination.AuthenticationScheme; var destinationClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, convention: conventions, apiKey: destination.ApiKey, credentials: credentials); RavenJObject destinationMetadata; try { destinationMetadata = await destinationClient.GetMetadataForAsync(fileName).ConfigureAwait(false); } catch (Exception ex) { var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.Url; Log.WarnException(exceptionMessage, ex); return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(exceptionMessage, ex) }; } RavenJObject localMetadata = GetLocalMetadata(fileName); NoSyncReason reason; SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason); if (work == null) { Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.Url, reason.GetDescription()); return new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(reason.GetDescription()) }; } return await PerformSynchronizationAsync(destinationClient, work).ConfigureAwait(false); }
public async Task<DestinationSyncResult> CreateDestinationResult(SynchronizationDestination destination, IEnumerable<Task<SynchronizationReport>> synchronizations) { try { var reports = await Task.WhenAll(synchronizations).ConfigureAwait(false); var destinationSyncResult = new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem }; if (reports.Length > 0) { var successfulSynchronizationsCount = reports.Count(x => x.Exception == null); var failedSynchronizationsCount = reports.Count(x => x.Exception != null); if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0) { Log.Debug( "Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed", destination.Url, successfulSynchronizationsCount, failedSynchronizationsCount); } destinationSyncResult.Reports = reports; } return destinationSyncResult; } catch (Exception ex) { Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex); return new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem, Exception = ex }; } }
protected bool Equals(SynchronizationDestination other) { return(string.Equals(serverUrl, other.serverUrl) && string.Equals(ApiKey, other.ApiKey) && string.Equals(Domain, other.Domain) && string.Equals(Password, other.Password) && string.Equals(Username, other.Username) && string.Equals(FileSystem, other.FileSystem)); }
private IEnumerable<Task<SynchronizationReport>> SynchronizePendingFilesAsync(SynchronizationDestination destination, IAsyncFilesSynchronizationCommands destinationCommands, bool forceSyncingContinuation) { var commands = (IAsyncFilesCommandsImpl)destinationCommands.Commands; var destinationUrl = commands.UrlFor(); for (var i = 0; i < AvailableSynchronizationRequestsTo(destinationUrl); i++) { SynchronizationWorkItem work; if (!synchronizationQueue.TryDequePendingSynchronization(destinationUrl, out work)) break; if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destinationUrl)) { Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName, destinationCommands); if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) // add it again at the end of the queue { // add it again at the end of the queue publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Enqueue, Direction = SynchronizationDirection.Outgoing }); } } else { var workTask = PerformSynchronizationAsync(destinationCommands, work); if (forceSyncingContinuation) workTask.ContinueWith(async t => { if (CanSynchronizeTo(destinationUrl)) await SynchronizeDestinationAsync(destination, true); }); yield return workTask; } } }
private async Task<DestinationSyncResult> SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingContinuation) { try { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var destinationClient = new AsyncFilesServerClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization; var lastETag = await destinationClient.GetLastSynchronizationFromAsync(storage.Id); var activeTasks = synchronizationQueue.Active; var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList(); var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, destinationClient); var needSyncingAgain = new List<FileHeader>(); foreach (var confirmation in confirmations) { if (confirmation.Status == FileStatus.Safe) { Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName); RemoveSyncingConfiguration(confirmation.FileName, destination.Url); } else { storage.Batch(accessor => { var fileHeader = accessor.ReadFile(confirmation.FileName); if (fileHeader != null) { needSyncingAgain.Add(fileHeader); Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status); } }); } } await EnqueueMissingUpdatesAsync(destinationClient, lastETag, needSyncingAgain); var reports = await Task.WhenAll(SynchronizePendingFilesAsync(destination, destinationClient, forceSyncingContinuation)); var destinationSyncResult = new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem }; if (reports.Length > 0) { var successfulSynchronizationsCount = reports.Count(x => x.Exception == null); var failedSynchronizationsCount = reports.Count(x => x.Exception != null); if (successfulSynchronizationsCount > 0 || failedSynchronizationsCount > 0) { Log.Debug( "Synchronization to a destination {0} has completed. {1} file(s) were synchronized successfully, {2} synchronization(s) were failed", destination.Url, successfulSynchronizationsCount, failedSynchronizationsCount); } destinationSyncResult.Reports = reports; } return destinationSyncResult; } catch (Exception ex) { Log.WarnException(string.Format("Failed to perform a synchronization to a destination {0}", destination), ex); return new DestinationSyncResult { DestinationServer = destination.ServerUrl, DestinationFileSystem = destination.FileSystem, Exception = ex }; } }
public async Task MultipleConflictListeners_ConflictNotResolved() { var store = this.NewStore(1); var anotherStore = this.NewStore(2); var takeLocalConflictListener = new TakeLocalConflictListener(); var noOpListener = new NoOpConflictListener(); anotherStore.Listeners.RegisterListener(noOpListener); anotherStore.Listeners.RegisterListener(takeLocalConflictListener); using (var sessionDestination1 = store.OpenAsyncSession()) using (var sessionDestination2 = anotherStore.OpenAsyncSession()) { sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130)); await sessionDestination2.SaveChangesAsync(); sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128)); await sessionDestination1.SaveChangesAsync(); var notificationTask = WaitForConflictDetected(anotherStore, 1, 10); var syncDestinatios = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() }; await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinatios); await sessionDestination1.Commands.Synchronization.StartAsync(); await notificationTask; Assert.Equal(1, noOpListener.DetectedCount); Assert.Equal(1, takeLocalConflictListener.DetectedCount); Assert.Equal(0, takeLocalConflictListener.ResolvedCount + noOpListener.ResolvedCount); // try to change content of file in destination2 sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(140)); // Assert an exception is thrown because the conflict is still there var aggregateException = Assert.Throws<AggregateException>(() => sessionDestination2.SaveChangesAsync().Wait()); Assert.IsType<NotSupportedException>(aggregateException.InnerException); // try to change content of file in destination2 sessionDestination2.RegisterRename("test1.file", "test2.file"); // Assert an exception is thrown because the conflict is still there aggregateException = Assert.Throws<AggregateException>(() => sessionDestination2.SaveChangesAsync().Wait()); Assert.IsType<NotSupportedException>(aggregateException.InnerException); } }
public async Task MultipleConflictListeners_MultipleResolutionListeners() { var store = this.NewStore(1); var anotherStore = this.NewStore(2); var conflictsListener = new TakeLocalConflictListener(); var noOpListener = new NoOpConflictListener(); anotherStore.Listeners.RegisterListener(noOpListener); anotherStore.Listeners.RegisterListener(conflictsListener); using (var sessionDestination1 = store.OpenAsyncSession()) using (var sessionDestination2 = anotherStore.OpenAsyncSession()) { sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130)); await sessionDestination2.SaveChangesAsync(); sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128)); await sessionDestination1.SaveChangesAsync(); var notificationTask = await WaitForConflictResolved(anotherStore, 1, 5); var syncDestinatios = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() }; await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinatios); await sessionDestination1.Commands.Synchronization.StartAsync(); await notificationTask; Assert.Equal(1, conflictsListener.DetectedCount); Assert.Equal(1, conflictsListener.ResolvedCount); Assert.Equal(1, noOpListener.DetectedCount); Assert.Equal(1, noOpListener.ResolvedCount); } }
public async Task ConflictListeners_RemoteVersion() { var filename = FileHeader.Canonize("test1.file"); int firstStreamSize = 130; int secondStreamSize = 128; var store = this.NewStore(1); var anotherStore = this.NewStore(2); var conflictsListener = new TakeNewestConflictsListener(); anotherStore.Listeners.RegisterListener(conflictsListener); using (var sessionDestination1 = store.OpenAsyncSession()) using (var sessionDestination2 = anotherStore.OpenAsyncSession()) { sessionDestination2.RegisterUpload(filename, CreateUniformFileStream(firstStreamSize)); await sessionDestination2.SaveChangesAsync(); await sessionDestination1.Commands.Synchronization.StartAsync(); sessionDestination1.RegisterUpload(filename, CreateUniformFileStream(secondStreamSize)); await sessionDestination1.SaveChangesAsync(); var file = await sessionDestination1.LoadFileAsync(filename); var file2 = await sessionDestination2.LoadFileAsync(filename); Assert.Equal(secondStreamSize, file.TotalSize); Assert.Equal(firstStreamSize, file2.TotalSize); var notificationTask = await WaitForConflictResolved(anotherStore, 1, 10); var syncDestinations = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() }; await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinations); await sessionDestination1.Commands.Synchronization.StartAsync(); //We need to sync again after conflict resolution because strategy was to resolve with remote await sessionDestination1.Commands.Synchronization.StartAsync(); await notificationTask; Assert.Equal(1, conflictsListener.DetectedCount); Assert.Equal(1, conflictsListener.ResolvedCount); file = await sessionDestination1.LoadFileAsync(filename); file2 = await sessionDestination2.LoadFileAsync(filename); Assert.Equal(secondStreamSize, file.TotalSize); Assert.Equal(secondStreamSize, file2.TotalSize); } }
protected bool Equals(SynchronizationDestination other) { return string.Equals(serverUrl, other.serverUrl) && string.Equals(ApiKey, other.ApiKey) && string.Equals(Domain, other.Domain) && string.Equals(Password, other.Password) && string.Equals(Username, other.Username) && string.Equals(FileSystem, other.FileSystem); }
public async Task ConflictListeners_LocalVersion() { var store = this.NewStore(1); var anotherStore = this.NewStore(2); var conflictsListener = new TakeLocalConflictListener(); anotherStore.Listeners.RegisterListener(conflictsListener); using (var sessionDestination1 = store.OpenAsyncSession()) using (var sessionDestination2 = anotherStore.OpenAsyncSession()) { sessionDestination1.RegisterUpload("test1.file", CreateUniformFileStream(128)); await sessionDestination1.SaveChangesAsync(); sessionDestination2.RegisterUpload("test1.file", CreateUniformFileStream(130)); await sessionDestination2.SaveChangesAsync(); var notificationTask = await WaitForConflictResolved(anotherStore, 1, 10); var syncDestinations = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() }; await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinations); await sessionDestination1.Commands.Synchronization.SynchronizeAsync(); await notificationTask; Assert.Equal(1, conflictsListener.DetectedCount); Assert.Equal(1, conflictsListener.ResolvedCount); var file = await sessionDestination1.LoadFileAsync("test1.file"); var file2 = await sessionDestination2.LoadFileAsync("test1.file"); Assert.Equal(128, file.TotalSize); Assert.Equal(130, file2.TotalSize); } }
public async Task ConflictListeners_RemoteVersion() { var filename = FileHeader.Canonize("test1.file"); int firstStreamSize = 130; int secondStreamSize = 128; var store = this.NewStore(1); var anotherStore = this.NewStore(2); var conflictsListener = new TakeNewestConflictsListener(); anotherStore.Listeners.RegisterListener(conflictsListener); using (var sessionDestination1 = store.OpenAsyncSession()) using (var sessionDestination2 = anotherStore.OpenAsyncSession()) { sessionDestination2.RegisterUpload(filename, CreateUniformFileStream(firstStreamSize)); await sessionDestination2.SaveChangesAsync(); sessionDestination1.RegisterUpload(filename, CreateUniformFileStream(secondStreamSize)); await sessionDestination1.SaveChangesAsync(); var notificationTask = await WaitForConflictResolved(anotherStore, 1, 30); var syncDestinations = new SynchronizationDestination[] { sessionDestination2.Commands.ToSynchronizationDestination() }; await sessionDestination1.Commands.Synchronization.SetDestinationsAsync(syncDestinations); var syncResult = await sessionDestination1.Commands.Synchronization.StartAsync(); Assert.Equal(string.Format("File {0} is conflicted", filename), syncResult[0].Reports.ToList()[0].Exception.Message); // conflict should be resolved by the registered listener Assert.True(SpinWait.SpinUntil(() => conflictsListener.DetectedCount == 1 && conflictsListener.ResolvedCount == 1, TimeSpan.FromMinutes(1)), string.Format("DetectedCount: {0}, ResolvedCount: {1}", conflictsListener.DetectedCount, conflictsListener.ResolvedCount)); // We need to sync again after conflict resolution because the strategy was to resolve with remote await sessionDestination1.Commands.Synchronization.StartAsync(); await notificationTask; Assert.Equal(1, conflictsListener.DetectedCount); Assert.Equal(1, conflictsListener.ResolvedCount); var file = await sessionDestination1.LoadFileAsync(filename); var file2 = await sessionDestination2.LoadFileAsync(filename); Assert.Equal(secondStreamSize, file.TotalSize); Assert.Equal(secondStreamSize, file2.TotalSize); } }