private async Task<SynchronizationReport> PerformSynchronizationAsync(SynchronizationClient destination, SynchronizationWorkItem work) { Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}", work.GetType().Name, work.FileName, destination.FileSystemUrl); if (!CanSynchronizeTo(destination.FileSystemUrl)) { Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destination.FileSystemUrl, work.FileName); synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work); return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = new SynchronizationException(string.Format( "The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destination.FileSystemUrl, work.FileName)) }; } string fileName = work.FileName; synchronizationQueue.SynchronizationStarted(work, destination.FileSystemUrl); publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destination.FileSystemUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Start, SynchronizationDirection = SynchronizationDirection.Outgoing }); SynchronizationReport report; try { report = await work.PerformAsync(destination); } catch (Exception ex) { report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = ex, }; } var synchronizationCancelled = false; if (report.Exception == null) { var moreDetails = string.Empty; if (work.SynchronizationType == SynchronizationType.ContentUpdate) { moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}", report.BytesTransfered, report.BytesCopied, report.NeedListLength); } UpdateSuccessfulSynchronizationTime(); Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destination.FileSystemUrl, moreDetails); } else { if (work.IsCancelled || report.Exception is TaskCanceledException) { synchronizationCancelled = true; Log.DebugException(string.Format("{0} to {1} was cancelled", work, destination.FileSystemUrl), report.Exception); } else { Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destination.FileSystemUrl), report.Exception); } } Queue.SynchronizationFinished(work, destination.FileSystemUrl); if (!synchronizationCancelled) CreateSyncingConfiguration(fileName, work.FileETag, destination.FileSystemUrl, work.SynchronizationType); publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destination.FileSystemUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Finish, SynchronizationDirection = SynchronizationDirection.Outgoing }); return report; }
private async Task <SynchronizationReport> PerformSynchronizationAsync(IAsyncFilesSynchronizationCommands destination, SynchronizationWorkItem work) { var commands = (IAsyncFilesCommandsImpl)destination.Commands; string destinationUrl = commands.UrlFor(); Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}", work.GetType().Name, work.FileName, destinationUrl); if (!CanSynchronizeTo(destinationUrl)) { Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destinationUrl, work.FileName); if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) { publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Enqueue, SynchronizationDirection = SynchronizationDirection.Outgoing }); } return(new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = new SynchronizationException(string.Format( "The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destinationUrl, work.FileName)) }); } string fileName = work.FileName; synchronizationQueue.SynchronizationStarted(work, destinationUrl); publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Start, SynchronizationDirection = SynchronizationDirection.Outgoing }); SynchronizationReport report; try { report = await work.PerformAsync(destination); } catch (Exception ex) { report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = ex, }; } var synchronizationCancelled = false; if (report.Exception == null) { var moreDetails = string.Empty; if (work.SynchronizationType == SynchronizationType.ContentUpdate) { moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}", report.BytesTransfered, report.BytesCopied, report.NeedListLength); } UpdateSuccessfulSynchronizationTime(); Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destinationUrl, moreDetails); } else { if (work.IsCancelled || report.Exception is TaskCanceledException) { synchronizationCancelled = true; Log.DebugException(string.Format("{0} to {1} was cancelled", work, destinationUrl), report.Exception); } else { Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception); } } Queue.SynchronizationFinished(work, destinationUrl); if (!synchronizationCancelled) { CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType); } publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Finish, SynchronizationDirection = SynchronizationDirection.Outgoing }); return(report); }
public bool EnqueueSynchronization(string destinationFileSystemUrl, SynchronizationWorkItem workItem) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()) .EnterUpgradeableReadLock(); try { var pendingForDestination = pendingSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentQueue<SynchronizationWorkItem>()); // if delete work is enqueued and there are other synchronization works for a given file then remove them from a queue if (workItem.SynchronizationType == SynchronizationType.Delete && pendingForDestination.Any(x => x.FileName == workItem.FileName && x.SynchronizationType != SynchronizationType.Delete)) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).EnterWriteLock(); try { var modifiedQueue = new ConcurrentQueue<SynchronizationWorkItem>(); foreach (var pendingWork in pendingForDestination) { if (pendingWork.FileName != workItem.FileName) modifiedQueue.Enqueue(pendingWork); } modifiedQueue.Enqueue(workItem); pendingForDestination = pendingSynchronizations.AddOrUpdate(destinationFileSystemUrl, modifiedQueue, (key, value) => modifiedQueue); } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitWriteLock(); } } foreach (var pendingWork in pendingForDestination) { // if there is a file in pending synchronizations do not add it again if (pendingWork.Equals(workItem)) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in a pending queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return false; } // if there is a work for a file of the same type but with lower file ETag just refresh existing work metadata and do not enqueue again if (pendingWork.FileName == workItem.FileName && pendingWork.SynchronizationType == workItem.SynchronizationType && Buffers.Compare(workItem.FileETag.ToByteArray(), pendingWork.FileETag.ToByteArray()) > 0) { pendingWork.RefreshMetadata(); Log.Debug( "{0} for a file {1} and a destination {2} was already existed in a pending queue but with older ETag, it's metadata has been refreshed", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return false; } } var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentDictionary<string, SynchronizationWorkItem> ()); // if there is a work in an active synchronizations do not add it again if (activeForDestination.ContainsKey(workItem.FileName) && activeForDestination[workItem.FileName].Equals(workItem)) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in an active queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return false; } pendingForDestination.Enqueue(workItem); Log.Debug("{0} for a file {1} and a destination {2} was enqueued", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitUpgradeableReadLock(); } return true; }
public bool EnqueueSynchronization(string destinationFileSystemUrl, SynchronizationWorkItem workItem) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()) .EnterUpgradeableReadLock(); try { var pendingForDestination = pendingSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentQueue <SynchronizationWorkItem>()); // if delete work is enqueued and there are other synchronization works for a given file then remove them from a queue if (workItem.SynchronizationType == SynchronizationType.Delete && pendingForDestination.Any(x => x.FileName == workItem.FileName && x.SynchronizationType != SynchronizationType.Delete)) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).EnterWriteLock(); try { var modifiedQueue = new ConcurrentQueue <SynchronizationWorkItem>(); foreach (var pendingWork in pendingForDestination) { if (pendingWork.FileName != workItem.FileName) { modifiedQueue.Enqueue(pendingWork); } } modifiedQueue.Enqueue(workItem); pendingForDestination = pendingSynchronizations.AddOrUpdate(destinationFileSystemUrl, modifiedQueue, (key, value) => modifiedQueue); } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitWriteLock(); } } foreach (var pendingWork in pendingForDestination) { // if there is a file in pending synchronizations do not add it again if (pendingWork.Equals(workItem)) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in a pending queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return(false); } // if there is a work for a file of the same type but with lower file ETag just refresh existing work metadata and do not enqueue again if (pendingWork.FileName == workItem.FileName && pendingWork.SynchronizationType == workItem.SynchronizationType && Buffers.Compare(workItem.FileETag.ToByteArray(), pendingWork.FileETag.ToByteArray()) > 0) { pendingWork.RefreshMetadata(); Log.Debug( "{0} for a file {1} and a destination {2} was already existed in a pending queue but with older ETag, it's metadata has been refreshed", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return(false); } } var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentDictionary <string, SynchronizationWorkItem> ()); // if there is a work in an active synchronizations do not add it again if (activeForDestination.ContainsKey(workItem.FileName) && activeForDestination[workItem.FileName].Equals(workItem)) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in an active queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return(false); } pendingForDestination.Enqueue(workItem); Log.Debug("{0} for a file {1} and a destination {2} was enqueued", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitUpgradeableReadLock(); } return(true); }