public async Task <SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination) { ICredentials credentials = null; if (string.IsNullOrEmpty(destination.Username) == false) { credentials = string.IsNullOrEmpty(destination.Domain) ? new NetworkCredential(destination.Username, destination.Password) : new NetworkCredential(destination.Username, destination.Password, destination.Domain); } var conventions = new FilesConvention(); if (string.IsNullOrEmpty(destination.AuthenticationScheme) == false) { conventions.AuthenticationScheme = destination.AuthenticationScheme; } var destinationClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, convention: conventions, apiKey: destination.ApiKey, credentials: credentials); RavenJObject destinationMetadata; try { destinationMetadata = await destinationClient.GetMetadataForAsync(fileName).ConfigureAwait(false); } catch (Exception ex) { var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.Url; Log.WarnException(exceptionMessage, ex); return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(exceptionMessage, ex) }); } RavenJObject localMetadata = GetLocalMetadata(fileName); NoSyncReason reason; SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason); if (work == null) { Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.Url, reason.GetDescription()); return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown) { Exception = new SynchronizationException(reason.GetDescription()) }); } return(await PerformSynchronizationAsync(destinationClient, work).ConfigureAwait(false)); }
public void SynchronizationStarted(SynchronizationWorkItem work, string destinationFileSystemUrl) { var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentDictionary <string, SynchronizationWorkItem>()); if (activeForDestination.TryAdd(work.FileName, work)) { Log.Debug("File '{0}' with ETag {1} was added to an active synchronization queue for a destination {2}", work.FileName, work.FileETag, destinationFileSystemUrl); } }
public bool IsDifferentWorkForTheSameFileBeingPerformed(SynchronizationWorkItem work, string destinationFileSystemUrl) { ConcurrentDictionary <string, SynchronizationWorkItem> activeForDestination; if (!activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeForDestination)) { return(false); } SynchronizationWorkItem activeWork; return(activeForDestination.TryGetValue(work.FileName, out activeWork) && !activeWork.Equals(work)); }
public void SynchronizationFinished(SynchronizationWorkItem work, string destinationFileSystemUrl) { ConcurrentDictionary <string, SynchronizationWorkItem> activeDestinationTasks; if (activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeDestinationTasks) == false) { Log.Warn("Could not get an active synchronization queue for {0}", destinationFileSystemUrl); return; } SynchronizationWorkItem removingItem; if (activeDestinationTasks.TryRemove(work.FileName, out removingItem)) { Log.Debug("File '{0}' with ETag {1} was removed from an active synchronization queue for a destination {2}", work.FileName, work.FileETag, destinationFileSystemUrl); } }
public bool TryDequePending(string destinationFileSystemUrl, out SynchronizationWorkItem workItem) { var readerWriterLockSlim = pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()); readerWriterLockSlim.EnterReadLock(); try { ConcurrentQueue <SynchronizationWorkItem> pendingForDestination; if (pendingSynchronizations.TryGetValue(destinationFileSystemUrl, out pendingForDestination) == false) { workItem = null; return(false); } return(pendingForDestination.TryDequeue(out workItem)); } finally { readerWriterLockSlim.ExitReadLock(); } }
public bool EnqueueSynchronization(string destinationFileSystemUrl, SynchronizationWorkItem workItem) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()) .EnterUpgradeableReadLock(); try { var pendingForDestination = pendingSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentQueue<SynchronizationWorkItem>()); // if delete work is enqueued and there are other synchronization works for a given file then remove them from a queue if (workItem.SynchronizationType == SynchronizationType.Delete && pendingForDestination.Any(x => x.FileName == workItem.FileName && x.SynchronizationType != SynchronizationType.Delete)) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).EnterWriteLock(); try { var modifiedQueue = new ConcurrentQueue<SynchronizationWorkItem>(); foreach (var pendingWork in pendingForDestination) { if (pendingWork.FileName != workItem.FileName) modifiedQueue.Enqueue(pendingWork); } modifiedQueue.Enqueue(workItem); pendingForDestination = pendingSynchronizations.AddOrUpdate(destinationFileSystemUrl, modifiedQueue, (key, value) => modifiedQueue); } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitWriteLock(); } } foreach (var pendingWork in pendingForDestination) { // if there is a file in pending synchronizations do not add it again if (pendingWork.Equals(workItem)) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in a pending queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return false; } // if there is a work for a file of the same type but with lower file ETag just refresh existing work metadata and do not enqueue again if (pendingWork.FileName == workItem.FileName && pendingWork.SynchronizationType == workItem.SynchronizationType && Buffers.Compare(workItem.FileETag.ToByteArray(), pendingWork.FileETag.ToByteArray()) > 0) { pendingWork.RefreshMetadata(); Log.Debug( "{0} for a file {1} and a destination {2} was already existed in a pending queue but with older ETag, it's metadata has been refreshed", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return false; } } var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentDictionary<string, SynchronizationWorkItem> ()); // if there is a work in an active synchronizations do not add it again if (activeForDestination.ContainsKey(workItem.FileName) && activeForDestination[workItem.FileName].Equals(workItem)) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in an active queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); return false; } pendingForDestination.Enqueue(workItem); Log.Debug("{0} for a file {1} and a destination {2} was enqueued", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitUpgradeableReadLock(); } return true; }
public void SynchronizationFinished(SynchronizationWorkItem work, string destinationFileSystemUrl) { ConcurrentDictionary<string, SynchronizationWorkItem> activeDestinationTasks; if (activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeDestinationTasks) == false) { Log.Warn("Could not get an active synchronization queue for {0}", destinationFileSystemUrl); return; } SynchronizationWorkItem removingItem; if (activeDestinationTasks.TryRemove(work.FileName, out removingItem)) { Log.Debug("File '{0}' with ETag {1} was removed from an active synchronization queue for a destination {2}", work.FileName, work.FileETag, destinationFileSystemUrl); } }
public void SynchronizationStarted(SynchronizationWorkItem work, string destinationFileSystemUrl) { var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentDictionary<string, SynchronizationWorkItem>()); if (activeForDestination.TryAdd(work.FileName, work)) { Log.Debug("File '{0}' with ETag {1} was added to an active synchronization queue for a destination {2}", work.FileName, work.FileETag, destinationFileSystemUrl); } }
public bool IsDifferentWorkForTheSameFileBeingPerformed(SynchronizationWorkItem work, string destinationFileSystemUrl) { ConcurrentDictionary<string, SynchronizationWorkItem> activeForDestination; if (!activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeForDestination)) return false; SynchronizationWorkItem activeWork; return activeForDestination.TryGetValue(work.FileName, out activeWork) && !activeWork.Equals(work); }
public bool TryDequePending(string destinationFileSystemUrl, out SynchronizationWorkItem workItem) { var readerWriterLockSlim = pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()); readerWriterLockSlim.EnterReadLock(); try { ConcurrentQueue<SynchronizationWorkItem> pendingForDestination; if (pendingSynchronizations.TryGetValue(destinationFileSystemUrl, out pendingForDestination) == false) { workItem = null; return false; } return pendingForDestination.TryDequeue(out workItem); } finally { readerWriterLockSlim.ExitReadLock(); } }
public bool EnqueueSynchronization(string destinationFileSystemUrl, SynchronizationWorkItem workItem) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()) .EnterUpgradeableReadLock(); try { var pendingForDestination = pendingSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentQueue <SynchronizationWorkItem>()); // if delete work is enqueued and there are other synchronization works for a given file then remove them from a queue if (workItem.SynchronizationType == SynchronizationType.Delete && pendingForDestination.Any(x => x.FileName == workItem.FileName && x.SynchronizationType != SynchronizationType.Delete)) { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).EnterWriteLock(); try { var modifiedQueue = new ConcurrentQueue <SynchronizationWorkItem>(); foreach (var pendingWork in pendingForDestination) { if (pendingWork.FileName != workItem.FileName) { modifiedQueue.Enqueue(pendingWork); } } modifiedQueue.Enqueue(workItem); pendingForDestination = pendingSynchronizations.AddOrUpdate(destinationFileSystemUrl, modifiedQueue, (key, value) => modifiedQueue); } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitWriteLock(); } } foreach (var pendingWork in pendingForDestination) { // if there is a file in pending synchronizations do not add it again if (pendingWork.Equals(workItem)) { if (Log.IsDebugEnabled) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in a pending queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); } return(false); } // if there is a work for a file of the same type but with lower file ETag just refresh existing work metadata and do not enqueue again if (pendingWork.FileName == workItem.FileName && pendingWork.SynchronizationType == workItem.SynchronizationType && Buffers.Compare(workItem.FileETag.ToByteArray(), pendingWork.FileETag.ToByteArray()) > 0) { pendingWork.RefreshMetadata(); if (Log.IsDebugEnabled) { Log.Debug( "{0} for a file {1} and a destination {2} was already existed in a pending queue but with older ETag, it's metadata has been refreshed", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); } return(false); } } var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl, new ConcurrentDictionary <string, SynchronizationWorkItem> ()); // if there is a work in an active synchronizations do not add it again if (activeForDestination.ContainsKey(workItem.FileName) && activeForDestination[workItem.FileName].Equals(workItem)) { if (Log.IsDebugEnabled) { Log.Debug("{0} for a file {1} and a destination {2} was already existed in an active queue", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); } return(false); } pendingForDestination.Enqueue(workItem); if (Log.IsDebugEnabled) { Log.Debug("{0} for a file {1} and a destination {2} was enqueued", workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl); } } finally { pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitUpgradeableReadLock(); } return(true); }
private async Task<SynchronizationReport> PerformSynchronizationAsync(ISynchronizationServerClient synchronizationServerClient, SynchronizationWorkItem work) { var destinationUrl = synchronizationServerClient.BaseUrl; Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}", work.GetType().Name, work.FileName, destinationUrl); if (AvailableSynchronizationRequestsTo(destinationUrl) <= 0) { Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destinationUrl, work.FileName); if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) { publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Enqueue, Direction = SynchronizationDirection.Outgoing }); } return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = new SynchronizationException(string.Format( "The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destinationUrl, work.FileName)) }; } string fileName = work.FileName; synchronizationQueue.SynchronizationStarted(work, destinationUrl); publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Start, Direction = SynchronizationDirection.Outgoing }); SynchronizationReport report; try { report = await work.PerformAsync(synchronizationServerClient).ConfigureAwait(false); } catch (Exception ex) { report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = ex, }; } var synchronizationCancelled = false; if (report.Exception == null) { var moreDetails = string.Empty; if (work.SynchronizationType == SynchronizationType.ContentUpdate) { moreDetails = string.Format(". {0} bytes were transferred and {1} bytes copied. Need list length was {2}", report.BytesTransfered, report.BytesCopied, report.NeedListLength); } context.UpdateSuccessfulSynchronizationTime(); Log.Debug("{0} to {1} has finished successfully{2}", work, destinationUrl, moreDetails); } else { if (work.IsCancelled || report.Exception is TaskCanceledException) { synchronizationCancelled = true; Log.DebugException(string.Format("{0} to {1} was canceled", work, destinationUrl), report.Exception); } else { Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception); } } Queue.SynchronizationFinished(work, destinationUrl); if (synchronizationCancelled == false) CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType); publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Finish, Direction = SynchronizationDirection.Outgoing }); return report; }
private async Task <SynchronizationReport> PerformSynchronizationAsync(ISynchronizationServerClient synchronizationServerClient, SynchronizationWorkItem work) { var destinationUrl = synchronizationServerClient.BaseUrl; if (Log.IsDebugEnabled) { Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}", work.GetType().Name, work.FileName, destinationUrl); } if (AvailableSynchronizationRequestsTo(destinationUrl) <= 0) { if (Log.IsDebugEnabled) { Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destinationUrl, work.FileName); } if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) { publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Enqueue, Direction = SynchronizationDirection.Outgoing }); } return(new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = new SynchronizationException(string.Format( "The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destinationUrl, work.FileName)) }); } string fileName = work.FileName; synchronizationQueue.SynchronizationStarted(work, destinationUrl); publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Start, Direction = SynchronizationDirection.Outgoing }); SynchronizationReport report; try { report = await work.PerformAsync(synchronizationServerClient).ConfigureAwait(false); } catch (Exception ex) { report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType) { Exception = ex, }; } var synchronizationCancelled = false; if (report.Exception == null) { var moreDetails = string.Empty; if (work.SynchronizationType == SynchronizationType.ContentUpdate) { moreDetails = string.Format(". {0} bytes were transferred and {1} bytes copied. Need list length was {2}", report.BytesTransfered, report.BytesCopied, report.NeedListLength); } context.UpdateSuccessfulSynchronizationTime(); if (Log.IsDebugEnabled) { Log.Debug("{0} to {1} has finished successfully{2}", work, destinationUrl, moreDetails); } } else { if (work.IsCancelled || report.Exception is TaskCanceledException) { synchronizationCancelled = true; if (Log.IsDebugEnabled) { Log.DebugException(string.Format("{0} to {1} was canceled", work, destinationUrl), report.Exception); } } else { Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception); } } Queue.SynchronizationFinished(work, destinationUrl); if (synchronizationCancelled == false) { CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType); } publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Finish, Direction = SynchronizationDirection.Outgoing }); return(report); }