public void SynchronizationStarted(SynchronizationWorkItem work, string destinationFileSystemUrl)
{
var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentDictionary <string, SynchronizationWorkItem>());
if (activeForDestination.TryAdd(work.FileName, work))
{
Log.Debug("File '{0}' with ETag {1} was added to an active synchronization queue for a destination {2}",
work.FileName,
work.FileETag, destinationFileSystemUrl);
}
}
public bool IsDifferentWorkForTheSameFileBeingPerformed(SynchronizationWorkItem work, string destinationFileSystemUrl)
{
ConcurrentDictionary <string, SynchronizationWorkItem> activeForDestination;
if (!activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeForDestination))
{
return(false);
}
SynchronizationWorkItem activeWork;
return(activeForDestination.TryGetValue(work.FileName, out activeWork) && !activeWork.Equals(work));
}
public async Task <SynchronizationReport> SynchronizeFileToAsync(string fileName, SynchronizationDestination destination)
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var destinationClient = new AsyncFilesServerClient(destination.ServerUrl, destination.FileSystem, apiKey: destination.ApiKey, credentials: credentials).Synchronization;
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationClient.Commands.GetMetadataForAsync(fileName);
}
catch (Exception ex)
{
var exceptionMessage = "Could not get metadata details for " + fileName + " from " + destination.FileSystemUrl;
Log.WarnException(exceptionMessage, ex);
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(exceptionMessage, ex)
});
}
RavenJObject localMetadata = GetLocalMetadata(fileName);
NoSyncReason reason;
SynchronizationWorkItem work = synchronizationStrategy.DetermineWork(fileName, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", fileName, destination.FileSystemUrl, reason.GetDescription());
return(new SynchronizationReport(fileName, Guid.Empty, SynchronizationType.Unknown)
{
Exception = new SynchronizationException(reason.GetDescription())
});
}
return(await PerformSynchronizationAsync(destinationClient, work));
}
public void SynchronizationFinished(SynchronizationWorkItem work, string destinationFileSystemUrl)
{
ConcurrentDictionary <string, SynchronizationWorkItem> activeDestinationTasks;
if (activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeDestinationTasks) == false)
{
Log.Warn("Could not get an active synchronization queue for {0}", destinationFileSystemUrl);
return;
}
SynchronizationWorkItem removingItem;
if (activeDestinationTasks.TryRemove(work.FileName, out removingItem))
{
Log.Debug("File '{0}' with ETag {1} was removed from an active synchronization queue for a destination {2}",
work.FileName,
work.FileETag, destinationFileSystemUrl);
}
}
public bool TryDequePendingSynchronization(string destinationFileSystemUrl, out SynchronizationWorkItem workItem)
{
var readerWriterLockSlim = pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim());
readerWriterLockSlim.EnterReadLock();
try
{
ConcurrentQueue <SynchronizationWorkItem> pendingForDestination;
if (pendingSynchronizations.TryGetValue(destinationFileSystemUrl, out pendingForDestination) == false)
{
workItem = null;
return(false);
}
return(pendingForDestination.TryDequeue(out workItem));
}
finally
{
readerWriterLockSlim.ExitReadLock();
}
}
private async Task<SynchronizationReport> PerformSynchronizationAsync(SynchronizationClient destination,
SynchronizationWorkItem work)
{
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
work.GetType().Name, work.FileName, destination.FileSystemUrl);
if (!CanSynchronizeTo(destination.FileSystemUrl))
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName);
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work);
return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName))
};
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destination.FileSystemUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(destination);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
UpdateSuccessfulSynchronizationTime();
Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destination.FileSystemUrl, moreDetails);
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
Log.DebugException(string.Format("{0} to {1} was cancelled", work, destination.FileSystemUrl), report.Exception);
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destination.FileSystemUrl),
report.Exception);
}
}
Queue.SynchronizationFinished(work, destination.FileSystemUrl);
if (!synchronizationCancelled)
CreateSyncingConfiguration(fileName, work.FileETag, destination.FileSystemUrl, work.SynchronizationType);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
return report;
}
public bool EnqueueSynchronization(string destinationFileSystemUrl, SynchronizationWorkItem workItem)
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim())
.EnterUpgradeableReadLock();
try
{
var pendingForDestination = pendingSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentQueue<SynchronizationWorkItem>());
// if delete work is enqueued and there are other synchronization works for a given file then remove them from a queue
if (workItem.SynchronizationType == SynchronizationType.Delete &&
pendingForDestination.Any(x => x.FileName == workItem.FileName && x.SynchronizationType != SynchronizationType.Delete))
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).EnterWriteLock();
try
{
var modifiedQueue = new ConcurrentQueue<SynchronizationWorkItem>();
foreach (var pendingWork in pendingForDestination)
{
if (pendingWork.FileName != workItem.FileName)
modifiedQueue.Enqueue(pendingWork);
}
modifiedQueue.Enqueue(workItem);
pendingForDestination = pendingSynchronizations.AddOrUpdate(destinationFileSystemUrl, modifiedQueue,
(key, value) => modifiedQueue);
}
finally
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitWriteLock();
}
}
foreach (var pendingWork in pendingForDestination)
{
// if there is a file in pending synchronizations do not add it again
if (pendingWork.Equals(workItem))
{
Log.Debug("{0} for a file {1} and a destination {2} was already existed in a pending queue",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
return false;
}
// if there is a work for a file of the same type but with lower file ETag just refresh existing work metadata and do not enqueue again
if (pendingWork.FileName == workItem.FileName &&
pendingWork.SynchronizationType == workItem.SynchronizationType &&
Buffers.Compare(workItem.FileETag.ToByteArray(), pendingWork.FileETag.ToByteArray()) > 0)
{
pendingWork.RefreshMetadata();
Log.Debug(
"{0} for a file {1} and a destination {2} was already existed in a pending queue but with older ETag, it's metadata has been refreshed",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
return false;
}
}
var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentDictionary<string, SynchronizationWorkItem>
());
// if there is a work in an active synchronizations do not add it again
if (activeForDestination.ContainsKey(workItem.FileName) && activeForDestination[workItem.FileName].Equals(workItem))
{
Log.Debug("{0} for a file {1} and a destination {2} was already existed in an active queue",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
return false;
}
pendingForDestination.Enqueue(workItem);
Log.Debug("{0} for a file {1} and a destination {2} was enqueued", workItem.GetType().Name, workItem.FileName,
destinationFileSystemUrl);
}
finally
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitUpgradeableReadLock();
}
return true;
}
public void SynchronizationFinished(SynchronizationWorkItem work, string destinationFileSystemUrl)
{
ConcurrentDictionary<string, SynchronizationWorkItem> activeDestinationTasks;
if (activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeDestinationTasks) == false)
{
Log.Warn("Could not get an active synchronization queue for {0}", destinationFileSystemUrl);
return;
}
SynchronizationWorkItem removingItem;
if (activeDestinationTasks.TryRemove(work.FileName, out removingItem))
{
Log.Debug("File '{0}' with ETag {1} was removed from an active synchronization queue for a destination {2}",
work.FileName,
work.FileETag, destinationFileSystemUrl);
}
}
public void SynchronizationStarted(SynchronizationWorkItem work, string destinationFileSystemUrl)
{
var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentDictionary<string, SynchronizationWorkItem>());
if (activeForDestination.TryAdd(work.FileName, work))
{
Log.Debug("File '{0}' with ETag {1} was added to an active synchronization queue for a destination {2}",
work.FileName,
work.FileETag, destinationFileSystemUrl);
}
}
public bool IsDifferentWorkForTheSameFileBeingPerformed(SynchronizationWorkItem work, string destinationFileSystemUrl)
{
ConcurrentDictionary<string, SynchronizationWorkItem> activeForDestination;
if (!activeSynchronizations.TryGetValue(destinationFileSystemUrl, out activeForDestination))
return false;
SynchronizationWorkItem activeWork;
return activeForDestination.TryGetValue(work.FileName, out activeWork) && !activeWork.Equals(work);
}
public bool TryDequePendingSynchronization(string destinationFileSystemUrl, out SynchronizationWorkItem workItem)
{
var readerWriterLockSlim = pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim());
readerWriterLockSlim.EnterReadLock();
try
{
ConcurrentQueue<SynchronizationWorkItem> pendingForDestination;
if (pendingSynchronizations.TryGetValue(destinationFileSystemUrl, out pendingForDestination) == false)
{
workItem = null;
return false;
}
return pendingForDestination.TryDequeue(out workItem);
}
finally
{
readerWriterLockSlim.ExitReadLock();
}
}
private async Task <SynchronizationReport> PerformSynchronizationAsync(IAsyncFilesSynchronizationCommands destination,
SynchronizationWorkItem work)
{
var commands = (IAsyncFilesCommandsImpl)destination.Commands;
string destinationUrl = commands.UrlFor();
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
work.GetType().Name, work.FileName, destinationUrl);
if (!CanSynchronizeTo(destinationUrl))
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destinationUrl, work.FileName);
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
}
return(new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destinationUrl, work.FileName))
});
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destinationUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(destination);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
UpdateSuccessfulSynchronizationTime();
Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destinationUrl, moreDetails);
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
Log.DebugException(string.Format("{0} to {1} was cancelled", work, destinationUrl), report.Exception);
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl),
report.Exception);
}
}
Queue.SynchronizationFinished(work, destinationUrl);
if (!synchronizationCancelled)
{
CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType);
}
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
return(report);
}
public bool EnqueueSynchronization(string destinationFileSystemUrl, SynchronizationWorkItem workItem)
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim())
.EnterUpgradeableReadLock();
try
{
var pendingForDestination = pendingSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentQueue <SynchronizationWorkItem>());
// if delete work is enqueued and there are other synchronization works for a given file then remove them from a queue
if (workItem.SynchronizationType == SynchronizationType.Delete &&
pendingForDestination.Any(x => x.FileName == workItem.FileName && x.SynchronizationType != SynchronizationType.Delete))
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).EnterWriteLock();
try
{
var modifiedQueue = new ConcurrentQueue <SynchronizationWorkItem>();
foreach (var pendingWork in pendingForDestination)
{
if (pendingWork.FileName != workItem.FileName)
{
modifiedQueue.Enqueue(pendingWork);
}
}
modifiedQueue.Enqueue(workItem);
pendingForDestination = pendingSynchronizations.AddOrUpdate(destinationFileSystemUrl, modifiedQueue,
(key, value) => modifiedQueue);
}
finally
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitWriteLock();
}
}
foreach (var pendingWork in pendingForDestination)
{
// if there is a file in pending synchronizations do not add it again
if (pendingWork.Equals(workItem))
{
Log.Debug("{0} for a file {1} and a destination {2} was already existed in a pending queue",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
return(false);
}
// if there is a work for a file of the same type but with lower file ETag just refresh existing work metadata and do not enqueue again
if (pendingWork.FileName == workItem.FileName &&
pendingWork.SynchronizationType == workItem.SynchronizationType &&
Buffers.Compare(workItem.FileETag.ToByteArray(), pendingWork.FileETag.ToByteArray()) > 0)
{
pendingWork.RefreshMetadata();
Log.Debug(
"{0} for a file {1} and a destination {2} was already existed in a pending queue but with older ETag, it's metadata has been refreshed",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
return(false);
}
}
var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentDictionary <string, SynchronizationWorkItem>
());
// if there is a work in an active synchronizations do not add it again
if (activeForDestination.ContainsKey(workItem.FileName) && activeForDestination[workItem.FileName].Equals(workItem))
{
Log.Debug("{0} for a file {1} and a destination {2} was already existed in an active queue",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
return(false);
}
pendingForDestination.Enqueue(workItem);
Log.Debug("{0} for a file {1} and a destination {2} was enqueued", workItem.GetType().Name, workItem.FileName,
destinationFileSystemUrl);
}
finally
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitUpgradeableReadLock();
}
return(true);
}