private Task <IEnumerable <SynchronizationConfirmation> > ConfirmPushedFiles(
IList <SynchronizationDetails> filesNeedConfirmation, SynchronizationClient destinationClient)
{
if (filesNeedConfirmation.Count == 0)
{
return(new CompletedTask <IEnumerable <SynchronizationConfirmation> >(Enumerable.Empty <SynchronizationConfirmation>()));
}
return
(destinationClient.ConfirmFilesAsync(
filesNeedConfirmation.Select(x => new Tuple <string, Guid>(x.FileName, x.FileETag))));
}
private Task<IEnumerable<SynchronizationConfirmation>> ConfirmPushedFiles(
IList<SynchronizationDetails> filesNeedConfirmation, SynchronizationClient destinationClient)
{
if (filesNeedConfirmation.Count == 0)
{
return new CompletedTask<IEnumerable<SynchronizationConfirmation>>(Enumerable.Empty<SynchronizationConfirmation>());
}
return
destinationClient.ConfirmFilesAsync(
filesNeedConfirmation.Select(x => new Tuple<string, Guid>(x.FileName, x.FileETag)));
}
private async Task<SynchronizationReport> PerformSynchronizationAsync(SynchronizationClient destination,
SynchronizationWorkItem work)
{
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
work.GetType().Name, work.FileName, destination.FileSystemUrl);
if (!CanSynchronizeTo(destination.FileSystemUrl))
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName);
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work);
return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName))
};
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destination.FileSystemUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(destination);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
UpdateSuccessfulSynchronizationTime();
Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destination.FileSystemUrl, moreDetails);
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
Log.DebugException(string.Format("{0} to {1} was cancelled", work, destination.FileSystemUrl), report.Exception);
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destination.FileSystemUrl),
report.Exception);
}
}
Queue.SynchronizationFinished(work, destination.FileSystemUrl);
if (!synchronizationCancelled)
CreateSyncingConfiguration(fileName, work.FileETag, destination.FileSystemUrl, work.SynchronizationType);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
return report;
}
private IEnumerable<Task<SynchronizationReport>> SynchronizePendingFilesAsync(SynchronizationClient destination, bool forceSyncingContinuation)
{
for (var i = 0; i < AvailableSynchronizationRequestsTo(destination.FileSystemUrl); i++)
{
SynchronizationWorkItem work;
if (!synchronizationQueue.TryDequePendingSynchronization(destination.FileSystemUrl, out work))
break;
if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destination.FileSystemUrl))
{
Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName,
destination);
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work); // add it again at the end of the queue
}
else
{
var workTask = PerformSynchronizationAsync(destination, work);
if (forceSyncingContinuation)
{
workTask.ContinueWith(t => SynchronizePendingFilesAsync(destination, true).ToArray());
}
yield return workTask;
}
}
}
private async Task EnqueueMissingUpdatesAsync(SynchronizationClient destination,
SourceSynchronizationInformation lastEtag,
IList<FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet<FileHeader>(GetFilesToSynchronization(lastEtag, 100),
new FileHeaderNameEqualityComparer());
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization =
filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}",
filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
return;
foreach (var fileHeader in filteredFilesToSynchronization)
{
var file = fileHeader.Name;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destination.GetMetadataForAsync(file);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destination.FileSystemUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destination.FileSystemUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value<Guid>("ETag");
await destination.IncrementLastETagAsync(storage.Id, FileSystemUrl, etag);
RemoveSyncingConfiguration(file, destination.FileSystemUrl);
}
continue;
}
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work);
}
}
private async Task <SynchronizationReport> PerformSynchronizationAsync(SynchronizationClient destination,
SynchronizationWorkItem work)
{
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
work.GetType().Name, work.FileName, destination.FileSystemUrl);
if (!CanSynchronizeTo(destination.FileSystemUrl))
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName);
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work);
return(new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName))
});
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destination.FileSystemUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(destination);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
UpdateSuccessfulSynchronizationTime();
Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destination.FileSystemUrl, moreDetails);
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
Log.DebugException(string.Format("{0} to {1} was cancelled", work, destination.FileSystemUrl), report.Exception);
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destination.FileSystemUrl),
report.Exception);
}
}
Queue.SynchronizationFinished(work, destination.FileSystemUrl);
if (!synchronizationCancelled)
{
CreateSyncingConfiguration(fileName, work.FileETag, destination.FileSystemUrl, work.SynchronizationType);
}
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
return(report);
}
private IEnumerable <Task <SynchronizationReport> > SynchronizePendingFilesAsync(SynchronizationClient destination, bool forceSyncingContinuation)
{
for (var i = 0; i < AvailableSynchronizationRequestsTo(destination.FileSystemUrl); i++)
{
SynchronizationWorkItem work;
if (!synchronizationQueue.TryDequePendingSynchronization(destination.FileSystemUrl, out work))
{
break;
}
if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destination.FileSystemUrl))
{
Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName,
destination);
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work); // add it again at the end of the queue
}
else
{
var workTask = PerformSynchronizationAsync(destination, work);
if (forceSyncingContinuation)
{
workTask.ContinueWith(t => SynchronizePendingFilesAsync(destination, true).ToArray());
}
yield return(workTask);
}
}
}
private async Task EnqueueMissingUpdatesAsync(SynchronizationClient destination,
SourceSynchronizationInformation lastEtag,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(lastEtag, 100),
new FileHeaderNameEqualityComparer());
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization =
filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}",
filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
{
return;
}
foreach (var fileHeader in filteredFilesToSynchronization)
{
var file = fileHeader.Name;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destination.GetMetadataForAsync(file);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destination.FileSystemUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destination.FileSystemUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value <Guid>("ETag");
await destination.IncrementLastETagAsync(storage.Id, FileSystemUrl, etag);
RemoveSyncingConfiguration(file, destination.FileSystemUrl);
}
continue;
}
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work);
}
}
