public override Task<SynchronizationReport> PerformAsync(ISynchronizationServerClient synchronizationServerClient)
{
FileAndPagesInformation fileAndPages = null;
Storage.Batch(accessor => fileAndPages = accessor.GetFile(FileName, 0, 0));
return synchronizationServerClient.RenameAsync(FileName, rename, fileAndPages.Metadata, FileSystemInfo);
}
protected async Task <SynchronizationReport> HandleConflict(ISynchronizationServerClient synchronizationServerClient, ConflictItem conflict, ILog log)
{
var conflictResolutionStrategy = await synchronizationServerClient.GetResolutionStrategyFromDestinationResolvers(conflict, FileMetadata);
switch (conflictResolutionStrategy)
{
case ConflictResolutionStrategy.NoResolution:
return(await ApplyConflictOnDestinationAsync(conflict, FileMetadata, synchronizationServerClient, FileSystemInfo.Url, log));
case ConflictResolutionStrategy.CurrentVersion:
await ApplyConflictOnDestinationAsync(conflict, FileMetadata, synchronizationServerClient, FileSystemInfo.Url, log);
await synchronizationServerClient.ResolveConflictAsync(FileName, conflictResolutionStrategy);
return(new SynchronizationReport(FileName, FileETag, SynchronizationType));
case ConflictResolutionStrategy.RemoteVersion:
// we can push the file even though it conflicted, the conflict will be automatically resolved on the destination side
return(null);
default:
return(new SynchronizationReport(FileName, FileETag, SynchronizationType)
{
Exception = new SynchronizationException(string.Format("Unknown resolution strategy: {0}", conflictResolutionStrategy)),
});
}
}
public override async Task <SynchronizationReport> PerformAsync(ISynchronizationServerClient synchronizationServerClient)
{
AssertLocalFileExistsAndIsNotConflicted(FileMetadata);
var destinationMetadata = await synchronizationServerClient.GetMetadataForAsync(FileName);
if (destinationMetadata == null)
{
// if file doesn't exist on destination server - upload it there
return(await UploadToAsync(synchronizationServerClient));
}
var destinationServerRdcStats = await synchronizationServerClient.GetRdcStatsAsync();
if (!IsRemoteRdcCompatible(destinationServerRdcStats))
{
throw new SynchronizationException("Incompatible RDC version detected on destination server");
}
var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, FileSystemInfo.Url);
if (conflict != null)
{
var report = await HandleConflict(synchronizationServerClient, conflict, log);
if (report != null)
{
return(report);
}
}
using (var localSignatureRepository = new StorageSignatureRepository(Storage, FileName))
using (var remoteSignatureCache = new VolatileSignatureRepository(FileName))
{
var localRdcManager = new LocalRdcManager(localSignatureRepository, Storage, sigGenerator);
var destinationRdcManager = new RemoteRdcManager(synchronizationServerClient, localSignatureRepository, remoteSignatureCache);
log.Debug("Starting to retrieve signatures of a local file '{0}'.", FileName);
Cts.Token.ThrowIfCancellationRequested();
// first we need to create a local file signatures before we synchronize with remote ones
var localSignatureManifest = await localRdcManager.GetSignatureManifestAsync(FileDataInfo);
log.Debug("Number of a local file '{0}' signatures was {1}.", FileName, localSignatureManifest.Signatures.Count);
if (localSignatureManifest.Signatures.Any())
{
var destinationSignatureManifest = await destinationRdcManager.SynchronizeSignaturesAsync(FileDataInfo, Cts.Token);
if (destinationSignatureManifest.Signatures.Any())
{
return(await SynchronizeTo(synchronizationServerClient, localSignatureRepository, remoteSignatureCache, localSignatureManifest, destinationSignatureManifest));
}
}
return(await UploadToAsync(synchronizationServerClient));
}
}
public RemoteRdcManager(ISynchronizationServerClient synchronizationServerClient,
ISignatureRepository localSignatureRepository,
ISignatureRepository remoteCacheSignatureRepository)
{
this.localSignatureRepository = localSignatureRepository;
this.remoteCacheSignatureRepository = remoteCacheSignatureRepository;
this.synchronizationServerClient = synchronizationServerClient;
}
public override Task <SynchronizationReport> PerformAsync(ISynchronizationServerClient synchronizationServerClient)
{
FileAndPagesInformation fileAndPages = null;
Storage.Batch(accessor => fileAndPages = accessor.GetFile(FileName, 0, 0));
return(synchronizationServerClient.DeleteAsync(FileName, fileAndPages.Metadata, FileSystemInfo));
}
public RemoteRdcManager(ISynchronizationServerClient synchronizationServerClient,
ISignatureRepository localSignatureRepository,
ISignatureRepository remoteCacheSignatureRepository)
{
this.localSignatureRepository = localSignatureRepository;
this.remoteCacheSignatureRepository = remoteCacheSignatureRepository;
this.synchronizationServerClient = synchronizationServerClient;
}
public SynchronizationMultipartRequest(ISynchronizationServerClient synchronizationServerClient, FileSystemInfo fileSystemInfo, string fileName,
RavenJObject sourceMetadata, Stream sourceStream, IList <RdcNeed> needList)
{
this.synchronizationServerClient = synchronizationServerClient;
this.fileSystemInfo = fileSystemInfo;
this.fileName = fileName;
this.sourceMetadata = sourceMetadata;
this.sourceStream = sourceStream;
this.needList = needList;
syncingBoundary = "syncing";
}
public SynchronizationMultipartRequest(ISynchronizationServerClient synchronizationServerClient, FileSystemInfo fileSystemInfo, string fileName,
RavenJObject sourceMetadata, Stream sourceStream, IList<RdcNeed> needList)
{
this.synchronizationServerClient = synchronizationServerClient;
this.fileSystemInfo = fileSystemInfo;
this.fileName = fileName;
this.sourceMetadata = sourceMetadata;
this.sourceStream = sourceStream;
this.needList = needList;
syncingBoundary = "syncing";
}
public override async Task<SynchronizationReport> PerformAsync(ISynchronizationServerClient synchronizationServerClient)
{
AssertLocalFileExistsAndIsNotConflicted(FileMetadata);
var destinationMetadata = await synchronizationServerClient.GetMetadataForAsync(FileName);
if (destinationMetadata == null)
{
// if file doesn't exist on destination server - upload it there
return await UploadToAsync(synchronizationServerClient);
}
var destinationServerRdcStats = await synchronizationServerClient.GetRdcStatsAsync();
if (!IsRemoteRdcCompatible(destinationServerRdcStats))
throw new SynchronizationException("Incompatible RDC version detected on destination server");
var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, FileSystemInfo.Url);
if (conflict != null)
{
var report = await HandleConflict(synchronizationServerClient, conflict, log);
if (report != null)
return report;
}
using (var localSignatureRepository = new StorageSignatureRepository(Storage, FileName))
using (var remoteSignatureCache = new VolatileSignatureRepository(FileName))
{
var localRdcManager = new LocalRdcManager(localSignatureRepository, Storage, sigGenerator);
var destinationRdcManager = new RemoteRdcManager(synchronizationServerClient, localSignatureRepository, remoteSignatureCache);
log.Debug("Starting to retrieve signatures of a local file '{0}'.", FileName);
Cts.Token.ThrowIfCancellationRequested();
// first we need to create a local file signatures before we synchronize with remote ones
var localSignatureManifest = await localRdcManager.GetSignatureManifestAsync(FileDataInfo);
log.Debug("Number of a local file '{0}' signatures was {1}.", FileName, localSignatureManifest.Signatures.Count);
if (localSignatureManifest.Signatures.Any())
{
var destinationSignatureManifest = await destinationRdcManager.SynchronizeSignaturesAsync(FileDataInfo, Cts.Token);
if (destinationSignatureManifest.Signatures.Any())
{
return await SynchronizeTo(synchronizationServerClient, localSignatureRepository, remoteSignatureCache, localSignatureManifest, destinationSignatureManifest);
}
}
return await UploadToAsync(synchronizationServerClient);
}
}
private Task <SynchronizationReport> PushByUsingMultipartRequest(ISynchronizationServerClient synchronizationServerClient, Stream sourceFileStream,
IList <RdcNeed> needList)
{
Cts.Token.ThrowIfCancellationRequested();
multipartRequest = new SynchronizationMultipartRequest(synchronizationServerClient, FileSystemInfo, FileName, FileMetadata, sourceFileStream, needList);
var bytesToTransferCount = needList.Where(x => x.BlockType == RdcNeedType.Source).Sum(x => (double)x.BlockLength);
log.Debug(
"Synchronizing a file '{0}' (ETag {1}) to {2} by using multipart request. Need list length is {3}. Number of bytes that needs to be transfered is {4}",
FileName, FileETag, synchronizationServerClient, needList.Count, bytesToTransferCount);
return(multipartRequest.PushChangesAsync(Cts.Token));
}
public override async Task<SynchronizationReport> PerformAsync(ISynchronizationServerClient synchronizationServerClient)
{
AssertLocalFileExistsAndIsNotConflicted(FileMetadata);
var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, FileSystemInfo.Url);
if (conflict != null)
{
var report = await HandleConflict(synchronizationServerClient, conflict, log);
if (report != null)
return report;
}
return await synchronizationServerClient.UpdateMetadataAsync(FileName, FileMetadata, FileSystemInfo);
}
public override async Task <SynchronizationReport> PerformAsync(ISynchronizationServerClient synchronizationServerClient)
{
AssertLocalFileExistsAndIsNotConflicted(FileMetadata);
var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, FileSystemInfo.Url);
if (conflict != null)
{
var report = await HandleConflict(synchronizationServerClient, conflict, log);
if (report != null)
{
return(report);
}
}
return(await synchronizationServerClient.UpdateMetadataAsync(FileName, FileMetadata, FileSystemInfo));
}
public async Task <SynchronizationReport> UploadToAsync(ISynchronizationServerClient synchronizationServerClient)
{
using (var sourceFileStream = StorageStream.Reading(Storage, FileName))
{
var fileSize = sourceFileStream.Length;
var onlySourceNeed = new List <RdcNeed>
{
new RdcNeed
{
BlockType = RdcNeedType.Source,
BlockLength = (ulong)fileSize,
FileOffset = 0
}
};
return(await PushByUsingMultipartRequest(synchronizationServerClient, sourceFileStream, onlySourceNeed));
}
}
private async Task <SynchronizationReport> SynchronizeTo(ISynchronizationServerClient synchronizationServerClient,
ISignatureRepository localSignatureRepository,
ISignatureRepository remoteSignatureRepository,
SignatureManifest sourceSignatureManifest,
SignatureManifest destinationSignatureManifest)
{
var seedSignatureInfo = SignatureInfo.Parse(destinationSignatureManifest.Signatures.Last().Name);
var sourceSignatureInfo = SignatureInfo.Parse(sourceSignatureManifest.Signatures.Last().Name);
using (var localFile = StorageStream.Reading(Storage, FileName))
{
IList <RdcNeed> needList;
using (var needListGenerator = new NeedListGenerator(remoteSignatureRepository, localSignatureRepository))
{
needList = needListGenerator.CreateNeedsList(seedSignatureInfo, sourceSignatureInfo, Cts.Token);
}
return(await PushByUsingMultipartRequest(synchronizationServerClient, localFile, needList));
}
}
public async Task<SynchronizationReport> UploadToAsync(ISynchronizationServerClient synchronizationServerClient)
{
using (var sourceFileStream = StorageStream.Reading(Storage, FileName))
{
var fileSize = sourceFileStream.Length;
var onlySourceNeed = new List<RdcNeed>
{
new RdcNeed
{
BlockType = RdcNeedType.Source,
BlockLength = (ulong) fileSize,
FileOffset = 0
}
};
return await PushByUsingMultipartRequest(synchronizationServerClient, sourceFileStream, onlySourceNeed);
}
}
private async Task<SynchronizationReport> SynchronizeTo(ISynchronizationServerClient synchronizationServerClient,
ISignatureRepository localSignatureRepository,
ISignatureRepository remoteSignatureRepository,
SignatureManifest sourceSignatureManifest,
SignatureManifest destinationSignatureManifest)
{
var seedSignatureInfo = SignatureInfo.Parse(destinationSignatureManifest.Signatures.Last().Name);
var sourceSignatureInfo = SignatureInfo.Parse(sourceSignatureManifest.Signatures.Last().Name);
using (var localFile = StorageStream.Reading(Storage, FileName))
{
IList<RdcNeed> needList;
using (var needListGenerator = new NeedListGenerator(remoteSignatureRepository, localSignatureRepository))
{
needList = needListGenerator.CreateNeedsList(seedSignatureInfo, sourceSignatureInfo, Cts.Token);
}
return await PushByUsingMultipartRequest(synchronizationServerClient, localFile, needList);
}
}
private async Task <SynchronizationReport> PerformSynchronizationAsync(ISynchronizationServerClient synchronizationServerClient,
SynchronizationWorkItem work)
{
var destinationUrl = synchronizationServerClient.BaseUrl;
if (Log.IsDebugEnabled)
{
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}", work.GetType().Name, work.FileName, destinationUrl);
}
if (AvailableSynchronizationRequestsTo(destinationUrl) <= 0)
{
if (Log.IsDebugEnabled)
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.", destinationUrl, work.FileName);
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
return(new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destinationUrl, work.FileName))
});
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destinationUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
Direction = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(synchronizationServerClient).ConfigureAwait(false);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transferred and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
context.UpdateSuccessfulSynchronizationTime();
if (Log.IsDebugEnabled)
{
Log.Debug("{0} to {1} has finished successfully{2}", work, destinationUrl, moreDetails);
}
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
if (Log.IsDebugEnabled)
{
Log.DebugException(string.Format("{0} to {1} was canceled", work, destinationUrl), report.Exception);
}
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception);
}
}
Queue.SynchronizationFinished(work, destinationUrl);
if (synchronizationCancelled == false)
{
CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType);
}
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
Direction = SynchronizationDirection.Outgoing
});
return(report);
}
private async Task <SynchronizationConfirmation[]> ConfirmPushedFiles(IList <SynchronizationDetails> filesNeedConfirmation, ISynchronizationServerClient synchronizationServerClient)
{
if (!filesNeedConfirmation.Any())
{
return(new SynchronizationConfirmation[0]);
}
return(await synchronizationServerClient.GetConfirmationForFilesAsync(filesNeedConfirmation.Select(x => new Tuple <string, Etag>(x.FileName, x.FileETag))).ConfigureAwait(false));
}
private async Task<SynchronizationReport> ApplyConflictOnDestinationAsync(ConflictItem conflict, RavenJObject remoteMetadata, ISynchronizationServerClient synchronizationServerClient, string localServerUrl, ILog log)
{
log.Debug("File '{0}' is in conflict with destination version from {1}. Applying conflict on destination", FileName, synchronizationServerClient.BaseUrl);
try
{
var version = conflict.RemoteHistory.Last().Version;
var serverId = conflict.RemoteHistory.Last().ServerId;
var history = new List<HistoryItem>(conflict.RemoteHistory);
history.RemoveAt(conflict.RemoteHistory.Count - 1);
await synchronizationServerClient.ApplyConflictAsync(FileName, version, serverId, remoteMetadata, localServerUrl);
}
catch (Exception ex)
{
log.WarnException(string.Format("Failed to apply conflict on {0} for file '{1}'", synchronizationServerClient, FileName), ex);
}
return new SynchronizationReport(FileName, FileETag, SynchronizationType)
{
Exception = new SynchronizationException(string.Format("File {0} is conflicted", FileName)),
};
}
private IEnumerable <Task <SynchronizationReport> > SynchronizePendingFilesAsync(ISynchronizationServerClient destinationCommands, bool forceSyncingAll)
{
var destinationUrl = destinationCommands.BaseUrl;
while (AvailableSynchronizationRequestsTo(destinationUrl) > 0)
{
context.CancellationToken.ThrowIfCancellationRequested();
SynchronizationWorkItem work;
if (synchronizationQueue.TryDequePending(destinationUrl, out work) == false)
{
break;
}
if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destinationUrl))
{
if (Log.IsDebugEnabled)
{
Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName, destinationCommands);
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) // add it again at the end of the queue
{
// add it again at the end of the queue
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
continue;
}
var workTask = PerformSynchronizationAsync(destinationCommands, work);
if (forceSyncingAll)
{
workTask.ContinueWith(_ => context.NotifyAboutWork()); // synchronization slot released, next file can be synchronized
}
yield return(workTask);
}
}
public RemoteSignaturePartialAccess(ISynchronizationServerClient synchronizationServerClient, string fileName)
{
this.synchronizationServerClient = synchronizationServerClient;
_fileName = fileName;
}
private async Task<bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient,
SourceSynchronizationInformation synchronizationInfo,
IList<FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet<FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization = filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
if (filteredFilesToSynchronization.Count == 0)
{
var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault();
if (lastFileBeforeFiltering == null)
return true; // there are no more files that need
if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag)
return true; // already updated etag on destination side
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false);
return false; // all docs has been filtered out, update etag on destination side and retry
}
var destinationUrl = destinationSyncClient.BaseUrl;
bool enqueued = true;
foreach (var fileHeader in filteredFilesToSynchronization)
{
context.CancellationToken.ThrowIfCancellationRequested();
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destinationUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value<Guid>(Constants.MetadataEtagField);
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, etag).ConfigureAwait(false);
RemoveSyncingConfiguration(file, destinationUrl);
enqueued = false;
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
enqueued = true;
}
return enqueued;
}
private async Task <bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient,
SourceSynchronizationInformation synchronizationInfo,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization = filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
{
var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault();
if (lastFileBeforeFiltering == null)
{
return(true); // there are no more files that need
}
if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag)
{
return(true); // already updated etag on destination side
}
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false);
return(false); // all docs has been filtered out, update etag on destination side and retry
}
var destinationUrl = destinationSyncClient.BaseUrl;
bool enqueued = false;
var maxEtagOfFilteredDoc = Etag.Empty;
foreach (var fileHeader in filteredFilesToSynchronization)
{
context.CancellationToken.ThrowIfCancellationRequested();
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destinationUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription());
switch (reason)
{
case NoSyncReason.ContainedInDestinationHistory:
case NoSyncReason.DestinationFileConflicted:
case NoSyncReason.NoNeedToDeleteNonExistigFile:
var localEtag = Etag.Parse(localMetadata.Value <string>(Constants.MetadataEtagField));
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
RemoveSyncingConfiguration(file, destinationUrl);
}
else if (reason == NoSyncReason.DestinationFileConflicted)
{
if (needSyncingAgain.Contains(fileHeader, FileHeaderNameEqualityComparer.Instance) == false)
{
CreateSyncingConfiguration(fileHeader.Name, fileHeader.Etag, destinationUrl, SynchronizationType.Unknown);
}
}
else if (reason == NoSyncReason.NoNeedToDeleteNonExistigFile)
{
// after the upgrade to newer build there can be still an existing syncing configuration for it
RemoveSyncingConfiguration(file, destinationUrl);
}
if (EtagUtil.IsGreaterThan(localEtag, maxEtagOfFilteredDoc))
{
maxEtagOfFilteredDoc = localEtag;
}
break;
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
enqueued = true;
}
if (enqueued == false && EtagUtil.IsGreaterThan(maxEtagOfFilteredDoc, synchronizationInfo.LastSourceFileEtag))
{
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, maxEtagOfFilteredDoc).ConfigureAwait(false);
return(false); // we bumped the last synced etag on a destination server, let it know it need to repeat the operation
}
return(true);
}
private Task<SynchronizationReport> PushByUsingMultipartRequest(ISynchronizationServerClient synchronizationServerClient, Stream sourceFileStream,
IList<RdcNeed> needList)
{
Cts.Token.ThrowIfCancellationRequested();
multipartRequest = new SynchronizationMultipartRequest(synchronizationServerClient, FileSystemInfo, FileName, FileMetadata, sourceFileStream, needList);
var bytesToTransferCount = needList.Where(x => x.BlockType == RdcNeedType.Source).Sum(x => (double)x.BlockLength);
log.Debug(
"Synchronizing a file '{0}' (ETag {1}) to {2} by using multipart request. Need list length is {3}. Number of bytes that needs to be transfered is {4}",
FileName, FileETag, synchronizationServerClient, needList.Count, bytesToTransferCount);
return multipartRequest.PushChangesAsync(Cts.Token);
}
protected async Task<SynchronizationReport> HandleConflict(ISynchronizationServerClient synchronizationServerClient, ConflictItem conflict, ILog log)
{
var conflictResolutionStrategy = await synchronizationServerClient.GetResolutionStrategyFromDestinationResolvers(conflict, FileMetadata);
switch (conflictResolutionStrategy)
{
case ConflictResolutionStrategy.NoResolution:
return await ApplyConflictOnDestinationAsync(conflict, FileMetadata, synchronizationServerClient, FileSystemInfo.Url, log);
case ConflictResolutionStrategy.CurrentVersion:
await ApplyConflictOnDestinationAsync(conflict, FileMetadata, synchronizationServerClient, FileSystemInfo.Url, log);
await synchronizationServerClient.ResolveConflictAsync(FileName, conflictResolutionStrategy);
return new SynchronizationReport(FileName, FileETag, SynchronizationType);
case ConflictResolutionStrategy.RemoteVersion:
// we can push the file even though it conflicted, the conflict will be automatically resolved on the destination side
return null;
default:
return new SynchronizationReport(FileName, FileETag, SynchronizationType)
{
Exception = new SynchronizationException(string.Format("Unknown resolution strategy: {0}", conflictResolutionStrategy)),
};
}
}
public abstract Task<SynchronizationReport> PerformAsync(ISynchronizationServerClient destination);
private async Task<SynchronizationReport> PerformSynchronizationAsync(ISynchronizationServerClient synchronizationServerClient,
SynchronizationWorkItem work)
{
var destinationUrl = synchronizationServerClient.BaseUrl;
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
work.GetType().Name, work.FileName, destinationUrl);
if (AvailableSynchronizationRequestsTo(destinationUrl) <= 0)
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destinationUrl, work.FileName);
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destinationUrl, work.FileName))
};
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destinationUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
Direction = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(synchronizationServerClient).ConfigureAwait(false);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transferred and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
context.UpdateSuccessfulSynchronizationTime();
Log.Debug("{0} to {1} has finished successfully{2}", work, destinationUrl, moreDetails);
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
Log.DebugException(string.Format("{0} to {1} was canceled", work, destinationUrl), report.Exception);
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception);
}
}
Queue.SynchronizationFinished(work, destinationUrl);
if (synchronizationCancelled == false)
CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
Direction = SynchronizationDirection.Outgoing
});
return report;
}
private async Task EnqueueMissingUpdatesAsync(ISynchronizationServerClient synchronizationServerClient,
SourceSynchronizationInformation lastEtag,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(lastEtag, 100),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization =
filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
}
// Early break. There are no files to synchronize to the selected destination.
if (!filteredFilesToSynchronization.Any())
{
return;
}
var baseUrl = synchronizationServerClient.BaseUrl;
foreach (var fileHeader in filteredFilesToSynchronization)
{
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await synchronizationServerClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
baseUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, baseUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value <Guid>(Constants.MetadataEtagField);
await synchronizationServerClient.IncrementLastETagAsync(storage.Id, baseUrl, etag).ConfigureAwait(false);
RemoveSyncingConfiguration(file, baseUrl);
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(baseUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = baseUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
}
}
private async Task EnqueueMissingUpdatesAsync(ISynchronizationServerClient synchronizationServerClient,
SourceSynchronizationInformation lastEtag,
IList<FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet<FileHeader>(GetFilesToSynchronization(lastEtag, 100),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization =
filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
// Early break. There are no files to synchronize to the selected destination.
if (!filteredFilesToSynchronization.Any())
return;
var baseUrl = synchronizationServerClient.BaseUrl;
foreach (var fileHeader in filteredFilesToSynchronization)
{
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await synchronizationServerClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
baseUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, baseUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value<Guid>(Constants.MetadataEtagField);
await synchronizationServerClient.IncrementLastETagAsync(storage.Id, baseUrl, etag).ConfigureAwait(false);
RemoveSyncingConfiguration(file, baseUrl);
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(baseUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = baseUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
}
}
public abstract Task <SynchronizationReport> PerformAsync(ISynchronizationServerClient destination);
private async Task <bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient,
SourceSynchronizationInformation synchronizationInfo,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization = filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
{
var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault();
if (lastFileBeforeFiltering == null)
{
return(true); // there are no more files that need
}
if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag)
{
return(true); // already updated etag on destination side
}
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false);
return(false); // all docs has been filtered out, update etag on destination side and retry
}
var destinationUrl = destinationSyncClient.BaseUrl;
bool enqueued = true;
foreach (var fileHeader in filteredFilesToSynchronization)
{
context.CancellationToken.ThrowIfCancellationRequested();
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destinationUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription());
}
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value <Guid>(Constants.MetadataEtagField);
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, etag).ConfigureAwait(false);
RemoveSyncingConfiguration(file, destinationUrl);
enqueued = false;
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
enqueued = true;
}
return(enqueued);
}
private IEnumerable<Task<SynchronizationReport>> SynchronizePendingFilesAsync(ISynchronizationServerClient destinationCommands, bool forceSyncingAll)
{
var destinationUrl = destinationCommands.BaseUrl;
while (AvailableSynchronizationRequestsTo(destinationUrl) > 0)
{
context.CancellationToken.ThrowIfCancellationRequested();
SynchronizationWorkItem work;
if (synchronizationQueue.TryDequePending(destinationUrl, out work) == false)
break;
if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destinationUrl))
{
Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName,
destinationCommands);
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) // add it again at the end of the queue
{
// add it again at the end of the queue
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
continue;
}
var workTask = PerformSynchronizationAsync(destinationCommands, work);
if (forceSyncingAll)
{
workTask.ContinueWith(_ => context.NotifyAboutWork()); // synchronization slot released, next file can be synchronized
}
yield return workTask;
}
}
public RemoteSignaturePartialAccess(ISynchronizationServerClient synchronizationServerClient, string fileName)
{
this.synchronizationServerClient = synchronizationServerClient;
_fileName = fileName;
}
private async Task<SynchronizationConfirmation[]> ConfirmPushedFiles(IList<SynchronizationDetails> filesNeedConfirmation, ISynchronizationServerClient synchronizationServerClient)
{
if (!filesNeedConfirmation.Any())
return new SynchronizationConfirmation[0];
return await synchronizationServerClient.GetConfirmationForFilesAsync(filesNeedConfirmation.Select(x => new Tuple<string, Etag>(x.FileName, x.FileETag))).ConfigureAwait(false);
}
private async Task <SynchronizationReport> ApplyConflictOnDestinationAsync(ConflictItem conflict, RavenJObject remoteMetadata, ISynchronizationServerClient synchronizationServerClient, string localServerUrl, ILog log)
{
log.Debug("File '{0}' is in conflict with destination version from {1}. Applying conflict on destination", FileName, synchronizationServerClient.BaseUrl);
try
{
var version = conflict.RemoteHistory.Last().Version;
var serverId = conflict.RemoteHistory.Last().ServerId;
var history = new List <HistoryItem>(conflict.RemoteHistory);
history.RemoveAt(conflict.RemoteHistory.Count - 1);
await synchronizationServerClient.ApplyConflictAsync(FileName, version, serverId, remoteMetadata, localServerUrl);
}
catch (Exception ex)
{
log.WarnException(string.Format("Failed to apply conflict on {0} for file '{1}'", synchronizationServerClient, FileName), ex);
}
return(new SynchronizationReport(FileName, FileETag, SynchronizationType)
{
Exception = new SynchronizationException(string.Format("File {0} is conflicted", FileName)),
});
}
