public SourceSynchronizationInformation GetLastSynchronization(Guid from)
{
SourceSynchronizationInformation info = null;
try
{
Storage.Batch(accessor =>
{
info = accessor.GetConfig(SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + from)
.JsonDeserialization <SourceSynchronizationInformation>();
});
}
catch (FileNotFoundException)
{
info = new SourceSynchronizationInformation
{
LastSourceFileEtag = Etag.Empty,
DestinationServerId = Storage.Id
};
}
return(info);
}
private IEnumerable <FileHeader> GetFilesToSynchronization(SourceSynchronizationInformation destinationsSynchronizationInformationForSource, int take)
{
var filesToSynchronization = new List <FileHeader>();
Log.Debug("Getting files to synchronize with ETag greater than {0} [parameter take = {1}]",
destinationsSynchronizationInformationForSource.LastSourceFileEtag, take);
try
{
storage.Batch(
accessor =>
filesToSynchronization =
accessor.GetFilesAfter(destinationsSynchronizationInformationForSource.LastSourceFileEtag, take).ToList());
}
catch (Exception e)
{
Log.WarnException(
string.Format("Could not get files to synchronize after: " +
destinationsSynchronizationInformationForSource.LastSourceFileEtag), e);
}
return(filesToSynchronization);
}
private void SaveSynchronizationSourceInformation(FileSystemInfo sourceFileSystem, Etag lastSourceEtag)
{
var lastSynchronizationInformation = GetLastSynchronization(sourceFileSystem.Id);
if (EtagUtil.IsGreaterThan(lastSynchronizationInformation.LastSourceFileEtag, lastSourceEtag))
{
return;
}
var synchronizationSourceInfo = new SourceSynchronizationInformation
{
LastSourceFileEtag = lastSourceEtag,
SourceServerUrl = sourceFileSystem.Url,
DestinationServerId = Storage.Id
};
var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceFileSystem.Id;
Storage.Batch(accessor => accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo)));
Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceFileSystem.Url, sourceFileSystem.Id);
}
private async Task <bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient,
SourceSynchronizationInformation synchronizationInfo,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization = filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
{
var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault();
if (lastFileBeforeFiltering == null)
{
return(true); // there are no more files that need
}
if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag)
{
return(true); // already updated etag on destination side
}
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false);
return(false); // all docs has been filtered out, update etag on destination side and retry
}
var destinationUrl = destinationSyncClient.BaseUrl;
bool enqueued = true;
foreach (var fileHeader in filteredFilesToSynchronization)
{
context.CancellationToken.ThrowIfCancellationRequested();
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destinationUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription());
}
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value <Guid>(Constants.MetadataEtagField);
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, etag).ConfigureAwait(false);
RemoveSyncingConfiguration(file, destinationUrl);
enqueued = false;
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
enqueued = true;
}
return(enqueued);
}
private async Task EnqueueMissingUpdatesAsync(IAsyncFilesSynchronizationCommands destination,
SourceSynchronizationInformation lastEtag,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var commands = (IAsyncFilesCommandsImpl)destination.Commands;
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(lastEtag, 100),
new FileHeaderNameEqualityComparer());
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization =
filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
}
// Early break. There are no files to synchronize to the selected destination.
if (!filteredFilesToSynchronization.Any())
{
return;
}
var baseUrl = commands.UrlFor();
foreach (var fileHeader in filteredFilesToSynchronization)
{
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destination.Commands.GetMetadataForAsync(file);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
baseUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, baseUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value <Guid>(Constants.MetadataEtagField);
await destination.IncrementLastETagAsync(storage.Id, baseUrl, etag);
RemoveSyncingConfiguration(file, baseUrl);
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(baseUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = baseUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
}
}
private async Task <bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient,
SourceSynchronizationInformation synchronizationInfo,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization),
FileHeaderNameEqualityComparer.Instance);
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization = filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
{
var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault();
if (lastFileBeforeFiltering == null)
{
return(true); // there are no more files that need
}
if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag)
{
return(true); // already updated etag on destination side
}
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false);
return(false); // all docs has been filtered out, update etag on destination side and retry
}
var destinationUrl = destinationSyncClient.BaseUrl;
bool enqueued = false;
var maxEtagOfFilteredDoc = Etag.Empty;
foreach (var fileHeader in filteredFilesToSynchronization)
{
context.CancellationToken.ThrowIfCancellationRequested();
var file = fileHeader.FullPath;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destinationUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' was not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription());
switch (reason)
{
case NoSyncReason.ContainedInDestinationHistory:
case NoSyncReason.DestinationFileConflicted:
case NoSyncReason.NoNeedToDeleteNonExistigFile:
var localEtag = Etag.Parse(localMetadata.Value <string>(Constants.MetadataEtagField));
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
RemoveSyncingConfiguration(file, destinationUrl);
}
else if (reason == NoSyncReason.DestinationFileConflicted)
{
if (needSyncingAgain.Contains(fileHeader, FileHeaderNameEqualityComparer.Instance) == false)
{
CreateSyncingConfiguration(fileHeader.Name, fileHeader.Etag, destinationUrl, SynchronizationType.Unknown);
}
}
else if (reason == NoSyncReason.NoNeedToDeleteNonExistigFile)
{
// after the upgrade to newer build there can be still an existing syncing configuration for it
RemoveSyncingConfiguration(file, destinationUrl);
}
if (EtagUtil.IsGreaterThan(localEtag, maxEtagOfFilteredDoc))
{
maxEtagOfFilteredDoc = localEtag;
}
break;
}
continue;
}
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
enqueued = true;
}
if (enqueued == false && EtagUtil.IsGreaterThan(maxEtagOfFilteredDoc, synchronizationInfo.LastSourceFileEtag))
{
await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, maxEtagOfFilteredDoc).ConfigureAwait(false);
return(false); // we bumped the last synced etag on a destination server, let it know it need to repeat the operation
}
return(true);
}
private async Task EnqueueMissingUpdatesAsync(SynchronizationClient destination,
SourceSynchronizationInformation lastEtag,
IList <FileHeader> needSyncingAgain)
{
LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
needSyncingAgain);
var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(lastEtag, 100),
new FileHeaderNameEqualityComparer());
LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
filesToSynchronization);
foreach (FileHeader needSyncing in needSyncingAgain)
{
filesToSynchronization.Add(needSyncing);
}
var filteredFilesToSynchronization =
filesToSynchronization.Where(
x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();
if (filesToSynchronization.Count > 0)
{
LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}",
filteredFilesToSynchronization);
}
if (filteredFilesToSynchronization.Count == 0)
{
return;
}
foreach (var fileHeader in filteredFilesToSynchronization)
{
var file = fileHeader.Name;
var localMetadata = GetLocalMetadata(file);
RavenJObject destinationMetadata;
try
{
destinationMetadata = await destination.GetMetadataForAsync(file);
}
catch (Exception ex)
{
Log.WarnException(
string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
destination.FileSystemUrl), ex);
continue;
}
NoSyncReason reason;
var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
if (work == null)
{
Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destination.FileSystemUrl, reason.GetDescription());
if (reason == NoSyncReason.ContainedInDestinationHistory)
{
var etag = localMetadata.Value <Guid>("ETag");
await destination.IncrementLastETagAsync(storage.Id, FileSystemUrl, etag);
RemoveSyncingConfiguration(file, destination.FileSystemUrl);
}
continue;
}
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work);
}
}
