private IEnumerable<FileHeader> GetFilesToSynchronization(SourceSynchronizationInformation destinationsSynchronizationInformationForSource, int take) { var filesToSynchronization = new List<FileHeader>(); Log.Debug("Getting files to synchronize with ETag greater than {0} [parameter take = {1}]", destinationsSynchronizationInformationForSource.LastSourceFileEtag, take); try { storage.Batch( accessor => filesToSynchronization = accessor.GetFilesAfter(destinationsSynchronizationInformationForSource.LastSourceFileEtag, take).ToList()); } catch (Exception e) { Log.WarnException( string.Format("Could not get files to synchronize after: " + destinationsSynchronizationInformationForSource.LastSourceFileEtag), e); } return filesToSynchronization; }
private async Task<bool> EnqueueMissingUpdatesAsync(ISynchronizationServerClient destinationSyncClient, SourceSynchronizationInformation synchronizationInfo, IList<FileHeader> needSyncingAgain) { LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}", needSyncingAgain); var filesToSynchronization = new HashSet<FileHeader>(GetFilesToSynchronization(synchronizationInfo.LastSourceFileEtag, NumberOfFilesToCheckForSynchronization), FileHeaderNameEqualityComparer.Instance); LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}", filesToSynchronization); foreach (FileHeader needSyncing in needSyncingAgain) { filesToSynchronization.Add(needSyncing); } var filteredFilesToSynchronization = filesToSynchronization.Where( x => synchronizationStrategy.Filter(x, synchronizationInfo.DestinationServerId, filesToSynchronization)).ToList(); if (filesToSynchronization.Count > 0) LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization); if (filteredFilesToSynchronization.Count == 0) { var lastFileBeforeFiltering = filesToSynchronization.LastOrDefault(); if (lastFileBeforeFiltering == null) return true; // there are no more files that need if (lastFileBeforeFiltering.Etag == synchronizationInfo.LastSourceFileEtag) return true; // already updated etag on destination side await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, lastFileBeforeFiltering.Etag).ConfigureAwait(false); return false; // all docs has been filtered out, update etag on destination side and retry } var destinationUrl = destinationSyncClient.BaseUrl; bool enqueued = true; foreach (var fileHeader in filteredFilesToSynchronization) { context.CancellationToken.ThrowIfCancellationRequested(); var file = fileHeader.FullPath; var localMetadata = GetLocalMetadata(file); RavenJObject destinationMetadata; try { destinationMetadata = await destinationSyncClient.GetMetadataForAsync(file).ConfigureAwait(false); } catch (Exception ex) { Log.WarnException( string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file, destinationUrl), ex); continue; } NoSyncReason reason; var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason); if (work == null) { Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, destinationUrl, reason.GetDescription()); if (reason == NoSyncReason.ContainedInDestinationHistory) { var etag = localMetadata.Value<Guid>(Constants.MetadataEtagField); await destinationSyncClient.IncrementLastETagAsync(storage.Id, FileSystemUrl, etag).ConfigureAwait(false); RemoveSyncingConfiguration(file, destinationUrl); enqueued = false; } continue; } if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) { publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = destinationUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Enqueue, Direction = SynchronizationDirection.Outgoing }); } enqueued = true; } return enqueued; }
private async Task EnqueueMissingUpdatesAsync(IAsyncFilesSynchronizationCommands destination, SourceSynchronizationInformation lastEtag, IList<FileHeader> needSyncingAgain) { LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}", needSyncingAgain); var commands = (IAsyncFilesCommandsImpl)destination.Commands; var filesToSynchronization = new HashSet<FileHeader>(GetFilesToSynchronization(lastEtag, 100), new FileHeaderNameEqualityComparer()); LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}", filesToSynchronization); foreach (FileHeader needSyncing in needSyncingAgain) { filesToSynchronization.Add(needSyncing); } var filteredFilesToSynchronization = filesToSynchronization.Where( x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList(); if (filesToSynchronization.Count > 0) LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization); // Early break. There are no files to synchronize to the selected destination. if (!filteredFilesToSynchronization.Any()) return; var baseUrl = commands.UrlFor(); foreach (var fileHeader in filteredFilesToSynchronization) { var file = fileHeader.FullPath; var localMetadata = GetLocalMetadata(file); RavenJObject destinationMetadata; try { destinationMetadata = await destination.Commands.GetMetadataForAsync(file); } catch (Exception ex) { Log.WarnException( string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file, baseUrl), ex); continue; } NoSyncReason reason; var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason); if (work == null) { Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, baseUrl, reason.GetDescription()); if (reason == NoSyncReason.ContainedInDestinationHistory) { var etag = localMetadata.Value<Guid>(Constants.MetadataEtagField); await destination.IncrementLastETagAsync(storage.Id, baseUrl, etag); RemoveSyncingConfiguration(file, baseUrl); } continue; } if (synchronizationQueue.EnqueueSynchronization(baseUrl, work)) { publisher.Publish(new SynchronizationUpdateNotification { FileName = work.FileName, DestinationFileSystemUrl = baseUrl, SourceServerId = storage.Id, SourceFileSystemUrl = FileSystemUrl, Type = work.SynchronizationType, Action = SynchronizationAction.Enqueue, Direction = SynchronizationDirection.Outgoing }); } } }
private void SaveSynchronizationSourceInformation(FileSystemInfo sourceFileSystem, Etag lastSourceEtag) { var lastSynchronizationInformation = GetLastSynchronization(sourceFileSystem.Id); if (EtagUtil.IsGreaterThan(lastSynchronizationInformation.LastSourceFileEtag, lastSourceEtag)) { return; } var synchronizationSourceInfo = new SourceSynchronizationInformation { LastSourceFileEtag = lastSourceEtag, SourceServerUrl = sourceFileSystem.Url, DestinationServerId = Storage.Id }; var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceFileSystem.Id; Storage.Batch(accessor => accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo))); Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceFileSystem.Url, sourceFileSystem.Id); }
public SourceSynchronizationInformation GetLastSynchronization(Guid from) { SourceSynchronizationInformation info = null; try { Storage.Batch(accessor => { info = accessor.GetConfig(SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + from) .JsonDeserialization<SourceSynchronizationInformation>(); }); } catch (FileNotFoundException) { info = new SourceSynchronizationInformation { LastSourceFileEtag = Etag.Empty, DestinationServerId = Storage.Id }; } return info; }
private void SaveSynchronizationSourceInformation(ServerInfo sourceServer, Guid lastSourceEtag, IStorageActionsAccessor accessor) { var lastSynchronizationInformation = GetLastSynchronization(sourceServer.Id, accessor); if (Buffers.Compare(lastSynchronizationInformation.LastSourceFileEtag.ToByteArray(), lastSourceEtag.ToByteArray()) > 0) { return; } var synchronizationSourceInfo = new SourceSynchronizationInformation { LastSourceFileEtag = lastSourceEtag, SourceServerUrl = sourceServer.FileSystemUrl, DestinationServerId = Storage.Id }; var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceServer.Id; accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo)); Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceServer.FileSystemUrl, sourceServer.Id); }
private SourceSynchronizationInformation GetLastSynchronization(Guid from, IStorageActionsAccessor accessor) { SourceSynchronizationInformation info; try { info = accessor.GetConfig(SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + from) .JsonDeserialization<SourceSynchronizationInformation>(); } catch (FileNotFoundException) { info = new SourceSynchronizationInformation { LastSourceFileEtag = Guid.Empty, DestinationServerId = Storage.Id }; } return info; }