public override async Task SmuggleAsync(FileSystemSmugglerOperationState state, CancellationToken cancellationToken) { var written = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); Notifications.ShowProgress("Exporting Configurations"); using (var writer = Destination.WriteConfigurations()) { while (true) { bool hasConfigs = false; foreach (var configItem in await Source.GetConfigurations(written, Options.BatchSize).ConfigureAwait(false)) { if (internalConfigs.IsMatch(configItem.Key)) { continue; } if (Options.StripReplicationInformation) { if (configItem.Key.Equals(SynchronizationConstants.RavenSynchronizationVersionHiLo, StringComparison.OrdinalIgnoreCase)) { continue; } } hasConfigs = true; var config = configItem.Value; if (string.Equals(configItem.Key, SynchronizationConstants.RavenSynchronizationDestinations, StringComparison.OrdinalIgnoreCase)) { config = DisableSynchronizationDestinations(config); } await writer.WriteConfigurationAsync(configItem.Key, config).ConfigureAwait(false); written++; if (written % 100 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { Notifications.ShowProgress("Exported {0} configurations. ", written); lastReport = SystemTime.UtcNow; } } if (hasConfigs == false) { break; } } } Notifications.ShowProgress("Done with exporting configurations"); }
public Task AfterExecuteAsync(FileSystemSmugglerOperationState state) { if (incremental) { WriteLastEtagsToFile(state, Path.GetDirectoryName(path)); } state.OutputPath = path; return(new CompletedTask()); }
public static void WriteLastEtagsToFile(FileSystemSmugglerOperationState state, string backupPath) { var etagFileLocation = Path.Combine(backupPath, IncrementalExportStateFile); using (var streamWriter = new StreamWriter(File.Create(etagFileLocation))) { new RavenJObject { { "LastFileEtag", state.LastFileEtag.ToString() }, { "LastDeletedFileEtag", state.LastDeletedFileEtag.ToString() }, }.WriteTo(new JsonTextWriter(streamWriter)); } }
public async Task <FileSystemSmugglerOperationState> ExecuteAsync(IFileSystemSmugglerSource source, IFileSystemSmugglerDestination destination, CancellationToken cancellationToken = default(CancellationToken)) { using (source) // TODO arek using (destination) { FileSystemSmugglerOperationState state = null; try { await source .InitializeAsync(options, cancellationToken) .ConfigureAwait(false); await destination .InitializeAsync(options, Notifications, cancellationToken) .ConfigureAwait(false); state = new FileSystemSmugglerOperationState { LastFileEtag = options.StartFilesEtag, LastDeletedFileEtag = options.StartFilesDeletionEtag, }; Debug.Assert(state.LastFileEtag != null); //TODO arek - for refactoring purposes Debug.Assert(state.LastDeletedFileEtag != null); await ProcessAsync(source, destination, state, cancellationToken).ConfigureAwait(false); await destination.AfterExecuteAsync(state).ConfigureAwait(false); return(state); } catch (SmugglerException e) { // TODO arek //source.OnException(e); destination.OnException(e); throw; } } }
public override async Task SmuggleAsync(FileSystemSmugglerOperationState state, CancellationToken cancellationToken) { var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); Notifications.ShowProgress("Exporting Files"); using (var writer = Destination.WriteFiles()) { try { while (true) { bool hasDocs = false; using (var files = await Source.GetFilesAsync(state.LastFileEtag, Options.BatchSize, cancellationToken).ConfigureAwait(false)) { while (await files.MoveNextAsync().ConfigureAwait(false)) { hasDocs = true; var file = files.Current; if (file.IsTombstone) { state.LastFileEtag = file.Etag; continue; } var tempLastEtag = file.Etag; // TODO arek - why do we need temp variable if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0) { break; } if (Options.StripReplicationInformation) { file.Metadata = StripReplicationInformationFromMetadata(file.Metadata); } if (Options.ShouldDisableVersioningBundle) { file.Metadata = DisableVersioning(file.Metadata); } using (var fileStream = await Source.DownloadFileAsync(file).ConfigureAwait(false)) { await writer.WriteFileAsync(file, fileStream).ConfigureAwait(false); } state.LastFileEtag = tempLastEtag; totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { //TODO: Show also the MB/sec and total GB exported. // TODO arek Notifications.ShowProgress("Exported {0} files. ", totalCount); lastReport = SystemTime.UtcNow; } } } if (hasDocs == false) { break; } } } catch (Exception e) { Notifications.ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); Notifications.ShowProgress("Done with reading files, total: {0}, lastEtag: {1}", totalCount, state.LastFileEtag); throw new SmugglerException(e.Message, e) { LastEtag = state.LastFileEtag }; } } Notifications.ShowProgress("Done with reading files, total: {0}, lastEtag: {1}", totalCount, state.LastFileEtag); }
private async Task ProcessAsync(IFileSystemSmugglerSource source, IFileSystemSmugglerDestination destination, FileSystemSmugglerOperationState state, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(source.DisplayName) == false) { Notifications.ShowProgress("Processing source: {0}", source.DisplayName); } var maxEtags = await source .FetchCurrentMaxEtagsAsync() .ConfigureAwait(false); foreach (var type in source.GetItemsToSmuggle()) { switch (type) { case SmuggleType.None: return; case SmuggleType.File: await new FileSmuggler(source, destination, options, Notifications, maxEtags.LastFileEtag).SmuggleAsync(state, cancellationToken).ConfigureAwait(false); continue; case SmuggleType.Configuration: await new ConfigurationSmuggler(source, destination, options, Notifications).SmuggleAsync(state, cancellationToken).ConfigureAwait(false); continue; default: throw new NotSupportedException(type.ToString()); } } }
public Task AfterExecuteAsync(FileSystemSmugglerOperationState state) { return(new CompletedTask()); }
public abstract Task SmuggleAsync(FileSystemSmugglerOperationState state, CancellationToken cancellationToken);