private static async Task <DatabaseSmugglerOperationState> GetOperationStateAsync(DatabaseSmugglerOptions options, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination, CancellationToken cancellationToken) { DatabaseSmugglerOperationState state = null; if (destination.SupportsOperationState) { state = await destination .LoadOperationStateAsync(options, cancellationToken) .ConfigureAwait(false); } if (state == null) { state = new DatabaseSmugglerOperationState { LastDocsEtag = options.StartDocsEtag, LastDocDeleteEtag = options.StartDocsDeletionEtag, }; } Debug.Assert(state.LastDocsEtag != null); Debug.Assert(state.LastDocDeleteEtag != null); return(state); }
private async Task SaveOperationStateAsync(DatabaseSmugglerOperationState state, Etag etag, CancellationToken cancellationToken) { if (Destination.SupportsOperationState) { if (etag.CompareTo(state.LastDocsEtag) > 0) { state.LastDocsEtag = etag; } await Destination.SaveOperationStateAsync(Options, state, cancellationToken).ConfigureAwait(false); } }
private static void WriteLastEtagsToFile(DatabaseSmugglerOperationState state, string etagFileLocation) { using (var streamWriter = new StreamWriter(File.Create(etagFileLocation))) { new RavenJObject { { "LastDocEtag", state.LastDocsEtag.ToString() }, { "LastDocDeleteEtag", state.LastDocDeleteEtag.ToString() }, }.WriteTo(new JsonTextWriter(streamWriter)); streamWriter.Flush(); } }
public async Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(_options.ContinuationToken) == false) { var continuationDocId = "Raven/Smuggler/Continuation/" + _options.ContinuationToken; try { await _store .AsyncDatabaseCommands .PutAsync(continuationDocId, null, RavenJObject.FromObject(state), null, cancellationToken) .ConfigureAwait(false); } catch (Exception e) { if (options.IgnoreErrorsAndContinue == false) { throw; } _notifications.ShowProgress("Failed saving continuation state. Message: {0}", e.Message); } } }
public Task AfterExecuteAsync(DatabaseSmugglerOperationState state) { return(new CompletedTask()); }
private async Task ProcessSourceAsync(IDatabaseSmugglerSource source, DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(source.DisplayName) == false) { Notifications.ShowProgress("Processing source: {0}", source.DisplayName); } var maxEtags = await source .FetchCurrentMaxEtagsAsync(cancellationToken) .ConfigureAwait(false); while (true) { var type = await source .GetNextSmuggleTypeAsync(cancellationToken) .ConfigureAwait(false); switch (type) { case SmuggleType.None: return; case SmuggleType.Index: await new IndexSmuggler(_options, _notifications, source, _destination) .SmuggleAsync(state, cancellationToken) .ConfigureAwait(false); continue; case SmuggleType.Document: await new DocumentSmuggler(_options, _notifications, source, _destination, maxEtags) .SmuggleAsync(state, cancellationToken) .ConfigureAwait(false); continue; case SmuggleType.Transformer: await new TransformerSmuggler(_options, _notifications, source, _destination) .SmuggleAsync(state, cancellationToken) .ConfigureAwait(false); continue; case SmuggleType.DocumentDeletion: await new DocumentDeletionsSmuggler(_options, _notifications, source, _destination, maxEtags) .SmuggleAsync(state, cancellationToken) .ConfigureAwait(false); continue; case SmuggleType.Identity: await new IdentitySmuggler(_options, _notifications, source, _destination) .SmuggleAsync(state, cancellationToken) .ConfigureAwait(false); continue; case SmuggleType.Attachment: await new AttachmentSmuggler(_options, _notifications, source, _destination) .SmuggleAsync(state, cancellationToken) .ConfigureAwait(false); continue; case SmuggleType.AttachmentDeletion: await new AttachmentDeletionsSmuggler(_options, _notifications, source, _destination) .SmuggleAsync(state, cancellationToken) .ConfigureAwait(false); continue; default: throw new NotSupportedException(type.ToString()); } } }
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { await Source.SkipAttachmentsAsync(cancellationToken).ConfigureAwait(false); }
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { using (var actions = Destination.IndexActions()) { if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes) == false) { await Source.SkipIndexesAsync(cancellationToken).ConfigureAwait(false); return; } var count = 0; var retries = Source.SupportsRetries ? DatabaseSmuggler.NumberOfRetries : 1; var pageSize = Source.SupportsPaging ? Options.BatchSize : int.MaxValue; do { List <IndexDefinition> indexes; try { indexes = await Source.ReadIndexesAsync(count, pageSize, cancellationToken).ConfigureAwait(false); } catch (Exception e) { if (retries-- == 0 && Options.IgnoreErrorsAndContinue) { Notifications.ShowProgress("Failed getting indexes too much times, stopping the index export entirely. Message: {0}", e.Message); return; } if (Options.IgnoreErrorsAndContinue == false) { throw new SmugglerException(e.Message, e); } Notifications.ShowProgress("Failed fetching indexes. {0} retries remaining. Message: {1}", retries, e.Message); continue; } if (indexes.Count == 0) { Notifications.ShowProgress("Done with reading indexes, total: {0}", count); break; } count += indexes.Count; Notifications.ShowProgress("Reading batch of {0,3} indexes, read so far: {1,10:#,#;;0}", indexes.Count, count); foreach (var index in indexes) { try { if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes)) { await actions.WriteIndexAsync(index, cancellationToken).ConfigureAwait(false); } } catch (Exception e) { if (Options.IgnoreErrorsAndContinue == false) { throw new SmugglerException(e.Message, e); } Notifications.ShowProgress("Failed to export index {0}. Message: {1}", index, e.Message); } } } while (Source.SupportsPaging || Source.SupportsRetries); } }
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { using (var actions = Destination.IdentityActions()) { if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false) { await Source.SkipIdentitiesAsync(cancellationToken).ConfigureAwait(false); return; } int readCount = 0, filteredCount = 0, writeCount = 0; var retries = Source.SupportsRetries ? DatabaseSmuggler.NumberOfRetries : 1; do { List <KeyValuePair <string, long> > identities; try { identities = await Source.ReadIdentitiesAsync(cancellationToken).ConfigureAwait(false); } catch (Exception e) { if (retries-- == 0 && Options.IgnoreErrorsAndContinue) { Notifications.ShowProgress("Failed getting identities too much times, stopping the identity export entirely. Message: {0}", e.Message); return; } if (Options.IgnoreErrorsAndContinue == false) { throw new SmugglerException(e.Message, e); } Notifications.ShowProgress("Failed fetching identities. {0} retries remaining. Message: {1}", retries, e.Message); continue; } readCount += identities.Count; Notifications.ShowProgress("Exported {0} following identities: {1}", identities.Count, string.Join(", ", identities.Select(x => x.Key))); var filteredIdentities = identities.Where(x => FilterIdentity(x.Key, Options.OperateOnTypes)).ToList(); filteredCount += filteredIdentities.Count; Notifications.ShowProgress("After filtering {0} identities need to be exported: {1}", filteredIdentities.Count, string.Join(", ", filteredIdentities.Select(x => x.Key))); foreach (var identity in filteredIdentities) { try { await actions.WriteIdentityAsync(identity.Key, identity.Value, cancellationToken).ConfigureAwait(false); writeCount++; } catch (Exception e) { if (Options.IgnoreErrorsAndContinue == false) { throw new SmugglerException(e.Message, e); } Notifications.ShowProgress("Failed to export identity {0}. Message: {1}", identity, e.Message); } } break; } while (Source.SupportsRetries); Notifications.ShowProgress("IDENTITY. Read: {0}. Filtered: {1}. Wrote: {2}", readCount, readCount - filteredCount, writeCount); } }
public Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { throw new NotSupportedException(); }
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { using (var actions = Destination.DocumentActions()) { if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false) { await Source.SkipDocumentsAsync(cancellationToken).ConfigureAwait(false); return; } var afterEtag = state.LastDocsEtag; var maxEtag = _maxEtags.LastDocsEtag; var now = SystemTime.UtcNow; var totalCount = 0; var reachedMaxEtag = false; var affectedCollections = new List <string>(); Options.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { affectedCollections.Add(collectionName); }); } }); do { var hasDocs = false; try { var maxRecords = Options.Limit - totalCount; if (maxRecords > 0 && reachedMaxEtag == false) { var pageSize = Source.SupportsPaging ? Math.Min(Options.BatchSize, maxRecords) : int.MaxValue; using (var documents = await Source.ReadDocumentsAfterAsync(afterEtag, pageSize, cancellationToken).ConfigureAwait(false)) { while (await documents.MoveNextAsync().ConfigureAwait(false)) { hasDocs = true; var currentDocument = documents.Current; var currentEtag = Etag.Parse(currentDocument.Value <RavenJObject>("@metadata").Value <string>("@etag")); var currentKey = currentDocument["@metadata"].Value <string>("@id"); Notifications.OnDocumentRead(this, currentKey); if (maxEtag != null && currentEtag.CompareTo(maxEtag) > 0) { reachedMaxEtag = true; break; } afterEtag = currentEtag; if (Options.MatchFilters(currentDocument) == false) { if (affectedCollections.Count <= 0) { continue; } if (currentDocument.ContainsKey("@metadata") == false) { continue; } if (currentKey == null) { continue; } var isHilo = currentKey.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase); if (isHilo && Source.SupportsReadingHiLoDocuments) { continue; } if (isHilo == false || affectedCollections.Any(x => currentKey.EndsWith("/" + x, StringComparison.OrdinalIgnoreCase)) == false) { continue; } } if (Options.ShouldExcludeExpired && Options.ExcludeExpired(currentDocument, now)) { continue; } if (string.IsNullOrEmpty(Options.TransformScript) == false) { currentDocument = await TransformDocumentAsync(currentDocument).ConfigureAwait(false); } // If document is null after a transform we skip it. if (currentDocument == null) { continue; } var metadata = currentDocument["@metadata"] as RavenJObject; if (metadata != null) { if (Options.SkipConflicted && metadata.ContainsKey(Constants.RavenReplicationConflictDocument)) { continue; } if (Options.StripReplicationInformation) { currentDocument["@metadata"] = SmugglerHelper.StripReplicationInformationFromMetadata(metadata); //TODO arek - why there three methods are in SmugglerHelper, they are used just here } if (Options.ShouldDisableVersioningBundle) { currentDocument["@metadata"] = SmugglerHelper.DisableVersioning(metadata); } currentDocument["@metadata"] = SmugglerHelper.HandleConflictDocuments(metadata); } try { await actions.WriteDocumentAsync(currentDocument, cancellationToken).ConfigureAwait(false); Notifications.OnDocumentWrite(this, currentKey); } catch (Exception e) { if (Options.IgnoreErrorsAndContinue == false) { throw; } Notifications.ShowProgress(DatabaseSmugglerMessages.Documents_Write_Failure, currentKey, e.Message); } totalCount++; if (totalCount % Options.BatchSize == 0) { await SaveOperationStateAsync(state, currentEtag, cancellationToken).ConfigureAwait(false); // Wait for the batch to be indexed before continue. if (Destination.SupportsWaitingForIndexing) { await Destination.WaitForIndexingAsOfLastWriteAsync(cancellationToken).ConfigureAwait(false); } } } } if (hasDocs) { continue; } } if (Source.SupportsReadingHiLoDocuments) { // Load HiLo documents for selected collections foreach (var filter in Options.Filters) { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase) == false) { continue; } foreach (var collectionName in filter.Values) { var doc = await Source .ReadDocumentAsync("Raven/HiLo/" + collectionName, cancellationToken) .ConfigureAwait(false); if (doc == null) { continue; } await actions.WriteDocumentAsync(doc, cancellationToken).ConfigureAwait(false); totalCount++; } } } await SaveOperationStateAsync(state, afterEtag, cancellationToken).ConfigureAwait(false); state.LastDocsEtag = afterEtag; return; } catch (Exception e) { throw new SmugglerException(e.Message, e) { LastEtag = afterEtag }; } } while (Source.SupportsPaging); } }
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { using (var actions = Destination.DocumentDeletionActions()) { if (Source.SupportsDocumentDeletions == false) { return; } if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false) { await Source.SkipDocumentDeletionsAsync(cancellationToken).ConfigureAwait(false); return; } var maxEtag = _maxEtags.LastDocDeleteEtag?.IncrementBy(1); List <KeyValuePair <string, Etag> > deletions = null; try { deletions = await Source .ReadDocumentDeletionsAsync(state.LastDocDeleteEtag, maxEtag, cancellationToken) .ConfigureAwait(false); } catch (Exception e) { if (Options.IgnoreErrorsAndContinue == false) { throw; } Notifications.ShowProgress(DatabaseSmugglerMessages.DocumentDeletions_Read_Failure, e.Message); } if (deletions == null || deletions.Count == 0) { return; } var lastEtag = state.LastDocDeleteEtag; foreach (var deletion in deletions) { Notifications.OnDocumentDeletionRead(this, deletion.Key); try { await actions .WriteDocumentDeletionAsync(deletion.Key, cancellationToken) .ConfigureAwait(false); } catch (Exception e) { if (Options.IgnoreErrorsAndContinue == false) { throw; } Notifications.ShowProgress(DatabaseSmugglerMessages.DocumentDeletions_Write_Failure, deletion.Key, e.Message); } Notifications.OnDocumentDeletionWrite(this, deletion.Key); lastEtag = deletion.Value; } state.LastDocDeleteEtag = lastEtag; } }
public abstract Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken);
public override Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { if (_options.Incremental) { var etagFileLocation = Path.Combine(_path, IncrementalExportStateFile); WriteLastEtagsToFile(state, etagFileLocation); } return(new CompletedTask()); }
public override Task AfterExecuteAsync(DatabaseSmugglerOperationState state) { state.FilePath = _outputFilePath; return(base.AfterExecuteAsync(state)); }