private static async Task <DatabaseSmugglerOperationState> GetOperationStateAsync(DatabaseSmugglerOptions options, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination, CancellationToken cancellationToken)
{
DatabaseSmugglerOperationState state = null;
if (destination.SupportsOperationState)
{
state = await destination
.LoadOperationStateAsync(options, cancellationToken)
.ConfigureAwait(false);
}
if (state == null)
{
state = new DatabaseSmugglerOperationState
{
LastDocsEtag = options.StartDocsEtag,
LastDocDeleteEtag = options.StartDocsDeletionEtag,
};
}
Debug.Assert(state.LastDocsEtag != null);
Debug.Assert(state.LastDocDeleteEtag != null);
return(state);
}
private async Task SaveOperationStateAsync(DatabaseSmugglerOperationState state, Etag etag, CancellationToken cancellationToken)
{
if (Destination.SupportsOperationState)
{
if (etag.CompareTo(state.LastDocsEtag) > 0)
{
state.LastDocsEtag = etag;
}
await Destination.SaveOperationStateAsync(Options, state, cancellationToken).ConfigureAwait(false);
}
}
private static void WriteLastEtagsToFile(DatabaseSmugglerOperationState state, string etagFileLocation)
{
using (var streamWriter = new StreamWriter(File.Create(etagFileLocation)))
{
new RavenJObject
{
{ "LastDocEtag", state.LastDocsEtag.ToString() },
{ "LastDocDeleteEtag", state.LastDocDeleteEtag.ToString() },
}.WriteTo(new JsonTextWriter(streamWriter));
streamWriter.Flush();
}
}
public async Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(_options.ContinuationToken) == false)
{
var continuationDocId = "Raven/Smuggler/Continuation/" + _options.ContinuationToken;
try
{
await _store
.AsyncDatabaseCommands
.PutAsync(continuationDocId, null, RavenJObject.FromObject(state), null, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception e)
{
if (options.IgnoreErrorsAndContinue == false)
{
throw;
}
_notifications.ShowProgress("Failed saving continuation state. Message: {0}", e.Message);
}
}
}
public Task AfterExecuteAsync(DatabaseSmugglerOperationState state)
{
return(new CompletedTask());
}
private async Task ProcessSourceAsync(IDatabaseSmugglerSource source, DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(source.DisplayName) == false)
{
Notifications.ShowProgress("Processing source: {0}", source.DisplayName);
}
var maxEtags = await source
.FetchCurrentMaxEtagsAsync(cancellationToken)
.ConfigureAwait(false);
while (true)
{
var type = await source
.GetNextSmuggleTypeAsync(cancellationToken)
.ConfigureAwait(false);
switch (type)
{
case SmuggleType.None:
return;
case SmuggleType.Index:
await new IndexSmuggler(_options, _notifications, source, _destination)
.SmuggleAsync(state, cancellationToken)
.ConfigureAwait(false);
continue;
case SmuggleType.Document:
await new DocumentSmuggler(_options, _notifications, source, _destination, maxEtags)
.SmuggleAsync(state, cancellationToken)
.ConfigureAwait(false);
continue;
case SmuggleType.Transformer:
await new TransformerSmuggler(_options, _notifications, source, _destination)
.SmuggleAsync(state, cancellationToken)
.ConfigureAwait(false);
continue;
case SmuggleType.DocumentDeletion:
await new DocumentDeletionsSmuggler(_options, _notifications, source, _destination, maxEtags)
.SmuggleAsync(state, cancellationToken)
.ConfigureAwait(false);
continue;
case SmuggleType.Identity:
await new IdentitySmuggler(_options, _notifications, source, _destination)
.SmuggleAsync(state, cancellationToken)
.ConfigureAwait(false);
continue;
case SmuggleType.Attachment:
await new AttachmentSmuggler(_options, _notifications, source, _destination)
.SmuggleAsync(state, cancellationToken)
.ConfigureAwait(false);
continue;
case SmuggleType.AttachmentDeletion:
await new AttachmentDeletionsSmuggler(_options, _notifications, source, _destination)
.SmuggleAsync(state, cancellationToken)
.ConfigureAwait(false);
continue;
default:
throw new NotSupportedException(type.ToString());
}
}
}
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
await Source.SkipAttachmentsAsync(cancellationToken).ConfigureAwait(false);
}
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
using (var actions = Destination.IndexActions())
{
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes) == false)
{
await Source.SkipIndexesAsync(cancellationToken).ConfigureAwait(false);
return;
}
var count = 0;
var retries = Source.SupportsRetries ? DatabaseSmuggler.NumberOfRetries : 1;
var pageSize = Source.SupportsPaging ? Options.BatchSize : int.MaxValue;
do
{
List <IndexDefinition> indexes;
try
{
indexes = await Source.ReadIndexesAsync(count, pageSize, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
if (retries-- == 0 && Options.IgnoreErrorsAndContinue)
{
Notifications.ShowProgress("Failed getting indexes too much times, stopping the index export entirely. Message: {0}", e.Message);
return;
}
if (Options.IgnoreErrorsAndContinue == false)
{
throw new SmugglerException(e.Message, e);
}
Notifications.ShowProgress("Failed fetching indexes. {0} retries remaining. Message: {1}", retries, e.Message);
continue;
}
if (indexes.Count == 0)
{
Notifications.ShowProgress("Done with reading indexes, total: {0}", count);
break;
}
count += indexes.Count;
Notifications.ShowProgress("Reading batch of {0,3} indexes, read so far: {1,10:#,#;;0}", indexes.Count, count);
foreach (var index in indexes)
{
try
{
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes))
{
await actions.WriteIndexAsync(index, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception e)
{
if (Options.IgnoreErrorsAndContinue == false)
{
throw new SmugglerException(e.Message, e);
}
Notifications.ShowProgress("Failed to export index {0}. Message: {1}", index, e.Message);
}
}
} while (Source.SupportsPaging || Source.SupportsRetries);
}
}
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
using (var actions = Destination.IdentityActions())
{
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false)
{
await Source.SkipIdentitiesAsync(cancellationToken).ConfigureAwait(false);
return;
}
int readCount = 0, filteredCount = 0, writeCount = 0;
var retries = Source.SupportsRetries ? DatabaseSmuggler.NumberOfRetries : 1;
do
{
List <KeyValuePair <string, long> > identities;
try
{
identities = await Source.ReadIdentitiesAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
if (retries-- == 0 && Options.IgnoreErrorsAndContinue)
{
Notifications.ShowProgress("Failed getting identities too much times, stopping the identity export entirely. Message: {0}", e.Message);
return;
}
if (Options.IgnoreErrorsAndContinue == false)
{
throw new SmugglerException(e.Message, e);
}
Notifications.ShowProgress("Failed fetching identities. {0} retries remaining. Message: {1}", retries, e.Message);
continue;
}
readCount += identities.Count;
Notifications.ShowProgress("Exported {0} following identities: {1}", identities.Count, string.Join(", ", identities.Select(x => x.Key)));
var filteredIdentities = identities.Where(x => FilterIdentity(x.Key, Options.OperateOnTypes)).ToList();
filteredCount += filteredIdentities.Count;
Notifications.ShowProgress("After filtering {0} identities need to be exported: {1}", filteredIdentities.Count, string.Join(", ", filteredIdentities.Select(x => x.Key)));
foreach (var identity in filteredIdentities)
{
try
{
await actions.WriteIdentityAsync(identity.Key, identity.Value, cancellationToken).ConfigureAwait(false);
writeCount++;
}
catch (Exception e)
{
if (Options.IgnoreErrorsAndContinue == false)
{
throw new SmugglerException(e.Message, e);
}
Notifications.ShowProgress("Failed to export identity {0}. Message: {1}", identity, e.Message);
}
}
break;
} while (Source.SupportsRetries);
Notifications.ShowProgress("IDENTITY. Read: {0}. Filtered: {1}. Wrote: {2}", readCount, readCount - filteredCount, writeCount);
}
}
public Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
throw new NotSupportedException();
}
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
using (var actions = Destination.DocumentActions())
{
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false)
{
await Source.SkipDocumentsAsync(cancellationToken).ConfigureAwait(false);
return;
}
var afterEtag = state.LastDocsEtag;
var maxEtag = _maxEtags.LastDocsEtag;
var now = SystemTime.UtcNow;
var totalCount = 0;
var reachedMaxEtag = false;
var affectedCollections = new List <string>();
Options.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
affectedCollections.Add(collectionName);
});
}
});
do
{
var hasDocs = false;
try
{
var maxRecords = Options.Limit - totalCount;
if (maxRecords > 0 && reachedMaxEtag == false)
{
var pageSize = Source.SupportsPaging ? Math.Min(Options.BatchSize, maxRecords) : int.MaxValue;
using (var documents = await Source.ReadDocumentsAfterAsync(afterEtag, pageSize, cancellationToken).ConfigureAwait(false))
{
while (await documents.MoveNextAsync().ConfigureAwait(false))
{
hasDocs = true;
var currentDocument = documents.Current;
var currentEtag = Etag.Parse(currentDocument.Value <RavenJObject>("@metadata").Value <string>("@etag"));
var currentKey = currentDocument["@metadata"].Value <string>("@id");
Notifications.OnDocumentRead(this, currentKey);
if (maxEtag != null && currentEtag.CompareTo(maxEtag) > 0)
{
reachedMaxEtag = true;
break;
}
afterEtag = currentEtag;
if (Options.MatchFilters(currentDocument) == false)
{
if (affectedCollections.Count <= 0)
{
continue;
}
if (currentDocument.ContainsKey("@metadata") == false)
{
continue;
}
if (currentKey == null)
{
continue;
}
var isHilo = currentKey.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase);
if (isHilo && Source.SupportsReadingHiLoDocuments)
{
continue;
}
if (isHilo == false || affectedCollections.Any(x => currentKey.EndsWith("/" + x, StringComparison.OrdinalIgnoreCase)) == false)
{
continue;
}
}
if (Options.ShouldExcludeExpired && Options.ExcludeExpired(currentDocument, now))
{
continue;
}
if (string.IsNullOrEmpty(Options.TransformScript) == false)
{
currentDocument = await TransformDocumentAsync(currentDocument).ConfigureAwait(false);
}
// If document is null after a transform we skip it.
if (currentDocument == null)
{
continue;
}
var metadata = currentDocument["@metadata"] as RavenJObject;
if (metadata != null)
{
if (Options.SkipConflicted && metadata.ContainsKey(Constants.RavenReplicationConflictDocument))
{
continue;
}
if (Options.StripReplicationInformation)
{
currentDocument["@metadata"] = SmugglerHelper.StripReplicationInformationFromMetadata(metadata); //TODO arek - why there three methods are in SmugglerHelper, they are used just here
}
if (Options.ShouldDisableVersioningBundle)
{
currentDocument["@metadata"] = SmugglerHelper.DisableVersioning(metadata);
}
currentDocument["@metadata"] = SmugglerHelper.HandleConflictDocuments(metadata);
}
try
{
await actions.WriteDocumentAsync(currentDocument, cancellationToken).ConfigureAwait(false);
Notifications.OnDocumentWrite(this, currentKey);
}
catch (Exception e)
{
if (Options.IgnoreErrorsAndContinue == false)
{
throw;
}
Notifications.ShowProgress(DatabaseSmugglerMessages.Documents_Write_Failure, currentKey, e.Message);
}
totalCount++;
if (totalCount % Options.BatchSize == 0)
{
await SaveOperationStateAsync(state, currentEtag, cancellationToken).ConfigureAwait(false);
// Wait for the batch to be indexed before continue.
if (Destination.SupportsWaitingForIndexing)
{
await Destination.WaitForIndexingAsOfLastWriteAsync(cancellationToken).ConfigureAwait(false);
}
}
}
}
if (hasDocs)
{
continue;
}
}
if (Source.SupportsReadingHiLoDocuments)
{
// Load HiLo documents for selected collections
foreach (var filter in Options.Filters)
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase) == false)
{
continue;
}
foreach (var collectionName in filter.Values)
{
var doc = await Source
.ReadDocumentAsync("Raven/HiLo/" + collectionName, cancellationToken)
.ConfigureAwait(false);
if (doc == null)
{
continue;
}
await actions.WriteDocumentAsync(doc, cancellationToken).ConfigureAwait(false);
totalCount++;
}
}
}
await SaveOperationStateAsync(state, afterEtag, cancellationToken).ConfigureAwait(false);
state.LastDocsEtag = afterEtag;
return;
}
catch (Exception e)
{
throw new SmugglerException(e.Message, e)
{
LastEtag = afterEtag
};
}
} while (Source.SupportsPaging);
}
}
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
using (var actions = Destination.DocumentDeletionActions())
{
if (Source.SupportsDocumentDeletions == false)
{
return;
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false)
{
await Source.SkipDocumentDeletionsAsync(cancellationToken).ConfigureAwait(false);
return;
}
var maxEtag = _maxEtags.LastDocDeleteEtag?.IncrementBy(1);
List <KeyValuePair <string, Etag> > deletions = null;
try
{
deletions = await Source
.ReadDocumentDeletionsAsync(state.LastDocDeleteEtag, maxEtag, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception e)
{
if (Options.IgnoreErrorsAndContinue == false)
{
throw;
}
Notifications.ShowProgress(DatabaseSmugglerMessages.DocumentDeletions_Read_Failure, e.Message);
}
if (deletions == null || deletions.Count == 0)
{
return;
}
var lastEtag = state.LastDocDeleteEtag;
foreach (var deletion in deletions)
{
Notifications.OnDocumentDeletionRead(this, deletion.Key);
try
{
await actions
.WriteDocumentDeletionAsync(deletion.Key, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception e)
{
if (Options.IgnoreErrorsAndContinue == false)
{
throw;
}
Notifications.ShowProgress(DatabaseSmugglerMessages.DocumentDeletions_Write_Failure, deletion.Key, e.Message);
}
Notifications.OnDocumentDeletionWrite(this, deletion.Key);
lastEtag = deletion.Value;
}
state.LastDocDeleteEtag = lastEtag;
}
}
public abstract Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken);
public override Task SaveOperationStateAsync(DatabaseSmugglerOptions options, DatabaseSmugglerOperationState state, CancellationToken cancellationToken)
{
if (_options.Incremental)
{
var etagFileLocation = Path.Combine(_path, IncrementalExportStateFile);
WriteLastEtagsToFile(state, etagFileLocation);
}
return(new CompletedTask());
}
public override Task AfterExecuteAsync(DatabaseSmugglerOperationState state)
{
state.FilePath = _outputFilePath;
return(base.AfterExecuteAsync(state));
}
