public override async Task ImportData(Stream stream, SmugglerOptions options)
{
SmugglerJintHelper.Initialize(options ?? SmugglerOptions);
var batchSize = options != null ? options.BatchSize : SmugglerOptions.BatchSize;
using (store = CreateStore())
{
Task disposeTask = null;
try
{
operation = store.BulkInsert(options: new BulkInsertOptions
{
BatchSize = batchSize,
CheckForUpdates = true
});
operation.Report += text => ShowProgress(text);
await base.ImportData(stream, options);
}
finally
{
disposeTask = operation.DisposeAsync();
}
if (disposeTask != null)
{
await disposeTask;
}
}
}
public void SmugglerTransformShouldRecognizeNumericPropertiesEvenThoughTheyHaveTheSameNames()
{
var helper = new SmugglerJintHelper();
helper.Initialize(new SmugglerOptions
{
TransformScript = emptyTransform
});
var testObject = new RavenJObject
{
{ "Range", new RavenJArray {
new RavenJObject {
{ "Min", 2.4 }
}
} },
{ "Min", 1 }
};
var transformedObject = helper.Transform(emptyTransform, testObject);
Assert.Equal(testObject["Min"].Type, transformedObject["Min"].Type);
Assert.Equal(((RavenJObject)((RavenJArray)testObject["Range"])[0])["Min"].Type, ((RavenJObject)((RavenJArray)transformedObject["Range"])[0])["Min"].Type);
Assert.True(RavenJToken.DeepEquals(testObject, transformedObject));
}
public override async Task ImportData(SmugglerImportOptions importOptions, SmugglerOptions options, Stream stream)
{
SetSmugglerOptions(options);
SmugglerJintHelper.Initialize(options);
using (store = CreateStore(importOptions.To))
{
Task disposeTask;
try
{
operation = new ChunkedBulkInsertOperation(store.DefaultDatabase, store, store.Listeners, new BulkInsertOptions
{
BatchSize = options.BatchSize,
OverwriteExisting = true
}, store.Changes(), options.ChunkSize, SmugglerOptions.DefaultDocumentSizeInChunkLimitInBytes);
operation.Report += text => ShowProgress(text);
await base.ImportData(importOptions, options, stream);
}
finally
{
disposeTask = operation.DisposeAsync();
}
if (disposeTask != null)
{
await disposeTask;
}
}
}
public DocumentSmuggler(DatabaseSmugglerOptions options, DatabaseSmugglerNotifications notifications, IDatabaseSmugglerSource source, IDatabaseSmugglerDestination destination, DatabaseLastEtagsInfo maxEtags)
: base(options, notifications, source, destination)
{
_maxEtags = maxEtags;
_patcher = new SmugglerJintHelper();
_patcher.Initialize(options);
}
public void SmugglerTransformShouldRecognizeNumericPropertiesEvenThoughTheyHaveTheSameNames()
{
var helper = new SmugglerJintHelper();
helper.Initialize(new SmugglerOptions
{
TransformScript = emptyTransform
});
var testObject = new RavenJObject
{
{"Range", new RavenJArray {new RavenJObject {{"Min", 2.4}}}},
{"Min", 1}
};
var transformedObject = helper.Transform(emptyTransform, testObject);
Assert.Equal(testObject["Min"].Type, transformedObject["Min"].Type);
Assert.Equal(((RavenJObject)((RavenJArray)testObject["Range"])[0])["Min"].Type, ((RavenJObject)((RavenJArray)transformedObject["Range"])[0])["Min"].Type);
Assert.True(RavenJToken.DeepEquals(testObject, transformedObject));
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
try
{
while (true)
{
try
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
{
continue;
}
metadata = document.Value <RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.Empty);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
{
return;
}
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
public RavenDB_1733()
{
jintHelper = new SmugglerJintHelper();
}
protected override Task <RavenJObject> TransformDocument(RavenJObject document, string transformScript)
{
return(new CompletedTask <RavenJObject>(SmugglerJintHelper.Transform(transformScript, document)));
}
private static async Task <Tuple <int, Etag, DateTime> > TransferStreamedDocuments(DocumentStore exportStore,
SmugglerDatabaseOptions databaseOptions,
DateTime now,
SmugglerJintHelper jintHelper,
BulkInsertOperation bulkInsertOperation,
TimeSpan reportInterval,
int totalCount,
string fromEtag,
DateTime lastReport)
{
Etag lastReadEtag = fromEtag;
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastReadEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
{
continue;
}
metadata = document.Value <RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
return(Tuple.Create(totalCount, lastReadEtag, lastReport));
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
var lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
var isLastLoop = false;
try
{
while (true)
{
try
{
var beforeCount = totalCount;
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
var res = await TransferStreamedDocuments(exportStore,
databaseOptions,
now,
jintHelper,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var res = await TransferDocumentsWithoutStreaming(exportStore,
databaseOptions,
exportBatchSize,
operationMetadata,
now,
bulkInsertOperation,
reportInterval,
totalCount,
lastEtag,
lastReport);
totalCount = res.Item1;
lastEtag = res.Item2;
lastReport = res.Item3;
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.Empty);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) <= 0 && !isLastLoop)
{
if (totalCount == beforeCount)
{
isLastLoop = true;
ShowProgress("Got no new results , trying one more loop from: {0}", lastEtag);
}
else
{
ShowProgress("Finished streaming batch, but haven't reached an end (last reached etag = {0})", lastEtag);
}
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
{
return;
}
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
