private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = databaseOptions.Limit - totalCount;
if (maxRecords > 0)
{
var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords);
using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false))
{
while (await documents.MoveNextAsync().ConfigureAwait(false))
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));
lastEtag = tempLastEtag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false);
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer);
if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0)
{
lastEtag = databaseStatistics.LastDocEtag;
}
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
var jsonDoc = doc.ToJson();
AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc)));
totalCount++;
}
});
}
});
await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
try
{
while (true)
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (RavenJObject document in documents)
{
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
break;
}
catch (Exception e)
{
if (retries-- == 0)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
private static async Task<Tuple<int, Etag, DateTime>> TransferStreamedDocuments(DocumentStore exportStore,
SmugglerDatabaseOptions databaseOptions,
DateTime now,
SmugglerJintHelper jintHelper,
BulkInsertOperation bulkInsertOperation,
TimeSpan reportInterval,
int totalCount,
string fromEtag,
DateTime lastReport)
{
Etag lastReadEtag = fromEtag;
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastReadEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
continue;
metadata = document.Value<RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
return Tuple.Create(totalCount, lastReadEtag, lastReport);
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
try
{
while (true)
{
try
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
{
continue;
}
metadata = document.Value <RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
{
return(Etag.Empty);
}
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
{
return;
}
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
private static async Task<Tuple<int, string, DateTime>> TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport)
{
var documents = await ((AsyncServerClient) exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject) jToken;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
fromEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
return Tuple.Create(totalCount, fromEtag, lastReport);
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
var jintHelper = new SmugglerJintHelper();
jintHelper.Initialize(databaseOptions);
try
{
while (true)
{
try
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
continue;
metadata = document.Value<RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
lastEtag = etag;
if (!databaseOptions.MatchFilters(document))
continue;
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
continue;
if (databaseOptions.StripReplicationInformation)
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
if (databaseOptions.ShouldDisableVersioningBundle)
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
break;
}
catch (Exception e)
{
if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
return Etag.Empty;
if (databaseOptions.IgnoreErrorsAndContinue == false)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
if (doc == null)
return;
doc.Metadata["@id"] = doc.Key;
bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
totalCount++;
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
try
{
while (true)
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
document.Remove("@metadata");
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = databaseOptions.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (RavenJObject document in documents)
{
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
break;
}
catch (Exception e)
{
if (retries-- == 0)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return(lastEtag);
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
private static async Task <Tuple <int, Etag, DateTime> > TransferStreamedDocuments(DocumentStore exportStore,
SmugglerDatabaseOptions databaseOptions,
DateTime now,
SmugglerJintHelper jintHelper,
BulkInsertOperation bulkInsertOperation,
TimeSpan reportInterval,
int totalCount,
string fromEtag,
DateTime lastReport)
{
Etag lastReadEtag = fromEtag;
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
lastReadEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
{
document = jintHelper.Transform(databaseOptions.TransformScript, document);
if (document == null)
{
continue;
}
metadata = document.Value <RavenJObject>("@metadata");
}
document.Remove("@metadata");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
return(Tuple.Create(totalCount, lastReadEtag, lastReport));
}
private static async Task <Tuple <int, string, DateTime> > TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport)
{
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata);
foreach (var jToken in documents)
{
var document = (RavenJObject)jToken;
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
fromEtag = etag;
if (!databaseOptions.MatchFilters(document))
{
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
try
{
bulkInsertOperation.Store(document, metadata, id);
}
catch (Exception e)
{
if (databaseOptions.IgnoreErrorsAndContinue == false)
{
throw;
}
ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
}
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
return(Tuple.Create(totalCount, fromEtag, lastReport));
}