private async Task <int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
{
var now = SystemTime.UtcNow;
var count = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = DocumentHelpers.GetRoughSize(document);
if (size > 1024 * 1024)
{
Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value <string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
{
continue;
}
if (options.MatchFilters(document) == false)
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
if (!string.IsNullOrEmpty(options.TransformScript))
{
document = await TransformDocument(document, options.TransformScript);
}
if (document == null)
{
continue;
}
PutDocument(document, options, (int)size);
count++;
if (count % options.BatchSize == 0)
{
ShowProgress("Read {0} documents", count);
}
}
PutDocument(null, options, -1); // force flush
return(count);
}
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0)
{
using (var documents = await GetDocuments(lastEtag, Math.Min(maxRecords, options.BatchSize)))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
{
errorcount++;
}
watch.Start();
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
protected virtual async Task <Etag> ExportDocuments(RavenConnectionStringOptions src, SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorsCount = 0;
var reachedMaxEtag = false;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try {
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0 && reachedMaxEtag == false)
{
using (var documents = await GetDocuments(src, lastEtag, Math.Min(options.BatchSize, maxRecords)))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
reachedMaxEtag = true;
break;
}
lastEtag = tempLastEtag;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
{
errorsCount++;
}
watch.Start();
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
options.Filters.ForEach(filter =>
{
if (filter.Path == "@metadata.Raven-Entity-Name")
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
doc.ToJson().WriteTo(jsonWriter);
totalCount++;
}
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try {
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0)
{
using (var documents = await GetDocuments(lastEtag, maxRecords))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
errorcount++;
watch.Start();
}
}
if (hasDocs)
continue;
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
}
private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
{
var now = SystemTime.UtcNow;
var count = 0;
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Docs", jsonReader.Value) == false)
throw new InvalidDataException("Docs property was expected");
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = GetRoughSize(document);
if (size > 1024 * 1024)
{
Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value<string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
continue;
if (options.MatchFilters(document) == false)
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
continue;
if (!string.IsNullOrEmpty(options.TransformScript))
document = await TransformDocument(document, options.TransformScript);
if (document == null)
continue;
await PutDocument(document);
count++;
if (count % options.BatchSize == 0)
{
ShowProgress("Read {0} documents", count);
}
}
await PutDocument(null); // force flush
return count;
}
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
using (var documents = await GetDocuments(lastEtag))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
var document = documents.Current;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
if (watch.ElapsedMilliseconds > 100)
{
errorcount++;
}
watch.Start();
}
}
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return(lastEtag);
}
}
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerOptions options, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = options.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
try
{
while (true)
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
continue;
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = options.Timeout.Seconds;
if (timeout < 30)
timeout = 30;
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (RavenJObject document in documents)
{
var metadata = document.Value<RavenJObject>("@metadata");
var id = metadata.Value<string>("@id");
var etag = Etag.Parse(metadata.Value<string>("@etag"));
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
continue;
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
break;
}
catch (Exception e)
{
if (retries-- == 0)
throw;
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
using (var documents = await GetDocuments(lastEtag))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
var document = documents.Current;
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
if (watch.ElapsedMilliseconds > 100)
errorcount++;
watch.Start();
}
}
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
}