public void ImportData(Stream stream, SmugglerOptions options)
{
var sw = Stopwatch.StartNew();
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
var streamReader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress));
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
catch (InvalidDataException)
{
stream.Seek(0, SeekOrigin.Begin);
var streamReader = new StreamReader(stream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
if (jsonReader.TokenType != JsonToken.StartObject)
throw new InvalidDataException("StartObject was expected");
// should read indexes now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Indexes", jsonReader.Value) == false)
throw new InvalidDataException("Indexes property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = RavenJToken.ReadFrom(jsonReader);
if (options.OperateOnTypes.HasFlag(ItemType.Indexes) == false)
continue;
var indexName = index.Value<string>("name");
if (indexName.StartsWith("Raven/") || indexName.StartsWith("Temp/"))
continue;
var request = CreateRequest("indexes/" + indexName, "PUT");
request.Write(index.Value<RavenJObject>("definition"));
request.ExecuteRequest();
}
// should read documents now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Docs", jsonReader.Value) == false)
throw new InvalidDataException("Docs property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
var batch = new List<RavenJObject>();
int totalCount = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
if (options.OperateOnTypes.HasFlag(ItemType.Documents) == false)
continue;
if (options.MatchFilters(document) == false)
continue;
totalCount += 1;
batch.Add(document);
if (batch.Count >= 128)
FlushBatch(batch);
}
FlushBatch(batch);
var attachmentCount = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Attachments", jsonReader.Value) == false)
throw new InvalidDataException("Attachment property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
attachmentCount += 1;
var item = RavenJToken.ReadFrom(jsonReader);
if (options.OperateOnTypes.HasFlag(ItemType.Attachments) == false)
continue;
var attachmentExportInfo =
new JsonSerializer
{
Converters = { new TrivialJsonToJsonJsonConverter() }
}.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item));
Console.WriteLine("Importing attachment {0}", attachmentExportInfo.Key);
var request = CreateRequest("static/" + attachmentExportInfo.Key, "PUT");
if (attachmentExportInfo.Metadata != null)
{
foreach (var header in attachmentExportInfo.Metadata)
{
request.WebRequest.Headers.Add(header.Key, StripQuotesIfNeeded(header.Value));
}
}
request.Write(attachmentExportInfo.Data);
request.ExecuteRequest();
}
Console.WriteLine("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
}
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerOptions options, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
{
var now = SystemTime.UtcNow;
string lastEtag = options.StartDocsEtag;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
{
BatchSize = importBatchSize,
OverwriteExisting = true,
});
bulkInsertOperation.Report += text => ShowProgress(text);
try
{
while (true)
{
if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
{
ShowProgress("Streaming documents from " + lastEtag);
using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
{
while (await documentsEnumerator.MoveNextAsync())
{
var document = documentsEnumerator.Current;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
document.Remove("@metadata");
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
}
}
else
{
int retries = RetriesCount;
var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
var timeout = options.Timeout.Seconds;
if (timeout < 30)
{
timeout = 30;
}
try
{
var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);
while (true)
{
try
{
ShowProgress("Get documents from " + lastEtag);
var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
foreach (RavenJObject document in documents)
{
var metadata = document.Value <RavenJObject>("@metadata");
var id = metadata.Value <string>("@id");
var etag = Etag.Parse(metadata.Value <string>("@etag"));
document.Remove("@metadata");
metadata.Remove("@id");
metadata.Remove("@etag");
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
bulkInsertOperation.Store(document, metadata, id);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = etag;
}
break;
}
catch (Exception e)
{
if (retries-- == 0)
{
throw;
}
exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
}
}
}
finally
{
exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
}
}
// In a case that we filter all the results, the formEtag hasn't updaed to the latest,
// but we still need to continue until we finish all the docs.
var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return(lastEtag);
}
}
finally
{
bulkInsertOperation.Dispose();
}
}
public void ImportData(Stream stream, SmugglerOptions options)
{
EnsureDatabaseExists();
var sw = Stopwatch.StartNew();
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
var streamReader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress));
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
catch (InvalidDataException)
{
stream.Seek(0, SeekOrigin.Begin);
var streamReader = new StreamReader(stream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
// should read indexes now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Indexes", jsonReader.Value) == false)
{
throw new InvalidDataException("Indexes property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = RavenJToken.ReadFrom(jsonReader);
if (options.OperateOnTypes.HasFlag(ItemType.Indexes) == false)
{
continue;
}
var indexName = index.Value <string>("name");
if (indexName.StartsWith("Raven/") || indexName.StartsWith("Temp/"))
{
continue;
}
var request = CreateRequest("/indexes/" + indexName, "PUT");
request.Write(index.Value <RavenJObject>("definition"));
request.ExecuteRequest();
}
// should read documents now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Docs", jsonReader.Value) == false)
{
throw new InvalidDataException("Docs property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
var batch = new List <RavenJObject>();
int totalCount = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
if (options.OperateOnTypes.HasFlag(ItemType.Documents) == false)
{
continue;
}
if (options.MatchFilters(document) == false)
{
continue;
}
totalCount += 1;
batch.Add(document);
if (batch.Count >= 128)
{
FlushBatch(batch);
}
}
FlushBatch(batch);
var attachmentCount = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Attachments", jsonReader.Value) == false)
{
throw new InvalidDataException("Attachment property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
attachmentCount += 1;
var item = RavenJToken.ReadFrom(jsonReader);
if (options.OperateOnTypes.HasFlag(ItemType.Attachments) == false)
{
continue;
}
var attachmentExportInfo =
new JsonSerializer
{
Converters = { new TrivialJsonToJsonJsonConverter() }
}.Deserialize <AttachmentExportInfo>(new RavenJTokenReader(item));
Console.WriteLine("Importing attachment {0}", attachmentExportInfo.Key);
var request = CreateRequest("/static/" + attachmentExportInfo.Key, "PUT");
if (attachmentExportInfo.Metadata != null)
{
foreach (var header in attachmentExportInfo.Metadata)
{
switch (header.Key)
{
case "Content-Type":
request.WebRequest.ContentType = header.Value.Value <string>();
break;
default:
request.WebRequest.Headers.Add(header.Key, StripQuotesIfNeeded(header.Value));
break;
}
}
}
request.Write(attachmentExportInfo.Data);
request.ExecuteRequest();
}
Console.WriteLine("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
}