| public MatchFilters ( RavenJToken item ) : bool | ||
| item | RavenJToken | |
| return | bool | |
private async Task <int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
{
var now = SystemTime.UtcNow;
var count = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = DocumentHelpers.GetRoughSize(document);
if (size > 1024 * 1024)
{
Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value <string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
{
continue;
}
if (options.MatchFilters(document) == false)
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
if (!string.IsNullOrEmpty(options.TransformScript))
{
document = await TransformDocument(document, options.TransformScript);
}
if (document == null)
{
continue;
}
PutDocument(document, options, (int)size);
count++;
if (count % options.BatchSize == 0)
{
ShowProgress("Read {0} documents", count);
}
}
PutDocument(null, options, -1); // force flush
return(count);
}
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0)
{
using (var documents = await GetDocuments(lastEtag, Math.Min(maxRecords, options.BatchSize)))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
{
errorcount++;
}
watch.Start();
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true)
{
EnsureDatabaseExists();
var sw = Stopwatch.StartNew();
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
var streamReader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress));
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
catch (InvalidDataException)
{
stream.Seek(0, SeekOrigin.Begin);
var streamReader = new StreamReader(stream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
if (jsonReader.TokenType != JsonToken.StartObject)
throw new InvalidDataException("StartObject was expected");
// should read indexes now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Indexes", jsonReader.Value) == false)
throw new InvalidDataException("Indexes property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes)
continue;
var indexName = index.Value<string>("name");
if (indexName.StartsWith("Raven/") || indexName.StartsWith("Temp/"))
continue;
PutIndex(indexName, index);
}
// should read documents now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Docs", jsonReader.Value) == false)
throw new InvalidDataException("Docs property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
var batch = new List<RavenJObject>();
int totalCount = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
continue;
if (options.MatchFilters(document) == false)
continue;
totalCount += 1;
batch.Add(document);
if (batch.Count >= 128)
FlushBatch(batch);
}
FlushBatch(batch);
var attachmentCount = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Attachments", jsonReader.Value) == false)
throw new InvalidDataException("Attachment property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
attachmentCount += 1;
var item = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments)
continue;
var attachmentExportInfo =
new JsonSerializer
{
Converters = { new TrivialJsonToJsonJsonConverter() }
}.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item));
ShowProgress("Importing attachment {0}", attachmentExportInfo.Key);
PutAttachment(attachmentExportInfo);
}
ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
}
protected virtual async Task <Etag> ExportDocuments(RavenConnectionStringOptions src, SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
var now = SystemTime.UtcNow;
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorsCount = 0;
var reachedMaxEtag = false;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try {
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0 && reachedMaxEtag == false)
{
using (var documents = await GetDocuments(src, lastEtag, Math.Min(options.BatchSize, maxRecords)))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));
if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0)
{
reachedMaxEtag = true;
break;
}
lastEtag = tempLastEtag;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
{
errorsCount++;
}
watch.Start();
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
options.Filters.ForEach(filter =>
{
if (filter.Path == "@metadata.Raven-Entity-Name")
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
doc.ToJson().WriteTo(jsonWriter);
totalCount++;
}
});
}
});
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try {
var maxRecords = options.Limit - totalCount;
if (maxRecords > 0)
{
using (var documents = await GetDocuments(lastEtag, maxRecords))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
hasDocs = true;
var document = documents.Current;
lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
if (watch.ElapsedMilliseconds > 100)
errorcount++;
watch.Start();
}
}
if (hasDocs)
continue;
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return lastEtag;
}
}
private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
{
var count = 0;
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Docs", jsonReader.Value) == false)
throw new InvalidDataException("Docs property was expected");
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = GetRoughSize(document);
if (size > 1024 * 1024)
{
Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value<string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
continue;
if (options.MatchFilters(document) == false)
continue;
if (!string.IsNullOrEmpty(options.TransformScript))
document = await TransformDocument(document, options.TransformScript);
if (document == null)
continue;
await PutDocument(document);
count++;
if (count % options.BatchSize == 0)
{
ShowProgress("Read {0} documents", count);
}
}
await PutDocument(null); // force flush
return count;
}
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
using (var documents = await GetDocuments(lastEtag))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
var document = documents.Current;
if (!options.MatchFilters(document))
{
continue;
}
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
{
continue;
}
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
if (watch.ElapsedMilliseconds > 100)
{
errorcount++;
}
watch.Start();
}
}
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return(lastEtag);
}
}
private async Task <int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
{
var count = 0;
if (jsonReader.Read() == false)
{
return(count);
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Docs", jsonReader.Value) == false)
{
throw new InvalidDataException("Docs property was expected");
}
if (jsonReader.Read() == false)
{
return(count);
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = GetRoughSize(document);
if (size > 1024 * 1024)
{
Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value <string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
{
continue;
}
if (options.MatchFilters(document) == false)
{
continue;
}
if (!string.IsNullOrEmpty(options.TransformScript))
{
document = await TransformDocument(document, options.TransformScript);
}
if (document == null)
{
continue;
}
await PutDocument(document);
count++;
if (count % options.BatchSize == 0)
{
ShowProgress("Read {0} documents", count);
}
}
await PutDocument(null); // force flush
return(count);
}
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true)
{
EnsureDatabaseExists();
Stream sizeStream;
var sw = Stopwatch.StartNew();
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));
var streamReader = new StreamReader(sizeStream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
catch (InvalidDataException)
{
sizeStream = stream;
stream.Seek(0, SeekOrigin.Begin);
var streamReader = new StreamReader(stream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
if (jsonReader.TokenType != JsonToken.StartObject)
throw new InvalidDataException("StartObject was expected");
// should read indexes now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Indexes", jsonReader.Value) == false)
throw new InvalidDataException("Indexes property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes)
continue;
var indexName = index.Value<string>("name");
if (indexName.StartsWith("Temp/"))
continue;
if (index.Value<RavenJObject>("definition").Value<bool>("IsCompiled"))
continue; // can't import compiled indexes
PutIndex(indexName, index);
}
// should read documents now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Docs", jsonReader.Value) == false)
throw new InvalidDataException("Docs property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
var batch = new List<RavenJObject>();
int totalCount = 0;
long lastFlushedAt = 0;
int batchCount = 0;
long sizeOnDisk = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var before = sizeStream.Position;
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = sizeStream.Position - before;
if (size > 1024 * 1024)
{
Console.WriteLine("{0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value<string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
continue;
if (options.MatchFilters(document) == false)
continue;
totalCount += 1;
batch.Add(document);
sizeOnDisk = (sizeStream.Position - lastFlushedAt);
if (batch.Count >= smugglerOptions.BatchSize ||
sizeOnDisk >= MaxSizeOfUncompressedSizeToSendToDatabase)
{
lastFlushedAt = sizeStream.Position;
HandleBatch(options,batch, sizeOnDisk);
sizeOnDisk = 0;
if (++batchCount % 10 == 0)
{
OutputIndexingDistance();
}
}
}
HandleBatch(options, batch, sizeOnDisk);
OutputIndexingDistance();
var attachmentCount = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Attachments", jsonReader.Value) == false)
throw new InvalidDataException("Attachment property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
attachmentCount += 1;
var item = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments)
continue;
var attachmentExportInfo =
new JsonSerializer
{
Converters = { new JsonToJsonConverter() }
}.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item));
ShowProgress("Importing attachment {0}", attachmentExportInfo.Key);
PutAttachment(attachmentExportInfo);
}
ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
}
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true)
{
EnsureDatabaseExists();
Stream sizeStream;
var sw = Stopwatch.StartNew();
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));
var streamReader = new StreamReader(sizeStream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
catch (InvalidDataException)
{
sizeStream = stream;
stream.Seek(0, SeekOrigin.Begin);
var streamReader = new StreamReader(stream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
// should read indexes now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Indexes", jsonReader.Value) == false)
{
throw new InvalidDataException("Indexes property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes)
{
continue;
}
var indexName = index.Value <string>("name");
if (indexName.StartsWith("Temp/"))
{
continue;
}
if (index.Value <RavenJObject>("definition").Value <bool>("IsCompiled"))
{
continue; // can't import compiled indexes
}
PutIndex(indexName, index);
}
// should read documents now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Docs", jsonReader.Value) == false)
{
throw new InvalidDataException("Docs property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
var batch = new List <RavenJObject>();
int totalCount = 0;
long lastFlushedAt = 0;
int batchCount = 0;
long sizeOnDisk = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var before = sizeStream.Position;
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = sizeStream.Position - before;
if (size > 1024 * 1024)
{
Console.WriteLine("{0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value <string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
{
continue;
}
if (options.MatchFilters(document) == false)
{
continue;
}
totalCount += 1;
batch.Add(document);
sizeOnDisk = (sizeStream.Position - lastFlushedAt);
if (batch.Count >= smugglerOptions.BatchSize ||
sizeOnDisk >= MaxSizeOfUncomressedSizeToSendToDatabase)
{
lastFlushedAt = sizeStream.Position;
HandleBatch(options, batch, sizeOnDisk);
sizeOnDisk = 0;
if (++batchCount % 10 == 0)
{
OutputIndexingDistance();
}
}
}
HandleBatch(options, batch, sizeOnDisk);
OutputIndexingDistance();
var attachmentCount = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Attachments", jsonReader.Value) == false)
{
throw new InvalidDataException("Attachment property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
attachmentCount += 1;
var item = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments)
{
continue;
}
var attachmentExportInfo =
new JsonSerializer
{
Converters = { new JsonToJsonConverter() }
}.Deserialize <AttachmentExportInfo>(new RavenJTokenReader(item));
ShowProgress("Importing attachment {0}", attachmentExportInfo.Key);
PutAttachment(attachmentExportInfo);
}
ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
}
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
{
var totalCount = 0;
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
var errorcount = 0;
ShowProgress("Exporting Documents");
while (true)
{
using (var documents = await GetDocuments(lastEtag))
{
var watch = Stopwatch.StartNew();
while (await documents.MoveNextAsync())
{
var document = documents.Current;
if (!options.MatchFilters(document))
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
continue;
document.WriteTo(jsonWriter);
totalCount++;
if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
if (watch.ElapsedMilliseconds > 100)
errorcount++;
watch.Start();
}
}
var databaseStatistics = await GetStats();
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
ShowProgress("Done with reading documents, total: {0}", totalCount);
return lastEtag;
}
}
private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
{
var now = SystemTime.UtcNow;
var count = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = GetRoughSize(document);
if (size > 1024 * 1024)
{
Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value<string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
continue;
if (options.MatchFilters(document) == false)
continue;
if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
continue;
if (!string.IsNullOrEmpty(options.TransformScript))
document = await TransformDocument(document, options.TransformScript);
if (document == null)
continue;
PutDocument(document, options);
count++;
if (count % options.BatchSize == 0)
{
ShowProgress("Read {0} documents", count);
}
}
PutDocument(null, options); // force flush
return count;
}
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true)
{
EnsureDatabaseExists();
var sw = Stopwatch.StartNew();
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
var streamReader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress));
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
catch (InvalidDataException)
{
stream.Seek(0, SeekOrigin.Begin);
var streamReader = new StreamReader(stream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
// should read indexes now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Indexes", jsonReader.Value) == false)
{
throw new InvalidDataException("Indexes property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes)
{
continue;
}
var indexName = index.Value <string>("name");
if (indexName.StartsWith("Raven/") || indexName.StartsWith("Temp/"))
{
continue;
}
PutIndex(indexName, index);
}
// should read documents now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Docs", jsonReader.Value) == false)
{
throw new InvalidDataException("Docs property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
var batch = new List <RavenJObject>();
int totalCount = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
{
continue;
}
if (options.MatchFilters(document) == false)
{
continue;
}
totalCount += 1;
batch.Add(document);
if (batch.Count >= 128)
{
FlushBatch(batch);
}
}
FlushBatch(batch);
var attachmentCount = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Attachments", jsonReader.Value) == false)
{
throw new InvalidDataException("Attachment property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
attachmentCount += 1;
var item = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments)
{
continue;
}
var attachmentExportInfo =
new JsonSerializer
{
Converters = { new TrivialJsonToJsonJsonConverter() }
}.Deserialize <AttachmentExportInfo>(new RavenJTokenReader(item));
ShowProgress("Importing attachment {0}", attachmentExportInfo.Key);
PutAttachment(attachmentExportInfo);
}
ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
}