private async Task <int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options) { var now = SystemTime.UtcNow; var count = 0; while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var size = DocumentHelpers.GetRoughSize(document); if (size > 1024 * 1024) { Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}", (double)size / 1024, document["@metadata"].Value <string>("@id")); } if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) { continue; } if (options.MatchFilters(document) == false) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } if (!string.IsNullOrEmpty(options.TransformScript)) { document = await TransformDocument(document, options.TransformScript); } if (document == null) { continue; } PutDocument(document, options, (int)size); count++; if (count % options.BatchSize == 0) { ShowProgress("Read {0} documents", count); } } PutDocument(null, options, -1); // force flush return(count); }
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var now = SystemTime.UtcNow; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; if (maxRecords > 0) { using (var documents = await GetDocuments(lastEtag, Math.Min(maxRecords, options.BatchSize))) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) { errorcount++; } watch.Start(); } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true) { EnsureDatabaseExists(); var sw = Stopwatch.StartNew(); // Try to read the stream compressed, otherwise continue uncompressed. JsonTextReader jsonReader; try { var streamReader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress)); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) return; } catch (InvalidDataException) { stream.Seek(0, SeekOrigin.Begin); var streamReader = new StreamReader(stream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) return; } if (jsonReader.TokenType != JsonToken.StartObject) throw new InvalidDataException("StartObject was expected"); // should read indexes now if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Indexes", jsonReader.Value) == false) throw new InvalidDataException("Indexes property was expected"); if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var index = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes) continue; var indexName = index.Value<string>("name"); if (indexName.StartsWith("Raven/") || indexName.StartsWith("Temp/")) continue; PutIndex(indexName, index); } // should read documents now if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Docs", jsonReader.Value) == false) throw new InvalidDataException("Docs property was expected"); if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); var batch = new List<RavenJObject>(); int totalCount = 0; while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) continue; if (options.MatchFilters(document) == false) continue; totalCount += 1; batch.Add(document); if (batch.Count >= 128) FlushBatch(batch); } FlushBatch(batch); var attachmentCount = 0; if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject) return; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Attachments", jsonReader.Value) == false) throw new InvalidDataException("Attachment property was expected"); if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { attachmentCount += 1; var item = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments) continue; var attachmentExportInfo = new JsonSerializer { Converters = { new TrivialJsonToJsonJsonConverter() } }.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item)); ShowProgress("Importing attachment {0}", attachmentExportInfo.Key); PutAttachment(attachmentExportInfo); } ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds); }
protected virtual async Task <Etag> ExportDocuments(RavenConnectionStringOptions src, SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { var now = SystemTime.UtcNow; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorsCount = 0; var reachedMaxEtag = false; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; if (maxRecords > 0 && reachedMaxEtag == false) { using (var documents = await GetDocuments(src, lastEtag, Math.Min(options.BatchSize, maxRecords))) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id"))); if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0) { reachedMaxEtag = true; break; } lastEtag = tempLastEtag; if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) { errorsCount++; } watch.Start(); } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } // Load HiLo documents for selected collections options.Filters.ForEach(filter => { if (filter.Path == "@metadata.Raven-Entity-Name") { filter.Values.ForEach(collectionName => { JsonDocument doc = GetDocument("Raven/Hilo/" + collectionName); if (doc != null) { doc.Metadata["@id"] = doc.Key; doc.ToJson().WriteTo(jsonWriter); totalCount++; } }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; if (maxRecords > 0) { using (var documents = await GetDocuments(lastEtag, maxRecords)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag")); if (!options.MatchFilters(document)) continue; if (options.ShouldExcludeExpired && options.ExcludeExpired(document)) continue; document.WriteTo(jsonWriter); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) errorcount++; watch.Start(); } } if (hasDocs) continue; // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } }
private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options) { var count = 0; if (jsonReader.Read() == false) return count; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Docs", jsonReader.Value) == false) throw new InvalidDataException("Docs property was expected"); if (jsonReader.Read() == false) return count; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var size = GetRoughSize(document); if (size > 1024 * 1024) { Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}", (double)size / 1024, document["@metadata"].Value<string>("@id")); } if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) continue; if (options.MatchFilters(document) == false) continue; if (!string.IsNullOrEmpty(options.TransformScript)) document = await TransformDocument(document, options.TransformScript); if (document == null) continue; await PutDocument(document); count++; if (count % options.BatchSize == 0) { ShowProgress("Read {0} documents", count); } } await PutDocument(null); // force flush return count; }
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { using (var documents = await GetDocuments(lastEtag)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { var document = documents.Current; if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document)) { continue; } document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); if (watch.ElapsedMilliseconds > 100) { errorcount++; } watch.Start(); } } var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return(lastEtag); } }
private async Task <int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options) { var count = 0; if (jsonReader.Read() == false) { return(count); } if (jsonReader.TokenType != JsonToken.PropertyName) { throw new InvalidDataException("PropertyName was expected"); } if (Equals("Docs", jsonReader.Value) == false) { throw new InvalidDataException("Docs property was expected"); } if (jsonReader.Read() == false) { return(count); } if (jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var size = GetRoughSize(document); if (size > 1024 * 1024) { Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}", (double)size / 1024, document["@metadata"].Value <string>("@id")); } if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) { continue; } if (options.MatchFilters(document) == false) { continue; } if (!string.IsNullOrEmpty(options.TransformScript)) { document = await TransformDocument(document, options.TransformScript); } if (document == null) { continue; } await PutDocument(document); count++; if (count % options.BatchSize == 0) { ShowProgress("Read {0} documents", count); } } await PutDocument(null); // force flush return(count); }
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true) { EnsureDatabaseExists(); Stream sizeStream; var sw = Stopwatch.StartNew(); // Try to read the stream compressed, otherwise continue uncompressed. JsonTextReader jsonReader; try { sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress)); var streamReader = new StreamReader(sizeStream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) return; } catch (InvalidDataException) { sizeStream = stream; stream.Seek(0, SeekOrigin.Begin); var streamReader = new StreamReader(stream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) return; } if (jsonReader.TokenType != JsonToken.StartObject) throw new InvalidDataException("StartObject was expected"); // should read indexes now if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Indexes", jsonReader.Value) == false) throw new InvalidDataException("Indexes property was expected"); if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var index = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes) continue; var indexName = index.Value<string>("name"); if (indexName.StartsWith("Temp/")) continue; if (index.Value<RavenJObject>("definition").Value<bool>("IsCompiled")) continue; // can't import compiled indexes PutIndex(indexName, index); } // should read documents now if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Docs", jsonReader.Value) == false) throw new InvalidDataException("Docs property was expected"); if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); var batch = new List<RavenJObject>(); int totalCount = 0; long lastFlushedAt = 0; int batchCount = 0; long sizeOnDisk = 0; while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var before = sizeStream.Position; var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var size = sizeStream.Position - before; if (size > 1024 * 1024) { Console.WriteLine("{0:#,#.##;;0} kb - {1}", (double)size / 1024, document["@metadata"].Value<string>("@id")); } if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) continue; if (options.MatchFilters(document) == false) continue; totalCount += 1; batch.Add(document); sizeOnDisk = (sizeStream.Position - lastFlushedAt); if (batch.Count >= smugglerOptions.BatchSize || sizeOnDisk >= MaxSizeOfUncompressedSizeToSendToDatabase) { lastFlushedAt = sizeStream.Position; HandleBatch(options,batch, sizeOnDisk); sizeOnDisk = 0; if (++batchCount % 10 == 0) { OutputIndexingDistance(); } } } HandleBatch(options, batch, sizeOnDisk); OutputIndexingDistance(); var attachmentCount = 0; if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject) return; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Attachments", jsonReader.Value) == false) throw new InvalidDataException("Attachment property was expected"); if (jsonReader.Read() == false) return; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { attachmentCount += 1; var item = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments) continue; var attachmentExportInfo = new JsonSerializer { Converters = { new JsonToJsonConverter() } }.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item)); ShowProgress("Importing attachment {0}", attachmentExportInfo.Key); PutAttachment(attachmentExportInfo); } ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds); }
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true) { EnsureDatabaseExists(); Stream sizeStream; var sw = Stopwatch.StartNew(); // Try to read the stream compressed, otherwise continue uncompressed. JsonTextReader jsonReader; try { sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress)); var streamReader = new StreamReader(sizeStream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) { return; } } catch (InvalidDataException) { sizeStream = stream; stream.Seek(0, SeekOrigin.Begin); var streamReader = new StreamReader(stream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) { return; } } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidDataException("StartObject was expected"); } // should read indexes now if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.PropertyName) { throw new InvalidDataException("PropertyName was expected"); } if (Equals("Indexes", jsonReader.Value) == false) { throw new InvalidDataException("Indexes property was expected"); } if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var index = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes) { continue; } var indexName = index.Value <string>("name"); if (indexName.StartsWith("Temp/")) { continue; } if (index.Value <RavenJObject>("definition").Value <bool>("IsCompiled")) { continue; // can't import compiled indexes } PutIndex(indexName, index); } // should read documents now if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.PropertyName) { throw new InvalidDataException("PropertyName was expected"); } if (Equals("Docs", jsonReader.Value) == false) { throw new InvalidDataException("Docs property was expected"); } if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } var batch = new List <RavenJObject>(); int totalCount = 0; long lastFlushedAt = 0; int batchCount = 0; long sizeOnDisk = 0; while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var before = sizeStream.Position; var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var size = sizeStream.Position - before; if (size > 1024 * 1024) { Console.WriteLine("{0:#,#.##;;0} kb - {1}", (double)size / 1024, document["@metadata"].Value <string>("@id")); } if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) { continue; } if (options.MatchFilters(document) == false) { continue; } totalCount += 1; batch.Add(document); sizeOnDisk = (sizeStream.Position - lastFlushedAt); if (batch.Count >= smugglerOptions.BatchSize || sizeOnDisk >= MaxSizeOfUncomressedSizeToSendToDatabase) { lastFlushedAt = sizeStream.Position; HandleBatch(options, batch, sizeOnDisk); sizeOnDisk = 0; if (++batchCount % 10 == 0) { OutputIndexingDistance(); } } } HandleBatch(options, batch, sizeOnDisk); OutputIndexingDistance(); var attachmentCount = 0; if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject) { return; } if (jsonReader.TokenType != JsonToken.PropertyName) { throw new InvalidDataException("PropertyName was expected"); } if (Equals("Attachments", jsonReader.Value) == false) { throw new InvalidDataException("Attachment property was expected"); } if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { attachmentCount += 1; var item = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments) { continue; } var attachmentExportInfo = new JsonSerializer { Converters = { new JsonToJsonConverter() } }.Deserialize <AttachmentExportInfo>(new RavenJTokenReader(item)); ShowProgress("Importing attachment {0}", attachmentExportInfo.Key); PutAttachment(attachmentExportInfo); } ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds); }
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { using (var documents = await GetDocuments(lastEtag)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { var document = documents.Current; if (!options.MatchFilters(document)) continue; if (options.ShouldExcludeExpired && options.ExcludeExpired(document)) continue; document.WriteTo(jsonWriter); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag")); if (watch.ElapsedMilliseconds > 100) errorcount++; watch.Start(); } } var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return lastEtag; } }
private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options) { var now = SystemTime.UtcNow; var count = 0; while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var size = GetRoughSize(document); if (size > 1024 * 1024) { Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}", (double)size / 1024, document["@metadata"].Value<string>("@id")); } if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) continue; if (options.MatchFilters(document) == false) continue; if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) continue; if (!string.IsNullOrEmpty(options.TransformScript)) document = await TransformDocument(document, options.TransformScript); if (document == null) continue; PutDocument(document, options); count++; if (count % options.BatchSize == 0) { ShowProgress("Read {0} documents", count); } } PutDocument(null, options); // force flush return count; }
public void ImportData(Stream stream, SmugglerOptions options, bool importIndexes = true) { EnsureDatabaseExists(); var sw = Stopwatch.StartNew(); // Try to read the stream compressed, otherwise continue uncompressed. JsonTextReader jsonReader; try { var streamReader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress)); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) { return; } } catch (InvalidDataException) { stream.Seek(0, SeekOrigin.Begin); var streamReader = new StreamReader(stream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) { return; } } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidDataException("StartObject was expected"); } // should read indexes now if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.PropertyName) { throw new InvalidDataException("PropertyName was expected"); } if (Equals("Indexes", jsonReader.Value) == false) { throw new InvalidDataException("Indexes property was expected"); } if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var index = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Indexes) != ItemType.Indexes) { continue; } var indexName = index.Value <string>("name"); if (indexName.StartsWith("Raven/") || indexName.StartsWith("Temp/")) { continue; } PutIndex(indexName, index); } // should read documents now if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.PropertyName) { throw new InvalidDataException("PropertyName was expected"); } if (Equals("Docs", jsonReader.Value) == false) { throw new InvalidDataException("Docs property was expected"); } if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } var batch = new List <RavenJObject>(); int totalCount = 0; while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) { continue; } if (options.MatchFilters(document) == false) { continue; } totalCount += 1; batch.Add(document); if (batch.Count >= 128) { FlushBatch(batch); } } FlushBatch(batch); var attachmentCount = 0; if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject) { return; } if (jsonReader.TokenType != JsonToken.PropertyName) { throw new InvalidDataException("PropertyName was expected"); } if (Equals("Attachments", jsonReader.Value) == false) { throw new InvalidDataException("Attachment property was expected"); } if (jsonReader.Read() == false) { return; } if (jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { attachmentCount += 1; var item = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments) { continue; } var attachmentExportInfo = new JsonSerializer { Converters = { new TrivialJsonToJsonJsonConverter() } }.Deserialize <AttachmentExportInfo>(new RavenJTokenReader(item)); ShowProgress("Importing attachment {0}", attachmentExportInfo.Key); PutAttachment(attachmentExportInfo); } ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds); }