private IEnumerable<JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action<int> increaseDocumentsCount)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
var metadata = doc.Value<RavenJObject>("@metadata");
if (metadata == null)
throw new InvalidOperationException("Could not find metadata for document");
var id = metadata.Value<string>("@id");
if (string.IsNullOrEmpty(id))
throw new InvalidOperationException("Could not get id from metadata");
if (id.Equals(Constants.BulkImportHeartbeatDocKey, StringComparison.InvariantCultureIgnoreCase))
continue; //its just a token document, should not get written into the database
//the purpose of the heartbeat document is to make sure that the connection doesn't time-out
//during long pauses in the bulk insert operation.
// Currently used by smuggler to make sure that the connection doesn't time out if there is a
//continuation token and lots of document skips
doc.Remove("@metadata");
yield return new JsonDocument
{
Key = id,
DataAsJson = doc,
Metadata = metadata
};
}
increaseDocumentsCount(count);
}
}
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout)
{
timeout.Delay();
item.WriteTo(writer);
}
private IEnumerable<JsonDocument> YieldDocumentsInBatch(CancellationTimeout timeout, Stream partialStream, Action<int> increaseDocumentsCount)
{
using (var stream = new GZipStream(partialStream, CompressionMode.Decompress, leaveOpen: true))
{
var reader = new BinaryReader(stream);
var count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
timeout.Delay();
var doc = (RavenJObject)RavenJToken.ReadFrom(new BsonReader(reader)
{
DateTimeKindHandling = DateTimeKind.Unspecified
});
var metadata = doc.Value<RavenJObject>("@metadata");
if (metadata == null)
throw new InvalidOperationException("Could not find metadata for document");
var id = metadata.Value<string>("@id");
if (string.IsNullOrEmpty(id))
throw new InvalidOperationException("Could not get id from metadata");
doc.Remove("@metadata");
yield return new JsonDocument
{
Key = id,
DataAsJson = doc,
Metadata = metadata
};
}
increaseDocumentsCount(count);
}
}
