private void WriteToServer(string database, Stream serverStream)
{
var streamNetworkBuffer = serverStream;
var writeToStreamBuffer = new byte[32 * 1024];
var header = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new TcpConnectionHeaderMessage
{
DatabaseName = database,
Operation = TcpConnectionHeaderMessage.OperationTypes.BulkInsert
}));
streamNetworkBuffer.Write(header, 0, header.Length);
while (_documents.IsCompleted == false)
{
_cts.Token.ThrowIfCancellationRequested();
Tuple <object, string> doc;
try
{
doc = _documents.Take();
}
catch (InvalidOperationException)
{
break;
}
var needToThrottle = _throttlingEvent.Wait(0) == false;
_jsonOperationContext.ResetAndRenew();
var documentInfo = new DocumentInfo();
var tag = _store.Conventions.GetDynamicTagName(doc.Item1);
var metadata = new DynamicJsonValue();
if (tag != null)
{
metadata[Constants.Headers.RavenEntityName] = tag;
}
metadata[Constants.Metadata.Id] = doc.Item2;
documentInfo.Metadata = _jsonOperationContext.ReadObject(metadata, doc.Item2);
var data = _entityToBlittable.ConvertEntityToBlittable(doc.Item1, _store.Conventions, _jsonOperationContext, documentInfo);
WriteVariableSizeInt(streamNetworkBuffer, data.Size);
WriteToStream(streamNetworkBuffer, data, writeToStreamBuffer);
if (needToThrottle)
{
streamNetworkBuffer.Flush();
_throttlingEvent.Wait(500);
}
}
streamNetworkBuffer.WriteByte(0); //done
streamNetworkBuffer.Flush();
}
public void ResetAndRenew()
{
MultiDocumentParser.Parser.ResetStream();
Context.ResetAndRenew();
MultiDocumentParser.Parser.SetStream();
}
private void WriteToServer(string url, Stream serverStream)
{
const string debugTag = "bulk/insert/document";
var jsonParserState = new JsonParserState();
//var streamNetworkBuffer = new BufferedStream(serverStream, 32 * 1024);
var streamNetworkBuffer = serverStream;
var writeToStreamBuffer = new byte[32 * 1024];
var header = Encoding.UTF8.GetBytes(RavenJObject.FromObject(new TcpConnectionHeaderMessage
{
DatabaseName = MultiDatabase.GetDatabaseName(url),
Operation = TcpConnectionHeaderMessage.OperationTypes.BulkInsert
}).ToString());
streamNetworkBuffer.Write(header, 0, header.Length);
JsonOperationContext.ManagedPinnedBuffer bytes;
using (_jsonOperationContext.GetManagedBuffer(out bytes))
{
while (_documents.IsCompleted == false)
{
_cts.Token.ThrowIfCancellationRequested();
MemoryStream jsonBuffer;
try
{
jsonBuffer = _documents.Take();
}
catch (InvalidOperationException)
{
break;
}
var needToThrottle = _throttlingEvent.Wait(0) == false;
_jsonOperationContext.ResetAndRenew();
using (var jsonParser = new UnmanagedJsonParser(_jsonOperationContext, jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(_jsonOperationContext,
BlittableJsonDocumentBuilder.UsageMode.ToDisk, debugTag,
jsonParser, jsonParserState))
{
_jsonOperationContext.CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
var read = jsonBuffer.Read(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
jsonParser.SetBuffer(bytes, read);
if (builder.Read())
{
break;
}
}
_buffers.Add(jsonBuffer);
builder.FinalizeDocument();
WriteVariableSizeInt(streamNetworkBuffer, builder.SizeInBytes);
WriteToStream(streamNetworkBuffer, builder, writeToStreamBuffer);
}
if (needToThrottle)
{
streamNetworkBuffer.Flush();
_throttlingEvent.Wait(500);
}
}
streamNetworkBuffer.WriteByte(0); //done
streamNetworkBuffer.Flush();
}
}