private void WriteToServer(string database, Stream serverStream) { var streamNetworkBuffer = serverStream; var writeToStreamBuffer = new byte[32 * 1024]; var header = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new TcpConnectionHeaderMessage { DatabaseName = database, Operation = TcpConnectionHeaderMessage.OperationTypes.BulkInsert })); streamNetworkBuffer.Write(header, 0, header.Length); while (_documents.IsCompleted == false) { _cts.Token.ThrowIfCancellationRequested(); Tuple <object, string> doc; try { doc = _documents.Take(); } catch (InvalidOperationException) { break; } var needToThrottle = _throttlingEvent.Wait(0) == false; _jsonOperationContext.ResetAndRenew(); var documentInfo = new DocumentInfo(); var tag = _store.Conventions.GetDynamicTagName(doc.Item1); var metadata = new DynamicJsonValue(); if (tag != null) { metadata[Constants.Headers.RavenEntityName] = tag; } metadata[Constants.Metadata.Id] = doc.Item2; documentInfo.Metadata = _jsonOperationContext.ReadObject(metadata, doc.Item2); var data = _entityToBlittable.ConvertEntityToBlittable(doc.Item1, _store.Conventions, _jsonOperationContext, documentInfo); WriteVariableSizeInt(streamNetworkBuffer, data.Size); WriteToStream(streamNetworkBuffer, data, writeToStreamBuffer); if (needToThrottle) { streamNetworkBuffer.Flush(); _throttlingEvent.Wait(500); } } streamNetworkBuffer.WriteByte(0); //done streamNetworkBuffer.Flush(); }
public void ResetAndRenew() { MultiDocumentParser.Parser.ResetStream(); Context.ResetAndRenew(); MultiDocumentParser.Parser.SetStream(); }
private void WriteToServer(string url, Stream serverStream) { const string debugTag = "bulk/insert/document"; var jsonParserState = new JsonParserState(); //var streamNetworkBuffer = new BufferedStream(serverStream, 32 * 1024); var streamNetworkBuffer = serverStream; var writeToStreamBuffer = new byte[32 * 1024]; var header = Encoding.UTF8.GetBytes(RavenJObject.FromObject(new TcpConnectionHeaderMessage { DatabaseName = MultiDatabase.GetDatabaseName(url), Operation = TcpConnectionHeaderMessage.OperationTypes.BulkInsert }).ToString()); streamNetworkBuffer.Write(header, 0, header.Length); JsonOperationContext.ManagedPinnedBuffer bytes; using (_jsonOperationContext.GetManagedBuffer(out bytes)) { while (_documents.IsCompleted == false) { _cts.Token.ThrowIfCancellationRequested(); MemoryStream jsonBuffer; try { jsonBuffer = _documents.Take(); } catch (InvalidOperationException) { break; } var needToThrottle = _throttlingEvent.Wait(0) == false; _jsonOperationContext.ResetAndRenew(); using (var jsonParser = new UnmanagedJsonParser(_jsonOperationContext, jsonParserState, debugTag)) using (var builder = new BlittableJsonDocumentBuilder(_jsonOperationContext, BlittableJsonDocumentBuilder.UsageMode.ToDisk, debugTag, jsonParser, jsonParserState)) { _jsonOperationContext.CachedProperties.NewDocument(); builder.ReadObjectDocument(); while (true) { var read = jsonBuffer.Read(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } jsonParser.SetBuffer(bytes, read); if (builder.Read()) { break; } } _buffers.Add(jsonBuffer); builder.FinalizeDocument(); WriteVariableSizeInt(streamNetworkBuffer, builder.SizeInBytes); WriteToStream(streamNetworkBuffer, builder, writeToStreamBuffer); } if (needToThrottle) { streamNetworkBuffer.Flush(); _throttlingEvent.Wait(500); } } streamNetworkBuffer.WriteByte(0); //done streamNetworkBuffer.Flush(); } }