public JsonOperationContext()
{
_arenaAllocator = new ArenaMemoryAllocator();
_arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator();
_jsonParserState = new JsonParserState();
_documentBuilder = new BlittableJsonDocumentBuilder(_jsonParserState, null);
}
private BlittableJsonReaderObject ReadObjectInternal(object builder, string documentId, BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
using (var parser = new ObjectJsonParser(state, builder, this))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, state);
try
{
CachedProperties.NewDocument();
writer.ReadObject();
if (writer.Read() == false)
{
throw new InvalidOperationException("Partial content in object json parser shouldn't happen");
}
writer.FinalizeDocument();
_disposables.Add(writer);
return(writer.CreateReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public unsafe BlittableJsonReaderObject ParseBuffer(byte *buffer, int length, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
{
CachedProperties.NewDocument();
builder.ReadObjectDocument();
parser.SetBuffer(buffer, length);
if (builder.Read() == false)
{
throw new EndOfStreamException("Buffer ended without reaching end of json content");
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
}
public BlittableJsonReaderObject ReadObjectWithExternalProperties(DynamicJsonValue obj, string debugTag)
{
var state = new JsonParserState();
using (var parser = new ObjectJsonParser(state, obj, this))
{
var writer = new BlittableJsonDocumentBuilder(this, BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, state);
try
{
writer.ReadObject();
if (writer.Read() == false)
{
throw new InvalidOperationException("Partial json content in object json parser shouldn't happen");
}
writer.FinalizeDocumentWithoutProperties(CachedProperties.Version);
_disposables.Add(writer);
return(writer.CreateReader(CachedProperties));
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderObject> ReadFromWebSocket(
WebSocket webSocket,
string debugTag,
CancellationToken cancellationToken)
{
var jsonParserState = new JsonParserState();
using (var parser = new UnmanagedJsonParser(this, jsonParserState, debugTag))
{
var buffer = new ArraySegment <byte>(GetManagedBuffer());
var writer = new BlittableJsonDocumentBuilder(this,
BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, jsonParserState);
writer.ReadObject();
var result = await webSocket.ReceiveAsync(buffer, cancellationToken);
if (result.MessageType == WebSocketMessageType.Close)
{
return(null);
}
parser.SetBuffer(buffer.Array, result.Count);
while (writer.Read() == false)
{
result = await webSocket.ReceiveAsync(buffer, cancellationToken);
parser.SetBuffer(buffer.Array, result.Count);
}
writer.FinalizeDocument();
return(writer.CreateReader());
}
}
public MultiDocumentParser(JsonOperationContext context, Stream stream)
{
_context = context;
_stream = stream;
var state = new JsonParserState();
_returnManagedBuffer = context.GetManagedBuffer(out _buffer);
_parser = new UnmanagedJsonParser(context, state, "parse/multi");
_writer = new BlittableJsonDocumentBuilder(_context, state, _parser);
}
public async ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(WebSocket webSocket, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode,
ManagedPinnedBuffer bytes,
CancellationToken token = default(CancellationToken)
)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
UnmanagedJsonParser parser = null;
BlittableJsonDocumentBuilder builder = null;
var generation = _generation;
try
{
parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag);
builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState);
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
if (bytes.Valid == bytes.Used)
{
var read = await webSocket.ReceiveAsync(bytes.Buffer, token);
EnsureNotDisposed();
if (read.Count == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read.Count;
bytes.Used = 0;
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
finally
{
DisposeIfNeeded(generation, parser, builder);
}
}
public JsonOperationContext(int initialSize, int longLivedSize, SharedMultipleUseFlag lowMemoryFlag)
{
Debug.Assert(lowMemoryFlag != null);
_disposeOnceRunner = new DisposeOnce <ExceptionRetry>(() =>
{
#if MEM_GUARD_STACK
ElectricFencedMemory.DecrementConext();
ElectricFencedMemory.UnRegisterContextAllocation(this);
#endif
Reset(true);
_documentBuilder.Dispose();
_arenaAllocator.Dispose();
_arenaAllocatorForLongLivedValues?.Dispose();
if (_managedBuffers != null)
{
foreach (var managedPinnedBuffer in _managedBuffers)
{
managedPinnedBuffer.Dispose();
}
_managedBuffers = null;
}
if (_pinnedObjects != null)
{
foreach (var pinnedObject in _pinnedObjects)
{
pinnedObject.Free();
}
_pinnedObjects = null;
}
});
_initialSize = initialSize;
_longLivedSize = longLivedSize;
_arenaAllocator = new ArenaMemoryAllocator(lowMemoryFlag, initialSize);
_arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator(lowMemoryFlag, longLivedSize);
CachedProperties = new CachedProperties(this);
_jsonParserState = new JsonParserState();
_objectJsonParser = new ObjectJsonParser(_jsonParserState, this);
_documentBuilder = new BlittableJsonDocumentBuilder(this, _jsonParserState, _objectJsonParser);
LowMemoryFlag = lowMemoryFlag;
#if MEM_GUARD_STACK
ElectricFencedMemory.IncrementConext();
ElectricFencedMemory.RegisterContextAllocation(this, Environment.StackTrace);
#endif
}
private void DisposeIfNeeded(int generation, UnmanagedJsonParser parser, BlittableJsonDocumentBuilder builder)
{
// if the generation has changed, that means that we had reset the context
// this can happen if we were waiting on an async call for a while, got timed out / error / something
// and the context was reset before we got back from the async call
// since the full context was reset, there is no point in trying to dispose things, they were already
// taken care of
if (generation == _generation)
{
parser?.Dispose();
builder?.Dispose();
}
}
public JsonOperationContext(int initialSize, int longLivedSize)
{
_initialSize = initialSize;
_longLivedSize = longLivedSize;
_arenaAllocator = new ArenaMemoryAllocator(initialSize);
_arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator(longLivedSize);
Encoding = new UTF8Encoding();
CachedProperties = new CachedProperties(this);
_jsonParserState = new JsonParserState();
_objectJsonParser = new ObjectJsonParser(_jsonParserState, this);
_documentBuilder = new BlittableJsonDocumentBuilder(this, _jsonParserState, _objectJsonParser);
}
public BlittableJsonReaderObject(byte *mem, int size, JsonOperationContext context,
BlittableJsonDocumentBuilder builder = null,
CachedProperties cachedProperties = null)
{
_builder = builder;
_cachedProperties = cachedProperties;
_mem = mem; // get beginning of memory pointer
_size = size; // get document size
_context = context;
byte offset;
var propOffsetStart = _size - 2;
var propsOffset = ReadVariableSizeIntInReverse(_mem, propOffsetStart, out offset);
// init document level properties
if (_cachedProperties == null)
{
SetupPropertiesAccess(mem, propsOffset);
}
else
{
if (_cachedProperties.Version != propsOffset)
{
throw new InvalidOperationException(
$"This object requires an external properties cache with version {propsOffset}, but got one with {_cachedProperties.Version}");
}
}
// get pointer to property names array on document level
// init root level object properties
var objStartOffset = ReadVariableSizeIntInReverse(_mem, propOffsetStart - offset, out offset);
// get offset of beginning of data of the main object
byte propCountOffset;
_propCount = ReadVariableSizeInt(objStartOffset, out propCountOffset); // get main object properties count
_objStart = objStartOffset + mem;
_metadataPtr = objStartOffset + mem + propCountOffset;
// get pointer to current objects property tags metadata collection
var currentType = (BlittableJsonToken)(*(mem + size - sizeof(byte)));
// get current type byte flags
// analyze main object type and it's offset and propertyIds flags
_currentOffsetSize = ProcessTokenOffsetFlags(currentType);
_currentPropertyIdSize = ProcessTokenPropertyFlags(currentType);
}
public JsonOperationContext(int initialSize, int longLivedSize, SharedMultipleUseFlag lowMemoryFlag)
{
Debug.Assert(lowMemoryFlag != null);
_initialSize = initialSize;
_longLivedSize = longLivedSize;
_arenaAllocator = new ArenaMemoryAllocator(lowMemoryFlag, initialSize);
_arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator(lowMemoryFlag, longLivedSize);
CachedProperties = new CachedProperties(this);
_jsonParserState = new JsonParserState();
_objectJsonParser = new ObjectJsonParser(_jsonParserState, this);
_documentBuilder = new BlittableJsonDocumentBuilder(this, _jsonParserState, _objectJsonParser);
LowMemoryFlag = lowMemoryFlag;
#if MEM_GUARD_STACK
ElectricFencedMemory.IncrementConext();
ElectricFencedMemory.RegisterContextAllocation(this, Environment.StackTrace);
#endif
}
public BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode,
ManagedPinnedBuffer bytes, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
{
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
if (bytes.Valid == bytes.Used)
{
var read = stream.Read(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
}
private async Task <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode)
{
_jsonParserState.Reset();
ManagedPinnedBuffer bytes;
using (GetManagedBuffer(out bytes))
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, documentId))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState);
try
{
CachedProperties.NewDocument();
writer.ReadObjectDocument();
while (true)
{
var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(bytes, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
var reader = writer.CreateReader();
RegisterLiveReader(reader);
return(reader);
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public unsafe BlittableJsonReaderArray ParseBufferToArray(string value, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
using (GetManagedBuffer(out var buffer))
{
CachedProperties.NewDocument();
builder.ReadArrayDocument();
var maxChars = buffer.Length / 8; //utf8 max size is 8 bytes, must consider worst case possiable
bool lastReadResult = false;
for (int i = 0; i < value.Length; i += maxChars)
{
var charsToRead = Math.Min(value.Length - i, maxChars);
var length = Encodings.Utf8.GetBytes(value, i,
charsToRead,
buffer.Buffer.Array,
buffer.Buffer.Offset);
parser.SetBuffer(buffer.Pointer, length);
lastReadResult = builder.Read();
}
if (lastReadResult == false)
{
throw new EndOfStreamException("Buffer ended without reaching end of json content");
}
builder.FinalizeDocument();
var reader = builder.CreateArrayReader(false);
return(reader);
}
}
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode)
{
_jsonParserState.Reset();
ManagedPinnedBuffer bytes;
using (GetManagedBuffer(out bytes))
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState);
try
{
CachedProperties.NewDocument();
writer.ReadArrayDocument();
while (true)
{
var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(bytes, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
// here we "leak" the memory used by the array, in practice this is used
// in short scoped context, so we don't care
return(writer.CreateArrayReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderObject> ReadFromWebSocket(
WebSocket webSocket,
string debugTag,
CancellationToken cancellationToken)
{
_jsonParserState.Reset();
ManagedPinnedBuffer bytes;
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (GetManagedBuffer(out bytes))
{
var writer = new BlittableJsonDocumentBuilder(this,
BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, _jsonParserState);
try
{
writer.ReadObjectDocument();
var result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken);
if (result.MessageType == WebSocketMessageType.Close)
{
return(null);
}
parser.SetBuffer(bytes, result.Count);
while (writer.Read() == false)
{
result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken);
parser.SetBuffer(bytes, result.Count);
}
writer.FinalizeDocument();
return(writer.CreateReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
var buffer = GetParsingBuffer();
using (var parser = new UnmanagedJsonParser(this, state, debugTag))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state);
try
{
CachedProperties.NewDocument();
writer.ReadArray();
while (true)
{
var read = await stream.ReadAsync(buffer, 0, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
_disposables.Add(writer);
return(writer.CreateArrayReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderObject> ParseAsync(BlittableJsonDocumentBuilder.UsageMode mode, string debugTag)
{
var writer = new BlittableJsonDocumentBuilder(_context, mode, debugTag, _parser, _state);
try
{
writer.ReadObject();
_context.CachedProperties.NewDocument();
while (true)
{
if (_parser.BufferOffset == _parser.BufferSize)
{
var read = await _stream.ReadAsync(_buffer, 0, _buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
_parser.SetBuffer(_buffer, read);
}
else
{
_parser.SetBuffer(new ArraySegment <byte>(_buffer, _parser.BufferOffset, _parser.BufferSize));
}
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
return(writer.CreateReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
private BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
var buffer = GetParsingBuffer();
using (var parser = new UnmanagedJsonParser(this, state, debugTag))
{
var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state);
try
{
CachedProperties.NewDocument();
builder.ReadObject();
while (true)
{
var read = stream.Read(buffer, 0, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
if (builder.Read())
{
break;
}
}
builder.FinalizeDocument();
_disposables.Add(builder);
return(builder.CreateReader());
}
catch (Exception)
{
builder.Dispose();
throw;
}
}
}
public async ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode, ManagedPinnedBuffer bytes,
CancellationToken?token = null,
int maxSize = int.MaxValue)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
UnmanagedJsonParser parser = null;
BlittableJsonDocumentBuilder builder = null;
var generation = _generation;
var streamDisposer = token?.Register(stream.Dispose);
try
{
parser = new UnmanagedJsonParser(this, _jsonParserState, documentId);
builder = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState);
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
token?.ThrowIfCancellationRequested();
if (bytes.Valid == bytes.Used)
{
var read = token.HasValue
? await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length, token.Value)
: await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
maxSize -= read;
if (maxSize < 0)
{
throw new ArgumentException($"The maximum size allowed for {documentId} ({maxSize}) has been exceeded, aborting");
}
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
finally
{
streamDisposer?.Dispose();
DisposeIfNeeded(generation, parser, builder);
}
}
public async Task <BlittableJsonReaderObject> ReadFromWebSocket(
WebSocket webSocket,
string debugTag,
CancellationToken cancellationToken)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
UnmanagedJsonParser parser = null;
BlittableJsonDocumentBuilder builder = null;
var managedBuffer = default(ReturnBuffer);
var generation = _generation;
try
{
parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag);
builder = new BlittableJsonDocumentBuilder(this,
BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, _jsonParserState);
managedBuffer = GetManagedBuffer(out var bytes);
try
{
builder.ReadObjectDocument();
var result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken);
EnsureNotDisposed();
if (result.MessageType == WebSocketMessageType.Close)
{
return(null);
}
bytes.Valid = result.Count;
bytes.Used = 0;
parser.SetBuffer(bytes);
while (true)
{
var read = builder.Read();
bytes.Used += parser.BufferOffset;
if (read)
{
break;
}
result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken);
bytes.Valid = result.Count;
bytes.Used = 0;
parser.SetBuffer(bytes);
}
builder.FinalizeDocument();
return(builder.CreateReader());
}
catch (Exception)
{
builder.Dispose();
throw;
}
}
finally
{
DisposeIfNeeded(generation, parser, builder);
if (generation == _generation)
{
managedBuffer.Dispose();
}
}
}