public JsonOperationContext() { _arenaAllocator = new ArenaMemoryAllocator(); _arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator(); _jsonParserState = new JsonParserState(); _documentBuilder = new BlittableJsonDocumentBuilder(_jsonParserState, null); }
private BlittableJsonReaderObject ReadObjectInternal(object builder, string documentId, BlittableJsonDocumentBuilder.UsageMode mode) { var state = new JsonParserState(); using (var parser = new ObjectJsonParser(state, builder, this)) { var writer = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, state); try { CachedProperties.NewDocument(); writer.ReadObject(); if (writer.Read() == false) { throw new InvalidOperationException("Partial content in object json parser shouldn't happen"); } writer.FinalizeDocument(); _disposables.Add(writer); return(writer.CreateReader()); } catch (Exception) { writer.Dispose(); throw; } } }
public unsafe BlittableJsonReaderObject ParseBuffer(byte *buffer, int length, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null) { if (_disposed) { ThrowObjectDisposed(); } _jsonParserState.Reset(); using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag)) using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier)) { CachedProperties.NewDocument(); builder.ReadObjectDocument(); parser.SetBuffer(buffer, length); if (builder.Read() == false) { throw new EndOfStreamException("Buffer ended without reaching end of json content"); } builder.FinalizeDocument(); var reader = builder.CreateReader(); return(reader); } }
public BlittableJsonReaderObject ReadObjectWithExternalProperties(DynamicJsonValue obj, string debugTag) { var state = new JsonParserState(); using (var parser = new ObjectJsonParser(state, obj, this)) { var writer = new BlittableJsonDocumentBuilder(this, BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, state); try { writer.ReadObject(); if (writer.Read() == false) { throw new InvalidOperationException("Partial json content in object json parser shouldn't happen"); } writer.FinalizeDocumentWithoutProperties(CachedProperties.Version); _disposables.Add(writer); return(writer.CreateReader(CachedProperties)); } catch (Exception) { writer.Dispose(); throw; } } }
public async Task <BlittableJsonReaderObject> ReadFromWebSocket( WebSocket webSocket, string debugTag, CancellationToken cancellationToken) { var jsonParserState = new JsonParserState(); using (var parser = new UnmanagedJsonParser(this, jsonParserState, debugTag)) { var buffer = new ArraySegment <byte>(GetManagedBuffer()); var writer = new BlittableJsonDocumentBuilder(this, BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, jsonParserState); writer.ReadObject(); var result = await webSocket.ReceiveAsync(buffer, cancellationToken); if (result.MessageType == WebSocketMessageType.Close) { return(null); } parser.SetBuffer(buffer.Array, result.Count); while (writer.Read() == false) { result = await webSocket.ReceiveAsync(buffer, cancellationToken); parser.SetBuffer(buffer.Array, result.Count); } writer.FinalizeDocument(); return(writer.CreateReader()); } }
public MultiDocumentParser(JsonOperationContext context, Stream stream) { _context = context; _stream = stream; var state = new JsonParserState(); _returnManagedBuffer = context.GetManagedBuffer(out _buffer); _parser = new UnmanagedJsonParser(context, state, "parse/multi"); _writer = new BlittableJsonDocumentBuilder(_context, state, _parser); }
public async ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(WebSocket webSocket, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode, ManagedPinnedBuffer bytes, CancellationToken token = default(CancellationToken) ) { if (_disposed) { ThrowObjectDisposed(); } _jsonParserState.Reset(); UnmanagedJsonParser parser = null; BlittableJsonDocumentBuilder builder = null; var generation = _generation; try { parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag); builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState); CachedProperties.NewDocument(); builder.ReadObjectDocument(); while (true) { if (bytes.Valid == bytes.Used) { var read = await webSocket.ReceiveAsync(bytes.Buffer, token); EnsureNotDisposed(); if (read.Count == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } bytes.Valid = read.Count; bytes.Used = 0; } parser.SetBuffer(bytes); var result = builder.Read(); bytes.Used += parser.BufferOffset; if (result) { break; } } builder.FinalizeDocument(); var reader = builder.CreateReader(); return(reader); } finally { DisposeIfNeeded(generation, parser, builder); } }
public JsonOperationContext(int initialSize, int longLivedSize, SharedMultipleUseFlag lowMemoryFlag) { Debug.Assert(lowMemoryFlag != null); _disposeOnceRunner = new DisposeOnce <ExceptionRetry>(() => { #if MEM_GUARD_STACK ElectricFencedMemory.DecrementConext(); ElectricFencedMemory.UnRegisterContextAllocation(this); #endif Reset(true); _documentBuilder.Dispose(); _arenaAllocator.Dispose(); _arenaAllocatorForLongLivedValues?.Dispose(); if (_managedBuffers != null) { foreach (var managedPinnedBuffer in _managedBuffers) { managedPinnedBuffer.Dispose(); } _managedBuffers = null; } if (_pinnedObjects != null) { foreach (var pinnedObject in _pinnedObjects) { pinnedObject.Free(); } _pinnedObjects = null; } }); _initialSize = initialSize; _longLivedSize = longLivedSize; _arenaAllocator = new ArenaMemoryAllocator(lowMemoryFlag, initialSize); _arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator(lowMemoryFlag, longLivedSize); CachedProperties = new CachedProperties(this); _jsonParserState = new JsonParserState(); _objectJsonParser = new ObjectJsonParser(_jsonParserState, this); _documentBuilder = new BlittableJsonDocumentBuilder(this, _jsonParserState, _objectJsonParser); LowMemoryFlag = lowMemoryFlag; #if MEM_GUARD_STACK ElectricFencedMemory.IncrementConext(); ElectricFencedMemory.RegisterContextAllocation(this, Environment.StackTrace); #endif }
private void DisposeIfNeeded(int generation, UnmanagedJsonParser parser, BlittableJsonDocumentBuilder builder) { // if the generation has changed, that means that we had reset the context // this can happen if we were waiting on an async call for a while, got timed out / error / something // and the context was reset before we got back from the async call // since the full context was reset, there is no point in trying to dispose things, they were already // taken care of if (generation == _generation) { parser?.Dispose(); builder?.Dispose(); } }
public JsonOperationContext(int initialSize, int longLivedSize) { _initialSize = initialSize; _longLivedSize = longLivedSize; _arenaAllocator = new ArenaMemoryAllocator(initialSize); _arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator(longLivedSize); Encoding = new UTF8Encoding(); CachedProperties = new CachedProperties(this); _jsonParserState = new JsonParserState(); _objectJsonParser = new ObjectJsonParser(_jsonParserState, this); _documentBuilder = new BlittableJsonDocumentBuilder(this, _jsonParserState, _objectJsonParser); }
public BlittableJsonReaderObject(byte *mem, int size, JsonOperationContext context, BlittableJsonDocumentBuilder builder = null, CachedProperties cachedProperties = null) { _builder = builder; _cachedProperties = cachedProperties; _mem = mem; // get beginning of memory pointer _size = size; // get document size _context = context; byte offset; var propOffsetStart = _size - 2; var propsOffset = ReadVariableSizeIntInReverse(_mem, propOffsetStart, out offset); // init document level properties if (_cachedProperties == null) { SetupPropertiesAccess(mem, propsOffset); } else { if (_cachedProperties.Version != propsOffset) { throw new InvalidOperationException( $"This object requires an external properties cache with version {propsOffset}, but got one with {_cachedProperties.Version}"); } } // get pointer to property names array on document level // init root level object properties var objStartOffset = ReadVariableSizeIntInReverse(_mem, propOffsetStart - offset, out offset); // get offset of beginning of data of the main object byte propCountOffset; _propCount = ReadVariableSizeInt(objStartOffset, out propCountOffset); // get main object properties count _objStart = objStartOffset + mem; _metadataPtr = objStartOffset + mem + propCountOffset; // get pointer to current objects property tags metadata collection var currentType = (BlittableJsonToken)(*(mem + size - sizeof(byte))); // get current type byte flags // analyze main object type and it's offset and propertyIds flags _currentOffsetSize = ProcessTokenOffsetFlags(currentType); _currentPropertyIdSize = ProcessTokenPropertyFlags(currentType); }
public JsonOperationContext(int initialSize, int longLivedSize, SharedMultipleUseFlag lowMemoryFlag) { Debug.Assert(lowMemoryFlag != null); _initialSize = initialSize; _longLivedSize = longLivedSize; _arenaAllocator = new ArenaMemoryAllocator(lowMemoryFlag, initialSize); _arenaAllocatorForLongLivedValues = new ArenaMemoryAllocator(lowMemoryFlag, longLivedSize); CachedProperties = new CachedProperties(this); _jsonParserState = new JsonParserState(); _objectJsonParser = new ObjectJsonParser(_jsonParserState, this); _documentBuilder = new BlittableJsonDocumentBuilder(this, _jsonParserState, _objectJsonParser); LowMemoryFlag = lowMemoryFlag; #if MEM_GUARD_STACK ElectricFencedMemory.IncrementConext(); ElectricFencedMemory.RegisterContextAllocation(this, Environment.StackTrace); #endif }
public BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode, ManagedPinnedBuffer bytes, IBlittableDocumentModifier modifier = null) { if (_disposed) { ThrowObjectDisposed(); } _jsonParserState.Reset(); using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag)) using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier)) { CachedProperties.NewDocument(); builder.ReadObjectDocument(); while (true) { if (bytes.Valid == bytes.Used) { var read = stream.Read(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length); EnsureNotDisposed(); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } bytes.Valid = read; bytes.Used = 0; } parser.SetBuffer(bytes); var result = builder.Read(); bytes.Used += parser.BufferOffset; if (result) { break; } } builder.FinalizeDocument(); var reader = builder.CreateReader(); return(reader); } }
private async Task <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode) { _jsonParserState.Reset(); ManagedPinnedBuffer bytes; using (GetManagedBuffer(out bytes)) using (var parser = new UnmanagedJsonParser(this, _jsonParserState, documentId)) { var writer = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState); try { CachedProperties.NewDocument(); writer.ReadObjectDocument(); while (true) { var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } parser.SetBuffer(bytes, read); if (writer.Read()) { break; } } writer.FinalizeDocument(); var reader = writer.CreateReader(); RegisterLiveReader(reader); return(reader); } catch (Exception) { writer.Dispose(); throw; } } }
public unsafe BlittableJsonReaderArray ParseBufferToArray(string value, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null) { if (_disposed) { ThrowObjectDisposed(); } _jsonParserState.Reset(); using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag)) using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier)) using (GetManagedBuffer(out var buffer)) { CachedProperties.NewDocument(); builder.ReadArrayDocument(); var maxChars = buffer.Length / 8; //utf8 max size is 8 bytes, must consider worst case possiable bool lastReadResult = false; for (int i = 0; i < value.Length; i += maxChars) { var charsToRead = Math.Min(value.Length - i, maxChars); var length = Encodings.Utf8.GetBytes(value, i, charsToRead, buffer.Buffer.Array, buffer.Buffer.Offset); parser.SetBuffer(buffer.Pointer, length); lastReadResult = builder.Read(); } if (lastReadResult == false) { throw new EndOfStreamException("Buffer ended without reaching end of json content"); } builder.FinalizeDocument(); var reader = builder.CreateArrayReader(false); return(reader); } }
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode) { _jsonParserState.Reset(); ManagedPinnedBuffer bytes; using (GetManagedBuffer(out bytes)) using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag)) { var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState); try { CachedProperties.NewDocument(); writer.ReadArrayDocument(); while (true) { var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } parser.SetBuffer(bytes, read); if (writer.Read()) { break; } } writer.FinalizeDocument(); // here we "leak" the memory used by the array, in practice this is used // in short scoped context, so we don't care return(writer.CreateArrayReader()); } catch (Exception) { writer.Dispose(); throw; } } }
public async Task <BlittableJsonReaderObject> ReadFromWebSocket( WebSocket webSocket, string debugTag, CancellationToken cancellationToken) { _jsonParserState.Reset(); ManagedPinnedBuffer bytes; using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag)) using (GetManagedBuffer(out bytes)) { var writer = new BlittableJsonDocumentBuilder(this, BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, _jsonParserState); try { writer.ReadObjectDocument(); var result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken); if (result.MessageType == WebSocketMessageType.Close) { return(null); } parser.SetBuffer(bytes, result.Count); while (writer.Read() == false) { result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken); parser.SetBuffer(bytes, result.Count); } writer.FinalizeDocument(); return(writer.CreateReader()); } catch (Exception) { writer.Dispose(); throw; } } }
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode) { var state = new JsonParserState(); var buffer = GetParsingBuffer(); using (var parser = new UnmanagedJsonParser(this, state, debugTag)) { var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state); try { CachedProperties.NewDocument(); writer.ReadArray(); while (true) { var read = await stream.ReadAsync(buffer, 0, buffer.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } parser.SetBuffer(buffer, read); if (writer.Read()) { break; } } writer.FinalizeDocument(); _disposables.Add(writer); return(writer.CreateArrayReader()); } catch (Exception) { writer.Dispose(); throw; } } }
public async Task <BlittableJsonReaderObject> ParseAsync(BlittableJsonDocumentBuilder.UsageMode mode, string debugTag) { var writer = new BlittableJsonDocumentBuilder(_context, mode, debugTag, _parser, _state); try { writer.ReadObject(); _context.CachedProperties.NewDocument(); while (true) { if (_parser.BufferOffset == _parser.BufferSize) { var read = await _stream.ReadAsync(_buffer, 0, _buffer.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } _parser.SetBuffer(_buffer, read); } else { _parser.SetBuffer(new ArraySegment <byte>(_buffer, _parser.BufferOffset, _parser.BufferSize)); } if (writer.Read()) { break; } } writer.FinalizeDocument(); return(writer.CreateReader()); } catch (Exception) { writer.Dispose(); throw; } }
private BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode) { var state = new JsonParserState(); var buffer = GetParsingBuffer(); using (var parser = new UnmanagedJsonParser(this, state, debugTag)) { var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state); try { CachedProperties.NewDocument(); builder.ReadObject(); while (true) { var read = stream.Read(buffer, 0, buffer.Length); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } parser.SetBuffer(buffer, read); if (builder.Read()) { break; } } builder.FinalizeDocument(); _disposables.Add(builder); return(builder.CreateReader()); } catch (Exception) { builder.Dispose(); throw; } } }
public async ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode, ManagedPinnedBuffer bytes, CancellationToken?token = null, int maxSize = int.MaxValue) { if (_disposed) { ThrowObjectDisposed(); } _jsonParserState.Reset(); UnmanagedJsonParser parser = null; BlittableJsonDocumentBuilder builder = null; var generation = _generation; var streamDisposer = token?.Register(stream.Dispose); try { parser = new UnmanagedJsonParser(this, _jsonParserState, documentId); builder = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState); CachedProperties.NewDocument(); builder.ReadObjectDocument(); while (true) { token?.ThrowIfCancellationRequested(); if (bytes.Valid == bytes.Used) { var read = token.HasValue ? await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length, token.Value) : await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length); EnsureNotDisposed(); if (read == 0) { throw new EndOfStreamException("Stream ended without reaching end of json content"); } bytes.Valid = read; bytes.Used = 0; maxSize -= read; if (maxSize < 0) { throw new ArgumentException($"The maximum size allowed for {documentId} ({maxSize}) has been exceeded, aborting"); } } parser.SetBuffer(bytes); var result = builder.Read(); bytes.Used += parser.BufferOffset; if (result) { break; } } builder.FinalizeDocument(); var reader = builder.CreateReader(); return(reader); } finally { streamDisposer?.Dispose(); DisposeIfNeeded(generation, parser, builder); } }
public async Task <BlittableJsonReaderObject> ReadFromWebSocket( WebSocket webSocket, string debugTag, CancellationToken cancellationToken) { if (_disposed) { ThrowObjectDisposed(); } _jsonParserState.Reset(); UnmanagedJsonParser parser = null; BlittableJsonDocumentBuilder builder = null; var managedBuffer = default(ReturnBuffer); var generation = _generation; try { parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag); builder = new BlittableJsonDocumentBuilder(this, BlittableJsonDocumentBuilder.UsageMode.None, debugTag, parser, _jsonParserState); managedBuffer = GetManagedBuffer(out var bytes); try { builder.ReadObjectDocument(); var result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken); EnsureNotDisposed(); if (result.MessageType == WebSocketMessageType.Close) { return(null); } bytes.Valid = result.Count; bytes.Used = 0; parser.SetBuffer(bytes); while (true) { var read = builder.Read(); bytes.Used += parser.BufferOffset; if (read) { break; } result = await webSocket.ReceiveAsync(bytes.Buffer, cancellationToken); bytes.Valid = result.Count; bytes.Used = 0; parser.SetBuffer(bytes); } builder.FinalizeDocument(); return(builder.CreateReader()); } catch (Exception) { builder.Dispose(); throw; } } finally { DisposeIfNeeded(generation, parser, builder); if (generation == _generation) { managedBuffer.Dispose(); } } }