private void PrepareForNewDocumentParse(BlittableJsonDocumentBuilder.UsageMode mode, string debugTag)
{
_parser.NewDocument();
_writer.Renew(debugTag, mode);
_writer.ReadObjectDocument();
_context.CachedProperties.NewDocument();
}
public BlittableJsonReaderObject Parse(BlittableJsonDocumentBuilder.UsageMode mode, string debugTag)
{
PrepareForNewDocumentParse(mode, debugTag);
while (true)
{
if (_parser.BufferOffset == _parser.BufferSize)
{
var read = _stream.Read(_buffer.Buffer.Array, _buffer.Buffer.Offset, _buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content.");
}
_parser.SetBuffer(_buffer, read);
_bufferOffset = 0;
}
else
{
_bufferOffset += _parser.BufferOffset;
_parser.SetBuffer(_buffer, _bufferOffset, _parser.BufferSize - _parser.BufferOffset);
}
if (_writer.Read())
{
break;
}
}
_writer.FinalizeDocument();
return(_writer.CreateReader());
}
public unsafe BlittableJsonReaderObject ParseBuffer(byte *buffer, int length, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
{
CachedProperties.NewDocument();
builder.ReadObjectDocument();
parser.SetBuffer(buffer, length);
if (builder.Read() == false)
{
throw new EndOfStreamException("Buffer ended without reaching end of json content");
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
}
public async ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(WebSocket webSocket, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode,
ManagedPinnedBuffer bytes,
CancellationToken token = default(CancellationToken)
)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
UnmanagedJsonParser parser = null;
BlittableJsonDocumentBuilder builder = null;
var generation = _generation;
try
{
parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag);
builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState);
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
if (bytes.Valid == bytes.Used)
{
var read = await webSocket.ReceiveAsync(bytes.Buffer, token);
EnsureNotDisposed();
if (read.Count == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read.Count;
bytes.Used = 0;
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
finally
{
DisposeIfNeeded(generation, parser, builder);
}
}
//Todo Fixing the clone implementation to support this situation or throw clear error
public BlittableJsonReaderArray Clone(JsonOperationContext context, BlittableJsonDocumentBuilder.UsageMode usageMode = BlittableJsonDocumentBuilder.UsageMode.None)
{
AssertContextNotDisposed();
using (var builder = new ManualBlittableJsonDocumentBuilder <UnmanagedWriteBuffer>(context))
{
builder.Reset(usageMode);
builder.StartArrayDocument();
builder.StartWriteArray();
using (var itr = new BlittableJsonArrayEnumerator(this))
{
while (itr.MoveNext())
{
switch (itr.Current)
{
case BlittableJsonReaderObject item:
var clone = item.CloneOnTheSameContext();
builder.WriteEmbeddedBlittableDocument(clone.BasePointer, clone.Size);
break;
case LazyStringValue item:
builder.WriteValue(item);
break;
case long item:
builder.WriteValue(item);
break;
case LazyNumberValue item:
builder.WriteValue(item);
break;
case LazyCompressedStringValue item:
builder.WriteValue(item);
break;
default:
throw new InvalidDataException($"Actual value type is {itr.Current.GetType()}. Should be known serialized type and should not happen. ");
}
}
}
builder.WriteArrayEnd();
builder.FinalizeDocument();
return(builder.CreateArrayReader());
}
}
private BlittableJsonReaderObject ReadObjectInternal(object builder, string documentId,
BlittableJsonDocumentBuilder.UsageMode mode)
{
_jsonParserState.Reset();
_objectJsonParser.Reset(builder);
_documentBuilder.Renew(documentId, mode);
CachedProperties.NewDocument();
_documentBuilder.ReadObjectDocument();
if (_documentBuilder.Read() == false)
{
throw new InvalidOperationException("Partial content in object json parser shouldn't happen");
}
_documentBuilder.FinalizeDocument();
var reader = _documentBuilder.CreateReader();
RegisterLiveReader(reader);
return(reader);
}
public static BlittableJsonReaderObject ParseToMemory(
this JsonOperationContext.SyncJsonOperationContext syncContext,
Stream stream,
string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode,
JsonOperationContext.MemoryBuffer bytes,
IBlittableDocumentModifier modifier = null)
{
syncContext.EnsureNotDisposed();
syncContext.JsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(syncContext.Context, syncContext.JsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(syncContext.Context, mode, debugTag, parser, syncContext.JsonParserState, modifier: modifier))
{
syncContext.Context.CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
if (bytes.Valid == bytes.Used)
{
var read = stream.Read(bytes.Memory.Memory.Span);
syncContext.EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
}
public BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode,
ManagedPinnedBuffer bytes, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
{
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
if (bytes.Valid == bytes.Used)
{
var read = stream.Read(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
}
public unsafe BlittableJsonReaderArray ParseBufferToArray(string value, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
using (var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState, modifier: modifier))
using (GetManagedBuffer(out var buffer))
{
CachedProperties.NewDocument();
builder.ReadArrayDocument();
var maxChars = buffer.Length / 8; //utf8 max size is 8 bytes, must consider worst case possiable
bool lastReadResult = false;
for (int i = 0; i < value.Length; i += maxChars)
{
var charsToRead = Math.Min(value.Length - i, maxChars);
var length = Encodings.Utf8.GetBytes(value, i,
charsToRead,
buffer.Buffer.Array,
buffer.Buffer.Offset);
parser.SetBuffer(buffer.Pointer, length);
lastReadResult = builder.Read();
}
if (lastReadResult == false)
{
throw new EndOfStreamException("Buffer ended without reaching end of json content");
}
builder.FinalizeDocument();
var reader = builder.CreateArrayReader(false);
return(reader);
}
}
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode)
{
_jsonParserState.Reset();
ManagedPinnedBuffer bytes;
using (GetManagedBuffer(out bytes))
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, debugTag))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, _jsonParserState);
try
{
CachedProperties.NewDocument();
writer.ReadArrayDocument();
while (true)
{
var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(bytes, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
// here we "leak" the memory used by the array, in practice this is used
// in short scoped context, so we don't care
return(writer.CreateArrayReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public async Task <BlittableJsonReaderObject> ParseAsync(BlittableJsonDocumentBuilder.UsageMode mode, string debugTag)
{
var writer = new BlittableJsonDocumentBuilder(_context, mode, debugTag, _parser, _state);
try
{
writer.ReadObject();
_context.CachedProperties.NewDocument();
while (true)
{
if (_parser.BufferOffset == _parser.BufferSize)
{
var read = await _stream.ReadAsync(_buffer, 0, _buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
_parser.SetBuffer(_buffer, read);
}
else
{
_parser.SetBuffer(new ArraySegment <byte>(_buffer, _parser.BufferOffset, _parser.BufferSize));
}
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
return(writer.CreateReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
public async Task <BlittableJsonReaderArray> ParseArrayToMemoryAsync(Stream stream, string debugTag,
BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
var buffer = GetParsingBuffer();
using (var parser = new UnmanagedJsonParser(this, state, debugTag))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state);
try
{
CachedProperties.NewDocument();
writer.ReadArray();
while (true)
{
var read = await stream.ReadAsync(buffer, 0, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
_disposables.Add(writer);
return(writer.CreateArrayReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
private ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode, CancellationToken?token = null)
{
using (GetManagedBuffer(out ManagedPinnedBuffer bytes))
return(ParseToMemoryAsync(stream, documentId, mode, bytes, token));
}
private BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null)
{
ManagedPinnedBuffer bytes;
using (GetManagedBuffer(out bytes))
{
return(ParseToMemory(stream, debugTag, mode, bytes, modifier));
}
}
public BlittableJsonReaderObject ReadObject(BlittableJsonReaderObject obj, string documentId,
BlittableJsonDocumentBuilder.UsageMode mode = BlittableJsonDocumentBuilder.UsageMode.None)
{
return(ReadObjectInternal(obj, documentId, mode));
}
public BlittableJsonReaderObject ReadObject(DynamicJsonValue builder, string documentId,
BlittableJsonDocumentBuilder.UsageMode mode = BlittableJsonDocumentBuilder.UsageMode.None, IBlittableDocumentModifier modifier = null)
{
return(ReadObjectInternal(builder, documentId, mode, modifier));
}
private async Task <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode)
{
_jsonParserState.Reset();
ManagedPinnedBuffer bytes;
using (GetManagedBuffer(out bytes))
using (var parser = new UnmanagedJsonParser(this, _jsonParserState, documentId))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState);
try
{
CachedProperties.NewDocument();
writer.ReadObjectDocument();
while (true)
{
var read = await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(bytes, read);
if (writer.Read())
{
break;
}
}
writer.FinalizeDocument();
var reader = writer.CreateReader();
RegisterLiveReader(reader);
return(reader);
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public static BlittableJsonReaderObject Translate(JsonOperationContext context, Engine scriptEngine, ObjectInstance objectInstance, IResultModifier modifier = null, BlittableJsonDocumentBuilder.UsageMode usageMode = BlittableJsonDocumentBuilder.UsageMode.None)
{
if (objectInstance == null)
{
return(null);
}
if (objectInstance is BlittableObjectInstance boi && boi.Changed == false)
{
return(boi.Blittable.Clone(context));
}
using (var writer = new ManualBlittableJsonDocumentBuilder <UnmanagedWriteBuffer>(context))
{
writer.Reset(usageMode);
writer.StartWriteObjectDocument();
var blittableBridge = new JsBlittableBridge(writer, usageMode, scriptEngine);
blittableBridge.WriteInstance(objectInstance, modifier);
writer.FinalizeDocument();
return(writer.CreateReader());
}
}
public async ValueTask <BlittableJsonReaderObject> ParseToMemoryAsync(Stream stream, string documentId, BlittableJsonDocumentBuilder.UsageMode mode, ManagedPinnedBuffer bytes,
CancellationToken?token = null,
int maxSize = int.MaxValue)
{
if (_disposed)
{
ThrowObjectDisposed();
}
_jsonParserState.Reset();
UnmanagedJsonParser parser = null;
BlittableJsonDocumentBuilder builder = null;
var generation = _generation;
var streamDisposer = token?.Register(stream.Dispose);
try
{
parser = new UnmanagedJsonParser(this, _jsonParserState, documentId);
builder = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, _jsonParserState);
CachedProperties.NewDocument();
builder.ReadObjectDocument();
while (true)
{
token?.ThrowIfCancellationRequested();
if (bytes.Valid == bytes.Used)
{
var read = token.HasValue
? await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length, token.Value)
: await stream.ReadAsync(bytes.Buffer.Array, bytes.Buffer.Offset, bytes.Length);
EnsureNotDisposed();
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
bytes.Valid = read;
bytes.Used = 0;
maxSize -= read;
if (maxSize < 0)
{
throw new ArgumentException($"The maximum size allowed for {documentId} ({maxSize}) has been exceeded, aborting");
}
}
parser.SetBuffer(bytes);
var result = builder.Read();
bytes.Used += parser.BufferOffset;
if (result)
{
break;
}
}
builder.FinalizeDocument();
var reader = builder.CreateReader();
return(reader);
}
finally
{
streamDisposer?.Dispose();
DisposeIfNeeded(generation, parser, builder);
}
}
private BlittableJsonReaderObject ParseToMemory(Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
var buffer = GetParsingBuffer();
using (var parser = new UnmanagedJsonParser(this, state, debugTag))
{
var builder = new BlittableJsonDocumentBuilder(this, mode, debugTag, parser, state);
try
{
CachedProperties.NewDocument();
builder.ReadObject();
while (true)
{
var read = stream.Read(buffer, 0, buffer.Length);
if (read == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of json content");
}
parser.SetBuffer(buffer, read);
if (builder.Read())
{
break;
}
}
builder.FinalizeDocument();
_disposables.Add(builder);
return(builder.CreateReader());
}
catch (Exception)
{
builder.Dispose();
throw;
}
}
}
private BlittableJsonReaderObject ReadObjectInternal(object builder, string documentId, BlittableJsonDocumentBuilder.UsageMode mode)
{
var state = new JsonParserState();
using (var parser = new ObjectJsonParser(state, builder, this))
{
var writer = new BlittableJsonDocumentBuilder(this, mode, documentId, parser, state);
try
{
CachedProperties.NewDocument();
writer.ReadObject();
if (writer.Read() == false)
{
throw new InvalidOperationException("Partial content in object json parser shouldn't happen");
}
writer.FinalizeDocument();
_disposables.Add(writer);
return(writer.CreateReader());
}
catch (Exception)
{
writer.Dispose();
throw;
}
}
}
public JsBlittableBridge(ManualBlittableJsonDocumentBuilder <UnmanagedWriteBuffer> writer, BlittableJsonDocumentBuilder.UsageMode usageMode, Engine scriptEngine)
{
_writer = writer;
_usageMode = usageMode;
_scriptEngine = scriptEngine;
}
public BlittableJsonReaderObject TranslateToObject(JsonOperationContext context, JsBlittableBridge.IResultModifier modifier = null, BlittableJsonDocumentBuilder.UsageMode usageMode = BlittableJsonDocumentBuilder.UsageMode.None)
{
if (IsNull)
{
return(null);
}
var obj = _instance.AsObject();
return(JsBlittableBridge.Translate(context, _parent.ScriptEngine, obj, modifier, usageMode));
}
public object Translate(ScriptRunnerResult result, JsonOperationContext context, JsBlittableBridge.IResultModifier modifier = null, BlittableJsonDocumentBuilder.UsageMode usageMode = BlittableJsonDocumentBuilder.UsageMode.None)
{
var val = result.RawJsValue;
if (val.IsString())
{
return(val.AsString());
}
if (val.IsBoolean())
{
return(val.AsBoolean());
}
if (val.IsObject())
{
return(result.TranslateToObject(context, modifier, usageMode));
}
if (val.IsNumber())
{
return(val.AsNumber());
}
if (val.IsNull() || val.IsUndefined())
{
return(null);
}
if (val.IsArray())
{
throw new InvalidOperationException("Returning arrays from scripts is not supported, only objects or primitves");
}
throw new NotSupportedException("Unable to translate " + val.Type);
}
private static BlittableJsonReaderObject ParseToMemory(JsonOperationContext.SyncJsonOperationContext syncContext, Stream stream, string debugTag, BlittableJsonDocumentBuilder.UsageMode mode, IBlittableDocumentModifier modifier = null)
{
using (syncContext.Context.GetMemoryBuffer(out var bytes))
return(ParseToMemory(syncContext, stream, debugTag, mode, bytes, modifier));
}
