private static unsafe bool CompareStringsWithEscapePositions(JsonOperationContext context, BlittableJsonReaderObject.PropertyDetails oldProp,
BlittableJsonReaderObject.PropertyDetails newProp)
{
// this is called if the values are NOT equal, but we need to check if the oldProp was read from network and already resolved
// the escape characters
if (oldProp.Value is LazyStringValue lsv)
{
int pos = lsv.Size;
int numOfEscapePositions = BlittableJsonReaderBase.ReadVariableSizeInt(lsv.Buffer, ref pos);
if (numOfEscapePositions == 0)
{
return(false);
}
using (var memoryStream = new MemoryStream())
{
using (var textWriter = new BlittableJsonTextWriter(context, memoryStream))
{
textWriter.WriteString(lsv);
textWriter.Flush();
}
memoryStream.TryGetBuffer(out var bytes);
fixed(byte *pBuff = bytes.Array)
{
// need to ignore the quote marks
using var str = context.AllocateStringValue(null, pBuff + bytes.Offset + 1, (int)memoryStream.Length - 2);
return(newProp.Value.Equals(str));
}
}
}
return(false);
}
private static LazyStringValue ExtractDbId(JsonOperationContext context, LazyStringValue counterKey)
{
var offset = counterKey.Size - CountersStorage.DbIdAsBase64Size;
var p = counterKey.Buffer + offset;
return(context.AllocateStringValue(null, p, CountersStorage.DbIdAsBase64Size));
}
public LazyStringValue ReadStringLazily(int pos)
{
byte offset;
var size = ReadVariableSizeInt(pos, out offset);
return(_context.AllocateStringValue(null, _mem + pos + offset, size));
}
/// <summary>
/// Returns uncompressed data in form of LazyStringValue
/// </summary>
/// <returns></returns>
public LazyStringValue ToLazyStringValue()
{
var allocatedUncompressedData = DecompressToAllocatedMemoryData(_context);
var lazyStringValue = _context.AllocateStringValue(null, allocatedUncompressedData.Address, UncompressedSize);
lazyStringValue.AllocatedMemoryData = allocatedUncompressedData;
return(lazyStringValue);
}
private unsafe LazyStringValue CreateLazyStringValueFromParserState()
{
var lazyStringValueFromParserState = _context.AllocateStringValue(null, _state.StringBuffer, _state.StringSize);
if (_state.EscapePositions.Count <= 0)
{
return(lazyStringValueFromParserState);
}
lazyStringValueFromParserState.EscapePositions = _state.EscapePositions.ToArray();
return(lazyStringValueFromParserState);
}
public static unsafe string ReadString(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.ManagedPinnedBuffer buffer)
{
if (Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJson(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowInvalidJson(peepingTomStream);
}
return(context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString());
}
private static (LazyStringValue Doc, LazyStringValue Name) ExtractDocIdAndNameFromCounterTombstone(JsonOperationContext context, LazyStringValue counterKey)
{
var p = counterKey.Buffer;
var size = counterKey.Size;
int sizeOfDocId = 0;
for (; sizeOfDocId < size; sizeOfDocId++)
{
if (p[sizeOfDocId] == SpecialChars.RecordSeparator)
{
break;
}
}
var doc = context.AllocateStringValue(null, p, sizeOfDocId);
sizeOfDocId++;
p += sizeOfDocId;
int sizeOfName = size - sizeOfDocId - 1;
var name = context.AllocateStringValue(null, p, sizeOfName);
return(doc, name);
}
private unsafe LazyStringValue CreateLazyStringValueFromParserState(JsonParserState state)
{
int escapePositionsCount = state.EscapePositions.Count;
var maxSizeOfEscapePos = escapePositionsCount * 5 // max size of var int
+ JsonParserState.VariableSizeIntSize(escapePositionsCount);
var mem = _ctx.GetMemory(maxSizeOfEscapePos + state.StringSize);
_allocations.Add(mem);
Memory.Copy(mem.Address, state.StringBuffer, state.StringSize);
var lazyStringValueFromParserState = _ctx.AllocateStringValue(null, mem.Address, state.StringSize);
if (escapePositionsCount > 0)
{
lazyStringValueFromParserState.EscapePositions = state.EscapePositions.ToArray();
}
return(lazyStringValueFromParserState);
}
private static (LazyStringValue Doc, LazyStringValue Name) ExtractDocIdAndName(JsonOperationContext context, TableValueReader tvr)
{
var p = tvr.Read((int)CountersTable.CounterKey, out var size);
Debug.Assert(size > DbIdAsBase64Size + 2 /* record separators */);
int sizeOfDocId = 0;
for (; sizeOfDocId < size; sizeOfDocId++)
{
if (p[sizeOfDocId] == SpecialChars.RecordSeparator)
{
break;
}
}
var doc = context.AllocateStringValue(null, p, sizeOfDocId);
var name = ExtractCounterName(context, tvr);
return(doc, name);
}
public static (LazyStringValue DocId, string CounterName) ExtractDocIdAndCounterNameFromTombstone(JsonOperationContext context,
LazyStringValue counterTombstoneId)
{
var p = counterTombstoneId.Buffer;
var size = counterTombstoneId.Size;
int sizeOfDocId = 0;
for (; sizeOfDocId < size; sizeOfDocId++)
{
if (p[sizeOfDocId] == SpecialChars.RecordSeparator)
{
break;
}
}
var doc = context.AllocateStringValue(null, p, sizeOfDocId);
var name = Encoding.UTF8.GetString(p + sizeOfDocId + 1, size - (sizeOfDocId + 2));
return(doc, name);
}
private static unsafe void ThrowInvalidProperty(JsonParserState state, JsonOperationContext ctx)
{
throw new InvalidOperationException("Invalid property name: " +
ctx.AllocateStringValue(null, state.StringBuffer, state.StringSize));
}
public unsafe void ReadDataTypesTest()
{
using (var context = new JsonOperationContext(1024, 1024 * 4, SharedMultipleUseFlag.None))
{
BlittableJsonReaderObject embeddedReader;
using (var builder = new ManualBlittableJsonDocumentBuilder <UnmanagedWriteBuffer>(context))
{
builder.Reset(BlittableJsonDocumentBuilder.UsageMode.None);
builder.StartWriteObjectDocument();
builder.StartWriteObject();
builder.WritePropertyName("Value");
builder.WriteValue(1000);
builder.WriteObjectEnd();
builder.FinalizeDocument();
embeddedReader = builder.CreateReader();
}
using (var builder = new ManualBlittableJsonDocumentBuilder <UnmanagedWriteBuffer>(context))
{
var lonEscapedCharsString = string.Join(",", Enumerable.Repeat("\"Cool\"", 200).ToArray());
var longEscapedCharsAndNonAsciiString = string.Join(",", Enumerable.Repeat("\"מגניב\"", 200).ToArray());
builder.Reset(BlittableJsonDocumentBuilder.UsageMode.None);
builder.StartWriteObjectDocument();
builder.StartWriteObject();
builder.WritePropertyName("FloatMin");
builder.WriteValue(float.MinValue);
builder.WritePropertyName("FloatMax");
builder.WriteValue(float.MaxValue);
builder.WritePropertyName("UshortMin");
builder.WriteValue(ushort.MinValue);
builder.WritePropertyName("UshortMax");
builder.WriteValue(ushort.MaxValue);
builder.WritePropertyName("UintMin");
builder.WriteValue(uint.MinValue);
builder.WritePropertyName("UintMax");
builder.WriteValue(uint.MaxValue);
builder.WritePropertyName("DoubleMin");
builder.WriteValue(double.MinValue);
builder.WritePropertyName("DoubleMax");
builder.WriteValue(double.MaxValue);
builder.WritePropertyName("LongMin");
builder.WriteValue(long.MinValue);
builder.WritePropertyName("LongMax");
builder.WriteValue(long.MaxValue);
builder.WritePropertyName("StringEmpty");
builder.WriteValue(string.Empty);
builder.WritePropertyName("StringSimple");
builder.WriteValue("StringSimple");
builder.WritePropertyName("StringEscapedChars");
builder.WriteValue("\"Cool\"");
builder.WritePropertyName("StringLongEscapedChars");
builder.WriteValue(lonEscapedCharsString);
builder.WritePropertyName("StringEscapedCharsAndNonAscii");
builder.WriteValue(longEscapedCharsAndNonAsciiString);
var lsvString = "\"fooאbar\"";
var lsvStringBytes = Encoding.UTF8.GetBytes(lsvString);
fixed(byte *b = lsvStringBytes)
{
var escapePositionsMaxSize = JsonParserState.FindEscapePositionsMaxSize(lsvString);
var lsv = context.AllocateStringValue(null, b, lsvStringBytes.Length);
var escapePositions = new FastList <int>();
var len = lsvStringBytes.Length;
JsonParserState.FindEscapePositionsIn(escapePositions, b, ref len, escapePositionsMaxSize);
lsv.EscapePositions = escapePositions.ToArray();
builder.WritePropertyName("LSVString");
builder.WriteValue(lsv);
}
builder.WritePropertyName("Embedded");
builder.WriteEmbeddedBlittableDocument(embeddedReader);
builder.WriteObjectEnd();
builder.FinalizeDocument();
var reader = builder.CreateReader();
reader.BlittableValidation();
Assert.Equal(17, reader.Count);
Assert.Equal(float.MinValue, float.Parse(reader["FloatMin"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(float.MaxValue, float.Parse(reader["FloatMax"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(ushort.MinValue, ushort.Parse(reader["UshortMin"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(ushort.MaxValue, ushort.Parse(reader["UshortMax"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(uint.MinValue, uint.Parse(reader["UintMin"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(uint.MaxValue, uint.Parse(reader["UintMax"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(double.MinValue, double.Parse(reader["DoubleMin"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(double.MaxValue, double.Parse(reader["DoubleMax"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(long.MinValue, long.Parse(reader["LongMin"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(long.MaxValue, long.Parse(reader["LongMax"].ToString(), CultureInfo.InvariantCulture));
Assert.Equal(string.Empty, reader["StringEmpty"].ToString());
Assert.Equal("StringSimple", reader["StringSimple"].ToString());
Assert.Equal("\"Cool\"", reader["StringEscapedChars"].ToString());
Assert.Equal(lonEscapedCharsString, reader["StringLongEscapedChars"].ToString());
Assert.Equal(longEscapedCharsAndNonAsciiString, reader["StringEscapedCharsAndNonAscii"].ToString());
Assert.Equal(lsvString, reader["LSVString"].ToString());
Assert.Equal(1000, int.Parse((reader["Embedded"] as BlittableJsonReaderObject)["Value"].ToString(), CultureInfo.InvariantCulture));
}
}
}
private static unsafe GetResponse ReadResponse(JsonOperationContext context, PeepingTomStream peepingTomStream, UnmanagedJsonParser parser, JsonParserState state, JsonOperationContext.MemoryBuffer buffer)
{
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var getResponse = new GetResponse();
while (true)
{
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.EndObject)
{
break;
}
if (state.CurrentTokenType != JsonParserToken.String)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
var property = context.AllocateStringValue(null, state.StringBuffer, state.StringSize).ToString();
switch (property)
{
case nameof(GetResponse.Result):
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.Null)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
{
UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
getResponse.Result = builder.CreateReader();
}
continue;
case nameof(GetResponse.Headers):
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType == JsonParserToken.Null)
{
continue;
}
if (state.CurrentTokenType != JsonParserToken.StartObject)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
using (var builder = new BlittableJsonDocumentBuilder(context, BlittableJsonDocumentBuilder.UsageMode.None, "multi_get/result", parser, state))
{
UnmanagedJsonParserHelper.ReadObject(builder, peepingTomStream, parser, buffer);
using (var headersJson = builder.CreateReader())
{
foreach (var propertyName in headersJson.GetPropertyNames())
{
getResponse.Headers[propertyName] = headersJson[propertyName].ToString();
}
}
}
continue;
case nameof(GetResponse.StatusCode):
if (UnmanagedJsonParserHelper.Read(peepingTomStream, parser, state, buffer) == false)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
if (state.CurrentTokenType != JsonParserToken.Integer)
{
ThrowInvalidJsonResponse(peepingTomStream);
}
getResponse.StatusCode = (HttpStatusCode)state.Long;
continue;
default:
ThrowInvalidJsonResponse(peepingTomStream);
break;
}
}
return(getResponse);
}
private static unsafe LazyStringValue ReadCompareExchangeKey(JsonOperationContext context, TableValueReader reader)
{
var ptr = reader.Read((int)ClusterStateMachine.CompareExchangeTable.Key, out var size);
return(context.AllocateStringValue(null, ptr, size));
}
private static unsafe void ThrowInvalidCommandType(JsonParserState state, JsonOperationContext ctx)
{
throw new InvalidCommandTypeException("Invalid command type: "
+ ctx.AllocateStringValue(null, state.StringBuffer, state.StringSize));
}