private static int SetPropertyIdSizeFlag(ref BlittableJsonToken objectToken, int maxPropId)
{
int propertyIdSize;
if (maxPropId <= byte.MaxValue)
{
propertyIdSize = sizeof(byte);
objectToken |= BlittableJsonToken.PropertyIdSizeByte;
}
else
{
if (maxPropId <= ushort.MaxValue)
{
propertyIdSize = sizeof(short);
objectToken |= BlittableJsonToken.PropertyIdSizeShort;
}
else
{
propertyIdSize = sizeof(int);
objectToken |= BlittableJsonToken.PropertyIdSizeInt;
}
}
return(propertyIdSize);
}
public int ProcessTokenOffsetFlags(BlittableJsonToken currentType)
{
// process part of byte flags that responsible for offset sizes
const BlittableJsonToken mask =
BlittableJsonToken.OffsetSizeByte |
BlittableJsonToken.OffsetSizeShort |
BlittableJsonToken.OffsetSizeInt;
switch (currentType &
mask)
{
case BlittableJsonToken.OffsetSizeByte:
return(sizeof(byte));
case BlittableJsonToken.OffsetSizeShort:
return(sizeof(short));
case BlittableJsonToken.OffsetSizeInt:
return(sizeof(int));
default:
throw new ArgumentException("Illegal offset size");
}
}
private static int SetOffsetSizeFlag(ref BlittableJsonToken objectToken, int distanceFromFirstProperty)
{
int positionSize;
if (distanceFromFirstProperty <= byte.MaxValue)
{
positionSize = sizeof(byte);
objectToken |= BlittableJsonToken.OffsetSizeByte;
}
else
{
if (distanceFromFirstProperty <= ushort.MaxValue)
{
positionSize = sizeof(short);
objectToken |= BlittableJsonToken.OffsetSizeShort;
}
else
{
positionSize = sizeof(int);
objectToken |= BlittableJsonToken.OffsetSizeInt;
}
}
return(positionSize);
}
private static void ThrowOutOfRangeException(BlittableJsonToken token)
{
throw new ArgumentOutOfRangeException(
$"Property names offset flag should be either byte, short of int, instead of {token}");
}
private int PropertiesValidation(BlittableJsonToken rootTokenTypen, int mainPropOffsetSize, int mainPropIdSize,
int objStartOffset, int numberOfPropsNames)
{
byte offset;
var numberOfProperties = ReadVariableSizeInt(_mem + objStartOffset, 0, out offset);
var current = objStartOffset + offset;
if (numberOfProperties < 0)
{
ThrowInvalidNumbeOfProperties();
}
for (var i = 1; i <= numberOfProperties; i++)
{
var propOffset = ReadNumber(_mem + current, mainPropOffsetSize);
if ((propOffset > objStartOffset) || (propOffset < 0))
{
ThrowInvalidPropertiesOffest();
}
current += mainPropOffsetSize;
if (rootTokenTypen == BlittableJsonToken.StartObject)
{
var id = ReadNumber(_mem + current, mainPropIdSize);
if ((id > numberOfPropsNames) || (id < 0))
{
ThrowInvalidPropertiesId();
}
current += mainPropIdSize;
}
int propOffsetSize;
int propIdSize;
var tokenType = TokenValidation(*(_mem + current), out propOffsetSize, out propIdSize);
current++;
var propValueOffset = objStartOffset - propOffset;
switch (tokenType)
{
case BlittableJsonToken.StartObject:
PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames);
break;
case BlittableJsonToken.StartArray:
PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames);
break;
case BlittableJsonToken.Integer:
ReadVariableSizeLong(propValueOffset);
break;
case BlittableJsonToken.LazyNumber:
var numberLength = ReadVariableSizeInt(propValueOffset, out byte lengthOffset);
var escCount = ReadVariableSizeInt(propValueOffset + lengthOffset + numberLength, out byte escOffset);
// if number has any non-ascii symbols, we rull it out immediately
if (escCount > 0)
{
ThrowInvalidNumber(propValueOffset);
}
var numberCharsStart = _mem + objStartOffset - propOffset + lengthOffset;
// try and validate number using double's validation
if (_context.TryParseDouble(numberCharsStart, numberLength, out _) == false)
{
ThrowInvalidNumber(propValueOffset);
}
break;
case BlittableJsonToken.String:
StringValidation(propValueOffset);
break;
case BlittableJsonToken.CompressedString:
var stringLength = ReadVariableSizeInt(propValueOffset, out offset);
var compressedStringLength = ReadVariableSizeInt(propValueOffset + offset, out offset);
if ((compressedStringLength > stringLength) ||
(compressedStringLength < 0) ||
(stringLength < 0))
{
ThrowInvalidCompressedString();
}
break;
case BlittableJsonToken.Boolean:
var boolProp = ReadNumber(_mem + propValueOffset, 1);
if ((boolProp != 0) && (boolProp != 1))
{
ThrowInvalidBool();
}
break;
case BlittableJsonToken.Null:
if (ReadNumber(_mem + propValueOffset, 1) != 0)
{
ThrowInvalidNull();
}
break;
case BlittableJsonToken.EmbeddedBlittable:
byte offsetLen;
stringLength = ReadVariableSizeInt(propValueOffset, out offsetLen);
var blittableJsonReaderObject = new BlittableJsonReaderObject(_mem + propValueOffset + offsetLen, stringLength, _context);
blittableJsonReaderObject.BlittableValidation();
break;
default:
ThrowInvalidTokenType();
break;
}
}
return(current);
}
public unsafe int WriteValue(byte *buffer, int size, out BlittableJsonToken token,
UsageMode mode, int?initialCompressedSize)
{
return(WriteValue(buffer, size, null, out token, mode, initialCompressedSize));
}
public unsafe int WriteValue(byte *buffer, int size, FastList <int> escapePositions, out BlittableJsonToken token, UsageMode mode, int?initialCompressedSize)
{
int position = _position;
int startPos = position;
token = BlittableJsonToken.String;
position += WriteVariableSizeInt(size);
// if we are more than this size, we want to abort the compression early and just use
// the verbatim string
int maxGoodCompressionSize = size - sizeof(int) * 2;
if (maxGoodCompressionSize > 0)
{
size = TryCompressValue(ref buffer, ref position, size, ref token, mode, initialCompressedSize, maxGoodCompressionSize);
}
_unmanagedWriteBuffer.Write(buffer, size);
position += size;
if (escapePositions == null)
{
position += WriteVariableSizeInt(0);
goto Finish;
}
// we write the number of the escape sequences required
// and then we write the distance to the _next_ escape sequence
position += WriteVariableSizeInt(escapePositions.Count);
// PERF: Use indexer to avoid the allocation and overhead of the foreach.
int count = escapePositions.Count;
for (int i = 0; i < count; i++)
{
position += WriteVariableSizeInt(escapePositions[i]);
}
Finish:
_position = position;
return(startPos);
}
public WriteToken(int position, BlittableJsonToken token)
{
ValuePos = position;
WrittenToken = token;
}
public unsafe BlittableJsonReaderObject(int pos, BlittableJsonReaderObject parent, BlittableJsonToken type)
{
_parent = parent;
_context = parent._context;
_mem = parent._mem;
_size = parent._size;
_propNames = parent._propNames;
_cachedProperties = parent._cachedProperties;
var propNamesOffsetFlag = (BlittableJsonToken)(*_propNames);
switch (propNamesOffsetFlag)
{
case BlittableJsonToken.OffsetSizeByte:
_propNamesDataOffsetSize = sizeof(byte);
break;
case BlittableJsonToken.OffsetSizeShort:
_propNamesDataOffsetSize = sizeof(short);
break;
case BlittableJsonToken.OffsetSizeInt:
_propNamesDataOffsetSize = sizeof(int);
break;
default:
throw new ArgumentOutOfRangeException(
$"Property names offset flag should be either byte, short of int, instead of {propNamesOffsetFlag}");
}
_objStart = _mem + pos;
byte propCountOffset;
_propCount = ReadVariableSizeInt(pos, out propCountOffset);
_metadataPtr = _objStart + propCountOffset;
// analyze main object type and it's offset and propertyIds flags
_currentOffsetSize = ProcessTokenOffsetFlags(type);
_currentPropertyIdSize = ProcessTokenPropertyFlags(type);
}
private static int ThrowInvalidOffsetSize(BlittableJsonToken currentType)
{
throw new ArgumentException($"Illegal offset size {currentType}");
}
private unsafe int TryCompressValue(ref byte *buffer, ref int position, int size, ref BlittableJsonToken token, int?initialCompressedSize, int maxGoodCompressionSize)
{
return(size);
}
public unsafe int WriteValue(LazyStringValue str, out BlittableJsonToken token,
int?initialCompressedSize)
{
token = BlittableJsonToken.String;
return(1);
}
public unsafe int WriteValue(string str, out BlittableJsonToken token)
{
token = BlittableJsonToken.String;
return(1);
}
public unsafe int WriteValue(byte *buffer, int size, ICollection <int> escapePositions, out BlittableJsonToken token, UsageMode mode, int?initialCompressedSize)
{
var startPos = _position;
token = BlittableJsonToken.String;
_position += WriteVariableSizeInt(size);
var maxGoodCompressionSize =
// if we are more than this size, we want to abort the compression early and just use
// the verbatim string
size - sizeof(int) * 2;
var shouldCompress =
initialCompressedSize.HasValue ||
(((mode & UsageMode.CompressStrings) == UsageMode.CompressStrings) && (size > 128)) ||
((mode & UsageMode.CompressSmallStrings) == UsageMode.CompressSmallStrings) && (size <= 128);
if (maxGoodCompressionSize > 0 && shouldCompress)
{
int compressedSize;
byte *compressionBuffer;
if (initialCompressedSize.HasValue)
{
// we already have compressed data here
compressedSize = initialCompressedSize.Value;
compressionBuffer = buffer;
}
else
{
compressionBuffer = CompressBuffer(buffer, size, maxGoodCompressionSize, out compressedSize);
}
if (compressedSize > 0)// only if we actually save more than space
{
token = BlittableJsonToken.CompressedString;
buffer = compressionBuffer;
size = compressedSize;
_position += WriteVariableSizeInt(compressedSize);
}
}
_unmanagedWriteBuffer.Write(buffer, size);
_position += size;
if (escapePositions == null)
{
_position += WriteVariableSizeInt(0);
return(startPos);
}
// we write the number of the escape sequences required
// and then we write the distance to the _next_ escape sequence
_position += WriteVariableSizeInt(escapePositions.Count);
foreach (int escapePos in escapePositions)
{
_position += WriteVariableSizeInt(escapePos);
}
return(startPos);
}
private static void ThrowInvalidType(BlittableJsonToken currentType)
{
throw new ArgumentException($"Illegal type {currentType}");
}
private bool SetToken(BlittableJsonToken token, object value)
{
switch (token & BlittableJsonReaderBase.TypesMask)
{
case BlittableJsonToken.EmbeddedBlittable:
SetToken(JsonToken.Raw, value);
return(true);
case BlittableJsonToken.StartObject:
var obj = (BlittableJsonReaderObject)value;
var newObject = new CurrentItem
{
Object = (BlittableJsonReaderObject)value,
Buffers = obj.GetPropertiesByInsertionOrder()
};
_items.Push(newObject);
//The value is passed in case the field/property should remains BlittableJsonReaderObject
SetToken(JsonToken.StartObject, value);
return(true);
case BlittableJsonToken.StartArray:
var newArray = new CurrentItem
{
Array = (BlittableJsonReaderArray)value
};
_items.Push(newArray);
//The value is passed in case the field/property should remains BlittableJsonReaderArray
SetToken(JsonToken.StartArray, value);
return(true);
case BlittableJsonToken.Integer:
SetToken(JsonToken.Integer, (long)value);
return(true);
case BlittableJsonToken.LazyNumber:
if (_readAsLazyNumber)
{
SetToken(JsonToken.Float, value);
}
else
{
LazyNumberValue lnv = (LazyNumberValue)value;
if (lnv.TryParseULong(out var ulongValue))
{
SetToken(JsonToken.Integer, ulongValue);
}
else if (lnv.TryParseDecimal(out var decimalValue))
{
SetToken(JsonToken.Float, decimalValue);
}
else
{
SetToken(JsonToken.Float, (double)lnv);
}
}
return(true);
case BlittableJsonToken.String:
case BlittableJsonToken.CompressedString:
SetToken(JsonToken.String, value.ToString());
return(true);
case BlittableJsonToken.Boolean:
SetToken(JsonToken.Boolean, (bool)value);
return(true);
case BlittableJsonToken.Null:
SetToken(JsonToken.Null);
return(true);
default:
throw new ArgumentOutOfRangeException();
}
}
private int PropertiesValidation(BlittableJsonToken rootTokenTypen, int mainPropOffsetSize, int mainPropIdSize,
int objStartOffset, int numberOfPropsNames)
{
byte offset;
var numberOfProperties = ReadVariableSizeInt(_mem + objStartOffset, 0, out offset);
var current = objStartOffset + offset;
if (numberOfProperties < 0)
{
throw new InvalidDataException("Number of properties not valid");
}
for (var i = 1; i <= numberOfProperties; i++)
{
var propOffset = ReadNumber(_mem + current, mainPropOffsetSize);
if ((propOffset > objStartOffset) || (propOffset < 0))
{
throw new InvalidDataException("Properties offset not valid");
}
current += mainPropOffsetSize;
if (rootTokenTypen == BlittableJsonToken.StartObject)
{
var id = ReadNumber(_mem + current, mainPropIdSize);
if ((id > numberOfPropsNames) || (id < 0))
{
throw new InvalidDataException("Properties id not valid");
}
current += mainPropIdSize;
}
int propOffsetSize;
int propIdSize;
var tokenType = TokenValidation(*(_mem + current), out propOffsetSize, out propIdSize);
current++;
var propValueOffset = objStartOffset - propOffset;
switch (tokenType)
{
case BlittableJsonToken.StartObject:
PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames);
break;
case BlittableJsonToken.StartArray:
PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames);
break;
case BlittableJsonToken.Integer:
ReadVariableSizeLong(propValueOffset);
break;
case BlittableJsonToken.Float:
var floatLen = ReadNumber(_mem + objStartOffset - propOffset, 1);
var floatStringBuffer = new string(' ', floatLen);
fixed(char *pChars = floatStringBuffer)
{
for (int j = 0; j < floatLen; j++)
{
pChars[j] = (char)ReadNumber((_mem + objStartOffset - propOffset + 1 + j), sizeof(byte));
}
}
double _double;
var result = double.TryParse(floatStringBuffer, NumberStyles.Float, CultureInfo.InvariantCulture, out _double);
if (!(result))
{
throw new InvalidDataException("Double not valid (" + floatStringBuffer + ")");
}
break;
case BlittableJsonToken.String:
StringValidation(propValueOffset);
break;
case BlittableJsonToken.CompressedString:
var stringLength = ReadVariableSizeInt(propValueOffset, out offset);
var compressedStringLength = ReadVariableSizeInt(propValueOffset + offset, out offset);
if ((compressedStringLength > stringLength) ||
(compressedStringLength < 0) ||
(stringLength < 0))
{
throw new InvalidDataException("Compressed string not valid");
}
break;
case BlittableJsonToken.Boolean:
var boolProp = ReadNumber(_mem + propValueOffset, 1);
if ((boolProp != 0) && (boolProp != 1))
{
throw new InvalidDataException("Bool not valid");
}
break;
case BlittableJsonToken.Null:
if (ReadNumber(_mem + propValueOffset, 1) != 0)
{
throw new InvalidDataException("Null not valid");
}
break;
default:
throw new InvalidDataException("Token type not valid");
}
}
return(current);
}
public int WriteArrayMetadata(FastList <int> positions, FastList <BlittableJsonToken> types, ref BlittableJsonToken listToken)
{
var arrayInfoStart = _position;
_position += WriteVariableSizeInt(positions.Count);
if (positions.Count == 0)
{
listToken |= BlittableJsonToken.OffsetSizeByte;
}
else
{
var distanceFromFirstItem = arrayInfoStart - positions[0];
var distanceTypeSize = SetOffsetSizeFlag(ref listToken, distanceFromFirstItem);
for (var i = 0; i < positions.Count; i++)
{
WriteNumber(arrayInfoStart - positions[i], distanceTypeSize);
_position += distanceTypeSize;
_unmanagedWriteBuffer.WriteByte((byte)types[i]);
_position++;
}
}
return(arrayInfoStart);
}
private int PropertiesValidation(BlittableJsonToken rootTokenTypen, int mainPropOffsetSize, int mainPropIdSize,
int objStartOffset, int numberOfPropsNames)
{
byte offset;
var numberOfProperties = ReadVariableSizeInt(_mem + objStartOffset, 0, out offset);
var current = objStartOffset + offset;
if (numberOfProperties < 0)
{
ThrowInvalidNumbeOfProperties();
}
for (var i = 1; i <= numberOfProperties; i++)
{
var propOffset = ReadNumber(_mem + current, mainPropOffsetSize);
if ((propOffset > objStartOffset) || (propOffset < 0))
{
ThrowInvalidPropertiesOffest();
}
current += mainPropOffsetSize;
if (rootTokenTypen == BlittableJsonToken.StartObject)
{
var id = ReadNumber(_mem + current, mainPropIdSize);
if ((id > numberOfPropsNames) || (id < 0))
{
ThrowInvalidPropertiesId();
}
current += mainPropIdSize;
}
int propOffsetSize;
int propIdSize;
var tokenType = TokenValidation(*(_mem + current), out propOffsetSize, out propIdSize);
current++;
var propValueOffset = objStartOffset - propOffset;
switch (tokenType)
{
case BlittableJsonToken.StartObject:
PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames);
break;
case BlittableJsonToken.StartArray:
PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames);
break;
case BlittableJsonToken.Integer:
ReadVariableSizeLong(propValueOffset);
break;
case BlittableJsonToken.LazyNumber:
var floatLen = ReadNumber(_mem + objStartOffset - propOffset, 1);
var floatStringBuffer = new string(' ', floatLen);
fixed(char *pChars = floatStringBuffer)
{
for (int j = 0; j < floatLen; j++)
{
pChars[j] = (char)ReadNumber((_mem + objStartOffset - propOffset + 1 + j), sizeof(byte));
}
}
double _double;
var result = double.TryParse(floatStringBuffer, NumberStyles.Float, CultureInfo.InvariantCulture, out _double);
if (!(result))
{
ThrowInvalidDouble(floatStringBuffer);
}
break;
case BlittableJsonToken.String:
StringValidation(propValueOffset);
break;
case BlittableJsonToken.CompressedString:
var stringLength = ReadVariableSizeInt(propValueOffset, out offset);
var compressedStringLength = ReadVariableSizeInt(propValueOffset + offset, out offset);
if ((compressedStringLength > stringLength) ||
(compressedStringLength < 0) ||
(stringLength < 0))
{
ThrowInvalidCompressedString();
}
break;
case BlittableJsonToken.Boolean:
var boolProp = ReadNumber(_mem + propValueOffset, 1);
if ((boolProp != 0) && (boolProp != 1))
{
ThrowInvalidBool();
}
break;
case BlittableJsonToken.Null:
if (ReadNumber(_mem + propValueOffset, 1) != 0)
{
ThrowInvalidNull();
}
break;
case BlittableJsonToken.EmbeddedBlittable:
byte offsetLen;
stringLength = ReadVariableSizeInt(propValueOffset, out offsetLen);
var blittableJsonReaderObject = new BlittableJsonReaderObject(_mem + propValueOffset + offsetLen, stringLength, _context);
blittableJsonReaderObject.BlittableValidation();
break;
default:
ThrowInvalidTokenType();
break;
}
}
return(current);
}
private unsafe int TryCompressValue(ref byte *buffer, ref int position, int size, ref BlittableJsonToken token, UsageMode mode, int?initialCompressedSize, int maxGoodCompressionSize)
{
bool shouldCompress = initialCompressedSize.HasValue ||
(((mode & UsageMode.CompressStrings) == UsageMode.CompressStrings) && (size > 128)) ||
((mode & UsageMode.CompressSmallStrings) == UsageMode.CompressSmallStrings) && (size <= 128);
if (shouldCompress)
{
int compressedSize;
byte *compressionBuffer;
if (initialCompressedSize.HasValue)
{
// we already have compressed data here
compressedSize = initialCompressedSize.Value;
compressionBuffer = buffer;
}
else
{
compressionBuffer = CompressBuffer(buffer, size, maxGoodCompressionSize, out compressedSize);
}
if (compressedSize > 0) // only if we actually save more than space
{
token = BlittableJsonToken.CompressedString;
buffer = compressionBuffer;
size = compressedSize;
position += WriteVariableSizeInt(compressedSize);
}
}
return(size);
}
public unsafe int WriteValue(LazyStringValue str, out BlittableJsonToken token,
UsageMode mode, int?initialCompressedSize)
{
return(WriteValue(str.Buffer, str.Size, str.EscapePositions, out token, mode, initialCompressedSize));
}
