private static int SetPropertyIdSizeFlag(ref BlittableJsonToken objectToken, int maxPropId) { int propertyIdSize; if (maxPropId <= byte.MaxValue) { propertyIdSize = sizeof(byte); objectToken |= BlittableJsonToken.PropertyIdSizeByte; } else { if (maxPropId <= ushort.MaxValue) { propertyIdSize = sizeof(short); objectToken |= BlittableJsonToken.PropertyIdSizeShort; } else { propertyIdSize = sizeof(int); objectToken |= BlittableJsonToken.PropertyIdSizeInt; } } return(propertyIdSize); }
public int ProcessTokenOffsetFlags(BlittableJsonToken currentType) { // process part of byte flags that responsible for offset sizes const BlittableJsonToken mask = BlittableJsonToken.OffsetSizeByte | BlittableJsonToken.OffsetSizeShort | BlittableJsonToken.OffsetSizeInt; switch (currentType & mask) { case BlittableJsonToken.OffsetSizeByte: return(sizeof(byte)); case BlittableJsonToken.OffsetSizeShort: return(sizeof(short)); case BlittableJsonToken.OffsetSizeInt: return(sizeof(int)); default: throw new ArgumentException("Illegal offset size"); } }
private static int SetOffsetSizeFlag(ref BlittableJsonToken objectToken, int distanceFromFirstProperty) { int positionSize; if (distanceFromFirstProperty <= byte.MaxValue) { positionSize = sizeof(byte); objectToken |= BlittableJsonToken.OffsetSizeByte; } else { if (distanceFromFirstProperty <= ushort.MaxValue) { positionSize = sizeof(short); objectToken |= BlittableJsonToken.OffsetSizeShort; } else { positionSize = sizeof(int); objectToken |= BlittableJsonToken.OffsetSizeInt; } } return(positionSize); }
private static void ThrowOutOfRangeException(BlittableJsonToken token) { throw new ArgumentOutOfRangeException( $"Property names offset flag should be either byte, short of int, instead of {token}"); }
private int PropertiesValidation(BlittableJsonToken rootTokenTypen, int mainPropOffsetSize, int mainPropIdSize, int objStartOffset, int numberOfPropsNames) { byte offset; var numberOfProperties = ReadVariableSizeInt(_mem + objStartOffset, 0, out offset); var current = objStartOffset + offset; if (numberOfProperties < 0) { ThrowInvalidNumbeOfProperties(); } for (var i = 1; i <= numberOfProperties; i++) { var propOffset = ReadNumber(_mem + current, mainPropOffsetSize); if ((propOffset > objStartOffset) || (propOffset < 0)) { ThrowInvalidPropertiesOffest(); } current += mainPropOffsetSize; if (rootTokenTypen == BlittableJsonToken.StartObject) { var id = ReadNumber(_mem + current, mainPropIdSize); if ((id > numberOfPropsNames) || (id < 0)) { ThrowInvalidPropertiesId(); } current += mainPropIdSize; } int propOffsetSize; int propIdSize; var tokenType = TokenValidation(*(_mem + current), out propOffsetSize, out propIdSize); current++; var propValueOffset = objStartOffset - propOffset; switch (tokenType) { case BlittableJsonToken.StartObject: PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames); break; case BlittableJsonToken.StartArray: PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames); break; case BlittableJsonToken.Integer: ReadVariableSizeLong(propValueOffset); break; case BlittableJsonToken.LazyNumber: var numberLength = ReadVariableSizeInt(propValueOffset, out byte lengthOffset); var escCount = ReadVariableSizeInt(propValueOffset + lengthOffset + numberLength, out byte escOffset); // if number has any non-ascii symbols, we rull it out immediately if (escCount > 0) { ThrowInvalidNumber(propValueOffset); } var numberCharsStart = _mem + objStartOffset - propOffset + lengthOffset; // try and validate number using double's validation if (_context.TryParseDouble(numberCharsStart, numberLength, out _) == false) { ThrowInvalidNumber(propValueOffset); } break; case BlittableJsonToken.String: StringValidation(propValueOffset); break; case BlittableJsonToken.CompressedString: var stringLength = ReadVariableSizeInt(propValueOffset, out offset); var compressedStringLength = ReadVariableSizeInt(propValueOffset + offset, out offset); if ((compressedStringLength > stringLength) || (compressedStringLength < 0) || (stringLength < 0)) { ThrowInvalidCompressedString(); } break; case BlittableJsonToken.Boolean: var boolProp = ReadNumber(_mem + propValueOffset, 1); if ((boolProp != 0) && (boolProp != 1)) { ThrowInvalidBool(); } break; case BlittableJsonToken.Null: if (ReadNumber(_mem + propValueOffset, 1) != 0) { ThrowInvalidNull(); } break; case BlittableJsonToken.EmbeddedBlittable: byte offsetLen; stringLength = ReadVariableSizeInt(propValueOffset, out offsetLen); var blittableJsonReaderObject = new BlittableJsonReaderObject(_mem + propValueOffset + offsetLen, stringLength, _context); blittableJsonReaderObject.BlittableValidation(); break; default: ThrowInvalidTokenType(); break; } } return(current); }
public unsafe int WriteValue(byte *buffer, int size, out BlittableJsonToken token, UsageMode mode, int?initialCompressedSize) { return(WriteValue(buffer, size, null, out token, mode, initialCompressedSize)); }
public unsafe int WriteValue(byte *buffer, int size, FastList <int> escapePositions, out BlittableJsonToken token, UsageMode mode, int?initialCompressedSize) { int position = _position; int startPos = position; token = BlittableJsonToken.String; position += WriteVariableSizeInt(size); // if we are more than this size, we want to abort the compression early and just use // the verbatim string int maxGoodCompressionSize = size - sizeof(int) * 2; if (maxGoodCompressionSize > 0) { size = TryCompressValue(ref buffer, ref position, size, ref token, mode, initialCompressedSize, maxGoodCompressionSize); } _unmanagedWriteBuffer.Write(buffer, size); position += size; if (escapePositions == null) { position += WriteVariableSizeInt(0); goto Finish; } // we write the number of the escape sequences required // and then we write the distance to the _next_ escape sequence position += WriteVariableSizeInt(escapePositions.Count); // PERF: Use indexer to avoid the allocation and overhead of the foreach. int count = escapePositions.Count; for (int i = 0; i < count; i++) { position += WriteVariableSizeInt(escapePositions[i]); } Finish: _position = position; return(startPos); }
public WriteToken(int position, BlittableJsonToken token) { ValuePos = position; WrittenToken = token; }
public unsafe BlittableJsonReaderObject(int pos, BlittableJsonReaderObject parent, BlittableJsonToken type) { _parent = parent; _context = parent._context; _mem = parent._mem; _size = parent._size; _propNames = parent._propNames; _cachedProperties = parent._cachedProperties; var propNamesOffsetFlag = (BlittableJsonToken)(*_propNames); switch (propNamesOffsetFlag) { case BlittableJsonToken.OffsetSizeByte: _propNamesDataOffsetSize = sizeof(byte); break; case BlittableJsonToken.OffsetSizeShort: _propNamesDataOffsetSize = sizeof(short); break; case BlittableJsonToken.OffsetSizeInt: _propNamesDataOffsetSize = sizeof(int); break; default: throw new ArgumentOutOfRangeException( $"Property names offset flag should be either byte, short of int, instead of {propNamesOffsetFlag}"); } _objStart = _mem + pos; byte propCountOffset; _propCount = ReadVariableSizeInt(pos, out propCountOffset); _metadataPtr = _objStart + propCountOffset; // analyze main object type and it's offset and propertyIds flags _currentOffsetSize = ProcessTokenOffsetFlags(type); _currentPropertyIdSize = ProcessTokenPropertyFlags(type); }
private static int ThrowInvalidOffsetSize(BlittableJsonToken currentType) { throw new ArgumentException($"Illegal offset size {currentType}"); }
private unsafe int TryCompressValue(ref byte *buffer, ref int position, int size, ref BlittableJsonToken token, int?initialCompressedSize, int maxGoodCompressionSize) { return(size); }
public unsafe int WriteValue(LazyStringValue str, out BlittableJsonToken token, int?initialCompressedSize) { token = BlittableJsonToken.String; return(1); }
public unsafe int WriteValue(string str, out BlittableJsonToken token) { token = BlittableJsonToken.String; return(1); }
public unsafe int WriteValue(byte *buffer, int size, ICollection <int> escapePositions, out BlittableJsonToken token, UsageMode mode, int?initialCompressedSize) { var startPos = _position; token = BlittableJsonToken.String; _position += WriteVariableSizeInt(size); var maxGoodCompressionSize = // if we are more than this size, we want to abort the compression early and just use // the verbatim string size - sizeof(int) * 2; var shouldCompress = initialCompressedSize.HasValue || (((mode & UsageMode.CompressStrings) == UsageMode.CompressStrings) && (size > 128)) || ((mode & UsageMode.CompressSmallStrings) == UsageMode.CompressSmallStrings) && (size <= 128); if (maxGoodCompressionSize > 0 && shouldCompress) { int compressedSize; byte *compressionBuffer; if (initialCompressedSize.HasValue) { // we already have compressed data here compressedSize = initialCompressedSize.Value; compressionBuffer = buffer; } else { compressionBuffer = CompressBuffer(buffer, size, maxGoodCompressionSize, out compressedSize); } if (compressedSize > 0)// only if we actually save more than space { token = BlittableJsonToken.CompressedString; buffer = compressionBuffer; size = compressedSize; _position += WriteVariableSizeInt(compressedSize); } } _unmanagedWriteBuffer.Write(buffer, size); _position += size; if (escapePositions == null) { _position += WriteVariableSizeInt(0); return(startPos); } // we write the number of the escape sequences required // and then we write the distance to the _next_ escape sequence _position += WriteVariableSizeInt(escapePositions.Count); foreach (int escapePos in escapePositions) { _position += WriteVariableSizeInt(escapePos); } return(startPos); }
private static void ThrowInvalidType(BlittableJsonToken currentType) { throw new ArgumentException($"Illegal type {currentType}"); }
private bool SetToken(BlittableJsonToken token, object value) { switch (token & BlittableJsonReaderBase.TypesMask) { case BlittableJsonToken.EmbeddedBlittable: SetToken(JsonToken.Raw, value); return(true); case BlittableJsonToken.StartObject: var obj = (BlittableJsonReaderObject)value; var newObject = new CurrentItem { Object = (BlittableJsonReaderObject)value, Buffers = obj.GetPropertiesByInsertionOrder() }; _items.Push(newObject); //The value is passed in case the field/property should remains BlittableJsonReaderObject SetToken(JsonToken.StartObject, value); return(true); case BlittableJsonToken.StartArray: var newArray = new CurrentItem { Array = (BlittableJsonReaderArray)value }; _items.Push(newArray); //The value is passed in case the field/property should remains BlittableJsonReaderArray SetToken(JsonToken.StartArray, value); return(true); case BlittableJsonToken.Integer: SetToken(JsonToken.Integer, (long)value); return(true); case BlittableJsonToken.LazyNumber: if (_readAsLazyNumber) { SetToken(JsonToken.Float, value); } else { LazyNumberValue lnv = (LazyNumberValue)value; if (lnv.TryParseULong(out var ulongValue)) { SetToken(JsonToken.Integer, ulongValue); } else if (lnv.TryParseDecimal(out var decimalValue)) { SetToken(JsonToken.Float, decimalValue); } else { SetToken(JsonToken.Float, (double)lnv); } } return(true); case BlittableJsonToken.String: case BlittableJsonToken.CompressedString: SetToken(JsonToken.String, value.ToString()); return(true); case BlittableJsonToken.Boolean: SetToken(JsonToken.Boolean, (bool)value); return(true); case BlittableJsonToken.Null: SetToken(JsonToken.Null); return(true); default: throw new ArgumentOutOfRangeException(); } }
private int PropertiesValidation(BlittableJsonToken rootTokenTypen, int mainPropOffsetSize, int mainPropIdSize, int objStartOffset, int numberOfPropsNames) { byte offset; var numberOfProperties = ReadVariableSizeInt(_mem + objStartOffset, 0, out offset); var current = objStartOffset + offset; if (numberOfProperties < 0) { throw new InvalidDataException("Number of properties not valid"); } for (var i = 1; i <= numberOfProperties; i++) { var propOffset = ReadNumber(_mem + current, mainPropOffsetSize); if ((propOffset > objStartOffset) || (propOffset < 0)) { throw new InvalidDataException("Properties offset not valid"); } current += mainPropOffsetSize; if (rootTokenTypen == BlittableJsonToken.StartObject) { var id = ReadNumber(_mem + current, mainPropIdSize); if ((id > numberOfPropsNames) || (id < 0)) { throw new InvalidDataException("Properties id not valid"); } current += mainPropIdSize; } int propOffsetSize; int propIdSize; var tokenType = TokenValidation(*(_mem + current), out propOffsetSize, out propIdSize); current++; var propValueOffset = objStartOffset - propOffset; switch (tokenType) { case BlittableJsonToken.StartObject: PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames); break; case BlittableJsonToken.StartArray: PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames); break; case BlittableJsonToken.Integer: ReadVariableSizeLong(propValueOffset); break; case BlittableJsonToken.Float: var floatLen = ReadNumber(_mem + objStartOffset - propOffset, 1); var floatStringBuffer = new string(' ', floatLen); fixed(char *pChars = floatStringBuffer) { for (int j = 0; j < floatLen; j++) { pChars[j] = (char)ReadNumber((_mem + objStartOffset - propOffset + 1 + j), sizeof(byte)); } } double _double; var result = double.TryParse(floatStringBuffer, NumberStyles.Float, CultureInfo.InvariantCulture, out _double); if (!(result)) { throw new InvalidDataException("Double not valid (" + floatStringBuffer + ")"); } break; case BlittableJsonToken.String: StringValidation(propValueOffset); break; case BlittableJsonToken.CompressedString: var stringLength = ReadVariableSizeInt(propValueOffset, out offset); var compressedStringLength = ReadVariableSizeInt(propValueOffset + offset, out offset); if ((compressedStringLength > stringLength) || (compressedStringLength < 0) || (stringLength < 0)) { throw new InvalidDataException("Compressed string not valid"); } break; case BlittableJsonToken.Boolean: var boolProp = ReadNumber(_mem + propValueOffset, 1); if ((boolProp != 0) && (boolProp != 1)) { throw new InvalidDataException("Bool not valid"); } break; case BlittableJsonToken.Null: if (ReadNumber(_mem + propValueOffset, 1) != 0) { throw new InvalidDataException("Null not valid"); } break; default: throw new InvalidDataException("Token type not valid"); } } return(current); }
public int WriteArrayMetadata(FastList <int> positions, FastList <BlittableJsonToken> types, ref BlittableJsonToken listToken) { var arrayInfoStart = _position; _position += WriteVariableSizeInt(positions.Count); if (positions.Count == 0) { listToken |= BlittableJsonToken.OffsetSizeByte; } else { var distanceFromFirstItem = arrayInfoStart - positions[0]; var distanceTypeSize = SetOffsetSizeFlag(ref listToken, distanceFromFirstItem); for (var i = 0; i < positions.Count; i++) { WriteNumber(arrayInfoStart - positions[i], distanceTypeSize); _position += distanceTypeSize; _unmanagedWriteBuffer.WriteByte((byte)types[i]); _position++; } } return(arrayInfoStart); }
private int PropertiesValidation(BlittableJsonToken rootTokenTypen, int mainPropOffsetSize, int mainPropIdSize, int objStartOffset, int numberOfPropsNames) { byte offset; var numberOfProperties = ReadVariableSizeInt(_mem + objStartOffset, 0, out offset); var current = objStartOffset + offset; if (numberOfProperties < 0) { ThrowInvalidNumbeOfProperties(); } for (var i = 1; i <= numberOfProperties; i++) { var propOffset = ReadNumber(_mem + current, mainPropOffsetSize); if ((propOffset > objStartOffset) || (propOffset < 0)) { ThrowInvalidPropertiesOffest(); } current += mainPropOffsetSize; if (rootTokenTypen == BlittableJsonToken.StartObject) { var id = ReadNumber(_mem + current, mainPropIdSize); if ((id > numberOfPropsNames) || (id < 0)) { ThrowInvalidPropertiesId(); } current += mainPropIdSize; } int propOffsetSize; int propIdSize; var tokenType = TokenValidation(*(_mem + current), out propOffsetSize, out propIdSize); current++; var propValueOffset = objStartOffset - propOffset; switch (tokenType) { case BlittableJsonToken.StartObject: PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames); break; case BlittableJsonToken.StartArray: PropertiesValidation(tokenType, propOffsetSize, propIdSize, propValueOffset, numberOfPropsNames); break; case BlittableJsonToken.Integer: ReadVariableSizeLong(propValueOffset); break; case BlittableJsonToken.LazyNumber: var floatLen = ReadNumber(_mem + objStartOffset - propOffset, 1); var floatStringBuffer = new string(' ', floatLen); fixed(char *pChars = floatStringBuffer) { for (int j = 0; j < floatLen; j++) { pChars[j] = (char)ReadNumber((_mem + objStartOffset - propOffset + 1 + j), sizeof(byte)); } } double _double; var result = double.TryParse(floatStringBuffer, NumberStyles.Float, CultureInfo.InvariantCulture, out _double); if (!(result)) { ThrowInvalidDouble(floatStringBuffer); } break; case BlittableJsonToken.String: StringValidation(propValueOffset); break; case BlittableJsonToken.CompressedString: var stringLength = ReadVariableSizeInt(propValueOffset, out offset); var compressedStringLength = ReadVariableSizeInt(propValueOffset + offset, out offset); if ((compressedStringLength > stringLength) || (compressedStringLength < 0) || (stringLength < 0)) { ThrowInvalidCompressedString(); } break; case BlittableJsonToken.Boolean: var boolProp = ReadNumber(_mem + propValueOffset, 1); if ((boolProp != 0) && (boolProp != 1)) { ThrowInvalidBool(); } break; case BlittableJsonToken.Null: if (ReadNumber(_mem + propValueOffset, 1) != 0) { ThrowInvalidNull(); } break; case BlittableJsonToken.EmbeddedBlittable: byte offsetLen; stringLength = ReadVariableSizeInt(propValueOffset, out offsetLen); var blittableJsonReaderObject = new BlittableJsonReaderObject(_mem + propValueOffset + offsetLen, stringLength, _context); blittableJsonReaderObject.BlittableValidation(); break; default: ThrowInvalidTokenType(); break; } } return(current); }
private unsafe int TryCompressValue(ref byte *buffer, ref int position, int size, ref BlittableJsonToken token, UsageMode mode, int?initialCompressedSize, int maxGoodCompressionSize) { bool shouldCompress = initialCompressedSize.HasValue || (((mode & UsageMode.CompressStrings) == UsageMode.CompressStrings) && (size > 128)) || ((mode & UsageMode.CompressSmallStrings) == UsageMode.CompressSmallStrings) && (size <= 128); if (shouldCompress) { int compressedSize; byte *compressionBuffer; if (initialCompressedSize.HasValue) { // we already have compressed data here compressedSize = initialCompressedSize.Value; compressionBuffer = buffer; } else { compressionBuffer = CompressBuffer(buffer, size, maxGoodCompressionSize, out compressedSize); } if (compressedSize > 0) // only if we actually save more than space { token = BlittableJsonToken.CompressedString; buffer = compressionBuffer; size = compressedSize; position += WriteVariableSizeInt(compressedSize); } } return(size); }
public unsafe int WriteValue(LazyStringValue str, out BlittableJsonToken token, UsageMode mode, int?initialCompressedSize) { return(WriteValue(str.Buffer, str.Size, str.EscapePositions, out token, mode, initialCompressedSize)); }