private static void UnicodeGetLowerKeySliceAndStorageKey(JsonOperationContext context, string str, out byte *lowerKey, out int lowerSize, out byte *key, out int keySize) { // See comment in GetLowerKeySliceAndStorageKey for the format _jsonParserState.Reset(); var byteCount = Utf8.GetMaxByteCount(str.Length); _jsonParserState.FindEscapePositionsIn(str); var maxKeyLenSize = JsonParserState.VariableSizeIntSize(byteCount); var escapePositionsSize = _jsonParserState.GetEscapePositionsSize(); var buffer = context.GetNativeTempBuffer( sizeof(char) * str.Length // for the lower calls + byteCount // lower key + maxKeyLenSize // the size of var int for the len of the key + byteCount // actual key + escapePositionsSize); fixed(char *pChars = str) { var destChars = (char *)buffer; for (var i = 0; i < str.Length; i++) { destChars[i] = char.ToLowerInvariant(pChars[i]); } lowerKey = buffer + str.Length * sizeof(char); lowerSize = Utf8.GetBytes(destChars, str.Length, lowerKey, byteCount); if (lowerSize > 512) { ThrowKeyTooBig(str); } key = buffer + str.Length * sizeof(char) + byteCount; var writePos = key; keySize = Utf8.GetBytes(pChars, str.Length, writePos + maxKeyLenSize, byteCount); var actualKeyLenSize = JsonParserState.VariableSizeIntSize(keySize); if (actualKeyLenSize < maxKeyLenSize) { var movePtr = maxKeyLenSize - actualKeyLenSize; key += movePtr; writePos += movePtr; } JsonParserState.WriteVariableSizeInt(ref writePos, keySize); _jsonParserState.WriteEscapePositionsTo(writePos + keySize); keySize += escapePositionsSize + maxKeyLenSize; } }
private unsafe LazyStringValue CreateLazyStringValueFromParserState(JsonParserState state) { int escapePositionsCount = state.EscapePositions.Count; var maxSizeOfEscapePos = escapePositionsCount * 5 // max size of var int + JsonParserState.VariableSizeIntSize(escapePositionsCount); var mem = _ctx.GetMemory(maxSizeOfEscapePos + state.StringSize); _allocations.Add(mem); Memory.Copy(mem.Address, state.StringBuffer, state.StringSize); var lazyStringValueFromParserState = _ctx.AllocateStringValue(null, mem.Address, state.StringSize); if (escapePositionsCount > 0) { lazyStringValueFromParserState.EscapePositions = state.EscapePositions.ToArray(); } return(lazyStringValueFromParserState); }
private static ByteStringContext.InternalScope UnicodeGetLowerIdAndStorageKey <TTransaction>( TransactionOperationContext <TTransaction> context, string str, out Slice lowerIdSlice, out Slice idSlice, int maxStrSize, int escapePositionsSize) where TTransaction : RavenTransaction { // See comment in GetLowerIdSliceAndStorageKey for the format var maxIdLenSize = JsonParserState.VariableSizeIntSize(maxStrSize); int strLength = str.Length; var scope = context.Allocator.Allocate( sizeof(char) * strLength // for the lower calls + maxStrSize // lower ID + maxIdLenSize // the size of var int for the len of the ID + maxStrSize // actual ID + escapePositionsSize, out ByteString buffer); fixed(char *pChars = str) { var size = Encoding.GetBytes(pChars, strLength, buffer.Ptr, maxStrSize); _jsonParserState.FindEscapePositionsIn(buffer.Ptr, size, escapePositionsSize); var destChars = (char *)buffer.Ptr; for (var i = 0; i < strLength; i++) { destChars[i] = char.ToLowerInvariant(pChars[i]); } byte *lowerId = buffer.Ptr + strLength * sizeof(char); int lowerSize = Encoding.GetBytes(destChars, strLength, lowerId, maxStrSize); if (lowerSize > 512) { ThrowDocumentIdTooBig(str); } byte *id = buffer.Ptr + strLength * sizeof(char) + maxStrSize; byte *writePos = id; int idSize = Encoding.GetBytes(pChars, strLength, writePos + maxIdLenSize, maxStrSize); var actualIdLenSize = JsonParserState.VariableSizeIntSize(idSize); if (actualIdLenSize < maxIdLenSize) { var movePtr = maxIdLenSize - actualIdLenSize; id += movePtr; writePos += movePtr; } JsonParserState.WriteVariableSizeInt(ref writePos, idSize); escapePositionsSize = _jsonParserState.WriteEscapePositionsTo(writePos + idSize); idSize += escapePositionsSize + maxIdLenSize; Slice.External(context.Allocator, id, idSize, out idSlice); Slice.External(context.Allocator, lowerId, lowerSize, out lowerIdSlice); return(scope); } }
public static ByteStringContext.InternalScope GetLowerIdSliceAndStorageKey <TTransaction>( TransactionOperationContext <TTransaction> context, string str, out Slice lowerIdSlice, out Slice idSlice) where TTransaction : RavenTransaction { // Because we need to also store escape positions for the key when we store it // we need to store it as a lazy string value. // But lazy string value has two lengths, one is the string length, and the other // is the actual data size with the escape positions // In order to resolve this, we process the key to find escape positions, then store it // in the table using the following format: // // [var int - string len, string bytes, number of escape positions, escape positions] // // The total length of the string is stored in the actual table (and include the var int size // prefix. if (_jsonParserState == null) { _jsonParserState = new JsonParserState(); } _jsonParserState.Reset(); int strLength = str.Length; int maxStrSize = Encoding.GetMaxByteCount(strLength); int idSize = JsonParserState.VariableSizeIntSize(strLength); int escapePositionsSize = JsonParserState.FindEscapePositionsMaxSize(str); var scope = context.Allocator.Allocate(maxStrSize // lower key + idSize // the size of var int for the len of the key + maxStrSize // actual key + escapePositionsSize, out ByteString buffer); byte *ptr = buffer.Ptr; fixed(char *pChars = str) { for (var i = 0; i < strLength; i++) { uint ch = pChars[i]; // PERF: Trick to avoid multiple compare instructions on hot loops. // This is the same as (ch >= 65 && ch <= 90) if (ch - 65 <= 90 - 65) { ptr[i] = (byte)(ch | 0x20); } else { if (ch > 127) // not ASCII, use slower mode { goto UnlikelyUnicode; } ptr[i] = (byte)ch; } ptr[i + idSize + maxStrSize] = (byte)ch; } _jsonParserState.FindEscapePositionsIn(ptr, strLength, escapePositionsSize); } var writePos = ptr + maxStrSize; JsonParserState.WriteVariableSizeInt(ref writePos, strLength); escapePositionsSize = _jsonParserState.WriteEscapePositionsTo(writePos + strLength); idSize = escapePositionsSize + strLength + idSize; Slice.External(context.Allocator, ptr + maxStrSize, idSize, out idSlice); Slice.External(context.Allocator, ptr, str.Length, out lowerIdSlice); return(scope); UnlikelyUnicode: scope.Dispose(); return(UnicodeGetLowerIdAndStorageKey(context, str, out lowerIdSlice, out idSlice, maxStrSize, escapePositionsSize)); }
public static ByteStringContext.InternalScope GetLowerIdSliceAndStorageKey <TTransaction>( TransactionOperationContext <TTransaction> context, string str, out Slice lowerIdSlice, out Slice idSlice) where TTransaction : RavenTransaction { // Because we need to also store escape positions for the key when we store it // we need to store it as a lazy string value. // But lazy string value has two lengths, one is the string length, and the other // is the actual data size with the escape positions // In order to resolve this, we process the key to find escape positions, then store it // in the table using the following format: // // [var int - string len, string bytes, number of escape positions, escape positions] // // The total length of the string is stored in the actual table (and include the var int size // prefix. if (_jsonParserState == null) { _jsonParserState = new JsonParserState(); } _jsonParserState.Reset(); int originalStrLength = str.Length; int strLength = originalStrLength; if (strLength > MaxIdSize) { ThrowDocumentIdTooBig(str); } int escapePositionsSize = JsonParserState.FindEscapePositionsMaxSize(str, out var escapedCount); /* * add the size of all control characters * this is to treat case when we have 2+ control character in a row * GetMaxByteCount returns smaller size than the actual size with escaped control characters * For example: string with two control characters such as '\0\0' will be converted to '\u0000\u0000' (another example: '\b\b' => '\u000b\u000b') * string size = 2, GetMaxByteCount = 9, converted string size = 12, maxStrSize = 19 */ var maxStrSize = Encoding.GetMaxByteCount(strLength) + JsonParserState.ControlCharacterItemSize * escapedCount; var originalMaxStrSize = maxStrSize; int idSize = JsonParserState.VariableSizeIntSize(maxStrSize); var scope = context.Allocator.Allocate(maxStrSize // lower key + idSize // the size of var int for the len of the key + maxStrSize // actual key + escapePositionsSize, out ByteString buffer); byte *ptr = buffer.Ptr; fixed(char *pChars = str) { for (var i = 0; i < strLength; i++) { uint ch = pChars[i]; // PERF: Trick to avoid multiple compare instructions on hot loops. // This is the same as (ch >= 65 && ch <= 90) if (ch - 65 <= 90 - 65) { ptr[i] = (byte)(ch | 0x20); } else { if (ch > 127) // not ASCII, use slower mode { goto UnlikelyUnicode; } ptr[i] = (byte)ch; } ptr[i + idSize + maxStrSize] = (byte)ch; } _jsonParserState.FindEscapePositionsIn(ptr, ref strLength, escapePositionsSize); if (strLength != originalStrLength) { var anotherStrLength = originalStrLength; _jsonParserState.FindEscapePositionsIn(ptr + idSize + maxStrSize, ref anotherStrLength, escapePositionsSize); #if DEBUG if (strLength != anotherStrLength) { throw new InvalidOperationException($"String length mismatch between Id ({str}) and it's lowercased counterpart after finding escape positions. Original: {anotherStrLength}. Lowercased: {strLength}"); } #endif } } var writePos = ptr + maxStrSize; Debug.Assert(strLength <= originalMaxStrSize, $"Calculated {nameof(originalMaxStrSize)} value {originalMaxStrSize}, was smaller than actually {nameof(strLength)} value {strLength}"); // in case there were no control characters the idSize could be smaller var sizeDifference = idSize - JsonParserState.VariableSizeIntSize(strLength); writePos += sizeDifference; idSize -= sizeDifference; JsonParserState.WriteVariableSizeInt(ref writePos, strLength); escapePositionsSize = _jsonParserState.WriteEscapePositionsTo(writePos + strLength); idSize = escapePositionsSize + strLength + idSize; Slice.External(context.Allocator, ptr + maxStrSize + sizeDifference, idSize, out idSlice); Slice.External(context.Allocator, ptr, strLength, out lowerIdSlice); return(scope); UnlikelyUnicode: scope.Dispose(); return(UnicodeGetLowerIdAndStorageKey(context, str, out lowerIdSlice, out idSlice, maxStrSize, escapePositionsSize)); }
public static void GetLowerKeySliceAndStorageKey(JsonOperationContext context, string str, out byte *lowerKey, out int lowerSize, out byte *key, out int keySize) { // Because we need to also store escape positions for the key when we store it // we need to store it as a lazy string value. // But lazy string value has two lengths, one is the string length, and the other // is the actual data size with the escape positions // In order to resolve this, we process the key to find escape positions, then store it // in the table using the following format: // // [var int - string len, string bytes, number of escape positions, escape positions] // // The total length of the string is stored in the actual table (and include the var int size // prefix. if (_jsonParserState == null) { _jsonParserState = new JsonParserState(); } _jsonParserState.Reset(); keySize = JsonParserState.VariableSizeIntSize(str.Length); _jsonParserState.FindEscapePositionsIn(str); var escapePositionsSize = _jsonParserState.GetEscapePositionsSize(); var buffer = context.GetNativeTempBuffer( str.Length // lower key + keySize // the size of var int for the len of the key + str.Length // actual key + escapePositionsSize); for (var i = 0; i < str.Length; i++) { var ch = str[i]; if (ch > 127) // not ASCII, use slower mode { goto UnlikelyUnicode; } if ((ch >= 65) && (ch <= 90)) { buffer[i] = (byte)(ch | 0x20); } else { buffer[i] = (byte)ch; } buffer[i + keySize + str.Length] = (byte)ch; } var writePos = buffer + str.Length; JsonParserState.WriteVariableSizeInt(ref writePos, str.Length); _jsonParserState.WriteEscapePositionsTo(writePos + str.Length); keySize = escapePositionsSize + str.Length + keySize; key = buffer + str.Length; lowerKey = buffer; lowerSize = str.Length; return; UnlikelyUnicode: UnicodeGetLowerKeySliceAndStorageKey(context, str, out lowerKey, out lowerSize, out key, out keySize); }