private void SetStringBuffer(string str) { // max possible size - we avoid using GetByteCount because profiling showed it to take 2% of runtime // the buffer might be a bit longer, but we'll reuse it, and it is better than the computing cost int byteCount = Encodings.Utf8.GetMaxByteCount(str.Length); int escapePositionsSize = JsonParserState.FindEscapePositionsMaxSize(str, out _); // If we do not have a buffer or the buffer is too small, return the memory and get more. var size = byteCount + escapePositionsSize; if (_currentStateBuffer == null || _currentStateBuffer.SizeInBytes < size) { if (_currentStateBuffer != null) { _ctx.ReturnMemory(_currentStateBuffer); } _currentStateBuffer = _ctx.GetMemory(size); Debug.Assert(_currentStateBuffer != null && _currentStateBuffer.Address != null); } _state.StringBuffer = _currentStateBuffer.Address; fixed(char *pChars = str) { _state.StringSize = Encodings.Utf8.GetBytes(pChars, str.Length, _state.StringBuffer, byteCount); _state.CompressedSize = null; // don't even try _state.FindEscapePositionsIn(_state.StringBuffer, ref _state.StringSize, escapePositionsSize); var escapePos = _state.StringBuffer + _state.StringSize; _state.WriteEscapePositionsTo(escapePos); } }
private unsafe LazyStringValue GetLazyString(StringSegment field, bool longLived) { var state = new JsonParserState(); var maxByteCount = Encodings.Utf8.GetMaxByteCount(field.Length); int escapePositionsSize = JsonParserState.FindEscapePositionsMaxSize(field); int memorySize = maxByteCount + escapePositionsSize; var memory = longLived ? GetLongLivedMemory(memorySize) : GetMemory(memorySize); fixed(char *pField = field.Buffer) { var address = memory.Address; var actualSize = Encodings.Utf8.GetBytes(pField + field.Offset, field.Length, address, memory.SizeInBytes); state.FindEscapePositionsIn(address, actualSize, escapePositionsSize); state.WriteEscapePositionsTo(address + actualSize); LazyStringValue result = longLived == false?AllocateStringValue(field, address, actualSize) : new LazyStringValue(field, address, actualSize, this); result.AllocatedMemoryData = memory; if (state.EscapePositions.Count > 0) { result.EscapePositions = state.EscapePositions.ToArray(); } return(result); } }
public unsafe LazyStringValue GetLazyString(char[] chars, int start, int count) { LazyStringValue value; var state = new JsonParserState(); state.FindEscapePositionsIn(chars, start, count); var maxByteCount = Encoding.GetMaxByteCount(count); var memory = GetMemory(maxByteCount + state.GetEscapePositionsSize()); try { fixed(char *pChars = chars) { var address = (byte *)memory.Address; var actualSize = Encoding.GetBytes(pChars + start, count, address, memory.SizeInBytes); state.WriteEscapePositionsTo(address + actualSize); value = new LazyStringValue(null, address, actualSize, this); } } catch (Exception) { ReturnMemory(memory); throw; } return(value); }
public unsafe LazyStringValue GetLazyString(string field) { var state = new JsonParserState(); state.FindEscapePositionsIn(field); var maxByteCount = Encoding.GetMaxByteCount(field.Length); var memory = GetMemory(maxByteCount + state.GetEscapePositionsSize()); try { fixed(char *pField = field) { var address = (byte *)memory.Address; var actualSize = Encoding.GetBytes(pField, field.Length, address, memory.SizeInBytes); state.WriteEscapePositionsTo(address + actualSize); return(new LazyStringValue(field, address, actualSize, this) { AllocatedMemoryData = memory }); } } catch (Exception) { ReturnMemory(memory); throw; } }
private unsafe LazyStringValue GetLazyString(StringSegment field, bool longLived) { var state = new JsonParserState(); state.FindEscapePositionsIn(field); var maxByteCount = Encoding.GetMaxByteCount(field.Length); var memory = GetMemory(maxByteCount + state.GetEscapePositionsSize(), longLived: longLived); fixed(char *pField = field.String) { var address = memory.Address; var actualSize = Encoding.GetBytes(pField + field.Start, field.Length, address, memory.SizeInBytes); state.WriteEscapePositionsTo(address + actualSize); var result = new LazyStringValue(field, address, actualSize, this) { AllocatedMemoryData = memory, }; if (state.EscapePositions.Count > 0) { result.EscapePositions = state.EscapePositions.ToArray(); } return(result); } }
public static ByteStringContext.InternalScope GetLowerIdSliceAndStorageKey <TTransaction>( TransactionOperationContext <TTransaction> context, string str, out Slice lowerIdSlice, out Slice idSlice) where TTransaction : RavenTransaction { // Because we need to also store escape positions for the key when we store it // we need to store it as a lazy string value. // But lazy string value has two lengths, one is the string length, and the other // is the actual data size with the escape positions // In order to resolve this, we process the key to find escape positions, then store it // in the table using the following format: // // [var int - string len, string bytes, number of escape positions, escape positions] // // The total length of the string is stored in the actual table (and include the var int size // prefix. if (_jsonParserState == null) { _jsonParserState = new JsonParserState(); } _jsonParserState.Reset(); int strLength = str.Length; int maxStrSize = Encoding.GetMaxByteCount(strLength); int idSize = JsonParserState.VariableSizeIntSize(strLength); int escapePositionsSize = JsonParserState.FindEscapePositionsMaxSize(str); var scope = context.Allocator.Allocate(maxStrSize // lower key + idSize // the size of var int for the len of the key + maxStrSize // actual key + escapePositionsSize, out ByteString buffer); byte *ptr = buffer.Ptr; fixed(char *pChars = str) { for (var i = 0; i < strLength; i++) { uint ch = pChars[i]; // PERF: Trick to avoid multiple compare instructions on hot loops. // This is the same as (ch >= 65 && ch <= 90) if (ch - 65 <= 90 - 65) { ptr[i] = (byte)(ch | 0x20); } else { if (ch > 127) // not ASCII, use slower mode { goto UnlikelyUnicode; } ptr[i] = (byte)ch; } ptr[i + idSize + maxStrSize] = (byte)ch; } _jsonParserState.FindEscapePositionsIn(ptr, strLength, escapePositionsSize); } var writePos = ptr + maxStrSize; JsonParserState.WriteVariableSizeInt(ref writePos, strLength); escapePositionsSize = _jsonParserState.WriteEscapePositionsTo(writePos + strLength); idSize = escapePositionsSize + strLength + idSize; Slice.External(context.Allocator, ptr + maxStrSize, idSize, out idSlice); Slice.External(context.Allocator, ptr, str.Length, out lowerIdSlice); return(scope); UnlikelyUnicode: scope.Dispose(); return(UnicodeGetLowerIdAndStorageKey(context, str, out lowerIdSlice, out idSlice, maxStrSize, escapePositionsSize)); }
public static ByteStringContext.InternalScope GetLowerIdSliceAndStorageKey <TTransaction>( TransactionOperationContext <TTransaction> context, string str, out Slice lowerIdSlice, out Slice idSlice) where TTransaction : RavenTransaction { // Because we need to also store escape positions for the key when we store it // we need to store it as a lazy string value. // But lazy string value has two lengths, one is the string length, and the other // is the actual data size with the escape positions // In order to resolve this, we process the key to find escape positions, then store it // in the table using the following format: // // [var int - string len, string bytes, number of escape positions, escape positions] // // The total length of the string is stored in the actual table (and include the var int size // prefix. if (_jsonParserState == null) { _jsonParserState = new JsonParserState(); } _jsonParserState.Reset(); int originalStrLength = str.Length; int strLength = originalStrLength; if (strLength > MaxIdSize) { ThrowDocumentIdTooBig(str); } int escapePositionsSize = JsonParserState.FindEscapePositionsMaxSize(str, out var escapedCount); /* * add the size of all control characters * this is to treat case when we have 2+ control character in a row * GetMaxByteCount returns smaller size than the actual size with escaped control characters * For example: string with two control characters such as '\0\0' will be converted to '\u0000\u0000' (another example: '\b\b' => '\u000b\u000b') * string size = 2, GetMaxByteCount = 9, converted string size = 12, maxStrSize = 19 */ var maxStrSize = Encoding.GetMaxByteCount(strLength) + JsonParserState.ControlCharacterItemSize * escapedCount; var originalMaxStrSize = maxStrSize; int idSize = JsonParserState.VariableSizeIntSize(maxStrSize); var scope = context.Allocator.Allocate(maxStrSize // lower key + idSize // the size of var int for the len of the key + maxStrSize // actual key + escapePositionsSize, out ByteString buffer); byte *ptr = buffer.Ptr; fixed(char *pChars = str) { for (var i = 0; i < strLength; i++) { uint ch = pChars[i]; // PERF: Trick to avoid multiple compare instructions on hot loops. // This is the same as (ch >= 65 && ch <= 90) if (ch - 65 <= 90 - 65) { ptr[i] = (byte)(ch | 0x20); } else { if (ch > 127) // not ASCII, use slower mode { goto UnlikelyUnicode; } ptr[i] = (byte)ch; } ptr[i + idSize + maxStrSize] = (byte)ch; } _jsonParserState.FindEscapePositionsIn(ptr, ref strLength, escapePositionsSize); if (strLength != originalStrLength) { var anotherStrLength = originalStrLength; _jsonParserState.FindEscapePositionsIn(ptr + idSize + maxStrSize, ref anotherStrLength, escapePositionsSize); #if DEBUG if (strLength != anotherStrLength) { throw new InvalidOperationException($"String length mismatch between Id ({str}) and it's lowercased counterpart after finding escape positions. Original: {anotherStrLength}. Lowercased: {strLength}"); } #endif } } var writePos = ptr + maxStrSize; Debug.Assert(strLength <= originalMaxStrSize, $"Calculated {nameof(originalMaxStrSize)} value {originalMaxStrSize}, was smaller than actually {nameof(strLength)} value {strLength}"); // in case there were no control characters the idSize could be smaller var sizeDifference = idSize - JsonParserState.VariableSizeIntSize(strLength); writePos += sizeDifference; idSize -= sizeDifference; JsonParserState.WriteVariableSizeInt(ref writePos, strLength); escapePositionsSize = _jsonParserState.WriteEscapePositionsTo(writePos + strLength); idSize = escapePositionsSize + strLength + idSize; Slice.External(context.Allocator, ptr + maxStrSize + sizeDifference, idSize, out idSlice); Slice.External(context.Allocator, ptr, strLength, out lowerIdSlice); return(scope); UnlikelyUnicode: scope.Dispose(); return(UnicodeGetLowerIdAndStorageKey(context, str, out lowerIdSlice, out idSlice, maxStrSize, escapePositionsSize)); }
public static void GetLowerKeySliceAndStorageKey(JsonOperationContext context, string str, out byte *lowerKey, out int lowerSize, out byte *key, out int keySize) { // Because we need to also store escape positions for the key when we store it // we need to store it as a lazy string value. // But lazy string value has two lengths, one is the string length, and the other // is the actual data size with the escape positions // In order to resolve this, we process the key to find escape positions, then store it // in the table using the following format: // // [var int - string len, string bytes, number of escape positions, escape positions] // // The total length of the string is stored in the actual table (and include the var int size // prefix. if (_jsonParserState == null) { _jsonParserState = new JsonParserState(); } _jsonParserState.Reset(); keySize = JsonParserState.VariableSizeIntSize(str.Length); _jsonParserState.FindEscapePositionsIn(str); var escapePositionsSize = _jsonParserState.GetEscapePositionsSize(); var buffer = context.GetNativeTempBuffer( str.Length // lower key + keySize // the size of var int for the len of the key + str.Length // actual key + escapePositionsSize); for (var i = 0; i < str.Length; i++) { var ch = str[i]; if (ch > 127) // not ASCII, use slower mode { goto UnlikelyUnicode; } if ((ch >= 65) && (ch <= 90)) { buffer[i] = (byte)(ch | 0x20); } else { buffer[i] = (byte)ch; } buffer[i + keySize + str.Length] = (byte)ch; } var writePos = buffer + str.Length; JsonParserState.WriteVariableSizeInt(ref writePos, str.Length); _jsonParserState.WriteEscapePositionsTo(writePos + str.Length); keySize = escapePositionsSize + str.Length + keySize; key = buffer + str.Length; lowerKey = buffer; lowerSize = str.Length; return; UnlikelyUnicode: UnicodeGetLowerKeySliceAndStorageKey(context, str, out lowerKey, out lowerSize, out key, out keySize); }