private static void WriteUInt64Slow(ref SliceWriter writer, ulong value)
{
// We are only called for values >= 256
// determine the number of bytes needed to encode the value
int bytes = NumberOfBytes(value);
writer.EnsureBytes(bytes + 1);
var buffer = writer.Buffer;
int p = writer.Position;
// simple case (ulong can only be positive)
buffer[p++] = (byte)(FdbTupleTypes.IntBase + bytes);
if (bytes > 0)
{
// head
--bytes;
int shift = bytes << 3;
while (bytes-- > 0)
{
buffer[p++] = (byte)(value >> shift);
shift -= 8;
}
// last
buffer[p++] = (byte)value;
}
writer.Position = p;
}
/// <summary>Writes a char encoded in UTF-8</summary>
public static void WriteChar(ref SliceWriter writer, char value)
{
if (value == 0)
{ // NUL => "00 0F"
// note: \0 is the only unicode character that will produce a zero byte when converted in UTF-8
writer.WriteByte4(FdbTupleTypes.Utf8, 0x00, 0xFF, 0x00);
}
else if (value < 0x80)
{ // 0x00..0x7F => 0xxxxxxx
writer.WriteByte3(FdbTupleTypes.Utf8, (byte)value, 0x00);
}
else if (value < 0x800)
{ // 0x80..0x7FF => 110xxxxx 10xxxxxx => two bytes
writer.WriteByte4(FdbTupleTypes.Utf8, (byte)(0xC0 | (value >> 6)), (byte)(0x80 | (value & 0x3F)), 0x00);
}
else
{ // 0x800..0xFFFF => 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
// note: System.Char is 16 bits, and thus cannot represent UNICODE chars above 0xFFFF.
// => This means that a System.Char will never take more than 3 bytes in UTF-8 !
var tmp = Encoding.UTF8.GetBytes(new string(value, 1));
writer.EnsureBytes(tmp.Length + 2);
writer.UnsafeWriteByte(FdbTupleTypes.Utf8);
writer.UnsafeWriteBytes(tmp, 0, tmp.Length);
writer.UnsafeWriteByte(0x00);
}
}
/// <summary>Writes a 64-bit UUID</summary>
public static void WriteUuid64(ref SliceWriter writer, Uuid64 value)
{
writer.EnsureBytes(9);
writer.UnsafeWriteByte(FdbTupleTypes.Uuid64);
unsafe
{
byte *ptr = stackalloc byte[8];
value.WriteTo(ptr);
writer.UnsafeWriteBytes(ptr, 8);
}
}
/// <summary>Writes a RFC 4122 encoded 128-bit UUID</summary>
public static void WriteUuid128(ref SliceWriter writer, Uuid128 value)
{
writer.EnsureBytes(17);
writer.UnsafeWriteByte(FdbTupleTypes.Uuid128);
unsafe
{
byte *ptr = stackalloc byte[16];
value.WriteTo(ptr);
writer.UnsafeWriteBytes(ptr, 16);
}
}
/// <summary>Writes a RFC 4122 encoded 16-byte Microsoft GUID</summary>
public static void WriteGuid(ref SliceWriter writer, Guid value)
{
writer.EnsureBytes(17);
writer.UnsafeWriteByte(FdbTupleTypes.Uuid128);
unsafe
{
// UUIDs are stored using the RFC 4122 standard, so we need to swap some parts of the System.Guid
byte *ptr = stackalloc byte[16];
Uuid128.Write(value, ptr);
writer.UnsafeWriteBytes(ptr, 16);
}
}
private static void WriteInt64Slow(ref SliceWriter writer, long value)
{
// we are only called for values <= -256 or >= 256
// determine the number of bytes needed to encode the absolute value
int bytes = NumberOfBytes(value);
writer.EnsureBytes(bytes + 1);
var buffer = writer.Buffer;
int p = writer.Position;
ulong v;
if (value > 0)
{ // simple case
buffer[p++] = (byte)(FdbTupleTypes.IntBase + bytes);
v = (ulong)value;
}
else
{ // we will encode the one's complement of the absolute value
// -1 => 0xFE
// -256 => 0xFFFE
// -65536 => 0xFFFFFE
buffer[p++] = (byte)(FdbTupleTypes.IntBase - bytes);
v = (ulong)(~(-value));
}
if (bytes > 0)
{
// head
--bytes;
int shift = bytes << 3;
while (bytes-- > 0)
{
buffer[p++] = (byte)(v >> shift);
shift -= 8;
}
// last
buffer[p++] = (byte)v;
}
writer.Position = p;
}
/// <summary>Writes a buffer with all instances of 0 escaped as '00 FF'</summary>
internal static void WriteNulEscapedBytes(ref SliceWriter writer, byte type, [NotNull] byte[] value, int offset, int count)
{
int n = count;
// we need to know if there are any NUL chars (\0) that need escaping...
// (we will also need to add 1 byte to the buffer size per NUL)
for (int i = offset, end = offset + count; i < end; ++i)
{
if (value[i] == 0)
{
++n;
}
}
writer.EnsureBytes(n + 2);
var buffer = writer.Buffer;
int p = writer.Position;
buffer[p++] = type;
if (n > 0)
{
if (n == count)
{ // no NULs in the string, can copy all at once
SliceHelpers.CopyBytesUnsafe(buffer, p, value, offset, n);
p += n;
}
else
{ // we need to escape all NULs
for (int i = offset, end = offset + count; i < end; ++i)
{
byte b = value[i];
buffer[p++] = b;
if (b == 0)
{
buffer[p++] = 0xFF;
}
}
}
}
buffer[p] = FdbTupleTypes.Nil;
writer.Position = p + 1;
}
/// <summary>Writes a buffer with all instances of 0 escaped as '00 FF'</summary>
private static void WriteNulEscapedBytes(ref SliceWriter writer, byte type, [NotNull] byte[] value)
{
int n = value.Length;
// we need to know if there are any NUL chars (\0) that need escaping...
// (we will also need to add 1 byte to the buffer size per NUL)
foreach (byte b in value)
{
if (b == 0)
{
++n;
}
}
writer.EnsureBytes(n + 2);
var buffer = writer.Buffer;
int p = writer.Position;
buffer[p++] = type;
if (n > 0)
{
if (n == value.Length)
{ // no NULs in the string, can copy all at once
SliceHelpers.CopyBytesUnsafe(buffer, p, value, 0, n);
p += n;
}
else
{ // we need to escape all NULs
foreach (byte b in value)
{
buffer[p++] = b;
if (b == 0)
{
buffer[p++] = 0xFF;
}
}
}
}
buffer[p++] = FdbTupleTypes.Nil;
writer.Position = p;
}
/// <summary>Writes an Double at the end, and advance the cursor</summary>
/// <param name="writer">Target buffer</param>
/// <param name="value">IEEE Floating point, 64 bits, High Endian</param>
public static void WriteDouble(ref SliceWriter writer, double value)
{
// The double is converted to its Big-Endian IEEE binary representation
// - If the sign bit is set, flip all the bits
// - If the sign bit is not set, just flip the sign bit
// This ensures that all negative numbers have their first byte < 0x80, and all positive numbers have their first byte >= 0x80
// Special case for NaN: All variants are normalized to float.NaN !
if (double.IsNaN(value))
{
value = double.NaN;
}
// note: we could use BitConverter.DoubleToInt64Bits(...), but it does the same thing, and also it does not exist for floats...
ulong bits;
unsafe { bits = *((ulong *)&value); }
if ((bits & 0x8000000000000000UL) != 0)
{ // negative
bits = ~bits;
}
else
{ // postive
bits |= 0x8000000000000000UL;
}
writer.EnsureBytes(9);
var buffer = writer.Buffer;
int p = writer.Position;
buffer[p] = FdbTupleTypes.Double;
buffer[p + 1] = (byte)(bits >> 56);
buffer[p + 2] = (byte)(bits >> 48);
buffer[p + 3] = (byte)(bits >> 40);
buffer[p + 4] = (byte)(bits >> 32);
buffer[p + 5] = (byte)(bits >> 24);
buffer[p + 6] = (byte)(bits >> 16);
buffer[p + 7] = (byte)(bits >> 8);
buffer[p + 8] = (byte)(bits);
writer.Position = p + 9;
}
private static unsafe void WriteUnescapedAsciiChars(ref SliceWriter writer, char *chars, int count)
{
Contract.Requires(chars != null && count >= 0);
// copy and convert an ASCII string directly into the destination buffer
writer.EnsureBytes(2 + count);
int pos = writer.Position;
char *end = chars + count;
fixed(byte *buffer = writer.Buffer)
{
buffer[pos++] = FdbTupleTypes.Utf8;
while (chars < end)
{
buffer[pos++] = (byte)(*chars++);
}
buffer[pos] = 0x00;
writer.Position = pos + 1;
}
}
/// <summary>Writes an Single at the end, and advance the cursor</summary>
/// <param name="writer">Target buffer</param>
/// <param name="value">IEEE Floating point, 32 bits, High Endian</param>
public static void WriteSingle(ref SliceWriter writer, float value)
{
// The double is converted to its Big-Endian IEEE binary representation
// - If the sign bit is set, flip all the bits
// - If the sign bit is not set, just flip the sign bit
// This ensures that all negative numbers have their first byte < 0x80, and all positive numbers have their first byte >= 0x80
// Special case for NaN: All variants are normalized to float.NaN !
if (float.IsNaN(value))
{
value = float.NaN;
}
// note: there is no BitConverter.SingleToInt32Bits(...), so we have to do it ourselves...
uint bits;
unsafe { bits = *((uint *)&value); }
if ((bits & 0x80000000U) != 0)
{ // negative
bits = ~bits;
}
else
{ // postive
bits |= 0x80000000U;
}
writer.EnsureBytes(5);
var buffer = writer.Buffer;
int p = writer.Position;
buffer[p + 0] = FdbTupleTypes.Single;
buffer[p + 1] = (byte)(bits >> 24);
buffer[p + 2] = (byte)(bits >> 16);
buffer[p + 3] = (byte)(bits >> 8);
buffer[p + 4] = (byte)(bits);
writer.Position = p + 5;
}
private static unsafe bool TryWriteUnescapedUtf8String(ref SliceWriter writer, char *chars, int count)
{
Contract.Requires(chars != null && count >= 0);
// Several observations:
// * Most strings will be keywords or ASCII-only with no zeroes. These can be copied directly to the buffer
// * We will only attempt to optimze strings that don't have any 00 to escape to 00 FF. For these, we will fallback to converting to byte[] then escaping.
// * Since .NET's strings are UTF-16, the max possible UNICODE value to encode is 0xFFFF, which takes 3 bytes in UTF-8 (EF BF BF)
// * Most western europe languages have only a few non-ASCII chars here and there, and most of them will only use 2 bytes (ex: 'é' => 'C3 A9')
// * More complex scripts with dedicated symbol pages (kanjis, arabic, ....) will take 2 or 3 bytes for each charecter.
// We will first do a pass to check for the presence of 00 and non-ASCII chars
// => if we find at least on 00, we fallback to escaping the result of Encoding.UTF8.GetBytes()
// => if we find only ASCII (1..127) chars, we have an optimized path that will truncate the chars to bytes
// => if not, we will use an UTF8Encoder to convert the string to UTF-8, in chunks, using a small buffer allocated on the stack
#region First pass: look for \0 and non-ASCII chars
// fastest way to check for non-ASCII, is to OR all the chars together, and look at bits 7 to 15. If they are not all zero, there is at least ONE non-ASCII char.
// also, we abort as soon as we find a \0
char *ptr = chars;
char *end = chars + count;
char mask = '\0', c;
while (ptr < end && (c = *ptr) != '\0')
{
mask |= c; ++ptr;
}
if (ptr < end)
{
return(false); // there is at least one \0 in the string
}
// bit 7-15 all unset means the string is pure ASCII
if ((mask >> 7) == 0)
{ // => directly dump the chars to the buffer
WriteUnescapedAsciiChars(ref writer, chars, count);
return(true);
}
#endregion
#region Second pass: encode the string to UTF-8, in chunks
// Here we know that there is at least one unicode char, and that there are no \0
// We will tterate through the string, filling as much of the buffer as possible
bool done;
int remaining = count;
ptr = chars;
// We need at most 3 * CHUNK_SIZE to encode the chunk
// > For small strings, we will allocated exactly string.Length * 3 bytes, and will be done in one chunk
// > For larger strings, we will call encoder.Convert(...) until it says it is done.
const int CHUNK_SIZE = 1024;
int bufLen = Encoding.UTF8.GetMaxByteCount(Math.Min(count, CHUNK_SIZE));
byte * buf = stackalloc byte[bufLen];
// We can not really predict the final size of the encoded string, but:
// * Western languages have a few chars that usually need 2 bytes. If we pre-allocate 50% more bytes, it should fit most of the time, without too much waste
// * Eastern langauges will have all chars encoded to 3 bytes. If we also pre-allocated 50% more, we should only need one resize of the buffer (150% x 2 = 300%), which is acceptable
writer.EnsureBytes(checked (2 + count + (count >> 1))); // preallocate 150% of the string + 2 bytes
writer.UnsafeWriteByte(FdbTupleTypes.Utf8);
var encoder = Encoding.UTF8.GetEncoder();
// note: encoder.Convert() tries to fill up the buffer as much as possible with complete chars, and will set 'done' to true when all chars have been converted.
do
{
int charsUsed, bytesUsed;
encoder.Convert(ptr, remaining, buf, bufLen, true, out charsUsed, out bytesUsed, out done);
if (bytesUsed > 0)
{
writer.WriteBytes(buf, bytesUsed);
}
remaining -= charsUsed;
ptr += charsUsed;
}while (!done);
Contract.Assert(remaining == 0 && ptr == end);
// close the string
writer.WriteByte(0x00);
#endregion
return(true);
}