/// <summary>Writes an Int32 at the end, and advance the cursor</summary> /// <param name="writer">Target buffer</param> /// <param name="value">Signed DWORD, 32 bits, High Endian</param> public static void WriteInt32(ref SliceWriter writer, int value) { if (value <= 255) { if (value == 0) { // zero writer.WriteByte(FdbTupleTypes.IntZero); return; } if (value > 0) { // 1..255: frequent for array index writer.WriteByte2(FdbTupleTypes.IntPos1, (byte)value); return; } if (value > -256) { // -255..-1 writer.WriteByte2(FdbTupleTypes.IntNeg1, (byte)(255 + value)); return; } } WriteInt64Slow(ref writer, (long)value); }
/// <summary>Create a new compressed bitmap writer, with a specific underlying buffer</summary> /// <param name="writer">Existing where the compressed words will be written to</param> /// <param name="ownsBuffer">If true, the buffer is still private and and can modified at will. If false, the buffer has been exposed publicly and a new buffer must be allocated before reusing this writer</param> internal CompressedBitmapWriter(SliceWriter writer, bool ownsBuffer) { m_writer = writer; m_head = writer.Position; m_ownsBuffer = ownsBuffer; Reset(); }
public SliceWriter OpenWriter(int extra = 32) { var key = GetKeyPrefix(); var sw = new SliceWriter(key.Count + extra); //TODO: BufferPool ? sw.WriteBytes(in key); return(sw); }
/// <summary>Writes an UInt8 at the end, and advance the cursor</summary> /// <param name="writer">Target buffer</param> /// <param name="value">Unsigned BYTE, 32 bits</param> public static void WriteInt8(ref SliceWriter writer, byte value) { if (value == 0) { // zero writer.WriteByte(FdbTupleTypes.IntZero); } else { // 1..255: frequent for array index writer.WriteByte2(FdbTupleTypes.IntPos1, value); } }
public static Slice[] Convert(SliceWriter writer, [NotNull, ItemNotNull] IEnumerable <TValue> values, Handler handler, TState state) { Contract.Requires(values != null && handler != null); //Note on performance: // - we will reuse the same buffer for each temp key, and copy them into a slice buffer // - doing it this way adds a memory copy (writer => buffer) but reduce the number of byte[] allocations (and reduce the GC overhead) int start = writer.Position; var buffer = new SliceBuffer(); var coll = values as ICollection <TValue>; if (coll != null) { // pre-allocate the final array with the correct size var res = new Slice[coll.Count]; int p = 0; foreach (var tuple in coll) { // reset position to just after the subspace prefix writer.Position = start; handler(ref writer, tuple, state); // copy full key in the buffer res[p++] = buffer.Intern(writer.ToSlice()); } Contract.Assert(p == res.Length); return(res); } else { // we won't now the array size until the end... var res = new List <Slice>(); foreach (var tuple in values) { // reset position to just after the subspace prefix writer.Position = start; handler(ref writer, tuple, state); // copy full key in the buffer res.Add(buffer.Intern(writer.ToSlice())); } return(res.ToArray()); } }
public SnapshotWriter(Win32SnapshotFile file, int levels, int pageSize, int bufferSize) { Contract.Requires(file != null && levels >= 0 && pageSize >= 0 && bufferSize >= pageSize); //TODO: && file.CanRead ? m_file = file; m_pageSize = pageSize; m_bufferSize = bufferSize; //TODO: verify pageSize is a power of two, and bufferSize is a multiple of pageSize! Contract.Assert(bufferSize % pageSize == 0); m_writer = new SliceWriter(bufferSize); m_levels = levels; m_jumpTable = new KeyValuePair<ulong, ulong>[levels]; for (int i = 0; i < levels; i++) { m_jumpTable[i] = new KeyValuePair<ulong, ulong>(0, 0); } }
private void InitializeDirectory(IFdbTransaction trans) { // Set the version key var writer = new SliceWriter(3 * 4); writer.WriteFixed32((uint)LayerVersion.Major); writer.WriteFixed32((uint)LayerVersion.Minor); writer.WriteFixed32((uint)LayerVersion.Build); trans.Set(this.RootNode.Pack(VersionKey), writer.ToSlice()); }
public DebugView(SliceWriter writer) { m_writer = writer; }
/// <summary>Writes a 64-bit UUID</summary> public static void WriteUuid64(ref SliceWriter writer, Uuid64 value) { writer.EnsureBytes(9); writer.UnsafeWriteByte(FdbTupleTypes.Uuid64); unsafe { byte* ptr = stackalloc byte[8]; value.WriteTo(ptr); writer.UnsafeWriteBytes(ptr, 8); } }
public abstract void EncodeOrderedSelfTerm(ref SliceWriter output, T value);
/// <summary>Writes a buffer with all instances of 0 escaped as '00 FF'</summary> internal static void WriteNulEscapedBytes(ref SliceWriter writer, byte type, byte[] value, int offset, int count) { int n = count; // we need to know if there are any NUL chars (\0) that need escaping... // (we will also need to add 1 byte to the buffer size per NUL) for (int i = offset, end = offset + count; i < end; ++i) { if (value[i] == 0) ++n; } writer.EnsureBytes(n + 2); var buffer = writer.Buffer; int p = writer.Position; buffer[p++] = type; if (n > 0) { if (n == count) { // no NULs in the string, can copy all at once SliceHelpers.CopyBytesUnsafe(buffer, p, value, offset, n); p += n; } else { // we need to escape all NULs for(int i = offset, end = offset + count; i < end; ++i) { byte b = value[i]; buffer[p++] = b; if (b == 0) buffer[p++] = 0xFF; } } } buffer[p] = FdbTupleTypes.Nil; writer.Position = p + 1; }
/// <summary>Writes a RFC 4122 encoded 16-byte Microsoft GUID</summary> public static void WriteGuid(ref SliceWriter writer, Guid value) { writer.EnsureBytes(17); writer.UnsafeWriteByte(FdbTupleTypes.Uuid128); unsafe { // UUIDs are stored using the RFC 4122 standard, so we need to swap some parts of the System.Guid byte* ptr = stackalloc byte[16]; Uuid128.Write(value, ptr); writer.UnsafeWriteBytes(ptr, 16); } }
/// <summary>Writes an Single at the end, and advance the cursor</summary> /// <param name="writer">Target buffer</param> /// <param name="value">IEEE Floating point, 32 bits, High Endian</param> public static void WriteSingle(ref SliceWriter writer, float value) { // The double is converted to its Big-Endian IEEE binary representation // - If the sign bit is set, flip all the bits // - If the sign bit is not set, just flip the sign bit // This ensures that all negative numbers have their first byte < 0x80, and all positive numbers have their first byte >= 0x80 // Special case for NaN: All variants are normalized to float.NaN ! if (float.IsNaN(value)) value = float.NaN; // note: there is no BitConverter.SingleToInt32Bits(...), so we have to do it ourselves... uint bits; unsafe { bits = *((uint*)&value); } if ((bits & 0x80000000U) != 0) { // negative bits = ~bits; } else { // postive bits |= 0x80000000U; } writer.EnsureBytes(5); var buffer = writer.Buffer; int p = writer.Position; buffer[p + 0] = FdbTupleTypes.Single; buffer[p + 1] = (byte)(bits >> 24); buffer[p + 2] = (byte)(bits >> 16); buffer[p + 3] = (byte)(bits >> 8); buffer[p + 4] = (byte)(bits); writer.Position = p + 5; }
/// <summary>Writes a binary string</summary> public static void WriteBytes(ref SliceWriter writer, byte[] value, int offset, int count) { WriteNulEscapedBytes(ref writer, FdbTupleTypes.Bytes, value, offset, count); }
/// <summary>Writes an UInt64 at the end, and advance the cursor</summary> /// <param name="writer">Target buffer</param> /// <param name="value">Signed QWORD, 64 bits, High Endian</param> public static void WriteUInt64(ref SliceWriter writer, ulong value) { if (value <= 255) { if (value == 0) { // 0 writer.WriteByte(FdbTupleTypes.IntZero); } else { // 1..255 writer.WriteByte2(FdbTupleTypes.IntPos1, (byte)value); } } else { // >= 256 WriteUInt64Slow(ref writer, value); } }
private static void WriteUInt64Slow(ref SliceWriter writer, ulong value) { // We are only called for values >= 256 // determine the number of bytes needed to encode the value int bytes = NumberOfBytes(value); writer.EnsureBytes(bytes + 1); var buffer = writer.Buffer; int p = writer.Position; // simple case (ulong can only be positive) buffer[p++] = (byte)(FdbTupleTypes.IntBase + bytes); if (bytes > 0) { // head --bytes; int shift = bytes << 3; while (bytes-- > 0) { buffer[p++] = (byte)(value >> shift); shift -= 8; } // last buffer[p++] = (byte)value; } writer.Position = p; }
private static void WriteInt64Slow(ref SliceWriter writer, long value) { // we are only called for values <= -256 or >= 256 // determine the number of bytes needed to encode the absolute value int bytes = NumberOfBytes(value); writer.EnsureBytes(bytes + 1); var buffer = writer.Buffer; int p = writer.Position; ulong v; if (value > 0) { // simple case buffer[p++] = (byte)(FdbTupleTypes.IntBase + bytes); v = (ulong)value; } else { // we will encode the one's complement of the absolute value // -1 => 0xFE // -256 => 0xFFFE // -65536 => 0xFFFFFE buffer[p++] = (byte)(FdbTupleTypes.IntBase - bytes); v = (ulong)(~(-value)); } if (bytes > 0) { // head --bytes; int shift = bytes << 3; while (bytes-- > 0) { buffer[p++] = (byte)(v >> shift); shift -= 8; } // last buffer[p++] = (byte)v; } writer.Position = p; }
public virtual void EncodeKey <T1, T2, T3, T4>(ref SliceWriter writer, T1 item1, T2 item2, T3 item3, T4 item4) { PackKey(ref writer, FdbTuple.Create(item1, item2, item3, item4)); }
public virtual void EncodeKey <T1, T2>(ref SliceWriter writer, T1 item1, T2 item2) { PackKey(ref writer, FdbTuple.Create(item1, item2)); }
/// <summary>Writes an Double at the end, and advance the cursor</summary> /// <param name="writer">Target buffer</param> /// <param name="value">IEEE Floating point, 64 bits, High Endian</param> public static void WriteDouble(ref SliceWriter writer, double value) { // The double is converted to its Big-Endian IEEE binary representation // - If the sign bit is set, flip all the bits // - If the sign bit is not set, just flip the sign bit // This ensures that all negative numbers have their first byte < 0x80, and all positive numbers have their first byte >= 0x80 // Special case for NaN: All variants are normalized to float.NaN ! if (double.IsNaN(value)) value = double.NaN; // note: we could use BitConverter.DoubleToInt64Bits(...), but it does the same thing, and also it does not exist for floats... ulong bits; unsafe { bits = *((ulong*)&value); } if ((bits & 0x8000000000000000UL) != 0) { // negative bits = ~bits; } else { // postive bits |= 0x8000000000000000UL; } writer.EnsureBytes(9); var buffer = writer.Buffer; int p = writer.Position; buffer[p] = FdbTupleTypes.Double; buffer[p + 1] = (byte)(bits >> 56); buffer[p + 2] = (byte)(bits >> 48); buffer[p + 3] = (byte)(bits >> 40); buffer[p + 4] = (byte)(bits >> 32); buffer[p + 5] = (byte)(bits >> 24); buffer[p + 6] = (byte)(bits >> 16); buffer[p + 7] = (byte)(bits >> 8); buffer[p + 8] = (byte)(bits); writer.Position = p + 9; }
/// <summary>Clear the content of the buffer, and start from scratch</summary> /// <remarks>Keeps the allocated buffer space.</remarks> public void Reset() { if (m_ownsBuffer) { m_writer.Position = m_head; } else { // buffer has been exposed and cannot be reused m_writer = new SliceWriter(); m_head = 0; m_ownsBuffer = true; } m_writer.WriteFixed32(0xFFFFFFFF); // incomplete m_current = NO_VALUE; m_counter = 0; m_words = 0; m_packed = false; }
/// <summary>Writes a binary string</summary> public static void WriteBytes(ref SliceWriter writer, byte[] value) { if (value == null) { writer.WriteByte(FdbTupleTypes.Nil); } else { WriteNulEscapedBytes(ref writer, FdbTupleTypes.Bytes, value); } }
/// <summary>Writes a binary string</summary> public static void WriteBytes(ref SliceWriter writer, ArraySegment<byte> value) { WriteNulEscapedBytes(ref writer, FdbTupleTypes.Bytes, value.Array, value.Offset, value.Count); }
/// <summary>Writes a string encoded in UTF-8</summary> public static unsafe void WriteString(ref SliceWriter writer, string value) { if (value == null) { // "00" writer.WriteByte(FdbTupleTypes.Nil); } else if (value.Length == 0) { // "02 00" writer.WriteByte2(FdbTupleTypes.Utf8, 0x00); } else { fixed(char* chars = value) { if (!TryWriteUnescapedUtf8String(ref writer, chars, value.Length)) { // the string contains \0 chars, we need to do it the hard way WriteNulEscapedBytes(ref writer, FdbTupleTypes.Utf8, Encoding.UTF8.GetBytes(value)); } } } }
/// <summary>Writes a buffer with all instances of 0 escaped as '00 FF'</summary> private static void WriteNulEscapedBytes(ref SliceWriter writer, byte type, byte[] value) { int n = value.Length; // we need to know if there are any NUL chars (\0) that need escaping... // (we will also need to add 1 byte to the buffer size per NUL) foreach (byte b in value) { if (b == 0) ++n; } writer.EnsureBytes(n + 2); var buffer = writer.Buffer; int p = writer.Position; buffer[p++] = type; if (n > 0) { if (n == value.Length) { // no NULs in the string, can copy all at once SliceHelpers.CopyBytesUnsafe(buffer, p, value, 0, n); p += n; } else { // we need to escape all NULs foreach (byte b in value) { buffer[p++] = b; if (b == 0) buffer[p++] = 0xFF; } } } buffer[p++] = FdbTupleTypes.Nil; writer.Position = p; }
/// <summary>Writes a char array encoded in UTF-8</summary> internal static unsafe void WriteChars(ref SliceWriter writer, char[] value, int offset, int count) { Contract.Requires(offset >= 0 && count >= 0); if (count == 0) { if (value == null) { // "00" writer.WriteByte(FdbTupleTypes.Nil); } else { // "02 00" writer.WriteByte2(FdbTupleTypes.Utf8, 0x00); } } else { fixed (char* chars = value) { if (TryWriteUnescapedUtf8String(ref writer, chars + offset, count)) return; } // the string contains \0 chars, we need to do it the hard way WriteNulEscapedBytes(ref writer, FdbTupleTypes.Utf8, Encoding.UTF8.GetBytes(value, 0, count)); } }
/// <summary>Writes a RFC 4122 encoded 128-bit UUID</summary> public static void WriteUuid128(ref SliceWriter writer, Uuid128 value) { writer.EnsureBytes(17); writer.UnsafeWriteByte(FdbTupleTypes.Uuid128); unsafe { byte* ptr = stackalloc byte[16]; value.WriteTo(ptr); writer.UnsafeWriteBytes(ptr, 16); } }
public abstract void PackKey <TTuple>(ref SliceWriter writer, TTuple items) where TTuple : IVarTuple;
public TupleWriter(int capacity) { this.Output = new SliceWriter(capacity); this.Depth = 0; }
public virtual void EncodeKey <T1, T2, T3>(ref SliceWriter writer, T1 item1, T2 item2, T3 item3) { PackKey(ref writer, STuple.Create(item1, item2, item3)); }
public virtual void EncodeUnorderedSelfTerm(ref SliceWriter output, T value) { EncodeOrderedSelfTerm(ref output, value); }
public virtual void EncodeKey <T1, T2, T3, T4, T5, T6, T7, T8>(ref SliceWriter writer, T1 item1, T2 item2, T3 item3, T4 item4, T5 item5, T6 item6, T7 item7, T8 item8) { PackKey(ref writer, STuple.Create(item1, item2, item3, item4, item5, item6, item7, item8)); }
public TupleWriter(SliceWriter buffer) { this.Output = buffer; this.Depth = 0; }
public static void Partial <T1>(ref SliceWriter writer, IOrderedTypeCodec <T1> codec1, T1 value1) { Contract.Assert(codec1 != null); codec1.EncodeOrderedSelfTerm(ref writer, value1); }
private static unsafe void WriteUnescapedAsciiChars(ref SliceWriter writer, char* chars, int count) { Contract.Requires(chars != null && count >= 0); // copy and convert an ASCII string directly into the destination buffer writer.EnsureBytes(2 + count); int pos = writer.Position; char* end = chars + count; fixed (byte* buffer = writer.Buffer) { buffer[pos++] = FdbTupleTypes.Utf8; while(chars < end) { buffer[pos++] = (byte)(*chars++); } buffer[pos] = 0x00; writer.Position = pos + 1; } }
public abstract void PackKey(ref SliceWriter writer, IFdbTuple items);
public virtual void EncodeKey <T1>(ref SliceWriter writer, T1 item1) { PackKey(ref writer, STuple.Create(item1)); }
private static unsafe bool TryWriteUnescapedUtf8String(ref SliceWriter writer, char* chars, int count) { Contract.Requires(chars != null && count >= 0); // Several observations: // * Most strings will be keywords or ASCII-only with no zeroes. These can be copied directly to the buffer // * We will only attempt to optimze strings that don't have any 00 to escape to 00 FF. For these, we will fallback to converting to byte[] then escaping. // * Since .NET's strings are UTF-16, the max possible UNICODE value to encode is 0xFFFF, which takes 3 bytes in UTF-8 (EF BF BF) // * Most western europe languages have only a few non-ASCII chars here and there, and most of them will only use 2 bytes (ex: 'é' => 'C3 A9') // * More complex scripts with dedicated symbol pages (kanjis, arabic, ....) will take 2 or 3 bytes for each charecter. // We will first do a pass to check for the presence of 00 and non-ASCII chars // => if we find at least on 00, we fallback to escaping the result of Encoding.UTF8.GetBytes() // => if we find only ASCII (1..127) chars, we have an optimized path that will truncate the chars to bytes // => if not, we will use an UTF8Encoder to convert the string to UTF-8, in chunks, using a small buffer allocated on the stack #region First pass: look for \0 and non-ASCII chars // fastest way to check for non-ASCII, is to OR all the chars together, and look at bits 7 to 15. If they are not all zero, there is at least ONE non-ASCII char. // also, we abort as soon as we find a \0 char* ptr = chars; char* end = chars + count; char mask = '\0', c; while (ptr < end && (c = *ptr) != '\0') { mask |= c; ++ptr; } if (ptr < end) return false; // there is at least one \0 in the string // bit 7-15 all unset means the string is pure ASCII if ((mask >> 7) == 0) { // => directly dump the chars to the buffer WriteUnescapedAsciiChars(ref writer, chars, count); return true; } #endregion #region Second pass: encode the string to UTF-8, in chunks // Here we know that there is at least one unicode char, and that there are no \0 // We will tterate through the string, filling as much of the buffer as possible bool done; int charsUsed, bytesUsed; int remaining = count; ptr = chars; // We need at most 3 * CHUNK_SIZE to encode the chunk // > For small strings, we will allocated exactly string.Length * 3 bytes, and will be done in one chunk // > For larger strings, we will call encoder.Convert(...) until it says it is done. const int CHUNK_SIZE = 1024; int bufLen = Encoding.UTF8.GetMaxByteCount(Math.Min(count, CHUNK_SIZE)); byte* buf = stackalloc byte[bufLen]; // We can not really predict the final size of the encoded string, but: // * Western languages have a few chars that usually need 2 bytes. If we pre-allocate 50% more bytes, it should fit most of the time, without too much waste // * Eastern langauges will have all chars encoded to 3 bytes. If we also pre-allocated 50% more, we should only need one resize of the buffer (150% x 2 = 300%), which is acceptable writer.EnsureBytes(checked(2 + count + (count >> 1))); // preallocate 150% of the string + 2 bytes writer.UnsafeWriteByte(FdbTupleTypes.Utf8); var encoder = Encoding.UTF8.GetEncoder(); // note: encoder.Convert() tries to fill up the buffer as much as possible with complete chars, and will set 'done' to true when all chars have been converted. do { encoder.Convert(ptr, remaining, buf, bufLen, true, out charsUsed, out bytesUsed, out done); if (bytesUsed > 0) { writer.WriteBytes(buf, bytesUsed); } remaining -= charsUsed; ptr += charsUsed; } while (!done); Contract.Assert(remaining == 0 && ptr == end); // close the string writer.WriteByte(0x00); #endregion return true; }
public virtual void EncodeKey <T1, T2, T3, T4, T5>(ref SliceWriter writer, T1 item1, T2 item2, T3 item3, T4 item4, T5 item5) { PackKey(ref writer, STuple.Create(item1, item2, item3, item4, item5)); }
/// <summary>Writes a null value at the end, and advance the cursor</summary> public static void WriteNil(ref SliceWriter writer) { writer.WriteByte(FdbTupleTypes.Nil); }
internal static Slice Pack([NotNull] CompressedWord[] words, int size, int highest) { Contract.Requires(size >= 0 && size <= words.Length); if (size == 0) { // empty bitmap return Slice.Empty; } var writer = new SliceWriter(checked((size + 1) << 2)); writer.WriteFixed32(CompressedWord.MakeHeader(highest)); for (int i = 0; i < size; i++) { writer.WriteFixed32(words[i].RawValue); } return writer.ToSlice(); }
/// <summary>Writes a char encoded in UTF-8</summary> public static void WriteChar(ref SliceWriter writer, char value) { if (value == 0) { // NUL => "00 0F" // note: \0 is the only unicode character that will produce a zero byte when converted in UTF-8 writer.WriteByte4(FdbTupleTypes.Utf8, 0x00, 0xFF, 0x00); } else if (value < 0x80) { // 0x00..0x7F => 0xxxxxxx writer.WriteByte3(FdbTupleTypes.Utf8, (byte)value, 0x00); } else if (value < 0x800) { // 0x80..0x7FF => 110xxxxx 10xxxxxx => two bytes writer.WriteByte4(FdbTupleTypes.Utf8, (byte)(0xC0 | (value >> 6)), (byte)(0x80 | (value & 0x3F)), 0x00); } else { // 0x800..0xFFFF => 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx // note: System.Char is 16 bits, and thus cannot represent UNICODE chars above 0xFFFF. // => This means that a System.Char will never take more than 3 bytes in UTF-8 ! var tmp = Encoding.UTF8.GetBytes(new string(value, 1)); writer.EnsureBytes(tmp.Length + 2); writer.UnsafeWriteByte(FdbTupleTypes.Utf8); writer.UnsafeWriteBytes(tmp, 0, tmp.Length); writer.UnsafeWriteByte(0x00); } }
public static void Encode <T1, T2>(ref SliceWriter writer, [NotNull] IOrderedTypeCodec <T1> codec1, T1 value1, [NotNull] IOrderedTypeCodec <T2> codec2, T2 value2) { Contract.Assert(codec1 != null && codec2 != null); codec1.EncodeOrderedSelfTerm(ref writer, value1); codec2.EncodeOrderedSelfTerm(ref writer, value2); }
private void InitializeDirectory([NotNull] IFdbTransaction trans) { Contract.Requires(trans != null); // Set the version key var writer = new SliceWriter(3 * 4); writer.WriteFixed32((uint)LayerVersion.Major); writer.WriteFixed32((uint)LayerVersion.Minor); writer.WriteFixed32((uint)LayerVersion.Build); trans.Set(this.RootNode.Keys.Encode(VersionKey), writer.ToSlice()); }