public void PackTo(ref SliceWriter writer)
{
if (m_packed.IsPresent)
{
writer.WriteBytes(m_packed);
}
}
void IUnorderedTypeCodec <TDocument> .EncodeUnorderedSelfTerm(ref SliceWriter output, TDocument value)
{
var packed = EncodeInternal(value);
Contract.Assert(packed.Count >= 0);
output.WriteVarint32((uint)packed.Count);
output.WriteBytes(packed);
}
internal static SliceWriter OpenWriter(this IKeySubspace self, int extra = 32)
{
var key = self.GetPrefix();
var sw = new SliceWriter(key.Count + extra); //TODO: BufferPool ?
sw.WriteBytes(key);
return(sw);
}
/// <summary>Encode a value into a key, with an additional prefix, using the specified encoder</summary>
public static Slice EncodeKey <T1>(this IKeyEncoder <T1> encoder, Slice prefix, [AllowNull] T1 value)
{
var writer = new SliceWriter(prefix.Count + 16); // ~16 bytes si T1 = Guid
writer.WriteBytes(prefix);
encoder.WriteKeyTo(ref writer, value);
return(writer.ToSlice());
}
/// <summary>Encode a pair of values into a key, with an additional prefix, using the specified encoder</summary>
public static Slice EncodeKey <T1, T2>(this ICompositeKeyEncoder <T1, T2> encoder, Slice prefix, T1 item1, T2 item2)
{
var writer = new SliceWriter(prefix.Count + 24);
writer.WriteBytes(prefix);
encoder.WriteKeyTo(ref writer, item1, item2);
return(writer.ToSlice());
}
public void PackTo(ref SliceWriter writer)
{
var slices = m_slices;
for (int n = m_count, p = m_offset; n > 0; n--)
{
writer.WriteBytes(slices[p++]);
}
}
public void WriteKeyTo(ref SliceWriter writer, TKey value)
{
if (this.Pack is Func <TKey, Slice> f)
{
writer.WriteBytes(f(value));
return;
}
throw new InvalidOperationException();
}
/// <summary>Pack a tuple into a key, with an additional prefix, using the specified encoder</summary>
public static Slice Pack <TTuple>(this IDynamicKeyEncoder encoder, Slice prefix, TTuple tuple)
where TTuple : IVarTuple
{
var writer = new SliceWriter(checked (prefix.Count + tuple.Count * 8));
writer.WriteBytes(prefix);
encoder.PackKey(ref writer, tuple);
return(writer.ToSlice());
}
/// <summary>Encode only the first part of a key, with an additional prefix, using the specified encoder</summary>
public static Slice EncodePartialKey <T1, T2>(this ICompositeKeyEncoder <T1, T2> encoder, Slice prefix, T1 item1)
{
var writer = new SliceWriter(prefix.Count + 16);
writer.WriteBytes(prefix);
var tuple = (item1, default(T2));
encoder.WriteKeyPartsTo(ref writer, 1, ref tuple);
return(writer.ToSlice());
}
public void Test_Tail()
{
var writer = new SliceWriter(64);
var slice = writer.Tail(0);
Assert.That(slice.Count, Is.EqualTo(0));
Assert.That(slice.Offset, Is.EqualTo(0));
//note: slice.Array is not guaranteed to be equal to writer.Buffer
Assert.That(() => writer.Head(1), Throws.InstanceOf <ArgumentOutOfRangeException>());
writer.WriteBytes(Slice.FromString("hello world!"));
slice = writer.Tail(6);
Assert.That(slice.Array, Is.SameAs(writer.Buffer));
Assert.That(slice.Offset, Is.EqualTo(6));
Assert.That(slice.Count, Is.EqualTo(6));
Assert.That(slice.ToStringAscii(), Is.EqualTo("world!"));
slice = writer.Tail(12);
Assert.That(slice.Array, Is.SameAs(writer.Buffer));
Assert.That(slice.Offset, Is.EqualTo(0));
Assert.That(slice.Count, Is.EqualTo(12));
Assert.That(slice.ToStringAscii(), Is.EqualTo("hello world!"));
Assert.That(() => writer.Tail(13), Throws.InstanceOf <ArgumentOutOfRangeException>());
Assert.That(() => writer.Tail(-1), Throws.InstanceOf <ArgumentOutOfRangeException>());
writer.WriteBytes(Slice.FromString("foo"));
slice = writer.Tail(3);
Assert.That(slice.Array, Is.SameAs(writer.Buffer));
Assert.That(slice.Offset, Is.EqualTo(12));
Assert.That(slice.Count, Is.EqualTo(3));
Assert.That(slice.ToStringAscii(), Is.EqualTo("foo"));
slice = writer.Tail(15);
Assert.That(slice.Array, Is.SameAs(writer.Buffer));
Assert.That(slice.Offset, Is.EqualTo(0));
Assert.That(slice.Count, Is.EqualTo(15));
Assert.That(slice.ToStringAscii(), Is.EqualTo("hello world!foo"));
Assert.That(() => writer.Tail(16), Throws.InstanceOf <ArgumentOutOfRangeException>());
}
public void Test_Skip()
{
var writer = new SliceWriter();
writer.WriteBytes(Slice.FromString("hello"));
Assert.That(writer.Position, Is.EqualTo(5));
Assert.That(writer.ToSlice().ToString(), Is.EqualTo("hello"));
// default pad is 255
Assert.That(writer.Skip(3), Is.EqualTo(5));
Assert.That(writer.Position, Is.EqualTo(8));
Assert.That(writer.ToSlice().ToString(), Is.EqualTo("hello<FF><FF><FF>"));
writer.WriteBytes(Slice.FromString("world"));
Assert.That(writer.Position, Is.EqualTo(13));
Assert.That(writer.ToSlice().ToString(), Is.EqualTo("hello<FF><FF><FF>world"));
// custom pad
Assert.That(writer.Skip(5, 42), Is.EqualTo(13));
Assert.That(writer.Position, Is.EqualTo(18));
Assert.That(writer.ToSlice().ToString(), Is.EqualTo("hello<FF><FF><FF>world*****"));
}
public void Test_WriteBytes_Resize_Buffer()
{
// check buffer resize occurs as intended
var original = new byte[32];
var writer = new SliceWriter(original);
Assert.That(writer.Buffer, Is.SameAs(original));
// first write should not resize the buffer
writer.WriteBytes(Slice.Repeat((byte)'a', 24));
Assert.That(writer.Buffer, Is.SameAs(original));
Assert.That(writer.ToSlice().ToStringAscii(), Is.EqualTo("aaaaaaaaaaaaaaaaaaaaaaaa"));
// second write should resize the buffer
writer.WriteBytes(Slice.Repeat((byte)'b', 24));
// buffer should have been replaced with larger one
Assert.That(writer.Buffer, Is.Not.SameAs(original));
Assert.That(writer.Buffer.Length, Is.GreaterThanOrEqualTo(48));
//but the content should be unchanged
Assert.That(writer.ToSlice().ToStringAscii(), Is.EqualTo("aaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbb"));
// adding exactly what is missing should not resize the buffer
writer = new SliceWriter(original);
writer.WriteBytes(Slice.Repeat((byte)'c', original.Length));
Assert.That(writer.Buffer, Is.SameAs(original));
Assert.That(writer.ToSlice().ToStringAscii(), Is.EqualTo("cccccccccccccccccccccccccccccccc"));
// adding nothing should not resize the buffer
writer.WriteBytes(Slice.Empty);
Assert.That(writer.Buffer, Is.SameAs(original));
Assert.That(writer.ToSlice().ToStringAscii(), Is.EqualTo("cccccccccccccccccccccccccccccccc"));
// adding a single byte should resize the buffer
writer.WriteBytes(Slice.FromChar('Z'));
Assert.That(writer.Buffer, Is.Not.SameAs(original));
Assert.That(writer.Buffer.Length, Is.GreaterThanOrEqualTo(33));
Assert.That(writer.ToSlice().ToStringAscii(), Is.EqualTo("ccccccccccccccccccccccccccccccccZ"));
}
public void Test_ToSlice()
{
var writer = new SliceWriter(64);
var slice = writer.ToSlice();
//note: slice.Array is not guaranteed to be equal to writer.Buffer
Assert.That(slice.Count, Is.EqualTo(0));
Assert.That(slice.Offset, Is.EqualTo(0));
writer.WriteBytes(Slice.FromString("hello world!"));
slice = writer.ToSlice();
Assert.That(slice.Array, Is.SameAs(writer.Buffer));
Assert.That(slice.Offset, Is.EqualTo(0));
Assert.That(slice.Count, Is.EqualTo(12));
Assert.That(slice.ToStringAscii(), Is.EqualTo("hello world!"));
writer.WriteBytes(Slice.FromString("foo"));
slice = writer.ToSlice();
Assert.That(slice.Array, Is.SameAs(writer.Buffer));
Assert.That(slice.Offset, Is.EqualTo(0));
Assert.That(slice.Count, Is.EqualTo(15));
Assert.That(slice.ToStringAscii(), Is.EqualTo("hello world!foo"));
}
public void Test_Flush()
{
var writer = new SliceWriter();
writer.WriteBytes(Slice.FromString("hello world!"));
Assert.That(writer.Position, Is.EqualTo(12));
writer.Flush(5);
Assert.That(writer.Position, Is.EqualTo(7));
Assert.That(writer.ToSlice().ToString(), Is.EqualTo(" world!"));
writer.Flush(1);
Assert.That(writer.Position, Is.EqualTo(6));
Assert.That(writer.ToSlice().ToString(), Is.EqualTo("world!"));
writer.Flush(0);
Assert.That(writer.Position, Is.EqualTo(6));
Assert.That(writer.ToSlice().ToString(), Is.EqualTo("world!"));
// REVIEW: should we throw if we flush more bytes than in the writer? (currently, it just clears it)
writer.Flush(7);
Assert.That(writer.Position, Is.EqualTo(0));
Assert.That(writer.ToSlice(), Is.EqualTo(Slice.Empty));
}
/// <summary>Write the header to the file</summary>
/// <param name="headerFlags"></param>
/// <param name="uid"></param>
/// <param name="sequence"></param>
/// <param name="count"></param>
/// <param name="timestamp"></param>
/// <param name="attributes"></param>
/// <remarks>This needs to be called before writing any level to the file</remarks>
public Task WriteHeaderAsync(SnapshotFormat.Flags headerFlags, Uuid128 uid, ulong sequence, long count, long timestamp, IDictionary <string, IFdbTuple> attributes)
{
// The header will be use on ore more "pages", to simplify the job of loading / peeking at a stream content (no need for fancy buffering, just need to read 4K pages)
// > The last page is padded with 0xAAs to detect corruption.
m_uid = uid;
m_sequence = sequence;
m_itemCount = count;
m_timestamp = timestamp;
// HEADER
// - DB_HEADER (64 bytes)
// - DB ATTRIBUTES (variable size list of k/v)
// - END_MARKER + HEADER_CRC
// - PADDING (to fill last page)
// DB Header
// "PNDB"
m_writer.WriteFixed32(SnapshotFormat.HEADER_MAGIC_NUMBER);
// v1.0
m_writer.WriteFixed16(1); // major
m_writer.WriteFixed16(0); // minor
// FLAGS
m_writer.WriteFixed64((ulong)headerFlags);
// Database ID
m_writer.WriteBytes(uid.ToSlice());
// Database Version
m_writer.WriteFixed64(sequence);
// Number of items in the database
m_writer.WriteFixed64((ulong)count);
// Database Timestamp
m_writer.WriteFixed64((ulong)timestamp);
// Page Size
m_writer.WriteFixed32(SnapshotFormat.PAGE_SIZE);
// Header Size (not known yet and will be filled in later)
int offsetToHeaderSize = m_writer.Skip(4);
// we should be at the 64 byte mark
Contract.Assert(m_writer.Position == SnapshotFormat.HEADER_METADATA_BYTES);
// DB Attributes
m_writer.WriteFixed32((uint)attributes.Count);
foreach (var kvp in attributes)
{
// Name
m_writer.WriteVarbytes(Slice.FromString(kvp.Key));
// Value
m_writer.WriteVarbytes(kvp.Value.ToSlice());
}
// Mark the end of the header
m_writer.WriteFixed32(uint.MaxValue);
// we now have the size of the header, and can fill in the blank
var headerEnd = m_writer.Position;
m_writer.Position = offsetToHeaderSize;
// write the header size (includes the CRC)
m_writer.WriteFixed32((uint)checked (headerEnd + SnapshotFormat.HEADER_CRC_SIZE));
m_writer.Position = headerEnd;
// now we can compute the actual CRC
uint headerChecksum = SnapshotFormat.ComputeChecksum(m_writer.ToSlice());
m_writer.WriteFixed32(headerChecksum);
m_headerChecksum = headerChecksum;
// optional padding to fill the rest of the page
PadPageIfNeeded(SnapshotFormat.PAGE_SIZE, 0xFD);
return(TaskHelpers.CompletedTask);
}
public void WriteKeyTo(ref SliceWriter writer, T key)
{
//TODO: PERF: optimize this!
writer.WriteBytes(m_codec.EncodeOrdered(key));
}
static void Test(ref SliceWriter writer, byte[] value) => writer.WriteBytes(value);
public void WriteKeyTo(ref SliceWriter writer, T value)
{
writer.WriteBytes(m_encoder(value));
}
public void WriteKeyTo(ref SliceWriter writer, Slice key)
{
writer.WriteBytes(key);
}
private static unsafe bool TryWriteUnescapedUtf8String(ref SliceWriter writer, char *chars, int count)
{
Contract.Requires(chars != null && count >= 0);
// Several observations:
// * Most strings will be keywords or ASCII-only with no zeroes. These can be copied directly to the buffer
// * We will only attempt to optimze strings that don't have any 00 to escape to 00 FF. For these, we will fallback to converting to byte[] then escaping.
// * Since .NET's strings are UTF-16, the max possible UNICODE value to encode is 0xFFFF, which takes 3 bytes in UTF-8 (EF BF BF)
// * Most western europe languages have only a few non-ASCII chars here and there, and most of them will only use 2 bytes (ex: 'é' => 'C3 A9')
// * More complex scripts with dedicated symbol pages (kanjis, arabic, ....) will take 2 or 3 bytes for each charecter.
// We will first do a pass to check for the presence of 00 and non-ASCII chars
// => if we find at least on 00, we fallback to escaping the result of Encoding.UTF8.GetBytes()
// => if we find only ASCII (1..127) chars, we have an optimized path that will truncate the chars to bytes
// => if not, we will use an UTF8Encoder to convert the string to UTF-8, in chunks, using a small buffer allocated on the stack
#region First pass: look for \0 and non-ASCII chars
// fastest way to check for non-ASCII, is to OR all the chars together, and look at bits 7 to 15. If they are not all zero, there is at least ONE non-ASCII char.
// also, we abort as soon as we find a \0
char *ptr = chars;
char *end = chars + count;
char mask = '\0', c;
while (ptr < end && (c = *ptr) != '\0')
{
mask |= c; ++ptr;
}
if (ptr < end)
{
return(false); // there is at least one \0 in the string
}
// bit 7-15 all unset means the string is pure ASCII
if ((mask >> 7) == 0)
{ // => directly dump the chars to the buffer
WriteUnescapedAsciiChars(ref writer, chars, count);
return(true);
}
#endregion
#region Second pass: encode the string to UTF-8, in chunks
// Here we know that there is at least one unicode char, and that there are no \0
// We will tterate through the string, filling as much of the buffer as possible
bool done;
int remaining = count;
ptr = chars;
// We need at most 3 * CHUNK_SIZE to encode the chunk
// > For small strings, we will allocated exactly string.Length * 3 bytes, and will be done in one chunk
// > For larger strings, we will call encoder.Convert(...) until it says it is done.
const int CHUNK_SIZE = 1024;
int bufLen = Encoding.UTF8.GetMaxByteCount(Math.Min(count, CHUNK_SIZE));
byte * buf = stackalloc byte[bufLen];
// We can not really predict the final size of the encoded string, but:
// * Western languages have a few chars that usually need 2 bytes. If we pre-allocate 50% more bytes, it should fit most of the time, without too much waste
// * Eastern langauges will have all chars encoded to 3 bytes. If we also pre-allocated 50% more, we should only need one resize of the buffer (150% x 2 = 300%), which is acceptable
writer.EnsureBytes(checked (2 + count + (count >> 1))); // preallocate 150% of the string + 2 bytes
writer.UnsafeWriteByte(FdbTupleTypes.Utf8);
var encoder = Encoding.UTF8.GetEncoder();
// note: encoder.Convert() tries to fill up the buffer as much as possible with complete chars, and will set 'done' to true when all chars have been converted.
do
{
int charsUsed, bytesUsed;
encoder.Convert(ptr, remaining, buf, bufLen, true, out charsUsed, out bytesUsed, out done);
if (bytesUsed > 0)
{
writer.WriteBytes(buf, bytesUsed);
}
remaining -= charsUsed;
ptr += charsUsed;
}while (!done);
Contract.Assert(remaining == 0 && ptr == end);
// close the string
writer.WriteByte(0x00);
#endregion
return(true);
}
public void PackTo(ref SliceWriter writer)
{
writer.WriteBytes(m_prefix);
m_items.PackTo(ref writer);
}