internal static ArraySegment <byte> UnescapeByteStringSlow([NotNull] byte[] buffer, int offset, int count, int offsetOfFirstZero = 0)
{
Contract.Requires(buffer != null && offset >= 0 && count >= 0);
var tmp = new byte[count];
int p = offset;
int end = offset + count;
int i = 0;
if (offsetOfFirstZero > 0)
{
SliceHelpers.CopyBytesUnsafe(tmp, 0, buffer, offset, offsetOfFirstZero);
p += offsetOfFirstZero;
i = offsetOfFirstZero;
}
while (p < end)
{
byte b = buffer[p++];
if (b == 0)
{ // skip next FF
//TODO: check that next byte really is 0xFF
++p;
}
tmp[i++] = b;
}
return(new ArraySegment <byte>(tmp, 0, i));
}
private void SetTagContent()
{
for (var i = 0; i < Document.Children.Count; i++)
{
ProcessTag(Document.Children[i]);
}
void ProcessTag(MustacheToken tag)
{
switch (tag)
{
case InlineToken inline:
inline.Content = new StringSlice(
content.Text,
inline.ContentStartPosition,
inline.ContentEndPosition - 1);
break;
case LiteralToken literal:
literal.Content = SliceHelpers.SplitSliceToLines(new StringSlice(
content.Text,
literal.ContentStartPosition,
literal.ContentEndPosition - 1)).ToArray();
break;
case BlockToken blockTag:
foreach (var child in blockTag.Children)
{
ProcessTag(child);
}
break;
}
}
}
public void WorksEvenWhenSliceSizeAndNumberOfSlicesDoNotEvenlyDivideData()
{
var originalData = new []
{
(byte)'C',
(byte)'a',
(byte)'t'
};
Slice
slice1 = SliceHelpers.CreateSlice(
coefficients: new [] { true, false },
data: new [] { originalData[0], originalData[1] }
),
slice2 = SliceHelpers.CreateSlice(
coefficients: new [] { true, true },
data: new [] { (byte)(originalData[0] ^ originalData[2]), originalData[1] }
);
var solver = new SliceSolver(2, 3);
solver.Remember(slice2);
solver.Remember(slice1);
var solved = solver.TrySolve(out var solution);
Assert.True(solved);
Assert.NotNull(solution);
Assert.Equal(solution.Length, 3);
Assert.True(solution.SequenceEqual(originalData));
}
/// <summary>Ceate a new slice buffer with the specified page size</summary>
/// <param name="pageSize">Initial page size</param>
public SliceBuffer(int pageSize)
{
if (pageSize < 0)
{
throw new ArgumentOutOfRangeException("pageSize", "Page size cannt be less than zero");
}
m_pageSize = pageSize == 0 ? DefaultPageSize : SliceHelpers.Align(pageSize);
}
public void ClonesASingleSlice()
{
var slice = SliceHelpers.CreateSlice(coefficients: new bool[5], data: new byte[5]);
var sequence = new [] { slice };
var mixed = sequence.Mix();
Assert.Equal(5, mixed.GetCoefficients().Count());
Assert.Equal(5, mixed.GetData().Count());
}
/// <summary>Copy a slice into the buffer, with optional alignement, and return a new identical slice.</summary>
/// <param name="data">Data to copy in the buffer</param>
/// <param name="aligned">If true, align the index of first byte of the slice with a multiple of 8 bytes</param>
/// <returns>Slice that is the equivalent of <paramref name="data"/>, backed by the buffer.</returns>
public Slice Intern(Slice data, bool aligned = false)
{
if (data.Count == 0)
{
// transform into the corresponding Slice.Nil / Slice.Empty singleton
return data.Memoize();
}
SliceHelpers.EnsureSliceIsValid(ref data);
// allocate the slice
var slice = Allocate(data.Count, aligned);
SliceHelpers.CopyBytesUnsafe(slice.Array, slice.Offset, data.Array, data.Offset, data.Count);
return slice;
}
public void Test_SliceHelpers_NextPowerOfTwo()
{
// 0 is a special case, to simplify bugger handling logic
Assert.That(SliceHelpers.NextPowerOfTwo(0), Is.EqualTo(1), "Special case for 0");
Assert.That(SliceHelpers.NextPowerOfTwo(1), Is.EqualTo(1));
Assert.That(SliceHelpers.NextPowerOfTwo(2), Is.EqualTo(2));
for (int i = 2; i < 31; i++)
{
Assert.That(SliceHelpers.NextPowerOfTwo((1 << i) - 1), Is.EqualTo(1 << i));
Assert.That(SliceHelpers.NextPowerOfTwo(1 << i), Is.EqualTo(1 << i));
}
Assert.That(() => SliceHelpers.NextPowerOfTwo(-1), Throws.InstanceOf <ArgumentOutOfRangeException>());
Assert.That(() => SliceHelpers.NextPowerOfTwo(-42), Throws.InstanceOf <ArgumentOutOfRangeException>());
}
/// <summary>Copy a slice into the buffer, immediately followed by a suffix, and return a new slice that is the concatenation of the two.</summary>
/// <param name="data">Data to copy in the buffer</param>
/// <param name="suffix">Suffix to copy immediately after <paramref name="data"/>.</param>
/// <param name="aligned">If true, align the index of first byte of the slice with a multiple of 8 bytes</param>
/// <returns>Slice that is the equivalent of <paramref name="data"/> plus <paramref name="suffix"/>, backed by the buffer.</returns>
/// <remarks>When <paramref name="data"/> is empty, <paramref name="suffix"/> is returned without being copied to the buffer itself.</remarks>
internal Slice Intern(Slice data, Slice suffix, bool aligned = false)
{
if (data.Count == 0)
{
// note: we don't memoize the suffix, because in most case, it comes from a constant, and it would be a waste to copy it other and other again...
return suffix.Count > 0 ? suffix : data.Array == null ? Slice.Nil : Slice.Empty;
}
SliceHelpers.EnsureSliceIsValid(ref data);
SliceHelpers.EnsureSliceIsValid(ref suffix);
var slice = Allocate(data.Count + suffix.Count, aligned);
SliceHelpers.CopyBytesUnsafe(slice.Array, slice.Offset, data.Array, data.Offset, data.Count);
SliceHelpers.CopyBytesUnsafe(slice.Array, slice.Offset + data.Count, suffix.Array, suffix.Offset, suffix.Count);
return slice;
}
public void EnsureCapacity(int minSize)
{
if (minSize > m_size)
{
int newSize = SliceHelpers.NextPowerOfTwo(minSize);
if (newSize < 0)
{
newSize = minSize;
}
if (newSize < 8)
{
newSize = 8;
}
Array.Resize(ref m_words, newSize);
}
}
public Task WritePixel(WritePixelCommand command)
{
var index = SliceHelpers.GetIndex(command.X, command.Y);
var relativeIndex = index - Offset;
var pixel = _slice.Pixels[relativeIndex];
//only replace pixel if new command is newer.
if (command.Timestamp >= pixel.LastTimestamp || !pixel.LastTimestamp.HasValue)
{
pixel.Color = command.Color;
pixel.LastAuthor = command.Author;
pixel.LastTimestamp = command.Timestamp;
}
return(TaskDone.Done);
}
public void PutsOneByteBackTogether()
{
const byte value = 0xF5;
var slice = SliceHelpers.CreateSlice(
coefficients: new [] { true },
data: new [] { value }
);
var solver = new SliceSolver(1, 1);
solver.Remember(slice);
var solved = solver.TrySolve(out var solution);
Assert.True(solved);
Assert.NotNull(solution);
Assert.Equal(solution.Length, 1);
Assert.Equal(solution[0], value);
}
public void Test_SliceHelpers_ComputeHashCode()
{
//note: if everything fails, check that the hashcode algorithm hasn't changed also !
Assert.That(SliceHelpers.ComputeHashCode(new byte[0], 0, 0), Is.EqualTo(-2128831035));
Assert.That(SliceHelpers.ComputeHashCode(new byte[1], 0, 1), Is.EqualTo(84696351));
Assert.That(SliceHelpers.ComputeHashCode(new byte[2], 0, 1), Is.EqualTo(84696351));
Assert.That(SliceHelpers.ComputeHashCode(new byte[2], 1, 1), Is.EqualTo(84696351));
Assert.That(SliceHelpers.ComputeHashCode(new byte[2], 0, 2), Is.EqualTo(292984781));
Assert.That(SliceHelpers.ComputeHashCode(Encoding.Default.GetBytes("hello"), 0, 5), Is.EqualTo(1335831723));
Assert.That(SliceHelpers.ComputeHashCodeUnsafe(new byte[0], 0, 0), Is.EqualTo(-2128831035));
Assert.That(SliceHelpers.ComputeHashCodeUnsafe(new byte[1], 0, 1), Is.EqualTo(84696351));
Assert.That(SliceHelpers.ComputeHashCodeUnsafe(new byte[2], 0, 1), Is.EqualTo(84696351));
Assert.That(SliceHelpers.ComputeHashCodeUnsafe(new byte[2], 1, 1), Is.EqualTo(84696351));
Assert.That(SliceHelpers.ComputeHashCodeUnsafe(new byte[2], 0, 2), Is.EqualTo(292984781));
Assert.That(SliceHelpers.ComputeHashCodeUnsafe(Encoding.Default.GetBytes("hello"), 0, 5), Is.EqualTo(1335831723));
}
/// <summary>Writes a buffer with all instances of 0 escaped as '00 FF'</summary>
internal static void WriteNulEscapedBytes(ref TupleWriter writer, byte type, [NotNull] byte[] value, int offset, int count)
{
int n = count;
// we need to know if there are any NUL chars (\0) that need escaping...
// (we will also need to add 1 byte to the buffer size per NUL)
for (int i = offset, end = offset + count; i < end; ++i)
{
if (value[i] == 0)
{
++n;
}
}
writer.Output.EnsureBytes(n + 2);
var buffer = writer.Output.Buffer;
int p = writer.Output.Position;
buffer[p++] = type;
if (n > 0)
{
if (n == count)
{ // no NULs in the string, can copy all at once
SliceHelpers.CopyBytesUnsafe(buffer, p, value, offset, n);
p += n;
}
else
{ // we need to escape all NULs
for (int i = offset, end = offset + count; i < end; ++i)
{
byte b = value[i];
buffer[p++] = b;
if (b == 0)
{
buffer[p++] = 0xFF;
}
}
}
}
buffer[p] = 0x00;
writer.Output.Position = p + 1;
}
/// <summary>Writes a buffer with all instances of 0 escaped as '00 FF'</summary>
private static void WriteNulEscapedBytes(ref TupleWriter writer, byte type, [NotNull] byte[] value)
{
int n = value.Length;
// we need to know if there are any NUL chars (\0) that need escaping...
// (we will also need to add 1 byte to the buffer size per NUL)
foreach (byte b in value)
{
if (b == 0)
{
++n;
}
}
writer.Output.EnsureBytes(n + 2);
var buffer = writer.Output.Buffer;
int p = writer.Output.Position;
buffer[p++] = type;
if (n > 0)
{
if (n == value.Length)
{ // no NULs in the string, can copy all at once
SliceHelpers.CopyBytesUnsafe(buffer, p, value, 0, n);
p += n;
}
else
{ // we need to escape all NULs
foreach (byte b in value)
{
buffer[p++] = b;
if (b == 0)
{
buffer[p++] = 0xFF;
}
}
}
}
buffer[p++] = 0x00;
writer.Output.Position = p;
}
public void EnsureCapacity(int minSize)
{
if (minSize > m_size)
{
int newSize = SliceHelpers.NextPowerOfTwo(minSize);
if (newSize < 0)
{
newSize = minSize;
}
if (newSize < 8)
{
newSize = 8;
}
//Console.WriteLine("> resize buffer to {0} words", newSize);
Array.Resize(ref m_words, newSize);
}
//else
//{
// Console.WriteLine("> buffer has enough capacity {0} for min size {1}", m_words.Length, minSize);
//}
}
internal static CompressedWord[] DecodeWords(Slice data, int size, BitRange bounds)
{
Contract.Requires(size >= 0 && data.Count >= 4 && (data.Count & 3) == 0);
int capacity = SliceHelpers.NextPowerOfTwo(size);
if (capacity < 0)
{
capacity = size;
}
var words = new CompressedWord[capacity];
var end = data.Offset + data.Count;
for (int i = 0, p = data.Offset + 4; p < end; i++, p += 4)
{
words[i] = new CompressedWord(data.ReadUInt32(p, 4));
}
return(words);
}
public void PutsTwoSequentialSlicesBackTogether()
{
const byte
value1 = 0xFF,
value2 = 0x55;
Slice
slice1 = SliceHelpers.CreateSlice(
coefficients: new [] { true, false },
data: new [] { value1 }),
slice2 = SliceHelpers.CreateSlice(
coefficients: new [] { false, true },
data: new [] { value2 });
var solver = new SliceSolver(1, 2);
solver.Remember(slice1);
solver.Remember(slice2);
var solved = solver.TrySolve(out var solution);
Assert.True(solved);
Assert.NotNull(solution);
Assert.Equal(solution.Length, 2);
Assert.Equal(solution[0], value1);
Assert.Equal(solution[1], value2);
}
public void Test_SliceHelpers_Align()
{
// Even though 0 is a multiple of 16, it is always rounded up to 16 to simplify buffer handling logic
Assert.That(SliceHelpers.Align(0), Is.EqualTo(16));
// 1..16 => 16
for (int i = 1; i <= 16; i++)
{
Assert.That(SliceHelpers.Align(i), Is.EqualTo(16), "Align({0}) => 16", i);
}
// 17..32 => 32
for (int i = 17; i <= 32; i++)
{
Assert.That(SliceHelpers.Align(i), Is.EqualTo(32), "Align({0}) => 32", i);
}
// 33..48 => 48
for (int i = 33; i <= 48; i++)
{
Assert.That(SliceHelpers.Align(i), Is.EqualTo(48), "Align({0}) => 48", i);
}
// 2^N-1
for (int i = 6; i < 30; i++)
{
Assert.That(SliceHelpers.Align((1 << i) - 1), Is.EqualTo(1 << i));
}
// largest non overflowing
Assert.That(() => SliceHelpers.Align(int.MaxValue - 15), Is.EqualTo((int.MaxValue - 15)));
// overflow
Assert.That(() => SliceHelpers.Align(int.MaxValue), Throws.InstanceOf <OverflowException>());
Assert.That(() => SliceHelpers.Align(int.MaxValue - 14), Throws.InstanceOf <OverflowException>());
// negative values
Assert.That(() => SliceHelpers.Align(-1), Throws.InstanceOf <ArgumentOutOfRangeException>());
Assert.That(() => SliceHelpers.Align(int.MinValue), Throws.InstanceOf <ArgumentOutOfRangeException>());
}
public void CanSolveWithACombinedSlice()
{
const byte
value1 = 0xFF,
value2 = 0x55;
Slice
slice1 = SliceHelpers.CreateSlice(
coefficients: new [] { true, false },
data: new [] { value1 }),
slice2 = SliceHelpers.CreateSlice(
coefficients: new [] { true, true },
data: new [] { (byte)(value2 ^ value1) });
var solver = new SliceSolver(1, 2);
solver.Remember(slice1);
solver.Remember(slice2);
var solved = solver.TrySolve(out var solution);
Assert.True(solved);
Assert.NotNull(solution);
Assert.Equal(solution.Length, 2);
Assert.Equal(solution[0], value1);
Assert.Equal(solution[1], value2);
}