// FloatArrayEncodeAll encodes src into b, returning b and any error encountered.
// The returned slice may be of a different length and capactity to b.
//
// Currently only the float compression scheme used in Facebook's Gorilla is
// supported, so this method implements a batch oriented version of that.
public (ByteWriter, string) EncodingAll(Span <double> src)
{
int length = src.Length;
//9=Enough room for the header and one value.
ByteWriter result = new ByteWriter(length * 10 + 9);
byte[] bytes = result.EndWrite();
Span <byte> b = new Span <byte>(bytes);
b.Fill(0);
b[0] = (floatCompressedGorilla << 4);
double first;
bool finished = false;
if (length > 0 && Double.IsNaN(src[0]))
{
result.Release();
return(null, "unsupported value: NaN");
}
else if (length == 0)
{
first = Double.NaN;// Write sentinal value to terminate batch.
finished = true;
}
else
{
first = src[0];
src = src.Slice(1);
}
int n = 8 + 64;//Number of bits written.
ulong prev = Float64bits(first);
// Write first value.
ByteWriter.WriteBigEndian(bytes, 1, prev);
int prevLeading = -1, prevTrailing = 0;
int leading, trailing;
ulong mask;
double sum = 0;
// Encode remaining values.
for (int i = 0; !finished; i++)
{
double x;
if (i < src.Length)
{
x = src[i];
sum += x;
}
else
{
// Encode sentinal value to terminate batch
x = double.NaN;
finished = true;
}
ulong cur = Float64bits(x);
ulong vDelta = cur ^ prev;
if (vDelta == 0)
{
n++;// Write a zero bit. Nothing else to do.
prev = cur;
continue;
}
// n&7 - current bit in current byte.
// n>>3 - the current byte.
b[n >> 3] |= (byte)(128 >> (n & 7));// Sets the current bit of the current byte.
n++;
// Write the delta to b.
// Determine the leading and trailing zeros.
leading = Encoding.Clz(vDelta);
trailing = Encoding.Ctz(vDelta);
// Clamp number of leading zeros to avoid overflow when encoding
leading &= 0x1F;
if (leading >= 32)
{
leading = 31;
}
if (prevLeading != -1 && leading >= prevLeading && trailing >= prevTrailing)
{
n++;// Write a zero bit.
// Write the l least significant bits of vDelta to b, most significant
// bit first.
int L = 64 - prevLeading - prevTrailing;
// Full value to write.
ulong v = (vDelta >> prevTrailing) << (64 - L); // l least signifciant bits of v.
int m = n & 7; // Current bit in current byte.
int written = 0;
if (m > 0) // In this case the current byte is not full.
{
written = 8 - m;
if (L < written)
{
written = L;
}
mask = v >> 56;// Move 8 MSB to 8 LSB
b[n >> 3] |= (byte)(mask >> m);
n += written;
if (L == written)
{
prev = cur;
continue;
}
}
var vv = v << written;// Move written bits out of the way.
ByteWriter.WriteBigEndian(bytes, n >> 3, vv);
n += (L - written);
}
else
{
prevLeading = leading;
prevTrailing = trailing;
// Set a single bit to indicate a value will follow.
b[n >> 3] |= (byte)(128 >> (n & 7));// Set current bit on current byte
n++;
int m = n & 7;
int L = 5;
ulong v = (ulong)leading << 59; // 5 LSB of leading.
mask = v >> 56; // Move 5 MSB to 8 LSB
if (m <= 3) // 5 bits fit into current byte.
{
b[n >> 3] |= (byte)(mask >> m);
n = n + L;
}
else// In this case there are fewer than 5 bits available in current byte.
{
// First step is to fill current byte
int written = 8 - m;
b[n >> 3] |= (byte)(mask >> m);// Some of mask will get lost.
n += written;
// Second step is to write the lost part of mask into the next byte.
mask = v << written; // Move written bits in previous byte out of way.
mask >>= 56;
m = n & 7; //Recompute current bit.
b[n >> 3] |= (byte)(mask >> m);
n += (L - written);
}
// Note that if leading == trailing == 0, then sigbits == 64. But that
// value doesn't actually fit into the 6 bits we have.
// Luckily, we never need to encode 0 significant bits, since that would
// put us in the other case (vdelta == 0). So instead we write out a 0 and
// adjust it back to 64 on unpacking.
int sigbits = 64 - leading - trailing;
m = n & 7;
L = 6;
v = (ulong)sigbits << 58; // Move 6 LSB of sigbits to MSB
mask = v >> 56; // Move 6 MSB to 8 LSB
if (m <= 2)
{
// The 6 bits fit into the current byte.
b[n >> 3] |= (byte)(mask >> m);
n += L;
}
else// In this case there are fewer than 6 bits available in current byte.
{
// First step is to fill the current byte.
int written = 8 - m;
b[n >> 3] |= (byte)(mask >> m);// Write to the current bit.
n += written;
// Second step is to write the lost part of mask into the next byte.
// Write l remaining bits into current byte.
mask = v << written; // Remove bits written in previous byte out of way.
mask >>= 56;
m = n & 7; // Recompute current bit.
b[n >> 3] |= (byte)(mask >> m);
n += L - written;
}
// Write final value.
m = n & 7;
L = sigbits;
v = (vDelta >> trailing) << (64 - L); // Move l LSB into MSB
int written2 = 0;
if (m > 0) // In this case the current byte is not full.
{
written2 = 8 - m;
if (L < written2)
{
written2 = L;
}
mask = v >> 56;//Move 8 MSB to 8 LSB
b[n >> 3] |= (byte)(mask >> m);
n += written2;
if (L - written2 == 0)
{
prev = cur;
continue;
}
}
// Shift remaining bits and write out in one go.
ulong vv = v << written2;// Remove bits written in previous byte.
ByteWriter.WriteBigEndian(bytes, n >> 3, vv);
n += (L - written2);
}
prev = cur;
}
if (Double.IsNaN(sum))
{
result.Release();
return(null, "unsupported value: NaN");
}
int blength = n >> 3;
if ((n & 7) > 0)
{
blength++; // Add an extra byte to capture overflowing bits.
}
result.Length = blength;
return(result, null);
}