public void WriteIndex()
{
lock (lockthis)
{
if (index.SidCount == 0)
{
throw new Exception(Constants.ErrNoValues);
}
int size = 8 + (int)index.Size;//8 for n
ByteWriter writer = new ByteWriter(size);
index.WriteTo(writer);
writer.Write(blocksize);
wrapped.Write(writer.EndWrite(), 0, writer.Length);
writer.Release();
}
}
private (ByteWriter, string error) encodePacked()
{
if (len == 0)
{
return(null, null);
}
// Encode all but the first value. Fist value is written unencoded
// using 8 bytes.
ByteWriter b = new ByteWriter(1 + 8 * len);
// 4 high bits of first byte store the encoding type for the block
b.Write((byte)(intCompressedSimple << 4));
// Write the first value since it's not part of the encoded values
b.Write(values[0]);
var error = Simple8bEncoder.EncodeAll(values, 1, len, b);
if (error != null)
{
b.Release();
return(null, error);
}
return(b, null);
}
public (ByteWriter, string) EncodingAll(Span <ulong> src)
{
int srclen = src.Length;
if (srclen == 0)
{
return(null, null); // Nothing to do
}
ulong max = 0, div = (ulong)1e12;
ulong[] deltaarray = ArrayPool <ulong> .Shared.Rent(srclen);
Span <ulong> deltas = new Span <ulong>(deltaarray);
deltas[0] = src[0];
ByteWriter result;
if (srclen > 1)
{
for (int i = srclen - 1; i > 0; i--)
{
deltas[i] = src[i] - src[i - 1];
if (deltas[i] > max)
{
max = deltas[i];
}
}
bool rle = true;
var delta1 = deltas[1];
for (int i = 2; i < srclen; i++)
{
if (delta1 != deltas[i])
{
rle = false;
break;
}
}
// Deltas are the same - encode with RLE
if (rle)
{
// Large varints can take up to 10 bytes. We're storing 3 + 1
// type byte.
result = new ByteWriter(31);
// 4 high bits used for the encoding type
result.Write((byte)(timeCompressedRLE << 4));
// The first value
result.Write(deltas[0]);
// The first delta, checking the divisor
// given all deltas are the same, we can do a single check for the divisor
ulong v = deltas[1];
while (div > 1 && v % div != 0)
{
div /= 10;
}
if (div > 1)
{
// 4 low bits are the log10 divisor
result.EndWrite()[0] |= (byte)Math.Log10(div);
result.Write(Varint.GetBytes(deltas[1] / div));
}
else
{
result.Write(Varint.GetBytes(deltas[1]));
}
// The number of times the delta is repeated
result.Write(Varint.GetBytes(srclen));
ArrayPool <ulong> .Shared.Return(deltaarray);
return(result, null);
}
}
// We can't compress this time-range, the deltas exceed 1 << 60
if (max > Simple8bEncoder.MaxValue)
{
// Encode uncompressed.
int sz = 1 + srclen * 8;
result = new ByteWriter(sz);
result.Write((byte)(timeUncompressed << 4));
for (int i = 0; i < srclen; i++)
{
result.Write(deltas[i]);
}
ArrayPool <ulong> .Shared.Return(deltaarray);
return(result, null);
}
// find divisor only if we're compressing with simple8b
for (int i = 1; i < srclen && div > 1; i++)
{
// If our value is divisible by 10, break. Otherwise, try the next smallest divisor.
var v = deltaarray[i];
while (div > 1 && v % div != 0)
{
div /= 10;
}
}
// Only apply the divisor if it's greater than 1 since division is expensive.
if (div > 1)
{
for (int i = 1; i < srclen; i++)
{
deltas[i] /= div;
}
}
result = new ByteWriter(1 + srclen * 8);
result.Write((byte)((timeCompressedPackedSimple << 4) | (byte)Math.Log10(div)));
// Write the first value since it's not part of the encoded values
result.Write(deltas[0]);
// Encode with simple8b - fist value is written unencoded using 8 bytes.
var error = Simple8bEncoder.EncodeAll(deltaarray, 1, srclen, result);
if (error != null)
{
result.Release();
return(null, error);
}
return(result, null);
}
public static (ByteWriter, string error) Encode2(IList <IClockValue> values, int startIndex, int count)
{
if (values == null || values.Count == 0)
{
return(null, null);
}
IClockValue value0 = values[0];
byte datatype = value0.DataType;
TimeEncoder tenc = (TimeEncoder)EncoderFactory.Get(DataTypeEnum.DateTime, count);
IntegerEncoder qenc = (IntegerEncoder)EncoderFactory.Get(DataTypeEnum.Integer, count);
IEncoder venc = EncoderFactory.Get(datatype, count);
if (datatype == DataTypeEnum.Double)
{
FloatEncoder encoder = (FloatEncoder)venc;
for (int i = startIndex; i < startIndex + count; i++)
{
ClockDouble value = (ClockDouble)values[i];
tenc.Write(value.Clock);
qenc.Write(value.Quality);
encoder.Write(value.Value);
}
}
else if (datatype == DataTypeEnum.Boolean)
{
BooleanEncoder encoder = (BooleanEncoder)venc;
for (int i = startIndex; i < startIndex + count; i++)
{
ClockBoolean value = (ClockBoolean)values[i];
tenc.Write(value.Clock);
qenc.Write(value.Quality);
encoder.Write(value.Value);
}
}
else if (datatype == DataTypeEnum.Integer)
{
IntegerEncoder encoder = (IntegerEncoder)venc;
for (int i = startIndex; i < startIndex + count; i++)
{
ClockInt64 value = (ClockInt64)values[i];
tenc.Write(value.Clock);
qenc.Write(value.Quality);
encoder.Write(value.Value);
}
}
else if (datatype == DataTypeEnum.String)
{
StringEncoder encoder = (StringEncoder)venc;
for (int i = startIndex; i < startIndex + count; i++)
{
ClockString value = (ClockString)values[i];
tenc.Write(value.Clock);
qenc.Write(value.Quality);
encoder.Write(value.Value);
}
}
var ts = tenc.Bytes();
venc.Flush();//for floatencoder
var qs = qenc.Bytes();
var vs = venc.Bytes();
EncoderFactory.Put(DataTypeEnum.DateTime, tenc);
EncoderFactory.Put(DataTypeEnum.Integer, qenc);
EncoderFactory.Put(datatype, venc);
if (ts.Item2 != null)
{
return(null, ts.Item2);
}
if (vs.Item2 != null)
{
return(null, vs.Item2);
}
if (qs.Item2 != null)
{
return(null, qs.Item2);
}
ByteWriter tswriter = ts.Item1;
ByteWriter vswriter = vs.Item1;
ByteWriter qswriter = qs.Item1;
ByteWriter result = new ByteWriter(1 + Varint.MaxVarintLen64 + tswriter.Length + vswriter.Length + qswriter.Length);
result.Write(datatype); //first byte valuetype
result.Write(Varint.GetBytes(tswriter.Length));
result.Write(tswriter.EndWrite(), 0, tswriter.Length);
tswriter.Release();
result.Write(Varint.GetBytes(vswriter.Length));
result.Write(vswriter.EndWrite(), 0, vswriter.Length);
vswriter.Release();
result.Write(qswriter.EndWrite(), 0, qswriter.Length);
qswriter.Release();
return(result, null);
}
public static (ByteWriter, string error) Encode(IList <IClockValue> values, int startIndex, int count)
{
if (values == null || values.Count == 0)
{
return(null, null);
}
IClockValue value0 = values[0];
byte datatype = value0.DataType;
ulong[] ts = ArrayPool <ulong> .Shared.Rent(count);
Span <ulong> ts_span = new Span <ulong>(ts, 0, count);
long[] qs = ArrayPool <long> .Shared.Rent(count);
Span <long> qs_span = new Span <long>(qs, 0, count);
ByteWriter tswriter = null, vswriter = null, qswriter = null;
string tserror = null, vserror = null, qserror = null;
if (datatype == DataTypeEnum.Double)
{
double[] vs = ArrayPool <double> .Shared.Rent(count);
Span <double> vs_span = new Span <double>(vs, 0, count);
int j = 0;
for (int i = startIndex; i < startIndex + count; i++, j++)
{
ClockDouble value = (ClockDouble)values[i];
ts_span[j] = (ulong)value.Clock;
vs_span[j] = value.Value;
qs_span[j] = value.Quality;
}
BatchDouble encoder = (BatchDouble)CoderFactory.Get(datatype);
(vswriter, vserror) = encoder.EncodingAll(vs_span);
ArrayPool <double> .Shared.Return(vs);
CoderFactory.Put(datatype, encoder);
}
else if (datatype == DataTypeEnum.Boolean)
{
bool[] vs = ArrayPool <bool> .Shared.Rent(count);
Span <bool> vs_span = new Span <bool>(vs, 0, count);
int j = 0;
for (int i = startIndex; i < startIndex + count; i++, j++)
{
ClockBoolean value = (ClockBoolean)values[i];
ts_span[j] = (ulong)value.Clock;
vs_span[j] = value.Value;
qs_span[j] = value.Quality;
}
BatchBoolean encoder = (BatchBoolean)CoderFactory.Get(datatype);
(vswriter, vserror) = encoder.EncodingAll(vs_span);
ArrayPool <bool> .Shared.Return(vs);
CoderFactory.Put(datatype, encoder);
}
else if (datatype == DataTypeEnum.Integer)
{
long[] vs = ArrayPool <long> .Shared.Rent(count);
Span <long> vs_span = new Span <long>(vs, 0, count);
int j = 0;
for (int i = startIndex; i < startIndex + count; i++, j++)
{
ClockInt64 value = (ClockInt64)values[i];
ts_span[j] = (ulong)value.Clock;
vs_span[j] = value.Value;
qs_span[j] = value.Quality;
}
BatchInt64 encoder = (BatchInt64)CoderFactory.Get(datatype);
(vswriter, vserror) = encoder.EncodingAll(vs_span);
ArrayPool <long> .Shared.Return(vs);
CoderFactory.Put(datatype, encoder);
}
else if (datatype == DataTypeEnum.String)
{
string[] vs = ArrayPool <string> .Shared.Rent(count);
Span <string> vs_span = new Span <string>(vs, 0, count);
int j = 0;
for (int i = startIndex; i < startIndex + count; i++, j++)
{
ClockString value = (ClockString)values[i];
ts_span[j] = (ulong)value.Clock;
vs_span[j] = value.Value;
qs_span[j] = value.Quality;
}
BatchString encoder = (BatchString)CoderFactory.Get(datatype);
(vswriter, vserror) = encoder.EncodingAll(vs_span);
ArrayPool <string> .Shared.Return(vs);
CoderFactory.Put(datatype, encoder);
}
bool good = (vserror == null && vswriter != null);
if (good)
{
BatchTimeStamp tenc = (BatchTimeStamp)CoderFactory.Get(DataTypeEnum.DateTime);
(tswriter, tserror) = tenc.EncodingAll(ts_span);
CoderFactory.Put(DataTypeEnum.DateTime, tenc);
good = (tswriter != null && tserror == null);
if (good)
{
BatchInt64 qenc = (BatchInt64)CoderFactory.Get(DataTypeEnum.Integer);
(qswriter, qserror) = qenc.EncodingAll(qs_span);
CoderFactory.Put(DataTypeEnum.Integer, qenc);
good = (qswriter != null && qserror == null);
}
}
ArrayPool <ulong> .Shared.Return(ts);
ArrayPool <long> .Shared.Return(qs);
if (good)
{
ByteWriter result = new ByteWriter(1 + Varint.MaxVarintLen64 + tswriter.Length + vswriter.Length + qswriter.Length);
result.Write(datatype); //first byte valuetype
result.Write(Varint.GetBytes(tswriter.Length));
result.Write(tswriter.EndWrite(), 0, tswriter.Length);
result.Write(Varint.GetBytes(vswriter.Length));
result.Write(vswriter.EndWrite(), 0, vswriter.Length);
result.Write(Varint.GetBytes(qswriter.Length));
result.Write(qswriter.EndWrite(), 0, qswriter.Length);
return(result, null);
}
if (tswriter != null)
{
tswriter.Release();
}
if (vswriter != null)
{
vswriter.Release();
}
if (qswriter != null)
{
qswriter.Release();
}
return(null, vserror != null ? vserror : tserror != null ? tserror : qserror);
}
public (ByteWriter, string) EncodingAll(Span <long> src)
{
int srclen = src.Length;
ulong max = 0;
ulong[] deltasarray = ArrayPool <ulong> .Shared.Rent(srclen);
Span <ulong> deltas = new Span <ulong>(deltasarray);
for (int i = srclen - 1; i > 0; i--)
{
long l = src[i] - src[i - 1];
ulong delta = Encoding.ZigZagEncode(l);
deltas[i] = delta;
if (delta > max)
{
max = delta;
}
}
deltas[0] = Encoding.ZigZagEncode(src[0]);
ByteWriter result;
if (srclen > 2)
{
var rle = true;
for (int i = 2; i < srclen; i++)
{
if (deltas[1] != deltas[i])
{
rle = false;
break;
}
}
if (rle)
{
// Large varints can take up to 10 bytes. We're storing 3 + 1
// type byte.
result = new ByteWriter(31);
// 4 high bits used for the encoding type
result.Write((byte)(intCompressedRLE << 4));
// The first value
result.Write(deltas[0]);
// The first delta
result.Write(Varint.GetBytes(deltas[1]));
// The number of times the delta is repeated
result.Write(Varint.GetBytes(srclen - 1));
ArrayPool <ulong> .Shared.Return(deltasarray);
return(result, null);
}
}
if (max > Simple8bEncoder.MaxValue)// There is an encoded value that's too big to simple8b encode.
{
// Encode uncompressed.
int sz = 1 + srclen * 8;
result = new ByteWriter(sz);
result.Write((byte)(intUncompressed << 4));
for (int i = 0; i < srclen; i++)
{
result.Write(deltas[i]);
}
ArrayPool <ulong> .Shared.Return(deltasarray);
return(result, null);
}
result = new ByteWriter(1 + srclen * 8);
result.Write((byte)(intCompressedSimple << 4));
// Write the first value since it's not part of the encoded values
result.Write(deltas[0]);
var error = Simple8bEncoder.EncodeAll(deltasarray, 1, srclen, result);
ArrayPool <ulong> .Shared.Return(deltasarray);
if (error != null)
{
result.Release();
return(null, error);
}
return(result, null);
}
// FloatArrayEncodeAll encodes src into b, returning b and any error encountered.
// The returned slice may be of a different length and capactity to b.
//
// Currently only the float compression scheme used in Facebook's Gorilla is
// supported, so this method implements a batch oriented version of that.
public (ByteWriter, string) EncodingAll(Span <double> src)
{
int length = src.Length;
//9=Enough room for the header and one value.
ByteWriter result = new ByteWriter(length * 10 + 9);
byte[] bytes = result.EndWrite();
Span <byte> b = new Span <byte>(bytes);
b.Fill(0);
b[0] = (floatCompressedGorilla << 4);
double first;
bool finished = false;
if (length > 0 && Double.IsNaN(src[0]))
{
result.Release();
return(null, "unsupported value: NaN");
}
else if (length == 0)
{
first = Double.NaN;// Write sentinal value to terminate batch.
finished = true;
}
else
{
first = src[0];
src = src.Slice(1);
}
int n = 8 + 64;//Number of bits written.
ulong prev = Float64bits(first);
// Write first value.
ByteWriter.WriteBigEndian(bytes, 1, prev);
int prevLeading = -1, prevTrailing = 0;
int leading, trailing;
ulong mask;
double sum = 0;
// Encode remaining values.
for (int i = 0; !finished; i++)
{
double x;
if (i < src.Length)
{
x = src[i];
sum += x;
}
else
{
// Encode sentinal value to terminate batch
x = double.NaN;
finished = true;
}
ulong cur = Float64bits(x);
ulong vDelta = cur ^ prev;
if (vDelta == 0)
{
n++;// Write a zero bit. Nothing else to do.
prev = cur;
continue;
}
// n&7 - current bit in current byte.
// n>>3 - the current byte.
b[n >> 3] |= (byte)(128 >> (n & 7));// Sets the current bit of the current byte.
n++;
// Write the delta to b.
// Determine the leading and trailing zeros.
leading = Encoding.Clz(vDelta);
trailing = Encoding.Ctz(vDelta);
// Clamp number of leading zeros to avoid overflow when encoding
leading &= 0x1F;
if (leading >= 32)
{
leading = 31;
}
if (prevLeading != -1 && leading >= prevLeading && trailing >= prevTrailing)
{
n++;// Write a zero bit.
// Write the l least significant bits of vDelta to b, most significant
// bit first.
int L = 64 - prevLeading - prevTrailing;
// Full value to write.
ulong v = (vDelta >> prevTrailing) << (64 - L); // l least signifciant bits of v.
int m = n & 7; // Current bit in current byte.
int written = 0;
if (m > 0) // In this case the current byte is not full.
{
written = 8 - m;
if (L < written)
{
written = L;
}
mask = v >> 56;// Move 8 MSB to 8 LSB
b[n >> 3] |= (byte)(mask >> m);
n += written;
if (L == written)
{
prev = cur;
continue;
}
}
var vv = v << written;// Move written bits out of the way.
ByteWriter.WriteBigEndian(bytes, n >> 3, vv);
n += (L - written);
}
else
{
prevLeading = leading;
prevTrailing = trailing;
// Set a single bit to indicate a value will follow.
b[n >> 3] |= (byte)(128 >> (n & 7));// Set current bit on current byte
n++;
int m = n & 7;
int L = 5;
ulong v = (ulong)leading << 59; // 5 LSB of leading.
mask = v >> 56; // Move 5 MSB to 8 LSB
if (m <= 3) // 5 bits fit into current byte.
{
b[n >> 3] |= (byte)(mask >> m);
n = n + L;
}
else// In this case there are fewer than 5 bits available in current byte.
{
// First step is to fill current byte
int written = 8 - m;
b[n >> 3] |= (byte)(mask >> m);// Some of mask will get lost.
n += written;
// Second step is to write the lost part of mask into the next byte.
mask = v << written; // Move written bits in previous byte out of way.
mask >>= 56;
m = n & 7; //Recompute current bit.
b[n >> 3] |= (byte)(mask >> m);
n += (L - written);
}
// Note that if leading == trailing == 0, then sigbits == 64. But that
// value doesn't actually fit into the 6 bits we have.
// Luckily, we never need to encode 0 significant bits, since that would
// put us in the other case (vdelta == 0). So instead we write out a 0 and
// adjust it back to 64 on unpacking.
int sigbits = 64 - leading - trailing;
m = n & 7;
L = 6;
v = (ulong)sigbits << 58; // Move 6 LSB of sigbits to MSB
mask = v >> 56; // Move 6 MSB to 8 LSB
if (m <= 2)
{
// The 6 bits fit into the current byte.
b[n >> 3] |= (byte)(mask >> m);
n += L;
}
else// In this case there are fewer than 6 bits available in current byte.
{
// First step is to fill the current byte.
int written = 8 - m;
b[n >> 3] |= (byte)(mask >> m);// Write to the current bit.
n += written;
// Second step is to write the lost part of mask into the next byte.
// Write l remaining bits into current byte.
mask = v << written; // Remove bits written in previous byte out of way.
mask >>= 56;
m = n & 7; // Recompute current bit.
b[n >> 3] |= (byte)(mask >> m);
n += L - written;
}
// Write final value.
m = n & 7;
L = sigbits;
v = (vDelta >> trailing) << (64 - L); // Move l LSB into MSB
int written2 = 0;
if (m > 0) // In this case the current byte is not full.
{
written2 = 8 - m;
if (L < written2)
{
written2 = L;
}
mask = v >> 56;//Move 8 MSB to 8 LSB
b[n >> 3] |= (byte)(mask >> m);
n += written2;
if (L - written2 == 0)
{
prev = cur;
continue;
}
}
// Shift remaining bits and write out in one go.
ulong vv = v << written2;// Remove bits written in previous byte.
ByteWriter.WriteBigEndian(bytes, n >> 3, vv);
n += (L - written2);
}
prev = cur;
}
if (Double.IsNaN(sum))
{
result.Release();
return(null, "unsupported value: NaN");
}
int blength = n >> 3;
if ((n & 7) > 0)
{
blength++; // Add an extra byte to capture overflowing bits.
}
result.Length = blength;
return(result, null);
}