public (ByteWriter, string) EncodingAll(Span <bool> src_span) { int srclen = src_span.Length; int sz = 1 + 8 + (srclen + 7) / 8;//header+num bools+bool data ByteWriter result = new ByteWriter(sz); // Store the encoding type in the 4 high bits of the first byte result.Write((byte)(booleanCompressedBitPacked << 4)); // Encode the number of booleans written. result.Write(Varint.GetBytes(srclen)); int bitcount = result.Length * 8; // Current bit in current byte. Span <byte> result_span = new Span <byte>(result.EndWrite()); //取出数组 for (int i = 0; i < srclen; i++) { bool v = src_span[i]; if (v) { result_span[n >> 3] |= (byte)(128 >> (bitcount & 7)); }// Set current bit on current byte. else { result_span[n >> 3] = (byte)(result_span[n >> 3] & ~(byte)(128 >> (bitcount & 7))); }// Clear current bit on current byte. bitcount++; } int length = bitcount >> 3; if ((bitcount & 7) > 0) { length++;// Add an extra byte to capture overflowing bits. } result.Length = length; return(result, null); }
public void WriteIndex() { lock (lockthis) { if (index.SidCount == 0) { throw new Exception(Constants.ErrNoValues); } int size = 8 + (int)index.Size;//8 for n ByteWriter writer = new ByteWriter(size); index.WriteTo(writer); writer.Write(blocksize); wrapped.Write(writer.EndWrite(), 0, writer.Length); writer.Release(); } }
public (ByteWriter, string) EncodingAll(Span <string> src) { int srcSz = 2 + src.Length * Varint.MaxVarintLen32;// strings should't be longer than 64kb for (int i = 0; i < src.Length; i++) { srcSz += src[i].Length; } // determine the maximum possible length needed for the buffer, which // includes the compressed size var compressSz = 0; if (src.Length > 0) { compressSz = SnappyMaxEncodedLen(srcSz) + 1;//header } int totSz = srcSz + compressSz; ByteWriter result = new ByteWriter(totSz); // Shortcut to snappy encoding nothing. if (src.Length == 0) { result.Write((byte)(stringCompressedSnappy << 4)); return(result, null); } for (int i = 0; i < src.Length; i++) { result.Write(Varint.GetBytes(src[i].Length)); result.Write(System.Text.Encoding.Default.GetBytes(src[i])); } byte[] compressed = SnappyPI.SnappyCodec.Compress(result.EndWrite(), 0, result.Length); result.Length = 0; result.Write((byte)(stringCompressedSnappy << 4)); result.Write(compressed); return(result, null); }
public (ByteWriter, string) EncodingAll(Span <ulong> src) { int srclen = src.Length; if (srclen == 0) { return(null, null); // Nothing to do } ulong max = 0, div = (ulong)1e12; ulong[] deltaarray = ArrayPool <ulong> .Shared.Rent(srclen); Span <ulong> deltas = new Span <ulong>(deltaarray); deltas[0] = src[0]; ByteWriter result; if (srclen > 1) { for (int i = srclen - 1; i > 0; i--) { deltas[i] = src[i] - src[i - 1]; if (deltas[i] > max) { max = deltas[i]; } } bool rle = true; var delta1 = deltas[1]; for (int i = 2; i < srclen; i++) { if (delta1 != deltas[i]) { rle = false; break; } } // Deltas are the same - encode with RLE if (rle) { // Large varints can take up to 10 bytes. We're storing 3 + 1 // type byte. result = new ByteWriter(31); // 4 high bits used for the encoding type result.Write((byte)(timeCompressedRLE << 4)); // The first value result.Write(deltas[0]); // The first delta, checking the divisor // given all deltas are the same, we can do a single check for the divisor ulong v = deltas[1]; while (div > 1 && v % div != 0) { div /= 10; } if (div > 1) { // 4 low bits are the log10 divisor result.EndWrite()[0] |= (byte)Math.Log10(div); result.Write(Varint.GetBytes(deltas[1] / div)); } else { result.Write(Varint.GetBytes(deltas[1])); } // The number of times the delta is repeated result.Write(Varint.GetBytes(srclen)); ArrayPool <ulong> .Shared.Return(deltaarray); return(result, null); } } // We can't compress this time-range, the deltas exceed 1 << 60 if (max > Simple8bEncoder.MaxValue) { // Encode uncompressed. int sz = 1 + srclen * 8; result = new ByteWriter(sz); result.Write((byte)(timeUncompressed << 4)); for (int i = 0; i < srclen; i++) { result.Write(deltas[i]); } ArrayPool <ulong> .Shared.Return(deltaarray); return(result, null); } // find divisor only if we're compressing with simple8b for (int i = 1; i < srclen && div > 1; i++) { // If our value is divisible by 10, break. Otherwise, try the next smallest divisor. var v = deltaarray[i]; while (div > 1 && v % div != 0) { div /= 10; } } // Only apply the divisor if it's greater than 1 since division is expensive. if (div > 1) { for (int i = 1; i < srclen; i++) { deltas[i] /= div; } } result = new ByteWriter(1 + srclen * 8); result.Write((byte)((timeCompressedPackedSimple << 4) | (byte)Math.Log10(div))); // Write the first value since it's not part of the encoded values result.Write(deltas[0]); // Encode with simple8b - fist value is written unencoded using 8 bytes. var error = Simple8bEncoder.EncodeAll(deltaarray, 1, srclen, result); if (error != null) { result.Release(); return(null, error); } return(result, null); }
public static (ByteWriter, string error) Encode2(IList <IClockValue> values, int startIndex, int count) { if (values == null || values.Count == 0) { return(null, null); } IClockValue value0 = values[0]; byte datatype = value0.DataType; TimeEncoder tenc = (TimeEncoder)EncoderFactory.Get(DataTypeEnum.DateTime, count); IntegerEncoder qenc = (IntegerEncoder)EncoderFactory.Get(DataTypeEnum.Integer, count); IEncoder venc = EncoderFactory.Get(datatype, count); if (datatype == DataTypeEnum.Double) { FloatEncoder encoder = (FloatEncoder)venc; for (int i = startIndex; i < startIndex + count; i++) { ClockDouble value = (ClockDouble)values[i]; tenc.Write(value.Clock); qenc.Write(value.Quality); encoder.Write(value.Value); } } else if (datatype == DataTypeEnum.Boolean) { BooleanEncoder encoder = (BooleanEncoder)venc; for (int i = startIndex; i < startIndex + count; i++) { ClockBoolean value = (ClockBoolean)values[i]; tenc.Write(value.Clock); qenc.Write(value.Quality); encoder.Write(value.Value); } } else if (datatype == DataTypeEnum.Integer) { IntegerEncoder encoder = (IntegerEncoder)venc; for (int i = startIndex; i < startIndex + count; i++) { ClockInt64 value = (ClockInt64)values[i]; tenc.Write(value.Clock); qenc.Write(value.Quality); encoder.Write(value.Value); } } else if (datatype == DataTypeEnum.String) { StringEncoder encoder = (StringEncoder)venc; for (int i = startIndex; i < startIndex + count; i++) { ClockString value = (ClockString)values[i]; tenc.Write(value.Clock); qenc.Write(value.Quality); encoder.Write(value.Value); } } var ts = tenc.Bytes(); venc.Flush();//for floatencoder var qs = qenc.Bytes(); var vs = venc.Bytes(); EncoderFactory.Put(DataTypeEnum.DateTime, tenc); EncoderFactory.Put(DataTypeEnum.Integer, qenc); EncoderFactory.Put(datatype, venc); if (ts.Item2 != null) { return(null, ts.Item2); } if (vs.Item2 != null) { return(null, vs.Item2); } if (qs.Item2 != null) { return(null, qs.Item2); } ByteWriter tswriter = ts.Item1; ByteWriter vswriter = vs.Item1; ByteWriter qswriter = qs.Item1; ByteWriter result = new ByteWriter(1 + Varint.MaxVarintLen64 + tswriter.Length + vswriter.Length + qswriter.Length); result.Write(datatype); //first byte valuetype result.Write(Varint.GetBytes(tswriter.Length)); result.Write(tswriter.EndWrite(), 0, tswriter.Length); tswriter.Release(); result.Write(Varint.GetBytes(vswriter.Length)); result.Write(vswriter.EndWrite(), 0, vswriter.Length); vswriter.Release(); result.Write(qswriter.EndWrite(), 0, qswriter.Length); qswriter.Release(); return(result, null); }
public static (ByteWriter, string error) Encode(IList <IClockValue> values, int startIndex, int count) { if (values == null || values.Count == 0) { return(null, null); } IClockValue value0 = values[0]; byte datatype = value0.DataType; ulong[] ts = ArrayPool <ulong> .Shared.Rent(count); Span <ulong> ts_span = new Span <ulong>(ts, 0, count); long[] qs = ArrayPool <long> .Shared.Rent(count); Span <long> qs_span = new Span <long>(qs, 0, count); ByteWriter tswriter = null, vswriter = null, qswriter = null; string tserror = null, vserror = null, qserror = null; if (datatype == DataTypeEnum.Double) { double[] vs = ArrayPool <double> .Shared.Rent(count); Span <double> vs_span = new Span <double>(vs, 0, count); int j = 0; for (int i = startIndex; i < startIndex + count; i++, j++) { ClockDouble value = (ClockDouble)values[i]; ts_span[j] = (ulong)value.Clock; vs_span[j] = value.Value; qs_span[j] = value.Quality; } BatchDouble encoder = (BatchDouble)CoderFactory.Get(datatype); (vswriter, vserror) = encoder.EncodingAll(vs_span); ArrayPool <double> .Shared.Return(vs); CoderFactory.Put(datatype, encoder); } else if (datatype == DataTypeEnum.Boolean) { bool[] vs = ArrayPool <bool> .Shared.Rent(count); Span <bool> vs_span = new Span <bool>(vs, 0, count); int j = 0; for (int i = startIndex; i < startIndex + count; i++, j++) { ClockBoolean value = (ClockBoolean)values[i]; ts_span[j] = (ulong)value.Clock; vs_span[j] = value.Value; qs_span[j] = value.Quality; } BatchBoolean encoder = (BatchBoolean)CoderFactory.Get(datatype); (vswriter, vserror) = encoder.EncodingAll(vs_span); ArrayPool <bool> .Shared.Return(vs); CoderFactory.Put(datatype, encoder); } else if (datatype == DataTypeEnum.Integer) { long[] vs = ArrayPool <long> .Shared.Rent(count); Span <long> vs_span = new Span <long>(vs, 0, count); int j = 0; for (int i = startIndex; i < startIndex + count; i++, j++) { ClockInt64 value = (ClockInt64)values[i]; ts_span[j] = (ulong)value.Clock; vs_span[j] = value.Value; qs_span[j] = value.Quality; } BatchInt64 encoder = (BatchInt64)CoderFactory.Get(datatype); (vswriter, vserror) = encoder.EncodingAll(vs_span); ArrayPool <long> .Shared.Return(vs); CoderFactory.Put(datatype, encoder); } else if (datatype == DataTypeEnum.String) { string[] vs = ArrayPool <string> .Shared.Rent(count); Span <string> vs_span = new Span <string>(vs, 0, count); int j = 0; for (int i = startIndex; i < startIndex + count; i++, j++) { ClockString value = (ClockString)values[i]; ts_span[j] = (ulong)value.Clock; vs_span[j] = value.Value; qs_span[j] = value.Quality; } BatchString encoder = (BatchString)CoderFactory.Get(datatype); (vswriter, vserror) = encoder.EncodingAll(vs_span); ArrayPool <string> .Shared.Return(vs); CoderFactory.Put(datatype, encoder); } bool good = (vserror == null && vswriter != null); if (good) { BatchTimeStamp tenc = (BatchTimeStamp)CoderFactory.Get(DataTypeEnum.DateTime); (tswriter, tserror) = tenc.EncodingAll(ts_span); CoderFactory.Put(DataTypeEnum.DateTime, tenc); good = (tswriter != null && tserror == null); if (good) { BatchInt64 qenc = (BatchInt64)CoderFactory.Get(DataTypeEnum.Integer); (qswriter, qserror) = qenc.EncodingAll(qs_span); CoderFactory.Put(DataTypeEnum.Integer, qenc); good = (qswriter != null && qserror == null); } } ArrayPool <ulong> .Shared.Return(ts); ArrayPool <long> .Shared.Return(qs); if (good) { ByteWriter result = new ByteWriter(1 + Varint.MaxVarintLen64 + tswriter.Length + vswriter.Length + qswriter.Length); result.Write(datatype); //first byte valuetype result.Write(Varint.GetBytes(tswriter.Length)); result.Write(tswriter.EndWrite(), 0, tswriter.Length); result.Write(Varint.GetBytes(vswriter.Length)); result.Write(vswriter.EndWrite(), 0, vswriter.Length); result.Write(Varint.GetBytes(qswriter.Length)); result.Write(qswriter.EndWrite(), 0, qswriter.Length); return(result, null); } if (tswriter != null) { tswriter.Release(); } if (vswriter != null) { vswriter.Release(); } if (qswriter != null) { qswriter.Release(); } return(null, vserror != null ? vserror : tserror != null ? tserror : qserror); }
// FloatArrayEncodeAll encodes src into b, returning b and any error encountered. // The returned slice may be of a different length and capactity to b. // // Currently only the float compression scheme used in Facebook's Gorilla is // supported, so this method implements a batch oriented version of that. public (ByteWriter, string) EncodingAll(Span <double> src) { int length = src.Length; //9=Enough room for the header and one value. ByteWriter result = new ByteWriter(length * 10 + 9); byte[] bytes = result.EndWrite(); Span <byte> b = new Span <byte>(bytes); b.Fill(0); b[0] = (floatCompressedGorilla << 4); double first; bool finished = false; if (length > 0 && Double.IsNaN(src[0])) { result.Release(); return(null, "unsupported value: NaN"); } else if (length == 0) { first = Double.NaN;// Write sentinal value to terminate batch. finished = true; } else { first = src[0]; src = src.Slice(1); } int n = 8 + 64;//Number of bits written. ulong prev = Float64bits(first); // Write first value. ByteWriter.WriteBigEndian(bytes, 1, prev); int prevLeading = -1, prevTrailing = 0; int leading, trailing; ulong mask; double sum = 0; // Encode remaining values. for (int i = 0; !finished; i++) { double x; if (i < src.Length) { x = src[i]; sum += x; } else { // Encode sentinal value to terminate batch x = double.NaN; finished = true; } ulong cur = Float64bits(x); ulong vDelta = cur ^ prev; if (vDelta == 0) { n++;// Write a zero bit. Nothing else to do. prev = cur; continue; } // n&7 - current bit in current byte. // n>>3 - the current byte. b[n >> 3] |= (byte)(128 >> (n & 7));// Sets the current bit of the current byte. n++; // Write the delta to b. // Determine the leading and trailing zeros. leading = Encoding.Clz(vDelta); trailing = Encoding.Ctz(vDelta); // Clamp number of leading zeros to avoid overflow when encoding leading &= 0x1F; if (leading >= 32) { leading = 31; } if (prevLeading != -1 && leading >= prevLeading && trailing >= prevTrailing) { n++;// Write a zero bit. // Write the l least significant bits of vDelta to b, most significant // bit first. int L = 64 - prevLeading - prevTrailing; // Full value to write. ulong v = (vDelta >> prevTrailing) << (64 - L); // l least signifciant bits of v. int m = n & 7; // Current bit in current byte. int written = 0; if (m > 0) // In this case the current byte is not full. { written = 8 - m; if (L < written) { written = L; } mask = v >> 56;// Move 8 MSB to 8 LSB b[n >> 3] |= (byte)(mask >> m); n += written; if (L == written) { prev = cur; continue; } } var vv = v << written;// Move written bits out of the way. ByteWriter.WriteBigEndian(bytes, n >> 3, vv); n += (L - written); } else { prevLeading = leading; prevTrailing = trailing; // Set a single bit to indicate a value will follow. b[n >> 3] |= (byte)(128 >> (n & 7));// Set current bit on current byte n++; int m = n & 7; int L = 5; ulong v = (ulong)leading << 59; // 5 LSB of leading. mask = v >> 56; // Move 5 MSB to 8 LSB if (m <= 3) // 5 bits fit into current byte. { b[n >> 3] |= (byte)(mask >> m); n = n + L; } else// In this case there are fewer than 5 bits available in current byte. { // First step is to fill current byte int written = 8 - m; b[n >> 3] |= (byte)(mask >> m);// Some of mask will get lost. n += written; // Second step is to write the lost part of mask into the next byte. mask = v << written; // Move written bits in previous byte out of way. mask >>= 56; m = n & 7; //Recompute current bit. b[n >> 3] |= (byte)(mask >> m); n += (L - written); } // Note that if leading == trailing == 0, then sigbits == 64. But that // value doesn't actually fit into the 6 bits we have. // Luckily, we never need to encode 0 significant bits, since that would // put us in the other case (vdelta == 0). So instead we write out a 0 and // adjust it back to 64 on unpacking. int sigbits = 64 - leading - trailing; m = n & 7; L = 6; v = (ulong)sigbits << 58; // Move 6 LSB of sigbits to MSB mask = v >> 56; // Move 6 MSB to 8 LSB if (m <= 2) { // The 6 bits fit into the current byte. b[n >> 3] |= (byte)(mask >> m); n += L; } else// In this case there are fewer than 6 bits available in current byte. { // First step is to fill the current byte. int written = 8 - m; b[n >> 3] |= (byte)(mask >> m);// Write to the current bit. n += written; // Second step is to write the lost part of mask into the next byte. // Write l remaining bits into current byte. mask = v << written; // Remove bits written in previous byte out of way. mask >>= 56; m = n & 7; // Recompute current bit. b[n >> 3] |= (byte)(mask >> m); n += L - written; } // Write final value. m = n & 7; L = sigbits; v = (vDelta >> trailing) << (64 - L); // Move l LSB into MSB int written2 = 0; if (m > 0) // In this case the current byte is not full. { written2 = 8 - m; if (L < written2) { written2 = L; } mask = v >> 56;//Move 8 MSB to 8 LSB b[n >> 3] |= (byte)(mask >> m); n += written2; if (L - written2 == 0) { prev = cur; continue; } } // Shift remaining bits and write out in one go. ulong vv = v << written2;// Remove bits written in previous byte. ByteWriter.WriteBigEndian(bytes, n >> 3, vv); n += (L - written2); } prev = cur; } if (Double.IsNaN(sum)) { result.Release(); return(null, "unsupported value: NaN"); } int blength = n >> 3; if ((n & 7) > 0) { blength++; // Add an extra byte to capture overflowing bits. } result.Length = blength; return(result, null); }