public void Send(Stream resultStream) { lock (_lockObject) { try { Varint.SetUInt64(_stream, (ulong)resultStream.Length); using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 32)) { long remain = resultStream.Length; while (remain > 0) { int readLength = resultStream.Read(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length)); _stream.Write(safeBuffer.Value, 0, readLength); _stream.Flush(); remain -= readLength; } } } finally { resultStream.Dispose(); } } }
public byte[] ObjectToBuffer(object instance) { try { if (instance == null) { return(_NullBuffer()); } var type = instance.GetType(); var describer = _GetDescriber(type); var id = describer.Id; var idCount = Varint.GetByteCount((ulong)id); var bufferCount = describer.GetByteCount(instance); var buffer = new byte[idCount + bufferCount]; var readCount = Varint.NumberToBuffer(buffer, 0, id); describer.ToBuffer(instance, buffer, readCount); return(buffer); } catch (DescriberException ex) { if (instance != null) { throw new SystemException(string.Format("ObjectToBuffer {0}", instance.GetType()), ex); } throw new SystemException("ObjectToBuffer null", ex); } }
private void Alive() { lock (_lockObject) { Varint.SetUInt64(_stream, 0); } }
// BlockCount returns the number of timestamps encoded in block. public static int BlockCount(byte[] bytes) { int tsLen, index; index = 1 + Varint.Read(bytes, 1, out tsLen); return(TimeDecoder.CountTimestamps(bytes, index, tsLen)); }
private static Stream RemoveHash(Stream stream) { if (stream == null) { throw new ArgumentNullException(nameof(stream)); } int type = (int)Varint.GetUInt64(stream); if (type == (int)ConvertHashAlgorithm.Sha256) { var value = new byte[32]; stream.Read(value, 0, value.Length); var dataStream = new RangeStream(stream, true); if (!Unsafe.Equals(value, Sha256.Compute(new WrapperStream(dataStream)))) { throw new ArgumentException("Hash"); } dataStream.Seek(0, SeekOrigin.Begin); return(dataStream); } else { throw new NotSupportedException(); } }
public static int CountTimestamps(byte[] b, int startindex, int len) { if (b == null || b.Length < startindex + len) { return(0); } // Encoding type is stored in the 4 high bits of the first byte byte encoding = (byte)(b[startindex] >> 4); switch (encoding) { case TimeEncoder.timeUncompressed: // Uncompressed timestamps are just 8 bytes each return((len - 1) / 8); case TimeEncoder.timeCompressedRLE: // First 9 bytes are the starting timestamp and scaling factor, skip over them int j = startindex + 9; // Next 1-10 bytes is our (scaled down by factor of 10) run length values ulong value; j += Varint.Read(b, j, out value); // Last 1-10 bytes is how many times the value repeats Varint.Read(b, j, out value); return((int)value); case TimeEncoder.timeCompressedPackedSimple: // First 9 bytes are the starting timestamp and scaling factor, skip over them var count = Simple8bDecoder.CountBytes(b, startindex + 9, len - 9); return(count.Item1 + 1); // +1 is for the first uncompressed timestamp, starting timestamep in b[1:9] default: return(0); } }
// SetBytes initializes the decoder with a new set of bytes to read from. // This must be called before calling any other methods. public string SetBytes(byte[] b, int startindex, int len) { err = null; if (b == null || len <= 0 || b.Length - startindex - len < 0) { return(null); } // First byte stores the encoding type, only have 1 bit-packet format // currently ignore for now. ulong boolcount; int bytecount = Varint.Read(b, startindex + 1, out boolcount); if (bytecount <= 0) { err = "BooleanDecoder: invalid count"; return(err); } byteIndex = bytecount + startindex + 1; i = -1; n = (int)boolcount; bytes = b; int min = len * 8; if (min < n) { // Shouldn't happen - TSM file was truncated/corrupted n = min; } return(null); }
int ITypeDescriber.ToBuffer(object instance, byte[] buffer, int begin) { try { var offset = begin; var validFields = _Fields.Select( (field, index) => new { field, index }) .Where(validField => object.Equals(_GetDescriber(validField.field).Default, validField.field.GetValue(instance)) == false) .ToArray(); offset += Varint.NumberToBuffer(buffer, offset, validFields.Length); foreach (var validField in validFields) { var index = validField.index; offset += Varint.NumberToBuffer(buffer, offset, index); var field = validField.field; var value = field.GetValue(instance); var describer = _GetDescriber(field); offset += describer.ToBuffer(value, buffer, offset); } return(offset - begin); } catch (Exception ex) { throw new DescriberException(typeof(ClassDescriber), _Type, _Id, "ToBuffer", ex); } }
int ITypeDescriber.GetByteCount(object instance) { try { var validFields = _Fields.Select( (field, index) => new { field, index }) .Where(validField => object.Equals(_GetDescriber(validField.field).Default, validField.field.GetValue(instance)) == false) .ToArray(); var validCount = Varint.GetByteCount(validFields.Length); var count = 0; for (var i = 0; i < validFields.Length; i++) { var validField = validFields[i]; var value = validField.field.GetValue(instance); var describer = _GetDescriber(validField.field); var byteCount = describer.GetByteCount(value); var indexCount = Varint.GetByteCount(validField.index); count += byteCount + indexCount; } return(count + validCount); } catch (Exception ex) { throw new DescriberException(typeof(ClassDescriber), _Type, _Id, "GetByteCount", ex); } }
public void PutGivesCorrectOutput() { var buf = new byte[Varint.MaxVarintLen]; var n = Varint.PutVarint(buf, 127); Assert.Equal(1, n); }
public static bool TryWrite(uint version, ReadOnlySequence <byte> sequence, IBufferWriter <byte> writer) { Varint.SetUInt32(version, writer); Varint.SetUInt32((uint)ConvertCompressionAlgorithm.Brotli, writer); using var encoder = new BrotliEncoder(0, 10); var reader = new SequenceReader <byte>(sequence); for (; ;) { var status = encoder.Compress(reader.UnreadSpan, writer.GetSpan(), out var bytesConsumed, out var bytesWritten, false); if (status == OperationStatus.InvalidData) { _logger.Warn("invalid data"); return(false); } reader.Advance(bytesConsumed); writer.Advance(bytesWritten); if (status == OperationStatus.Done) { return(true); } } }
public static ClockValues Decode2(byte[] bytes, int startindex) { byte datatype = bytes[startindex]; startindex++; int index; index = startindex + Varint.Read(bytes, startindex, out int tsLen);//时标段; TimeDecoder tdec = (TimeDecoder)DecoderFactory.Get(DataTypeEnum.DateTime); tdec.SetBytes(bytes, index, tsLen); index = index + tsLen; index = index + Varint.Read(bytes, index, out int vsLen);//数值段; IDecoder vdec = DecoderFactory.Get(datatype); vdec.SetBytes(bytes, index, vsLen); index = index + vsLen; index = index + Varint.Read(bytes, index, out int qsLen);//质量段; IntegerDecoder qdec = (IntegerDecoder)DecoderFactory.Get(DataTypeEnum.Integer); qdec.SetBytes(bytes, index, qsLen); ClockValues result = new ClockValues(); if (datatype == DataTypeEnum.Double) { FloatDecoder decoder = (FloatDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } else if (datatype == DataTypeEnum.Boolean) { BooleanDecoder decoder = (BooleanDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } else if (datatype == DataTypeEnum.Integer) { IntegerDecoder decoder = (IntegerDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } else if (datatype == DataTypeEnum.String) { StringDecoder decoder = (StringDecoder)vdec; while (tdec.Next() && vdec.Next() && qdec.Next()) { result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read())); } } DecoderFactory.Put(DataTypeEnum.DateTime, tdec); DecoderFactory.Put(DataTypeEnum.Integer, qdec); DecoderFactory.Put(datatype, vdec); return(result); }
public static byte[] Encode(IEnumerable <ReadOnlyMemory <byte> > collection) { var pipe = new Pipe(); { foreach (var value in collection) { Varint.SetUInt64((uint)value.Length, pipe.Writer); BytesOperations.Copy(value.Span, pipe.Writer.GetSpan(value.Length), value.Length); pipe.Writer.Advance(value.Length); } pipe.Writer.Complete(); } byte[] result; { pipe.Reader.TryRead(out var readResult); result = readResult.Buffer.ToArray(); pipe.Reader.Complete(); } return(result); }
public async Task Writing() { var ms = new MemoryStream(); var muxer = new Muxer { Channel = ms }; var stream = new Substream { Muxer = muxer }; var m1 = new byte[1]; stream.AddData(new byte[] { 10 }); Assert.IsTrue(stream.CanRead); Assert.IsTrue(stream.CanWrite); Assert.AreEqual(1, await stream.ReadAsync(m1, 0, 1)); await stream.WriteAsync(m1, 0, 1); stream.WriteByte(11); await stream.FlushAsync(); ms.Position = 0; var header = await Header.ReadAsync(ms); var length = await Varint.ReadVarint32Async(ms); var payload = new byte[length]; ms.Read(payload, 0, length); Assert.AreEqual(stream.Id, header.StreamId); Assert.AreEqual(2, payload.Length); CollectionAssert.AreEqual(new byte[] { 10, 11 }, payload); }
public (int, string) DecodeAll(Span <byte> src, Span <bool> to) { if (src.Length == 0) { return(0, null); } // First byte stores the encoding type, only have 1 bit-packet format // currently ignore for now. src = src.Slice(1); int nn = Varint.Read(src, out int count); src = src.Slice(nn); if (count <= 0) { return(0, "booleanBatchDecoder: invalid count"); } int j = 0; foreach (byte v in src) { for (byte i = 128; i > 0; i >>= 1) { to[j++] = (v & i) != 0; } } return(j, null); }
public void TooBig_Int32() { var bytes = Varint.Encode((long) int.MaxValue + 1); Assert.Throws <InvalidDataException>(() => Varint.DecodeInt32(bytes)); }
public object BufferToObject(byte[] buffer) { ulong id = 0; try { var readIdCount = Varint.BufferToNumber(buffer, 0, out id); if (id == 0) { return(null); } var describer = _GetDescriber((int)id); object instance; describer.ToObject(buffer, readIdCount, out instance); return(instance); } catch (DescriberException ex) { var describer = _GetDescriber((int)id); if (describer != null) { throw new SystemException(string.Format("BufferToObject {0}:{1}", id, describer.Type.FullName), ex); } throw new SystemException(string.Format("BufferToObject {0}:unkown", id), ex); } }
// Read returns the next value from the decoder. public string Read() { // Read the length of the string int length; int n = Varint.Read(bytes, i, out length); if (n <= 0) { err = "StringDecoder: invalid encoded string length"; return(String.Empty); } // The length of this string plus the length of the variable byte encoded length l = length + n; int lower = i + n; int upper = lower + length; if (upper < lower) { err = "StringDecoder: length overflow"; return(String.Empty); } if (upper > bytes.Length) { err = "StringDecoder: not enough data to represent encoded string"; return(String.Empty); } return(System.Text.Encoding.Default.GetString(bytes, lower, length)); }
/// <summary>Converts the given value into a byte array</summary> /// <param name="Receiver">The array that receives the final data.</param> /// <param name="Value">The object to convert</param> /// <param name="StartIndex">The startindex for the data</param> /// <returns>Converts the given value into a byte array</returns> public static Int32 OntoBytes(Byte[] Receiver, Int32 Value, Int32 StartIndex = 0) { if (Receiver is null) { throw new ArgumentNullException(nameof(Receiver)); } Int32 Count = Varint.ByteCount((UInt32)Value); Int32 Mark = Count - 1 + StartIndex; if (Mark >= Receiver.Length) { throw new ArgumentException($"Receiving byte array is not of an proper length"); } for (Int32 I = StartIndex; I < Mark; I++) { Receiver[I] = (Byte)(Value | Varint._SignalMask); Value >>= 7; } Receiver[Mark] = (Byte)(Value & Varint._NonSignalMask); return(Count); }
public override bool TryRead(ref byte[] buf) { TypeId = Varint.ReadInt(ref buf); X = Varint.ReadInt(ref buf); Z = Varint.ReadInt(ref buf); return(true); }
public (ByteWriter, string) EncodingAll(Span <bool> src_span) { int srclen = src_span.Length; int sz = 1 + 8 + (srclen + 7) / 8;//header+num bools+bool data ByteWriter result = new ByteWriter(sz); // Store the encoding type in the 4 high bits of the first byte result.Write((byte)(booleanCompressedBitPacked << 4)); // Encode the number of booleans written. result.Write(Varint.GetBytes(srclen)); int bitcount = result.Length * 8; // Current bit in current byte. Span <byte> result_span = new Span <byte>(result.EndWrite()); //取出数组 for (int i = 0; i < srclen; i++) { bool v = src_span[i]; if (v) { result_span[n >> 3] |= (byte)(128 >> (bitcount & 7)); }// Set current bit on current byte. else { result_span[n >> 3] = (byte)(result_span[n >> 3] & ~(byte)(128 >> (bitcount & 7))); }// Clear current bit on current byte. bitcount++; } int length = bitcount >> 3; if ((bitcount & 7) > 0) { length++;// Add an extra byte to capture overflowing bits. } result.Length = length; return(result, null); }
public (int, string) DecodeAll(Span <byte> b, Span <bool> dst) { if (b.Length == 0) { return(0, null); } // First byte stores the encoding type, only have 1 bit-packet format // currently ignore for now. b = b.Slice(1); int n = Varint.Read(b, out int count); if (n <= 0) { return(0, "booleanBatchDecoder: invalid count"); } b = b.Slice(n); int j = 0; foreach (byte v in b) { for (byte i = 128; i > 0 && j < dst.Length; i >>= 1) { dst[j++] = (v & i) != 0; } } return(j, null); }
public override byte[] ToByteArray() { return(ArrayBuilder.Build <byte>( Varint.Encode(TypeId), Varint.Encode(X), Varint.Encode(Z) )); }
public void MaxLong() { var x = "ffffffffffffffff7f".ToHexBuffer(); Assert.Equal(9, Varint.RequiredBytes(long.MaxValue)); x.Should().Contain(Varint.Encode(long.MaxValue)); Assert.Equal(long.MaxValue, Varint.DecodeInt64(x)); }
public void Example() { for (long v = 1; v <= 0xFFFFFFFL; v = v << 4) { Console.Write($"| {v} (0x{v.ToString("x")}) "); Console.WriteLine($"| {Varint.Encode(v).ToHexString()} |"); } }
public void MaxLong() { var x = "ffffffffffffffff7f".ToHexBuffer(); Assert.AreEqual(9, Varint.RequiredBytes(long.MaxValue)); CollectionAssert.AreEqual(x, Varint.Encode(long.MaxValue)); Assert.AreEqual(long.MaxValue, Varint.DecodeInt64(x)); }
public async ValueTask SendSessionUpdateWindowSizeMessageAsync(SessionUpdateWindowSizeMessage sessionUpdateWindowSizeMessage, CancellationToken token = default) { await _connection.EnqueueAsync((bufferWriter) => { Varint.SetUInt64((byte)PacketType.Connect, bufferWriter); sessionUpdateWindowSizeMessage.Export(bufferWriter, _bufferPool); }, token); }
private byte[] _NullBuffer() { var idCount = Varint.GetByteCount(0); var buffer = new byte[idCount]; Varint.NumberToBuffer(buffer, 0, 0); return(buffer); }
int ITypeDescriber.ToObject(byte[] buffer, int begin, out object instnace) { ulong value; var readed = Varint.BufferToNumber(buffer, begin, out value); instnace = Enum.ToObject(_Type, value); return(readed); }
public void Decode_From_Offset() { var x = new byte[] { 0x00, 0xAC, 0x02 }; Assert.Equal(300, Varint.DecodeInt32(x, 1)); }
public void TestSerialize() { int origin; Varint varint; byte[] data; int result; Random random = new Random(DateTime.Now.Millisecond); for (int i = 0; i < RANDOM_COUNT; i++) { origin = random.Next(int.MinValue, int.MaxValue); varint = origin; data = varint.Bytes; Varint newOne = new Varint(data,0); result = newOne; Assert.AreEqual(origin,result); } }