public void Send(Stream resultStream)
{
lock (_lockObject)
{
try
{
Varint.SetUInt64(_stream, (ulong)resultStream.Length);
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 32))
{
long remain = resultStream.Length;
while (remain > 0)
{
int readLength = resultStream.Read(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length));
_stream.Write(safeBuffer.Value, 0, readLength);
_stream.Flush();
remain -= readLength;
}
}
}
finally
{
resultStream.Dispose();
}
}
}
public byte[] ObjectToBuffer(object instance)
{
try
{
if (instance == null)
{
return(_NullBuffer());
}
var type = instance.GetType();
var describer = _GetDescriber(type);
var id = describer.Id;
var idCount = Varint.GetByteCount((ulong)id);
var bufferCount = describer.GetByteCount(instance);
var buffer = new byte[idCount + bufferCount];
var readCount = Varint.NumberToBuffer(buffer, 0, id);
describer.ToBuffer(instance, buffer, readCount);
return(buffer);
}
catch (DescriberException ex)
{
if (instance != null)
{
throw new SystemException(string.Format("ObjectToBuffer {0}", instance.GetType()), ex);
}
throw new SystemException("ObjectToBuffer null", ex);
}
}
private void Alive()
{
lock (_lockObject)
{
Varint.SetUInt64(_stream, 0);
}
}
// BlockCount returns the number of timestamps encoded in block.
public static int BlockCount(byte[] bytes)
{
int tsLen, index;
index = 1 + Varint.Read(bytes, 1, out tsLen);
return(TimeDecoder.CountTimestamps(bytes, index, tsLen));
}
private static Stream RemoveHash(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
int type = (int)Varint.GetUInt64(stream);
if (type == (int)ConvertHashAlgorithm.Sha256)
{
var value = new byte[32];
stream.Read(value, 0, value.Length);
var dataStream = new RangeStream(stream, true);
if (!Unsafe.Equals(value, Sha256.Compute(new WrapperStream(dataStream))))
{
throw new ArgumentException("Hash");
}
dataStream.Seek(0, SeekOrigin.Begin);
return(dataStream);
}
else
{
throw new NotSupportedException();
}
}
public static int CountTimestamps(byte[] b, int startindex, int len)
{
if (b == null || b.Length < startindex + len)
{
return(0);
}
// Encoding type is stored in the 4 high bits of the first byte
byte encoding = (byte)(b[startindex] >> 4);
switch (encoding)
{
case TimeEncoder.timeUncompressed:
// Uncompressed timestamps are just 8 bytes each
return((len - 1) / 8);
case TimeEncoder.timeCompressedRLE:
// First 9 bytes are the starting timestamp and scaling factor, skip over them
int j = startindex + 9;
// Next 1-10 bytes is our (scaled down by factor of 10) run length values
ulong value;
j += Varint.Read(b, j, out value);
// Last 1-10 bytes is how many times the value repeats
Varint.Read(b, j, out value);
return((int)value);
case TimeEncoder.timeCompressedPackedSimple:
// First 9 bytes are the starting timestamp and scaling factor, skip over them
var count = Simple8bDecoder.CountBytes(b, startindex + 9, len - 9);
return(count.Item1 + 1); // +1 is for the first uncompressed timestamp, starting timestamep in b[1:9]
default:
return(0);
}
}
// SetBytes initializes the decoder with a new set of bytes to read from.
// This must be called before calling any other methods.
public string SetBytes(byte[] b, int startindex, int len)
{
err = null;
if (b == null || len <= 0 || b.Length - startindex - len < 0)
{
return(null);
}
// First byte stores the encoding type, only have 1 bit-packet format
// currently ignore for now.
ulong boolcount;
int bytecount = Varint.Read(b, startindex + 1, out boolcount);
if (bytecount <= 0)
{
err = "BooleanDecoder: invalid count";
return(err);
}
byteIndex = bytecount + startindex + 1;
i = -1;
n = (int)boolcount;
bytes = b;
int min = len * 8;
if (min < n)
{
// Shouldn't happen - TSM file was truncated/corrupted
n = min;
}
return(null);
}
int ITypeDescriber.ToBuffer(object instance, byte[] buffer, int begin)
{
try
{
var offset = begin;
var validFields = _Fields.Select(
(field, index) => new
{
field,
index
})
.Where(validField => object.Equals(_GetDescriber(validField.field).Default, validField.field.GetValue(instance)) == false)
.ToArray();
offset += Varint.NumberToBuffer(buffer, offset, validFields.Length);
foreach (var validField in validFields)
{
var index = validField.index;
offset += Varint.NumberToBuffer(buffer, offset, index);
var field = validField.field;
var value = field.GetValue(instance);
var describer = _GetDescriber(field);
offset += describer.ToBuffer(value, buffer, offset);
}
return(offset - begin);
}
catch (Exception ex)
{
throw new DescriberException(typeof(ClassDescriber), _Type, _Id, "ToBuffer", ex);
}
}
int ITypeDescriber.GetByteCount(object instance)
{
try
{
var validFields = _Fields.Select(
(field, index) => new
{
field,
index
})
.Where(validField => object.Equals(_GetDescriber(validField.field).Default, validField.field.GetValue(instance)) == false)
.ToArray();
var validCount = Varint.GetByteCount(validFields.Length);
var count = 0;
for (var i = 0; i < validFields.Length; i++)
{
var validField = validFields[i];
var value = validField.field.GetValue(instance);
var describer = _GetDescriber(validField.field);
var byteCount = describer.GetByteCount(value);
var indexCount = Varint.GetByteCount(validField.index);
count += byteCount + indexCount;
}
return(count + validCount);
}
catch (Exception ex)
{
throw new DescriberException(typeof(ClassDescriber), _Type, _Id, "GetByteCount", ex);
}
}
public void PutGivesCorrectOutput()
{
var buf = new byte[Varint.MaxVarintLen];
var n = Varint.PutVarint(buf, 127);
Assert.Equal(1, n);
}
public static bool TryWrite(uint version, ReadOnlySequence <byte> sequence, IBufferWriter <byte> writer)
{
Varint.SetUInt32(version, writer);
Varint.SetUInt32((uint)ConvertCompressionAlgorithm.Brotli, writer);
using var encoder = new BrotliEncoder(0, 10);
var reader = new SequenceReader <byte>(sequence);
for (; ;)
{
var status = encoder.Compress(reader.UnreadSpan, writer.GetSpan(), out var bytesConsumed, out var bytesWritten, false);
if (status == OperationStatus.InvalidData)
{
_logger.Warn("invalid data");
return(false);
}
reader.Advance(bytesConsumed);
writer.Advance(bytesWritten);
if (status == OperationStatus.Done)
{
return(true);
}
}
}
public static ClockValues Decode2(byte[] bytes, int startindex)
{
byte datatype = bytes[startindex];
startindex++;
int index;
index = startindex + Varint.Read(bytes, startindex, out int tsLen);//时标段;
TimeDecoder tdec = (TimeDecoder)DecoderFactory.Get(DataTypeEnum.DateTime);
tdec.SetBytes(bytes, index, tsLen);
index = index + tsLen;
index = index + Varint.Read(bytes, index, out int vsLen);//数值段;
IDecoder vdec = DecoderFactory.Get(datatype);
vdec.SetBytes(bytes, index, vsLen);
index = index + vsLen;
index = index + Varint.Read(bytes, index, out int qsLen);//质量段;
IntegerDecoder qdec = (IntegerDecoder)DecoderFactory.Get(DataTypeEnum.Integer);
qdec.SetBytes(bytes, index, qsLen);
ClockValues result = new ClockValues();
if (datatype == DataTypeEnum.Double)
{
FloatDecoder decoder = (FloatDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
else if (datatype == DataTypeEnum.Boolean)
{
BooleanDecoder decoder = (BooleanDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
else if (datatype == DataTypeEnum.Integer)
{
IntegerDecoder decoder = (IntegerDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
else if (datatype == DataTypeEnum.String)
{
StringDecoder decoder = (StringDecoder)vdec;
while (tdec.Next() && vdec.Next() && qdec.Next())
{
result.Append(decoder.Create(tdec.Read(), decoder.Read(), (int)qdec.Read()));
}
}
DecoderFactory.Put(DataTypeEnum.DateTime, tdec);
DecoderFactory.Put(DataTypeEnum.Integer, qdec);
DecoderFactory.Put(datatype, vdec);
return(result);
}
public static byte[] Encode(IEnumerable <ReadOnlyMemory <byte> > collection)
{
var pipe = new Pipe();
{
foreach (var value in collection)
{
Varint.SetUInt64((uint)value.Length, pipe.Writer);
BytesOperations.Copy(value.Span, pipe.Writer.GetSpan(value.Length), value.Length);
pipe.Writer.Advance(value.Length);
}
pipe.Writer.Complete();
}
byte[] result;
{
pipe.Reader.TryRead(out var readResult);
result = readResult.Buffer.ToArray();
pipe.Reader.Complete();
}
return(result);
}
public async Task Writing()
{
var ms = new MemoryStream();
var muxer = new Muxer {
Channel = ms
};
var stream = new Substream {
Muxer = muxer
};
var m1 = new byte[1];
stream.AddData(new byte[] { 10 });
Assert.IsTrue(stream.CanRead);
Assert.IsTrue(stream.CanWrite);
Assert.AreEqual(1, await stream.ReadAsync(m1, 0, 1));
await stream.WriteAsync(m1, 0, 1);
stream.WriteByte(11);
await stream.FlushAsync();
ms.Position = 0;
var header = await Header.ReadAsync(ms);
var length = await Varint.ReadVarint32Async(ms);
var payload = new byte[length];
ms.Read(payload, 0, length);
Assert.AreEqual(stream.Id, header.StreamId);
Assert.AreEqual(2, payload.Length);
CollectionAssert.AreEqual(new byte[] { 10, 11 }, payload);
}
public (int, string) DecodeAll(Span <byte> src, Span <bool> to)
{
if (src.Length == 0)
{
return(0, null);
}
// First byte stores the encoding type, only have 1 bit-packet format
// currently ignore for now.
src = src.Slice(1);
int nn = Varint.Read(src, out int count);
src = src.Slice(nn);
if (count <= 0)
{
return(0, "booleanBatchDecoder: invalid count");
}
int j = 0;
foreach (byte v in src)
{
for (byte i = 128; i > 0; i >>= 1)
{
to[j++] = (v & i) != 0;
}
}
return(j, null);
}
public void TooBig_Int32()
{
var bytes = Varint.Encode((long)
int.MaxValue + 1);
Assert.Throws <InvalidDataException>(() => Varint.DecodeInt32(bytes));
}
public object BufferToObject(byte[] buffer)
{
ulong id = 0;
try
{
var readIdCount = Varint.BufferToNumber(buffer, 0, out id);
if (id == 0)
{
return(null);
}
var describer = _GetDescriber((int)id);
object instance;
describer.ToObject(buffer, readIdCount, out instance);
return(instance);
}
catch (DescriberException ex)
{
var describer = _GetDescriber((int)id);
if (describer != null)
{
throw new SystemException(string.Format("BufferToObject {0}:{1}", id, describer.Type.FullName), ex);
}
throw new SystemException(string.Format("BufferToObject {0}:unkown", id), ex);
}
}
// Read returns the next value from the decoder.
public string Read()
{
// Read the length of the string
int length;
int n = Varint.Read(bytes, i, out length);
if (n <= 0)
{
err = "StringDecoder: invalid encoded string length";
return(String.Empty);
}
// The length of this string plus the length of the variable byte encoded length
l = length + n;
int lower = i + n;
int upper = lower + length;
if (upper < lower)
{
err = "StringDecoder: length overflow";
return(String.Empty);
}
if (upper > bytes.Length)
{
err = "StringDecoder: not enough data to represent encoded string";
return(String.Empty);
}
return(System.Text.Encoding.Default.GetString(bytes, lower, length));
}
/// <summary>Converts the given value into a byte array</summary>
/// <param name="Receiver">The array that receives the final data.</param>
/// <param name="Value">The object to convert</param>
/// <param name="StartIndex">The startindex for the data</param>
/// <returns>Converts the given value into a byte array</returns>
public static Int32 OntoBytes(Byte[] Receiver, Int32 Value, Int32 StartIndex = 0)
{
if (Receiver is null)
{
throw new ArgumentNullException(nameof(Receiver));
}
Int32 Count = Varint.ByteCount((UInt32)Value);
Int32 Mark = Count - 1 + StartIndex;
if (Mark >= Receiver.Length)
{
throw new ArgumentException($"Receiving byte array is not of an proper length");
}
for (Int32 I = StartIndex; I < Mark; I++)
{
Receiver[I] = (Byte)(Value | Varint._SignalMask);
Value >>= 7;
}
Receiver[Mark] = (Byte)(Value & Varint._NonSignalMask);
return(Count);
}
public override bool TryRead(ref byte[] buf)
{
TypeId = Varint.ReadInt(ref buf);
X = Varint.ReadInt(ref buf);
Z = Varint.ReadInt(ref buf);
return(true);
}
public (ByteWriter, string) EncodingAll(Span <bool> src_span)
{
int srclen = src_span.Length;
int sz = 1 + 8 + (srclen + 7) / 8;//header+num bools+bool data
ByteWriter result = new ByteWriter(sz);
// Store the encoding type in the 4 high bits of the first byte
result.Write((byte)(booleanCompressedBitPacked << 4));
// Encode the number of booleans written.
result.Write(Varint.GetBytes(srclen));
int bitcount = result.Length * 8; // Current bit in current byte.
Span <byte> result_span = new Span <byte>(result.EndWrite()); //取出数组
for (int i = 0; i < srclen; i++)
{
bool v = src_span[i];
if (v)
{
result_span[n >> 3] |= (byte)(128 >> (bitcount & 7));
}// Set current bit on current byte.
else
{
result_span[n >> 3] = (byte)(result_span[n >> 3] & ~(byte)(128 >> (bitcount & 7)));
}// Clear current bit on current byte.
bitcount++;
}
int length = bitcount >> 3;
if ((bitcount & 7) > 0)
{
length++;// Add an extra byte to capture overflowing bits.
}
result.Length = length;
return(result, null);
}
public (int, string) DecodeAll(Span <byte> b, Span <bool> dst)
{
if (b.Length == 0)
{
return(0, null);
}
// First byte stores the encoding type, only have 1 bit-packet format
// currently ignore for now.
b = b.Slice(1);
int n = Varint.Read(b, out int count);
if (n <= 0)
{
return(0, "booleanBatchDecoder: invalid count");
}
b = b.Slice(n);
int j = 0;
foreach (byte v in b)
{
for (byte i = 128; i > 0 && j < dst.Length; i >>= 1)
{
dst[j++] = (v & i) != 0;
}
}
return(j, null);
}
public override byte[] ToByteArray()
{
return(ArrayBuilder.Build <byte>(
Varint.Encode(TypeId),
Varint.Encode(X),
Varint.Encode(Z)
));
}
public void MaxLong()
{
var x = "ffffffffffffffff7f".ToHexBuffer();
Assert.Equal(9, Varint.RequiredBytes(long.MaxValue));
x.Should().Contain(Varint.Encode(long.MaxValue));
Assert.Equal(long.MaxValue, Varint.DecodeInt64(x));
}
public void Example()
{
for (long v = 1; v <= 0xFFFFFFFL; v = v << 4)
{
Console.Write($"| {v} (0x{v.ToString("x")}) ");
Console.WriteLine($"| {Varint.Encode(v).ToHexString()} |");
}
}
public void MaxLong()
{
var x = "ffffffffffffffff7f".ToHexBuffer();
Assert.AreEqual(9, Varint.RequiredBytes(long.MaxValue));
CollectionAssert.AreEqual(x, Varint.Encode(long.MaxValue));
Assert.AreEqual(long.MaxValue, Varint.DecodeInt64(x));
}
public async ValueTask SendSessionUpdateWindowSizeMessageAsync(SessionUpdateWindowSizeMessage sessionUpdateWindowSizeMessage, CancellationToken token = default)
{
await _connection.EnqueueAsync((bufferWriter) =>
{
Varint.SetUInt64((byte)PacketType.Connect, bufferWriter);
sessionUpdateWindowSizeMessage.Export(bufferWriter, _bufferPool);
}, token);
}
private byte[] _NullBuffer()
{
var idCount = Varint.GetByteCount(0);
var buffer = new byte[idCount];
Varint.NumberToBuffer(buffer, 0, 0);
return(buffer);
}
int ITypeDescriber.ToObject(byte[] buffer, int begin, out object instnace)
{
ulong value;
var readed = Varint.BufferToNumber(buffer, begin, out value);
instnace = Enum.ToObject(_Type, value);
return(readed);
}
public void Decode_From_Offset()
{
var x = new byte[]
{
0x00, 0xAC, 0x02
};
Assert.Equal(300, Varint.DecodeInt32(x, 1));
}
public void TestSerialize()
{
int origin;
Varint varint;
byte[] data;
int result;
Random random = new Random(DateTime.Now.Millisecond);
for (int i = 0; i < RANDOM_COUNT; i++)
{
origin = random.Next(int.MinValue, int.MaxValue);
varint = origin;
data = varint.Bytes;
Varint newOne = new Varint(data,0);
result = newOne;
Assert.AreEqual(origin,result);
}
}
