public override void Serialize(RecyclableMemoryStream target)
{
base.Serialize(target);
target.Write((ushort)ContentTypeLength);
if (ContentTypeLength > 0)
{
target.Write(ContentType, 0, ContentTypeLength);
}
target.Write(checked ((short)ContentLength.Length));
foreach (var i in ContentLength)
{
target.Write7BitInt(i);
}
}
/// <summary>
/// Deserializes from byte array
/// </summary>
/// <typeparam name="T">Type to deserialize into</typeparam>
/// <param name="value">Byte-array to be deserialized</param>
/// <param name="gzipDecompress">Use gzip decompression, if your data is serialized with gzip</param>
/// <returns>Byte-array deserialized into type</returns>
public T FromByteArray <T>(
[NotNull] byte[] value,
bool gzipDecompress = false)
{
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
var manager = new RecyclableMemoryStreamManager();
using (var ms = new RecyclableMemoryStream(manager, "mytag"))//new MemoryStream(value)
{
ms.Write(value, 0, value.Length);
ms.Position = 0;
if (gzipDecompress)
{
using (var gzip = new GZipStream(ms, CompressionMode.Decompress, true))
{
return(Serializer.Deserialize <T>(gzip));
}
}
return(Serializer.Deserialize <T>(ms));
}
}
public RecyclableMemoryStream Add(string filename, byte[] bytes, int startIndex, int length)
{
RecyclableMemoryStream rms = new RecyclableMemoryStream(filename);
rms.Write(bytes, startIndex, length);
rms.Position = 0;
return(this.Add(filename, rms));
}
public void RecyclableMemoryStream()
{
using (var stream = new RecyclableMemoryStream(this.memoryManager))
{
for (int position = 0; position < this.Length; position += blockSize)
{
stream.Write(this.data, position, Math.Min(blockSize, this.Length - position));
}
}
}
private void EventLoop(CancellationToken token)
{
try
{
for (; ;)
{
token.ThrowIfCancellationRequested();
long length = (long)Varint.GetUInt64(_stream);
if (length == 0)
{
continue;
}
Stream resultSteram = null;
try
{
resultSteram = new RecyclableMemoryStream(_bufferManager);
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 32))
{
long remain = length;
while (remain > 0)
{
int readLength = _stream.Read(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length));
resultSteram.Write(safeBuffer.Value, 0, readLength);
remain -= readLength;
}
}
resultSteram.Seek(0, SeekOrigin.Begin);
this.OnReceive(resultSteram);
}
catch (Exception)
{
if (resultSteram != null)
{
resultSteram.Dispose();
resultSteram = null;
}
}
}
}
catch (Exception)
{
}
}
public void MessageUpload()
{
_callback.Invoke("----- CoreManager Message Upload Test (Start) -----");
_callback.Invoke("");
var random = RandomProvider.GetThreadRandom();
_coreManager.Resize((long)1024 * 1024 * 1024 * 32).Wait();
Metadata metadata = null;
Hash hash;
using (var stream = new RecyclableMemoryStream(_bufferManager))
{
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
for (long remain = (long)1024 * 1024 * 256; 0 < remain; remain -= safeBuffer.Value.Length)
{
int length = (int)Math.Min(remain, safeBuffer.Value.Length);
random.NextBytes(safeBuffer.Value);
stream.Write(safeBuffer.Value, 0, length);
}
}
stream.Seek(0, SeekOrigin.Begin);
hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(new WrapperStream(stream)));
stream.Seek(0, SeekOrigin.Begin);
metadata = _coreManager.VolatileSetStream(stream, new TimeSpan(1, 0, 0, 0), CancellationToken.None).Result;
}
using (var stream = metadata.Export(_bufferManager))
using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length))
{
stream.Read(safeBuffer.Value, 0, (int)stream.Length);
Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length));
}
using (var stream = hash.Export(_bufferManager))
using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length))
{
stream.Read(safeBuffer.Value, 0, (int)stream.Length);
Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length));
}
_callback.Invoke("----- CoreManager Message Upload Test (End) -----");
}
internal static byte[] UncompressBuffer(byte[] compressed)
{
using (var input = new MemoryStream(compressed))
using (var output = new RecyclableMemoryStream())
using (var decompressor = new ZlibStream(input, CompressionMode.Decompress))
using (Leayal.ByteBuffer buffer = new Leayal.ByteBuffer(1024))
{
int read = decompressor.Read(buffer, 0, buffer.Length);
while (read > 0)
{
output.Write(buffer, 0, read);
read = decompressor.Read(buffer, 0, buffer.Length);
}
return(output.ToArray());
}
}
private static Stream AddPadding(Stream stream, int size)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
try
{
var lengthStream = new RecyclableMemoryStream(_bufferManager);
Varint.SetUInt64(lengthStream, (ulong)stream.Length);
Stream paddingStream;
{
using (var random = RandomNumberGenerator.Create())
{
int paddingLength = size - (int)(lengthStream.Length + stream.Length);
paddingStream = new RecyclableMemoryStream(_bufferManager);
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
while (paddingLength > 0)
{
int writeSize = Math.Min(paddingLength, safeBuffer.Value.Length);
random.GetBytes(safeBuffer.Value);
paddingStream.Write(safeBuffer.Value, 0, writeSize);
paddingLength -= writeSize;
}
}
}
}
return(new UniteStream(lengthStream, stream, paddingStream));
}
catch (Exception e)
{
if (stream != null)
{
stream.Dispose();
}
throw new ArgumentException(e.Message, e);
}
}
/// <summary>
/// Retrieve a new MemoryStream object with the given tag and with contents copied from the provided
/// buffer. The provided buffer is not wrapped or used after construction.
/// </summary>
/// <remarks>The new stream's position is set to the beginning of the stream when returned.</remarks>
/// <param name="id">A unique identifier which can be used to trace usages of the stream.</param>
/// <param name="tag">A tag which can be used to track the source of the stream.</param>
/// <param name="buffer">The byte buffer to copy data from.</param>
/// <param name="offset">The offset from the start of the buffer to copy from.</param>
/// <param name="count">The number of bytes to copy from the buffer.</param>
/// <returns>A MemoryStream.</returns>
public RecyclableMemoryStream GetStream(Guid id, object tag, byte[] buffer, int offset, int count)
{
RecyclableMemoryStream stream = null;
try
{
stream = new RecyclableMemoryStream(this, id, tag, count);
stream.Write(buffer, offset, count);
stream.Position = 0;
return(stream);
}
catch
{
stream?.Dispose();
throw;
}
}
/// <summary>
/// Retrieve a new MemoryStream object with the given tag and with contents copied from the provided
/// buffer. The provided buffer is not wrapped or used after construction.
/// </summary>
/// <remarks>The new stream's position is set to the beginning of the stream when returned.</remarks>
/// <param name="id">A unique identifier which can be used to trace usages of the stream.</param>
/// <param name="tag">A tag which can be used to track the source of the stream.</param>
/// <param name="buffer">The byte buffer to copy data from.</param>
/// <returns>A MemoryStream.</returns>
public MemoryStream GetStream(Guid id, string tag, Memory <byte> buffer)
{
RecyclableMemoryStream stream = null;
try
{
stream = new RecyclableMemoryStream(this, id, tag, buffer.Length);
stream.Write(buffer.Span);
stream.Position = 0;
return(stream);
}
catch
{
stream?.Dispose();
throw;
}
}
private static Stream AddHash(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
var hashStream = new RecyclableMemoryStream(_bufferManager);
{
Varint.SetUInt64(hashStream, (uint)ConvertHashAlgorithm.Sha256);
var value = Sha256.Compute(new WrapperStream(stream));
hashStream.Write(value, 0, value.Length);
}
return(new UniteStream(hashStream, stream));
}
private Stream GetStream(string url)
{
var recyclableMemoryStream = new RecyclableMemoryStream(_bufferManager);
try
{
using (var client = new HttpClient())
{
using (var stream = client.GetStreamAsync(url).Result)
{
if (stream.Length > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
recyclableMemoryStream.Write(safeBuffer.Value, 0, length);
if (recyclableMemoryStream.Length > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
}
}
recyclableMemoryStream.Seek(0, SeekOrigin.Begin);
return(recyclableMemoryStream);
}
}
}
catch (Exception)
{
recyclableMemoryStream.Dispose();
throw;
}
}
public void PublicMethodThrowAfterDispose()
{
var stream = new RecyclableMemoryStream();
stream.Dispose();
byte[] buffer = new byte[1];
Assert.That(
() => stream.SetLength(10),
Throws.InstanceOf <ObjectDisposedException>());
Assert.That(
() => stream.WriteByte(0x00),
Throws.InstanceOf <ObjectDisposedException>());
Assert.That(
() => stream.Write(buffer, 0, 1),
Throws.InstanceOf <ObjectDisposedException>());
Assert.That(
() => stream.ReadByte(),
Throws.InstanceOf <ObjectDisposedException>());
Assert.That(
() => stream.Read(buffer, 0, 1),
Throws.InstanceOf <ObjectDisposedException>());
}
public void ToStringReturnsHelpfulDebugInfo()
{
var tag = "Nunit test";
var stream = new RecyclableMemoryStream(this.GetMemoryManager(), tag);
var buffer = this.GetRandomBuffer(1000);
stream.Write(buffer, 0, buffer.Length);
var debugInfo = stream.ToString();
Assert.That(debugInfo, Contains.Substring(stream.Id.ToString()));
Assert.That(debugInfo, Contains.Substring(tag));
Assert.That(debugInfo, Contains.Substring(buffer.Length.ToString("N0")));
}
private static Stream Decrypt(Stream stream, ExchangePrivateKey privateKey)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (privateKey == null)
{
throw new ArgumentNullException(nameof(privateKey));
}
try
{
int type = (int)Varint.GetUInt64(stream);
if (type == (int)ConvertCryptoAlgorithm.Aes256)
{
byte[] cryptoKey;
{
int length = (int)Varint.GetUInt64(stream);
var encryptedBuffer = new byte[length];
if (stream.Read(encryptedBuffer, 0, encryptedBuffer.Length) != encryptedBuffer.Length)
{
throw new ArgumentException();
}
cryptoKey = Exchange.Decrypt(privateKey, encryptedBuffer);
}
var iv = new byte[32];
stream.Read(iv, 0, iv.Length);
RecyclableMemoryStream outStream = null;
try
{
outStream = new RecyclableMemoryStream(_bufferManager);
using (var aes = Aes.Create())
{
aes.KeySize = 256;
aes.Mode = CipherMode.CBC;
aes.Padding = PaddingMode.PKCS7;
using (var inStream = new RangeStream(stream, stream.Position, stream.Length - stream.Position, true))
using (var cs = new CryptoStream(inStream, aes.CreateDecryptor(cryptoKey, iv), CryptoStreamMode.Read))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = cs.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
outStream.Write(safeBuffer.Value, 0, length);
}
}
}
outStream.Seek(0, SeekOrigin.Begin);
}
catch (Exception)
{
if (outStream != null)
{
outStream.Dispose();
}
throw;
}
return(outStream);
}
else
{
throw new NotSupportedException();
}
}
catch (Exception e)
{
throw new ArgumentException(e.Message, e);
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
private static Stream Encrypt(Stream stream, ExchangePublicKey publicKey)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (publicKey == null)
{
throw new ArgumentNullException(nameof(publicKey));
}
try
{
RecyclableMemoryStream outStream = null;
try
{
outStream = new RecyclableMemoryStream(_bufferManager);
Varint.SetUInt64(outStream, (uint)ConvertCryptoAlgorithm.Aes256);
var cryptoKey = new byte[32];
var iv = new byte[32];
using (var random = RandomNumberGenerator.Create())
{
random.GetBytes(cryptoKey);
random.GetBytes(iv);
}
{
var encryptedBuffer = Exchange.Encrypt(publicKey, cryptoKey);
Varint.SetUInt64(outStream, (uint)encryptedBuffer.Length);
outStream.Write(encryptedBuffer, 0, encryptedBuffer.Length);
}
outStream.Write(iv, 0, iv.Length);
using (var aes = Aes.Create())
{
aes.KeySize = 256;
aes.Mode = CipherMode.CBC;
aes.Padding = PaddingMode.PKCS7;
using (var inStream = new WrapperStream(stream))
using (var cs = new CryptoStream(inStream, aes.CreateEncryptor(cryptoKey, iv), CryptoStreamMode.Read))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = cs.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
outStream.Write(safeBuffer.Value, 0, length);
}
}
}
outStream.Seek(0, SeekOrigin.Begin);
}
catch (Exception)
{
if (outStream != null)
{
outStream.Dispose();
}
throw;
}
return(outStream);
}
catch (Exception e)
{
throw new ArgumentException(e.Message, e);
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
private static Stream Decompress(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
try
{
stream.Seek(0, SeekOrigin.Begin);
int type = (int)Varint.GetUInt64(stream);
if (type == (int)ConvertCompressionAlgorithm.None)
{
return(new RangeStream(stream));
}
else if (type == (int)ConvertCompressionAlgorithm.Deflate)
{
RecyclableMemoryStream deflateBufferStream = null;
try
{
deflateBufferStream = new RecyclableMemoryStream(_bufferManager);
using (var deflateStream = new DeflateStream(stream, CompressionMode.Decompress))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = deflateStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
deflateBufferStream.Write(safeBuffer.Value, 0, length);
if (deflateBufferStream.Length > 1024 * 1024 * 256)
{
throw new Exception("too large");
}
}
}
deflateBufferStream.Seek(0, SeekOrigin.Begin);
return(deflateBufferStream);
}
catch (Exception)
{
if (deflateBufferStream != null)
{
deflateBufferStream.Dispose();
}
throw;
}
}
else
{
throw new ArgumentException("ArgumentException");
}
}
catch (Exception e)
{
if (stream != null)
{
stream.Dispose();
}
throw new ArgumentException(e.Message, e);
}
}
private static T FromStream <T>(int version, Stream stream)
where T : MessageBase <T>
{
try
{
stream.Seek(0, SeekOrigin.Begin);
// Check
{
var verifyCrc = Crc32_Castagnoli.Compute(new RangeStream(stream, 0, stream.Length - 4, true));
var orignalCrc = new byte[4];
using (var crcStream = new RangeStream(stream, stream.Length - 4, 4, true))
{
crcStream.Read(orignalCrc, 0, orignalCrc.Length);
}
if (!Unsafe.Equals(verifyCrc, orignalCrc))
{
throw new ArgumentException("Crc Error");
}
}
stream.Seek(0, SeekOrigin.Begin);
if (version != (int)Varint.GetUInt64(stream))
{
throw new ArgumentException("version");
}
int type = (int)Varint.GetUInt64(stream);
using (var dataStream = new RangeStream(stream, stream.Position, stream.Length - stream.Position - 4, true))
{
if (type == (int)ConvertCompressionAlgorithm.None)
{
return(MessageBase <T> .Import(dataStream, _bufferManager));
}
else if (type == (int)ConvertCompressionAlgorithm.Deflate)
{
using (var deflateBufferStream = new RecyclableMemoryStream(_bufferManager))
{
using (var deflateStream = new DeflateStream(dataStream, CompressionMode.Decompress, true))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = deflateStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
deflateBufferStream.Write(safeBuffer.Value, 0, length);
if (deflateBufferStream.Length > 1024 * 1024 * 32)
{
throw new Exception("too large");
}
}
}
deflateBufferStream.Seek(0, SeekOrigin.Begin);
return(MessageBase <T> .Import(deflateBufferStream, _bufferManager));
}
}
else
{
throw new ArgumentException("ArgumentException");
}
}
}
catch (Exception e)
{
throw new ArgumentException(e.Message, e);
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
private int CreateHeader(out int contentLength)
{
contentLength = 0;
if (_header is ContentHeader content)
{
if (_message is Stream)
{
throw new NotSupportedException();
}
if (_message is byte[] buf)
{
throw new NotSupportedException();
}
_bodyStream = _contentStream;
var pos = _bodyStream.Position;
if (_message is IDynamicPayload dp)
{
content.PacketFlag |= PacketFlag.MultiContent;
var obj = dp.Deconstruct();
content.ContentLength = new int[obj.Length];
byte[] ct = null;
for (var i = 0; i < obj.Length; i++)
{
var cur = _bodyStream.Position;
Serializer.Serialize(obj[i], _bodyStream, out ct);
content.ContentLength[i] = checked ((int)(_bodyStream.Position - cur));
}
content.ContentType = ct;
}
else
{
byte[] contentType = ContentHeader.EmptyContent;
if (!content.PacketFlag.HasFlag(PacketFlag.NoContent))
{
Serializer.Serialize(_message, _bodyStream, out contentType);
}
content.ContentType = contentType;
content.ContentLength = new int[1] {
checked ((int)(_bodyStream.Position - pos))
};
}
_bodyStream.Position = pos;
contentLength = checked ((int)(_bodyStream.Length - _bodyStream.Position));
if (contentLength == 0)
{
content.PacketFlag |= PacketFlag.NoContent;
}
if (content.ContentLength.Length > 1)
{
content.PacketFlag |= PacketFlag.MultiContent;
}
}
else if (_header is RawDataHeader raw)
{
_bodyStream = (Stream)_message;
if (raw.ContentLength <= 0)
{
raw.ContentLength = checked ((int)(_bodyStream.Length - _bodyStream.Position));
}
contentLength = raw.ContentLength;
_disposeBodyStream = raw.DisposeStreamAfterSend;
}
_headerStream.TryExtend(512);
_headerStream.Position = sizeof(short);
_headerStream.Write(Version);
_header.Serialize(_headerStream);
var len = (short)_headerStream.Position;
if (_headerStream.Position > ushort.MaxValue)
{
throw new InvalidDataException("Invalid header");
}
_headerStream.Position = 0;
_headerStream.Write(len);
return(len);
}
public void EncodeBuffer(byte[] value, int start, int length)
{
ThrowIfDisposed();
m_stream.Write(value, start, length);
}
public void GetBufferAdjustsLargePoolFreeSize()
{
var stream = this.GetDefaultStream();
var memMgr = stream.MemoryManager;
var bufferLength = stream.MemoryManager.BlockSize * 4;
var buffer = this.GetRandomBuffer(bufferLength);
stream.Write(buffer, 0, buffer.Length);
var newBuffer = stream.GetBuffer();
stream.Dispose();
Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(newBuffer.Length));
var newStream = new RecyclableMemoryStream(memMgr);
newStream.Write(buffer, 0, buffer.Length);
var newBuffer2 = newStream.GetBuffer();
Assert.That(newBuffer2.Length, Is.EqualTo(newBuffer.Length));
Assert.That(memMgr.LargePoolFreeSize, Is.EqualTo(0));
}
public void WriteUpdatesLengthAndPosition()
{
const int expectedLength = 100;
var memoryManager = this.GetMemoryManager();
var stream = new RecyclableMemoryStream(memoryManager, string.Empty, expectedLength);
var buffer = this.GetRandomBuffer(expectedLength);
stream.Write(buffer, 0, buffer.Length);
Assert.That(stream.Length, Is.EqualTo(expectedLength));
Assert.That(stream.Position, Is.EqualTo(expectedLength));
}
public void ToStringWithNullTagIsOk()
{
var stream = new RecyclableMemoryStream(this.GetMemoryManager(), null);
var buffer = this.GetRandomBuffer(1000);
stream.Write(buffer, 0, buffer.Length);
var debugInfo = stream.ToString();
Assert.That(debugInfo, Contains.Substring(stream.Id.ToString()));
Assert.That(debugInfo, Contains.Substring(buffer.Length.ToString("N0")));
}
public override void Serialize(RecyclableMemoryStream target)
{
base.Serialize(target);
target.Write(RequestId);
}
