public void Uncompress_with_advanced_parameters_should_throw_if_parameters_incorrect(bool?provideCorrectCompressedBytes, int inputOffset, int inputLength, int?outputCount, int outputOffset, Type expectedException)
{
var uncompressedBytes = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); // 20 bytes
byte[] compressedBytes = null;
if (provideCorrectCompressedBytes.HasValue)
{
if (provideCorrectCompressedBytes.Value)
{
compressedBytes = SnappyCodec.Compress(uncompressedBytes);
}
else
{
compressedBytes = new byte[inputLength];
new Random(0).NextBytes(compressedBytes);
}
}
var outputBytes = outputCount.HasValue ? new byte[outputCount.Value] : null;
var outputLength = (outputBytes?.Length ?? 0) - outputOffset;
var exception = Record.Exception(() => SnappyCodec.Uncompress(compressedBytes, inputOffset, inputLength, outputBytes, outputOffset, outputLength));
exception.Should().BeOfType(expectedException);
}
public void CompressUncompressEmpty()
{
var compressed = SnappyCodec.Compress(new byte[0]);
Assert.That(compressed.Length, Is.GreaterThan(0));
Assert.AreEqual(0, SnappyCodec.Uncompress(compressed).Length);
}
private void SendMetricRequest(WriteRequest req)
{
var url = Configuration.GetValue <string>("prom_write_url");
var token = Configuration.GetValue <string>("prom_write_token");
if (!String.IsNullOrEmpty(url))
{
MemoryStream stream = new MemoryStream();
req.WriteTo(stream);
byte[] arrIn = stream.ToArray();
byte[] arr = SnappyCodec.Compress(arrIn);
var handler = new HttpClientHandler();
var httpClient = new HttpClient(handler);
HttpRequestMessage requestMessage = new HttpRequestMessage(new HttpMethod("POST"), url);
requestMessage.Headers.Add("Authorization", $"Bearer {token}");
requestMessage.Content = new ByteArrayContent(arr);
requestMessage.Content.Headers.ContentType = new MediaTypeHeaderValue("application/x-protobuf");
requestMessage.Content.Headers.ContentEncoding.Add("snappy");
HttpResponseMessage response = httpClient.SendAsync(requestMessage).Result;
}
}
void Flush(byte[] buff, int offset, int count)
{
var compressedSize = SnappyCodec.Compress(buff, offset, count, _compressedBuffer, 0);
BigEndianConverter.Write(_base, compressedSize);
_base.Write(_compressedBuffer, 0, compressedSize);
}
public void CompressUncompressEmpty()
{
var compressed = SnappyCodec.Compress(new byte[0]);
Assert.True(compressed.Length > 0);
Assert.Equal(0, SnappyCodec.Uncompress(compressed).Length);
}
private byte[] GetCompressed(byte[] buffer)
{
switch (Compression)
{
case CompressionType.None:
return(buffer);
case CompressionType.ZLib:
return(ZlibStream.CompressBuffer(buffer));
case CompressionType.Snappy:
return(SnappyCodec.Compress(buffer));
case CompressionType.Doboz:
return(DobozCodec.Encode(buffer, 0, buffer.Length));
case CompressionType.LZ4:
return(LZ4.LZ4Codec.Encode(buffer, 0, buffer.Length));
case CompressionType.LZ4HC:
return(LZ4.LZ4Codec.EncodeHC(buffer, 0, buffer.Length));
default:
throw new MissingCompressionException("Unhandled compression algorithm.")
{
};
}
}
public string Compress(object obj)
{
byte[] array = JsonSerializer.Serialize(obj);
byte[] compressed = SnappyCodec.Compress(array);
string str = Convert.ToBase64String(compressed, 0, compressed.Length, Base64FormattingOptions.None);
return(str);
}
public void CompressRange()
{
var input = Encoding.ASCII.GetBytes("ByeHelloBye");
var output = new byte[100];
var length = SnappyCodec.Compress(input, 3, 5, output, 10);
Assert.AreEqual("Hello", Encoding.ASCII.GetString(SnappyCodec.Uncompress(output.Skip(10).Take(length).ToArray())));
}
public void Validate_with_advanced_options_should_throw_if_parameters_incorrect(bool provideCompressedBytes, int inputOffset, int inputLength, Type expectedExceptionType)
{
var uncompressedBytes = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); // 20 bytes
var compressedBytes = provideCompressedBytes ? SnappyCodec.Compress(uncompressedBytes) : null; // 22 bytes
var exception = Record.Exception(() => SnappyCodec.Validate(compressedBytes, inputOffset, inputLength));
exception.Should().BeOfType(expectedExceptionType);
}
protected override void Encode(IChannelHandlerContext context, Packet message, List <object> output)
{
if (_logger.IsTrace)
{
_logger.Trace($"Compressing with Snappy a message of length {message.Data.Length}");
}
message.Data = SnappyCodec.Compress(message.Data);
output.Add(message);
}
public void Compress_with_advanced_options_should_throw_if_parameters_incorrect(int?inputCount, int inputOffset, int inputLength, int?outputCount, int outputOffset, Type expectedException)
{
var input = inputCount.HasValue ? new byte[inputCount.Value] : null;
var output = outputCount.HasValue ? new byte[outputCount.Value] : null;
var outputLength = (output?.Length ?? 0) - outputOffset;
var exception = Record.Exception(() => SnappyCodec.Compress(input, inputOffset, inputLength, output, outputOffset, outputLength));
exception.Should().BeOfType(expectedException);
}
public void UncompressRange()
{
var howdy = Encoding.ASCII.GetBytes("Howdy");
var padded = howdy.Take(3).Concat(SnappyCodec.Compress(Encoding.ASCII.GetBytes("Hello"))).Concat(howdy.Skip(3)).ToArray();
var output = new byte[100];
var length = SnappyCodec.Uncompress(padded, 3, padded.Length - 5, output, 10);
Assert.AreEqual(5, length);
Assert.AreEqual("Hello", Encoding.ASCII.GetString(output.Skip(10).Take(5).ToArray()));
}
public byte[] Serialize <T>(T instance)
{
if (instance == null)
{
throw new ArgumentNullException(nameof(instance));
}
var data = instance.XmlSerialize().ToString().GetBytes();
return(compressed ? SnappyCodec.Compress(data) : data);
}
public void CompressDecompress(int length)
{
byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
SnappyCodec codec = new SnappyCodec();
byte[] compressed = codec.Compress(data);
byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
}
public void Compress_should_uncompress_previously_compressed_message(string input)
{
var inputBytes = Encoding.ASCII.GetBytes(input);
var compressedBytes = SnappyCodec.Compress(inputBytes);
compressedBytes.Length.Should().BeGreaterThan(0);
var uncompressedBytes = SnappyCodec.Uncompress(compressedBytes);
var outputString = Encoding.ASCII.GetString(uncompressedBytes);
outputString.Should().Be(input);
}
public void ValidateExceptions()
{
var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?");
var compressed = SnappyCodec.Compress(uncompressed);
var buffer = new byte[100];
Assert.Throws <ArgumentNullException>(() => SnappyCodec.Validate(null));
Assert.Throws <ArgumentNullException>(() => SnappyCodec.Validate(null, 0, 3));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Validate(compressed, -1, uncompressed.Length));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Validate(compressed, 0, -1));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Validate(compressed, compressed.Length - 2, 4));
}
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info)
{
if (info.CompressionCodec != CompressionCodec.None)
{
stream.Write(Basics.Zero64, 0, 8);
using (var msgsetStream = stream.Pool.Reserve())
{
SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion);
using (var compressed = stream.Pool.Reserve())
{
if (info.CompressionCodec == CompressionCodec.Gzip)
{
using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true))
{
msgsetStream.WriteTo(gzip);
}
}
else // Snappy
{
#if NETSTANDARD1_3
throw new NotImplementedException();
#else
compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length));
{
int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0, (int)msgsetStream.Length,
compressed.GetBuffer(), 0);
compressed.SetLength(size);
}
#endif
}
var m = new Message
{
Value = compressed,
TimeStamp = Timestamp.Now
};
Basics.WriteSizeInBytes(stream, m,
new SerializationInfo
{
Serializers = SerializationConfig.ByteArraySerializers,
CompressionCodec = info.CompressionCodec,
MessageVersion = info.MessageVersion
}, SerializeMessageWithCodec);
}
}
}
else
{
SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion);
}
}
public byte[] Compress(byte[] data)
{
byte[] result = null;
try
{
result = SnappyCodec.Compress(data);
}
catch (Exception ex)
{
Console.WriteLine($"EXCEPTION: {ex.GetType().Name}, {ex.Message}");
}
return(result);
}
/// <summary>Compresses the specified input file.</summary>
/// <param name="input">The input file.</param>
/// <param name="output">The output file.</param>
private static void Compress(string input, string output)
{
byte[] original = File.ReadAllBytes(input);
var timer = Stopwatch.StartNew();
var compressed = SnappyCodec.Compress(original, 0, original.Length);
timer.Stop();
Console.WriteLine("Compression:");
Console.WriteLine(" Speed: {0:0.00}MB/s", (double)original.Length / 1024 / 1024 / timer.Elapsed.TotalSeconds);
Console.WriteLine(" Ratio: {0:0.00}%", (double)compressed.Length * 100 / original.Length);
Console.WriteLine(" Hash: {0}", original.MD5());
File.WriteAllBytes(output, compressed);
}
/// <summary>
/// Compresses the remainder of <paramref name="input"/>, writing the compressed data to
/// <paramref name="output"/>.
/// </summary>
/// <param name="input"> The input stream.</param>
/// <param name="output">The output stream.</param>
public void Compress(Stream input, Stream output)
{
#if NET452 || NETSTANDARD2_0
var uncompressedSize = (int)(input.Length - input.Position);
var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers
input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None);
var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize);
var compressedBytes = new byte[maxCompressedSize];
var compressedSize = SnappyCodec.Compress(uncompressedBytes, 0, uncompressedSize, compressedBytes, 0);
output.Write(compressedBytes, 0, compressedSize);
#else
throw new NotSupportedException();
#endif
}
public void Compress_with_advanced_options_should_work_as_expected(int inputOffset, int inputLength, int outputOffset, string expectedResult)
{
var input = Encoding.ASCII.GetBytes("ByeHelloBye");
var output = new byte[100];
var outputLength = output.Length - outputOffset;
var compressedLength = SnappyCodec.Compress(input, inputOffset, inputLength, output, outputOffset, outputLength);
var compressedOutputWithoutOffset = output.Skip(outputOffset).Take(compressedLength).ToArray();
var uncompressed = SnappyCodec.Uncompress(compressedOutputWithoutOffset);
var uncompressedString = Encoding.ASCII.GetString(uncompressed);
uncompressedString.Should().Be(expectedResult);
}
public void Validate()
{
var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?");
var compressed = SnappyCodec.Compress(uncompressed);
var buffer = new byte[100];
Assert.IsTrue(SnappyCodec.Validate(compressed));
var rubbish = new byte[10];
new Random(0).NextBytes(rubbish);
Assert.IsFalse(SnappyCodec.Validate(rubbish, 0, rubbish.Length));
Assert.IsFalse(SnappyCodec.Validate(compressed, 0, 0));
Assert.IsFalse(SnappyCodec.Validate(compressed, compressed.Length, 0));
}
public void Validation(string name)
{
Benchmark.Run("Validating", name, benchmark =>
{
var compressed = SnappyCodec.Compress(benchmark.Input);
bool ok = false;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
ok = SnappyCodec.Validate(compressed);
}
benchmark.Stopwatch.Stop();
Assert.IsTrue(ok);
});
}
public void Uncompression(string name)
{
Benchmark.Run("Uncompressing", name, benchmark =>
{
var compressed = SnappyCodec.Compress(benchmark.Input);
var roundtrip = new byte[benchmark.Input.Length];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
length = SnappyCodec.Uncompress(compressed, 0, compressed.Length, roundtrip, 0);
}
benchmark.Stopwatch.Stop();
CollectionAssert.AreEqual(benchmark.Input, roundtrip);
});
}
public void CompressExceptions()
{
var input = new byte[100];
var output = new byte[100];
Assert.Throws <ArgumentNullException>(() => SnappyCodec.Compress(null));
Assert.Throws <ArgumentNullException>(() => SnappyCodec.Compress(null, 0, 3, output, 0));
Assert.Throws <ArgumentNullException>(() => SnappyCodec.Compress(input, 0, 3, null, 0));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, -1, 3, output, 0));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, -1, output, 0));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 90, 20, output, 0));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 3, output, -1));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 3, output, 100));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 3, output, 101));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 100, new byte[3], 0));
}
public void CompressDecompressStream(int length)
{
byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray();
SnappyCodec codec = new SnappyCodec();
using (MemoryStream inputStream = new MemoryStream(data))
using (MemoryStream outputStream = new MemoryStream())
{
codec.Compress(inputStream, outputStream);
byte[] compressed = outputStream.ToArray();
byte[] uncompressed = codec.Decompress(compressed, compressed.Length);
Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed));
}
}
public void GetUncompressedLengthExceptions()
{
var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?");
var compressed = SnappyCodec.Compress(uncompressed);
var buffer = new byte[100];
Assert.Throws <ArgumentNullException>(() => SnappyCodec.GetUncompressedLength(null));
Assert.Throws <ArgumentNullException>(() => SnappyCodec.GetUncompressedLength(null, 0, 3));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, -1, uncompressed.Length));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, 0, -1));
Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, compressed.Length - 2, 4));
Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(compressed, 0, 0));
Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(compressed, compressed.Length, 0));
var rubbish = Enumerable.Repeat((byte)0xff, 10).ToArray();
Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(rubbish, 0, rubbish.Length));
}
public void Evaluate(int spreadMax)
{
if (FStreamIn.SliceCount == 0 || FStreamIn[0] == null || FStreamIn[0].Length == 0)
{
spreadMax = 0;
}
else
{
spreadMax = FStreamIn.SliceCount;
}
FStreamOut.ResizeAndDispose(spreadMax, () => new MemoryStream());
for (int i = 0; i < spreadMax; i++)
{
var inputStream = FStreamIn[i];
var outputStream = FStreamOut[i];
inputStream.Position = 0;
outputStream.Position = 0;
FStreamOut[i].SetLength(0);
var length = (int)inputStream.Length;
IntPtr contentPtr = Marshal.AllocHGlobal(length);
byte[] memory = new byte[length];
inputStream.Read(memory, 0, length);
Marshal.Copy(memory, 0, contentPtr, length);
int maxCompressedLength = SnappyCodec.GetMaximumCompressedLength(length);
this.compressedFrameData = new byte[maxCompressedLength];
fixed(byte *bptr = &this.compressedFrameData[0])
{
this.compressedSize = maxCompressedLength;
SnappyCodec.Compress((byte *)contentPtr, length, bptr, ref this.compressedSize);
byte[] output = new byte[this.compressedSize];
Marshal.Copy((IntPtr)bptr, output, 0, this.compressedSize);
outputStream.Write(this.compressedFrameData, 0, this.compressedSize);
}
Marshal.FreeHGlobal(contentPtr);
}
FStreamOut.Flush(true);
}
/// <summary>
/// Compresses the remainder of <paramref name="input"/>, writing the compressed data to
/// <paramref name="output"/>.
/// </summary>
/// <param name="input"> The input stream.</param>
/// <param name="output">The output stream.</param>
public void Compress(Stream input, Stream output)
{
var uncompressedSize = (int)(input.Length - input.Position);
var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers
input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None);
var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize);
var compressedBytes = new byte[maxCompressedSize];
var compressedSize = SnappyCodec.Compress(
input: uncompressedBytes,
inputOffset: 0,
inputLength: uncompressedSize,
output: compressedBytes,
outputOffset: 0,
outputLength: compressedBytes.Length); // output.Length - outputOffset
output.Write(compressedBytes, 0, compressedSize);
}
public void Compression(string name)
{
Benchmark.Run("Compressing", name, benchmark =>
{
var output = new byte[SnappyCodec.GetMaxCompressedLength(benchmark.Input.Length)];
int length = 0;
benchmark.Stopwatch.Start();
for (int i = 0; i < benchmark.Iterations; ++i)
{
length = SnappyCodec.Compress(benchmark.Input, 0, benchmark.Input.Length, output, 0);
}
benchmark.Stopwatch.Stop();
var roundtrip = new byte[benchmark.Input.Length];
var roundtripLength = SnappyCodec.Uncompress(output, 0, length, roundtrip, 0);
CollectionAssert.AreEqual(benchmark.Input, roundtrip.Take(roundtripLength));
benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length);
});
}
