public void Uncompress_with_advanced_parameters_should_throw_if_parameters_incorrect(bool?provideCorrectCompressedBytes, int inputOffset, int inputLength, int?outputCount, int outputOffset, Type expectedException) { var uncompressedBytes = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); // 20 bytes byte[] compressedBytes = null; if (provideCorrectCompressedBytes.HasValue) { if (provideCorrectCompressedBytes.Value) { compressedBytes = SnappyCodec.Compress(uncompressedBytes); } else { compressedBytes = new byte[inputLength]; new Random(0).NextBytes(compressedBytes); } } var outputBytes = outputCount.HasValue ? new byte[outputCount.Value] : null; var outputLength = (outputBytes?.Length ?? 0) - outputOffset; var exception = Record.Exception(() => SnappyCodec.Uncompress(compressedBytes, inputOffset, inputLength, outputBytes, outputOffset, outputLength)); exception.Should().BeOfType(expectedException); }
public void CompressUncompressEmpty() { var compressed = SnappyCodec.Compress(new byte[0]); Assert.That(compressed.Length, Is.GreaterThan(0)); Assert.AreEqual(0, SnappyCodec.Uncompress(compressed).Length); }
private void SendMetricRequest(WriteRequest req) { var url = Configuration.GetValue <string>("prom_write_url"); var token = Configuration.GetValue <string>("prom_write_token"); if (!String.IsNullOrEmpty(url)) { MemoryStream stream = new MemoryStream(); req.WriteTo(stream); byte[] arrIn = stream.ToArray(); byte[] arr = SnappyCodec.Compress(arrIn); var handler = new HttpClientHandler(); var httpClient = new HttpClient(handler); HttpRequestMessage requestMessage = new HttpRequestMessage(new HttpMethod("POST"), url); requestMessage.Headers.Add("Authorization", $"Bearer {token}"); requestMessage.Content = new ByteArrayContent(arr); requestMessage.Content.Headers.ContentType = new MediaTypeHeaderValue("application/x-protobuf"); requestMessage.Content.Headers.ContentEncoding.Add("snappy"); HttpResponseMessage response = httpClient.SendAsync(requestMessage).Result; } }
void Flush(byte[] buff, int offset, int count) { var compressedSize = SnappyCodec.Compress(buff, offset, count, _compressedBuffer, 0); BigEndianConverter.Write(_base, compressedSize); _base.Write(_compressedBuffer, 0, compressedSize); }
public void CompressUncompressEmpty() { var compressed = SnappyCodec.Compress(new byte[0]); Assert.True(compressed.Length > 0); Assert.Equal(0, SnappyCodec.Uncompress(compressed).Length); }
private byte[] GetCompressed(byte[] buffer) { switch (Compression) { case CompressionType.None: return(buffer); case CompressionType.ZLib: return(ZlibStream.CompressBuffer(buffer)); case CompressionType.Snappy: return(SnappyCodec.Compress(buffer)); case CompressionType.Doboz: return(DobozCodec.Encode(buffer, 0, buffer.Length)); case CompressionType.LZ4: return(LZ4.LZ4Codec.Encode(buffer, 0, buffer.Length)); case CompressionType.LZ4HC: return(LZ4.LZ4Codec.EncodeHC(buffer, 0, buffer.Length)); default: throw new MissingCompressionException("Unhandled compression algorithm.") { }; } }
public string Compress(object obj) { byte[] array = JsonSerializer.Serialize(obj); byte[] compressed = SnappyCodec.Compress(array); string str = Convert.ToBase64String(compressed, 0, compressed.Length, Base64FormattingOptions.None); return(str); }
public void CompressRange() { var input = Encoding.ASCII.GetBytes("ByeHelloBye"); var output = new byte[100]; var length = SnappyCodec.Compress(input, 3, 5, output, 10); Assert.AreEqual("Hello", Encoding.ASCII.GetString(SnappyCodec.Uncompress(output.Skip(10).Take(length).ToArray()))); }
public void Validate_with_advanced_options_should_throw_if_parameters_incorrect(bool provideCompressedBytes, int inputOffset, int inputLength, Type expectedExceptionType) { var uncompressedBytes = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); // 20 bytes var compressedBytes = provideCompressedBytes ? SnappyCodec.Compress(uncompressedBytes) : null; // 22 bytes var exception = Record.Exception(() => SnappyCodec.Validate(compressedBytes, inputOffset, inputLength)); exception.Should().BeOfType(expectedExceptionType); }
protected override void Encode(IChannelHandlerContext context, Packet message, List <object> output) { if (_logger.IsTrace) { _logger.Trace($"Compressing with Snappy a message of length {message.Data.Length}"); } message.Data = SnappyCodec.Compress(message.Data); output.Add(message); }
public void Compress_with_advanced_options_should_throw_if_parameters_incorrect(int?inputCount, int inputOffset, int inputLength, int?outputCount, int outputOffset, Type expectedException) { var input = inputCount.HasValue ? new byte[inputCount.Value] : null; var output = outputCount.HasValue ? new byte[outputCount.Value] : null; var outputLength = (output?.Length ?? 0) - outputOffset; var exception = Record.Exception(() => SnappyCodec.Compress(input, inputOffset, inputLength, output, outputOffset, outputLength)); exception.Should().BeOfType(expectedException); }
public void UncompressRange() { var howdy = Encoding.ASCII.GetBytes("Howdy"); var padded = howdy.Take(3).Concat(SnappyCodec.Compress(Encoding.ASCII.GetBytes("Hello"))).Concat(howdy.Skip(3)).ToArray(); var output = new byte[100]; var length = SnappyCodec.Uncompress(padded, 3, padded.Length - 5, output, 10); Assert.AreEqual(5, length); Assert.AreEqual("Hello", Encoding.ASCII.GetString(output.Skip(10).Take(5).ToArray())); }
public byte[] Serialize <T>(T instance) { if (instance == null) { throw new ArgumentNullException(nameof(instance)); } var data = instance.XmlSerialize().ToString().GetBytes(); return(compressed ? SnappyCodec.Compress(data) : data); }
public void CompressDecompress(int length) { byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); SnappyCodec codec = new SnappyCodec(); byte[] compressed = codec.Compress(data); byte[] uncompressed = codec.Decompress(compressed, compressed.Length); Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); }
public void Compress_should_uncompress_previously_compressed_message(string input) { var inputBytes = Encoding.ASCII.GetBytes(input); var compressedBytes = SnappyCodec.Compress(inputBytes); compressedBytes.Length.Should().BeGreaterThan(0); var uncompressedBytes = SnappyCodec.Uncompress(compressedBytes); var outputString = Encoding.ASCII.GetString(uncompressedBytes); outputString.Should().Be(input); }
public void ValidateExceptions() { var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); var compressed = SnappyCodec.Compress(uncompressed); var buffer = new byte[100]; Assert.Throws <ArgumentNullException>(() => SnappyCodec.Validate(null)); Assert.Throws <ArgumentNullException>(() => SnappyCodec.Validate(null, 0, 3)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Validate(compressed, -1, uncompressed.Length)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Validate(compressed, 0, -1)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Validate(compressed, compressed.Length - 2, 4)); }
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info) { if (info.CompressionCodec != CompressionCodec.None) { stream.Write(Basics.Zero64, 0, 8); using (var msgsetStream = stream.Pool.Reserve()) { SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion); using (var compressed = stream.Pool.Reserve()) { if (info.CompressionCodec == CompressionCodec.Gzip) { using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true)) { msgsetStream.WriteTo(gzip); } } else // Snappy { #if NETSTANDARD1_3 throw new NotImplementedException(); #else compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length)); { int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0, (int)msgsetStream.Length, compressed.GetBuffer(), 0); compressed.SetLength(size); } #endif } var m = new Message { Value = compressed, TimeStamp = Timestamp.Now }; Basics.WriteSizeInBytes(stream, m, new SerializationInfo { Serializers = SerializationConfig.ByteArraySerializers, CompressionCodec = info.CompressionCodec, MessageVersion = info.MessageVersion }, SerializeMessageWithCodec); } } } else { SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion); } }
public byte[] Compress(byte[] data) { byte[] result = null; try { result = SnappyCodec.Compress(data); } catch (Exception ex) { Console.WriteLine($"EXCEPTION: {ex.GetType().Name}, {ex.Message}"); } return(result); }
/// <summary>Compresses the specified input file.</summary> /// <param name="input">The input file.</param> /// <param name="output">The output file.</param> private static void Compress(string input, string output) { byte[] original = File.ReadAllBytes(input); var timer = Stopwatch.StartNew(); var compressed = SnappyCodec.Compress(original, 0, original.Length); timer.Stop(); Console.WriteLine("Compression:"); Console.WriteLine(" Speed: {0:0.00}MB/s", (double)original.Length / 1024 / 1024 / timer.Elapsed.TotalSeconds); Console.WriteLine(" Ratio: {0:0.00}%", (double)compressed.Length * 100 / original.Length); Console.WriteLine(" Hash: {0}", original.MD5()); File.WriteAllBytes(output, compressed); }
/// <summary> /// Compresses the remainder of <paramref name="input"/>, writing the compressed data to /// <paramref name="output"/>. /// </summary> /// <param name="input"> The input stream.</param> /// <param name="output">The output stream.</param> public void Compress(Stream input, Stream output) { #if NET452 || NETSTANDARD2_0 var uncompressedSize = (int)(input.Length - input.Position); var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None); var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize); var compressedBytes = new byte[maxCompressedSize]; var compressedSize = SnappyCodec.Compress(uncompressedBytes, 0, uncompressedSize, compressedBytes, 0); output.Write(compressedBytes, 0, compressedSize); #else throw new NotSupportedException(); #endif }
public void Compress_with_advanced_options_should_work_as_expected(int inputOffset, int inputLength, int outputOffset, string expectedResult) { var input = Encoding.ASCII.GetBytes("ByeHelloBye"); var output = new byte[100]; var outputLength = output.Length - outputOffset; var compressedLength = SnappyCodec.Compress(input, inputOffset, inputLength, output, outputOffset, outputLength); var compressedOutputWithoutOffset = output.Skip(outputOffset).Take(compressedLength).ToArray(); var uncompressed = SnappyCodec.Uncompress(compressedOutputWithoutOffset); var uncompressedString = Encoding.ASCII.GetString(uncompressed); uncompressedString.Should().Be(expectedResult); }
public void Validate() { var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); var compressed = SnappyCodec.Compress(uncompressed); var buffer = new byte[100]; Assert.IsTrue(SnappyCodec.Validate(compressed)); var rubbish = new byte[10]; new Random(0).NextBytes(rubbish); Assert.IsFalse(SnappyCodec.Validate(rubbish, 0, rubbish.Length)); Assert.IsFalse(SnappyCodec.Validate(compressed, 0, 0)); Assert.IsFalse(SnappyCodec.Validate(compressed, compressed.Length, 0)); }
public void Validation(string name) { Benchmark.Run("Validating", name, benchmark => { var compressed = SnappyCodec.Compress(benchmark.Input); bool ok = false; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { ok = SnappyCodec.Validate(compressed); } benchmark.Stopwatch.Stop(); Assert.IsTrue(ok); }); }
public void Uncompression(string name) { Benchmark.Run("Uncompressing", name, benchmark => { var compressed = SnappyCodec.Compress(benchmark.Input); var roundtrip = new byte[benchmark.Input.Length]; int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { length = SnappyCodec.Uncompress(compressed, 0, compressed.Length, roundtrip, 0); } benchmark.Stopwatch.Stop(); CollectionAssert.AreEqual(benchmark.Input, roundtrip); }); }
public void CompressExceptions() { var input = new byte[100]; var output = new byte[100]; Assert.Throws <ArgumentNullException>(() => SnappyCodec.Compress(null)); Assert.Throws <ArgumentNullException>(() => SnappyCodec.Compress(null, 0, 3, output, 0)); Assert.Throws <ArgumentNullException>(() => SnappyCodec.Compress(input, 0, 3, null, 0)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, -1, 3, output, 0)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, -1, output, 0)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 90, 20, output, 0)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 3, output, -1)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 3, output, 100)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 3, output, 101)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.Compress(input, 0, 100, new byte[3], 0)); }
public void CompressDecompressStream(int length) { byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); SnappyCodec codec = new SnappyCodec(); using (MemoryStream inputStream = new MemoryStream(data)) using (MemoryStream outputStream = new MemoryStream()) { codec.Compress(inputStream, outputStream); byte[] compressed = outputStream.ToArray(); byte[] uncompressed = codec.Decompress(compressed, compressed.Length); Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); } }
public void GetUncompressedLengthExceptions() { var uncompressed = Encoding.ASCII.GetBytes("Hello, hello, howdy?"); var compressed = SnappyCodec.Compress(uncompressed); var buffer = new byte[100]; Assert.Throws <ArgumentNullException>(() => SnappyCodec.GetUncompressedLength(null)); Assert.Throws <ArgumentNullException>(() => SnappyCodec.GetUncompressedLength(null, 0, 3)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, -1, uncompressed.Length)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, 0, -1)); Assert.Throws <ArgumentOutOfRangeException>(() => SnappyCodec.GetUncompressedLength(compressed, compressed.Length - 2, 4)); Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(compressed, 0, 0)); Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(compressed, compressed.Length, 0)); var rubbish = Enumerable.Repeat((byte)0xff, 10).ToArray(); Assert.Throws <InvalidDataException>(() => SnappyCodec.GetUncompressedLength(rubbish, 0, rubbish.Length)); }
public void Evaluate(int spreadMax) { if (FStreamIn.SliceCount == 0 || FStreamIn[0] == null || FStreamIn[0].Length == 0) { spreadMax = 0; } else { spreadMax = FStreamIn.SliceCount; } FStreamOut.ResizeAndDispose(spreadMax, () => new MemoryStream()); for (int i = 0; i < spreadMax; i++) { var inputStream = FStreamIn[i]; var outputStream = FStreamOut[i]; inputStream.Position = 0; outputStream.Position = 0; FStreamOut[i].SetLength(0); var length = (int)inputStream.Length; IntPtr contentPtr = Marshal.AllocHGlobal(length); byte[] memory = new byte[length]; inputStream.Read(memory, 0, length); Marshal.Copy(memory, 0, contentPtr, length); int maxCompressedLength = SnappyCodec.GetMaximumCompressedLength(length); this.compressedFrameData = new byte[maxCompressedLength]; fixed(byte *bptr = &this.compressedFrameData[0]) { this.compressedSize = maxCompressedLength; SnappyCodec.Compress((byte *)contentPtr, length, bptr, ref this.compressedSize); byte[] output = new byte[this.compressedSize]; Marshal.Copy((IntPtr)bptr, output, 0, this.compressedSize); outputStream.Write(this.compressedFrameData, 0, this.compressedSize); } Marshal.FreeHGlobal(contentPtr); } FStreamOut.Flush(true); }
/// <summary> /// Compresses the remainder of <paramref name="input"/>, writing the compressed data to /// <paramref name="output"/>. /// </summary> /// <param name="input"> The input stream.</param> /// <param name="output">The output stream.</param> public void Compress(Stream input, Stream output) { var uncompressedSize = (int)(input.Length - input.Position); var uncompressedBytes = new byte[uncompressedSize]; // does not include uncompressed message headers input.ReadBytes(uncompressedBytes, offset: 0, count: uncompressedSize, CancellationToken.None); var maxCompressedSize = SnappyCodec.GetMaxCompressedLength(uncompressedSize); var compressedBytes = new byte[maxCompressedSize]; var compressedSize = SnappyCodec.Compress( input: uncompressedBytes, inputOffset: 0, inputLength: uncompressedSize, output: compressedBytes, outputOffset: 0, outputLength: compressedBytes.Length); // output.Length - outputOffset output.Write(compressedBytes, 0, compressedSize); }
public void Compression(string name) { Benchmark.Run("Compressing", name, benchmark => { var output = new byte[SnappyCodec.GetMaxCompressedLength(benchmark.Input.Length)]; int length = 0; benchmark.Stopwatch.Start(); for (int i = 0; i < benchmark.Iterations; ++i) { length = SnappyCodec.Compress(benchmark.Input, 0, benchmark.Input.Length, output, 0); } benchmark.Stopwatch.Stop(); var roundtrip = new byte[benchmark.Input.Length]; var roundtripLength = SnappyCodec.Uncompress(output, 0, length, roundtrip, 0); CollectionAssert.AreEqual(benchmark.Input, roundtrip.Take(roundtripLength)); benchmark.Note = String.Format(" ({0:0.00 %})", length / (double)benchmark.Input.Length); }); }