public static BytesOwner ReadPageData(Stream nakedStream, Thrift.CompressionCodec compressionCodec, int compressedLength, int uncompressedLength) { if (!_codecToCompressionMethod.TryGetValue(compressionCodec, out CompressionMethod compressionMethod)) { throw new NotSupportedException($"reader for compression '{compressionCodec}' is not supported."); } int totalBytesRead = 0; int currentBytesRead = int.MinValue; byte[] data = BytesPool.Rent(compressedLength); // Some storage solutions (like Azure blobs) might require more than one 'Read' action to read the requested length. while (totalBytesRead < compressedLength && currentBytesRead != 0) { currentBytesRead = nakedStream.Read(data, totalBytesRead, compressedLength - totalBytesRead); totalBytesRead += currentBytesRead; } if (totalBytesRead != compressedLength) { throw new ParquetException($"expected {compressedLength} bytes in source stream but could read only {totalBytesRead}"); } switch (compressionMethod) { case CompressionMethod.None: //nothing to do, original data is the raw data break; case CompressionMethod.Gzip: using (var source = new MemoryStream(data, 0, compressedLength)) { byte[] unGzData = BytesPool.Rent(uncompressedLength); using (var dest = new MemoryStream(unGzData, 0, uncompressedLength)) { using (var gz = new GZipStream(source, CompressionMode.Decompress)) { gz.CopyTo(dest); } } BytesPool.Return(data); data = unGzData; } break; case CompressionMethod.Snappy: var snappy = new SnappyDecompressor(); byte[] unSnapData = snappy.Decompress(BytesPool, data, 0, compressedLength); BytesPool.Return(data); data = unSnapData; break; default: throw new NotSupportedException("method: " + compressionMethod); } return(new BytesOwner(data, 0, data.AsMemory(0, (int)uncompressedLength), d => BytesPool.Return(d))); }
public void onClickDecompress() { string text = this.input.text; Debug.Log("Depress:" + text); string[] txts = text.Trim().Split(','); Debug.Log("src Length:" + txts.Length); //byte[] byteArray = System.Text.Encoding.Default.GetBytes(text); byte[] compressed = new byte[txts.Length]; for (int i = 0; i < txts.Length; ++i) { compressed[i] = (byte)Convert.ToSByte(txts[i]); } var target = new SnappyDecompressor(); var data = target.Decompress(compressed, 0, compressed.Length); Debug.Log("Depress Length:" + data.Length); string result = System.Text.Encoding.UTF8.GetString(data); Debug.Log("result:" + result); }
private static CompressionResult RunDecompression(string fileName, int iterations) { long[] ticks = new long[iterations]; byte[] uncompressed = File.ReadAllBytes(fileName); var compressed = Sharp.Snappy.Compress(uncompressed); int size = compressed.Length; var target = new SnappyDecompressor(); var s = new Stopwatch(); for (int i = 0; i < iterations; i++) { s.Start(); var result = target.Decompress(compressed, 0, compressed.Length); s.Stop(); ticks[i] = s.ElapsedTicks; s.Reset(); } return(new CompressionResult { Direction = CompressionDirection.Decompress, FileName = FileMap[Path.GetFileName(fileName)], CompressedSize = size, FileBytes = uncompressed.Length, ElapsedTime = new TimeSpan(ticks.Sum()), StandardDeviation = StdDev(ticks), Iterations = iterations }); }
private static void VerifyRoundTrip(string fileName) { int size = 0; byte[] uncompressed = File.ReadAllBytes(fileName); var compressor = new SnappyCompressor(); var result = new byte[compressor.MaxCompressedLength(uncompressed.Length)]; size = compressor.Compress(uncompressed, 0, uncompressed.Length, result); Array.Resize(ref result, size); var decompressor = new SnappyDecompressor(); var decompressed = decompressor.Decompress(result, 0, size); byte[] source = File.ReadAllBytes(fileName); if (source.Length != decompressed.Length) { throw new Exception(string.Format("Decompressed length {0} does not match original {1}", decompressed.Length, source.Length)); } for (int i = 0; i < uncompressed.Length; i++) { if (source[i] != decompressed[i]) { throw new Exception(string.Format("Decompressed data did not match original at index {0}", i)); } } }
private void TestFindMatchBug_Internal(SnappyCompressor compressor) { var decompressor = new SnappyDecompressor(); var size = 1024; var data = new byte[size]; for (var i = 0; i < data.Length; ++i) { data[i] = (byte)(i & 0xff); } data[1021] = 5; data[1022] = 5; data[1023] = 5; var compressed = new byte[compressor.MaxCompressedLength(data.Length)]; var compressedLength = compressor.Compress(data, 0, data.Length, compressed, 0); var decompressed = decompressor.Decompress(compressed, 0, compressedLength); for (var i = 0; i < data.Length; ++i) { Assert.Equal(data[i], decompressed[i]); } }
public byte[] Read(Stream source, int count) { byte[] buffer = new byte[count]; source.Read(buffer, 0, count); byte[] uncompressedBytes = _snappyDecompressor.Decompress(buffer, 0, count); return(uncompressedBytes); }
public override int Read(byte[] buffer, int offset, int count) { if (compressionMode != CompressionMode.Decompress || decompressor == null) { throw new InvalidOperationException("Cannot read if not Set to decompression mode."); } int readCount = 0; int firstByte = stream.ReadByte(); // first byte can indicate stream header, we just read it and move on. if (firstByte == StreamIdentifier) { CheckStreamHeader(); } else if (firstByte == UncompressedType) { var length = GetChunkUncompressedLength(); readCount = ProcessRemainingInternalBuffer(buffer, offset, count); if (readCount != count) { stream.Read(internalBuffer, 0, length); Array.Copy(internalBuffer, 0, buffer, offset, count - readCount); internalBufferIndex = count - readCount; internalBufferLength = length; } } else if (firstByte == CompressedType) { var length = GetChunkUncompressedLength(); count = ProcessRemainingInternalBuffer(buffer, offset, count); // we at most have 64kb in the buffer to read byte[] tempBuffer = new byte[1 << (BLOCK_LOG + 1)]; stream.Read(tempBuffer, 0, tempBuffer.Length); decompressor.Decompress(tempBuffer, 0, tempBuffer.Length, internalBuffer, 0, length); Array.Copy(internalBuffer, 0, buffer, offset, count); internalBufferIndex = count; internalBufferLength = length; } else if (firstByte > 0x2 && firstByte < 0x7f) { throw new InvalidOperationException("Found unskippable chunk type that cannot be undertood."); } else { // getting the length and skipping the data. var length = GetChunkUncompressedLength(); stream.Seek(length, SeekOrigin.Current); readCount += length; } return(readCount); }
public static BytesOwner ReadPageData(Stream nakedStream, Thrift.CompressionCodec compressionCodec, int compressedLength, int uncompressedLength) { if (!_codecToCompressionMethod.TryGetValue(compressionCodec, out CompressionMethod compressionMethod)) { throw new NotSupportedException($"reader for compression '{compressionCodec}' is not supported."); } byte[] data = BytesPool.Rent(compressedLength); int read = nakedStream.Read(data, 0, compressedLength); if (read != compressedLength) { throw new ParquetException($"expected {compressedLength} bytes in source stream but could read only {read}"); } switch (compressionMethod) { case CompressionMethod.None: //nothing to do, original data is the raw data break; case CompressionMethod.Gzip: using (var source = new MemoryStream(data, 0, compressedLength)) { byte[] unGzData = BytesPool.Rent(uncompressedLength); using (var dest = new MemoryStream(unGzData, 0, uncompressedLength)) { using (var gz = new GZipStream(source, CompressionMode.Decompress)) { gz.CopyTo(dest); } } BytesPool.Return(data); data = unGzData; } break; case CompressionMethod.Snappy: var snappy = new SnappyDecompressor(); byte[] unSnapData = snappy.Decompress(BytesPool, data, 0, compressedLength); BytesPool.Return(data); data = unSnapData; break; default: throw new NotSupportedException("method: " + compressionMethod); } return(new BytesOwner(data, data.AsMemory(0, (int)uncompressedLength), d => BytesPool.Return(d))); }
/// <summary> /// 使用snappy算法解压缩 /// </summary> /// <param name="compressed"></param> /// <param name="offset"></param> /// <param name="length"></param> /// <returns></returns> public static byte[] DecompressInSnappy(byte[] compressed, int offset, int length) { SnappyDecompressor sd = new SnappyDecompressor(); try { return(sd.Decompress(compressed, offset, length)); } catch (Exception ex) { //啥情況? return(null); } }
public static byte[] DecompressBody(Stream body) { var ms = new MemoryStream(); body.CopyTo(ms); var source = ms.ToArray(); if (source.Length <= 0) { return(null); } var decompressed = _decompressor.Decompress(source, 0, source.Length); return(decompressed); }
public void round_trip_returns_original_data(string fileName) { byte[] uncompressed = File.ReadAllBytes(fileName); var target = new SnappyCompressor(); var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; int size = target.Compress(uncompressed, 0, uncompressed.Length, result); var target2 = new SnappyDecompressor(); var sizes = target2.ReadUncompressedLength(result, 0); var bytes = new byte[sizes[0]]; target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]); Assert.Equal(uncompressed, bytes); }
public void compress_random_data() { var data = GetRandomData(4096); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decomperssor = new SnappyDecompressor(); var bytes = decomperssor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_constant_data_is_one_literal_and_one_copy() { // 1024 a's var data = Encoding.Default.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.Equal(50, result); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public override void OnReceived(IConnection connection, byte[] buffer, int offset, int size) { byte b = buffer[offset]; if (b == Message.UnCompressed) { FireBuffReceived(connection, buffer, offset + 1, size - 1); } else { if (b == Message.Compressed) { byte[] array = decompressor.Decompress(buffer, offset + 1, size - 1); FireBuffReceived(connection, array, 0, array.Length); } } }
public void compress_constant_data_is_one_literal_and_one_copy() { // 1024 a's var data = Encoding.UTF8.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.Equal(52, result); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_returns_bytes_copied() { var data = Encoding.Default.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); Assert.Equal(15, result); // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them. var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Console.Write(Encoding.Default.GetString(bytes)); }
public void compress_returns_bytes_copied() { var data = Encoding.UTF8.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); Assert.Equal(12, result); // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them. var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Console.Write(Encoding.UTF8.GetString(bytes)); }
/// <summary> /// Decompress a block of Snappy data. This must be an entire block. /// </summary> /// <param name="input">Data to decompress.</param> /// <param name="output">Buffer to receive the decompressed data.</param> /// <returns>Number of bytes written to <paramref name="output"/>.</returns> /// <exception cref="InvalidDataException">Invalid Snappy block.</exception> /// <exception cref="ArgumentException">Output buffer is too small.</exception> public static int Decompress(ReadOnlySpan <byte> input, Span <byte> output) { var decompressor = new SnappyDecompressor(); decompressor.Decompress(input); if (!decompressor.AllDataDecompressed) { throw new InvalidDataException("Incomplete Snappy block."); } var read = decompressor.Read(output); if (!decompressor.EndOfFile) { throw new ArgumentException("Output buffer is too small.", nameof(output)); } return(read); }
public void compress_multiple_blocks() { /*! * this ends up being uncompressible because it is random. */ var rand = new Random().Next(1 << 4, 1 << 10); var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end. var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
/// <summary> /// 使用snappy算法解压缩 /// </summary> /// <param name="compressed"></param> /// <param name="offset"></param> /// <param name="length"></param> /// <returns></returns> public static byte[] DecompressInSnappy(byte[] compressed, int offset, int length) { SnappyDecompressor sd = new SnappyDecompressor(); return(sd.Decompress(compressed, offset, length)); }
static void Save(BinaryReader input, FileMetadata metadata, BinaryWriter output) { input.BaseStream.Position = metadata.offset; byte[] compressedData = input.ReadBytes((int)metadata.compressedSize); byte[] uncompressedData = new byte[metadata.uncompressedSize]; if (metadata.compressionMethod == 0) // Нет сжатия { if (metadata.compressedSize != metadata.uncompressedSize) { throw new Exception("metadata.compressedSize != metadata.uncompressedSize"); } uncompressedData = compressedData; } else if (metadata.compressionMethod == 1) // Deflate { Inflater inflater = new Inflater(false); inflater.SetInput(compressedData); int length = inflater.Inflate(uncompressedData); if (length != metadata.uncompressedSize) { throw new Exception("length != metadata.uncompressedSize"); } if (!inflater.IsFinished) { // Какой-то остаток в конце сжатых данных, но он не дает дополнительных распакованных данных byte[] remain = new byte[100]; int remainLength = inflater.Inflate(remain); if (remainLength != 0 || !inflater.IsFinished) { throw new Exception("remainLength != 0 || !inflater.IsFinished"); } } } else if (metadata.compressionMethod == 2) // Snappy { SnappyDecompressor decompressor = new SnappyDecompressor(); uncompressedData = decompressor.Decompress(compressedData, 0, compressedData.Length); } else if (metadata.compressionMethod == 3) // Doboz { uncompressedData = DobozDecoder.Decode(compressedData, 0, compressedData.Length); if (uncompressedData.Length != metadata.uncompressedSize) { throw new Exception("uncompressedData.Length != metadata.uncompressedSize"); } } else if (metadata.compressionMethod == 4 || metadata.compressionMethod == 5) // LZ4 { uncompressedData = LZ4Codec.Decode32(compressedData, 0, compressedData.Length, (int)metadata.uncompressedSize); if (uncompressedData.Length != metadata.uncompressedSize) { throw new Exception("uncompressedData.Length != metadata.uncompressedSize"); } } else { throw new Exception("Unknown compression method"); } // Проверяем, что распаковали данные правильно Crc32 calculatedCrc32 = new Crc32(); calculatedCrc32.Update(uncompressedData); if (calculatedCrc32.Value != metadata.crc32) { throw new Exception("calculatedCrc32.Value != metadata.crc32"); } output.Write(uncompressedData); }
public byte[] Decompress(byte[] src, int srcPos, int srcSize) { return(_decompressor.Decompress(src, srcPos, srcSize)); }
public static bool Uncompress(byte[] compressed, int offset, int size, byte[] uncompress, out int uncompressCount) { SnappyDecompressor decompressor = new SnappyDecompressor(); return(decompressor.Decompress(compressed, offset, size, uncompress, out uncompressCount)); }
public static byte[] Uncompress(byte[] compressed, int offset, int size) { SnappyDecompressor decompressor = new SnappyDecompressor(); return(decompressor.Decompress(compressed, offset, size)); }
public static byte[] Uncompress(byte[] compressed) { SnappyDecompressor decompressor = new SnappyDecompressor(); return(decompressor.Decompress(compressed, 0, compressed.Length)); }
public void Snappier() { var decompressor = new SnappyDecompressor(); decompressor.Decompress(_input.Span); }