public void RoundtripBinary() { byte[] originalBytes = new byte[] { 1, 2, 3, 4, 5, 6 }; using (MemoryStream memStream = new MemoryStream()) { using (Stream zip = Snappy.OpenWriter(memStream)) { zip.Write(originalBytes); } memStream.Seek(0, SeekOrigin.Begin); using (Stream zip = Snappy.OpenReader(memStream)) { byte[] buffer = new byte[1024]; int n = zip.Read(buffer); Assert.Equal(n, originalBytes.Length); for (int i = 0; i < n; i++) { Assert.Equal(originalBytes[i], buffer[i]); } } } }
public byte[] Read(Stream source, int count) { byte[] buffer = new byte[count]; source.Read(buffer, 0, count); byte[] uncompressedBytes = Snappy.Decode(buffer); return(uncompressedBytes); }
internal override byte[] Decompress(byte[] compressedData) { byte[] dataToDecompress = new byte[compressedData.Length - 4]; // last 4 bytes are CRC Array.Copy(compressedData, dataToDecompress, dataToDecompress.Length); return(Snappy.Decode(dataToDecompress)); }
/// <summary> /// Processes the packet within the context. Returns true whether the packet was processed or throttled. /// </summary> /// <param name="channel">The through which the packet is coming/going out.</param> /// <param name="context">The packet context for this operation.</param> /// <returns>True whether the packet was processed or throttled, false otherwise.</returns> public static ProcessingState Process(Emitter.Connection channel, ProcessingContext context) { // Get the buffer to decompress var input = context.Buffer.AsSegment(); try { // Reserve the buffer, we know exactly how many bytes to decompress var output = context.BufferReserve( (int)VarInt.UvarInt(input.Array, input.Offset).Value ); // Decompress var length = Snappy.Decode( input.Array, input.Offset, input.Count, output.Array, output.Offset, output.Size ); // Switch the buffer to the decompressed one context.SwitchBuffer(output); } catch (Exception ex) { ex.Log(); } return(ProcessingState.Success); }
public void RoundtripGoldenData() { byte[] goldenRaw = File.ReadAllBytes("TestData/Mark.Twain-Tom.Sawyer.txt"); byte[] compressed = Snappy.Encode(goldenRaw); byte[] uncompressed = Snappy.Decode(compressed); Assert.Equal(goldenRaw.Length, uncompressed.Length); Assert.Equal(goldenRaw, uncompressed); }
public void CompressAndDecompressString(string str) { var input = Encoding.UTF8.GetBytes(str); var compressed = Snappy.CompressToArray(input); var output = Snappy.DecompressToArray(compressed); Assert.Equal(input.Length, output.Length); Assert.Equal(input, output); }
public void EncodeGoldenInput() { byte[] got = Snappy.Encode(File.ReadAllBytes("TestData/Mark.Twain-Tom.Sawyer.txt")); byte[] want = File.ReadAllBytes("TestData/Mark.Twain-Tom.Sawyer.rawsnappy.txt"); Assert.Equal(want.Length, got.Length); Assert.Equal(want, got); }
internal override byte[] Compress(byte[] uncompressedData) { var compressedData = Snappy.Encode(uncompressedData); uint checksumUint = Crc32.Get(compressedData); byte[] checksumBytes = BitConverter.GetBytes(checksumUint); byte[] result = compressedData.Concat(checksumBytes).ToArray(); return(result); }
private void UpdateDocumentFromModel(CompanyDocuments target, ImagePickerResult source) { target.CompanyBlobId = source.blobId; target.CompanyGuid = source.guid; target.FileBlob = Snappy.Encode(source.ImageBytes); target.FileName = source.FileName; target.FileType = source.ContentType; target.FileCategoryId = source.FileCategoryId; target.UploadTime = DateTime.Now; target.FileLength = source.Size; }
public void CompressFiles() { var files = Directory.EnumerateFiles(@"..\..\..\Data"); foreach (var f in files) { var source = File.ReadAllBytes(f); var compressedData = Snappy.Compress(source); var decompressedData = Snappy.Decompress(compressedData, 0, compressedData.Length); Assert.True(decompressedData.SequenceEqual(source)); } }
public void RoundtripEncodeBytes() { byte[] bytes = File.ReadAllBytes("TestData/Mark.Shanghai-skyyearxp.bytes"); byte[] wants = File.ReadAllBytes("TestData/Mark.Shanghai-skyyearxp.snappy.bytes"); byte[] compressed = Snappy.Encode(bytes); Assert.Equal(wants.Length, compressed.Length); Assert.Equal(wants, compressed); byte[] uncompressed = Snappy.Decode(compressed); Assert.Equal(bytes.Length, uncompressed.Length); Assert.Equal(bytes, uncompressed); }
/// <summary> /// 打包 /// </summary> /// <param name="packet">输入的协议对象</param> /// <returns>需要发送的字节流</returns> public ArraySegment <byte> Pack(object packet) { // offset // --------------------------------------------------------- // | 包头 | 包体(Count) | // --------------------------------------------------------- // | ArraySegment(包头+包体) | // --------------------------------------------------------- // 返回的ArraySegment,Offset为包头偏移位置,Count为包体长度 var msgInfo = msgDefined.GetMsgByName(packet.GetType().FullName); if (msgInfo == null) { throw new NotSupportedException("Can not find msg info , msg name is [" + packet.GetType().FullName + "]"); } var sent = new ArraySegment <byte>(sentBuffer, Header.HeadSize, sentBuffer.Length - Header.HeadSize); packerPolicy.Packed(ref sent, msgInfo, packet); var header = new Header { MsgId = msgInfo.Id, }; header.FillMsgId(sent); var datalen = sent.Count; var destbuff = sentBuffer; header.Encrypt = true; if (datalen >= 100) { //compress header.Compress = true; destbuff = compSentBuffer; datalen = Snappy.Compress(sent.Array, sent.Offset - 2, sent.Count + 2, compSentBuffer, Header.HeadSize - 2); //Debug.LogError("send data compressed pre count->" + sent.Count + " after->" + datalen); header.BodyLength = datalen; } else { header.BodyLength = datalen + 2; } sent = new ArraySegment <byte>(destbuff, 0, header.BodyLength + Header.HeadSize - 2); header.Fill(sent); XorEncrypt.Encrypt(destbuff, Header.HeadSize - 2, header.BodyLength); return(sent); }
public void CompressRandomData() { var r = new Random(); var rng = new RNGCryptoServiceProvider(); for (int i = 0; i < 10000; i++) { var data = new byte[r.Next(UInt16.MaxValue)]; rng.GetNonZeroBytes(data); var compressedData = Snappy.Compress(data); var decompressedData = Snappy.Decompress(compressedData, 0, compressedData.Length); Assert.True(decompressedData.SequenceEqual(data)); } }
public void BadData_InsufficentOutputBuffer_ThrowsArgumentException() { var input = new byte[100000]; Array.Fill(input, (byte)'A'); var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)]; var compressedLength = Snappy.Compress(input, compressed); Assert.Throws <ArgumentException>(() => { var output = new byte[100]; Snappy.Decompress(compressed.AsSpan(0, compressedLength), output); }); }
public void decompress(ByteBuffer @in, ByteBuffer @out) { if (@in.isDirect() && @out.isDirect()) { directDecompress(@in, @out); return; } int inOffset = @in.position(); int uncompressLen = Snappy.uncompress(@in.array(), @in.arrayOffset() + inOffset, @in.limit() - inOffset, @out.array(), @out.arrayOffset() + @out.position()); @out.position(uncompressLen + @out.position()); @out.flip(); }
public void LoadToMemory() { using var resource = typeof(DecompressAlice).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.alice29.txt"); var input = new byte[65536]; // Just test the first 64KB // ReSharper disable once PossibleNullReferenceException resource.Read(input); var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)]; var compressedLength = Snappy.Compress(input, compressed); _input = compressed.AsMemory(0, compressedLength); }
/// <summary> /// Applies a compressin to the buffer and returns a result within a stream. /// </summary> /// <param name="buffer">The buffer to compress.</param> /// <param name="padding">The padding offset to pre-allocate.</param> /// <param name="context">The context of the processing.</param> /// <returns>The length segment</returns> public static BufferSegment Process(ByteStream stream, int padding, ProcessingContext context) { stream.Flush(); // Calculate the maximum compressed length var outSize = Snappy.MaxEncodedLen((int)stream.Length); var output = context.BufferReserve(outSize + padding); // Acquire a snappy encoder which comes in with a table for state using (var snappy = Snappy.Acquire()) { output.Size = snappy.Encode(stream.GetBuffer(), 0, (int)stream.Length, output, padding) + padding; return(output); } }
public void BadData_FromFile_ThrowsInvalidDataException(string filename) { using var resource = typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}"); Assert.NotNull(resource); var input = new byte[resource.Length]; resource.Read(input); Assert.Throws <InvalidDataException>(() => { var length = Snappy.GetUncompressedLength(input); Assert.InRange(length, 0, 1 << 20); var output = new byte[length]; Snappy.Decompress(input, output); }); }
public void BadData_LongLength_ThrowsInvalidDataException() { var input = new byte[1000]; Array.Fill(input, (byte)'A'); var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)]; var compressedLength = Snappy.Compress(input, compressed); var compressedSpan = compressed.AsSpan(0, compressedLength); // Set the length header to 16383 compressedSpan[0] = 255; compressedSpan[1] = 127; Assert.Throws <InvalidDataException>(() => { var output = new byte[1000]; Snappy.Decompress(compressed, output); }); }
public void CompressAndDecompressFile(string filename) { using var resource = typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}"); Assert.NotNull(resource); var input = new byte[resource.Length]; resource.Read(input); var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)]; var compressedLength = Snappy.Compress(input, compressed); var compressedSpan = compressed.AsSpan(0, compressedLength); var output = new byte[Snappy.GetUncompressedLength(compressedSpan)]; var outputLength = Snappy.Decompress(compressedSpan, output); Assert.Equal(input.Length, outputLength); Assert.Equal(input, output); }
public void BadData_SimpleCorruption_ThrowsInvalidDataException() { var input = Encoding.UTF8.GetBytes("making sure we don't crash with corrupted input"); var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)]; var compressedLength = Snappy.Compress(input, compressed); var compressedSpan = compressed.AsSpan(0, compressedLength); // corrupt the data a bit compressedSpan[1]--; compressedSpan[3]++; Assert.Throws <InvalidDataException>(() => { var length = Snappy.GetUncompressedLength(compressed.AsSpan(0, compressedLength)); Assert.InRange(length, 0, 1 << 20); var output = new byte[length]; Snappy.Decompress(compressed.AsSpan(0, compressedLength), output); }); }
public object Unpack(ArraySegment <byte> packet) { var header = new Header(packet); if (header.Encrypt) { XorEncrypt.Decrypt(packet.Array, Header.HeadSize - 2, header.BodyLength); } if (header.Compress) { var uncompdata = Snappy.Uncompress(packet.Array, Header.HeadSize - 2, header.BodyLength); header.Body = new ArraySegment <byte>(uncompdata, 2, uncompdata.Length - 2); header.MsgId = uncompdata[0] | (uint)uncompdata[1] << 8; } else if (header.Encrypt) { header.MsgId = packet.Array[packet.Offset + 4] | (uint)packet.Array[packet.Offset + 5] << 8; } //if (ModuleName == "GameBox.Cratos.GBoxSync") // Debug.LogError("Unpack msgid->" + header.MsgId + " buff len->" + packet.Count + " header.BodyLength->" + header.BodyLength + " header.Encrypt->" + header.Encrypt + " header.Compress->" + header.Compress); var msgInfo = msgDefined.GetMsgById(header.MsgId); if (msgInfo == null) { throw new NotSupportedException("Can not find msg info , msg id is [" + header.MsgId + "]"); } var result = packerPolicy.Unpacked(msgInfo, header.Body); if (msgInfo.Id != MsgConst.SyncMsgID) { return(result); } //MsgService.SyncMsg(result); return(null); }
static public CompanyModel CreateCompanyModelAsync(Company source, bool includeAllFields) { var model = new CompanyModel() { CompanyID = source.CompanyID, CompanyGuid = source.CompanyGuid, Name = string.IsNullOrEmpty(source.Name) ? source.Name : source.Name.Trim(), PhoneNoIsdCode = string.IsNullOrEmpty(source.PhoneNoIsdCode) ? source.PhoneNoIsdCode : source.PhoneNoIsdCode.Trim(), PhoneNo = string.IsNullOrEmpty(source.PhoneNo) ? source.PhoneNo : source.PhoneNo.Trim(), Email = string.IsNullOrEmpty(source.Email) ? source.Email : source.Email.Trim(), PAN = string.IsNullOrEmpty(source.PAN) ? source.PAN : source.PAN.Trim(), GSTIN = string.IsNullOrEmpty(source.GSTIN) ? source.GSTIN : source.GSTIN.Trim(), AddressLine1 = string.IsNullOrEmpty(source.AddressLine1) ? source.AddressLine1 : source.AddressLine1.Trim(), AddressLine2 = string.IsNullOrEmpty(source.AddressLine2) ? source.AddressLine2 : source.AddressLine2.Trim(), City = string.IsNullOrEmpty(source.City) ? source.City : source.City.Trim(), IsActive = source.IsActive, Pincode = string.IsNullOrEmpty(source.Pincode)? source.Pincode : source.Pincode.Trim() }; if (source.CompanyDocuments != null && source.CompanyDocuments.Count > 0) { ObservableCollection <ImagePickerResult> docs = new ObservableCollection <ImagePickerResult>(); foreach (var doc in source.CompanyDocuments) { docs.Add(new ImagePickerResult { blobId = doc.CompanyBlobId, guid = doc.CompanyGuid, FileName = doc.FileName, ImageBytes = Snappy.Decode(doc.FileBlob), ContentType = doc.FileType, Size = doc.FileLength, FileCategoryId = doc.FileCategoryId }); } model.CompanyDocuments = docs; } return(model); }
public void RoundtripGolden() { string text = File.ReadAllText("TestData/Mark.Twain-Tom.Sawyer.txt"); using (MemoryStream memStream = new MemoryStream()) { using (Stream zip = Snappy.OpenWriter(memStream)) using (StreamWriter writer = new StreamWriter(zip)) { writer.Write(text); } memStream.Seek(0, SeekOrigin.Begin); using (Stream zip = Snappy.OpenReader(memStream)) using (StreamReader reader = new StreamReader(zip)) { string result = reader.ReadToEnd(); Assert.Equal(text, result); } } }
public bool compress(ByteBuffer @in, ByteBuffer @out, ByteBuffer overflow) { int inBytes = @in.remaining(); // I should work on a patch for Snappy to support an overflow buffer // to prevent the extra buffer copy. byte[] compressed = new byte[Snappy.maxCompressedLength(inBytes)]; int outBytes = Snappy.compress(@in.array(), @in.arrayOffset() + @in.position(), inBytes, compressed, 0); if (outBytes < inBytes) { int remaining = @out.remaining(); if (remaining >= outBytes) { Array.Copy(compressed, 0, @out.array(), @out.arrayOffset() + @out.position(), outBytes); @out.position(@out.position() + outBytes); } else { Array.Copy(compressed, 0, @out.array(), @out.arrayOffset() + @out.position(), remaining); @out.position(@out.limit()); Array.Copy(compressed, remaining, overflow.array(), overflow.arrayOffset(), outBytes - remaining); overflow.position(outBytes - remaining); } return(true); } else { return(false); } }
public static BytesOwner ReadPageData(Stream nakedStream, Thrift.CompressionCodec compressionCodec, int compressedLength, int uncompressedLength) { if (!_codecToCompressionMethod.TryGetValue(compressionCodec, out CompressionMethod compressionMethod)) { throw new NotSupportedException($"reader for compression '{compressionCodec}' is not supported."); } int totalBytesRead = 0; int currentBytesRead = int.MinValue; byte[] data = BytesPool.Rent(compressedLength); bool dataRented = true; // Some storage solutions (like Azure blobs) might require more than one 'Read' action to read the requested length. while (totalBytesRead < compressedLength && currentBytesRead != 0) { currentBytesRead = nakedStream.Read(data, totalBytesRead, compressedLength - totalBytesRead); totalBytesRead += currentBytesRead; } if (totalBytesRead != compressedLength) { throw new ParquetException($"expected {compressedLength} bytes in source stream but could read only {totalBytesRead}"); } switch (compressionMethod) { case CompressionMethod.None: //nothing to do, original data is the raw data break; case CompressionMethod.Gzip: using (var source = new MemoryStream(data, 0, compressedLength)) { byte[] unGzData = BytesPool.Rent(uncompressedLength); using (var dest = new MemoryStream(unGzData, 0, uncompressedLength)) { using (var gz = new GZipStream(source, CompressionMode.Decompress)) { gz.CopyTo(dest); } } BytesPool.Return(data); data = unGzData; } break; case CompressionMethod.Snappy: byte[] uncompressed = Snappy.Decode(data.AsSpan(0, compressedLength)); BytesPool.Return(data); data = uncompressed; dataRented = false; break; default: throw new NotSupportedException("method: " + compressionMethod); } return(new BytesOwner(data, 0, data.AsMemory(0, uncompressedLength), d => BytesPool.Return(d), dataRented)); }
/// <inheritdoc /> public IMemoryOwner <byte> Decompress(ReadOnlyMemory <byte> input) => Snappy.DecompressToMemory(input.Span);
public void Write(byte[] buffer, Stream destination) { byte[] compressed = Snappy.Encode(buffer); destination.Write(compressed, 0, buffer.Length); }