public void RoundtripBinary()
{
byte[] originalBytes = new byte[] { 1, 2, 3, 4, 5, 6 };
using (MemoryStream memStream = new MemoryStream())
{
using (Stream zip = Snappy.OpenWriter(memStream))
{
zip.Write(originalBytes);
}
memStream.Seek(0, SeekOrigin.Begin);
using (Stream zip = Snappy.OpenReader(memStream))
{
byte[] buffer = new byte[1024];
int n = zip.Read(buffer);
Assert.Equal(n, originalBytes.Length);
for (int i = 0; i < n; i++)
{
Assert.Equal(originalBytes[i], buffer[i]);
}
}
}
}
public byte[] Read(Stream source, int count)
{
byte[] buffer = new byte[count];
source.Read(buffer, 0, count);
byte[] uncompressedBytes = Snappy.Decode(buffer);
return(uncompressedBytes);
}
internal override byte[] Decompress(byte[] compressedData)
{
byte[] dataToDecompress = new byte[compressedData.Length - 4]; // last 4 bytes are CRC
Array.Copy(compressedData, dataToDecompress, dataToDecompress.Length);
return(Snappy.Decode(dataToDecompress));
}
/// <summary>
/// Processes the packet within the context. Returns true whether the packet was processed or throttled.
/// </summary>
/// <param name="channel">The through which the packet is coming/going out.</param>
/// <param name="context">The packet context for this operation.</param>
/// <returns>True whether the packet was processed or throttled, false otherwise.</returns>
public static ProcessingState Process(Emitter.Connection channel, ProcessingContext context)
{
// Get the buffer to decompress
var input = context.Buffer.AsSegment();
try
{
// Reserve the buffer, we know exactly how many bytes to decompress
var output = context.BufferReserve(
(int)VarInt.UvarInt(input.Array, input.Offset).Value
);
// Decompress
var length = Snappy.Decode(
input.Array, input.Offset, input.Count,
output.Array, output.Offset, output.Size
);
// Switch the buffer to the decompressed one
context.SwitchBuffer(output);
}
catch (Exception ex)
{
ex.Log();
}
return(ProcessingState.Success);
}
public void RoundtripGoldenData()
{
byte[] goldenRaw = File.ReadAllBytes("TestData/Mark.Twain-Tom.Sawyer.txt");
byte[] compressed = Snappy.Encode(goldenRaw);
byte[] uncompressed = Snappy.Decode(compressed);
Assert.Equal(goldenRaw.Length, uncompressed.Length);
Assert.Equal(goldenRaw, uncompressed);
}
public void CompressAndDecompressString(string str)
{
var input = Encoding.UTF8.GetBytes(str);
var compressed = Snappy.CompressToArray(input);
var output = Snappy.DecompressToArray(compressed);
Assert.Equal(input.Length, output.Length);
Assert.Equal(input, output);
}
public void EncodeGoldenInput()
{
byte[] got = Snappy.Encode(File.ReadAllBytes("TestData/Mark.Twain-Tom.Sawyer.txt"));
byte[] want = File.ReadAllBytes("TestData/Mark.Twain-Tom.Sawyer.rawsnappy.txt");
Assert.Equal(want.Length, got.Length);
Assert.Equal(want, got);
}
internal override byte[] Compress(byte[] uncompressedData)
{
var compressedData = Snappy.Encode(uncompressedData);
uint checksumUint = Crc32.Get(compressedData);
byte[] checksumBytes = BitConverter.GetBytes(checksumUint);
byte[] result = compressedData.Concat(checksumBytes).ToArray();
return(result);
}
private void UpdateDocumentFromModel(CompanyDocuments target, ImagePickerResult source)
{
target.CompanyBlobId = source.blobId;
target.CompanyGuid = source.guid;
target.FileBlob = Snappy.Encode(source.ImageBytes);
target.FileName = source.FileName;
target.FileType = source.ContentType;
target.FileCategoryId = source.FileCategoryId;
target.UploadTime = DateTime.Now;
target.FileLength = source.Size;
}
public void CompressFiles()
{
var files = Directory.EnumerateFiles(@"..\..\..\Data");
foreach (var f in files)
{
var source = File.ReadAllBytes(f);
var compressedData = Snappy.Compress(source);
var decompressedData = Snappy.Decompress(compressedData, 0, compressedData.Length);
Assert.True(decompressedData.SequenceEqual(source));
}
}
public void RoundtripEncodeBytes()
{
byte[] bytes = File.ReadAllBytes("TestData/Mark.Shanghai-skyyearxp.bytes");
byte[] wants = File.ReadAllBytes("TestData/Mark.Shanghai-skyyearxp.snappy.bytes");
byte[] compressed = Snappy.Encode(bytes);
Assert.Equal(wants.Length, compressed.Length);
Assert.Equal(wants, compressed);
byte[] uncompressed = Snappy.Decode(compressed);
Assert.Equal(bytes.Length, uncompressed.Length);
Assert.Equal(bytes, uncompressed);
}
/// <summary>
/// 打包
/// </summary>
/// <param name="packet">输入的协议对象</param>
/// <returns>需要发送的字节流</returns>
public ArraySegment <byte> Pack(object packet)
{
// offset
// ---------------------------------------------------------
// | 包头 | 包体（Count） |
// ---------------------------------------------------------
// | ArraySegment(包头+包体) |
// ---------------------------------------------------------
// 返回的ArraySegment，Offset为包头偏移位置，Count为包体长度
var msgInfo = msgDefined.GetMsgByName(packet.GetType().FullName);
if (msgInfo == null)
{
throw new NotSupportedException("Can not find msg info , msg name is [" + packet.GetType().FullName + "]");
}
var sent = new ArraySegment <byte>(sentBuffer, Header.HeadSize, sentBuffer.Length - Header.HeadSize);
packerPolicy.Packed(ref sent, msgInfo, packet);
var header = new Header
{
MsgId = msgInfo.Id,
};
header.FillMsgId(sent);
var datalen = sent.Count;
var destbuff = sentBuffer;
header.Encrypt = true;
if (datalen >= 100)
{
//compress
header.Compress = true;
destbuff = compSentBuffer;
datalen = Snappy.Compress(sent.Array, sent.Offset - 2, sent.Count + 2, compSentBuffer, Header.HeadSize - 2);
//Debug.LogError("send data compressed pre count->" + sent.Count + " after->" + datalen);
header.BodyLength = datalen;
}
else
{
header.BodyLength = datalen + 2;
}
sent = new ArraySegment <byte>(destbuff, 0, header.BodyLength + Header.HeadSize - 2);
header.Fill(sent);
XorEncrypt.Encrypt(destbuff, Header.HeadSize - 2, header.BodyLength);
return(sent);
}
public void CompressRandomData()
{
var r = new Random();
var rng = new RNGCryptoServiceProvider();
for (int i = 0; i < 10000; i++)
{
var data = new byte[r.Next(UInt16.MaxValue)];
rng.GetNonZeroBytes(data);
var compressedData = Snappy.Compress(data);
var decompressedData = Snappy.Decompress(compressedData, 0, compressedData.Length);
Assert.True(decompressedData.SequenceEqual(data));
}
}
public void BadData_InsufficentOutputBuffer_ThrowsArgumentException()
{
var input = new byte[100000];
Array.Fill(input, (byte)'A');
var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)];
var compressedLength = Snappy.Compress(input, compressed);
Assert.Throws <ArgumentException>(() =>
{
var output = new byte[100];
Snappy.Decompress(compressed.AsSpan(0, compressedLength), output);
});
}
public void decompress(ByteBuffer @in, ByteBuffer @out)
{
if (@in.isDirect() && @out.isDirect())
{
directDecompress(@in, @out);
return;
}
int inOffset = @in.position();
int uncompressLen =
Snappy.uncompress(@in.array(), @in.arrayOffset() + inOffset,
@in.limit() - inOffset, @out.array(), @out.arrayOffset() + @out.position());
@out.position(uncompressLen + @out.position());
@out.flip();
}
public void LoadToMemory()
{
using var resource =
typeof(DecompressAlice).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.alice29.txt");
var input = new byte[65536]; // Just test the first 64KB
// ReSharper disable once PossibleNullReferenceException
resource.Read(input);
var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)];
var compressedLength = Snappy.Compress(input, compressed);
_input = compressed.AsMemory(0, compressedLength);
}
/// <summary>
/// Applies a compressin to the buffer and returns a result within a stream.
/// </summary>
/// <param name="buffer">The buffer to compress.</param>
/// <param name="padding">The padding offset to pre-allocate.</param>
/// <param name="context">The context of the processing.</param>
/// <returns>The length segment</returns>
public static BufferSegment Process(ByteStream stream, int padding, ProcessingContext context)
{
stream.Flush();
// Calculate the maximum compressed length
var outSize = Snappy.MaxEncodedLen((int)stream.Length);
var output = context.BufferReserve(outSize + padding);
// Acquire a snappy encoder which comes in with a table for state
using (var snappy = Snappy.Acquire())
{
output.Size = snappy.Encode(stream.GetBuffer(), 0, (int)stream.Length, output, padding) + padding;
return(output);
}
}
public void BadData_FromFile_ThrowsInvalidDataException(string filename)
{
using var resource =
typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}");
Assert.NotNull(resource);
var input = new byte[resource.Length];
resource.Read(input);
Assert.Throws <InvalidDataException>(() =>
{
var length = Snappy.GetUncompressedLength(input);
Assert.InRange(length, 0, 1 << 20);
var output = new byte[length];
Snappy.Decompress(input, output);
});
}
public void BadData_LongLength_ThrowsInvalidDataException()
{
var input = new byte[1000];
Array.Fill(input, (byte)'A');
var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)];
var compressedLength = Snappy.Compress(input, compressed);
var compressedSpan = compressed.AsSpan(0, compressedLength);
// Set the length header to 16383
compressedSpan[0] = 255;
compressedSpan[1] = 127;
Assert.Throws <InvalidDataException>(() =>
{
var output = new byte[1000];
Snappy.Decompress(compressed, output);
});
}
public void CompressAndDecompressFile(string filename)
{
using var resource =
typeof(SnappyTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}");
Assert.NotNull(resource);
var input = new byte[resource.Length];
resource.Read(input);
var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)];
var compressedLength = Snappy.Compress(input, compressed);
var compressedSpan = compressed.AsSpan(0, compressedLength);
var output = new byte[Snappy.GetUncompressedLength(compressedSpan)];
var outputLength = Snappy.Decompress(compressedSpan, output);
Assert.Equal(input.Length, outputLength);
Assert.Equal(input, output);
}
public void BadData_SimpleCorruption_ThrowsInvalidDataException()
{
var input = Encoding.UTF8.GetBytes("making sure we don't crash with corrupted input");
var compressed = new byte[Snappy.GetMaxCompressedLength(input.Length)];
var compressedLength = Snappy.Compress(input, compressed);
var compressedSpan = compressed.AsSpan(0, compressedLength);
// corrupt the data a bit
compressedSpan[1]--;
compressedSpan[3]++;
Assert.Throws <InvalidDataException>(() =>
{
var length = Snappy.GetUncompressedLength(compressed.AsSpan(0, compressedLength));
Assert.InRange(length, 0, 1 << 20);
var output = new byte[length];
Snappy.Decompress(compressed.AsSpan(0, compressedLength), output);
});
}
public object Unpack(ArraySegment <byte> packet)
{
var header = new Header(packet);
if (header.Encrypt)
{
XorEncrypt.Decrypt(packet.Array, Header.HeadSize - 2, header.BodyLength);
}
if (header.Compress)
{
var uncompdata = Snappy.Uncompress(packet.Array, Header.HeadSize - 2, header.BodyLength);
header.Body = new ArraySegment <byte>(uncompdata, 2, uncompdata.Length - 2);
header.MsgId = uncompdata[0] | (uint)uncompdata[1] << 8;
}
else if (header.Encrypt)
{
header.MsgId = packet.Array[packet.Offset + 4] | (uint)packet.Array[packet.Offset + 5] << 8;
}
//if (ModuleName == "GameBox.Cratos.GBoxSync")
// Debug.LogError("Unpack msgid->" + header.MsgId + " buff len->" + packet.Count + " header.BodyLength->" + header.BodyLength + " header.Encrypt->" + header.Encrypt + " header.Compress->" + header.Compress);
var msgInfo = msgDefined.GetMsgById(header.MsgId);
if (msgInfo == null)
{
throw new NotSupportedException("Can not find msg info , msg id is [" + header.MsgId + "]");
}
var result = packerPolicy.Unpacked(msgInfo, header.Body);
if (msgInfo.Id != MsgConst.SyncMsgID)
{
return(result);
}
//MsgService.SyncMsg(result);
return(null);
}
static public CompanyModel CreateCompanyModelAsync(Company source, bool includeAllFields)
{
var model = new CompanyModel()
{
CompanyID = source.CompanyID,
CompanyGuid = source.CompanyGuid,
Name = string.IsNullOrEmpty(source.Name) ? source.Name : source.Name.Trim(),
PhoneNoIsdCode = string.IsNullOrEmpty(source.PhoneNoIsdCode) ? source.PhoneNoIsdCode : source.PhoneNoIsdCode.Trim(),
PhoneNo = string.IsNullOrEmpty(source.PhoneNo) ? source.PhoneNo : source.PhoneNo.Trim(),
Email = string.IsNullOrEmpty(source.Email) ? source.Email : source.Email.Trim(),
PAN = string.IsNullOrEmpty(source.PAN) ? source.PAN : source.PAN.Trim(),
GSTIN = string.IsNullOrEmpty(source.GSTIN) ? source.GSTIN : source.GSTIN.Trim(),
AddressLine1 = string.IsNullOrEmpty(source.AddressLine1) ? source.AddressLine1 : source.AddressLine1.Trim(),
AddressLine2 = string.IsNullOrEmpty(source.AddressLine2) ? source.AddressLine2 : source.AddressLine2.Trim(),
City = string.IsNullOrEmpty(source.City) ? source.City : source.City.Trim(),
IsActive = source.IsActive,
Pincode = string.IsNullOrEmpty(source.Pincode)? source.Pincode : source.Pincode.Trim()
};
if (source.CompanyDocuments != null && source.CompanyDocuments.Count > 0)
{
ObservableCollection <ImagePickerResult> docs = new ObservableCollection <ImagePickerResult>();
foreach (var doc in source.CompanyDocuments)
{
docs.Add(new ImagePickerResult
{
blobId = doc.CompanyBlobId,
guid = doc.CompanyGuid,
FileName = doc.FileName,
ImageBytes = Snappy.Decode(doc.FileBlob),
ContentType = doc.FileType,
Size = doc.FileLength,
FileCategoryId = doc.FileCategoryId
});
}
model.CompanyDocuments = docs;
}
return(model);
}
public void RoundtripGolden()
{
string text = File.ReadAllText("TestData/Mark.Twain-Tom.Sawyer.txt");
using (MemoryStream memStream = new MemoryStream())
{
using (Stream zip = Snappy.OpenWriter(memStream))
using (StreamWriter writer = new StreamWriter(zip))
{
writer.Write(text);
}
memStream.Seek(0, SeekOrigin.Begin);
using (Stream zip = Snappy.OpenReader(memStream))
using (StreamReader reader = new StreamReader(zip))
{
string result = reader.ReadToEnd();
Assert.Equal(text, result);
}
}
}
public bool compress(ByteBuffer @in, ByteBuffer @out, ByteBuffer overflow)
{
int inBytes = @in.remaining();
// I should work on a patch for Snappy to support an overflow buffer
// to prevent the extra buffer copy.
byte[] compressed = new byte[Snappy.maxCompressedLength(inBytes)];
int outBytes =
Snappy.compress(@in.array(), @in.arrayOffset() + @in.position(), inBytes,
compressed, 0);
if (outBytes < inBytes)
{
int remaining = @out.remaining();
if (remaining >= outBytes)
{
Array.Copy(compressed, 0, @out.array(), @out.arrayOffset() +
@out.position(), outBytes);
@out.position(@out.position() + outBytes);
}
else
{
Array.Copy(compressed, 0, @out.array(), @out.arrayOffset() +
@out.position(), remaining);
@out.position(@out.limit());
Array.Copy(compressed, remaining, overflow.array(),
overflow.arrayOffset(), outBytes - remaining);
overflow.position(outBytes - remaining);
}
return(true);
}
else
{
return(false);
}
}
public static BytesOwner ReadPageData(Stream nakedStream, Thrift.CompressionCodec compressionCodec,
int compressedLength, int uncompressedLength)
{
if (!_codecToCompressionMethod.TryGetValue(compressionCodec, out CompressionMethod compressionMethod))
{
throw new NotSupportedException($"reader for compression '{compressionCodec}' is not supported.");
}
int totalBytesRead = 0;
int currentBytesRead = int.MinValue;
byte[] data = BytesPool.Rent(compressedLength);
bool dataRented = true;
// Some storage solutions (like Azure blobs) might require more than one 'Read' action to read the requested length.
while (totalBytesRead < compressedLength && currentBytesRead != 0)
{
currentBytesRead = nakedStream.Read(data, totalBytesRead, compressedLength - totalBytesRead);
totalBytesRead += currentBytesRead;
}
if (totalBytesRead != compressedLength)
{
throw new ParquetException($"expected {compressedLength} bytes in source stream but could read only {totalBytesRead}");
}
switch (compressionMethod)
{
case CompressionMethod.None:
//nothing to do, original data is the raw data
break;
case CompressionMethod.Gzip:
using (var source = new MemoryStream(data, 0, compressedLength))
{
byte[] unGzData = BytesPool.Rent(uncompressedLength);
using (var dest = new MemoryStream(unGzData, 0, uncompressedLength))
{
using (var gz = new GZipStream(source, CompressionMode.Decompress))
{
gz.CopyTo(dest);
}
}
BytesPool.Return(data);
data = unGzData;
}
break;
case CompressionMethod.Snappy:
byte[] uncompressed = Snappy.Decode(data.AsSpan(0, compressedLength));
BytesPool.Return(data);
data = uncompressed;
dataRented = false;
break;
default:
throw new NotSupportedException("method: " + compressionMethod);
}
return(new BytesOwner(data, 0, data.AsMemory(0, uncompressedLength), d => BytesPool.Return(d), dataRented));
}
/// <inheritdoc /> public IMemoryOwner <byte> Decompress(ReadOnlyMemory <byte> input) => Snappy.DecompressToMemory(input.Span);
public void Write(byte[] buffer, Stream destination)
{
byte[] compressed = Snappy.Encode(buffer);
destination.Write(compressed, 0, buffer.Length);
}
