static void Main(string[] args)
{
var data = Encoding.ASCII.GetBytes("Hello World!");
Console.WriteLine("SnappyCodec roundtrip: {0}", Encoding.ASCII.GetString(SnappyCodec.Uncompress(SnappyCodec.Compress(data))));
var buffer = new MemoryStream();
var stream = new SnappyStream(buffer, CompressionMode.Compress);
stream.Write(data, 0, data.Length);
stream.Close();
buffer = new MemoryStream(buffer.ToArray());
stream = new SnappyStream(buffer, CompressionMode.Decompress);
var roundtrip = new byte[data.Length];
int read = stream.Read(roundtrip, 0, data.Length);
if (read != data.Length)
{
throw new ApplicationException();
}
if (0 != stream.Read(roundtrip, 0, data.Length))
{
throw new ApplicationException();
}
Console.WriteLine("SnappyStream roundtrip: {0}", Encoding.ASCII.GetString(roundtrip));
}
/// <summary>A tool to test native library availability,</summary>
public static void Main(string[] args)
{
string usage = "NativeLibraryChecker [-a|-h]\n" + " -a use -a to check all libraries are available\n"
+ " by default just check hadoop library (and\n" + " winutils.exe on Windows OS) is available\n"
+ " exit with error code 1 if check failed\n" + " -h print this message\n";
if (args.Length > 1 || (args.Length == 1 && !(args[0].Equals("-a") || args[0].Equals
("-h"))))
{
System.Console.Error.WriteLine(usage);
ExitUtil.Terminate(1);
}
bool checkAll = false;
if (args.Length == 1)
{
if (args[0].Equals("-h"))
{
System.Console.Out.WriteLine(usage);
return;
}
checkAll = true;
}
Configuration conf = new Configuration();
bool nativeHadoopLoaded = NativeCodeLoader.IsNativeCodeLoaded();
bool zlibLoaded = false;
bool snappyLoaded = false;
// lz4 is linked within libhadoop
bool lz4Loaded = nativeHadoopLoaded;
bool bzip2Loaded = Bzip2Factory.IsNativeBzip2Loaded(conf);
bool openSslLoaded = false;
bool winutilsExists = false;
string openSslDetail = string.Empty;
string hadoopLibraryName = string.Empty;
string zlibLibraryName = string.Empty;
string snappyLibraryName = string.Empty;
string lz4LibraryName = string.Empty;
string bzip2LibraryName = string.Empty;
string winutilsPath = null;
if (nativeHadoopLoaded)
{
hadoopLibraryName = NativeCodeLoader.GetLibraryName();
zlibLoaded = ZlibFactory.IsNativeZlibLoaded(conf);
if (zlibLoaded)
{
zlibLibraryName = ZlibFactory.GetLibraryName();
}
snappyLoaded = NativeCodeLoader.BuildSupportsSnappy() && SnappyCodec.IsNativeCodeLoaded
();
if (snappyLoaded && NativeCodeLoader.BuildSupportsSnappy())
{
snappyLibraryName = SnappyCodec.GetLibraryName();
}
if (OpensslCipher.GetLoadingFailureReason() != null)
{
openSslDetail = OpensslCipher.GetLoadingFailureReason();
openSslLoaded = false;
}
else
{
openSslDetail = OpensslCipher.GetLibraryName();
openSslLoaded = true;
}
if (lz4Loaded)
{
lz4LibraryName = Lz4Codec.GetLibraryName();
}
if (bzip2Loaded)
{
bzip2LibraryName = Bzip2Factory.GetLibraryName(conf);
}
}
// winutils.exe is required on Windows
winutilsPath = Shell.GetWinUtilsPath();
if (winutilsPath != null)
{
winutilsExists = true;
}
else
{
winutilsPath = string.Empty;
}
System.Console.Out.WriteLine("Native library checking:");
System.Console.Out.Printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName
);
System.Console.Out.Printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName);
System.Console.Out.Printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName);
System.Console.Out.Printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName);
System.Console.Out.Printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName);
System.Console.Out.Printf("openssl: %b %s%n", openSslLoaded, openSslDetail);
if (Shell.Windows)
{
System.Console.Out.Printf("winutils: %b %s%n", winutilsExists, winutilsPath);
}
if ((!nativeHadoopLoaded) || (Shell.Windows && (!winutilsExists)) || (checkAll &&
!(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded)))
{
// return 1 to indicated check failed
ExitUtil.Terminate(1);
}
}
public void ReadMsg(BitStream stream)
{
//bool isFilenames;
if (stream.ReadChar() == ':')
{
//isFilenames = true;
}
else
{
stream.Seek(-8, System.IO.SeekOrigin.Current);
//isFilenames = false;
}
TableName = stream.ReadCString();
MaxEntries = stream.ReadUShort();
int encodeBits = ExtMath.Log2(MaxEntries);
Entries = stream.ReadUShort((byte)(encodeBits + 1));
ulong bitCount = stream.ReadVarUInt();
// userdatafixedsize
if (stream.ReadBool())
{
UserDataSize = stream.ReadUShort(12);
UserDataSizeBits = stream.ReadByte(4);
}
bool isCompressedData = stream.ReadBool();
Data = stream.Subsection(stream.Cursor, stream.Cursor + bitCount);
stream.Seek(bitCount, System.IO.SeekOrigin.Current);
if (isCompressedData)
{
uint decompressedNumBytes = Data.ReadUInt();
uint compressedNumBytes = Data.ReadUInt();
byte[] compressedData = Data.ReadBytes(compressedNumBytes);
char[] magic = Encoding.ASCII.GetChars(compressedData, 0, 4);
if (
magic[0] != 'S' ||
magic[1] != 'N' ||
magic[2] != 'A' ||
magic[3] != 'P')
{
throw new FormatException("Unknown format for compressed stringtable");
}
int snappyDecompressedNumBytes = SnappyCodec.GetUncompressedLength(compressedData, 4, compressedData.Length - 4);
if (snappyDecompressedNumBytes != decompressedNumBytes)
{
throw new FormatException("Mismatching decompressed data lengths");
}
byte[] decompressedData = new byte[snappyDecompressedNumBytes];
if (SnappyCodec.Uncompress(compressedData, 4, compressedData.Length - 4, decompressedData, 0) != decompressedNumBytes)
{
throw new FormatException("Snappy didn't decode all the bytes");
}
Data = new BitStream(decompressedData);
}
}
public void Extract(Stream output)
{
using (var file = MemoryMappedFile.CreateFromFile(Bundle.FileName, FileMode.Open))
{
using (var viewstream = file.CreateViewStream(PageOffset, ZSize, MemoryMappedFileAccess.Read))
{
switch (CompressionType)
{
case CompressionType.None:
{
viewstream.CopyTo(output);
break;
}
case CompressionType.LZ4:
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = LZ4Codec.Decode(buffer, 0, c, (int)Size);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case CompressionType.LZ4HC:
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = LZ4Codec.Decode(buffer, 0, c, (int)Size);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case CompressionType.Snappy:
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = SnappyCodec.Uncompress(buffer);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case CompressionType.Doboz:
{
var buffer = new byte[ZSize];
var c = viewstream.Read(buffer, 0, buffer.Length);
var uncompressed = DobozCodec.Decode(buffer, 0, c);
output.Write(uncompressed, 0, uncompressed.Length);
break;
}
case CompressionType.ZLib:
{
var zlib = new ZlibStream(viewstream, CompressionMode.Decompress);
zlib.CopyTo(output);
break;
}
default:
throw new MissingCompressionException("Unhandled compression algorithm.")
{
Compression = Compression
};
}
viewstream.Close();
}
}
}
/// <summary>
///
/// </summary>
/// <param name="Data"></param>
/// <returns></returns>
public static int GetDecompressedLength(byte[] Data, long DataOffset, long DataLength)
{
return(SnappyCodec.GetUncompressedLength(Data, (int)DataOffset, (int)DataLength));
}
public void CompressUncompressSimple()
{
Assert.Equal("Hello", Encoding.ASCII.GetString(SnappyCodec.Uncompress(SnappyCodec.Compress(Encoding.ASCII.GetBytes("Hello")))));
}
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info)
{
if (info.CompressionCodec != CompressionCodec.None)
{
stream.Write(Basics.Zero64, 0, 8);
using (var msgsetStream = stream.Pool.Reserve())
{
SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion);
using (var compressed = stream.Pool.Reserve())
{
switch (info.CompressionCodec)
{
case CompressionCodec.Gzip:
using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true))
{
msgsetStream.WriteTo(gzip);
}
break;
case CompressionCodec.Lz4:
KafkaLz4.Compress(compressed, msgsetStream.GetBuffer(), (int)msgsetStream.Length);
break;
case CompressionCodec.Snappy:
{
#if NET_CORE
throw new NotImplementedException();
#else
compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length));
{
int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0,
(int)msgsetStream.Length,
compressed.GetBuffer(), 0);
compressed.SetLength(size);
}
#endif
}
break;
}
var m = new Message
{
Value = compressed,
TimeStamp = Timestamp.Now
};
Basics.WriteSizeInBytes(stream, m,
new SerializationInfo
{
Serializers = SerializationConfig.ByteArraySerializers,
CompressionCodec = info.CompressionCodec,
MessageVersion = info.MessageVersion
}, SerializeMessageWithCodec);
}
}
}
else
{
SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion);
}
}
public byte[] Encode(byte[] source)
{
return(SnappyCodec.Compress(source));
}
protected override void ChannelRead0(IChannelHandlerContext ctx, ZeroPacket input)
{
IByteBuffer content = input.Content;
if (SnappyEnabled)
{
int uncompressedLength = SnappyCodec.GetUncompressedLength(content.Array, content.ArrayOffset + content.ReaderIndex, content.ReadableBytes);
if (uncompressedLength > SnappyParameters.MaxSnappyLength)
{
throw new Exception("Max message size exceeeded"); // TODO: disconnect here
}
if (content.ReadableBytes > SnappyParameters.MaxSnappyLength / 4)
{
if (_logger.IsWarn)
{
_logger.Warn($"Big Snappy message of length {content.ReadableBytes}");
}
}
else
{
if (_logger.IsTrace)
{
_logger.Trace($"Uncompressing with Snappy a message of length {content.ReadableBytes}");
}
}
IByteBuffer output = PooledByteBufferAllocator.Default.Buffer(uncompressedLength);
try
{
int length = SnappyCodec.Uncompress(content.Array, content.ArrayOffset + content.ReaderIndex, content.ReadableBytes, output.Array, output.ArrayOffset);
output.SetWriterIndex(output.WriterIndex + length);
}
catch (Exception)
{
if (content.ReadableBytes == 2 && content.ReadByte() == 193)
{
// this is a Parity disconnect sent as a non-snappy-encoded message
// e.g. 0xc103
}
else
{
content.SkipBytes(content.ReadableBytes);
output.SafeRelease();
throw;
}
}
content.SkipBytes(content.ReadableBytes);
ZeroPacket outputPacket = new(output);
try
{
outputPacket.PacketType = input.PacketType;
_session.ReceiveMessage(outputPacket);
}
finally
{
outputPacket.SafeRelease();
}
}
else
{
_session.ReceiveMessage(input);
}
}
public void HashCode()
{
SnappyCodec codec = new SnappyCodec();
Assert.AreNotEqual(0, codec.GetHashCode());
}
public void Compress_should_throw_if_parameter_null()
{
var exception = Record.Exception(() => SnappyCodec.Compress(null));
exception.Should().BeOfType <ArgumentNullException>();
}
