private static void VerifyRoundTrip(string fileName) { int size = 0; byte[] uncompressed = File.ReadAllBytes(fileName); var compressor = new SnappyCompressor(); var result = new byte[compressor.MaxCompressedLength(uncompressed.Length)]; size = compressor.Compress(uncompressed, 0, uncompressed.Length, result); Array.Resize(ref result, size); var decompressor = new SnappyDecompressor(); var decompressed = decompressor.Decompress(result, 0, size); byte[] source = File.ReadAllBytes(fileName); if (source.Length != decompressed.Length) { throw new Exception(string.Format("Decompressed length {0} does not match original {1}", decompressed.Length, source.Length)); } for (int i = 0; i < uncompressed.Length; i++) { if (source[i] != decompressed[i]) { throw new Exception(string.Format("Decompressed data did not match original at index {0}", i)); } } }
/// <summary> /// Initializes a new instance of the <see cref="SnappyStream"/> class. /// </summary> /// <param name="s">The stream.</param> /// <param name="mode">The compression mode.</param> /// <param name="leaveOpen">If Set to <c>true</c> leaves the stream open when complete.</param> /// <param name="checksum"><c>true</c> if checksums should be written to the stream </param> public SnappyStream(Stream s, CompressionMode mode, bool leaveOpen, bool checksum) { stream = s; compressionMode = mode; leaveStreamOpen = leaveOpen; writeChecksums = checksum; if (compressionMode == CompressionMode.Decompress) { if (!stream.CanRead) { throw new InvalidOperationException("Trying to decompress and underlying stream not readable."); } decompressor = new SnappyDecompressor(); CheckStreamIdentifier(); CheckStreamHeader(); } if (compressionMode == CompressionMode.Compress) { if (!stream.CanWrite) { throw new InvalidOperationException("Trying to compress and underlying stream is not writable."); } compressor = new SnappyCompressor(); stream.WriteByte(StreamIdentifier); stream.Write(StreamHeader, 0, StreamHeader.Length); } }
static void Main(string[] args) { var isPix = args[0] == "-p"; ICompressor compressor; switch (args[1]) { case "-l": compressor = new LZ4Compressor(); break; case "-s": compressor = new SnappyCompressor(); break; default: compressor = new NoCompression(); break; } if (isPix) { new PixServer(compressor).Start(); } else { new LiteServer(compressor).Start(); } }
static CompressionResult RunCompression(string fileName, int iterations) { int size = 0; long[] ticks = new long[iterations]; byte[] uncompressed = File.ReadAllBytes(fileName); var target = new SnappyCompressor(); var s = new Stopwatch(); for (int i = 0; i < iterations; i++) { var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; s.Start(); size = target.Compress(uncompressed, 0, uncompressed.Length, result); s.Stop(); ticks[i] = s.ElapsedTicks; s.Reset(); } return(new CompressionResult { Direction = CompressionDirection.Compress, FileName = FileMap[Path.GetFileName(fileName)], CompressedSize = size, FileBytes = uncompressed.Length, ElapsedTime = new TimeSpan(ticks.Sum()), StandardDeviation = StdDev(ticks), Iterations = iterations }); }
static void Main(string[] args) { var lz4 = new LZ4Compressor(); var snappy = new SnappyCompressor(); for (var i = 0; i < 100; i++) { var model = new WorldState2() { Players = Enumerable.Range(0, 1000).Select(_ => 1).ToArray() }; byte[] serialized; using (var m = new MemoryStream()) { ProtoBuf.Serializer.NonGeneric.Serialize(m, model); serialized = m.ToArray(); } var lz4Length = lz4.Compress(serialized).Length; var snappLength = snappy.Compress(serialized).Length; Console.WriteLine($"{serialized.Length}\t{lz4Length}\t{snappLength}\t{lz4Length - snappLength}"); } Console.ReadLine(); }
private void TestFindMatchBug_Internal(SnappyCompressor compressor) { var decompressor = new SnappyDecompressor(); var size = 1024; var data = new byte[size]; for (var i = 0; i < data.Length; ++i) { data[i] = (byte)(i & 0xff); } data[1021] = 5; data[1022] = 5; data[1023] = 5; var compressed = new byte[compressor.MaxCompressedLength(data.Length)]; var compressedLength = compressor.Compress(data, 0, data.Length, compressed, 0); var decompressed = decompressor.Decompress(compressed, 0, compressedLength); for (var i = 0; i < data.Length; ++i) { Assert.Equal(data[i], decompressed[i]); } }
/// <summary> /// Initializes the connection. /// </summary> /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception> /// <exception cref="AuthenticationException" /> /// <exception cref="UnsupportedProtocolVersionException"></exception> public async Task <Response> DoOpen() { _freeOperations = new ConcurrentStack <short>(Enumerable.Range(0, GetMaxConcurrentRequests(Serializer)).Select(s => (short)s).Reverse()); _pendingOperations = new ConcurrentDictionary <short, OperationState>(); _writeQueue = new ConcurrentQueue <OperationState>(); if (Options.CustomCompressor != null) { Compressor = Options.CustomCompressor; } else if (Options.Compression == CompressionType.LZ4) { #if NETFRAMEWORK Compressor = new LZ4Compressor(); #else throw new NotSupportedException("Lz4 compression not supported under .NETCore"); #endif } else if (Options.Compression == CompressionType.Snappy) { Compressor = new SnappyCompressor(); } //Init TcpSocket _tcpSocket.Init(); _tcpSocket.Error += CancelPending; _tcpSocket.Closing += () => CancelPending(null); //Read and write event handlers are going to be invoked using IO Threads _tcpSocket.Read += ReadHandler; _tcpSocket.WriteCompleted += WriteCompletedHandler; var protocolVersion = Serializer.ProtocolVersion; await _tcpSocket.Connect().ConfigureAwait(false); Response response; try { response = await Startup().ConfigureAwait(false); } catch (ProtocolErrorException ex) { // As we are starting up, check for protocol version errors. // There is no other way than checking the error message from Cassandra if (ex.Message.Contains("Invalid or unsupported protocol version")) { throw new UnsupportedProtocolVersionException(protocolVersion, Serializer.ProtocolVersion, ex); } throw; } if (response is AuthenticateResponse) { return(await StartAuthenticationFlow(((AuthenticateResponse)response).Authenticator) .ConfigureAwait(false)); } if (response is ReadyResponse) { return(response); } throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name); }
public static byte[] Compress(byte[] uncompressed) { var target = new SnappyCompressor(); var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; var count = target.Compress(uncompressed, 0, uncompressed.Length, result); return(result.Take(count).ToArray()); }
private static byte[] WriteLengthData(int value) { var target = new SnappyCompressor(); var data = new byte[10]; target.WriteUncomressedLength(data, 0, value); return(data); }
private static byte[] WriteLengthData(int value) { var target = new SnappyCompressor(); var data = new byte[10]; target.WriteUncomressedLength(data, 0, value); return data; }
private static object[] EmitLiteralTag(int dataSize, int resultSizeExtenstion) { var target = new SnappyCompressor(); var result = new byte[1 + resultSizeExtenstion]; int outputPosition = target.EmitLiteralTag(result, 0, dataSize); return(new object[] { outputPosition, result }); }
public void returns_index_into_buffer() { var target = new SnappyCompressor(); var data = new byte[10]; var result = target.WriteUncomressedLength(data, 0, 2097150); Assert.Equal(3, result); }
public void emit_literal_copies_bytes_to_destination(int dataSize, byte tagByteValue, int resultSizeExtension) { var target = new SnappyCompressor(); var data = GetRandomData(dataSize); var result = new byte[target.MaxCompressedLength(dataSize)]; var size = target.EmitLiteral(result, 0, data, 0, dataSize, true); Assert.Equal(data, result.Skip(size - dataSize).Take(dataSize)); }
public unsafe (long, bool) Regular() { ulong data = 0; fixed(byte *s1 = _array) { var s2 = s1 + 12; var s2Limit = s1 + _array.Length; return(SnappyCompressor.FindMatchLength(s1, s2, s2Limit, ref data)); } }
public static Byte[] CompressToByteArray(string str) { var data = Encoding.Default.GetBytes(str); var snappy = new SnappyCompressor(); int compressedSize = snappy.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = snappy.Compress(data, 0, data.Length, compressed); return(compressed.Take(result).ToArray()); }
/// <summary> /// 使用snappy算法压缩 /// </summary> /// <param name="uncompressed"></param> /// <param name="offset"></param> /// <param name="uncompressedLength"></param> /// <returns></returns> public static CompressedBytes CompressInSnappy(byte[] uncompressed, int offset, int uncompressedLength) { SnappyCompressor sc = new SnappyCompressor(); //var bytes = Encoding.ASCII.GetBytes("HelloWor134ertegsdfgsfdgsdfgsdfgsfdgsdfgsdfgsdfgsdfgdsfgsdfgdsfgdfgdsfgld"); byte[] outBytes = new byte[sc.MaxCompressedLength(uncompressed.Length)]; int actualLength = sc.Compress(uncompressed, 0, uncompressedLength, outBytes); return(new CompressedBytes() { ContentBytes = outBytes, Length = actualLength }); }
public void round_trip_returns_original_data(string fileName) { byte[] uncompressed = File.ReadAllBytes(fileName); var target = new SnappyCompressor(); var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; int size = target.Compress(uncompressed, 0, uncompressed.Length, result); var target2 = new SnappyDecompressor(); var sizes = target2.ReadUncompressedLength(result, 0); var bytes = new byte[sizes[0]]; target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]); Assert.Equal(uncompressed, bytes); }
public void compress_random_data() { var data = GetRandomData(4096); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decomperssor = new SnappyDecompressor(); var bytes = decomperssor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes) { var data = GetRandomData(dataSize); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; target.Compress(data, 0, data.Length, compressed); var decompressor = new SnappyDecompressor(); var result = decompressor.ReadUncompressedLength(compressed, 0); Assert.Equal(dataSize, result[0]); Assert.Equal(storageBytes, result[1]); }
public void compress_constant_data_is_one_literal_and_one_copy() { // 1024 a's var data = Encoding.Default.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.Equal(50, result); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_constant_data_is_one_literal_and_one_copy() { // 1024 a's var data = Encoding.UTF8.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.Equal(52, result); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
public void compress_returns_bytes_copied() { var data = Encoding.Default.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); Assert.Equal(15, result); // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them. var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Console.Write(Encoding.Default.GetString(bytes)); }
public unsafe void FindMatchLength(int expectedResult, string s1String, string s2String, int length) { var array = Encoding.ASCII.GetBytes(s1String + s2String + new string('\0', Math.Max(0, length - s2String.Length))); ulong data = 0; fixed(byte *s1 = array) { byte *s2 = s1 + s1String.Length; var result = SnappyCompressor.FindMatchLength(s1, s2, s2 + length, ref data); Assert.Equal(result.matchLength < 8, result.matchLengthLessThan8); Assert.Equal(expectedResult, result.matchLength); } }
/// <summary>Are the native snappy libraries loaded & initialized?</summary> public static void CheckNativeCodeLoaded() { if (!NativeCodeLoader.IsNativeCodeLoaded() || !NativeCodeLoader.BuildSupportsSnappy ()) { throw new RuntimeException("native snappy library not available: " + "this version of libhadoop was built without " + "snappy support."); } if (!SnappyCompressor.IsNativeCodeLoaded()) { throw new RuntimeException("native snappy library not available: " + "SnappyCompressor has not been loaded." ); } if (!SnappyDecompressor.IsNativeCodeLoaded()) { throw new RuntimeException("native snappy library not available: " + "SnappyDecompressor has not been loaded." ); } }
public void compress_returns_bytes_copied() { var data = Encoding.UTF8.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis"); var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); Assert.Equal(12, result); // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them. var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Console.Write(Encoding.UTF8.GetString(bytes)); }
public void compress_multiple_blocks() { /*! * this ends up being uncompressible because it is random. */ var rand = new Random().Next(1 << 4, 1 << 10); var data = GetRandomData((1 << 20) + rand); // 1MB + a bit random so there is an uneven block at end. var target = new SnappyCompressor(); int compressedSize = target.MaxCompressedLength(data.Length); var compressed = new byte[compressedSize]; int result = target.Compress(data, 0, data.Length, compressed); Assert.True(result < compressedSize); var decompressor = new SnappyDecompressor(); var bytes = decompressor.Decompress(compressed, 0, result); Assert.Equal(data, bytes); }
static void Main(string[] args) { var isPix = args[0] == "-p"; var rooms = args.Length == 3 ? int.Parse(args[2]) : 10; ICompressor compressor; switch (args[1]) { case "-l": compressor = new LZ4Compressor(); break; case "-s": compressor = new SnappyCompressor(); break; default: compressor = new NoCompression(); break; } for (var i = 0; i < rooms * 12; i++) { if (isPix) { Task.Run((Action) new PixClient(compressor).Start); } else { Task.Run((Action) new LiteClient(compressor).Start); } } Console.ReadLine(); }
/// <summary> /// Initializes the connection. /// </summary> /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception> /// <exception cref="AuthenticationException" /> /// <exception cref="UnsupportedProtocolVersionException"></exception> public Task<Response> Open() { _freeOperations = new ConcurrentStack<short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse()); _pendingOperations = new ConcurrentDictionary<short, OperationState>(); _writeQueue = new ConcurrentQueue<OperationState>(); if (Options.CustomCompressor != null) { Compressor = Options.CustomCompressor; } else if (Options.Compression == CompressionType.LZ4) { Compressor = new LZ4Compressor(); } else if (Options.Compression == CompressionType.Snappy) { Compressor = new SnappyCompressor(); } //Init TcpSocket _tcpSocket.Init(); _tcpSocket.Error += CancelPending; _tcpSocket.Closing += () => CancelPending(null, null); //Read and write event handlers are going to be invoked using IO Threads _tcpSocket.Read += ReadHandler; _tcpSocket.WriteCompleted += WriteCompletedHandler; return _tcpSocket .Connect() .Then(_ => Startup()) .ContinueWith(t => { if (t.IsFaulted && t.Exception != null) { //Adapt the inner exception and rethrow var ex = t.Exception.InnerException; if (ex is ProtocolErrorException) { //As we are starting up, check for protocol version errors //There is no other way than checking the error message from Cassandra if (ex.Message.Contains("Invalid or unsupported protocol version")) { throw new UnsupportedProtocolVersionException(ProtocolVersion, ex); } } if (ex is ServerErrorException && ProtocolVersion >= 3 && ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version")) { //For some versions of Cassandra, the error is wrapped into a server error //See CASSANDRA-9451 throw new UnsupportedProtocolVersionException(ProtocolVersion, ex); } throw ex; } return t.Result; }, TaskContinuationOptions.ExecuteSynchronously) .Then(response => { if (response is AuthenticateResponse) { return Authenticate(); } if (response is ReadyResponse) { return TaskHelper.ToTask(response); } throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name); }); }
/// <summary> /// Initializes the connection. Thread safe. /// </summary> /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception> /// <exception cref="AuthenticationException" /> /// <exception cref="UnsupportedProtocolVersionException"></exception> public virtual void Init() { _freeOperations = new ConcurrentStack<short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse()); _pendingOperations = new ConcurrentDictionary<short, OperationState>(); _writeQueue = new ConcurrentQueue<OperationState>(); if (Options.CustomCompressor != null) { Compressor = Options.CustomCompressor; } else if (Options.Compression == CompressionType.LZ4) { Compressor = new LZ4Compressor(); } else if (Options.Compression == CompressionType.Snappy) { Compressor = new SnappyCompressor(); } //Init TcpSocket _tcpSocket.Init(); _tcpSocket.Error += CancelPending; _tcpSocket.Closing += () => CancelPending(null, null); //Read and write event handlers are going to be invoked using IO Threads _tcpSocket.Read += ReadHandler; _tcpSocket.WriteCompleted += WriteCompletedHandler; _tcpSocket.Connect(); var startupTask = Startup(); try { TaskHelper.WaitToComplete(startupTask, _tcpSocket.Options.ConnectTimeoutMillis); } catch (ProtocolErrorException ex) { //As we are starting up, check for protocol version errors //There is no other way than checking the error message from Cassandra if (ex.Message.Contains("Invalid or unsupported protocol version")) { throw new UnsupportedProtocolVersionException(ProtocolVersion, ex); } throw; } catch (ServerErrorException ex) { if (ProtocolVersion >= 3 && ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version")) { //For some versions of Cassandra, the error is wrapped into a server error //See CASSANDRA-9451 throw new UnsupportedProtocolVersionException(ProtocolVersion, ex); } throw; } if (startupTask.Result is AuthenticateResponse) { Authenticate(); } else if (!(startupTask.Result is ReadyResponse)) { throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + startupTask.Result.GetType().Name); } }
public static int Compress(byte[] uncompressed, int offset, int size, byte[] output, int outputOffset) { var target = new SnappyCompressor(); return(target.Compress(uncompressed, offset, size, output, outputOffset)); }
public static int MaxCompressedLength(int sourceLength) { var compressor = new SnappyCompressor(); return(compressor.MaxCompressedLength(sourceLength)); }
/// <summary> /// Compress a block of Snappy data. /// </summary> /// <param name="input">Data to compress.</param> /// <param name="output">Buffer to receive the compressed data.</param> /// <returns>Number of bytes written to <paramref name="output"/>.</returns> /// <remarks> /// The output buffer must be large enough to contain the compressed output. /// </remarks> public static int Compress(ReadOnlySpan <byte> input, Span <byte> output) { var compressor = new SnappyCompressor(); return(compressor.Compress(input, output)); }
/// <summary> /// Initializes the connection. /// </summary> /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception> /// <exception cref="AuthenticationException" /> /// <exception cref="UnsupportedProtocolVersionException"></exception> public Task <Response> Open() { _freeOperations = new ConcurrentStack <short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse()); _pendingOperations = new ConcurrentDictionary <short, OperationState>(); _writeQueue = new ConcurrentQueue <OperationState>(); if (Options.CustomCompressor != null) { Compressor = Options.CustomCompressor; } else if (Options.Compression == CompressionType.LZ4) { #if !NETCORE Compressor = new LZ4Compressor(); #else return(TaskHelper.FromException <Response>(new NotSupportedException("Lz4 compression not supported under .NETCore"))); #endif } else if (Options.Compression == CompressionType.Snappy) { Compressor = new SnappyCompressor(); } //Init TcpSocket _tcpSocket.Init(); _tcpSocket.Error += CancelPending; _tcpSocket.Closing += () => CancelPending(null); //Read and write event handlers are going to be invoked using IO Threads _tcpSocket.Read += ReadHandler; _tcpSocket.WriteCompleted += WriteCompletedHandler; var protocolVersion = _serializer.ProtocolVersion; return(_tcpSocket .Connect() .Then(_ => Startup()) .ContinueWith(t => { if (t.IsFaulted && t.Exception != null) { //Adapt the inner exception and rethrow var ex = t.Exception.InnerException; if (ex is ProtocolErrorException) { //As we are starting up, check for protocol version errors //There is no other way than checking the error message from Cassandra if (ex.Message.Contains("Invalid or unsupported protocol version")) { throw new UnsupportedProtocolVersionException(protocolVersion, ex); } } if (ex is ServerErrorException && protocolVersion.CanStartupResponseErrorBeWrapped() && ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version")) { //For some versions of Cassandra, the error is wrapped into a server error //See CASSANDRA-9451 throw new UnsupportedProtocolVersionException(protocolVersion, ex); } // ReSharper disable once PossibleNullReferenceException throw ex; } return t.Result; }, TaskContinuationOptions.ExecuteSynchronously) .Then(response => { if (response is AuthenticateResponse) { return StartAuthenticationFlow(((AuthenticateResponse)response).Authenticator); } if (response is ReadyResponse) { return TaskHelper.ToTask(response); } throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name); })); }
public static bool IsNativeCodeLoaded() { return(SnappyCompressor.IsNativeCodeLoaded() && SnappyDecompressor.IsNativeCodeLoaded ()); }
public static string GetLibraryName() { return(SnappyCompressor.GetLibraryName()); }
/* HACK: this used to work, but doesn't now. as long as the round-trip stuff works will leave this out. [Theory] [PropertyData("DataFiles")] public void compression_same_as_cpp_output(string fileName) { string compressedFileName = Path.Combine(@"..\..\..\testdata_compressed", Path.GetFileName(fileName+ ".comp") ); byte[] uncompressed = File.ReadAllBytes(fileName); var target = new SnappyCompressor(); var result = new byte[target.MaxCompressedLength(uncompressed.Length)]; int size = target.Compress(uncompressed, 0, uncompressed.Length, result); byte[] compressed = File.ReadAllBytes(compressedFileName); Assert.Equal(Path.GetFileName(fileName + ".comp"), Path.GetFileName(compressedFileName)); Assert.Equal(compressed.Length, size); Assert.Equal(compressed, result.Take(size).ToArray()); } */ private static object[] EmitLiteralTag(int dataSize, int resultSizeExtenstion) { var target = new SnappyCompressor(); var result = new byte[1 + resultSizeExtenstion]; int outputPosition = target.EmitLiteralTag(result, 0, dataSize); return new object[] { outputPosition, result}; }