Пример #1
0
        private static void VerifyRoundTrip(string fileName)
        {
            int size = 0;

            byte[] uncompressed = File.ReadAllBytes(fileName);
            var    compressor   = new SnappyCompressor();
            var    result       = new byte[compressor.MaxCompressedLength(uncompressed.Length)];

            size = compressor.Compress(uncompressed, 0, uncompressed.Length, result);
            Array.Resize(ref result, size);

            var decompressor = new SnappyDecompressor();
            var decompressed = decompressor.Decompress(result, 0, size);

            byte[] source = File.ReadAllBytes(fileName);
            if (source.Length != decompressed.Length)
            {
                throw new Exception(string.Format("Decompressed length {0} does not match original {1}", decompressed.Length, source.Length));
            }
            for (int i = 0; i < uncompressed.Length; i++)
            {
                if (source[i] != decompressed[i])
                {
                    throw new Exception(string.Format("Decompressed data did not match original at index {0}", i));
                }
            }
        }
Пример #2
0
    /// <summary>
    /// Initializes a new instance of the <see cref="SnappyStream"/> class.
    /// </summary>
    /// <param name="s">The stream.</param>
    /// <param name="mode">The compression mode.</param>
    /// <param name="leaveOpen">If Set to <c>true</c> leaves the stream open when complete.</param>
    /// <param name="checksum"><c>true</c> if checksums should be written to the stream </param>
    public SnappyStream(Stream s, CompressionMode mode, bool leaveOpen, bool checksum)
    {
        stream          = s;
        compressionMode = mode;
        leaveStreamOpen = leaveOpen;
        writeChecksums  = checksum;

        if (compressionMode == CompressionMode.Decompress)
        {
            if (!stream.CanRead)
            {
                throw new InvalidOperationException("Trying to decompress and underlying stream not readable.");
            }

            decompressor = new SnappyDecompressor();

            CheckStreamIdentifier();
            CheckStreamHeader();
        }
        if (compressionMode == CompressionMode.Compress)
        {
            if (!stream.CanWrite)
            {
                throw new InvalidOperationException("Trying to compress and underlying stream is not writable.");
            }

            compressor = new SnappyCompressor();

            stream.WriteByte(StreamIdentifier);
            stream.Write(StreamHeader, 0, StreamHeader.Length);
        }
    }
Пример #3
0
        static void Main(string[] args)
        {
            var isPix = args[0] == "-p";

            ICompressor compressor;

            switch (args[1])
            {
            case "-l":
                compressor = new LZ4Compressor();
                break;

            case "-s":
                compressor = new SnappyCompressor();
                break;

            default:
                compressor = new NoCompression();
                break;
            }

            if (isPix)
            {
                new PixServer(compressor).Start();
            }
            else
            {
                new LiteServer(compressor).Start();
            }
        }
Пример #4
0
        static CompressionResult RunCompression(string fileName, int iterations)
        {
            int size = 0;

            long[] ticks        = new long[iterations];
            byte[] uncompressed = File.ReadAllBytes(fileName);

            var target = new SnappyCompressor();
            var s      = new Stopwatch();

            for (int i = 0; i < iterations; i++)
            {
                var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
                s.Start();
                size = target.Compress(uncompressed, 0, uncompressed.Length, result);
                s.Stop();
                ticks[i] = s.ElapsedTicks;
                s.Reset();
            }

            return(new CompressionResult
            {
                Direction = CompressionDirection.Compress,
                FileName = FileMap[Path.GetFileName(fileName)],
                CompressedSize = size,
                FileBytes = uncompressed.Length,
                ElapsedTime = new TimeSpan(ticks.Sum()),
                StandardDeviation = StdDev(ticks),
                Iterations = iterations
            });
        }
Пример #5
0
        static void Main(string[] args)
        {
            var lz4    = new LZ4Compressor();
            var snappy = new SnappyCompressor();

            for (var i = 0; i < 100; i++)
            {
                var model = new WorldState2()
                {
                    Players = Enumerable.Range(0, 1000).Select(_ => 1).ToArray()
                };
                byte[] serialized;
                using (var m = new MemoryStream())
                {
                    ProtoBuf.Serializer.NonGeneric.Serialize(m, model);
                    serialized = m.ToArray();
                }

                var lz4Length   = lz4.Compress(serialized).Length;
                var snappLength = snappy.Compress(serialized).Length;
                Console.WriteLine($"{serialized.Length}\t{lz4Length}\t{snappLength}\t{lz4Length - snappLength}");
            }

            Console.ReadLine();
        }
Пример #6
0
        private void TestFindMatchBug_Internal(SnappyCompressor compressor)
        {
            var decompressor = new SnappyDecompressor();

            var size = 1024;

            var data = new byte[size];

            for (var i = 0; i < data.Length; ++i)
            {
                data[i] = (byte)(i & 0xff);
            }

            data[1021] = 5;
            data[1022] = 5;
            data[1023] = 5;

            var compressed = new byte[compressor.MaxCompressedLength(data.Length)];

            var compressedLength = compressor.Compress(data, 0, data.Length, compressed, 0);
            var decompressed     = decompressor.Decompress(compressed, 0, compressedLength);

            for (var i = 0; i < data.Length; ++i)
            {
                Assert.Equal(data[i], decompressed[i]);
            }
        }
Пример #7
0
        /// <summary>
        /// Initializes the connection.
        /// </summary>
        /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
        /// <exception cref="AuthenticationException" />
        /// <exception cref="UnsupportedProtocolVersionException"></exception>
        public async Task <Response> DoOpen()
        {
            _freeOperations    = new ConcurrentStack <short>(Enumerable.Range(0, GetMaxConcurrentRequests(Serializer)).Select(s => (short)s).Reverse());
            _pendingOperations = new ConcurrentDictionary <short, OperationState>();
            _writeQueue        = new ConcurrentQueue <OperationState>();

            if (Options.CustomCompressor != null)
            {
                Compressor = Options.CustomCompressor;
            }
            else if (Options.Compression == CompressionType.LZ4)
            {
#if NETFRAMEWORK
                Compressor = new LZ4Compressor();
#else
                throw new NotSupportedException("Lz4 compression not supported under .NETCore");
#endif
            }
            else if (Options.Compression == CompressionType.Snappy)
            {
                Compressor = new SnappyCompressor();
            }

            //Init TcpSocket
            _tcpSocket.Init();
            _tcpSocket.Error   += CancelPending;
            _tcpSocket.Closing += () => CancelPending(null);
            //Read and write event handlers are going to be invoked using IO Threads
            _tcpSocket.Read           += ReadHandler;
            _tcpSocket.WriteCompleted += WriteCompletedHandler;
            var protocolVersion = Serializer.ProtocolVersion;
            await _tcpSocket.Connect().ConfigureAwait(false);

            Response response;
            try
            {
                response = await Startup().ConfigureAwait(false);
            }
            catch (ProtocolErrorException ex)
            {
                // As we are starting up, check for protocol version errors.
                // There is no other way than checking the error message from Cassandra
                if (ex.Message.Contains("Invalid or unsupported protocol version"))
                {
                    throw new UnsupportedProtocolVersionException(protocolVersion, Serializer.ProtocolVersion, ex);
                }
                throw;
            }
            if (response is AuthenticateResponse)
            {
                return(await StartAuthenticationFlow(((AuthenticateResponse)response).Authenticator)
                       .ConfigureAwait(false));
            }
            if (response is ReadyResponse)
            {
                return(response);
            }
            throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name);
        }
Пример #8
0
    public static byte[] Compress(byte[] uncompressed)
    {
        var target = new SnappyCompressor();
        var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
        var count  = target.Compress(uncompressed, 0, uncompressed.Length, result);

        return(result.Take(count).ToArray());
    }
        private static byte[] WriteLengthData(int value)
        {
            var target = new SnappyCompressor();
            var data   = new byte[10];

            target.WriteUncomressedLength(data, 0, value);
            return(data);
        }
Пример #10
0
        private static byte[] WriteLengthData(int value)
        {
            var target = new SnappyCompressor();
            var data = new byte[10];

            target.WriteUncomressedLength(data, 0, value);
            return data;
        }
Пример #11
0
        private static object[] EmitLiteralTag(int dataSize, int resultSizeExtenstion)
        {
            var target = new SnappyCompressor();
            var result = new byte[1 + resultSizeExtenstion];

            int outputPosition = target.EmitLiteralTag(result, 0, dataSize);

            return(new object[] { outputPosition, result });
        }
        public void returns_index_into_buffer()
        {
            var target = new SnappyCompressor();
            var data   = new byte[10];

            var result = target.WriteUncomressedLength(data, 0, 2097150);

            Assert.Equal(3, result);
        }
Пример #13
0
        public void returns_index_into_buffer()
        {
            var target = new SnappyCompressor();
            var data = new byte[10];

            var result = target.WriteUncomressedLength(data, 0, 2097150);

            Assert.Equal(3, result);
        }
Пример #14
0
        public void emit_literal_copies_bytes_to_destination(int dataSize, byte tagByteValue, int resultSizeExtension)
        {
            var target = new SnappyCompressor();
            var data   = GetRandomData(dataSize);
            var result = new byte[target.MaxCompressedLength(dataSize)];

            var size = target.EmitLiteral(result, 0, data, 0, dataSize, true);

            Assert.Equal(data, result.Skip(size - dataSize).Take(dataSize));
        }
Пример #15
0
        public unsafe (long, bool) Regular()
        {
            ulong data = 0;

            fixed(byte *s1 = _array)
            {
                var s2      = s1 + 12;
                var s2Limit = s1 + _array.Length;

                return(SnappyCompressor.FindMatchLength(s1, s2, s2Limit, ref data));
            }
        }
Пример #16
0
        public static Byte[] CompressToByteArray(string str)
        {
            var data   = Encoding.Default.GetBytes(str);
            var snappy = new SnappyCompressor();

            int compressedSize = snappy.MaxCompressedLength(data.Length);
            var compressed     = new byte[compressedSize];

            int result = snappy.Compress(data, 0, data.Length, compressed);

            return(compressed.Take(result).ToArray());
        }
Пример #17
0
        /// <summary>
        /// 使用snappy算法压缩
        /// </summary>
        /// <param name="uncompressed"></param>
        /// <param name="offset"></param>
        /// <param name="uncompressedLength"></param>
        /// <returns></returns>
        public static CompressedBytes CompressInSnappy(byte[] uncompressed, int offset, int uncompressedLength)
        {
            SnappyCompressor sc = new SnappyCompressor();

            //var bytes = Encoding.ASCII.GetBytes("HelloWor134ertegsdfgsfdgsdfgsdfgsfdgsdfgsdfgsdfgsdfgdsfgsdfgdsfgdfgdsfgld");
            byte[] outBytes = new byte[sc.MaxCompressedLength(uncompressed.Length)];

            int actualLength = sc.Compress(uncompressed, 0, uncompressedLength, outBytes);

            return(new CompressedBytes()
            {
                ContentBytes = outBytes, Length = actualLength
            });
        }
Пример #18
0
        public void round_trip_returns_original_data(string fileName)
        {
            byte[] uncompressed = File.ReadAllBytes(fileName);
            var target = new SnappyCompressor();
            var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
            int size = target.Compress(uncompressed, 0, uncompressed.Length, result);


            var target2 = new SnappyDecompressor();
            var sizes = target2.ReadUncompressedLength(result, 0);
            var bytes = new byte[sizes[0]];
            target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]);

            Assert.Equal(uncompressed, bytes);
        }
        public void compress_random_data()
        {
            var data = GetRandomData(4096);
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.True(result < compressedSize); 

            var decomperssor = new SnappyDecompressor();
            var bytes = decomperssor.Decompress(compressed, 0, result);
            Assert.Equal(data, bytes);
        }
Пример #20
0
        public void round_trip_returns_original_data(string fileName)
        {
            byte[] uncompressed = File.ReadAllBytes(fileName);
            var    target       = new SnappyCompressor();
            var    result       = new byte[target.MaxCompressedLength(uncompressed.Length)];
            int    size         = target.Compress(uncompressed, 0, uncompressed.Length, result);


            var target2 = new SnappyDecompressor();
            var sizes   = target2.ReadUncompressedLength(result, 0);
            var bytes   = new byte[sizes[0]];

            target2.Decompress(result, 0 + sizes[1], size - sizes[1], bytes, 0, sizes[1]);

            Assert.Equal(uncompressed, bytes);
        }
Пример #21
0
        public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes)
        {
            var data   = GetRandomData(dataSize);
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed     = new byte[compressedSize];

            target.Compress(data, 0, data.Length, compressed);

            var decompressor = new SnappyDecompressor();
            var result       = decompressor.ReadUncompressedLength(compressed, 0);

            Assert.Equal(dataSize, result[0]);
            Assert.Equal(storageBytes, result[1]);
        }
Пример #22
0
        public void compress_random_data()
        {
            var data   = GetRandomData(4096);
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed     = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.True(result < compressedSize);

            var decomperssor = new SnappyDecompressor();
            var bytes        = decomperssor.Decompress(compressed, 0, result);

            Assert.Equal(data, bytes);
        }
        public void compress_constant_data_is_one_literal_and_one_copy()
        {
            // 1024 a's
            var data = Encoding.Default.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); 
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.Equal(50, result);

            var decompressor = new SnappyDecompressor();
            var bytes = decompressor.Decompress(compressed, 0, result);
            Assert.Equal(data, bytes);
        }
Пример #24
0
        public void compress_constant_data_is_one_literal_and_one_copy()
        {
            // 1024 a's
            var data   = Encoding.UTF8.GetBytes("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed     = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.Equal(52, result);

            var decompressor = new SnappyDecompressor();
            var bytes        = decompressor.Decompress(compressed, 0, result);

            Assert.Equal(data, bytes);
        }
        public void compress_returns_bytes_copied()
        {
            var data = Encoding.Default.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis"); 
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.True(result < compressedSize);
            Assert.Equal(15, result);

            // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them.
            var decompressor = new SnappyDecompressor();
            var bytes = decompressor.Decompress(compressed, 0, result);
            Console.Write(Encoding.Default.GetString(bytes));
        }
Пример #26
0
        public unsafe void FindMatchLength(int expectedResult, string s1String, string s2String, int length)
        {
            var array = Encoding.ASCII.GetBytes(s1String + s2String
                                                + new string('\0', Math.Max(0, length - s2String.Length)));

            ulong data = 0;

            fixed(byte *s1 = array)
            {
                byte *s2 = s1 + s1String.Length;

                var result =
                    SnappyCompressor.FindMatchLength(s1, s2, s2 + length, ref data);

                Assert.Equal(result.matchLength < 8, result.matchLengthLessThan8);
                Assert.Equal(expectedResult, result.matchLength);
            }
        }
Пример #27
0
 /// <summary>Are the native snappy libraries loaded & initialized?</summary>
 public static void CheckNativeCodeLoaded()
 {
     if (!NativeCodeLoader.IsNativeCodeLoaded() || !NativeCodeLoader.BuildSupportsSnappy
             ())
     {
         throw new RuntimeException("native snappy library not available: " + "this version of libhadoop was built without "
                                    + "snappy support.");
     }
     if (!SnappyCompressor.IsNativeCodeLoaded())
     {
         throw new RuntimeException("native snappy library not available: " + "SnappyCompressor has not been loaded."
                                    );
     }
     if (!SnappyDecompressor.IsNativeCodeLoaded())
     {
         throw new RuntimeException("native snappy library not available: " + "SnappyDecompressor has not been loaded."
                                    );
     }
 }
Пример #28
0
        public void compress_returns_bytes_copied()
        {
            var data   = Encoding.UTF8.GetBytes("ThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThisThis");
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed     = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.True(result < compressedSize);
            Assert.Equal(12, result);

            // TODO: instead of decompressing, we should traverse the buffer looking for tag bytes and interpreting them.
            var decompressor = new SnappyDecompressor();
            var bytes        = decompressor.Decompress(compressed, 0, result);

            Console.Write(Encoding.UTF8.GetString(bytes));
        }
        public void compress_multiple_blocks()
        {
            /*!
             * this ends up being uncompressible because it is random.
             */
            var rand = new Random().Next(1 << 4, 1 << 10);
            var data = GetRandomData((1 << 20) + rand); // 1MB  + a bit random so there is an uneven block at end.
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.True(result < compressedSize); 

            var decompressor = new SnappyDecompressor();
            var bytes = decompressor.Decompress(compressed, 0, result);
            Assert.Equal(data, bytes);
        }
Пример #30
0
        public void compress_multiple_blocks()
        {
            /*!
             * this ends up being uncompressible because it is random.
             */
            var rand   = new Random().Next(1 << 4, 1 << 10);
            var data   = GetRandomData((1 << 20) + rand); // 1MB  + a bit random so there is an uneven block at end.
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed     = new byte[compressedSize];

            int result = target.Compress(data, 0, data.Length, compressed);

            Assert.True(result < compressedSize);

            var decompressor = new SnappyDecompressor();
            var bytes        = decompressor.Decompress(compressed, 0, result);

            Assert.Equal(data, bytes);
        }
Пример #31
0
        static void Main(string[] args)
        {
            var isPix = args[0] == "-p";
            var rooms = args.Length == 3 ? int.Parse(args[2]) : 10;

            ICompressor compressor;

            switch (args[1])
            {
            case "-l":
                compressor = new LZ4Compressor();
                break;

            case "-s":
                compressor = new SnappyCompressor();
                break;

            default:
                compressor = new NoCompression();
                break;
            }

            for (var i = 0; i < rooms * 12; i++)
            {
                if (isPix)
                {
                    Task.Run((Action) new PixClient(compressor).Start);
                }
                else
                {
                    Task.Run((Action) new LiteClient(compressor).Start);
                }
            }

            Console.ReadLine();
        }
Пример #32
0
        /// <summary>
        /// Initializes the connection.
        /// </summary>
        /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
        /// <exception cref="AuthenticationException" />
        /// <exception cref="UnsupportedProtocolVersionException"></exception>
        public Task<Response> Open()
        {
            _freeOperations = new ConcurrentStack<short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse());
            _pendingOperations = new ConcurrentDictionary<short, OperationState>();
            _writeQueue = new ConcurrentQueue<OperationState>();

            if (Options.CustomCompressor != null)
            {
                Compressor = Options.CustomCompressor;
            }
            else if (Options.Compression == CompressionType.LZ4)
            {
                Compressor = new LZ4Compressor();
            }
            else if (Options.Compression == CompressionType.Snappy)
            {
                Compressor = new SnappyCompressor();
            }

            //Init TcpSocket
            _tcpSocket.Init();
            _tcpSocket.Error += CancelPending;
            _tcpSocket.Closing += () => CancelPending(null, null);
            //Read and write event handlers are going to be invoked using IO Threads
            _tcpSocket.Read += ReadHandler;
            _tcpSocket.WriteCompleted += WriteCompletedHandler;
            return _tcpSocket
                .Connect()
                .Then(_ => Startup())
                .ContinueWith(t =>
                {
                    if (t.IsFaulted && t.Exception != null)
                    {
                        //Adapt the inner exception and rethrow
                        var ex = t.Exception.InnerException;
                        if (ex is ProtocolErrorException)
                        {
                            //As we are starting up, check for protocol version errors
                            //There is no other way than checking the error message from Cassandra
                            if (ex.Message.Contains("Invalid or unsupported protocol version"))
                            {
                                throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
                            }
                        }
                        if (ex is ServerErrorException && ProtocolVersion >= 3 && ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version"))
                        {
                            //For some versions of Cassandra, the error is wrapped into a server error
                            //See CASSANDRA-9451
                            throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
                        }
                        throw ex;
                    }
                    return t.Result;
                }, TaskContinuationOptions.ExecuteSynchronously)
                .Then(response =>
                {
                    if (response is AuthenticateResponse)
                    {
                        return Authenticate();
                    }
                    if (response is ReadyResponse)
                    {
                        return TaskHelper.ToTask(response);
                    }
                    throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name);
                });
        }
Пример #33
0
        /// <summary>
        /// Initializes the connection. Thread safe.
        /// </summary>
        /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
        /// <exception cref="AuthenticationException" />
        /// <exception cref="UnsupportedProtocolVersionException"></exception>
        public virtual void Init()
        {
            _freeOperations = new ConcurrentStack<short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse());
            _pendingOperations = new ConcurrentDictionary<short, OperationState>();
            _writeQueue = new ConcurrentQueue<OperationState>();

            if (Options.CustomCompressor != null)
            {
                Compressor = Options.CustomCompressor;
            }
            else if (Options.Compression == CompressionType.LZ4)
            {
                Compressor = new LZ4Compressor();
            }
            else if (Options.Compression == CompressionType.Snappy)
            {
                Compressor = new SnappyCompressor();
            }

            //Init TcpSocket
            _tcpSocket.Init();
            _tcpSocket.Error += CancelPending;
            _tcpSocket.Closing += () => CancelPending(null, null);
            //Read and write event handlers are going to be invoked using IO Threads
            _tcpSocket.Read += ReadHandler;
            _tcpSocket.WriteCompleted += WriteCompletedHandler;
            _tcpSocket.Connect();

            var startupTask = Startup();
            try
            {
                TaskHelper.WaitToComplete(startupTask, _tcpSocket.Options.ConnectTimeoutMillis);
            }
            catch (ProtocolErrorException ex)
            {
                //As we are starting up, check for protocol version errors
                //There is no other way than checking the error message from Cassandra
                if (ex.Message.Contains("Invalid or unsupported protocol version"))
                {
                    throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
                }
                throw;
            }
            catch (ServerErrorException ex)
            {
                if (ProtocolVersion >= 3 && ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version"))
                {
                    //For some versions of Cassandra, the error is wrapped into a server error
                    //See CASSANDRA-9451
                    throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
                }
                throw;
            }
            if (startupTask.Result is AuthenticateResponse)
            {
                Authenticate();
            }
            else if (!(startupTask.Result is ReadyResponse))
            {
                throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + startupTask.Result.GetType().Name);
            }
        }
Пример #34
0
    public static int Compress(byte[] uncompressed, int offset, int size, byte[] output, int outputOffset)
    {
        var target = new SnappyCompressor();

        return(target.Compress(uncompressed, offset, size, output, outputOffset));
    }
        public void emit_literal_copies_bytes_to_destination(int dataSize, byte tagByteValue, int resultSizeExtension)
        {
            var target = new SnappyCompressor();
            var data = GetRandomData(dataSize);
            var result = new byte[target.MaxCompressedLength(dataSize)];

            var size = target.EmitLiteral(result, 0, data, 0, dataSize, true);

            Assert.Equal(data, result.Skip(size - dataSize).Take(dataSize));
        }
Пример #36
0
    public static int MaxCompressedLength(int sourceLength)
    {
        var compressor = new SnappyCompressor();

        return(compressor.MaxCompressedLength(sourceLength));
    }
Пример #37
0
        /// <summary>
        /// Compress a block of Snappy data.
        /// </summary>
        /// <param name="input">Data to compress.</param>
        /// <param name="output">Buffer to receive the compressed data.</param>
        /// <returns>Number of bytes written to <paramref name="output"/>.</returns>
        /// <remarks>
        /// The output buffer must be large enough to contain the compressed output.
        /// </remarks>
        public static int Compress(ReadOnlySpan <byte> input, Span <byte> output)
        {
            var compressor = new SnappyCompressor();

            return(compressor.Compress(input, output));
        }
Пример #38
0
        /// <summary>
        /// Initializes the connection.
        /// </summary>
        /// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
        /// <exception cref="AuthenticationException" />
        /// <exception cref="UnsupportedProtocolVersionException"></exception>
        public Task <Response> Open()
        {
            _freeOperations    = new ConcurrentStack <short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse());
            _pendingOperations = new ConcurrentDictionary <short, OperationState>();
            _writeQueue        = new ConcurrentQueue <OperationState>();

            if (Options.CustomCompressor != null)
            {
                Compressor = Options.CustomCompressor;
            }
            else if (Options.Compression == CompressionType.LZ4)
            {
#if !NETCORE
                Compressor = new LZ4Compressor();
#else
                return(TaskHelper.FromException <Response>(new NotSupportedException("Lz4 compression not supported under .NETCore")));
#endif
            }
            else if (Options.Compression == CompressionType.Snappy)
            {
                Compressor = new SnappyCompressor();
            }

            //Init TcpSocket
            _tcpSocket.Init();
            _tcpSocket.Error   += CancelPending;
            _tcpSocket.Closing += () => CancelPending(null);
            //Read and write event handlers are going to be invoked using IO Threads
            _tcpSocket.Read           += ReadHandler;
            _tcpSocket.WriteCompleted += WriteCompletedHandler;
            var protocolVersion = _serializer.ProtocolVersion;
            return(_tcpSocket
                   .Connect()
                   .Then(_ => Startup())
                   .ContinueWith(t =>
            {
                if (t.IsFaulted && t.Exception != null)
                {
                    //Adapt the inner exception and rethrow
                    var ex = t.Exception.InnerException;
                    if (ex is ProtocolErrorException)
                    {
                        //As we are starting up, check for protocol version errors
                        //There is no other way than checking the error message from Cassandra
                        if (ex.Message.Contains("Invalid or unsupported protocol version"))
                        {
                            throw new UnsupportedProtocolVersionException(protocolVersion, ex);
                        }
                    }
                    if (ex is ServerErrorException && protocolVersion.CanStartupResponseErrorBeWrapped() &&
                        ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version"))
                    {
                        //For some versions of Cassandra, the error is wrapped into a server error
                        //See CASSANDRA-9451
                        throw new UnsupportedProtocolVersionException(protocolVersion, ex);
                    }
                    // ReSharper disable once PossibleNullReferenceException
                    throw ex;
                }
                return t.Result;
            }, TaskContinuationOptions.ExecuteSynchronously)
                   .Then(response =>
            {
                if (response is AuthenticateResponse)
                {
                    return StartAuthenticationFlow(((AuthenticateResponse)response).Authenticator);
                }
                if (response is ReadyResponse)
                {
                    return TaskHelper.ToTask(response);
                }
                throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name);
            }));
        }
Пример #39
0
 public static bool IsNativeCodeLoaded()
 {
     return(SnappyCompressor.IsNativeCodeLoaded() && SnappyDecompressor.IsNativeCodeLoaded
                ());
 }
        public void compress_writes_uncompressed_length_first(int dataSize, int storageBytes)
        {
            var data = GetRandomData(dataSize);
            var target = new SnappyCompressor();

            int compressedSize = target.MaxCompressedLength(data.Length);
            var compressed = new byte[compressedSize];

            target.Compress(data, 0, data.Length, compressed);

            var decompressor = new SnappyDecompressor();
            var result = decompressor.ReadUncompressedLength(compressed, 0);

            Assert.Equal(dataSize, result[0]);
            Assert.Equal(storageBytes, result[1]);
        }
Пример #41
0
 public static string GetLibraryName()
 {
     return(SnappyCompressor.GetLibraryName());
 }
        /*
        HACK: this used to work, but doesn't now. as long as the round-trip stuff works will leave this out.
        [Theory]
        [PropertyData("DataFiles")]
        public void compression_same_as_cpp_output(string fileName)
        {
            string compressedFileName = Path.Combine(@"..\..\..\testdata_compressed", Path.GetFileName(fileName+ ".comp") );
            byte[] uncompressed = File.ReadAllBytes(fileName);
            var target = new SnappyCompressor();
            var result = new byte[target.MaxCompressedLength(uncompressed.Length)];
            int size = target.Compress(uncompressed, 0, uncompressed.Length, result);

            byte[] compressed = File.ReadAllBytes(compressedFileName);

            Assert.Equal(Path.GetFileName(fileName + ".comp"), Path.GetFileName(compressedFileName));
            Assert.Equal(compressed.Length, size);
            Assert.Equal(compressed, result.Take(size).ToArray());
        }
        */

        private static object[] EmitLiteralTag(int dataSize, int resultSizeExtenstion)
        {
            var target = new SnappyCompressor();
            var result = new byte[1 + resultSizeExtenstion];

            int outputPosition = target.EmitLiteralTag(result, 0, dataSize);
            return new object[] { outputPosition, result};
        }