public void When_compressing_a_stream_aggresive()
{
var inputBytes = Encoding.UTF8.GetBytes(InputStr);
using (var streamIn = new MemoryStream(inputBytes))
using (ICompressor compressor = new LZ4Compressor())
using (var compressedStream = new MemoryStream())
{
compressedStream.Length.ShouldBe(0);
compressor.Compress(streamIn, compressedStream, CompressionLevel.Aggressive);
var compressedBytes = compressedStream.ToArray();
compressedBytes.Length.ShouldBe(84);
compressedStream.Position = 0;
using (var deCompressedStream = new MemoryStream())
{
compressor.DeCompress(compressedStream, deCompressedStream);
var deCompressedBytes = deCompressedStream.ToArray();
deCompressedBytes.ShouldBe(_inputBytes);
}
compressor.DeCompressAsString(compressedBytes, Encoding.UTF8).ShouldBe(InputStr);
}
}
static void Main(string[] args)
{
var isPix = args[0] == "-p";
ICompressor compressor;
switch (args[1])
{
case "-l":
compressor = new LZ4Compressor();
break;
case "-s":
compressor = new SnappyCompressor();
break;
default:
compressor = new NoCompression();
break;
}
if (isPix)
{
new PixServer(compressor).Start();
}
else
{
new LiteServer(compressor).Start();
}
}
static void Main(string[] args)
{
var lz4 = new LZ4Compressor();
var snappy = new SnappyCompressor();
for (var i = 0; i < 100; i++)
{
var model = new WorldState2()
{
Players = Enumerable.Range(0, 1000).Select(_ => 1).ToArray()
};
byte[] serialized;
using (var m = new MemoryStream())
{
ProtoBuf.Serializer.NonGeneric.Serialize(m, model);
serialized = m.ToArray();
}
var lz4Length = lz4.Compress(serialized).Length;
var snappLength = snappy.Compress(serialized).Length;
Console.WriteLine($"{serialized.Length}\t{lz4Length}\t{snappLength}\t{lz4Length - snappLength}");
}
Console.ReadLine();
}
public ConnectedState(RC4Encryptor encryptor, IMessageFactory messageFactory)
{
this.encryptor = encryptor;
this.messageFactory = messageFactory;
this.compressor = new LZ4Compressor();
}
/// <summary>
/// Initializes the connection.
/// </summary>
/// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
/// <exception cref="AuthenticationException" />
/// <exception cref="UnsupportedProtocolVersionException"></exception>
public async Task <Response> DoOpen()
{
_freeOperations = new ConcurrentStack <short>(Enumerable.Range(0, GetMaxConcurrentRequests(Serializer)).Select(s => (short)s).Reverse());
_pendingOperations = new ConcurrentDictionary <short, OperationState>();
_writeQueue = new ConcurrentQueue <OperationState>();
if (Options.CustomCompressor != null)
{
Compressor = Options.CustomCompressor;
}
else if (Options.Compression == CompressionType.LZ4)
{
#if NETFRAMEWORK
Compressor = new LZ4Compressor();
#else
throw new NotSupportedException("Lz4 compression not supported under .NETCore");
#endif
}
else if (Options.Compression == CompressionType.Snappy)
{
Compressor = new SnappyCompressor();
}
//Init TcpSocket
_tcpSocket.Init();
_tcpSocket.Error += CancelPending;
_tcpSocket.Closing += () => CancelPending(null);
//Read and write event handlers are going to be invoked using IO Threads
_tcpSocket.Read += ReadHandler;
_tcpSocket.WriteCompleted += WriteCompletedHandler;
var protocolVersion = Serializer.ProtocolVersion;
await _tcpSocket.Connect().ConfigureAwait(false);
Response response;
try
{
response = await Startup().ConfigureAwait(false);
}
catch (ProtocolErrorException ex)
{
// As we are starting up, check for protocol version errors.
// There is no other way than checking the error message from Cassandra
if (ex.Message.Contains("Invalid or unsupported protocol version"))
{
throw new UnsupportedProtocolVersionException(protocolVersion, Serializer.ProtocolVersion, ex);
}
throw;
}
if (response is AuthenticateResponse)
{
return(await StartAuthenticationFlow(((AuthenticateResponse)response).Authenticator)
.ConfigureAwait(false));
}
if (response is ReadyResponse)
{
return(response);
}
throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name);
}
private static void DeCompressStreamLZ4()
{
using (var compressor = new LZ4Compressor())
{
for (var i = 0; i < IterationCount; i++)
{
compressor.Compress(_bytes);
}
}
}
public async Task <byte[]> Decompress(CompressionType type, byte[] compressed)
{
if (type != CompressionType.LZ4)
{
throw new ArgumentException("Only LZ4 Compression is supported at the moment.");
}
return(await Task.Run(() =>
{
LZ4Compressor comp = new LZ4Compressor("", K4os.Compression.LZ4.LZ4Level.L12_MAX);
return comp.Decompress(compressed);
}));
}
public void When_decoding_bytes()
{
using (var compressor = new LZ4Compressor())
{
var compressedBytesStd = compressor.Wrap(_inputBytes);
var compressedBytesAggr = compressor.Wrap(_inputBytes, CompressionLevel.Aggressive);
compressedBytesStd.ShouldBe(compressedBytesAggr);
compressor.UnWrap(compressedBytesStd).ShouldBe(_inputBytes);
compressor.UnWrap(compressedBytesAggr).ShouldBe(_inputBytes);
}
}
public void When_compressing_bytes()
{
using (ICompressor compressor = new LZ4Compressor())
{
var compressedBytesStd = compressor.Compress(_inputBytes);
var compressedBytesAggr = compressor.Compress(_inputBytes, CompressionLevel.Aggressive);
compressedBytesStd.ShouldBe(compressedBytesAggr);
compressor.DeCompress(compressedBytesStd).ShouldBe(_inputBytes);
compressor.DeCompress(compressedBytesAggr).ShouldBe(_inputBytes);
}
}
internal void LZ4Compressor_CanCompressAndDecompress()
{
// Arrange
var message = "hello world!";
var compressor = new LZ4Compressor();
// Act
var messageBytes = Encoding.UTF8.GetBytes(message);
var compressed = compressor.Compress(messageBytes);
var decompressed = compressor.Decompress(compressed);
// Assert
Assert.Equal(message, Encoding.UTF8.GetString(decompressed));
this.Output.WriteLine(BitConverter.ToString(compressed));
}
internal void LZ4Compressor_DecompressFromPython()
{
// Arrange
var message = "hello world!";
var expected = Encoding.UTF8.GetBytes(message);
var compressor = new LZ4Compressor();
var fromPython = Convert.FromBase64String("BCJNGGhADAAAAAAAAABdDAAAgGhlbGxvIHdvcmxkIQAAAAA=");
this.Output.WriteLine(Encoding.UTF8.GetString(fromPython));
// Act
var decompressed = compressor.Decompress(fromPython);
// Assert
Assert.Equal(expected, decompressed);
}
public void When_compressing_string_with_default_method()
{
using (ICompressor compressor = new LZ4Compressor())
{
var compressedBytesStd = compressor.Compress(InputStr);
var compressedBytesAggr = compressor.Compress(InputStr, CompressionLevel.Aggressive);
compressedBytesStd.ShouldBe(compressedBytesAggr);
compressor.DeCompressAsString(compressedBytesStd).ShouldBe(InputStr);
compressor.DeCompressAsString(compressedBytesAggr).ShouldBe(InputStr);
compressor.DeCompress(compressedBytesStd).ShouldBe(_inputBytes);
compressor.DeCompress(compressedBytesAggr).ShouldBe(_inputBytes);
}
}
public void When_compressing_string_with_encoding()
{
using (ICompressor compressor = new LZ4Compressor())
{
var inputBytes = Encoding.UTF32.GetBytes(InputStr);
var compressedBytesStd = compressor.Compress(inputBytes);
var compressedBytesAggr = compressor.Compress(inputBytes, CompressionLevel.Aggressive);
compressedBytesStd.ShouldNotBe(compressedBytesAggr);
compressor.DeCompressAsString(compressedBytesStd, Encoding.UTF32).ShouldBe(InputStr);
compressor.DeCompressAsString(compressedBytesAggr, Encoding.UTF32).ShouldBe(InputStr);
compressor.DeCompress(compressedBytesStd).ShouldBe(inputBytes);
compressor.DeCompress(compressedBytesAggr).ShouldBe(inputBytes);
}
}
public GenericSession(Client client, IEventAggregator aggregator, IMessageFactory messageFactory, Handshake handshake)
{
this.client = client;
this.client.Disconnected += this.OnDisconnected;
this.aggregator = aggregator;
this.messageFactory = messageFactory;
this.handshake = handshake;
logger.Trace("Session created.");
this.handshake.HandshakeDone += delegate
{
this.encryptor = new RC4Encryptor(handshake.EncryptionKey);
this.compressor = new LZ4Compressor();
this.client.DataReceived += this.OnDataReceived;
logger.Trace("Handshake done.");
};
}
public void LZ4_Read_Write_Faster_Than_GZip()
{
var gzipCompressor = new GZipCompressor();
var lz4Compressor = new LZ4Compressor();
double gzipSeconds = 0;
double lz4Seconds = 0;
Stopwatch watch = new Stopwatch();
watch.Start();
for (int i = 0; i < 1000; i++)
{
byte[] data = TestHelper.GetRandomByteArray(10);
byte[] compressed = gzipCompressor.Compress(data);
byte[] deflated = gzipCompressor.Decompress(compressed);
Assert.IsTrue(data.SequenceEqual(deflated));
}
watch.Stop();
gzipSeconds = watch.Elapsed.TotalSeconds;
watch = new Stopwatch();
watch.Start();
for (int i = 0; i < 1000; i++)
{
byte[] data = TestHelper.GetRandomByteArray(10);
byte[] compressed = lz4Compressor.Compress(data);
byte[] deflated = lz4Compressor.Decompress(compressed);
Assert.IsTrue(data.SequenceEqual(deflated));
}
watch.Stop();
lz4Seconds = watch.Elapsed.TotalSeconds;
Assert.IsTrue(lz4Seconds < gzipSeconds);
}
static void Main(string[] args)
{
var isPix = args[0] == "-p";
var rooms = args.Length == 3 ? int.Parse(args[2]) : 10;
ICompressor compressor;
switch (args[1])
{
case "-l":
compressor = new LZ4Compressor();
break;
case "-s":
compressor = new SnappyCompressor();
break;
default:
compressor = new NoCompression();
break;
}
for (var i = 0; i < rooms * 12; i++)
{
if (isPix)
{
Task.Run((Action) new PixClient(compressor).Start);
}
else
{
Task.Run((Action) new LiteClient(compressor).Start);
}
}
Console.ReadLine();
}
/// <summary>
/// Initializes the connection.
/// </summary>
/// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
/// <exception cref="AuthenticationException" />
/// <exception cref="UnsupportedProtocolVersionException"></exception>
public Task<Response> Open()
{
_freeOperations = new ConcurrentStack<short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse());
_pendingOperations = new ConcurrentDictionary<short, OperationState>();
_writeQueue = new ConcurrentQueue<OperationState>();
if (Options.CustomCompressor != null)
{
Compressor = Options.CustomCompressor;
}
else if (Options.Compression == CompressionType.LZ4)
{
Compressor = new LZ4Compressor();
}
else if (Options.Compression == CompressionType.Snappy)
{
Compressor = new SnappyCompressor();
}
//Init TcpSocket
_tcpSocket.Init();
_tcpSocket.Error += CancelPending;
_tcpSocket.Closing += () => CancelPending(null, null);
//Read and write event handlers are going to be invoked using IO Threads
_tcpSocket.Read += ReadHandler;
_tcpSocket.WriteCompleted += WriteCompletedHandler;
return _tcpSocket
.Connect()
.Then(_ => Startup())
.ContinueWith(t =>
{
if (t.IsFaulted && t.Exception != null)
{
//Adapt the inner exception and rethrow
var ex = t.Exception.InnerException;
if (ex is ProtocolErrorException)
{
//As we are starting up, check for protocol version errors
//There is no other way than checking the error message from Cassandra
if (ex.Message.Contains("Invalid or unsupported protocol version"))
{
throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
}
}
if (ex is ServerErrorException && ProtocolVersion >= 3 && ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version"))
{
//For some versions of Cassandra, the error is wrapped into a server error
//See CASSANDRA-9451
throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
}
throw ex;
}
return t.Result;
}, TaskContinuationOptions.ExecuteSynchronously)
.Then(response =>
{
if (response is AuthenticateResponse)
{
return Authenticate();
}
if (response is ReadyResponse)
{
return TaskHelper.ToTask(response);
}
throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name);
});
}
/// <summary>
/// Initializes the connection. Thread safe.
/// </summary>
/// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
/// <exception cref="AuthenticationException" />
/// <exception cref="UnsupportedProtocolVersionException"></exception>
public virtual void Init()
{
_freeOperations = new ConcurrentStack<short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse());
_pendingOperations = new ConcurrentDictionary<short, OperationState>();
_writeQueue = new ConcurrentQueue<OperationState>();
if (Options.CustomCompressor != null)
{
Compressor = Options.CustomCompressor;
}
else if (Options.Compression == CompressionType.LZ4)
{
Compressor = new LZ4Compressor();
}
else if (Options.Compression == CompressionType.Snappy)
{
Compressor = new SnappyCompressor();
}
//Init TcpSocket
_tcpSocket.Init();
_tcpSocket.Error += CancelPending;
_tcpSocket.Closing += () => CancelPending(null, null);
//Read and write event handlers are going to be invoked using IO Threads
_tcpSocket.Read += ReadHandler;
_tcpSocket.WriteCompleted += WriteCompletedHandler;
_tcpSocket.Connect();
var startupTask = Startup();
try
{
TaskHelper.WaitToComplete(startupTask, _tcpSocket.Options.ConnectTimeoutMillis);
}
catch (ProtocolErrorException ex)
{
//As we are starting up, check for protocol version errors
//There is no other way than checking the error message from Cassandra
if (ex.Message.Contains("Invalid or unsupported protocol version"))
{
throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
}
throw;
}
catch (ServerErrorException ex)
{
if (ProtocolVersion >= 3 && ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version"))
{
//For some versions of Cassandra, the error is wrapped into a server error
//See CASSANDRA-9451
throw new UnsupportedProtocolVersionException(ProtocolVersion, ex);
}
throw;
}
if (startupTask.Result is AuthenticateResponse)
{
Authenticate();
}
else if (!(startupTask.Result is ReadyResponse))
{
throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + startupTask.Result.GetType().Name);
}
}
/// <summary>
/// Initializes the connection.
/// </summary>
/// <exception cref="SocketException">Throws a SocketException when the connection could not be established with the host</exception>
/// <exception cref="AuthenticationException" />
/// <exception cref="UnsupportedProtocolVersionException"></exception>
public Task <Response> Open()
{
_freeOperations = new ConcurrentStack <short>(Enumerable.Range(0, MaxConcurrentRequests).Select(s => (short)s).Reverse());
_pendingOperations = new ConcurrentDictionary <short, OperationState>();
_writeQueue = new ConcurrentQueue <OperationState>();
if (Options.CustomCompressor != null)
{
Compressor = Options.CustomCompressor;
}
else if (Options.Compression == CompressionType.LZ4)
{
#if !NETCORE
Compressor = new LZ4Compressor();
#else
return(TaskHelper.FromException <Response>(new NotSupportedException("Lz4 compression not supported under .NETCore")));
#endif
}
else if (Options.Compression == CompressionType.Snappy)
{
Compressor = new SnappyCompressor();
}
//Init TcpSocket
_tcpSocket.Init();
_tcpSocket.Error += CancelPending;
_tcpSocket.Closing += () => CancelPending(null);
//Read and write event handlers are going to be invoked using IO Threads
_tcpSocket.Read += ReadHandler;
_tcpSocket.WriteCompleted += WriteCompletedHandler;
var protocolVersion = _serializer.ProtocolVersion;
return(_tcpSocket
.Connect()
.Then(_ => Startup())
.ContinueWith(t =>
{
if (t.IsFaulted && t.Exception != null)
{
//Adapt the inner exception and rethrow
var ex = t.Exception.InnerException;
if (ex is ProtocolErrorException)
{
//As we are starting up, check for protocol version errors
//There is no other way than checking the error message from Cassandra
if (ex.Message.Contains("Invalid or unsupported protocol version"))
{
throw new UnsupportedProtocolVersionException(protocolVersion, ex);
}
}
if (ex is ServerErrorException && protocolVersion.CanStartupResponseErrorBeWrapped() &&
ex.Message.Contains("ProtocolException: Invalid or unsupported protocol version"))
{
//For some versions of Cassandra, the error is wrapped into a server error
//See CASSANDRA-9451
throw new UnsupportedProtocolVersionException(protocolVersion, ex);
}
// ReSharper disable once PossibleNullReferenceException
throw ex;
}
return t.Result;
}, TaskContinuationOptions.ExecuteSynchronously)
.Then(response =>
{
if (response is AuthenticateResponse)
{
return StartAuthenticationFlow(((AuthenticateResponse)response).Authenticator);
}
if (response is ReadyResponse)
{
return TaskHelper.ToTask(response);
}
throw new DriverInternalError("Expected READY or AUTHENTICATE, obtained " + response.GetType().Name);
}));
}
