private void OnTimeout(OperationState state)
{
var ex = new OperationTimedOutException(Address, Configuration.SocketOptions.ReadTimeoutMillis);
//Invoke if it hasn't been invoked yet
//Once the response is obtained, we decrement the timed out counter
var timedout = state.SetTimedOut(ex, () => Interlocked.Decrement(ref _timedOutOperations) );
if (!timedout)
{
//The response was obtained since the timer elapsed, move on
return;
}
//Increase timed-out counter
Interlocked.Increment(ref _timedOutOperations);
}
/// <summary>
/// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks.
/// Returns true if a full operation (streamId) has been processed and there is one available.
/// </summary>
/// <returns>True if a full operation (streamId) has been processed.</returns>
internal bool ReadParse(byte[] buffer, int length)
{
if (length <= 0)
{
return false;
}
if (_frameHeaderSize == 0)
{
//Read the first byte of the message to determine the version of the response
ProtocolVersion = FrameHeader.GetProtocolVersion(buffer);
_frameHeaderSize = FrameHeader.GetSize(ProtocolVersion);
}
//Use _readStream to buffer between messages, under low pressure, it should be null most of the times
var stream = Interlocked.Exchange(ref _readStream, null);
var operationCallbacks = new LinkedList<Action<MemoryStream>>();
var offset = 0;
if (_minimalBuffer != null)
{
//use a negative offset to identify that there is a previous header buffer
offset = -1 * _minimalBuffer.Length;
}
while (offset < length)
{
FrameHeader header;
//The remaining body length to read from this buffer
int remainingBodyLength;
if (_receivingHeader == null)
{
if (length - offset < _frameHeaderSize)
{
_minimalBuffer = offset >= 0 ?
Utils.SliceBuffer(buffer, offset, length - offset) :
//it should almost never be the case there isn't enough bytes to read the header more than once
// ReSharper disable once PossibleNullReferenceException
Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, 0, length);
break;
}
if (offset >= 0)
{
header = FrameHeader.ParseResponseHeader(ProtocolVersion, buffer, offset);
}
else
{
header = FrameHeader.ParseResponseHeader(ProtocolVersion, _minimalBuffer, buffer);
_minimalBuffer = null;
}
_logger.Verbose("Received #{0} from {1}", header.StreamId, Address);
offset += _frameHeaderSize;
remainingBodyLength = header.BodyLength;
}
else
{
header = _receivingHeader;
remainingBodyLength = header.BodyLength - (int) stream.Length;
_receivingHeader = null;
}
if (remainingBodyLength > length - offset)
{
//the buffer does not contains the body for this frame, buffer for later
MemoryStream nextMessageStream;
if (operationCallbacks.Count == 0 && stream != null)
{
//There hasn't been any operations completed with this buffer
//And there is a previous stream: reuse it
nextMessageStream = stream;
}
else
{
nextMessageStream = Configuration.BufferPool.GetStream(typeof(Connection) + "/Read");
}
nextMessageStream.Write(buffer, offset, length - offset);
Interlocked.Exchange(ref _readStream, nextMessageStream);
_receivingHeader = header;
break;
}
stream = stream ?? Configuration.BufferPool.GetStream(typeof (Connection) + "/Read");
OperationState state;
if (header.Opcode != EventResponse.OpCode)
{
state = RemoveFromPending(header.StreamId);
}
else
{
//Its an event
state = new OperationState(EventHandler);
}
stream.Write(buffer, offset, remainingBodyLength);
var callback = state.SetCompleted();
operationCallbacks.AddLast(CreateResponseAction(header, callback));
offset += remainingBodyLength;
}
return InvokeReadCallbacks(stream, operationCallbacks);
}
/// <summary>
/// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up.
/// </summary>
public OperationState Send(IRequest request, Action<Exception, Response> callback)
{
if (_isCanceled)
{
callback(new SocketException((int)SocketError.NotConnected), null);
}
var state = new OperationState(callback)
{
Request = request
};
_writeQueue.Enqueue(state);
RunWriteQueue();
return state;
}
/// <summary>
/// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks.
/// Returns true if a full operation (streamId) has been processed and there is one available.
/// </summary>
/// <param name="buffer">Byte buffer to read</param>
/// <param name="offset">Offset within the buffer</param>
/// <param name="count">Length of bytes to be read from the buffer</param>
/// <returns>True if a full operation (streamId) has been processed.</returns>
protected virtual bool ReadParse(byte[] buffer, int offset, int count)
{
OperationState state = _receivingOperation;
if (state == null)
{
if (_minimalBuffer != null)
{
buffer = Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, offset, count);
offset = 0;
count = buffer.Length;
}
var headerSize = FrameHeader.GetSize(ProtocolVersion);
if (count < headerSize)
{
//There is not enough data to read the header
_minimalBuffer = Utils.SliceBuffer(buffer, offset, count);
return false;
}
_minimalBuffer = null;
var header = FrameHeader.ParseResponseHeader(ProtocolVersion, buffer, offset);
if (!header.IsValidResponse())
{
_logger.Error("Not a response header");
}
offset += headerSize;
count -= headerSize;
if (header.Opcode != EventResponse.OpCode)
{
//Its a response to a previous request
state = _pendingOperations[header.StreamId];
}
else
{
//Its an event
state = new OperationState();
state.Callback = EventHandler;
}
state.Header = header;
_receivingOperation = state;
}
var countAdded = state.AppendBody(buffer, offset, count);
if (state.IsBodyComplete)
{
_logger.Verbose("Read #" + state.Header.StreamId + " for Opcode " + state.Header.Opcode);
//Stop reference it as the current receiving operation
_receivingOperation = null;
if (state.Header.Opcode != EventResponse.OpCode)
{
//Remove from pending
_pendingOperations.TryRemove(state.Header.StreamId, out state);
//Release the streamId
_freeOperations.Push(state.Header.StreamId);
}
try
{
var response = ReadParseResponse(state.Header, state.BodyStream);
state.InvokeCallback(null, response);
}
catch (Exception ex)
{
state.InvokeCallback(ex);
}
if (countAdded < count)
{
//There is more data, from the next frame
ReadParse(buffer, offset + countAdded, count - countAdded);
}
return true;
}
//There isn't enough data to read the whole frame.
//It is already buffered, carry on.
return false;
}
/// <summary>
/// Try to write the item provided. Thread safe.
/// </summary>
private void SendQueueProcess(OperationState state)
{
if (!_canWriteNext)
{
//Double-checked locking for best performance
_writeQueue.Enqueue(state);
return;
}
short streamId = -1;
lock (_writeQueueLock)
{
if (!_canWriteNext)
{
//We have to recheck as the world can change since the last instruction
_writeQueue.Enqueue(state);
return;
}
//Check if Cassandra can process a new operation
if (!_freeOperations.TryPop(out streamId))
{
//Queue it up for later.
//When receiving the next complete message, we can process it.
_writeQueue.Enqueue(state);
_logger.Info("Enqueued: " + _writeQueue.Count + ", if this message is recurrent consider configuring more connections per host or lower the pressure");
return;
}
//Prevent the next to process
_canWriteNext = false;
}
//At this point:
//We have a valid stream id
//Only 1 thread at a time can be here.
try
{
_logger.Verbose("Sending #" + streamId + " for " + state.Request.GetType().Name);
var frameStream = state.Request.GetFrame(streamId).Stream;
_pendingOperations.AddOrUpdate(streamId, state, (k, oldValue) => state);
//We will not use the request, stop reference it.
state.Request = null;
//Start sending it
_tcpSocket.Write(frameStream);
}
catch (Exception ex)
{
//Prevent dead locking
_canWriteNext = true;
_logger.Error(ex);
//The request was not written
_pendingOperations.TryRemove(streamId, out state);
_freeOperations.Push(streamId);
throw;
}
}
/// <summary>
/// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks.
/// Returns true if a full operation (streamId) has been processed and there is one available.
/// </summary>
/// <returns>True if a full operation (streamId) has been processed.</returns>
internal bool ReadParse(byte[] buffer, int length)
{
if (length <= 0)
{
return(false);
}
ProtocolVersion protocolVersion;
if (_frameHeaderSize == 0)
{
//The server replies the first message with the max protocol version supported
protocolVersion = FrameHeader.GetProtocolVersion(buffer);
_serializer.ProtocolVersion = protocolVersion;
_frameHeaderSize = FrameHeader.GetSize(protocolVersion);
}
else
{
protocolVersion = _serializer.ProtocolVersion;
}
//Use _readStream to buffer between messages, under low pressure, it should be null most of the times
var stream = Interlocked.Exchange(ref _readStream, null);
var operationCallbacks = new LinkedList <Action <MemoryStream> >();
var offset = 0;
if (_minimalBuffer != null)
{
//use a negative offset to identify that there is a previous header buffer
offset = -1 * _minimalBuffer.Length;
}
while (offset < length)
{
FrameHeader header;
//The remaining body length to read from this buffer
int remainingBodyLength;
if (_receivingHeader == null)
{
if (length - offset < _frameHeaderSize)
{
_minimalBuffer = offset >= 0 ?
Utils.SliceBuffer(buffer, offset, length - offset) :
//it should almost never be the case there isn't enough bytes to read the header more than once
// ReSharper disable once PossibleNullReferenceException
Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, 0, length);
break;
}
if (offset >= 0)
{
header = FrameHeader.ParseResponseHeader(protocolVersion, buffer, offset);
}
else
{
header = FrameHeader.ParseResponseHeader(protocolVersion, _minimalBuffer, buffer);
_minimalBuffer = null;
}
Logger.Verbose("Received #{0} from {1}", header.StreamId, Address);
offset += _frameHeaderSize;
remainingBodyLength = header.BodyLength;
}
else
{
header = _receivingHeader;
remainingBodyLength = header.BodyLength - (int)stream.Length;
_receivingHeader = null;
}
if (remainingBodyLength > length - offset)
{
//the buffer does not contains the body for this frame, buffer for later
MemoryStream nextMessageStream;
if (operationCallbacks.Count == 0 && stream != null)
{
//There hasn't been any operations completed with this buffer
//And there is a previous stream: reuse it
nextMessageStream = stream;
}
else
{
nextMessageStream = Configuration.BufferPool.GetStream(typeof(Connection) + "/Read");
}
nextMessageStream.Write(buffer, offset, length - offset);
Interlocked.Exchange(ref _readStream, nextMessageStream);
_receivingHeader = header;
break;
}
stream = stream ?? Configuration.BufferPool.GetStream(typeof(Connection) + "/Read");
OperationState state;
if (header.Opcode != EventResponse.OpCode)
{
state = RemoveFromPending(header.StreamId);
}
else
{
//Its an event
state = new OperationState(EventHandler);
}
stream.Write(buffer, offset, remainingBodyLength);
// State can be null when the Connection is being closed concurrently
// The original callback is being called with an error, use a Noop here
var callback = state != null?state.SetCompleted() : OperationState.Noop;
operationCallbacks.AddLast(CreateResponseAction(header, callback));
offset += remainingBodyLength;
}
return(InvokeReadCallbacks(stream, operationCallbacks));
}
/// <summary>
/// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up.
/// </summary>
public void Send(IRequest request, Action<Exception, AbstractResponse> callback)
{
if (_isCanceled)
{
callback(new SocketException((int)SocketError.NotConnected), null);
}
//thread safe write queue
var state = new OperationState
{
Request = request,
Callback = callback
};
SendQueueProcess(state);
}
/// <summary>
/// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks.
/// Returns true if a full operation (streamId) has been processed and there is one available.
/// </summary>
/// <param name="buffer">Byte buffer to read</param>
/// <param name="offset">Offset within the buffer</param>
/// <param name="count">Length of bytes to be read from the buffer</param>
/// <returns>True if a full operation (streamId) has been processed.</returns>
protected virtual bool ReadParse(byte[] buffer, int offset, int count)
{
var state = _receivingOperation;
if (state == null)
{
if (_minimalBuffer != null)
{
buffer = Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, offset, count);
offset = 0;
count = buffer.Length;
}
if (_frameHeaderSize == 0)
{
//Read the first byte of the message to determine the version of the response
ProtocolVersion = FrameHeader.GetProtocolVersion(buffer);
_frameHeaderSize = FrameHeader.GetSize(ProtocolVersion);
}
var headerSize = _frameHeaderSize;
if (count < headerSize)
{
//There is not enough data to read the header
_minimalBuffer = Utils.SliceBuffer(buffer, offset, count);
return(false);
}
_minimalBuffer = null;
var header = FrameHeader.ParseResponseHeader(ProtocolVersion, buffer, offset);
if (!header.IsValidResponse())
{
_logger.Error("Not a response header");
}
offset += headerSize;
count -= headerSize;
if (header.Opcode != EventResponse.OpCode)
{
//Its a response to a previous request
state = _pendingOperations[header.StreamId];
}
else
{
//Its an event
state = new OperationState(EventHandler);
}
state.Header = header;
_receivingOperation = state;
}
var countAdded = state.AppendBody(buffer, offset, count);
if (!state.IsBodyComplete)
{
//Nothing finished
return(false);
}
_logger.Verbose("Read #{0} for Opcode {1} from host {2}", state.Header.StreamId, state.Header.Opcode, Address);
//Stop reference it as the current receiving operation
_receivingOperation = null;
if (state.Header.Opcode != EventResponse.OpCode)
{
RemoveFromPending(state.Header.StreamId);
}
try
{
var response = ReadParseResponse(state.Header, state.BodyStream);
state.InvokeCallback(null, response);
}
catch (Exception ex)
{
state.InvokeCallback(ex);
}
if (countAdded < count)
{
//There is more data, from the next frame
ReadParse(buffer, offset + countAdded, count - countAdded);
}
return(true);
//There isn't enough data to read the whole frame.
//It is already buffered, carry on.
}
private void SendQueueProcessItem(OperationState state)
{
short streamId;
//Check if Cassandra can process a new operation
if (!_freeOperations.TryPop(out streamId))
{
//Queue it up for later.
//When receiving the next complete message, we can process it.
_writeQueue.Enqueue(state);
_logger.Info("Enqueued: {0}, if this message is recurrent consider configuring more connections per host or lower the pressure", _writeQueue.Count);
Interlocked.Exchange(ref _canWriteNext, 1);
return;
}
//We have a valid stream id
//Only 1 thread at a time can be here.
_logger.Verbose("Sending #" + streamId + " for " + state.Request.GetType().Name);
_pendingOperations.AddOrUpdate(streamId, state, (k, oldValue) => state);
try
{
var frameStream = state.Request.GetFrame(streamId).Stream;
//We will not use the request any more, stop reference it.
state.Request = null;
//Start sending it
_tcpSocket.Write(frameStream);
//Closure state variable
var delegateState = state;
if (Configuration.SocketOptions.ReadTimeoutMillis > 0 && Configuration.Timer != null)
{
state.Timeout = Configuration.Timer.NewTimeout(() => OnTimeout(delegateState), Configuration.SocketOptions.ReadTimeoutMillis);
}
}
catch (Exception ex)
{
//There was an error while serializing or begin sending
_logger.Error(ex);
//The request was not written, clear it from pending operations
RemoveFromPending(streamId);
//Callback with the Exception
state.InvokeCallback(ex);
}
}
/// <summary>
/// Try to write the item provided. Thread safe.
/// </summary>
/// <param name="state">The request and callback</param>
/// <param name="useInlining">Determines if the current thread can be used to start sending the request</param>
private void SendQueueProcess(OperationState state, bool useInlining)
{
var canWrite = Interlocked.CompareExchange(ref _canWriteNext, 0, 1);
if (canWrite == 1)
{
if (useInlining)
{
//Use the current thread to start the write operation
SendQueueProcessItem(state);
return;
}
//Start a new task using the TaskScheduler for writing
Task.Factory.StartNew(() => SendQueueProcessItem(state), CancellationToken.None, TaskCreationOptions.None, _writeScheduler);
}
else
{
_writeQueue.Enqueue(state);
}
}
/// <summary>
/// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up.
/// </summary>
public OperationState Send(IRequest request, Action<Exception, AbstractResponse> callback)
{
if (_isCanceled)
{
callback(new SocketException((int)SocketError.NotConnected), null);
}
var state = new OperationState(callback)
{
Request = request
};
SendQueueProcess(state, true);
return state;
}
/// <summary>
/// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up.
/// </summary>
public OperationState Send(IRequest request, Action<Exception, Response> callback, int timeoutMillis = Timeout.Infinite)
{
if (_isCanceled)
{
callback(new SocketException((int)SocketError.NotConnected), null);
return null;
}
var state = new OperationState(callback)
{
Request = request,
TimeoutMillis = timeoutMillis > 0 ? timeoutMillis : Configuration.SocketOptions.ReadTimeoutMillis
};
_writeQueue.Enqueue(state);
RunWriteQueue();
return state;
}