private void OnTimeout(OperationState state) { var ex = new OperationTimedOutException(Address, Configuration.SocketOptions.ReadTimeoutMillis); //Invoke if it hasn't been invoked yet //Once the response is obtained, we decrement the timed out counter var timedout = state.SetTimedOut(ex, () => Interlocked.Decrement(ref _timedOutOperations) ); if (!timedout) { //The response was obtained since the timer elapsed, move on return; } //Increase timed-out counter Interlocked.Increment(ref _timedOutOperations); }
/// <summary> /// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks. /// Returns true if a full operation (streamId) has been processed and there is one available. /// </summary> /// <returns>True if a full operation (streamId) has been processed.</returns> internal bool ReadParse(byte[] buffer, int length) { if (length <= 0) { return false; } if (_frameHeaderSize == 0) { //Read the first byte of the message to determine the version of the response ProtocolVersion = FrameHeader.GetProtocolVersion(buffer); _frameHeaderSize = FrameHeader.GetSize(ProtocolVersion); } //Use _readStream to buffer between messages, under low pressure, it should be null most of the times var stream = Interlocked.Exchange(ref _readStream, null); var operationCallbacks = new LinkedList<Action<MemoryStream>>(); var offset = 0; if (_minimalBuffer != null) { //use a negative offset to identify that there is a previous header buffer offset = -1 * _minimalBuffer.Length; } while (offset < length) { FrameHeader header; //The remaining body length to read from this buffer int remainingBodyLength; if (_receivingHeader == null) { if (length - offset < _frameHeaderSize) { _minimalBuffer = offset >= 0 ? Utils.SliceBuffer(buffer, offset, length - offset) : //it should almost never be the case there isn't enough bytes to read the header more than once // ReSharper disable once PossibleNullReferenceException Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, 0, length); break; } if (offset >= 0) { header = FrameHeader.ParseResponseHeader(ProtocolVersion, buffer, offset); } else { header = FrameHeader.ParseResponseHeader(ProtocolVersion, _minimalBuffer, buffer); _minimalBuffer = null; } _logger.Verbose("Received #{0} from {1}", header.StreamId, Address); offset += _frameHeaderSize; remainingBodyLength = header.BodyLength; } else { header = _receivingHeader; remainingBodyLength = header.BodyLength - (int) stream.Length; _receivingHeader = null; } if (remainingBodyLength > length - offset) { //the buffer does not contains the body for this frame, buffer for later MemoryStream nextMessageStream; if (operationCallbacks.Count == 0 && stream != null) { //There hasn't been any operations completed with this buffer //And there is a previous stream: reuse it nextMessageStream = stream; } else { nextMessageStream = Configuration.BufferPool.GetStream(typeof(Connection) + "/Read"); } nextMessageStream.Write(buffer, offset, length - offset); Interlocked.Exchange(ref _readStream, nextMessageStream); _receivingHeader = header; break; } stream = stream ?? Configuration.BufferPool.GetStream(typeof (Connection) + "/Read"); OperationState state; if (header.Opcode != EventResponse.OpCode) { state = RemoveFromPending(header.StreamId); } else { //Its an event state = new OperationState(EventHandler); } stream.Write(buffer, offset, remainingBodyLength); var callback = state.SetCompleted(); operationCallbacks.AddLast(CreateResponseAction(header, callback)); offset += remainingBodyLength; } return InvokeReadCallbacks(stream, operationCallbacks); }
/// <summary> /// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up. /// </summary> public OperationState Send(IRequest request, Action<Exception, Response> callback) { if (_isCanceled) { callback(new SocketException((int)SocketError.NotConnected), null); } var state = new OperationState(callback) { Request = request }; _writeQueue.Enqueue(state); RunWriteQueue(); return state; }
/// <summary> /// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks. /// Returns true if a full operation (streamId) has been processed and there is one available. /// </summary> /// <param name="buffer">Byte buffer to read</param> /// <param name="offset">Offset within the buffer</param> /// <param name="count">Length of bytes to be read from the buffer</param> /// <returns>True if a full operation (streamId) has been processed.</returns> protected virtual bool ReadParse(byte[] buffer, int offset, int count) { OperationState state = _receivingOperation; if (state == null) { if (_minimalBuffer != null) { buffer = Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, offset, count); offset = 0; count = buffer.Length; } var headerSize = FrameHeader.GetSize(ProtocolVersion); if (count < headerSize) { //There is not enough data to read the header _minimalBuffer = Utils.SliceBuffer(buffer, offset, count); return false; } _minimalBuffer = null; var header = FrameHeader.ParseResponseHeader(ProtocolVersion, buffer, offset); if (!header.IsValidResponse()) { _logger.Error("Not a response header"); } offset += headerSize; count -= headerSize; if (header.Opcode != EventResponse.OpCode) { //Its a response to a previous request state = _pendingOperations[header.StreamId]; } else { //Its an event state = new OperationState(); state.Callback = EventHandler; } state.Header = header; _receivingOperation = state; } var countAdded = state.AppendBody(buffer, offset, count); if (state.IsBodyComplete) { _logger.Verbose("Read #" + state.Header.StreamId + " for Opcode " + state.Header.Opcode); //Stop reference it as the current receiving operation _receivingOperation = null; if (state.Header.Opcode != EventResponse.OpCode) { //Remove from pending _pendingOperations.TryRemove(state.Header.StreamId, out state); //Release the streamId _freeOperations.Push(state.Header.StreamId); } try { var response = ReadParseResponse(state.Header, state.BodyStream); state.InvokeCallback(null, response); } catch (Exception ex) { state.InvokeCallback(ex); } if (countAdded < count) { //There is more data, from the next frame ReadParse(buffer, offset + countAdded, count - countAdded); } return true; } //There isn't enough data to read the whole frame. //It is already buffered, carry on. return false; }
/// <summary> /// Try to write the item provided. Thread safe. /// </summary> private void SendQueueProcess(OperationState state) { if (!_canWriteNext) { //Double-checked locking for best performance _writeQueue.Enqueue(state); return; } short streamId = -1; lock (_writeQueueLock) { if (!_canWriteNext) { //We have to recheck as the world can change since the last instruction _writeQueue.Enqueue(state); return; } //Check if Cassandra can process a new operation if (!_freeOperations.TryPop(out streamId)) { //Queue it up for later. //When receiving the next complete message, we can process it. _writeQueue.Enqueue(state); _logger.Info("Enqueued: " + _writeQueue.Count + ", if this message is recurrent consider configuring more connections per host or lower the pressure"); return; } //Prevent the next to process _canWriteNext = false; } //At this point: //We have a valid stream id //Only 1 thread at a time can be here. try { _logger.Verbose("Sending #" + streamId + " for " + state.Request.GetType().Name); var frameStream = state.Request.GetFrame(streamId).Stream; _pendingOperations.AddOrUpdate(streamId, state, (k, oldValue) => state); //We will not use the request, stop reference it. state.Request = null; //Start sending it _tcpSocket.Write(frameStream); } catch (Exception ex) { //Prevent dead locking _canWriteNext = true; _logger.Error(ex); //The request was not written _pendingOperations.TryRemove(streamId, out state); _freeOperations.Push(streamId); throw; } }
/// <summary> /// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks. /// Returns true if a full operation (streamId) has been processed and there is one available. /// </summary> /// <returns>True if a full operation (streamId) has been processed.</returns> internal bool ReadParse(byte[] buffer, int length) { if (length <= 0) { return(false); } ProtocolVersion protocolVersion; if (_frameHeaderSize == 0) { //The server replies the first message with the max protocol version supported protocolVersion = FrameHeader.GetProtocolVersion(buffer); _serializer.ProtocolVersion = protocolVersion; _frameHeaderSize = FrameHeader.GetSize(protocolVersion); } else { protocolVersion = _serializer.ProtocolVersion; } //Use _readStream to buffer between messages, under low pressure, it should be null most of the times var stream = Interlocked.Exchange(ref _readStream, null); var operationCallbacks = new LinkedList <Action <MemoryStream> >(); var offset = 0; if (_minimalBuffer != null) { //use a negative offset to identify that there is a previous header buffer offset = -1 * _minimalBuffer.Length; } while (offset < length) { FrameHeader header; //The remaining body length to read from this buffer int remainingBodyLength; if (_receivingHeader == null) { if (length - offset < _frameHeaderSize) { _minimalBuffer = offset >= 0 ? Utils.SliceBuffer(buffer, offset, length - offset) : //it should almost never be the case there isn't enough bytes to read the header more than once // ReSharper disable once PossibleNullReferenceException Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, 0, length); break; } if (offset >= 0) { header = FrameHeader.ParseResponseHeader(protocolVersion, buffer, offset); } else { header = FrameHeader.ParseResponseHeader(protocolVersion, _minimalBuffer, buffer); _minimalBuffer = null; } Logger.Verbose("Received #{0} from {1}", header.StreamId, Address); offset += _frameHeaderSize; remainingBodyLength = header.BodyLength; } else { header = _receivingHeader; remainingBodyLength = header.BodyLength - (int)stream.Length; _receivingHeader = null; } if (remainingBodyLength > length - offset) { //the buffer does not contains the body for this frame, buffer for later MemoryStream nextMessageStream; if (operationCallbacks.Count == 0 && stream != null) { //There hasn't been any operations completed with this buffer //And there is a previous stream: reuse it nextMessageStream = stream; } else { nextMessageStream = Configuration.BufferPool.GetStream(typeof(Connection) + "/Read"); } nextMessageStream.Write(buffer, offset, length - offset); Interlocked.Exchange(ref _readStream, nextMessageStream); _receivingHeader = header; break; } stream = stream ?? Configuration.BufferPool.GetStream(typeof(Connection) + "/Read"); OperationState state; if (header.Opcode != EventResponse.OpCode) { state = RemoveFromPending(header.StreamId); } else { //Its an event state = new OperationState(EventHandler); } stream.Write(buffer, offset, remainingBodyLength); // State can be null when the Connection is being closed concurrently // The original callback is being called with an error, use a Noop here var callback = state != null?state.SetCompleted() : OperationState.Noop; operationCallbacks.AddLast(CreateResponseAction(header, callback)); offset += remainingBodyLength; } return(InvokeReadCallbacks(stream, operationCallbacks)); }
/// <summary> /// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up. /// </summary> public void Send(IRequest request, Action<Exception, AbstractResponse> callback) { if (_isCanceled) { callback(new SocketException((int)SocketError.NotConnected), null); } //thread safe write queue var state = new OperationState { Request = request, Callback = callback }; SendQueueProcess(state); }
/// <summary> /// Parses the bytes received into a frame. Uses the internal operation state to do the callbacks. /// Returns true if a full operation (streamId) has been processed and there is one available. /// </summary> /// <param name="buffer">Byte buffer to read</param> /// <param name="offset">Offset within the buffer</param> /// <param name="count">Length of bytes to be read from the buffer</param> /// <returns>True if a full operation (streamId) has been processed.</returns> protected virtual bool ReadParse(byte[] buffer, int offset, int count) { var state = _receivingOperation; if (state == null) { if (_minimalBuffer != null) { buffer = Utils.JoinBuffers(_minimalBuffer, 0, _minimalBuffer.Length, buffer, offset, count); offset = 0; count = buffer.Length; } if (_frameHeaderSize == 0) { //Read the first byte of the message to determine the version of the response ProtocolVersion = FrameHeader.GetProtocolVersion(buffer); _frameHeaderSize = FrameHeader.GetSize(ProtocolVersion); } var headerSize = _frameHeaderSize; if (count < headerSize) { //There is not enough data to read the header _minimalBuffer = Utils.SliceBuffer(buffer, offset, count); return(false); } _minimalBuffer = null; var header = FrameHeader.ParseResponseHeader(ProtocolVersion, buffer, offset); if (!header.IsValidResponse()) { _logger.Error("Not a response header"); } offset += headerSize; count -= headerSize; if (header.Opcode != EventResponse.OpCode) { //Its a response to a previous request state = _pendingOperations[header.StreamId]; } else { //Its an event state = new OperationState(EventHandler); } state.Header = header; _receivingOperation = state; } var countAdded = state.AppendBody(buffer, offset, count); if (!state.IsBodyComplete) { //Nothing finished return(false); } _logger.Verbose("Read #{0} for Opcode {1} from host {2}", state.Header.StreamId, state.Header.Opcode, Address); //Stop reference it as the current receiving operation _receivingOperation = null; if (state.Header.Opcode != EventResponse.OpCode) { RemoveFromPending(state.Header.StreamId); } try { var response = ReadParseResponse(state.Header, state.BodyStream); state.InvokeCallback(null, response); } catch (Exception ex) { state.InvokeCallback(ex); } if (countAdded < count) { //There is more data, from the next frame ReadParse(buffer, offset + countAdded, count - countAdded); } return(true); //There isn't enough data to read the whole frame. //It is already buffered, carry on. }
private void SendQueueProcessItem(OperationState state) { short streamId; //Check if Cassandra can process a new operation if (!_freeOperations.TryPop(out streamId)) { //Queue it up for later. //When receiving the next complete message, we can process it. _writeQueue.Enqueue(state); _logger.Info("Enqueued: {0}, if this message is recurrent consider configuring more connections per host or lower the pressure", _writeQueue.Count); Interlocked.Exchange(ref _canWriteNext, 1); return; } //We have a valid stream id //Only 1 thread at a time can be here. _logger.Verbose("Sending #" + streamId + " for " + state.Request.GetType().Name); _pendingOperations.AddOrUpdate(streamId, state, (k, oldValue) => state); try { var frameStream = state.Request.GetFrame(streamId).Stream; //We will not use the request any more, stop reference it. state.Request = null; //Start sending it _tcpSocket.Write(frameStream); //Closure state variable var delegateState = state; if (Configuration.SocketOptions.ReadTimeoutMillis > 0 && Configuration.Timer != null) { state.Timeout = Configuration.Timer.NewTimeout(() => OnTimeout(delegateState), Configuration.SocketOptions.ReadTimeoutMillis); } } catch (Exception ex) { //There was an error while serializing or begin sending _logger.Error(ex); //The request was not written, clear it from pending operations RemoveFromPending(streamId); //Callback with the Exception state.InvokeCallback(ex); } }
/// <summary> /// Try to write the item provided. Thread safe. /// </summary> /// <param name="state">The request and callback</param> /// <param name="useInlining">Determines if the current thread can be used to start sending the request</param> private void SendQueueProcess(OperationState state, bool useInlining) { var canWrite = Interlocked.CompareExchange(ref _canWriteNext, 0, 1); if (canWrite == 1) { if (useInlining) { //Use the current thread to start the write operation SendQueueProcessItem(state); return; } //Start a new task using the TaskScheduler for writing Task.Factory.StartNew(() => SendQueueProcessItem(state), CancellationToken.None, TaskCreationOptions.None, _writeScheduler); } else { _writeQueue.Enqueue(state); } }
/// <summary> /// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up. /// </summary> public OperationState Send(IRequest request, Action<Exception, AbstractResponse> callback) { if (_isCanceled) { callback(new SocketException((int)SocketError.NotConnected), null); } var state = new OperationState(callback) { Request = request }; SendQueueProcess(state, true); return state; }
/// <summary> /// Sends a new request if possible and executes the callback when the response is parsed. If it is not possible it queues it up. /// </summary> public OperationState Send(IRequest request, Action<Exception, Response> callback, int timeoutMillis = Timeout.Infinite) { if (_isCanceled) { callback(new SocketException((int)SocketError.NotConnected), null); return null; } var state = new OperationState(callback) { Request = request, TimeoutMillis = timeoutMillis > 0 ? timeoutMillis : Configuration.SocketOptions.ReadTimeoutMillis }; _writeQueue.Enqueue(state); RunWriteQueue(); return state; }