public bool ShouldFail(INode node) { var now = DateTime.UtcNow; if (counter == 0) { LogTo.Trace("Never failed before, setting counter to 1."); counter = 1; } else { var diff = now - lastFailed; LogTo.Trace($"Last fail was {diff} ago with counter {counter}."); counter = diff <= ResetAfter ? (counter + 1) : 1; } lastFailed = now; if (counter == Threshold) { LogTo.Trace("Threshold reached, failing node."); counter = 0; return(true); } LogTo.Trace($"Threshold not reached, current value is {counter}"); return(false); }
/// <summary> /// Writes an operation to the output buffer. Handles the case where the op does not fit the buffer fully. /// </summary> /// <param name="data"></param> protected virtual void WriteOp(OpQueueEntry data) { if (currentWriteCopier != null) { throw new InvalidOperationException("Cannot write an operation while another is in progress."); } var request = data.Op.CreateRequest(); if (!request.WriteTo(socket.WriteBuffer)) // no pending IO => fully written { readQueue.Enqueue(data); request.Dispose(); #region Diagnostics npm.EnqueueReadOp(); CoreEventSource.EnqueueReadOp(name); LogTo.Trace($"Full send of {data.Op}"); #endregion } else { // it did not fit into the write buffer, so save the current op // as "in-progress"; DoRun will loop until it's fully sent currentWriteOp = data; currentWriteCopier = request; LogTo.Trace($"Partial send of {data.Op}"); } }
private void Worker() { while (!shutdownToken.IsCancellationRequested) { try { var node = ioQueue.Take(shutdownToken.Token); try { node.Run(); LogTo.Trace($"Node {node} finished IO"); if (shutdownToken.IsCancellationRequested) { break; } } catch (Exception e) { FailNode(node, e); } } catch (OperationCanceledException) { break; } } LogTo.Trace("shutdownToken was cancelled, finishing work"); workerIsDone.Set(); }
/// <summary> /// Sends the current chunked op. Happens when an op's data cannot fit the write buffer in one pass. /// </summary> /// <returns>returns true if further IO is required; false if no inprogress op present or the last chunk was successfully added to the buffer</returns> private bool ContinueWritingCurrentOp() { // check if we have an op in progress if (currentWriteCopier == null) { return(false); } if (currentWriteCopier.WriteTo(socket.WriteBuffer)) { return(true); } // last chunk was sent LogTo.Trace($"Sent & finished {currentWriteOp.Op}"); // op is sent fully; response can be expected readQueue.Enqueue(currentWriteOp); #region Diagnostics npm.EnqueueReadOp(); CoreEventSource.EnqueueReadOp(name); #endregion // clean up currentWriteCopier.Dispose(); currentWriteCopier = null; currentWriteOp = OpQueueEntry.Empty; return(false); }
protected virtual void FlushWriteBuffer() { npm.Flush(); socket.ScheduleSend(success => { if (success) { LogTo.Trace($"{name} send the write buffer successfully"); _FinishedWriting(); owner.NeedsIO(this); return; } // this is a soft fail (cannot throw from other thread) // so we requeue for IO and Run() will throw instead LogTo.Trace($"{name}'s FlushWriteBuffer failed"); FailMe(new IOException("send fail")); }); }
private void TryAskForMoreData() { // no data to process => read the socket if (socket.ReadBuffer.IsEmpty && !socket.IsReceiving) { LogTo.Trace("Read buffer is empty, ask for more."); socket.ScheduleReceive(success => { if (success) { LogTo.Trace($"{name} successfully received data"); _FinishedReading(); owner.NeedsIO(this); return; } // this is a soft fail (cannot throw from other thread), // so we requeue for IO and exception will be thrown by Receive() FailMe(new IOException("Failed receiving from " + endpoint)); }); } }
/// <summary> /// Put the node into the pending work queue. /// </summary> /// <param name="node"></param> public void NeedsIO(INode node) { LogTo.Trace($"Node {node} is requeued for IO"); ioQueue.Add(node); }
private void TryProcessReceivedData() { // process the commands in the readQueue while (readQueue.Count > 0) { // continue filling the previously unfinished response, // or create a new one var response = inprogressResponse ?? CreateResponse(); // continue filling the Response object from the buffer // Read() returns true if further data (IO) is required // (usually when the current response data is larger than the receive buffer size) if (response.Read(socket.ReadBuffer)) { inprogressResponse = response; LogTo.Trace("Response is not read fully, continue reading from the socket."); // refill the buffer _FinishedReading(); owner.NeedsIO(this); return; } // successfully read a response from the read buffer inprogressResponse = null; var isHandled = false; while (!isHandled && readQueue.Count > 0) { var data = readQueue.Peek(); Debug.Assert(!data.IsEmpty); // If the response does not matches the current op, it means it's a // response to a later command in the queue, so all commands before it // were silent commands without a response (== usually success). // So, successful silent ops will receive null as response (since // we have no real response (or we've ran into a bug)) isHandled = data.Op.Handles(response); LogTo.Trace($"Command {data.Op} handles reponse: {isHandled}"); // operations are allowed to hanndle subsequent responses // they returns false when no more IO (response) is required => done processing if (!data.Op.ProcessResponse(isHandled ? response : null)) { readQueue.Dequeue(); data.Task.TrySetResult(data.Op); #region Diagnostics npm.DequeueReadOp(); CoreEventSource.DequeueReadOp(name); #endregion } } response.Dispose(); } LogTo.Trace($"{name} fininshed RECEIVE, unlock read"); _FinishedReading(); }