public bool ShouldFail(INode node)
{
var now = DateTime.UtcNow;
if (counter == 0)
{
LogTo.Trace("Never failed before, setting counter to 1.");
counter = 1;
}
else
{
var diff = now - lastFailed;
LogTo.Trace($"Last fail was {diff} ago with counter {counter}.");
counter = diff <= ResetAfter ? (counter + 1) : 1;
}
lastFailed = now;
if (counter == Threshold)
{
LogTo.Trace("Threshold reached, failing node.");
counter = 0;
return(true);
}
LogTo.Trace($"Threshold not reached, current value is {counter}");
return(false);
}
/// <summary>
/// Writes an operation to the output buffer. Handles the case where the op does not fit the buffer fully.
/// </summary>
/// <param name="data"></param>
protected virtual void WriteOp(OpQueueEntry data)
{
if (currentWriteCopier != null)
{
throw new InvalidOperationException("Cannot write an operation while another is in progress.");
}
var request = data.Op.CreateRequest();
if (!request.WriteTo(socket.WriteBuffer)) // no pending IO => fully written
{
readQueue.Enqueue(data);
request.Dispose();
#region Diagnostics
npm.EnqueueReadOp();
CoreEventSource.EnqueueReadOp(name);
LogTo.Trace($"Full send of {data.Op}");
#endregion
}
else
{
// it did not fit into the write buffer, so save the current op
// as "in-progress"; DoRun will loop until it's fully sent
currentWriteOp = data;
currentWriteCopier = request;
LogTo.Trace($"Partial send of {data.Op}");
}
}
private void Worker()
{
while (!shutdownToken.IsCancellationRequested)
{
try
{
var node = ioQueue.Take(shutdownToken.Token);
try
{
node.Run();
LogTo.Trace($"Node {node} finished IO");
if (shutdownToken.IsCancellationRequested)
{
break;
}
}
catch (Exception e)
{
FailNode(node, e);
}
}
catch (OperationCanceledException)
{
break;
}
}
LogTo.Trace("shutdownToken was cancelled, finishing work");
workerIsDone.Set();
}
/// <summary>
/// Sends the current chunked op. Happens when an op's data cannot fit the write buffer in one pass.
/// </summary>
/// <returns>returns true if further IO is required; false if no inprogress op present or the last chunk was successfully added to the buffer</returns>
private bool ContinueWritingCurrentOp()
{
// check if we have an op in progress
if (currentWriteCopier == null)
{
return(false);
}
if (currentWriteCopier.WriteTo(socket.WriteBuffer))
{
return(true);
}
// last chunk was sent
LogTo.Trace($"Sent & finished {currentWriteOp.Op}");
// op is sent fully; response can be expected
readQueue.Enqueue(currentWriteOp);
#region Diagnostics
npm.EnqueueReadOp();
CoreEventSource.EnqueueReadOp(name);
#endregion
// clean up
currentWriteCopier.Dispose();
currentWriteCopier = null;
currentWriteOp = OpQueueEntry.Empty;
return(false);
}
protected virtual void FlushWriteBuffer()
{
npm.Flush();
socket.ScheduleSend(success =>
{
if (success)
{
LogTo.Trace($"{name} send the write buffer successfully");
_FinishedWriting();
owner.NeedsIO(this);
return;
}
// this is a soft fail (cannot throw from other thread)
// so we requeue for IO and Run() will throw instead
LogTo.Trace($"{name}'s FlushWriteBuffer failed");
FailMe(new IOException("send fail"));
});
}
private void TryAskForMoreData()
{
// no data to process => read the socket
if (socket.ReadBuffer.IsEmpty && !socket.IsReceiving)
{
LogTo.Trace("Read buffer is empty, ask for more.");
socket.ScheduleReceive(success =>
{
if (success)
{
LogTo.Trace($"{name} successfully received data");
_FinishedReading();
owner.NeedsIO(this);
return;
}
// this is a soft fail (cannot throw from other thread),
// so we requeue for IO and exception will be thrown by Receive()
FailMe(new IOException("Failed receiving from " + endpoint));
});
}
}
/// <summary>
/// Put the node into the pending work queue.
/// </summary>
/// <param name="node"></param>
public void NeedsIO(INode node)
{
LogTo.Trace($"Node {node} is requeued for IO");
ioQueue.Add(node);
}
private void TryProcessReceivedData()
{
// process the commands in the readQueue
while (readQueue.Count > 0)
{
// continue filling the previously unfinished response,
// or create a new one
var response = inprogressResponse ?? CreateResponse();
// continue filling the Response object from the buffer
// Read() returns true if further data (IO) is required
// (usually when the current response data is larger than the receive buffer size)
if (response.Read(socket.ReadBuffer))
{
inprogressResponse = response;
LogTo.Trace("Response is not read fully, continue reading from the socket.");
// refill the buffer
_FinishedReading();
owner.NeedsIO(this);
return;
}
// successfully read a response from the read buffer
inprogressResponse = null;
var isHandled = false;
while (!isHandled && readQueue.Count > 0)
{
var data = readQueue.Peek();
Debug.Assert(!data.IsEmpty);
// If the response does not matches the current op, it means it's a
// response to a later command in the queue, so all commands before it
// were silent commands without a response (== usually success).
// So, successful silent ops will receive null as response (since
// we have no real response (or we've ran into a bug))
isHandled = data.Op.Handles(response);
LogTo.Trace($"Command {data.Op} handles reponse: {isHandled}");
// operations are allowed to hanndle subsequent responses
// they returns false when no more IO (response) is required => done processing
if (!data.Op.ProcessResponse(isHandled ? response : null))
{
readQueue.Dequeue();
data.Task.TrySetResult(data.Op);
#region Diagnostics
npm.DequeueReadOp();
CoreEventSource.DequeueReadOp(name);
#endregion
}
}
response.Dispose();
}
LogTo.Trace($"{name} fininshed RECEIVE, unlock read");
_FinishedReading();
}
