public bool HandleNewConnection()
{
var workDone = false;
if (_queue.TryDequeue(out var connection))
{
workDone = true;
if (_connections.Count >= Constants.MaxActiveConnections)
{
// Signal that this connection will not be taken into account.
Span <byte> buffer = stackalloc byte[ProtocolErrorResponse.SizeInBytes];
var errorMessage = new ProtocolErrorResponse(
buffer, RequestId.MinRequestId, ClientId.MinClientId, ProtocolErrorStatus.TooManyActiveClients);
connection.Send(errorMessage.Span);
}
else
{
connection.OnRequestReceived = Wake;
_connections.Add(connection.ClientId, connection);
// Intended to start the threads associated to the connection.
OnConnectionAccepted(connection);
}
}
return(workDone);
}
public void QueueTooManyClients()
{
var socket = new MockSocket();
var dispatcherInbox = new BoundedInbox();
var dispatcher = new DispatchController(dispatcherInbox, new BoundedInbox(), new BoundedInbox[64],
new IdentityHash());
dispatcher.OnConnectionAccepted = (_) => { };
for (var i = 0; i < Constants.MaxActiveConnections; i++)
{
var client0 = new ConnectionController(dispatcherInbox, socket, ClientId.Next());
client0.OnRequestReceived = () => { };
dispatcher.AddConnection(client0);
dispatcher.HandleNewConnection();
}
var client1 = new ConnectionController(dispatcherInbox, socket, ClientId.Next());
client1.OnRequestReceived = () => { };
socket.ExpectSend(data =>
{
Assert.Equal(ProtocolErrorResponse.SizeInBytes, data.Length);
var errorMessage = new ProtocolErrorResponse(data);
Assert.Equal(MessageKind.ProtocolErrorResponse, errorMessage.MessageHeader.MessageKind);
Assert.Equal(ProtocolErrorStatus.TooManyActiveClients, errorMessage.Status);
return(data.Length);
});
dispatcher.AddConnection(client1);
dispatcher.HandleNewConnection();
client1.HandleResponse();
socket.ExpectAllDone();
}
public bool HandleRequest()
{
if (!_inbox.CanPeek)
{
return(false);
}
var next = _inbox.Peek().Span;
try
{
var message = new Message(next);
var mask = message.Header.ClientId.Mask;
var kind = message.Header.MessageKind;
var requestId = message.Header.RequestId;
var clientId = message.Header.ClientId;
if (!_connections.TryGetValue(clientId, out var connection))
{
// Outdated message, drop and move on.
return(true);
}
if (!connection.Connected || kind == MessageKind.CloseConnection)
{
// Client disconnected, drop message and remove connection.
_connections.Remove(clientId);
return(true);
}
if (kind.IsResponse())
{
// Forward responses to their respective 'ConnectionController'.
connection.Send(next);
return(true);
}
if (kind.IsForCoinController())
{
// Multiple coin controllers
Outpoint outpoint;
switch (kind)
{
case MessageKind.GetCoin:
outpoint = new GetCoinRequest(next, mask).Outpoint;
break;
case MessageKind.ProduceCoin:
outpoint = new ProduceCoinRequest(next, mask).Outpoint;
break;
case MessageKind.ConsumeCoin:
outpoint = new ConsumeCoinRequest(next, mask).Outpoint;
break;
case MessageKind.RemoveCoin:
outpoint = new RemoveCoinRequest(next, mask).Outpoint;
break;
default:
throw new NotSupportedException();
}
// Sharding based on the outpoint hash
// Beware: the factor 'BigPrime' is used to avoid accidental factor collision
// between the sharding performed at the dispatch controller level, and the
// sharding performed within the Sozu table.
// PERF: hashing the outpoint is repeated in the CoinController itself
const ulong BigPrime = 1_000_000_007;
var controllerIndex = (int)((_hash.Hash(ref outpoint) % BigPrime)
% (ulong)_coinControllerBoxes.Length);
var written = _coinControllerBoxes[controllerIndex].TryWrite(next);
if (written)
{
OnCoinMessageDispatched[controllerIndex]();
}
else
{
// Coin controller is saturated.
Span <byte> buffer = stackalloc byte[ProtocolErrorResponse.SizeInBytes];
var errorMessage = new ProtocolErrorResponse(
buffer, requestId, clientId, ProtocolErrorStatus.ServerBusy);
connection.Send(errorMessage.Span);
}
return(true);
}
{
// Block controller
var written = _chainControllerBox.TryWrite(next);
if (written)
{
OnBlockMessageDispatched();
}
else
{
// Block controller is saturated.
Span <byte> buffer = stackalloc byte[ProtocolErrorResponse.SizeInBytes];
var errorMessage = new ProtocolErrorResponse(
buffer, requestId, clientId, ProtocolErrorStatus.ServerBusy);
connection.Send(errorMessage.Span);
}
}
}
finally
{
_inbox.Next();
}
return(true);
}
public void FillChainControllerInbox()
{
var socket1 = new MockSocket();
var socket2 = new MockSocket();
var dispatcherInbox = new BoundedInbox();
var client1 = new ConnectionController(dispatcherInbox, socket1, ClientId.Next());
var client2 = new ConnectionController(dispatcherInbox, socket2, ClientId.Next());
client1.OnRequestReceived = () => { };
client2.OnRequestReceived = () => { };
Func <int, int, MockSocket.SpanToInt> func = (s1, s2) =>
{
return(data =>
{
data.Clear();
BinaryPrimitives.TryWriteInt32LittleEndian(data, s1);
// TODO: [vermorel] PingChainController has been removed, logic need to be upgraded.
//MessageKind.PingChainController.WriteTo(data.Slice(MessageHeaderHelper.MessageKindStart));
return s2;
});
};
socket2.ExpectReceive(func(LargeMessageSize, MessageHeader.SizeInBytes));
socket2.ExpectReceive(func(LargeMessageSize, LargeMessageSize));
socket1.ExpectReceive(func(LargeMessageSize, MessageHeader.SizeInBytes));
socket1.ExpectReceive(func(LargeMessageSize, LargeMessageSize));
// request too short
var bodyStart = sizeof(int) + RequestId.SizeInBytes + ClientId.SizeInBytes + sizeof(MessageKind);
socket2.ExpectReceive(func(bodyStart, bodyStart));
socket2.ExpectSend(data =>
{
Assert.Equal(ProtocolErrorResponse.SizeInBytes, data.Length);
var message = new ProtocolErrorResponse(data);
Assert.Equal(MessageKind.ProtocolErrorResponse, message.MessageHeader.MessageKind);
Assert.Equal(ProtocolErrorStatus.RequestTooShort, message.Status);
return(ProtocolErrorResponse.SizeInBytes);
});
socket2.ExpectConnected(() => true);
socket1.ExpectConnected(() => true);
var dispatcher = new DispatchController(dispatcherInbox,
new BoundedInbox(Constants.MaxResponseSize),
Enumerable.Range(0, 32).Select(x => new BoundedInbox()).ToArray(),
new IdentityHash());
// Nil handling of notifications
dispatcher.OnBlockMessageDispatched = () => { };
for (var i = 0; i < dispatcher.OnCoinMessageDispatched.Length; i++)
{
dispatcher.OnCoinMessageDispatched[i] = () => { }
}
;
dispatcher.OnConnectionAccepted = (_) => { };
dispatcher.AddConnection(client1);
dispatcher.AddConnection(client2);
dispatcher.HandleNewConnection();
dispatcher.HandleNewConnection();
client1.HandleRequest();
client2.HandleRequest();
client2.HandleRequest();
client2.HandleResponse();
dispatcher.HandleRequest();
dispatcher.HandleRequest();
dispatcher.HandleRequest();
socket1.ExpectAllDone();
socket2.ExpectAllDone();
}
public bool HandleResponse()
{
if (!_outbox.CanPeek)
{
return(false);
}
var next = _outbox.Peek().Span;
try
{
var message = new Message(next);
var kind = message.Header.MessageKind;
// Remove client ID from message
message.Header.ClientId = default;
if (_requestsInProgress >= ResponseBatchSize || _responseCountInPool > 0)
{
var nextResponse = _responsePool.GetSpan(next.Length);
next.CopyTo(nextResponse);
_responseCountInPool++;
if (_responseCountInPool >= ResponseBatchSize)
{
_socket.Send(_responsePool.Allocated());
_responsePool.Reset();
_responseCountInPool = 0;
}
}
else
{
_socket.Send(next);
}
Interlocked.Decrement(ref _requestsInProgress);
// Some responses trigger the termination of the controller.
if (kind == MessageKind.CloseConnectionResponse ||
kind == MessageKind.ProtocolErrorResponse)
{
if (kind == MessageKind.ProtocolErrorResponse)
{
var protocolResponse = new ProtocolErrorResponse(next);
_log?.Log(LogSeverity.Info, $"ConnectionController({ClientId}) on protocol error {protocolResponse.Status}.");
}
if (_responseCountInPool > 0)
{
_socket.Send(_responsePool.Allocated());
_responsePool.Reset();
_responseCountInPool = 0;
}
_tokenSource.Cancel();
}
}
finally
{
_outbox.Next();
}
return(true);
}
public bool HandleRequest()
{
// Blocking until header is received.
_socket.Receive(new Span <byte>(_bufferIn, 0, MessageHeader.SizeInBytes));
var message = new Message(_bufferIn);
// Request too short
if (message.SizeInBytes <= MessageHeader.SizeInBytes)
{
Span <byte> errorBuffer = stackalloc byte[ProtocolErrorResponse.SizeInBytes];
var errorMessage = new ProtocolErrorResponse(errorBuffer,
RequestId.MinRequestId, ClientId.MinClientId, ProtocolErrorStatus.RequestTooShort);
Send(errorMessage.Span);
return(false);
}
var kind = message.Header.MessageKind;
var requestId = message.Header.RequestId;
// Request too long
if (message.SizeInBytes >= Constants.MaxRequestSize)
{
Span <byte> errorBuffer = stackalloc byte[ProtocolErrorResponse.SizeInBytes];
var errorMessage = new ProtocolErrorResponse(errorBuffer,
requestId, ClientId.MinClientId, ProtocolErrorStatus.RequestTooLong);
Send(errorMessage.Span);
return(false);
}
// Invalid message kind
if (!kind.IsDefined() || kind.IsResponse())
{
// Unknown kind is invalid.
// Response kind is invalid.
Span <byte> buffer = stackalloc byte[ProtocolErrorResponse.SizeInBytes];
var errorMessage = new ProtocolErrorResponse(
buffer, requestId, ClientId.MinClientId, ProtocolErrorStatus.InvalidMessageKind);
Send(errorMessage.Span);
return(false);
}
// Blocking until the rest of the message is received.
_socket.Receive(new Span <byte>(_bufferIn,
MessageHeader.SizeInBytes,
message.SizeInBytes - MessageHeader.SizeInBytes));
message.Header.ClientId = _clientId;
// Client request to close the connection.
if (message.Header.MessageKind == MessageKind.CloseConnection)
{
Span <byte> buffer = stackalloc byte[CloseConnectionResponse.SizeInBytes];
var closeResponse = new CloseConnectionResponse(buffer, requestId, ClientId.MinClientId);
_outbox.TryWrite(closeResponse.Span);
}
// Forwards the message to the dispatch controller.
if (_dispatchInbox.TryWrite(message.Span))
{
Interlocked.Increment(ref _requestsInProgress);
return(true);
}
// The dispatch inbox is full.
{
Span <byte> errorBuffer = stackalloc byte[ProtocolErrorResponse.SizeInBytes];
var errorMessage = new ProtocolErrorResponse(errorBuffer,
requestId, ClientId.MinClientId, ProtocolErrorStatus.ServerBusy);
Send(errorMessage.Span);
}
return(false);
}
