public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
IOptions <LoadSheddingOptions> loadSheddingOptions,
IOptions <StorageOptions> azureStorageOptions,
ILocalSiloDetails siloDetails,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
IHostEnvironmentStatistics hostEnvironmentStatistics,
IAppEnvironmentStatistics appEnvironmentStatistics,
ILoggerFactory loggerFactory,
IOptions <SiloMessagingOptions> messagingOptions)
{
this.siloDetails = siloDetails;
this.storageOptions = azureStorageOptions.Value;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
this.hostEnvironmentStatistics = hostEnvironmentStatistics;
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.hostEnvironmentStatistics, appEnvironmentStatistics, loggerFactory, loadSheddingOptions);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
internal void RecordOpenedConnection(GatewayInboundConnection connection, GrainId clientId)
{
lock (lockable)
{
logger.LogInformation((int)ErrorCode.GatewayClientOpenedSocket, "Recorded opened connection from endpoint {EndPoint}, client ID {ClientId}.", connection.RemoteEndpoint, clientId);
ClientState clientState;
if (clients.TryGetValue(clientId, out clientState))
{
var oldSocket = clientState.Connection;
if (oldSocket != null)
{
// The old socket will be closed by itself later.
clientConnections.TryRemove(oldSocket, out _);
}
}
else
{
clientState = new ClientState(clientId, messagingOptions.ClientDropTimeout);
clients[clientId] = clientState;
MessagingStatisticsGroup.ConnectedClientCount.Increment();
}
clientState.RecordConnection(connection);
clientConnections[connection] = clientState;
clientRegistrar.ClientAdded(clientId);
NetworkingStatisticsGroup.OnOpenedGatewayDuplexSocket();
}
}
protected virtual void RecordClosedSocket(Socket sock)
{
if (TryRemoveClosedSocket(sock))
{
NetworkingStatisticsGroup.OnClosedReceivingSocket();
}
}
internal void RecordOpenedSocket(Socket sock, GrainId clientId)
{
lock (lockable)
{
logger.Info(ErrorCode.GatewayClientOpenedSocket, "Recorded opened socket from endpoint {0}, client ID {1}.", sock.RemoteEndPoint, clientId);
ClientState clientState;
if (clients.TryGetValue(clientId, out clientState))
{
var oldSocket = clientState.Socket;
if (oldSocket != null)
{
// The old socket will be closed by itself later.
ClientState ignore;
clientSockets.TryRemove(oldSocket, out ignore);
}
QueueRequest(clientState, null);
}
else
{
int gatewayToUse = nextGatewaySenderToUseForRoundRobin % senders.Length;
nextGatewaySenderToUseForRoundRobin++; // under Gateway lock
clientState = new ClientState(clientId, gatewayToUse);
clients[clientId] = clientState;
MessagingStatisticsGroup.ConnectedClientCount.Increment();
}
clientState.RecordConnection(sock);
clientSockets[sock] = clientState;
clientRegistrar.ClientAdded(clientId);
NetworkingStatisticsGroup.OnOpenedGatewayDuplexSocket();
}
}
protected virtual bool RecordOpenedSocket(Socket sock)
{
GrainId client;
if (!ReceiveSocketPreample(sock, false, out client)) return false;
NetworkingStatisticsGroup.OnOpenedReceiveSocket();
return true;
}
private static Task OnConnectedAsync(ConnectionContext context)
{
var connection = context.Features.Get <Connection>();
context.ConnectionClosed.Register(OnConnectionClosedDelegate, connection);
NetworkingStatisticsGroup.OnOpenedSocket(connection.ConnectionDirection);
return(connection.RunInternal());
}
private void CloseSocket(Socket socket)
{
SocketManager.CloseSocket(socket);
NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
if (Interlocked.Decrement(ref this.connectedCount) == 0)
{
MsgCenter.OnGatewayConnectionClosed();
}
}
private void CloseInternal(Exception exception)
{
if (!this.IsValid)
{
return;
}
lock (this.lockObj)
{
try
{
if (!this.IsValid)
{
return;
}
this.IsValid = false;
NetworkingStatisticsGroup.OnClosedSocket(this.ConnectionDirection);
this.closeRegistration.Dispose();
this.closeRegistration = default;
if (this.Log.IsEnabled(LogLevel.Information))
{
this.Log.LogInformation(
"Closing connection with remote endpoint {EndPoint}",
this.RemoteEndPoint,
Environment.StackTrace);
}
// Try to gracefully stop the reader/writer loops.
this.Context.Transport.Input.CancelPendingRead();
this.Context.Transport.Output.CancelPendingFlush();
this.outgoingMessageWriter.TryComplete();
if (exception is null)
{
this.Context.Abort();
}
else
{
var abortedException = exception as ConnectionAbortedException
?? new ConnectionAbortedException(
$"Connection closed. See {nameof(Exception.InnerException)}",
exception);
this.Context.Abort(abortedException);
}
}
catch (Exception innerException)
{
// Swallow any exceptions here.
this.Log.LogWarning(innerException, "Exception closing connection with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, innerException);
}
}
}
internal void RecordDisconnection()
{
if (Socket == null)
{
return;
}
DisconnectedSince = DateTime.UtcNow;
Socket = null;
NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
}
public SiloStatisticsManager(
IOptions <StatisticsOptions> statisticsOptions,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init();
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init();
StorageStatisticsGroup.Init();
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, loggerFactory);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
/* Temp fn only for testing Nekara service */
private static System.Threading.Tasks.Task OnConnectedAsync_temp(ConnectionContext context)
{
var connection = context.Features.Get <Connection>();
context.ConnectionClosed.Register(OnConnectionClosedDelegate, connection);
NetworkingStatisticsGroup.OnOpenedSocket(connection.ConnectionDirection);
var _t1 = connection.RunInternal().InnerTask;
return(_t1);
}
protected virtual bool RecordOpenedSocket(Socket sock)
{
Guid client;
if (!ReceiveSocketPreample(sock, false, out client))
{
return(false);
}
NetworkingStatisticsGroup.OnOpenedReceiveSocket();
return(true);
}
internal SiloStatisticsManager(GlobalConfiguration globalConfig, NodeConfiguration nodeConfig)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(globalConfig.ResponseTimeout);
SchedulerStatisticsGroup.Init();
StorageStatisticsGroup.Init();
runtimeStats = new RuntimeStatisticsGroup();
logStatistics = new LogStatistics(nodeConfig.StatisticsLogWriteInterval, true);
MetricsTable = new SiloPerformanceMetrics(runtimeStats, nodeConfig);
perfCountersPublisher = new PerfCountersStatistics(nodeConfig.StatisticsPerfCountersWriteInterval);
}
public SiloStatisticsManager(NodeConfiguration nodeConfiguration, ILocalSiloDetails siloDetails, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory, IOptions <MessagingOptions> messagingOptions)
{
this.siloDetails = siloDetails;
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
runtimeStats = new RuntimeStatisticsGroup(loggerFactory);
this.logStatistics = new LogStatistics(nodeConfiguration.StatisticsLogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, loggerFactory, nodeConfiguration);
this.countersPublisher = new CountersStatistics(nodeConfiguration.StatisticsPerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public SiloStatisticsManager(
IOptions <SiloStatisticsOptions> statisticsOptions,
SerializationManager serializationManager,
ITelemetryProducer telemetryProducer,
ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init();
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logStatistics = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
}
public SiloStatisticsManager(SiloInitializationParameters initializationParams)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(initializationParams.GlobalConfig.ResponseTimeout);
SchedulerStatisticsGroup.Init();
StorageStatisticsGroup.Init();
runtimeStats = new RuntimeStatisticsGroup();
this.logStatistics = new LogStatistics(initializationParams.NodeConfig.StatisticsLogWriteInterval, true);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, initializationParams.NodeConfig);
this.countersPublisher = new CountersStatistics(initializationParams.NodeConfig.StatisticsPerfCountersWriteInterval);
initializationParams.ClusterConfig.OnConfigChange(
"Defaults/LoadShedding",
() => this.MetricsTable.NodeConfig = initializationParams.NodeConfig,
false);
}
public SiloStatisticsManager(SiloInitializationParameters initializationParams, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
MessagingStatisticsGroup.Init(true);
MessagingProcessingStatisticsGroup.Init();
NetworkingStatisticsGroup.Init(true);
ApplicationRequestsStatisticsGroup.Init(initializationParams.ClusterConfig.Globals.ResponseTimeout);
SchedulerStatisticsGroup.Init(loggerFactory);
StorageStatisticsGroup.Init();
TransactionsStatisticsGroup.Init();
this.logger = new LoggerWrapper <SiloStatisticsManager>(loggerFactory);
runtimeStats = new RuntimeStatisticsGroup(loggerFactory);
this.logStatistics = new LogStatistics(initializationParams.NodeConfig.StatisticsLogWriteInterval, true, serializationManager, loggerFactory);
this.MetricsTable = new SiloPerformanceMetrics(this.runtimeStats, loggerFactory, initializationParams.NodeConfig);
this.countersPublisher = new CountersStatistics(initializationParams.NodeConfig.StatisticsPerfCountersWriteInterval, telemetryProducer, loggerFactory);
initializationParams.ClusterConfig.OnConfigChange(
"Defaults/LoadShedding",
() => this.MetricsTable.NodeConfig = initializationParams.NodeConfig,
false);
}
public override void Stop()
{
IsLive = false;
receiver.Stop();
base.Stop();
Socket s;
lock (Lockable)
{
s = Socket;
Socket = null;
}
if (s == null)
{
return;
}
SocketManager.CloseSocket(s);
NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
}
public override void Stop()
{
IsLive = false;
receiver.Stop();
base.Stop();
RuntimeClient.Current.BreakOutstandingMessagesToDeadSilo(Silo);
Socket s;
lock (Lockable)
{
s = Socket;
Socket = null;
}
if (s == null)
{
return;
}
SocketManager.CloseSocket(s);
NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
}
// passed the exact same socket on which it got SocketException. This way we prevent races between connect and disconnect.
public void MarkAsDisconnected(Socket socket2Disconnect)
{
Socket s = null;
lock (Lockable)
{
if (socket2Disconnect == null || Socket == null)
{
return;
}
if (Socket == socket2Disconnect) // handles races between connect and disconnect, since sometimes we grab the socket outside lock.
{
s = Socket;
Socket = null;
Log.Warn(ErrorCode.ProxyClient_MarkGatewayDisconnected, String.Format("Marking gateway at address {0} as Disconnected", Address));
if (MsgCenter != null && MsgCenter.GatewayManager != null)
{
// We need a refresh...
MsgCenter.GatewayManager.ExpediteUpdateLiveGatewaysSnapshot();
}
}
}
if (s != null)
{
SocketManager.CloseSocket(s);
NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
}
if (socket2Disconnect == s)
{
return;
}
SocketManager.CloseSocket(socket2Disconnect);
NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
}
private void CloseSocket(Socket socket)
{
SocketManager.CloseSocket(socket);
NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
MsgCenter.OnGatewayConnectionClosed();
}
public void Connect()
{
if (!MsgCenter.Running)
{
if (Log.IsVerbose)
{
Log.Verbose(ErrorCode.ProxyClient_MsgCtrNotRunning, "Ignoring connection attempt to gateway {0} because the proxy message center is not running", Address);
}
return;
}
// Yes, we take the lock around a Sleep. The point is to ensure that no more than one thread can try this at a time.
// There's still a minor problem as written -- if the sending thread and receiving thread both get here, the first one
// will try to reconnect. eventually do so, and then the other will try to reconnect even though it doesn't have to...
// Hopefully the initial "if" statement will prevent that.
lock (Lockable)
{
if (!IsLive)
{
if (Log.IsVerbose)
{
Log.Verbose(ErrorCode.ProxyClient_DeadGateway, "Ignoring connection attempt to gateway {0} because this gateway connection is already marked as non live", Address);
}
return; // if the connection is already marked as dead, don't try to reconnect. It has been doomed.
}
for (var i = 0; i < ProxiedMessageCenter.CONNECT_RETRY_COUNT; i++)
{
try
{
if (Socket != null)
{
if (Socket.Connected)
{
return;
}
MarkAsDisconnected(Socket); // clean up the socket before reconnecting.
}
if (lastConnect != new DateTime())
{
// We already tried at least once in the past to connect to this GW.
// If we are no longer connected to this GW and it is no longer in the list returned
// from the GatewayProvider, consider directly this connection dead.
if (!MsgCenter.GatewayManager.GetLiveGateways().Contains(Address))
{
break;
}
// Wait at least ProxiedMessageCenter.MINIMUM_INTERCONNECT_DELAY before reconnection tries
var millisecondsSinceLastAttempt = DateTime.UtcNow - lastConnect;
if (millisecondsSinceLastAttempt < ProxiedMessageCenter.MINIMUM_INTERCONNECT_DELAY)
{
var wait = ProxiedMessageCenter.MINIMUM_INTERCONNECT_DELAY - millisecondsSinceLastAttempt;
if (Log.IsVerbose)
{
Log.Verbose(ErrorCode.ProxyClient_PauseBeforeRetry, "Pausing for {0} before trying to connect to gateway {1} on trial {2}", wait, Address, i);
}
Thread.Sleep(wait);
}
}
lastConnect = DateTime.UtcNow;
Socket = new Socket(Silo.Endpoint.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
SocketManager.Connect(Socket, Silo.Endpoint, MsgCenter.MessagingConfiguration.OpenConnectionTimeout);
NetworkingStatisticsGroup.OnOpenedGatewayDuplexSocket();
MsgCenter.OnGatewayConnectionOpen();
SocketManager.WriteConnectionPreamble(Socket, MsgCenter.ClientId); // Identifies this client
Log.Info(ErrorCode.ProxyClient_Connected, "Connected to gateway at address {0} on trial {1}.", Address, i);
return;
}
catch (Exception ex)
{
Log.Warn(ErrorCode.ProxyClient_CannotConnect, $"Unable to connect to gateway at address {Address} on trial {i} (Exception: {ex.Message})");
MarkAsDisconnected(Socket);
}
}
// Failed too many times -- give up
MarkAsDead();
}
}
private void CloseInternal(Exception exception)
{
if (!this.IsValid)
{
return;
}
lock (this.lockObj)
{
try
{
if (!this.IsValid)
{
return;
}
this.IsValid = false;
NetworkingStatisticsGroup.OnClosedSocket(this.ConnectionDirection);
if (this.Log.IsEnabled(LogLevel.Information))
{
if (exception is null)
{
this.Log.LogInformation(
"Closing connection with remote endpoint {EndPoint}",
this.RemoteEndPoint);
}
else
{
this.Log.LogInformation(
exception,
"Closing connection with remote endpoint {EndPoint}. Exception: {Exception}",
this.RemoteEndPoint,
exception);
}
}
// Try to gracefully stop the reader/writer loops, if they are running.
try
{
if (_processIncomingTask is Task task && !task.IsCompleted)
{
this.Context.Transport.Input.CancelPendingRead();
}
}
catch (Exception cancelException)
{
// Swallow any exceptions here.
this.Log.LogWarning(cancelException, "Exception canceling pending read with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, cancelException);
}
try
{
if (_processOutgoingTask is Task task && !task.IsCompleted)
{
this.Context.Transport.Output.CancelPendingFlush();
}
}
catch (Exception cancelException)
{
// Swallow any exceptions here.
this.Log.LogWarning(cancelException, "Exception canceling pending flush with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, cancelException);
}
this.outgoingMessageWriter.TryComplete();
if (exception is null)
{
this.Context.Abort();
}
else
{
var abortedException = exception as ConnectionAbortedException
?? new ConnectionAbortedException(
$"Connection closed. See {nameof(Exception.InnerException)}",
exception);
this.Context.Abort(abortedException);
}
}
catch (Exception innerException)
{
// Swallow any exceptions here.
this.Log.LogWarning(innerException, "Exception closing connection with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, innerException);
}
}
}
/// <summary>
/// Close the connection. This method should only be called by <see cref="StartClosing(Exception)"/>.
/// </summary>
private async Task CloseAsync()
{
NetworkingStatisticsGroup.OnClosedSocket(this.ConnectionDirection);
// Signal the outgoing message processor to exit gracefully.
this.outgoingMessageWriter.TryComplete();
var transportFeature = Context.Features.Get <IUnderlyingTransportFeature>();
var transport = transportFeature?.Transport ?? _transport;
transport.Input.CancelPendingRead();
transport.Output.CancelPendingFlush();
// Try to gracefully stop the reader/writer loops, if they are running.
if (_processIncomingTask is { IsCompleted : false } incoming)
{
try
{
await incoming;
}
catch (Exception processIncomingException)
{
// Swallow any exceptions here.
this.Log.LogWarning(processIncomingException, "Exception processing incoming messages on connection {Connection}", this);
}
}
if (_processOutgoingTask is { IsCompleted : false } outgoing)
{
try
{
await outgoing;
}
catch (Exception processOutgoingException)
{
// Swallow any exceptions here.
this.Log.LogWarning(processOutgoingException, "Exception processing outgoing messages on connection {Connection}", this);
}
}
// Only wait for the transport to close if the connection actually started being processed.
if (_processIncomingTask is not null && _processOutgoingTask is not null)
{
// Wait for the transport to signal that it's closed before disposing it.
await _transportConnectionClosed.Task;
}
try
{
await this.Context.DisposeAsync();
}
catch (Exception abortException)
{
// Swallow any exceptions here.
this.Log.LogWarning(abortException, "Exception terminating connection {Connection}", this);
}
// Reject in-flight messages.
foreach (var message in this.inflight)
{
this.OnSendMessageFailure(message, "Connection terminated");
}
this.inflight.Clear();
// Reroute enqueued messages.
var i = 0;
while (this.outgoingMessages.Reader.TryRead(out var message))
{
if (i == 0 && Log.IsEnabled(LogLevel.Information))
{
this.Log.LogInformation(
"Rerouting messages for remote endpoint {EndPoint}",
this.RemoteEndPoint?.ToString() ?? "(never connected)");
}
++i;
this.RetryMessage(message);
}
if (i > 0 && this.Log.IsEnabled(LogLevel.Information))
{
this.Log.LogInformation(
"Rerouted {Count} messages for remote endpoint {EndPoint}",
i,
this.RemoteEndPoint?.ToString() ?? "(never connected)");
}
}