private void TcpConnectionClosed(TcpPackageConnection connection)
{
if (_state == ConnectionState.Init)
{
throw new Exception();
}
if (_state == ConnectionState.Closed || _connection != connection)
{
LogDebug("IGNORED (_state: {0}, _conn.ID: {1:B}, conn.ID: {2:B}): TCP connection to [{3}, L{4}] closed.",
_state, _connection == null ? Guid.Empty : _connection.ConnectionId, connection.ConnectionId,
connection.RemoteEndPoint, connection.LocalEndPoint);
return;
}
_state = ConnectionState.Connecting;
_connectingPhase = ConnectingPhase.Reconnecting;
LogDebug("TCP connection to [{0}, L{1}, {2:B}] closed.", connection.RemoteEndPoint, connection.LocalEndPoint, connection.ConnectionId);
_subscriptions.PurgeSubscribedAndDroppedSubscriptions(_connection.ConnectionId);
_reconnInfo = new ReconnectionInfo(_reconnInfo.ReconnectionAttempt, _stopwatch.Elapsed);
if (Interlocked.CompareExchange(ref _wasConnected, 0, 1) == 1)
{
RaiseDisconnected(connection.RemoteEndPoint);
}
}
internal void DropSubscription(SubscriptionDropReason reason, Exception exc, TcpPackageConnection connection = null)
{
if (Interlocked.CompareExchange(ref _unsubscribed, 1, 0) == 0)
{
if (_verboseLogging)
{
_log.Debug("Subscription {0:B} to {1}: closing subscription, reason: {2}, exception: {3}...",
_correlationId, _streamId == string.Empty ? "<all>" : _streamId, reason, exc);
}
if (reason != SubscriptionDropReason.UserInitiated)
{
if (exc == null)
{
throw new Exception(string.Format("No exception provided for subscription drop reason '{0}", reason));
}
_source.TrySetException(exc);
}
if (reason == SubscriptionDropReason.UserInitiated && _subscription != null && connection != null)
{
connection.EnqueueSend(CreateUnsubscriptionPackage());
}
if (_subscription != null)
{
ExecuteActionAsync(() => _subscriptionDropped(_subscription, reason, exc));
}
}
}
private void TcpConnectionEstablished(TcpPackageConnection connection)
{
if (_state != ConnectionState.Connecting || _connection != connection || connection.IsClosed)
{
LogDebug("IGNORED (_state {0}, _conn.Id {1:B}, conn.Id {2:B}, conn.closed {3}): TCP connection to [{4}, L{5}] established.",
_state, _connection == null ? Guid.Empty : _connection.ConnectionId, connection.ConnectionId,
connection.IsClosed, connection.RemoteEndPoint, connection.LocalEndPoint);
return;
}
LogDebug("TCP connection to [{0}, L{1}, {2:B}] established.", connection.RemoteEndPoint, connection.LocalEndPoint, connection.ConnectionId);
_heartbeatInfo = new HeartbeatInfo(_packageNumber, true, _stopwatch.Elapsed);
if (_settings.DefaultUserCredentials != null)
{
_connectingPhase = ConnectingPhase.Authentication;
_authInfo = new AuthInfo(Guid.NewGuid(), _stopwatch.Elapsed);
_connection.EnqueueSend(new TcpPackage(TcpCommand.Authenticate,
TcpFlags.Authenticated,
_authInfo.CorrelationId,
_settings.DefaultUserCredentials.Username,
_settings.DefaultUserCredentials.Password,
null));
}
else
{
GoToConnectedState();
}
}
public void TryScheduleWaitingOperations(TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
lock (_lock)
{
// We don't want to transmit or retain expired requests, so we trim any from before the cutoff implied by the current time
var cutoff = _settings.QueueTimeout == TimeSpan.Zero ? (DateTime?)null : DateTime.UtcNow - _settings.QueueTimeout;
OperationItem operation;
while (_activeOperations.Count < _settings.MaxConcurrentItems)
{
if (!_waitingOperations.TryDequeue(out operation))
{
break;
}
if (cutoff == null || !TryExpireItem(cutoff.Value, operation))
{
ExecuteOperation(operation, connection);
}
}
if (cutoff != null)
{
// In case the active operations queue is at capacity, we trim expired items from the front of the queue
while (_waitingOperations.TryPeek(out operation) && TryExpireItem(cutoff.Value, operation))
{
_waitingOperations.TryDequeue(out operation);
}
}
_totalOperationCount = _activeOperations.Count + _waitingOperations.Count;
}
}
public void DropSubscription(SubscriptionDropReason reason, Exception exc,
TcpPackageConnection connection = null)
{
if (Interlocked.CompareExchange(ref _unsubscribed, 1, 0) == 0)
{
if (_verboseLogging)
{
_log.Debug("Subscription {0:B} to {1}: closing subscription, reason: {2}, exception: {3}...",
_correlationId, _streamId == string.Empty ? "<all>" : _streamId, reason, exc);
}
if (reason != SubscriptionDropReason.UserInitiated)
{
var er = exc ?? new Exception(String.Format("Subscription dropped for {0}", reason));
_source.TrySetException(er);
}
if (reason == SubscriptionDropReason.UserInitiated && _subscription != null && connection != null)
{
connection.EnqueueSend(CreateUnsubscriptionPackage());
}
if (_subscription != null)
{
ExecuteActionAsync(() => {
_subscriptionDropped(_subscription, reason, exc);
return(TaskEx.CompletedTask);
});
}
}
}
public void StartSubscription(SubscriptionItem subscription, TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
if (subscription.IsSubscribed)
{
LogDebug("StartSubscription REMOVING due to already subscribed {0}.", subscription);
RemoveSubscription(subscription);
return;
}
subscription.CorrelationId = Guid.NewGuid();
subscription.ConnectionId = connection.ConnectionId;
subscription.LastUpdated = DateTime.UtcNow;
_activeSubscriptions.Add(subscription.CorrelationId, subscription);
if (!subscription.Operation.Subscribe(subscription.CorrelationId, connection))
{
LogDebug("StartSubscription REMOVING AS COULD NOT SUBSCRIBE {0}.", subscription);
RemoveSubscription(subscription);
}
else
{
LogDebug("StartSubscription SUBSCRIBING {0}.", subscription);
}
}
public void CheckTimeoutsAndRetry(TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
var retrySubscriptions = new List <SubscriptionItem>();
var removeSubscriptions = new List <SubscriptionItem>();
foreach (var subscription in _activeSubscriptions.Values)
{
if (subscription.IsSubscribed)
{
continue;
}
if (subscription.ConnectionId != connection.ConnectionId)
{
retrySubscriptions.Add(subscription);
}
else if (subscription.Timeout > TimeSpan.Zero && DateTime.UtcNow - subscription.LastUpdated > _settings.OperationTimeout)
{
var err = String.Format("EventStoreConnection '{0}': subscription never got confirmation from server.\n" +
"UTC now: {1:HH:mm:ss.fff}, operation: {2}.",
_connectionName, DateTime.UtcNow, subscription);
_settings.Log.Error(err);
if (_settings.FailOnNoServerResponse)
{
subscription.Operation.DropSubscription(SubscriptionDropReason.SubscribingError, new OperationTimedOutException(err));
removeSubscriptions.Add(subscription);
}
else
{
retrySubscriptions.Add(subscription);
}
}
}
foreach (var subscription in retrySubscriptions)
{
ScheduleSubscriptionRetry(subscription);
}
foreach (var subscription in removeSubscriptions)
{
RemoveSubscription(subscription);
}
if (_retryPendingSubscriptions.Count > 0)
{
foreach (var subscription in _retryPendingSubscriptions)
{
subscription.RetryCount += 1;
StartSubscription(subscription, connection);
}
_retryPendingSubscriptions.Clear();
}
while (_waitingSubscriptions.Count > 0)
{
StartSubscription(_waitingSubscriptions.Dequeue(), connection);
}
}
public void ScheduleWaitingOperations(TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
while (_waitingOperations.Count > 0 && _activeOperations.Count < _settings.MaxConcurrentItems)
{
ScheduleOperation(_waitingOperations.Dequeue(), connection);
}
_totalOperationCount = _activeOperations.Count + _waitingOperations.Count;
}
public void CheckTimeoutsAndRetry(TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
var retryOperations = new List <OperationItem>();
var removeOperations = new List <OperationItem>();
foreach (var operation in _activeOperations.Values)
{
if (operation.ConnectionId != connection.ConnectionId)
{
retryOperations.Add(operation);
}
else if (operation.Timeout > TimeSpan.Zero && DateTime.UtcNow - operation.LastUpdated > _settings.OperationTimeout)
{
var err = string.Format("EventStoreConnection '{0}': operation never got response from server.\n"
+ "UTC now: {1:HH:mm:ss.fff}, operation: {2}.",
_connectionName, DateTime.UtcNow, operation);
_settings.Log.Error(err);
if (_settings.FailOnNoServerResponse)
{
operation.Operation.Fail(new OperationTimedOutException(err));
removeOperations.Add(operation);
}
else
{
retryOperations.Add(operation);
}
}
}
foreach (var operation in retryOperations)
{
ScheduleOperationRetry(operation);
}
foreach (var operation in removeOperations)
{
RemoveOperation(operation);
}
if (_retryPendingOperations.Count > 0)
{
_retryPendingOperations.Sort(SeqNoComparer);
foreach (var operation in _retryPendingOperations)
{
var oldCorrId = operation.CorrelationId;
operation.CorrelationId = Guid.NewGuid();
operation.RetryCount += 1;
LogDebug("retrying, old corrId {0}, operation {1}.", oldCorrId, operation);
ScheduleOperation(operation, connection);
}
_retryPendingOperations.Clear();
}
ScheduleWaitingOperations(connection);
}
public void ExecuteOperation(OperationItem operation, TcpPackageConnection connection)
{
operation.ConnectionId = connection.ConnectionId;
operation.LastUpdated = DateTime.UtcNow;
_activeOperations.Add(operation.CorrelationId, operation);
var package = operation.Operation.CreateNetworkPackage(operation.CorrelationId);
LogDebug("ExecuteOperation package {0}, {1}, {2}.", package.Command, package.CorrelationId, operation);
connection.EnqueueSend(package);
}
public bool Subscribe(Guid correlationId, TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
if (_subscription != null || _unsubscribed != 0)
{
return(false);
}
_correlationId = correlationId;
connection.EnqueueSend(CreateSubscriptionPackage());
return(true);
}
private void CloseTcpConnection(string reason)
{
if (_connection == null)
{
LogDebug("CloseTcpConnection IGNORED because _connection == null");
return;
}
LogDebug("CloseTcpConnection");
_connection.Close(reason);
TcpConnectionClosed(_connection);
_connection = null;
}
private void TcpConnectionError(TcpPackageConnection connection, Exception exception)
{
if (_connection != connection)
{
return;
}
if (_state == ConnectionState.Closed)
{
return;
}
LogDebug("TcpConnectionError connId {0:B}, exc {1}.", connection.ConnectionId, exception);
CloseConnection("TCP connection error occurred.", exception);
}
private void EstablishTcpConnection(NodeEndPoints endPoints)
{
var endPoint = _compatibilityMode.IsAutoCompatibilityModeEnabled() || _settings.UseSslConnection
? endPoints.SecureTcpEndPoint ?? endPoints.TcpEndPoint
: endPoints.TcpEndPoint;
if (endPoint == null)
{
CloseConnection("No end point to node specified.");
return;
}
LogDebug("EstablishTcpConnection to [{0}]", endPoint);
if (_state != ConnectionState.Connecting)
{
LogDebug("EstablishTcpConnection to [{0}] skipped because expected state 'Connecting', was '{1}'",
endPoint, _state);
return;
}
if (_connectingPhase != ConnectingPhase.EndPointDiscovery)
{
LogDebug(
"EstablishTcpConnection to [{0}] skipped because expected connecting phase 'EndPointDiscovery', was '{1}'",
endPoint, _connectingPhase);
return;
}
var useSsl = _compatibilityMode.IsAutoCompatibilityModeEnabled()
? endPoints.SecureTcpEndPoint != null
: _settings.UseSslConnection;
_connectingPhase = ConnectingPhase.ConnectionEstablishing;
_connection = new TcpPackageConnection(
_settings.Log,
endPoint,
Guid.NewGuid(),
useSsl,
_settings.ValidateServer,
_settings.ClientConnectionTimeout,
(connection, package) => EnqueueMessage(new HandleTcpPackageMessage(connection, package)),
(connection, exc) => EnqueueMessage(new TcpConnectionErrorMessage(connection, exc)),
connection => EnqueueMessage(new TcpConnectionEstablishedMessage(connection)),
(connection, error) => EnqueueMessage(new TcpConnectionClosedMessage(connection, error)));
_connection.StartReceiving();
}
public void ScheduleOperation(OperationItem operation, TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
if (_activeOperations.Count >= _settings.MaxConcurrentItems)
{
LogDebug("ScheduleOperation WAITING for {0}.", operation);
_waitingOperations.Enqueue(operation);
}
else
{
operation.ConnectionId = connection.ConnectionId;
operation.LastUpdated = DateTime.UtcNow;
_activeOperations.Add(operation.CorrelationId, operation);
var package = operation.Operation.CreateNetworkPackage(operation.CorrelationId);
LogDebug("ScheduleOperation package {0}, {1}, {2}.", package.Command, package.CorrelationId, operation);
connection.EnqueueSend(package);
}
_totalOperationCount = _activeOperations.Count + _waitingOperations.Count;
}
private void EstablishTcpConnection(NodeEndPoints endPoints)
{
var endPoint = _settings.UseSslConnection
? endPoints.SecureTcpEndPoint ?? endPoints.TcpEndPoint
: endPoints.TcpEndPoint;
if (endPoint == null)
{
CloseConnection("No end point to node specified.");
return;
}
LogDebug("EstablishTcpConnection to [{0}]", endPoint);
if (_state != ConnectionState.Connecting)
{
return;
}
if (_connectingPhase != ConnectingPhase.EndPointDiscovery)
{
return;
}
_connectingPhase = ConnectingPhase.ConnectionEstablishing;
_connection = new TcpPackageConnection(
_settings.Log,
endPoint,
Guid.NewGuid(),
_settings.UseSslConnection,
_settings.TargetHost,
_settings.ValidateServer,
_settings.ClientConnectionTimeout,
(connection, package) => EnqueueMessage(new HandleTcpPackageMessage(connection, package)),
(connection, exc) => EnqueueMessage(new TcpConnectionErrorMessage(connection, exc)),
connection => EnqueueMessage(new TcpConnectionEstablishedMessage(connection)),
(connection, error) => EnqueueMessage(new TcpConnectionClosedMessage(connection, error)));
_connection.StartReceiving();
}
public HandleTcpPackageMessage(TcpPackageConnection connection, TcpPackage package)
{
Connection = connection;
Package = package;
}
public TcpConnectionErrorMessage(TcpPackageConnection connection, Exception exception)
{
Connection = connection;
Exception = exception;
}
public TcpConnectionEstablishedMessage(TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
Connection = connection;
}
public TcpConnectionClosedMessage(TcpPackageConnection connection, SocketError error)
{
Ensure.NotNull(connection, "connection");
Connection = connection;
Error = error;
}
private void HandleTcpPackage(TcpPackageConnection connection, TcpPackage package)
{
if (_connection != connection || _state == ConnectionState.Closed || _state == ConnectionState.Init)
{
LogDebug("IGNORED: HandleTcpPackage connId {0}, package {1}, {2}.", connection.ConnectionId, package.Command, package.CorrelationId);
return;
}
LogDebug("HandleTcpPackage connId {0}, package {1}, {2}.", _connection.ConnectionId, package.Command, package.CorrelationId);
_packageNumber += 1;
if (package.Command == TcpCommand.HeartbeatResponseCommand)
{
return;
}
if (package.Command == TcpCommand.HeartbeatRequestCommand)
{
_connection.EnqueueSend(new TcpPackage(TcpCommand.HeartbeatResponseCommand, package.CorrelationId, null));
return;
}
if (package.Command == TcpCommand.Authenticated || package.Command == TcpCommand.NotAuthenticated)
{
if (_state == ConnectionState.Connecting &&
_connectingPhase == ConnectingPhase.Authentication &&
_authInfo.CorrelationId == package.CorrelationId)
{
if (package.Command == TcpCommand.NotAuthenticated)
{
RaiseAuthenticationFailed("Not authenticated");
}
GoToConnectedState();
return;
}
}
if (package.Command == TcpCommand.BadRequest && package.CorrelationId == Guid.Empty)
{
string message = Helper.EatException(() => Helper.UTF8NoBom.GetString(package.Data.Array, package.Data.Offset, package.Data.Count));
var exc = new EventStoreConnectionException(
string.Format("Bad request received from server. Error: {0}", string.IsNullOrEmpty(message) ? "<no message>" : message));
CloseConnection("Connection-wide BadRequest received. Too dangerous to continue.", exc);
return;
}
OperationItem operation;
SubscriptionItem subscription;
if (_operations.TryGetActiveOperation(package.CorrelationId, out operation))
{
var result = operation.Operation.InspectPackage(package);
LogDebug("HandleTcpPackage OPERATION DECISION {0} ({1}), {2}", result.Decision, result.Description, operation);
switch (result.Decision)
{
case InspectionDecision.DoNothing: break;
case InspectionDecision.EndOperation:
_operations.RemoveOperation(operation);
break;
case InspectionDecision.Retry:
_operations.ScheduleOperationRetry(operation);
break;
case InspectionDecision.Reconnect:
ReconnectTo(new NodeEndPoints(result.TcpEndPoint, result.SecureTcpEndPoint));
_operations.ScheduleOperationRetry(operation);
break;
default: throw new Exception(string.Format("Unknown InspectionDecision: {0}", result.Decision));
}
if (_state == ConnectionState.Connected)
{
_operations.ScheduleWaitingOperations(connection);
}
}
else if (_subscriptions.TryGetActiveSubscription(package.CorrelationId, out subscription))
{
var result = subscription.Operation.InspectPackage(package);
LogDebug("HandleTcpPackage SUBSCRIPTION DECISION {0} ({1}), {2}", result.Decision, result.Description, subscription);
switch (result.Decision)
{
case InspectionDecision.DoNothing: break;
case InspectionDecision.EndOperation:
_subscriptions.RemoveSubscription(subscription);
break;
case InspectionDecision.Retry:
_subscriptions.ScheduleSubscriptionRetry(subscription);
break;
case InspectionDecision.Reconnect:
ReconnectTo(new NodeEndPoints(result.TcpEndPoint, result.SecureTcpEndPoint));
_subscriptions.ScheduleSubscriptionRetry(subscription);
break;
case InspectionDecision.Subscribed:
subscription.IsSubscribed = true;
break;
default: throw new Exception(string.Format("Unknown InspectionDecision: {0}", result.Decision));
}
}
else
{
LogDebug("HandleTcpPackage UNMAPPED PACKAGE with CorrelationId {0:B}, Command: {1}", package.CorrelationId, package.Command);
}
}
public void ScheduleOperation(OperationItem operation, TcpPackageConnection connection)
{
Ensure.NotNull(connection, "connection");
_waitingOperations.Enqueue(operation);
TryScheduleWaitingOperations(connection);
}
