public void Connect()
{
_connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
if (_executionThread == null)
{
_executionThread = new Thread(ExecuteUserCallbacks)
{
IsBackground = true,
Name = "SubscriptionsChannel user callbacks thread"
};
_executionThread.Start();
}
}
private void OnPackageReceived(Connection typedTcpConnection, TcpPackage package)
{
WorkItem workItem;
ITaskCompletionWrapper wrapper;
if (RemoveInProgressItem(package.CorrelationId, "PR", out workItem, out wrapper))
{
}
else
{
if (!_outstandingOperations.TryGetValue(package.CorrelationId, out wrapper))
{
// "SKIPPED [[ "
return;
}
}
var result = wrapper.Process(package);
switch (result.Status)
{
case ProcessResultStatus.Success:
wrapper.Complete();
break;
case ProcessResultStatus.Retry:
if (wrapper.UpdateForNextAttempt())
{
RegisterHandler(wrapper.SentPackage.CorrelationId, wrapper);
EnqueueForSend(wrapper.SentPackage);
}
else
{
wrapper.Fail(new Exception(string.Format("Retry is not supported in wrapper {0}.", wrapper.GetType())));
}
break;
case ProcessResultStatus.NotifyError:
wrapper.Fail(result.Exception);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public EventStoreConnection(IPEndPoint tcpEndPoint)
{
Ensure.NotNull(tcpEndPoint, "tcpEndPoint");
_tcpEndPoint = tcpEndPoint;
_connector = new TcpConnector(_tcpEndPoint);
_outstandingOperations = new ConcurrentDictionary <Guid, ITaskCompletionWrapper>();
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived, ConnectionEstablished, ConnectionClosed);
_timeoutCheckStopwatch.Start();
_monitoringThread = new Thread(MonitoringLoop)
{
IsBackground = true,
Name = string.Format("Monitoring thread")
};
_monitoringThread.Start();
}
private void ConnectionEstablished(Connection tcpTypedConnection)
{
}
private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection)
{
OnConnected();
lock (_connectionLock)
{
_reconnectionsCount = 0;
}
}
private void MainLoop()
{
while (!_stopping)
{
IClientOperation operation;
if (_inProgressCount < _settings.MaxConcurrentItems && _queue.TryDequeue(out operation))
{
Interlocked.Increment(ref _inProgressCount);
Send(new WorkItem(operation));
}
else
{
Thread.Sleep(1);
}
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= _settings.ReconnectionDelay)
{
OnReconnecting();
_reconnectionsCount += 1;
if (_reconnectionsCount > _settings.MaxReconnections)
Close();
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(_tcpEndPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > _settings.OperationTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
var retriable = new List<WorkItem>();
foreach (var workerItem in _inProgress.Values)
{
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > _settings.OperationTimeout)
{
if (lastUpdated >= _lastReconnectionTimestamp)
{
var err = string.Format("{0} never got response from server" +
"Last state update : {1}, last reconnect : {2}, now(utc) : {3}.",
workerItem,
lastUpdated,
_lastReconnectionTimestamp,
now);
if (TryRemoveWorkItem(workerItem))
{
_log.Error(err);
workerItem.Operation.Fail(new OperationTimedOutException(err));
}
_log.Error(err);
}
else
{
retriable.Add(workerItem);
}
}
}
foreach (var workItem in retriable.OrderBy(wi => wi.SeqNo))
{
Retry(workItem);
}
_timeoutCheckStopwatch.Restart();
}
}
}
public TcpTypedConnection CreateTcpConnection(Action<TcpTypedConnection, TcpPackage> handlePackage,
Action<TcpTypedConnection> connectionEstablished,
Action<TcpTypedConnection, IPEndPoint, SocketError> connectionClosed)
{
var connectionCreatedEvent = new AutoResetEvent(false);
TcpTypedConnection typedConnection = null;
var connection = _connector.ConnectTo(
_tcpEndpoint,
tcpConnection =>
{
_log.Debug("Connected to [{0}].", tcpConnection.EffectiveEndPoint);
connectionCreatedEvent.WaitOne(500);
connectionEstablished(typedConnection);
},
(conn, error) =>
{
var message = string.Format("Connection to [{0}] failed. Error: {1}.", conn.EffectiveEndPoint, error);
_log.Debug(message);
connectionClosed(null, conn.EffectiveEndPoint, error);
});
typedConnection = new TcpTypedConnection(connection);
typedConnection.ConnectionClosed +=
(conn, error) =>
{
_log.Debug("Connection [{0}] was closed {1}",
conn.EffectiveEndPoint,
error == SocketError.Success ? "cleanly." : "with error: " + error + ".");
connectionClosed(conn, conn.EffectiveEndPoint, error);
};
connectionCreatedEvent.Set();
typedConnection.ReceiveAsync((conn, pkg) =>
{
var package = new TcpPackage();
var valid = false;
try
{
package = TcpPackage.FromArraySegment(new ArraySegment<byte>(pkg));
valid = true;
if (package.Command == TcpCommand.HeartbeatRequestCommand)
{
var response = new TcpPackage(TcpCommand.HeartbeatResponseCommand, Guid.NewGuid(), null);
conn.EnqueueSend(response.AsByteArray());
return;
}
handlePackage(conn, package);
}
catch (Exception e)
{
var effectiveEndPoint = conn.EffectiveEndPoint;
var message = string.Format("[{0}] ERROR for {1}. Connection will be closed.",
effectiveEndPoint,
valid ? package.Command as object : "<invalid package>");
_log.Debug(e, message);
conn.Close();
}
});
return typedConnection;
}
private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection)
{
_connectedEvent.Set();
}
private void Connect(IPEndPoint endPoint)
{
_connection = _connector.CreateTcpConnection(endPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
}
private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error)
{
_connectedEvent.Reset();
foreach (var correlationId in _subscriptions.Keys)
{
DropSubscription(correlationId);
}
}
private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package)
{
SubscriptionTaskPair subscription;
if (!_subscriptions.TryGetValue(package.CorrelationId, out subscription))
{
_log.Error("Unexpected package received : {0} ({1})", package.CorrelationId, package.Command);
return;
}
try
{
switch (package.Command)
{
case TcpCommand.SubscriptionConfirmation:
{
var dto = package.Data.Deserialize<ClientMessage.SubscriptionConfirmation>();
subscription.Subscription.ConfirmSubscription(dto.LastCommitPosition, dto.LastEventNumber);
subscription.TaskCompletionSource.SetResult(subscription.Subscription);
break;
}
case TcpCommand.StreamEventAppeared:
{
var dto = package.Data.Deserialize<ClientMessage.StreamEventAppeared>();
ExecuteUserCallbackAsync(() => subscription.Subscription.EventAppeared(new ResolvedEvent(dto.Event)));
break;
}
case TcpCommand.SubscriptionDropped:
{
DropSubscription(package.CorrelationId);
break;
}
default:
throw new ArgumentOutOfRangeException(string.Format("Unexpected command : {0}", package.Command));
}
}
catch (Exception e)
{
_log.Error(e, "Error on package received");
}
}
public EventStoreConnection(IPEndPoint tcpEndPoint)
{
Ensure.NotNull(tcpEndPoint, "tcpEndPoint");
_tcpEndPoint = tcpEndPoint;
_connector = new TcpConnector(_tcpEndPoint);
_outstandingOperations = new ConcurrentDictionary<Guid, ITaskCompletionWrapper>();
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived, ConnectionEstablished, ConnectionClosed);
_timeoutCheckStopwatch.Start();
_monitoringThread = new Thread(MonitoringLoop)
{
IsBackground = true,
Name = string.Format("Monitoring thread")
};
_monitoringThread.Start();
}
private void OnPackageReceived(Connection typedTcpConnection, TcpPackage package)
{
WorkItem workItem;
ITaskCompletionWrapper wrapper;
if (RemoveInProgressItem(package.CorrelationId, "PR", out workItem, out wrapper))
{
}
else
{
if (!_outstandingOperations.TryGetValue(package.CorrelationId, out wrapper))
{
// "SKIPPED [[ "
return;
}
}
var result = wrapper.Process(package);
switch (result.Status)
{
case ProcessResultStatus.Success:
wrapper.Complete();
break;
case ProcessResultStatus.Retry:
if (wrapper.UpdateForNextAttempt())
{
RegisterHandler(wrapper.SentPackage.CorrelationId, wrapper);
EnqueueForSend(wrapper.SentPackage);
}
else
wrapper.Fail(new Exception(string.Format("Retry is not supported in wrapper {0}.", wrapper.GetType())));
break;
case ProcessResultStatus.NotifyError:
wrapper.Fail(result.Exception);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void MonitoringLoop()
{
while (!_monitoringThreadStop)
{
TcpPackage nextPackage;
if (_inProgressCount < TcpSentReceiveWindow && _sendQueue.TryDequeue(out nextPackage))
{
Interlocked.Increment(ref _inProgressCount);
var workItem = new WorkItem(nextPackage);
Send(workItem, null);
}
else
Thread.Sleep(1);
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
{
_reconnectionsCount += 1;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived,
ConnectionEstablished,
ConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
foreach (var kvp in _inProgress)
{
var correlationId = kvp.Key;
var workerItem = kvp.Value;
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > EventTimeoutDelay || _reconnectionsCount > 10)
{
if (lastUpdated > _lastReconnectionTimestamp || _reconnectionsCount > 10)
{
WorkItem workItem;
ITaskCompletionWrapper completionWrapper;
if (RemoveInProgressItem(correlationId, " -ML timeout", out workItem, out completionWrapper))
{
completionWrapper.Fail(
new Exception(
string.Format("Timed out event {0} which "
+ "never got response from server was discovered. "
+ "Last state update: {1}, last reconnect: {2}, now: {3}.",
workerItem.TcpPackage.CorrelationId,
lastUpdated,
_lastReconnectionTimestamp,
now)));
}
}
else
Retry(correlationId);
}
}
_timeoutCheckStopwatch.Restart();
}
}
}
private void Connect(IPEndPoint endPoint)
{
_connection = _connector.CreateTcpConnection(endPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
if (_executionThread == null)
{
_stopExecutionThread = false;
_executionThread = new Thread(ExecuteUserCallbacks)
{
IsBackground = true,
Name = "SubscriptionsChannel user callbacks thread"
};
_executionThread.Start();
}
}
private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error)
{
_connectedEvent.Reset();
var subscriptions = _subscriptions.Values;
_subscriptions.Clear();
foreach (var subscription in subscriptions)
{
subscription.Source.SetResult(null);
ExecuteUserCallbackAsync(subscription.SubscriptionDropped);
}
}
public Connection CreateTcpConnection(Action <Connection, TcpPackage> handlePackage,
Action <Connection> connectionEstablished,
Action <Connection, IPEndPoint, SocketError> connectionClosed)
{
var connectionCreatedEvent = new AutoResetEvent(false);
Connection typedConnection = null;
var connection = _connector.ConnectTo(
_tcpEndpoint,
tcpConnection =>
{
Log.Info("Connected to [{0}].", tcpConnection.EffectiveEndPoint);
connectionCreatedEvent.WaitOne(500);
connectionEstablished(typedConnection);
},
(conn, error) =>
{
var message = string.Format("Connection to [{0}] failed. Error: {1}.", conn.EffectiveEndPoint, error);
Log.Error(message);
connectionClosed(null, conn.EffectiveEndPoint, error);
});
typedConnection = new Connection(connection);
typedConnection.ConnectionClosed +=
(conn, error) =>
{
Log.Info("Connection [{0}] was closed {1}",
conn.EffectiveEndPoint,
error == SocketError.Success ? "cleanly." : "with error: " + error + ".");
connectionClosed(conn, conn.EffectiveEndPoint, error);
};
connectionCreatedEvent.Set();
typedConnection.ReceiveAsync((conn, pkg) =>
{
var package = new TcpPackage();
var valid = false;
try
{
package = TcpPackage.FromArraySegment(new ArraySegment <byte>(pkg));
valid = true;
if (package.Command == TcpCommand.HeartbeatRequestCommand)
{
var response = new TcpPackage(TcpCommand.HeartbeatResponseCommand, Guid.NewGuid(), null);
conn.EnqueueSend(response.AsByteArray());
return;
}
handlePackage(conn, package);
}
catch (Exception e)
{
var effectiveEndPoint = conn.EffectiveEndPoint;
var message = string.Format("[{0}] ERROR for {1}. Connection will be closed.",
effectiveEndPoint,
valid ? package.Command as object : "<invalid package>");
Log.Info(e, message);
conn.Close();
}
});
return(typedConnection);
}
private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package)
{
Subscription subscription;
if(!_subscriptions.TryGetValue(package.CorrelationId, out subscription))
{
_log.Error("Unexpected package received : {0} ({1})", package.CorrelationId, package.Command);
return;
}
try
{
switch (package.Command)
{
case TcpCommand.StreamEventAppeared:
var dto = package.Data.Deserialize<ClientMessage.StreamEventAppeared>();
var recordedEvent = new RecordedEvent(dto);
var commitPos = dto.CommitPosition;
var preparePos = dto.PreparePosition;
ExecuteUserCallbackAsync(() => subscription.EventAppeared(recordedEvent, new Position(commitPos, preparePos)));
break;
case TcpCommand.SubscriptionDropped:
case TcpCommand.SubscriptionToAllDropped:
Subscription removed;
if(_subscriptions.TryRemove(subscription.Id, out removed))
{
removed.Source.SetResult(null);
ExecuteUserCallbackAsync(removed.SubscriptionDropped);
}
break;
default:
throw new ArgumentOutOfRangeException(string.Format("Unexpected command : {0}", package.Command));
}
}
catch (Exception e)
{
_log.Error(e, "Error on package received");
}
}
private void ConnectionEstablished(Connection tcpTypedConnection)
{
}
private Task EstablishConnectionAsync(IPEndPoint tcpEndPoint)
{
lock (_connectionLock)
{
if (_active)
throw new InvalidOperationException("EventStoreConnection is already active");
if (_stopping)
throw new InvalidOperationException("EventStoreConnection has been closed");
_active = true;
_tcpEndPoint = tcpEndPoint;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(_tcpEndPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
_timeoutCheckStopwatch.Start();
_worker = new Thread(MainLoop) {IsBackground = true, Name = "Worker thread"};
_worker.Start();
return Tasks.CreateCompleted();
}
}
private void ConnectionClosed(Connection tcpTypedConnection, IPEndPoint endPoint, SocketError socketError)
{
lock (_connectionLock)
_reconnectionStopwatch.Restart();
}
private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package)
{
var corrId = package.CorrelationId;
WorkItem workItem;
if (!_inProgress.TryGetValue(corrId, out workItem))
{
_log.Error("Unexpected CorrelationId received {{{0}}}", corrId);
return;
}
var result = workItem.Operation.InspectPackage(package);
switch (result.Decision)
{
case InspectionDecision.Succeed:
{
if (TryRemoveWorkItem(workItem))
workItem.Operation.Complete();
break;
}
case InspectionDecision.Retry:
{
Retry(workItem);
break;
}
case InspectionDecision.Reconnect:
{
Reconnect(workItem, (IPEndPoint) result.Data);
break;
}
case InspectionDecision.NotifyError:
{
if (TryRemoveWorkItem(workItem))
workItem.Operation.Fail(result.Error);
break;
}
default:
throw new ArgumentOutOfRangeException();
}
}
private void MonitoringLoop()
{
while (!_monitoringThreadStop)
{
TcpPackage nextPackage;
if (_inProgressCount < TcpSentReceiveWindow && _sendQueue.TryDequeue(out nextPackage))
{
Interlocked.Increment(ref _inProgressCount);
var workItem = new WorkItem(nextPackage);
Send(workItem, null);
}
else
{
Thread.Sleep(1);
}
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
{
_reconnectionsCount += 1;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived,
ConnectionEstablished,
ConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
foreach (var kvp in _inProgress)
{
var correlationId = kvp.Key;
var workerItem = kvp.Value;
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > EventTimeoutDelay || _reconnectionsCount > 10)
{
if (lastUpdated > _lastReconnectionTimestamp || _reconnectionsCount > 10)
{
WorkItem workItem;
ITaskCompletionWrapper completionWrapper;
if (RemoveInProgressItem(correlationId, " -ML timeout", out workItem, out completionWrapper))
{
completionWrapper.Fail(
new Exception(
string.Format("Timed out event {0} which "
+ "never got response from server was discovered. "
+ "Last state update: {1}, last reconnect: {2}, now: {3}.",
workerItem.TcpPackage.CorrelationId,
lastUpdated,
_lastReconnectionTimestamp,
now)));
}
}
else
{
Retry(correlationId);
}
}
}
_timeoutCheckStopwatch.Restart();
}
}
}
private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error)
{
OnDisconnected();
lock (_connectionLock)
{
_reconnectionStopwatch.Restart();
}
}
private void ConnectionClosed(Connection tcpTypedConnection, IPEndPoint endPoint, SocketError socketError)
{
lock (_connectionLock)
_reconnectionStopwatch.Restart();
}