private void OnPackageReceived(Connection typedTcpConnection, TcpPackage package)
{
WorkItem workItem;
ITaskCompletionWrapper wrapper;
if (RemoveInProgressItem(package.CorrelationId, "PR", out workItem, out wrapper))
{
}
else
{
if (!_outstandingOperations.TryGetValue(package.CorrelationId, out wrapper))
{
// "SKIPPED [[ "
return;
}
}
var result = wrapper.Process(package);
switch (result.Status)
{
case ProcessResultStatus.Success:
wrapper.Complete();
break;
case ProcessResultStatus.Retry:
if (wrapper.UpdateForNextAttempt())
{
RegisterHandler(wrapper.SentPackage.CorrelationId, wrapper);
EnqueueForSend(wrapper.SentPackage);
}
else
{
wrapper.Fail(new Exception(string.Format("Retry is not supported in wrapper {0}.", wrapper.GetType())));
}
break;
case ProcessResultStatus.NotifyError:
wrapper.Fail(result.Exception);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public EventStoreConnection(IPEndPoint tcpEndPoint)
{
Ensure.NotNull(tcpEndPoint, "tcpEndPoint");
_tcpEndPoint = tcpEndPoint;
_connector = new TcpConnector(_tcpEndPoint);
_outstandingOperations = new ConcurrentDictionary <Guid, ITaskCompletionWrapper>();
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived, ConnectionEstablished, ConnectionClosed);
_timeoutCheckStopwatch.Start();
_monitoringThread = new Thread(MonitoringLoop)
{
IsBackground = true,
Name = string.Format("Monitoring thread")
};
_monitoringThread.Start();
}
public Connection CreateTcpConnection(Action <Connection, TcpPackage> handlePackage,
Action <Connection> connectionEstablished,
Action <Connection, IPEndPoint, SocketError> connectionClosed)
{
var connectionCreatedEvent = new AutoResetEvent(false);
Connection typedConnection = null;
var connection = _connector.ConnectTo(
_tcpEndpoint,
tcpConnection =>
{
Log.Info("Connected to [{0}].", tcpConnection.EffectiveEndPoint);
connectionCreatedEvent.WaitOne(500);
connectionEstablished(typedConnection);
},
(conn, error) =>
{
var message = string.Format("Connection to [{0}] failed. Error: {1}.", conn.EffectiveEndPoint, error);
Log.Error(message);
connectionClosed(null, conn.EffectiveEndPoint, error);
});
typedConnection = new Connection(connection);
typedConnection.ConnectionClosed +=
(conn, error) =>
{
Log.Info("Connection [{0}] was closed {1}",
conn.EffectiveEndPoint,
error == SocketError.Success ? "cleanly." : "with error: " + error + ".");
connectionClosed(conn, conn.EffectiveEndPoint, error);
};
connectionCreatedEvent.Set();
typedConnection.ReceiveAsync((conn, pkg) =>
{
var package = new TcpPackage();
var valid = false;
try
{
package = TcpPackage.FromArraySegment(new ArraySegment <byte>(pkg));
valid = true;
if (package.Command == TcpCommand.HeartbeatRequestCommand)
{
var response = new TcpPackage(TcpCommand.HeartbeatResponseCommand, Guid.NewGuid(), null);
conn.EnqueueSend(response.AsByteArray());
return;
}
handlePackage(conn, package);
}
catch (Exception e)
{
var effectiveEndPoint = conn.EffectiveEndPoint;
var message = string.Format("[{0}] ERROR for {1}. Connection will be closed.",
effectiveEndPoint,
valid ? package.Command as object : "<invalid package>");
Log.Info(e, message);
conn.Close();
}
});
return(typedConnection);
}
public Connection CreateTcpConnection(Action<Connection, TcpPackage> handlePackage,
Action<Connection> connectionEstablished,
Action<Connection, IPEndPoint, SocketError> connectionClosed)
{
var connectionCreatedEvent = new AutoResetEvent(false);
Connection typedConnection = null;
var connection = _connector.ConnectTo(
_tcpEndpoint,
tcpConnection =>
{
Log.Info("Connected to [{0}].", tcpConnection.EffectiveEndPoint);
connectionCreatedEvent.WaitOne(500);
connectionEstablished(typedConnection);
},
(conn, error) =>
{
var message = string.Format("Connection to [{0}] failed. Error: {1}.", conn.EffectiveEndPoint, error);
Log.Error(message);
connectionClosed(null, conn.EffectiveEndPoint, error);
});
typedConnection = new Connection(connection);
typedConnection.ConnectionClosed +=
(conn, error) =>
{
Log.Info("Connection [{0}] was closed {1}",
conn.EffectiveEndPoint,
error == SocketError.Success ? "cleanly." : "with error: " + error + ".");
connectionClosed(conn, conn.EffectiveEndPoint, error);
};
connectionCreatedEvent.Set();
typedConnection.ReceiveAsync((conn, pkg) =>
{
var package = new TcpPackage();
var valid = false;
try
{
package = TcpPackage.FromArraySegment(new ArraySegment<byte>(pkg));
valid = true;
if (package.Command == TcpCommand.HeartbeatRequestCommand)
{
var response = new TcpPackage(TcpCommand.HeartbeatResponseCommand, Guid.NewGuid(), null);
conn.EnqueueSend(response.AsByteArray());
return;
}
handlePackage(conn, package);
}
catch (Exception e)
{
var effectiveEndPoint = conn.EffectiveEndPoint;
var message = string.Format("[{0}] ERROR for {1}. Connection will be closed.",
effectiveEndPoint,
valid ? package.Command as object : "<invalid package>");
Log.Info(e, message);
conn.Close();
}
});
return typedConnection;
}
private void MonitoringLoop()
{
while (!_monitoringThreadStop)
{
TcpPackage nextPackage;
if (_inProgressCount < TcpSentReceiveWindow && _sendQueue.TryDequeue(out nextPackage))
{
Interlocked.Increment(ref _inProgressCount);
var workItem = new WorkItem(nextPackage);
Send(workItem, null);
}
else
{
Thread.Sleep(1);
}
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
{
_reconnectionsCount += 1;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived,
ConnectionEstablished,
ConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
foreach (var kvp in _inProgress)
{
var correlationId = kvp.Key;
var workerItem = kvp.Value;
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > EventTimeoutDelay || _reconnectionsCount > 10)
{
if (lastUpdated > _lastReconnectionTimestamp || _reconnectionsCount > 10)
{
WorkItem workItem;
ITaskCompletionWrapper completionWrapper;
if (RemoveInProgressItem(correlationId, " -ML timeout", out workItem, out completionWrapper))
{
completionWrapper.Fail(
new Exception(
string.Format("Timed out event {0} which "
+ "never got response from server was discovered. "
+ "Last state update: {1}, last reconnect: {2}, now: {3}.",
workerItem.TcpPackage.CorrelationId,
lastUpdated,
_lastReconnectionTimestamp,
now)));
}
}
else
{
Retry(correlationId);
}
}
}
_timeoutCheckStopwatch.Restart();
}
}
}
private void ConnectionEstablished(Connection tcpTypedConnection)
{
}
private void ConnectionClosed(Connection tcpTypedConnection, IPEndPoint endPoint, SocketError socketError)
{
lock (_connectionLock)
_reconnectionStopwatch.Restart();
}
public EventStoreConnection(IPEndPoint tcpEndPoint)
{
Ensure.NotNull(tcpEndPoint, "tcpEndPoint");
_tcpEndPoint = tcpEndPoint;
_connector = new TcpConnector(_tcpEndPoint);
_outstandingOperations = new ConcurrentDictionary<Guid, ITaskCompletionWrapper>();
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived, ConnectionEstablished, ConnectionClosed);
_timeoutCheckStopwatch.Start();
_monitoringThread = new Thread(MonitoringLoop)
{
IsBackground = true,
Name = string.Format("Monitoring thread")
};
_monitoringThread.Start();
}
private void OnPackageReceived(Connection typedTcpConnection, TcpPackage package)
{
WorkItem workItem;
ITaskCompletionWrapper wrapper;
if (RemoveInProgressItem(package.CorrelationId, "PR", out workItem, out wrapper))
{
}
else
{
if (!_outstandingOperations.TryGetValue(package.CorrelationId, out wrapper))
{
// "SKIPPED [[ "
return;
}
}
var result = wrapper.Process(package);
switch (result.Status)
{
case ProcessResultStatus.Success:
wrapper.Complete();
break;
case ProcessResultStatus.Retry:
if (wrapper.UpdateForNextAttempt())
{
RegisterHandler(wrapper.SentPackage.CorrelationId, wrapper);
EnqueueForSend(wrapper.SentPackage);
}
else
wrapper.Fail(new Exception(string.Format("Retry is not supported in wrapper {0}.", wrapper.GetType())));
break;
case ProcessResultStatus.NotifyError:
wrapper.Fail(result.Exception);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void MonitoringLoop()
{
while (!_monitoringThreadStop)
{
TcpPackage nextPackage;
if (_inProgressCount < TcpSentReceiveWindow && _sendQueue.TryDequeue(out nextPackage))
{
Interlocked.Increment(ref _inProgressCount);
var workItem = new WorkItem(nextPackage);
Send(workItem, null);
}
else
Thread.Sleep(1);
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
{
_reconnectionsCount += 1;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived,
ConnectionEstablished,
ConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
foreach (var kvp in _inProgress)
{
var correlationId = kvp.Key;
var workerItem = kvp.Value;
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > EventTimeoutDelay || _reconnectionsCount > 10)
{
if (lastUpdated > _lastReconnectionTimestamp || _reconnectionsCount > 10)
{
WorkItem workItem;
ITaskCompletionWrapper completionWrapper;
if (RemoveInProgressItem(correlationId, " -ML timeout", out workItem, out completionWrapper))
{
completionWrapper.Fail(
new Exception(
string.Format("Timed out event {0} which "
+ "never got response from server was discovered. "
+ "Last state update: {1}, last reconnect: {2}, now: {3}.",
workerItem.TcpPackage.CorrelationId,
lastUpdated,
_lastReconnectionTimestamp,
now)));
}
}
else
Retry(correlationId);
}
}
_timeoutCheckStopwatch.Restart();
}
}
}
private void ConnectionEstablished(Connection tcpTypedConnection)
{
}
private void ConnectionClosed(Connection tcpTypedConnection, IPEndPoint endPoint, SocketError socketError)
{
lock (_connectionLock)
_reconnectionStopwatch.Restart();
}
