public EventStoreConnection(IPEndPoint tcpEndPoint,
int maxConcurrentRequests = 50,
int maxAttemptsForOperation = 10,
int maxReconnections = 10,
ILogger logger = null)
{
Ensure.NotNull(tcpEndPoint, "tcpEndPoint");
Ensure.Positive(maxConcurrentRequests, "maxConcurrentRequests");
Ensure.Nonnegative(maxAttemptsForOperation, "maxAttemptsForOperation");
Ensure.Nonnegative(maxReconnections, "maxReconnections");
_tcpEndPoint = tcpEndPoint;
_maxConcurrentItems = maxConcurrentRequests;
_maxAttempts = maxAttemptsForOperation;
_maxReconnections = maxReconnections;
LogManager.RegisterLogger(logger);
_log = LogManager.GetLogger();
_connector = new TcpConnector(_tcpEndPoint);
_subscriptionsChannel = new SubscriptionsChannel(_connector);
//TODO TD: WAT?
_projectionsManager = new ProjectionsManager(new IPEndPoint(_tcpEndPoint.Address, _tcpEndPoint.Port + 1000));
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
_timeoutCheckStopwatch.Start();
_worker = new Thread(MainLoop)
{
IsBackground = true,
Name = "Worker thread"
};
_worker.Start();
}
private Task EstablishConnectionAsync(IPEndPoint tcpEndPoint, IPEndPoint httpEndPoint)
{
lock (_connectionLock)
{
if (_isActive)
{
throw new InvalidOperationException("EventStoreConnection is already active");
}
_isActive = true;
_tcpEndPoint = tcpEndPoint;
_httpEndPoint = httpEndPoint;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(_tcpEndPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
_timeoutCheckStopwatch.Start();
_worker = new Thread(MainLoop)
{
IsBackground = true, Name = "Worker thread"
};
_worker.Start();
return(Tasks.CreateCompleted());
}
}
private void OnConnectionClosed(TcpTypedConnection <byte[]> connection, SocketError errorCode)
{
lock (_connectionLock)
{
_reconnectionStopwatch.Restart();
}
}
public void Start()
{
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _coordinator.CreateConnection(OnPackageArrived, OnConnectionEstablished, OnConnectionClosed);
_timeoutCheckStopwatch.Start();
_stoppedEvent.Reset();
_thread.Start();
}
private void MainLoop()
{
while (!_stop)
{
VerificationEvent evnt;
if (_inProgressCount < _maxConcurrentItems && TryGetNext(out evnt))
{
Interlocked.Increment(ref _inProgressCount);
Send(evnt);
}
else
{
Thread.Sleep(1);
}
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
{
_reconnectionsCount += 1;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
foreach (var workerItem in _events.Values)
{
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > EventTimeoutDelay)
{
if (lastUpdated > _lastReconnectionTimestamp)
{
SignalWorkerFailed(
error: string.Format("Worker {0} discovered timed out event which "
+ "never got response from server. "
+ "Last state update: {1}, last reconnect: {2}, now: {3}.",
Name,
lastUpdated,
_lastReconnectionTimestamp,
now));
TryRemoveWorkItem(workerItem);
}
else
{
Retry(workerItem);
}
}
}
_timeoutCheckStopwatch.Restart();
}
}
_stoppedEvent.Set();
}
private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package)
{
var corrId = package.CorrelationId;
WorkerItem workItem;
if (!_items.TryGetValue(corrId, out workItem))
{
_coordinator.SignalWorkerFailed(null,
string.Format(
"Worker {0} received unexpected CorrId: {1}, no item with such CorrId is in progress.",
_name, corrId));
return;
}
var result = workItem.Task.CheckStepExpectations(package);
switch (result.Status)
{
case Status.MeetsExpectations:
if (TryRemoveWorkItem(workItem))
{
NotifyItemProcessed(workItem);
}
break;
case Status.Retry:
Retry(workItem);
break;
case Status.CheckStreamDeleted:
if (_coordinator.IsDeleted(result.Description) && TryRemoveWorkItem(workItem))
{
NotifyItemProcessed(workItem);
}
else
{
Retry(workItem);
_coordinator.SignalPossibleFailure(
string.Format(
"Got stream deleted ({0}), but it's not confirmed. Retrying ({1})", result.Description,
workItem.Attempt));
}
break;
case Status.Ignore:
break;
case Status.FailFast:
_coordinator.SignalWorkerFailed(null, result.Description);
TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error)
{
_connectedEvent.Reset();
var subscriptions = _subscriptions.Values;
_subscriptions.Clear();
foreach (var subscription in subscriptions)
{
subscription.Source.SetResult(null);
ExecuteUserCallbackAsync(subscription.SubscriptionDropped);
}
}
public void Connect()
{
_connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
if (_executionThread == null)
{
_executionThread = new Thread(ExecuteUserCallbacks)
{
IsBackground = true,
Name = "SubscriptionsChannel user callbacks thread"
};
_executionThread.Start();
}
}
private void Connect(IPEndPoint endPoint)
{
_connection = _connector.CreateTcpConnection(endPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
if (_executionThread == null)
{
_stopExecutionThread = false;
_executionThread = new Thread(ExecuteUserCallbacks)
{
IsBackground = true,
Name = "SubscriptionsChannel user callbacks thread"
};
_executionThread.Start();
}
}
private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package)
{
var corrId = package.CorrelationId;
WorkItem workItem;
if (!_inProgress.TryGetValue(corrId, out workItem))
{
_log.Error("Unexpected corrid received {0}", corrId);
return;
}
var result = workItem.Operation.InspectPackage(package);
switch (result.Decision)
{
case InspectionDecision.Succeed:
if (TryRemoveWorkItem(workItem))
{
workItem.Operation.Complete();
}
break;
case InspectionDecision.Retry:
Retry(workItem);
break;
case InspectionDecision.Reconnect:
Reconnect(workItem, (EndpointsPair)result.Data);
break;
case InspectionDecision.NotifyError:
if (TryRemoveWorkItem(workItem))
{
workItem.Operation.Fail(result.Error);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package)
{
Subscription subscription;
if (!_subscriptions.TryGetValue(package.CorrelationId, out subscription))
{
_log.Error("Unexpected package received : {0} ({1})", package.CorrelationId, package.Command);
return;
}
try
{
switch (package.Command)
{
case TcpCommand.StreamEventAppeared:
var dto = package.Data.Deserialize <ClientMessage.StreamEventAppeared>();
var recordedEvent = new RecordedEvent(dto);
var commitPos = dto.CommitPosition;
var preparePos = dto.PreparePosition;
ExecuteUserCallbackAsync(() => subscription.EventAppeared(recordedEvent, new Position(commitPos, preparePos)));
break;
case TcpCommand.SubscriptionDropped:
case TcpCommand.SubscriptionToAllDropped:
Subscription removed;
if (_subscriptions.TryRemove(subscription.Id, out removed))
{
removed.Source.SetResult(null);
ExecuteUserCallbackAsync(removed.SubscriptionDropped);
}
break;
default:
throw new ArgumentOutOfRangeException(string.Format("Unexpected command : {0}", package.Command));
}
}
catch (Exception e)
{
_log.Error(e, "Error on package received");
}
}
private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package)
{
var corrId = package.CorrelationId;
WorkerItem workItem;
if (!_items.TryGetValue(corrId, out workItem))
{
_coordinator.SignalWorkerFailed(null,
string.Format(
"Worker {0} received unexpected CorrId: {1}, no item with such CorrId is in progress.",
_name, corrId));
return;
}
var result = workItem.Task.CheckStepExpectations(package);
switch (result.Status)
{
case Status.MeetsExpectations:
if (TryRemoveWorkItem(workItem))
NotifyItemProcessed(workItem);
break;
case Status.Retry:
Retry(workItem);
break;
case Status.CheckStreamDeleted:
if (_coordinator.IsDeleted(result.Description) && TryRemoveWorkItem(workItem))
NotifyItemProcessed(workItem);
else
{
Retry(workItem);
_coordinator.SignalPossibleFailure(
string.Format(
"Got stream deleted ({0}), but it's not confirmed. Retrying ({1})", result.Description,
workItem.Attempt));
}
break;
case Status.Ignore:
break;
case Status.FailFast:
_coordinator.SignalWorkerFailed(null, result.Description);
TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void MainLoop()
{
while (!_stopping)
{
IClientOperation operation;
if (_inProgressCount < _maxConcurrentItems && _queue.TryDequeue(out operation))
{
Interlocked.Increment(ref _inProgressCount);
Send(new WorkItem(operation));
}
else
{
Thread.Sleep(1);
}
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
{
_reconnectionsCount += 1;
if (_reconnectionsCount > _maxReconnections)
{
throw new CannotEstablishConnectionException();
}
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > OperationTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
var retriable = new List <WorkItem>();
foreach (var workerItem in _inProgress.Values)
{
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > OperationTimeout)
{
Console.WriteLine("Timed out, corrId: {0}.", workerItem.Operation.CorrelationId);
//Debugger.Break();
if (lastUpdated >= _lastReconnectionTimestamp)
{
var err = string.Format("{0} never got response from server" +
"Last state update : {1}, last reconnect : {2}, now(utc) : {3}.",
workerItem,
lastUpdated,
_lastReconnectionTimestamp,
now);
if (TryRemoveWorkItem(workerItem))
{
_log.Error(err);
workerItem.Operation.Fail(new OperationTimedOutException(err));
}
_log.Error(err);
}
else
{
retriable.Add(workerItem);
}
}
}
foreach (var workItem in retriable.OrderBy(wi => wi.SeqNo))
{
Retry(workItem);
}
_timeoutCheckStopwatch.Restart();
}
}
}
private void OnConnectionEstablished(TcpTypedConnection <byte[]> connection)
{
}
private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package)
{
if (package.Command != TcpCommand.ReadEventCompleted)
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
_worker.Name,
TcpCommand.ReadEventCompleted,
package.Command));
return;
}
var dto = package.Data.Deserialize <ClientMessageDto.ReadEventCompleted>();
var corrId = new Guid(dto.CorrelationId);
WorkerItem workItem;
if (!_worker.TryGetWorkItem(corrId, out workItem))
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
_worker.Name,
corrId));
return;
}
var readResult = (SingleReadResult)dto.Result;
switch (readResult)
{
case SingleReadResult.Success:
var cmp = workItem.Event.VerifyThat(new Event(workItem.Event.Event.EventId, dto.EventType, false, dto.Data, dto.Metadata),
dto.EventNumber);
switch (cmp.Status)
{
case ComparisonStatus.Success:
if (_worker.TryRemoveWorkItem(workItem))
{
_worker.NotifyItemProcessed(workItem);
}
break;
case ComparisonStatus.Fail:
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received invalid event. Details : {1}.",
_worker,
cmp.Description));
_worker.TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
break;
case SingleReadResult.NotFound:
case SingleReadResult.NoStream:
case SingleReadResult.StreamDeleted:
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected response : {1}.",
_worker.Name,
readResult));
_worker.TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package)
{
if (package.Command != TcpCommand.ReadEventCompleted)
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
_worker.Name,
TcpCommand.ReadEventCompleted,
package.Command));
return;
}
var dto = package.Data.Deserialize<ClientMessageDto.ReadEventCompleted>();
var corrId = new Guid(dto.CorrelationId);
WorkerItem workItem;
if (!_worker.TryGetWorkItem(corrId, out workItem))
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
_worker.Name,
corrId));
return;
}
var readResult = (SingleReadResult) dto.Result;
switch (readResult)
{
case SingleReadResult.Success:
var cmp = workItem.Event.VerifyThat(new Event(workItem.Event.Event.EventId, dto.EventType, false, dto.Data, dto.Metadata),
dto.EventNumber);
switch (cmp.Status)
{
case ComparisonStatus.Success:
if (_worker.TryRemoveWorkItem(workItem))
_worker.NotifyItemProcessed(workItem);
break;
case ComparisonStatus.Fail:
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received invalid event. Details : {1}.",
_worker,
cmp.Description));
_worker.TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
break;
case SingleReadResult.NotFound:
case SingleReadResult.NoStream:
case SingleReadResult.StreamDeleted:
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected response : {1}.",
_worker.Name,
readResult));
_worker.TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void OnConnectionEstablished(TcpTypedConnection<byte[]> connection) { }
private void OnConnectionClosed(TcpTypedConnection<byte[]> connection, SocketError errorCode)
{
lock (_connectionLock)
{
_reconnectionStopwatch.Restart();
}
}
private void MainLoop()
{
while (!_stop)
{
VerificationEvent evnt;
if (_inProgressCount < _maxConcurrentItems && TryGetNext(out evnt))
{
Interlocked.Increment(ref _inProgressCount);
Send(evnt);
}
else
Thread.Sleep(1);
lock (_connectionLock)
{
if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
{
_reconnectionsCount += 1;
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed);
_reconnectionStopwatch.Stop();
}
}
if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
{
var now = DateTime.UtcNow;
foreach (var workerItem in _events.Values)
{
var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
if (now - lastUpdated > EventTimeoutDelay)
{
if (lastUpdated > _lastReconnectionTimestamp)
{
SignalWorkerFailed(
error: string.Format("Worker {0} discovered timed out event which "
+ "never got response from server. "
+ "Last state update: {1}, last reconnect: {2}, now: {3}.",
Name,
lastUpdated,
_lastReconnectionTimestamp,
now));
TryRemoveWorkItem(workerItem);
}
else
Retry(workerItem);
}
}
_timeoutCheckStopwatch.Restart();
}
}
_stoppedEvent.Set();
}
public void Start()
{
_lastReconnectionTimestamp = DateTime.UtcNow;
_connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed);
_timeoutCheckStopwatch.Start();
_stoppedEvent.Reset();
_thread.Start();
}
private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection)
{
lock (_connectionLock)
_reconnectionsCount = 0;
}
private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error)
{
lock (_connectionLock)
_reconnectionStopwatch.Restart();
}
private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection)
{
_connectedEvent.Set();
}
private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package)
{
if (package.Command != TcpCommand.WriteEventsCompleted)
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
_worker.Name,
TcpCommand.WriteEventsCompleted,
package.Command));
return;
}
var dto = package.Data.Deserialize<ClientMessageDto.WriteEventsCompleted>();
var corrId = new Guid(dto.CorrelationId);
WorkerItem workItem;
if (!_worker.TryGetWorkItem(corrId, out workItem))
{
_worker.SignalWorkerFailed(
error:
string.Format(
"Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
_worker.Name,
corrId));
return;
}
switch ((OperationErrorCode) dto.ErrorCode)
{
case OperationErrorCode.Success:
{
if (_worker.TryRemoveWorkItem(workItem))
_worker.NotifyItemProcessed(workItem);
break;
}
case OperationErrorCode.PrepareTimeout:
case OperationErrorCode.CommitTimeout:
case OperationErrorCode.ForwardTimeout:
_worker.Retry(workItem);
break;
case OperationErrorCode.WrongExpectedVersion:
if (workItem.Event.ExpectedVersion == ExpectedVersion.Any)
{
if (workItem.Event.ShouldBeVersion != 1)
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received WrongExpectedVersion for event with "
+ "ExpectedVersion = EventNumber.Any and ShouldBeVersion = 1, "
+ "which shouldn't happen ever!",
_worker.Name));
_worker.TryRemoveWorkItem(workItem);
return;
}
}
_worker.Retry(workItem);
break;
case OperationErrorCode.StreamDeleted:
case OperationErrorCode.InvalidTransaction:
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected OperationErrorCode: {1}.",
_worker.Name,
(OperationErrorCode) dto.ErrorCode));
_worker.TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package)
{
if (package.Command != TcpCommand.WriteEventsCompleted)
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
_worker.Name,
TcpCommand.WriteEventsCompleted,
package.Command));
return;
}
var dto = package.Data.Deserialize <ClientMessageDto.WriteEventsCompleted>();
var corrId = new Guid(dto.CorrelationId);
WorkerItem workItem;
if (!_worker.TryGetWorkItem(corrId, out workItem))
{
_worker.SignalWorkerFailed(
error:
string.Format(
"Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
_worker.Name,
corrId));
return;
}
switch ((OperationErrorCode)dto.ErrorCode)
{
case OperationErrorCode.Success:
{
if (_worker.TryRemoveWorkItem(workItem))
{
_worker.NotifyItemProcessed(workItem);
}
break;
}
case OperationErrorCode.PrepareTimeout:
case OperationErrorCode.CommitTimeout:
case OperationErrorCode.ForwardTimeout:
_worker.Retry(workItem);
break;
case OperationErrorCode.WrongExpectedVersion:
if (workItem.Event.ExpectedVersion == ExpectedVersion.Any)
{
if (workItem.Event.ShouldBeVersion != 1)
{
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received WrongExpectedVersion for event with "
+ "ExpectedVersion = EventNumber.Any and ShouldBeVersion = 1, "
+ "which shouldn't happen ever!",
_worker.Name));
_worker.TryRemoveWorkItem(workItem);
return;
}
}
_worker.Retry(workItem);
break;
case OperationErrorCode.StreamDeleted:
case OperationErrorCode.InvalidTransaction:
_worker.SignalWorkerFailed(
error: string.Format("Worker {0} received unexpected OperationErrorCode: {1}.",
_worker.Name,
(OperationErrorCode)dto.ErrorCode));
_worker.TryRemoveWorkItem(workItem);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
