Example #1
0
        public EventStoreConnection(IPEndPoint tcpEndPoint,
                                    int maxConcurrentRequests   = 50,
                                    int maxAttemptsForOperation = 10,
                                    int maxReconnections        = 10,
                                    ILogger logger = null)
        {
            Ensure.NotNull(tcpEndPoint, "tcpEndPoint");
            Ensure.Positive(maxConcurrentRequests, "maxConcurrentRequests");
            Ensure.Nonnegative(maxAttemptsForOperation, "maxAttemptsForOperation");
            Ensure.Nonnegative(maxReconnections, "maxReconnections");

            _tcpEndPoint        = tcpEndPoint;
            _maxConcurrentItems = maxConcurrentRequests;
            _maxAttempts        = maxAttemptsForOperation;
            _maxReconnections   = maxReconnections;

            LogManager.RegisterLogger(logger);
            _log = LogManager.GetLogger();

            _connector            = new TcpConnector(_tcpEndPoint);
            _subscriptionsChannel = new SubscriptionsChannel(_connector);
            //TODO TD: WAT?
            _projectionsManager = new ProjectionsManager(new IPEndPoint(_tcpEndPoint.Address, _tcpEndPoint.Port + 1000));

            _lastReconnectionTimestamp = DateTime.UtcNow;
            _connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
            _timeoutCheckStopwatch.Start();

            _worker = new Thread(MainLoop)
            {
                IsBackground = true,
                Name         = "Worker thread"
            };
            _worker.Start();
        }
        private Task EstablishConnectionAsync(IPEndPoint tcpEndPoint, IPEndPoint httpEndPoint)
        {
            lock (_connectionLock)
            {
                if (_isActive)
                {
                    throw new InvalidOperationException("EventStoreConnection is already active");
                }
                _isActive = true;

                _tcpEndPoint  = tcpEndPoint;
                _httpEndPoint = httpEndPoint;

                _lastReconnectionTimestamp = DateTime.UtcNow;
                _connection = _connector.CreateTcpConnection(_tcpEndPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
                _timeoutCheckStopwatch.Start();

                _worker = new Thread(MainLoop)
                {
                    IsBackground = true, Name = "Worker thread"
                };
                _worker.Start();

                return(Tasks.CreateCompleted());
            }
        }
Example #3
0
 private void OnConnectionClosed(TcpTypedConnection <byte[]> connection, SocketError errorCode)
 {
     lock (_connectionLock)
     {
         _reconnectionStopwatch.Restart();
     }
 }
Example #4
0
 public void Start()
 {
     _lastReconnectionTimestamp = DateTime.UtcNow;
     _connection = _coordinator.CreateConnection(OnPackageArrived, OnConnectionEstablished, OnConnectionClosed);
     _timeoutCheckStopwatch.Start();
     _stoppedEvent.Reset();
     _thread.Start();
 }
Example #5
0
        private void MainLoop()
        {
            while (!_stop)
            {
                VerificationEvent evnt;
                if (_inProgressCount < _maxConcurrentItems && TryGetNext(out evnt))
                {
                    Interlocked.Increment(ref _inProgressCount);
                    Send(evnt);
                }
                else
                {
                    Thread.Sleep(1);
                }

                lock (_connectionLock)
                {
                    if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
                    {
                        _reconnectionsCount       += 1;
                        _lastReconnectionTimestamp = DateTime.UtcNow;
                        _connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed);
                        _reconnectionStopwatch.Stop();
                    }
                }

                if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
                {
                    var now = DateTime.UtcNow;
                    foreach (var workerItem in _events.Values)
                    {
                        var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
                        if (now - lastUpdated > EventTimeoutDelay)
                        {
                            if (lastUpdated > _lastReconnectionTimestamp)
                            {
                                SignalWorkerFailed(
                                    error: string.Format("Worker {0} discovered timed out event which "
                                                         + "never got response from server. "
                                                         + "Last state update: {1}, last reconnect: {2}, now: {3}.",
                                                         Name,
                                                         lastUpdated,
                                                         _lastReconnectionTimestamp,
                                                         now));
                                TryRemoveWorkItem(workerItem);
                            }
                            else
                            {
                                Retry(workerItem);
                            }
                        }
                    }
                    _timeoutCheckStopwatch.Restart();
                }
            }

            _stoppedEvent.Set();
        }
Example #6
0
        private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package)
        {
            var corrId = package.CorrelationId;

            WorkerItem workItem;

            if (!_items.TryGetValue(corrId, out workItem))
            {
                _coordinator.SignalWorkerFailed(null,
                                                string.Format(
                                                    "Worker {0} received unexpected CorrId: {1}, no item with such CorrId is in progress.",
                                                    _name, corrId));
                return;
            }

            var result = workItem.Task.CheckStepExpectations(package);

            switch (result.Status)
            {
            case Status.MeetsExpectations:
                if (TryRemoveWorkItem(workItem))
                {
                    NotifyItemProcessed(workItem);
                }
                break;

            case Status.Retry:
                Retry(workItem);
                break;

            case Status.CheckStreamDeleted:
                if (_coordinator.IsDeleted(result.Description) && TryRemoveWorkItem(workItem))
                {
                    NotifyItemProcessed(workItem);
                }
                else
                {
                    Retry(workItem);
                    _coordinator.SignalPossibleFailure(
                        string.Format(
                            "Got stream deleted ({0}), but it's not confirmed. Retrying ({1})", result.Description,
                            workItem.Attempt));
                }
                break;

            case Status.Ignore:
                break;

            case Status.FailFast:
                _coordinator.SignalWorkerFailed(null, result.Description);
                TryRemoveWorkItem(workItem);
                break;

            default:
                throw new ArgumentOutOfRangeException();
            }
        }
Example #7
0
        private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error)
        {
            _connectedEvent.Reset();

            var subscriptions = _subscriptions.Values;

            _subscriptions.Clear();

            foreach (var subscription in subscriptions)
            {
                subscription.Source.SetResult(null);
                ExecuteUserCallbackAsync(subscription.SubscriptionDropped);
            }
        }
Example #8
0
        public void Connect()
        {
            _connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);

            if (_executionThread == null)
            {
                _executionThread = new Thread(ExecuteUserCallbacks)
                {
                    IsBackground = true,
                    Name         = "SubscriptionsChannel user callbacks thread"
                };
                _executionThread.Start();
            }
        }
Example #9
0
        private void Connect(IPEndPoint endPoint)
        {
            _connection = _connector.CreateTcpConnection(endPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);

            if (_executionThread == null)
            {
                _stopExecutionThread = false;
                _executionThread     = new Thread(ExecuteUserCallbacks)
                {
                    IsBackground = true,
                    Name         = "SubscriptionsChannel user callbacks thread"
                };
                _executionThread.Start();
            }
        }
        private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package)
        {
            var      corrId = package.CorrelationId;
            WorkItem workItem;

            if (!_inProgress.TryGetValue(corrId, out workItem))
            {
                _log.Error("Unexpected corrid received {0}", corrId);
                return;
            }

            var result = workItem.Operation.InspectPackage(package);

            switch (result.Decision)
            {
            case InspectionDecision.Succeed:
                if (TryRemoveWorkItem(workItem))
                {
                    workItem.Operation.Complete();
                }
                break;

            case InspectionDecision.Retry:
                Retry(workItem);
                break;

            case InspectionDecision.Reconnect:
                Reconnect(workItem, (EndpointsPair)result.Data);
                break;

            case InspectionDecision.NotifyError:
                if (TryRemoveWorkItem(workItem))
                {
                    workItem.Operation.Fail(result.Error);
                }
                break;

            default:
                throw new ArgumentOutOfRangeException();
            }
        }
Example #11
0
        private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package)
        {
            Subscription subscription;

            if (!_subscriptions.TryGetValue(package.CorrelationId, out subscription))
            {
                _log.Error("Unexpected package received : {0} ({1})", package.CorrelationId, package.Command);
                return;
            }

            try
            {
                switch (package.Command)
                {
                case TcpCommand.StreamEventAppeared:
                    var dto           = package.Data.Deserialize <ClientMessage.StreamEventAppeared>();
                    var recordedEvent = new RecordedEvent(dto);
                    var commitPos     = dto.CommitPosition;
                    var preparePos    = dto.PreparePosition;
                    ExecuteUserCallbackAsync(() => subscription.EventAppeared(recordedEvent, new Position(commitPos, preparePos)));
                    break;

                case TcpCommand.SubscriptionDropped:
                case TcpCommand.SubscriptionToAllDropped:
                    Subscription removed;
                    if (_subscriptions.TryRemove(subscription.Id, out removed))
                    {
                        removed.Source.SetResult(null);
                        ExecuteUserCallbackAsync(removed.SubscriptionDropped);
                    }
                    break;

                default:
                    throw new ArgumentOutOfRangeException(string.Format("Unexpected command : {0}", package.Command));
                }
            }
            catch (Exception e)
            {
                _log.Error(e, "Error on package received");
            }
        }
Example #12
0
        private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package)
        {
            var corrId = package.CorrelationId;

            WorkerItem workItem;
            if (!_items.TryGetValue(corrId, out workItem))
            {
                _coordinator.SignalWorkerFailed(null, 
                                                string.Format(
                                                "Worker {0} received unexpected CorrId: {1}, no item with such CorrId is in progress.",
                                                _name, corrId));
                return;
            }

            var result = workItem.Task.CheckStepExpectations(package);

            switch (result.Status)
            {
                case Status.MeetsExpectations:
                    if (TryRemoveWorkItem(workItem))
                        NotifyItemProcessed(workItem);
                    break;
                case Status.Retry:
                    Retry(workItem);
                    break;
                case Status.CheckStreamDeleted:
                    if (_coordinator.IsDeleted(result.Description) && TryRemoveWorkItem(workItem))
                        NotifyItemProcessed(workItem);
                    else
                    {
                        Retry(workItem);
                        _coordinator.SignalPossibleFailure(
                            string.Format(
                                "Got stream deleted ({0}), but it's not confirmed. Retrying ({1})", result.Description,
                                workItem.Attempt));
                    }
                    break;
                case Status.Ignore:
                    break;
                case Status.FailFast:
                    _coordinator.SignalWorkerFailed(null, result.Description);
                    TryRemoveWorkItem(workItem);
                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }
        }
Example #13
0
        private void MainLoop()
        {
            while (!_stopping)
            {
                IClientOperation operation;
                if (_inProgressCount < _maxConcurrentItems && _queue.TryDequeue(out operation))
                {
                    Interlocked.Increment(ref _inProgressCount);
                    Send(new WorkItem(operation));
                }
                else
                {
                    Thread.Sleep(1);
                }

                lock (_connectionLock)
                {
                    if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
                    {
                        _reconnectionsCount += 1;
                        if (_reconnectionsCount > _maxReconnections)
                        {
                            throw new CannotEstablishConnectionException();
                        }

                        _lastReconnectionTimestamp = DateTime.UtcNow;
                        _connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed);
                        _reconnectionStopwatch.Stop();
                    }
                }

                if (_timeoutCheckStopwatch.Elapsed > OperationTimeoutCheckPeriod)
                {
                    var now       = DateTime.UtcNow;
                    var retriable = new List <WorkItem>();
                    foreach (var workerItem in _inProgress.Values)
                    {
                        var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
                        if (now - lastUpdated > OperationTimeout)
                        {
                            Console.WriteLine("Timed out, corrId: {0}.", workerItem.Operation.CorrelationId);
                            //Debugger.Break();
                            if (lastUpdated >= _lastReconnectionTimestamp)
                            {
                                var err = string.Format("{0} never got response from server" +
                                                        "Last state update : {1}, last reconnect : {2}, now(utc) : {3}.",
                                                        workerItem,
                                                        lastUpdated,
                                                        _lastReconnectionTimestamp,
                                                        now);
                                if (TryRemoveWorkItem(workerItem))
                                {
                                    _log.Error(err);
                                    workerItem.Operation.Fail(new OperationTimedOutException(err));
                                }
                                _log.Error(err);
                            }
                            else
                            {
                                retriable.Add(workerItem);
                            }
                        }
                    }

                    foreach (var workItem in retriable.OrderBy(wi => wi.SeqNo))
                    {
                        Retry(workItem);
                    }

                    _timeoutCheckStopwatch.Restart();
                }
            }
        }
Example #14
0
 private void OnConnectionEstablished(TcpTypedConnection <byte[]> connection)
 {
 }
        private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package)
        {
            if (package.Command != TcpCommand.ReadEventCompleted)
            {
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
                                         _worker.Name,
                                         TcpCommand.ReadEventCompleted,
                                         package.Command));
                return;
            }

            var dto    = package.Data.Deserialize <ClientMessageDto.ReadEventCompleted>();
            var corrId = new Guid(dto.CorrelationId);

            WorkerItem workItem;

            if (!_worker.TryGetWorkItem(corrId, out workItem))
            {
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
                                         _worker.Name,
                                         corrId));
                return;
            }

            var readResult = (SingleReadResult)dto.Result;

            switch (readResult)
            {
            case SingleReadResult.Success:
                var cmp = workItem.Event.VerifyThat(new Event(workItem.Event.Event.EventId, dto.EventType, false, dto.Data, dto.Metadata),
                                                    dto.EventNumber);
                switch (cmp.Status)
                {
                case ComparisonStatus.Success:
                    if (_worker.TryRemoveWorkItem(workItem))
                    {
                        _worker.NotifyItemProcessed(workItem);
                    }
                    break;

                case ComparisonStatus.Fail:
                    _worker.SignalWorkerFailed(
                        error: string.Format("Worker {0} received invalid event. Details : {1}.",
                                             _worker,
                                             cmp.Description));
                    _worker.TryRemoveWorkItem(workItem);
                    break;

                default:
                    throw new ArgumentOutOfRangeException();
                }
                break;

            case SingleReadResult.NotFound:
            case SingleReadResult.NoStream:
            case SingleReadResult.StreamDeleted:
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected response : {1}.",
                                         _worker.Name,
                                         readResult));
                _worker.TryRemoveWorkItem(workItem);
                break;

            default:
                throw new ArgumentOutOfRangeException();
            }
        }
        private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package)
        {
            if (package.Command != TcpCommand.ReadEventCompleted)
            {
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
                                         _worker.Name,
                                         TcpCommand.ReadEventCompleted,
                                         package.Command));
                return;
            }

            var dto = package.Data.Deserialize<ClientMessageDto.ReadEventCompleted>();
            var corrId = new Guid(dto.CorrelationId);

            WorkerItem workItem;
            if (!_worker.TryGetWorkItem(corrId, out workItem))
            {
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
                                         _worker.Name,
                                         corrId));
                return;
            }

            var readResult = (SingleReadResult) dto.Result;
            switch (readResult)
            {
                case SingleReadResult.Success:
                    var cmp = workItem.Event.VerifyThat(new Event(workItem.Event.Event.EventId, dto.EventType, false,  dto.Data, dto.Metadata),
                                                        dto.EventNumber);
                    switch (cmp.Status)
                    {
                        case ComparisonStatus.Success:
                            if (_worker.TryRemoveWorkItem(workItem))
                                _worker.NotifyItemProcessed(workItem);
                            break;
                        case ComparisonStatus.Fail:
                            _worker.SignalWorkerFailed(
                                error: string.Format("Worker {0} received invalid event. Details : {1}.",
                                                     _worker,
                                                     cmp.Description));
                            _worker.TryRemoveWorkItem(workItem);
                            break;
                        default:
                            throw new ArgumentOutOfRangeException();
                    }
                    break;

                case SingleReadResult.NotFound:
                case SingleReadResult.NoStream:
                case SingleReadResult.StreamDeleted:
                    _worker.SignalWorkerFailed(
                        error: string.Format("Worker {0} received unexpected response : {1}.",
                                             _worker.Name,
                                             readResult));
                    _worker.TryRemoveWorkItem(workItem);
                    break;

                default:
                    throw new ArgumentOutOfRangeException();
            }
        }
Example #17
0
 private void OnConnectionEstablished(TcpTypedConnection<byte[]> connection) { }
Example #18
0
 private void OnConnectionClosed(TcpTypedConnection<byte[]> connection, SocketError errorCode)
 {
     lock (_connectionLock)
     {
         _reconnectionStopwatch.Restart();
     }
 }
Example #19
0
        private void MainLoop()
        {
            while (!_stop)
            {
                VerificationEvent evnt;
                if (_inProgressCount < _maxConcurrentItems && TryGetNext(out evnt))
                {
                    Interlocked.Increment(ref _inProgressCount);
                    Send(evnt);
                }
                else
                    Thread.Sleep(1);

                lock (_connectionLock)
                {
                    if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay)
                    {
                        _reconnectionsCount += 1;
                        _lastReconnectionTimestamp = DateTime.UtcNow;
                        _connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed);
                        _reconnectionStopwatch.Stop();
                    }
                }

                if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod)
                {
                    var now = DateTime.UtcNow;
                    foreach (var workerItem in _events.Values)
                    {
                        var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks));
                        if (now - lastUpdated > EventTimeoutDelay)
                        {
                            if (lastUpdated > _lastReconnectionTimestamp)
                            {
                                SignalWorkerFailed(
                                    error: string.Format("Worker {0} discovered timed out event which "
                                                         + "never got response from server. "
                                                         + "Last state update: {1}, last reconnect: {2}, now: {3}.",
                                                         Name,
                                                         lastUpdated,
                                                         _lastReconnectionTimestamp,
                                                         now));
                                TryRemoveWorkItem(workerItem);
                            }
                            else
                                Retry(workerItem);
                        }
                    }
                    _timeoutCheckStopwatch.Restart();
                }
            }

            _stoppedEvent.Set();
        }
Example #20
0
 public void Start()
 {
     _lastReconnectionTimestamp = DateTime.UtcNow;
     _connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed);
     _timeoutCheckStopwatch.Start();
     _stoppedEvent.Reset();
     _thread.Start();
 }
Example #21
0
 private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection)
 {
     lock (_connectionLock)
         _reconnectionsCount = 0;
 }
Example #22
0
 private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error)
 {
     lock (_connectionLock)
         _reconnectionStopwatch.Restart();
 }
Example #23
0
 private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection)
 {
     _connectedEvent.Set();
 }
Example #24
0
        private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package)
        {
            if (package.Command != TcpCommand.WriteEventsCompleted)
            {
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
                                         _worker.Name,
                                         TcpCommand.WriteEventsCompleted,
                                         package.Command));
                return;
            }

            var dto = package.Data.Deserialize<ClientMessageDto.WriteEventsCompleted>();
            var corrId = new Guid(dto.CorrelationId);

            WorkerItem workItem;
            if (!_worker.TryGetWorkItem(corrId, out workItem))
            {
                _worker.SignalWorkerFailed(
                    error:
                        string.Format(
                            "Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
                            _worker.Name,
                            corrId));
                return;
            }

            switch ((OperationErrorCode) dto.ErrorCode)
            {
                case OperationErrorCode.Success:
                    {
                        if (_worker.TryRemoveWorkItem(workItem))
                            _worker.NotifyItemProcessed(workItem);
                        break;
                    }
                case OperationErrorCode.PrepareTimeout:
                case OperationErrorCode.CommitTimeout:
                case OperationErrorCode.ForwardTimeout:
                    _worker.Retry(workItem);
                    break;
                case OperationErrorCode.WrongExpectedVersion:
                    if (workItem.Event.ExpectedVersion == ExpectedVersion.Any)
                    {
                        if (workItem.Event.ShouldBeVersion != 1)
                        {
                            _worker.SignalWorkerFailed(
                                error: string.Format("Worker {0} received WrongExpectedVersion for event with "
                                                     + "ExpectedVersion = EventNumber.Any and ShouldBeVersion = 1, "
                                                     + "which shouldn't happen ever!",
                                                     _worker.Name));

                            _worker.TryRemoveWorkItem(workItem);
                            return;
                        }
                    }
                    _worker.Retry(workItem);
                    break;
                case OperationErrorCode.StreamDeleted:
                case OperationErrorCode.InvalidTransaction:
                    _worker.SignalWorkerFailed(
                        error: string.Format("Worker {0} received unexpected OperationErrorCode: {1}.",
                                             _worker.Name,
                                             (OperationErrorCode) dto.ErrorCode));
                    _worker.TryRemoveWorkItem(workItem);
                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }
        }
Example #25
0
        private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package)
        {
            if (package.Command != TcpCommand.WriteEventsCompleted)
            {
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.",
                                         _worker.Name,
                                         TcpCommand.WriteEventsCompleted,
                                         package.Command));
                return;
            }

            var dto    = package.Data.Deserialize <ClientMessageDto.WriteEventsCompleted>();
            var corrId = new Guid(dto.CorrelationId);

            WorkerItem workItem;

            if (!_worker.TryGetWorkItem(corrId, out workItem))
            {
                _worker.SignalWorkerFailed(
                    error:
                    string.Format(
                        "Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.",
                        _worker.Name,
                        corrId));
                return;
            }

            switch ((OperationErrorCode)dto.ErrorCode)
            {
            case OperationErrorCode.Success:
            {
                if (_worker.TryRemoveWorkItem(workItem))
                {
                    _worker.NotifyItemProcessed(workItem);
                }
                break;
            }

            case OperationErrorCode.PrepareTimeout:
            case OperationErrorCode.CommitTimeout:
            case OperationErrorCode.ForwardTimeout:
                _worker.Retry(workItem);
                break;

            case OperationErrorCode.WrongExpectedVersion:
                if (workItem.Event.ExpectedVersion == ExpectedVersion.Any)
                {
                    if (workItem.Event.ShouldBeVersion != 1)
                    {
                        _worker.SignalWorkerFailed(
                            error: string.Format("Worker {0} received WrongExpectedVersion for event with "
                                                 + "ExpectedVersion = EventNumber.Any and ShouldBeVersion = 1, "
                                                 + "which shouldn't happen ever!",
                                                 _worker.Name));

                        _worker.TryRemoveWorkItem(workItem);
                        return;
                    }
                }
                _worker.Retry(workItem);
                break;

            case OperationErrorCode.StreamDeleted:
            case OperationErrorCode.InvalidTransaction:
                _worker.SignalWorkerFailed(
                    error: string.Format("Worker {0} received unexpected OperationErrorCode: {1}.",
                                         _worker.Name,
                                         (OperationErrorCode)dto.ErrorCode));
                _worker.TryRemoveWorkItem(workItem);
                break;

            default:
                throw new ArgumentOutOfRangeException();
            }
        }