public EventStoreConnection(IPEndPoint tcpEndPoint, int maxConcurrentRequests = 50, int maxAttemptsForOperation = 10, int maxReconnections = 10, ILogger logger = null) { Ensure.NotNull(tcpEndPoint, "tcpEndPoint"); Ensure.Positive(maxConcurrentRequests, "maxConcurrentRequests"); Ensure.Nonnegative(maxAttemptsForOperation, "maxAttemptsForOperation"); Ensure.Nonnegative(maxReconnections, "maxReconnections"); _tcpEndPoint = tcpEndPoint; _maxConcurrentItems = maxConcurrentRequests; _maxAttempts = maxAttemptsForOperation; _maxReconnections = maxReconnections; LogManager.RegisterLogger(logger); _log = LogManager.GetLogger(); _connector = new TcpConnector(_tcpEndPoint); _subscriptionsChannel = new SubscriptionsChannel(_connector); //TODO TD: WAT? _projectionsManager = new ProjectionsManager(new IPEndPoint(_tcpEndPoint.Address, _tcpEndPoint.Port + 1000)); _lastReconnectionTimestamp = DateTime.UtcNow; _connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed); _timeoutCheckStopwatch.Start(); _worker = new Thread(MainLoop) { IsBackground = true, Name = "Worker thread" }; _worker.Start(); }
private Task EstablishConnectionAsync(IPEndPoint tcpEndPoint, IPEndPoint httpEndPoint) { lock (_connectionLock) { if (_isActive) { throw new InvalidOperationException("EventStoreConnection is already active"); } _isActive = true; _tcpEndPoint = tcpEndPoint; _httpEndPoint = httpEndPoint; _lastReconnectionTimestamp = DateTime.UtcNow; _connection = _connector.CreateTcpConnection(_tcpEndPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed); _timeoutCheckStopwatch.Start(); _worker = new Thread(MainLoop) { IsBackground = true, Name = "Worker thread" }; _worker.Start(); return(Tasks.CreateCompleted()); } }
private void OnConnectionClosed(TcpTypedConnection <byte[]> connection, SocketError errorCode) { lock (_connectionLock) { _reconnectionStopwatch.Restart(); } }
public void Start() { _lastReconnectionTimestamp = DateTime.UtcNow; _connection = _coordinator.CreateConnection(OnPackageArrived, OnConnectionEstablished, OnConnectionClosed); _timeoutCheckStopwatch.Start(); _stoppedEvent.Reset(); _thread.Start(); }
private void MainLoop() { while (!_stop) { VerificationEvent evnt; if (_inProgressCount < _maxConcurrentItems && TryGetNext(out evnt)) { Interlocked.Increment(ref _inProgressCount); Send(evnt); } else { Thread.Sleep(1); } lock (_connectionLock) { if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay) { _reconnectionsCount += 1; _lastReconnectionTimestamp = DateTime.UtcNow; _connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed); _reconnectionStopwatch.Stop(); } } if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod) { var now = DateTime.UtcNow; foreach (var workerItem in _events.Values) { var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks)); if (now - lastUpdated > EventTimeoutDelay) { if (lastUpdated > _lastReconnectionTimestamp) { SignalWorkerFailed( error: string.Format("Worker {0} discovered timed out event which " + "never got response from server. " + "Last state update: {1}, last reconnect: {2}, now: {3}.", Name, lastUpdated, _lastReconnectionTimestamp, now)); TryRemoveWorkItem(workerItem); } else { Retry(workerItem); } } } _timeoutCheckStopwatch.Restart(); } } _stoppedEvent.Set(); }
private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package) { var corrId = package.CorrelationId; WorkerItem workItem; if (!_items.TryGetValue(corrId, out workItem)) { _coordinator.SignalWorkerFailed(null, string.Format( "Worker {0} received unexpected CorrId: {1}, no item with such CorrId is in progress.", _name, corrId)); return; } var result = workItem.Task.CheckStepExpectations(package); switch (result.Status) { case Status.MeetsExpectations: if (TryRemoveWorkItem(workItem)) { NotifyItemProcessed(workItem); } break; case Status.Retry: Retry(workItem); break; case Status.CheckStreamDeleted: if (_coordinator.IsDeleted(result.Description) && TryRemoveWorkItem(workItem)) { NotifyItemProcessed(workItem); } else { Retry(workItem); _coordinator.SignalPossibleFailure( string.Format( "Got stream deleted ({0}), but it's not confirmed. Retrying ({1})", result.Description, workItem.Attempt)); } break; case Status.Ignore: break; case Status.FailFast: _coordinator.SignalWorkerFailed(null, result.Description); TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } }
private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error) { _connectedEvent.Reset(); var subscriptions = _subscriptions.Values; _subscriptions.Clear(); foreach (var subscription in subscriptions) { subscription.Source.SetResult(null); ExecuteUserCallbackAsync(subscription.SubscriptionDropped); } }
public void Connect() { _connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed); if (_executionThread == null) { _executionThread = new Thread(ExecuteUserCallbacks) { IsBackground = true, Name = "SubscriptionsChannel user callbacks thread" }; _executionThread.Start(); } }
private void Connect(IPEndPoint endPoint) { _connection = _connector.CreateTcpConnection(endPoint, OnPackageReceived, OnConnectionEstablished, OnConnectionClosed); if (_executionThread == null) { _stopExecutionThread = false; _executionThread = new Thread(ExecuteUserCallbacks) { IsBackground = true, Name = "SubscriptionsChannel user callbacks thread" }; _executionThread.Start(); } }
private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package) { var corrId = package.CorrelationId; WorkItem workItem; if (!_inProgress.TryGetValue(corrId, out workItem)) { _log.Error("Unexpected corrid received {0}", corrId); return; } var result = workItem.Operation.InspectPackage(package); switch (result.Decision) { case InspectionDecision.Succeed: if (TryRemoveWorkItem(workItem)) { workItem.Operation.Complete(); } break; case InspectionDecision.Retry: Retry(workItem); break; case InspectionDecision.Reconnect: Reconnect(workItem, (EndpointsPair)result.Data); break; case InspectionDecision.NotifyError: if (TryRemoveWorkItem(workItem)) { workItem.Operation.Fail(result.Error); } break; default: throw new ArgumentOutOfRangeException(); } }
private void OnPackageReceived(TcpTypedConnection connection, TcpPackage package) { Subscription subscription; if (!_subscriptions.TryGetValue(package.CorrelationId, out subscription)) { _log.Error("Unexpected package received : {0} ({1})", package.CorrelationId, package.Command); return; } try { switch (package.Command) { case TcpCommand.StreamEventAppeared: var dto = package.Data.Deserialize <ClientMessage.StreamEventAppeared>(); var recordedEvent = new RecordedEvent(dto); var commitPos = dto.CommitPosition; var preparePos = dto.PreparePosition; ExecuteUserCallbackAsync(() => subscription.EventAppeared(recordedEvent, new Position(commitPos, preparePos))); break; case TcpCommand.SubscriptionDropped: case TcpCommand.SubscriptionToAllDropped: Subscription removed; if (_subscriptions.TryRemove(subscription.Id, out removed)) { removed.Source.SetResult(null); ExecuteUserCallbackAsync(removed.SubscriptionDropped); } break; default: throw new ArgumentOutOfRangeException(string.Format("Unexpected command : {0}", package.Command)); } } catch (Exception e) { _log.Error(e, "Error on package received"); } }
private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package) { var corrId = package.CorrelationId; WorkerItem workItem; if (!_items.TryGetValue(corrId, out workItem)) { _coordinator.SignalWorkerFailed(null, string.Format( "Worker {0} received unexpected CorrId: {1}, no item with such CorrId is in progress.", _name, corrId)); return; } var result = workItem.Task.CheckStepExpectations(package); switch (result.Status) { case Status.MeetsExpectations: if (TryRemoveWorkItem(workItem)) NotifyItemProcessed(workItem); break; case Status.Retry: Retry(workItem); break; case Status.CheckStreamDeleted: if (_coordinator.IsDeleted(result.Description) && TryRemoveWorkItem(workItem)) NotifyItemProcessed(workItem); else { Retry(workItem); _coordinator.SignalPossibleFailure( string.Format( "Got stream deleted ({0}), but it's not confirmed. Retrying ({1})", result.Description, workItem.Attempt)); } break; case Status.Ignore: break; case Status.FailFast: _coordinator.SignalWorkerFailed(null, result.Description); TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } }
private void MainLoop() { while (!_stopping) { IClientOperation operation; if (_inProgressCount < _maxConcurrentItems && _queue.TryDequeue(out operation)) { Interlocked.Increment(ref _inProgressCount); Send(new WorkItem(operation)); } else { Thread.Sleep(1); } lock (_connectionLock) { if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay) { _reconnectionsCount += 1; if (_reconnectionsCount > _maxReconnections) { throw new CannotEstablishConnectionException(); } _lastReconnectionTimestamp = DateTime.UtcNow; _connection = _connector.CreateTcpConnection(OnPackageReceived, OnConnectionEstablished, OnConnectionClosed); _reconnectionStopwatch.Stop(); } } if (_timeoutCheckStopwatch.Elapsed > OperationTimeoutCheckPeriod) { var now = DateTime.UtcNow; var retriable = new List <WorkItem>(); foreach (var workerItem in _inProgress.Values) { var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks)); if (now - lastUpdated > OperationTimeout) { Console.WriteLine("Timed out, corrId: {0}.", workerItem.Operation.CorrelationId); //Debugger.Break(); if (lastUpdated >= _lastReconnectionTimestamp) { var err = string.Format("{0} never got response from server" + "Last state update : {1}, last reconnect : {2}, now(utc) : {3}.", workerItem, lastUpdated, _lastReconnectionTimestamp, now); if (TryRemoveWorkItem(workerItem)) { _log.Error(err); workerItem.Operation.Fail(new OperationTimedOutException(err)); } _log.Error(err); } else { retriable.Add(workerItem); } } } foreach (var workItem in retriable.OrderBy(wi => wi.SeqNo)) { Retry(workItem); } _timeoutCheckStopwatch.Restart(); } } }
private void OnConnectionEstablished(TcpTypedConnection <byte[]> connection) { }
private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package) { if (package.Command != TcpCommand.ReadEventCompleted) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.", _worker.Name, TcpCommand.ReadEventCompleted, package.Command)); return; } var dto = package.Data.Deserialize <ClientMessageDto.ReadEventCompleted>(); var corrId = new Guid(dto.CorrelationId); WorkerItem workItem; if (!_worker.TryGetWorkItem(corrId, out workItem)) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.", _worker.Name, corrId)); return; } var readResult = (SingleReadResult)dto.Result; switch (readResult) { case SingleReadResult.Success: var cmp = workItem.Event.VerifyThat(new Event(workItem.Event.Event.EventId, dto.EventType, false, dto.Data, dto.Metadata), dto.EventNumber); switch (cmp.Status) { case ComparisonStatus.Success: if (_worker.TryRemoveWorkItem(workItem)) { _worker.NotifyItemProcessed(workItem); } break; case ComparisonStatus.Fail: _worker.SignalWorkerFailed( error: string.Format("Worker {0} received invalid event. Details : {1}.", _worker, cmp.Description)); _worker.TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } break; case SingleReadResult.NotFound: case SingleReadResult.NoStream: case SingleReadResult.StreamDeleted: _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected response : {1}.", _worker.Name, readResult)); _worker.TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } }
private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package) { if (package.Command != TcpCommand.ReadEventCompleted) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.", _worker.Name, TcpCommand.ReadEventCompleted, package.Command)); return; } var dto = package.Data.Deserialize<ClientMessageDto.ReadEventCompleted>(); var corrId = new Guid(dto.CorrelationId); WorkerItem workItem; if (!_worker.TryGetWorkItem(corrId, out workItem)) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.", _worker.Name, corrId)); return; } var readResult = (SingleReadResult) dto.Result; switch (readResult) { case SingleReadResult.Success: var cmp = workItem.Event.VerifyThat(new Event(workItem.Event.Event.EventId, dto.EventType, false, dto.Data, dto.Metadata), dto.EventNumber); switch (cmp.Status) { case ComparisonStatus.Success: if (_worker.TryRemoveWorkItem(workItem)) _worker.NotifyItemProcessed(workItem); break; case ComparisonStatus.Fail: _worker.SignalWorkerFailed( error: string.Format("Worker {0} received invalid event. Details : {1}.", _worker, cmp.Description)); _worker.TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } break; case SingleReadResult.NotFound: case SingleReadResult.NoStream: case SingleReadResult.StreamDeleted: _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected response : {1}.", _worker.Name, readResult)); _worker.TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } }
private void OnConnectionEstablished(TcpTypedConnection<byte[]> connection) { }
private void OnConnectionClosed(TcpTypedConnection<byte[]> connection, SocketError errorCode) { lock (_connectionLock) { _reconnectionStopwatch.Restart(); } }
private void MainLoop() { while (!_stop) { VerificationEvent evnt; if (_inProgressCount < _maxConcurrentItems && TryGetNext(out evnt)) { Interlocked.Increment(ref _inProgressCount); Send(evnt); } else Thread.Sleep(1); lock (_connectionLock) { if (_reconnectionStopwatch.IsRunning && _reconnectionStopwatch.Elapsed >= ReconnectionDelay) { _reconnectionsCount += 1; _lastReconnectionTimestamp = DateTime.UtcNow; _connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed); _reconnectionStopwatch.Stop(); } } if (_timeoutCheckStopwatch.Elapsed > EventTimeoutCheckPeriod) { var now = DateTime.UtcNow; foreach (var workerItem in _events.Values) { var lastUpdated = new DateTime(Interlocked.Read(ref workerItem.LastUpdatedTicks)); if (now - lastUpdated > EventTimeoutDelay) { if (lastUpdated > _lastReconnectionTimestamp) { SignalWorkerFailed( error: string.Format("Worker {0} discovered timed out event which " + "never got response from server. " + "Last state update: {1}, last reconnect: {2}, now: {3}.", Name, lastUpdated, _lastReconnectionTimestamp, now)); TryRemoveWorkItem(workerItem); } else Retry(workerItem); } } _timeoutCheckStopwatch.Restart(); } } _stoppedEvent.Set(); }
public void Start() { _lastReconnectionTimestamp = DateTime.UtcNow; _connection = _coordinator.CreateConnection(_onPackageArrived, OnConnectionEstablished, OnConnectionClosed); _timeoutCheckStopwatch.Start(); _stoppedEvent.Reset(); _thread.Start(); }
private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection) { lock (_connectionLock) _reconnectionsCount = 0; }
private void OnConnectionClosed(TcpTypedConnection connection, IPEndPoint endPoint, SocketError error) { lock (_connectionLock) _reconnectionStopwatch.Restart(); }
private void OnConnectionEstablished(TcpTypedConnection tcpTypedConnection) { _connectedEvent.Set(); }
private void OnPackageArrived(TcpTypedConnection<byte[]> connection, TcpPackage package) { if (package.Command != TcpCommand.WriteEventsCompleted) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.", _worker.Name, TcpCommand.WriteEventsCompleted, package.Command)); return; } var dto = package.Data.Deserialize<ClientMessageDto.WriteEventsCompleted>(); var corrId = new Guid(dto.CorrelationId); WorkerItem workItem; if (!_worker.TryGetWorkItem(corrId, out workItem)) { _worker.SignalWorkerFailed( error: string.Format( "Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.", _worker.Name, corrId)); return; } switch ((OperationErrorCode) dto.ErrorCode) { case OperationErrorCode.Success: { if (_worker.TryRemoveWorkItem(workItem)) _worker.NotifyItemProcessed(workItem); break; } case OperationErrorCode.PrepareTimeout: case OperationErrorCode.CommitTimeout: case OperationErrorCode.ForwardTimeout: _worker.Retry(workItem); break; case OperationErrorCode.WrongExpectedVersion: if (workItem.Event.ExpectedVersion == ExpectedVersion.Any) { if (workItem.Event.ShouldBeVersion != 1) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received WrongExpectedVersion for event with " + "ExpectedVersion = EventNumber.Any and ShouldBeVersion = 1, " + "which shouldn't happen ever!", _worker.Name)); _worker.TryRemoveWorkItem(workItem); return; } } _worker.Retry(workItem); break; case OperationErrorCode.StreamDeleted: case OperationErrorCode.InvalidTransaction: _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected OperationErrorCode: {1}.", _worker.Name, (OperationErrorCode) dto.ErrorCode)); _worker.TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } }
private void OnPackageArrived(TcpTypedConnection <byte[]> connection, TcpPackage package) { if (package.Command != TcpCommand.WriteEventsCompleted) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected command. Expected: {1}, received: {2}.", _worker.Name, TcpCommand.WriteEventsCompleted, package.Command)); return; } var dto = package.Data.Deserialize <ClientMessageDto.WriteEventsCompleted>(); var corrId = new Guid(dto.CorrelationId); WorkerItem workItem; if (!_worker.TryGetWorkItem(corrId, out workItem)) { _worker.SignalWorkerFailed( error: string.Format( "Worker {0} received unexpected CorrId: {1}, no event with such CorrId is in progress.", _worker.Name, corrId)); return; } switch ((OperationErrorCode)dto.ErrorCode) { case OperationErrorCode.Success: { if (_worker.TryRemoveWorkItem(workItem)) { _worker.NotifyItemProcessed(workItem); } break; } case OperationErrorCode.PrepareTimeout: case OperationErrorCode.CommitTimeout: case OperationErrorCode.ForwardTimeout: _worker.Retry(workItem); break; case OperationErrorCode.WrongExpectedVersion: if (workItem.Event.ExpectedVersion == ExpectedVersion.Any) { if (workItem.Event.ShouldBeVersion != 1) { _worker.SignalWorkerFailed( error: string.Format("Worker {0} received WrongExpectedVersion for event with " + "ExpectedVersion = EventNumber.Any and ShouldBeVersion = 1, " + "which shouldn't happen ever!", _worker.Name)); _worker.TryRemoveWorkItem(workItem); return; } } _worker.Retry(workItem); break; case OperationErrorCode.StreamDeleted: case OperationErrorCode.InvalidTransaction: _worker.SignalWorkerFailed( error: string.Format("Worker {0} received unexpected OperationErrorCode: {1}.", _worker.Name, (OperationErrorCode)dto.ErrorCode)); _worker.TryRemoveWorkItem(workItem); break; default: throw new ArgumentOutOfRangeException(); } }