public EventProcessor( IStreamStore streamStore, AcceptStreamMessageFilter filter, EnvelopeFactory envelopeFactory, ConnectedProjectionHandlerResolver <EditorContext> resolver, Func <EditorContext> dbContextFactory, Scheduler scheduler, ILogger <EventProcessor> logger) { if (streamStore == null) { throw new ArgumentNullException(nameof(streamStore)); } if (filter == null) { throw new ArgumentNullException(nameof(filter)); } if (envelopeFactory == null) { throw new ArgumentNullException(nameof(envelopeFactory)); } if (resolver == null) { throw new ArgumentNullException(nameof(resolver)); } if (dbContextFactory == null) { throw new ArgumentNullException(nameof(dbContextFactory)); } _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _messagePumpCancellation = new CancellationTokenSource(); _messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false, AllowSynchronousContinuations = false }); _messagePump = Task.Factory.StartNew(async() => { IAllStreamSubscription subscription = null; try { logger.LogInformation("EventProcessor message pump entered ..."); while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false)) { while (_messageChannel.Reader.TryRead(out var message)) { switch (message) { case Resume _: logger.LogInformation("Resuming ..."); await using (var resumeContext = dbContextFactory()) { var projection = await resumeContext.ProjectionStates .SingleOrDefaultAsync( item => item.Name == RoadRegistryEditorProjectionHost, _messagePumpCancellation.Token) .ConfigureAwait(false); var after = projection?.Position; var head = await streamStore.ReadHeadPosition(); if (head == Position.Start || (after.HasValue ? head - after.Value <= CatchUpThreshold : head - CatchUpThreshold <= 0)) { await _messageChannel.Writer .WriteAsync(new Subscribe(after), _messagePumpCancellation.Token) .ConfigureAwait(false); } else { await _messageChannel.Writer .WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token) .ConfigureAwait(false); } } break; case CatchUp catchUp: logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L); var observedMessageCount = 0; var catchUpPosition = catchUp.AfterPosition ?? Position.Start; var context = dbContextFactory(); var page = await streamStore .ReadAllForwards( catchUpPosition, catchUp.BatchSize, true, _messagePumpCancellation.Token) .ConfigureAwait(false); while (!page.IsEnd) { foreach (var streamMessage in page.Messages) { if (catchUp.AfterPosition.HasValue && streamMessage.Position == catchUp.AfterPosition.Value) { continue; // skip already processed message } if (filter(streamMessage)) { logger.LogInformation("Catching up on {MessageType} at {Position}", streamMessage.Type, streamMessage.Position); var envelope = envelopeFactory.Create(streamMessage); var handlers = resolver(envelope); foreach (var handler in handlers) { await handler .Handler(context, envelope, _messagePumpCancellation.Token) .ConfigureAwait(false); } } observedMessageCount++; catchUpPosition = streamMessage.Position; if (observedMessageCount % CatchUpBatchSize == 0) { logger.LogInformation( "Flushing catch up position of {0} and persisting changes ...", catchUpPosition); await context .UpdateProjectionState( RoadRegistryEditorProjectionHost, catchUpPosition, _messagePumpCancellation.Token) .ConfigureAwait(false); await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); await context.DisposeAsync().ConfigureAwait(false); context = dbContextFactory(); observedMessageCount = 0; } } page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false); } if (observedMessageCount > 0) // case where we just read the last page and pending work in memory needs to be flushed { logger.LogInformation( "Flushing catch up position of {Position} and persisting changes ...", catchUpPosition); await context .UpdateProjectionState( RoadRegistryEditorProjectionHost, catchUpPosition, _messagePumpCancellation.Token) .ConfigureAwait(false); await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } await context.DisposeAsync().ConfigureAwait(false); //switch to subscription as of the last page await _messageChannel.Writer .WriteAsync( new Subscribe(catchUpPosition), _messagePumpCancellation.Token) .ConfigureAwait(false); break; case Subscribe subscribe: logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L); subscription?.Dispose(); subscription = streamStore.SubscribeToAll( subscribe.AfterPosition, async(_, streamMessage, token) => { if (filter(streamMessage)) { logger.LogInformation("Observing {0} at {1}", streamMessage.Type, streamMessage.Position); var command = new ProcessStreamMessage(streamMessage); await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false); await command.Completion.ConfigureAwait(false); } else if (streamMessage.Position % RecordPositionThreshold == 0 && !_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync(new RecordPosition(streamMessage), token) .ConfigureAwait(false); } }, async(_, reason, exception) => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync( new SubscriptionDropped(reason, exception), _messagePumpCancellation.Token) .ConfigureAwait(false); } }, prefetchJsonData: false, name: "RoadRegistry.Editor.ProjectionHost.EventProcessor"); break; case RecordPosition record: try { logger.LogInformation("Recording position of {MessageType} at {Position}.", record.Message.Type, record.Message.Position); await using (var recordContext = dbContextFactory()) { await recordContext .UpdateProjectionState( RoadRegistryEditorProjectionHost, record.Message.Position, _messagePumpCancellation.Token) .ConfigureAwait(false); await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } } catch (Exception exception) { logger.LogError(exception, exception.Message); } break; case ProcessStreamMessage process: try { logger.LogInformation("Processing {MessageType} at {Position}", process.Message.Type, process.Message.Position); var envelope = envelopeFactory.Create(process.Message); var handlers = resolver(envelope); await using (var processContext = dbContextFactory()) { foreach (var handler in handlers) { await handler .Handler(processContext, envelope, _messagePumpCancellation.Token) .ConfigureAwait(false); } await processContext.UpdateProjectionState( RoadRegistryEditorProjectionHost, process.Message.Position, _messagePumpCancellation.Token).ConfigureAwait(false); await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } process.Complete(); } catch (Exception exception) { logger.LogError(exception, exception.Message); // how are we going to recover from this? do we even need to recover from this? // prediction: it's going to be a serialization error, a data quality error, or a bug process.Fault(exception); } break; case SubscriptionDropped dropped: if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a stream store error"); await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a subscriber error"); if (dropped.Exception != null && dropped.Exception is SqlException sqlException && sqlException.Number == -2 /* timeout */) { await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } } break; } } } } catch (TaskCanceledException) { logger.LogInformation("EventProcessor message pump is exiting due to cancellation"); } catch (OperationCanceledException) { logger.LogInformation("EventProcessor message pump is exiting due to cancellation"); } catch (Exception exception) { logger.LogError(exception, "EventProcessor message pump is exiting due to a bug"); } finally { subscription?.Dispose(); } }, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default); }
public EventProcessor( IStreamStore streamStore, IEventProcessorPositionStore positionStore, AcceptStreamMessageFilter filter, EventHandlerDispatcher dispatcher, Scheduler scheduler, ILogger <EventProcessor> logger) { if (streamStore == null) { throw new ArgumentNullException(nameof(streamStore)); } if (positionStore == null) { throw new ArgumentNullException(nameof(positionStore)); } if (filter == null) { throw new ArgumentNullException(nameof(filter)); } if (dispatcher == null) { throw new ArgumentNullException(nameof(dispatcher)); } _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _messagePumpCancellation = new CancellationTokenSource(); _messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false, AllowSynchronousContinuations = false }); _messagePump = Task.Factory.StartNew(async() => { IAllStreamSubscription subscription = null; try { logger.LogInformation("EventProcessor message pump entered ..."); while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false)) { while (_messageChannel.Reader.TryRead(out var message)) { switch (message) { case Subscribe _: logger.LogInformation("Subscribing ..."); subscription?.Dispose(); var position = await positionStore .ReadPosition(RoadNetworkArchiveEventQueue, _messagePumpCancellation.Token) .ConfigureAwait(false); logger.LogInformation("Subscribing as of {0}", position ?? -1L); subscription = streamStore.SubscribeToAll( position, async(_, streamMessage, token) => { if (filter(streamMessage)) { var command = new ProcessStreamMessage(streamMessage); await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false); await command.Completion.ConfigureAwait(false); } else if (streamMessage.Position % RecordPositionThreshold == 0 && !_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync(new RecordPosition(streamMessage), token) .ConfigureAwait(false); } else { logger.LogInformation("Skipping {MessageType} at {Position}", streamMessage.Type, streamMessage.Position); } }, async(_, reason, exception) => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync(new SubscriptionDropped(reason, exception), _messagePumpCancellation.Token) .ConfigureAwait(false); } }, prefetchJsonData: false, name: "RoadRegistry.BackOffice.EventHost.EventProcessor"); break; case RecordPosition record: try { logger.LogInformation("Recording position of {MessageType} at {Position}.", record.Message.Type, record.Message.Position); await positionStore .WritePosition( RoadNetworkArchiveEventQueue, record.Message.Position, _messagePumpCancellation.Token) .ConfigureAwait(false); } catch (Exception exception) { logger.LogError(exception, exception.Message); } break; case ProcessStreamMessage process: try { logger.LogInformation("Processing {MessageType} at {Position}", process.Message.Type, process.Message.Position); var body = JsonConvert.DeserializeObject( await process.Message.GetJsonData(_messagePumpCancellation.Token).ConfigureAwait(false), EventMapping.GetEventType(process.Message.Type), SerializerSettings); var @event = new Event(body).WithMessageId(process.Message.MessageId); await dispatcher(@event, _messagePumpCancellation.Token).ConfigureAwait(false); await positionStore .WritePosition( RoadNetworkArchiveEventQueue, process.Message.Position, _messagePumpCancellation.Token) .ConfigureAwait(false); process.Complete(); } catch (Exception exception) { logger.LogError(exception, exception.Message); // how are we going to recover from this? do we even need to recover from this? // prediction: it's going to be a serialization error, a data quality error, or a bug process.Fault(exception); } break; case SubscriptionDropped dropped: if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a stream store error"); await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a subscriber error"); if (dropped.Exception != null && dropped.Exception is SqlException sqlException && sqlException.Number == -2 /* timeout */) { await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } } break; } } } } catch (TaskCanceledException) { if (logger.IsEnabled(LogLevel.Information)) { logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation."); } } catch (OperationCanceledException) { if (logger.IsEnabled(LogLevel.Information)) { logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation."); } } catch (Exception exception) { logger.LogError(exception, "EventProcessor message pump is exiting due to a bug."); } finally { subscription?.Dispose(); } }, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default); }