public EventProcessor(
IStreamStore streamStore,
AcceptStreamMessageFilter filter,
EnvelopeFactory envelopeFactory,
ConnectedProjectionHandlerResolver <EditorContext> resolver,
Func <EditorContext> dbContextFactory,
Scheduler scheduler,
ILogger <EventProcessor> logger)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
if (envelopeFactory == null)
{
throw new ArgumentNullException(nameof(envelopeFactory));
}
if (resolver == null)
{
throw new ArgumentNullException(nameof(resolver));
}
if (dbContextFactory == null)
{
throw new ArgumentNullException(nameof(dbContextFactory));
}
_scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_messagePumpCancellation = new CancellationTokenSource();
_messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = false
});
_messagePump = Task.Factory.StartNew(async() =>
{
IAllStreamSubscription subscription = null;
try
{
logger.LogInformation("EventProcessor message pump entered ...");
while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
{
while (_messageChannel.Reader.TryRead(out var message))
{
switch (message)
{
case Resume _:
logger.LogInformation("Resuming ...");
await using (var resumeContext = dbContextFactory())
{
var projection =
await resumeContext.ProjectionStates
.SingleOrDefaultAsync(
item => item.Name == RoadRegistryEditorProjectionHost,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
var after = projection?.Position;
var head = await streamStore.ReadHeadPosition();
if (head == Position.Start || (after.HasValue
? head - after.Value <= CatchUpThreshold
: head - CatchUpThreshold <= 0))
{
await _messageChannel.Writer
.WriteAsync(new Subscribe(after), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
else
{
await _messageChannel.Writer
.WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
break;
case CatchUp catchUp:
logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L);
var observedMessageCount = 0;
var catchUpPosition = catchUp.AfterPosition ?? Position.Start;
var context = dbContextFactory();
var page = await streamStore
.ReadAllForwards(
catchUpPosition,
catchUp.BatchSize,
true,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
while (!page.IsEnd)
{
foreach (var streamMessage in page.Messages)
{
if (catchUp.AfterPosition.HasValue &&
streamMessage.Position == catchUp.AfterPosition.Value)
{
continue; // skip already processed message
}
if (filter(streamMessage))
{
logger.LogInformation("Catching up on {MessageType} at {Position}",
streamMessage.Type, streamMessage.Position);
var envelope = envelopeFactory.Create(streamMessage);
var handlers = resolver(envelope);
foreach (var handler in handlers)
{
await handler
.Handler(context, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
observedMessageCount++;
catchUpPosition = streamMessage.Position;
if (observedMessageCount % CatchUpBatchSize == 0)
{
logger.LogInformation(
"Flushing catch up position of {0} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
await context.DisposeAsync().ConfigureAwait(false);
context = dbContextFactory();
observedMessageCount = 0;
}
}
page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false);
}
if (observedMessageCount > 0) // case where we just read the last page and pending work in memory needs to be flushed
{
logger.LogInformation(
"Flushing catch up position of {Position} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
await context.DisposeAsync().ConfigureAwait(false);
//switch to subscription as of the last page
await _messageChannel.Writer
.WriteAsync(
new Subscribe(catchUpPosition),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
break;
case Subscribe subscribe:
logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L);
subscription?.Dispose();
subscription = streamStore.SubscribeToAll(
subscribe.AfterPosition, async(_, streamMessage, token) =>
{
if (filter(streamMessage))
{
logger.LogInformation("Observing {0} at {1}", streamMessage.Type,
streamMessage.Position);
var command = new ProcessStreamMessage(streamMessage);
await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false);
await command.Completion.ConfigureAwait(false);
}
else if (streamMessage.Position % RecordPositionThreshold == 0 &&
!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new RecordPosition(streamMessage), token)
.ConfigureAwait(false);
}
}, async(_, reason, exception) =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(
new SubscriptionDropped(reason, exception),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
}
},
prefetchJsonData: false,
name: "RoadRegistry.Editor.ProjectionHost.EventProcessor");
break;
case RecordPosition record:
try
{
logger.LogInformation("Recording position of {MessageType} at {Position}.",
record.Message.Type, record.Message.Position);
await using (var recordContext = dbContextFactory())
{
await recordContext
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
record.Message.Position, _messagePumpCancellation.Token)
.ConfigureAwait(false);
await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
}
break;
case ProcessStreamMessage process:
try
{
logger.LogInformation("Processing {MessageType} at {Position}",
process.Message.Type, process.Message.Position);
var envelope = envelopeFactory.Create(process.Message);
var handlers = resolver(envelope);
await using (var processContext = dbContextFactory())
{
foreach (var handler in handlers)
{
await handler
.Handler(processContext, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
await processContext.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
process.Message.Position,
_messagePumpCancellation.Token).ConfigureAwait(false);
await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
process.Complete();
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
// how are we going to recover from this? do we even need to recover from this?
// prediction: it's going to be a serialization error, a data quality error, or a bug
process.Fault(exception);
}
break;
case SubscriptionDropped dropped:
if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a stream store error");
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a subscriber error");
if (dropped.Exception != null &&
dropped.Exception is SqlException sqlException &&
sqlException.Number == -2 /* timeout */)
{
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
}
break;
}
}
}
}
catch (TaskCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (OperationCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (Exception exception)
{
logger.LogError(exception, "EventProcessor message pump is exiting due to a bug");
}
finally
{
subscription?.Dispose();
}
}, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
}
public EventProcessor(
IStreamStore streamStore,
IEventProcessorPositionStore positionStore,
AcceptStreamMessageFilter filter,
EventHandlerDispatcher dispatcher,
Scheduler scheduler,
ILogger <EventProcessor> logger)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
if (positionStore == null)
{
throw new ArgumentNullException(nameof(positionStore));
}
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
if (dispatcher == null)
{
throw new ArgumentNullException(nameof(dispatcher));
}
_scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_messagePumpCancellation = new CancellationTokenSource();
_messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = false
});
_messagePump = Task.Factory.StartNew(async() =>
{
IAllStreamSubscription subscription = null;
try
{
logger.LogInformation("EventProcessor message pump entered ...");
while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
{
while (_messageChannel.Reader.TryRead(out var message))
{
switch (message)
{
case Subscribe _:
logger.LogInformation("Subscribing ...");
subscription?.Dispose();
var position = await positionStore
.ReadPosition(RoadNetworkArchiveEventQueue, _messagePumpCancellation.Token)
.ConfigureAwait(false);
logger.LogInformation("Subscribing as of {0}", position ?? -1L);
subscription = streamStore.SubscribeToAll(
position, async(_, streamMessage, token) =>
{
if (filter(streamMessage))
{
var command = new ProcessStreamMessage(streamMessage);
await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false);
await command.Completion.ConfigureAwait(false);
}
else if (streamMessage.Position % RecordPositionThreshold == 0 &&
!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new RecordPosition(streamMessage), token)
.ConfigureAwait(false);
}
else
{
logger.LogInformation("Skipping {MessageType} at {Position}", streamMessage.Type, streamMessage.Position);
}
}, async(_, reason, exception) =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new SubscriptionDropped(reason, exception), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
},
prefetchJsonData: false,
name: "RoadRegistry.BackOffice.EventHost.EventProcessor");
break;
case RecordPosition record:
try
{
logger.LogInformation("Recording position of {MessageType} at {Position}.",
record.Message.Type, record.Message.Position);
await positionStore
.WritePosition(
RoadNetworkArchiveEventQueue,
record.Message.Position,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
}
break;
case ProcessStreamMessage process:
try
{
logger.LogInformation("Processing {MessageType} at {Position}",
process.Message.Type, process.Message.Position);
var body = JsonConvert.DeserializeObject(
await process.Message.GetJsonData(_messagePumpCancellation.Token).ConfigureAwait(false),
EventMapping.GetEventType(process.Message.Type),
SerializerSettings);
var @event = new Event(body).WithMessageId(process.Message.MessageId);
await dispatcher(@event, _messagePumpCancellation.Token).ConfigureAwait(false);
await positionStore
.WritePosition(
RoadNetworkArchiveEventQueue,
process.Message.Position,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
process.Complete();
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
// how are we going to recover from this? do we even need to recover from this?
// prediction: it's going to be a serialization error, a data quality error, or a bug
process.Fault(exception);
}
break;
case SubscriptionDropped dropped:
if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a stream store error");
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a subscriber error");
if (dropped.Exception != null &&
dropped.Exception is SqlException sqlException &&
sqlException.Number == -2 /* timeout */)
{
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
}
break;
}
}
}
}
catch (TaskCanceledException)
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation.");
}
}
catch (OperationCanceledException)
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation.");
}
}
catch (Exception exception)
{
logger.LogError(exception, "EventProcessor message pump is exiting due to a bug.");
}
finally
{
subscription?.Dispose();
}
}, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
}
