public override void Dispose()
{
_subscription?.Dispose();
GC.SuppressFinalize(this);
base.Dispose();
}
protected override Task DisconnectAsync()
{
if (_subscription == null && _allSubscription == null)
{
return(Task.CompletedTask);
}
_cancellationRequestedByUser = true;
if (_subscription != null)
{
_subscription.Dispose();
}
if (_allSubscription != null)
{
_allSubscription.Dispose();
}
if (_store != null)
{
_store.Dispose();
}
_status = SubscriptionConnectionStatus.Disconnected;
return(Task.CompletedTask);
}
protected override Task DisconnectAsync()
{
if (_allSubscription != null)
{
_cancellationRequestedByUser = true;
_allSubscription.Dispose();
}
else if (_streamSubscription != null)
{
_cancellationRequestedByUser = true;
_streamSubscription.Dispose();
}
return(Task.CompletedTask);
}
public async Task StopAsync(CancellationToken cancellationToken)
{
_running = false;
if (_measureTask != null)
{
_cts?.Cancel();
try {
await _measureTask;
}
catch (OperationCanceledException) {
// Expected
}
}
_subscription?.Dispose();
_log.LogInformation("Stopped subscription {Subscription}", _subscriptionName);
}
public EventProcessor(
IStreamStore streamStore,
AcceptStreamMessageFilter filter,
EnvelopeFactory envelopeFactory,
ConnectedProjectionHandlerResolver <EditorContext> resolver,
Func <EditorContext> dbContextFactory,
Scheduler scheduler,
ILogger <EventProcessor> logger)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
if (envelopeFactory == null)
{
throw new ArgumentNullException(nameof(envelopeFactory));
}
if (resolver == null)
{
throw new ArgumentNullException(nameof(resolver));
}
if (dbContextFactory == null)
{
throw new ArgumentNullException(nameof(dbContextFactory));
}
_scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_messagePumpCancellation = new CancellationTokenSource();
_messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = false
});
_messagePump = Task.Factory.StartNew(async() =>
{
IAllStreamSubscription subscription = null;
try
{
logger.LogInformation("EventProcessor message pump entered ...");
while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
{
while (_messageChannel.Reader.TryRead(out var message))
{
switch (message)
{
case Resume _:
logger.LogInformation("Resuming ...");
await using (var resumeContext = dbContextFactory())
{
var projection =
await resumeContext.ProjectionStates
.SingleOrDefaultAsync(
item => item.Name == RoadRegistryEditorProjectionHost,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
var after = projection?.Position;
var head = await streamStore.ReadHeadPosition();
if (head == Position.Start || (after.HasValue
? head - after.Value <= CatchUpThreshold
: head - CatchUpThreshold <= 0))
{
await _messageChannel.Writer
.WriteAsync(new Subscribe(after), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
else
{
await _messageChannel.Writer
.WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
break;
case CatchUp catchUp:
logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L);
var observedMessageCount = 0;
var catchUpPosition = catchUp.AfterPosition ?? Position.Start;
var context = dbContextFactory();
var page = await streamStore
.ReadAllForwards(
catchUpPosition,
catchUp.BatchSize,
true,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
while (!page.IsEnd)
{
foreach (var streamMessage in page.Messages)
{
if (catchUp.AfterPosition.HasValue &&
streamMessage.Position == catchUp.AfterPosition.Value)
{
continue; // skip already processed message
}
if (filter(streamMessage))
{
logger.LogInformation("Catching up on {MessageType} at {Position}",
streamMessage.Type, streamMessage.Position);
var envelope = envelopeFactory.Create(streamMessage);
var handlers = resolver(envelope);
foreach (var handler in handlers)
{
await handler
.Handler(context, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
observedMessageCount++;
catchUpPosition = streamMessage.Position;
if (observedMessageCount % CatchUpBatchSize == 0)
{
logger.LogInformation(
"Flushing catch up position of {0} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
await context.DisposeAsync().ConfigureAwait(false);
context = dbContextFactory();
observedMessageCount = 0;
}
}
page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false);
}
if (observedMessageCount > 0) // case where we just read the last page and pending work in memory needs to be flushed
{
logger.LogInformation(
"Flushing catch up position of {Position} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
await context.DisposeAsync().ConfigureAwait(false);
//switch to subscription as of the last page
await _messageChannel.Writer
.WriteAsync(
new Subscribe(catchUpPosition),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
break;
case Subscribe subscribe:
logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L);
subscription?.Dispose();
subscription = streamStore.SubscribeToAll(
subscribe.AfterPosition, async(_, streamMessage, token) =>
{
if (filter(streamMessage))
{
logger.LogInformation("Observing {0} at {1}", streamMessage.Type,
streamMessage.Position);
var command = new ProcessStreamMessage(streamMessage);
await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false);
await command.Completion.ConfigureAwait(false);
}
else if (streamMessage.Position % RecordPositionThreshold == 0 &&
!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new RecordPosition(streamMessage), token)
.ConfigureAwait(false);
}
}, async(_, reason, exception) =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(
new SubscriptionDropped(reason, exception),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
}
},
prefetchJsonData: false,
name: "RoadRegistry.Editor.ProjectionHost.EventProcessor");
break;
case RecordPosition record:
try
{
logger.LogInformation("Recording position of {MessageType} at {Position}.",
record.Message.Type, record.Message.Position);
await using (var recordContext = dbContextFactory())
{
await recordContext
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
record.Message.Position, _messagePumpCancellation.Token)
.ConfigureAwait(false);
await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
}
break;
case ProcessStreamMessage process:
try
{
logger.LogInformation("Processing {MessageType} at {Position}",
process.Message.Type, process.Message.Position);
var envelope = envelopeFactory.Create(process.Message);
var handlers = resolver(envelope);
await using (var processContext = dbContextFactory())
{
foreach (var handler in handlers)
{
await handler
.Handler(processContext, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
await processContext.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
process.Message.Position,
_messagePumpCancellation.Token).ConfigureAwait(false);
await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
process.Complete();
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
// how are we going to recover from this? do we even need to recover from this?
// prediction: it's going to be a serialization error, a data quality error, or a bug
process.Fault(exception);
}
break;
case SubscriptionDropped dropped:
if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a stream store error");
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a subscriber error");
if (dropped.Exception != null &&
dropped.Exception is SqlException sqlException &&
sqlException.Number == -2 /* timeout */)
{
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
}
break;
}
}
}
}
catch (TaskCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (OperationCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (Exception exception)
{
logger.LogError(exception, "EventProcessor message pump is exiting due to a bug");
}
finally
{
subscription?.Dispose();
}
}, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
}
public EventProcessor(
IStreamStore streamStore,
IEventProcessorPositionStore positionStore,
AcceptStreamMessageFilter filter,
EventHandlerDispatcher dispatcher,
Scheduler scheduler,
ILogger <EventProcessor> logger)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
if (positionStore == null)
{
throw new ArgumentNullException(nameof(positionStore));
}
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
if (dispatcher == null)
{
throw new ArgumentNullException(nameof(dispatcher));
}
_scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_messagePumpCancellation = new CancellationTokenSource();
_messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = false
});
_messagePump = Task.Factory.StartNew(async() =>
{
IAllStreamSubscription subscription = null;
try
{
logger.LogInformation("EventProcessor message pump entered ...");
while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
{
while (_messageChannel.Reader.TryRead(out var message))
{
switch (message)
{
case Subscribe _:
logger.LogInformation("Subscribing ...");
subscription?.Dispose();
var position = await positionStore
.ReadPosition(RoadNetworkArchiveEventQueue, _messagePumpCancellation.Token)
.ConfigureAwait(false);
logger.LogInformation("Subscribing as of {0}", position ?? -1L);
subscription = streamStore.SubscribeToAll(
position, async(_, streamMessage, token) =>
{
if (filter(streamMessage))
{
var command = new ProcessStreamMessage(streamMessage);
await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false);
await command.Completion.ConfigureAwait(false);
}
else if (streamMessage.Position % RecordPositionThreshold == 0 &&
!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new RecordPosition(streamMessage), token)
.ConfigureAwait(false);
}
else
{
logger.LogInformation("Skipping {MessageType} at {Position}", streamMessage.Type, streamMessage.Position);
}
}, async(_, reason, exception) =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new SubscriptionDropped(reason, exception), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
},
prefetchJsonData: false,
name: "RoadRegistry.BackOffice.EventHost.EventProcessor");
break;
case RecordPosition record:
try
{
logger.LogInformation("Recording position of {MessageType} at {Position}.",
record.Message.Type, record.Message.Position);
await positionStore
.WritePosition(
RoadNetworkArchiveEventQueue,
record.Message.Position,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
}
break;
case ProcessStreamMessage process:
try
{
logger.LogInformation("Processing {MessageType} at {Position}",
process.Message.Type, process.Message.Position);
var body = JsonConvert.DeserializeObject(
await process.Message.GetJsonData(_messagePumpCancellation.Token).ConfigureAwait(false),
EventMapping.GetEventType(process.Message.Type),
SerializerSettings);
var @event = new Event(body).WithMessageId(process.Message.MessageId);
await dispatcher(@event, _messagePumpCancellation.Token).ConfigureAwait(false);
await positionStore
.WritePosition(
RoadNetworkArchiveEventQueue,
process.Message.Position,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
process.Complete();
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
// how are we going to recover from this? do we even need to recover from this?
// prediction: it's going to be a serialization error, a data quality error, or a bug
process.Fault(exception);
}
break;
case SubscriptionDropped dropped:
if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a stream store error");
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a subscriber error");
if (dropped.Exception != null &&
dropped.Exception is SqlException sqlException &&
sqlException.Number == -2 /* timeout */)
{
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
}
break;
}
}
}
}
catch (TaskCanceledException)
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation.");
}
}
catch (OperationCanceledException)
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation.");
}
}
catch (Exception exception)
{
logger.LogError(exception, "EventProcessor message pump is exiting due to a bug.");
}
finally
{
subscription?.Dispose();
}
}, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
}
public Consumer(int id, IStreamStore store, IScheduler scheduler)
{
Id = id;
Store = store ?? throw new ArgumentNullException(nameof(store));
Scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
MessagePumpCancellation = new CancellationTokenSource();
Mailbox = new BufferBlock <object>(new DataflowBlockOptions
{
BoundedCapacity = int.MaxValue,
MaxMessagesPerTask = 1,
CancellationToken = MessagePumpCancellation.Token
});
MessagePump = Task.Run(async() =>
{
try
{
var random = new Random();
long?position = null;
IAllStreamSubscription subscription = null;
while (!MessagePumpCancellation.Token.IsCancellationRequested)
{
var message = await Mailbox.ReceiveAsync(MessagePumpCancellation.Token).ConfigureAwait(false);
switch (message)
{
case SubscribeToAll subscribe:
Console.WriteLine("[{0}]SubscribeToAll", Id);
position = subscribe.ContinueAfter;
if (subscription != null)
{
subscription.Dispose();
}
subscription = Store.SubscribeToAll(
subscribe.ContinueAfter,
async(_, received, token) =>
{
var completion = new TaskCompletionSource <bool>();
await Mailbox.SendAsync(new MessageReceived {
Position = received.Position, Completion = completion
}, MessagePumpCancellation.Token).ConfigureAwait(false);
await completion.Task.ConfigureAwait(false);
},
(_, reason, exception) =>
Mailbox.SendAsync(new SubscriptionDropped {
Reason = reason, Exception = exception
})
);
break;
case MessageReceived received:
if (position.HasValue && position.Value > received.Position)
{
Console.WriteLine("Received message at position {0} after message at position {1}.", received.Position, position.Value);
}
position = received.Position;
received.Completion.TrySetResult(true); //signal that the next message can be pushed into the mailbox
break;
case SubscriptionDropped dropped:
if (!MessagePumpCancellation.IsCancellationRequested)
{
if (dropped.Exception == null)
{
Console.WriteLine("[{0}]Subscription dropped because {1}", Id, dropped.Reason);
}
else
{
Console.WriteLine("[{0}]Subscription dropped because {1}:{2}", Id, dropped.Reason, dropped.Exception);
}
await Scheduler
.ScheduleTellOnceAsync(
() => Mailbox.Post(new SubscribeToAll {
ContinueAfter = position
}),
TimeSpan.FromMilliseconds(random.Next(100, 5000)), // consume resubscribes in between 100 and 5000ms
MessagePumpCancellation.Token)
.ConfigureAwait(false);
position = null;
}
break;
}
}
}
catch (OperationCanceledException) { }
}, MessagePumpCancellation.Token);
}
public void Dispose()
{
_subscription.Dispose();
}
