protected override async Task ConnectAsync() { //Get subscription position _position = await _subscriptionManager.GetSubscriptionPosition(_subscriptionId, true); _store = await _storeProvider.GetStreamStore(); if (_streamId == null) { _allSubscription = _store.SubscribeToAll(_position, HandleSubscriptionEvent, HandleSubscriptionDropped); _status = SubscriptionConnectionStatus.Connected; _startDate = DateTime.UtcNow; } else { int?intPos = (_position != null) ? Convert.ToInt32(_position) : default(int?); _subscription = _store.SubscribeToStream(_streamId.Id, intPos, HandleSubscriptionEvent, HandleSubscriptionDropped); _status = SubscriptionConnectionStatus.Connected; _startDate = DateTime.UtcNow; } }
public Task SubscribeToAllAsync(Func <IEvent, Task> handler, CancellationToken cancellationToken = default) { var are = new AutoResetEvent(false); var sub = _store.SubscribeToAll(null, (s, m, t) => MessageReceived(s, m, handler, are, t), null, null, true, "x"); are.WaitOne(); return(Task.CompletedTask); }
private void SubscriptionDropped(IAllStreamSubscription _, SubscriptionDroppedReason reason, Exception exception) { if (reason == SubscriptionDroppedReason.StreamStoreError) { // Transient errors should re-create the subscription if (exception is SqlException) { _subscription = _streamStore.SubscribeToAll(_currentPosition, StreamMessageReceived, SubscriptionDropped, IsCaughtUp); } } }
public void Start() { long?actualPosition; using (var scope = PaymentsCompositionRoot.BeginLifetimeScope()) { var checkpointStore = scope.Resolve <ICheckpointStore>(); actualPosition = checkpointStore.GetCheckpoint(SubscriptionCode.All); } _streamStore.SubscribeToAll(actualPosition, StreamMessageReceived); }
public Task Subscribe() { var tsc = new TaskCompletionSource <bool>(); var subscription = _streamStore.SubscribeToAll(0, StreamMessageReceived, null, up => { if (up) { tsc.SetResult(true); } }); subscription.MaxCountPerRead = 500; return(tsc.Task); }
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IHostingEnvironment env, IStreamStore store, IServiceProvider serviceProvider) { app.MigrateDbContext <ReadContext>(); store.SubscribeToAll(Position.None, async(subscription, message) => { try { using (var scope = serviceProvider.CreateScope()) { var context = scope.ServiceProvider.GetRequiredService <ReadContext>(); var data = await message.GetJsonData(); var command = JsonConvert.DeserializeObject <Command>(data); var readModel = context.ReadModels.Find(command.Id); if (readModel == null) { context.Add(new ReadModel(command.Value, message.CreatedUtc)); } else { readModel.UpdateValue(command.Value, message.CreatedUtc); } context.SaveChanges(); } } catch (Exception ex) { throw ex; } }); if (store is MsSqlStreamStore msStore) { msStore.CreateSchema().GetAwaiter().GetResult(); } if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); } app.UseMvc(); }
protected override async Task ConnectAsync() { _store = await _storeProvider.GetStreamStore(); if (_streamId == null) { _allSubscription = _store.SubscribeToAll( _startPosition == StreamPosition.End ? -1 : 0, (_, message, cancellationToken) => { return(HandleEvent(message, cancellationToken)); }, (sub, reason, ex) => { HandleSubscriptionDropped(sub, reason, ex); }); _status = SubscriptionConnectionStatus.Connected; _startDate = DateTime.UtcNow; } else { _streamSubscription = _store.SubscribeToStream( _streamId.Id, _startPosition == StreamPosition.End ? -1 : 0, (_, message, cancellationToken) => { return(HandleEvent(message, cancellationToken)); }, (sub, reason, ex) => { HandleSubscriptionDropped(sub, reason, ex); }); _status = SubscriptionConnectionStatus.Connected; _startDate = DateTime.UtcNow; } }
public EventProcessor( IStreamStore streamStore, AcceptStreamMessageFilter filter, EnvelopeFactory envelopeFactory, ConnectedProjectionHandlerResolver <EditorContext> resolver, Func <EditorContext> dbContextFactory, Scheduler scheduler, ILogger <EventProcessor> logger) { if (streamStore == null) { throw new ArgumentNullException(nameof(streamStore)); } if (filter == null) { throw new ArgumentNullException(nameof(filter)); } if (envelopeFactory == null) { throw new ArgumentNullException(nameof(envelopeFactory)); } if (resolver == null) { throw new ArgumentNullException(nameof(resolver)); } if (dbContextFactory == null) { throw new ArgumentNullException(nameof(dbContextFactory)); } _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _messagePumpCancellation = new CancellationTokenSource(); _messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false, AllowSynchronousContinuations = false }); _messagePump = Task.Factory.StartNew(async() => { IAllStreamSubscription subscription = null; try { logger.LogInformation("EventProcessor message pump entered ..."); while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false)) { while (_messageChannel.Reader.TryRead(out var message)) { switch (message) { case Resume _: logger.LogInformation("Resuming ..."); await using (var resumeContext = dbContextFactory()) { var projection = await resumeContext.ProjectionStates .SingleOrDefaultAsync( item => item.Name == RoadRegistryEditorProjectionHost, _messagePumpCancellation.Token) .ConfigureAwait(false); var after = projection?.Position; var head = await streamStore.ReadHeadPosition(); if (head == Position.Start || (after.HasValue ? head - after.Value <= CatchUpThreshold : head - CatchUpThreshold <= 0)) { await _messageChannel.Writer .WriteAsync(new Subscribe(after), _messagePumpCancellation.Token) .ConfigureAwait(false); } else { await _messageChannel.Writer .WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token) .ConfigureAwait(false); } } break; case CatchUp catchUp: logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L); var observedMessageCount = 0; var catchUpPosition = catchUp.AfterPosition ?? Position.Start; var context = dbContextFactory(); var page = await streamStore .ReadAllForwards( catchUpPosition, catchUp.BatchSize, true, _messagePumpCancellation.Token) .ConfigureAwait(false); while (!page.IsEnd) { foreach (var streamMessage in page.Messages) { if (catchUp.AfterPosition.HasValue && streamMessage.Position == catchUp.AfterPosition.Value) { continue; // skip already processed message } if (filter(streamMessage)) { logger.LogInformation("Catching up on {MessageType} at {Position}", streamMessage.Type, streamMessage.Position); var envelope = envelopeFactory.Create(streamMessage); var handlers = resolver(envelope); foreach (var handler in handlers) { await handler .Handler(context, envelope, _messagePumpCancellation.Token) .ConfigureAwait(false); } } observedMessageCount++; catchUpPosition = streamMessage.Position; if (observedMessageCount % CatchUpBatchSize == 0) { logger.LogInformation( "Flushing catch up position of {0} and persisting changes ...", catchUpPosition); await context .UpdateProjectionState( RoadRegistryEditorProjectionHost, catchUpPosition, _messagePumpCancellation.Token) .ConfigureAwait(false); await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); await context.DisposeAsync().ConfigureAwait(false); context = dbContextFactory(); observedMessageCount = 0; } } page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false); } if (observedMessageCount > 0) // case where we just read the last page and pending work in memory needs to be flushed { logger.LogInformation( "Flushing catch up position of {Position} and persisting changes ...", catchUpPosition); await context .UpdateProjectionState( RoadRegistryEditorProjectionHost, catchUpPosition, _messagePumpCancellation.Token) .ConfigureAwait(false); await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } await context.DisposeAsync().ConfigureAwait(false); //switch to subscription as of the last page await _messageChannel.Writer .WriteAsync( new Subscribe(catchUpPosition), _messagePumpCancellation.Token) .ConfigureAwait(false); break; case Subscribe subscribe: logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L); subscription?.Dispose(); subscription = streamStore.SubscribeToAll( subscribe.AfterPosition, async(_, streamMessage, token) => { if (filter(streamMessage)) { logger.LogInformation("Observing {0} at {1}", streamMessage.Type, streamMessage.Position); var command = new ProcessStreamMessage(streamMessage); await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false); await command.Completion.ConfigureAwait(false); } else if (streamMessage.Position % RecordPositionThreshold == 0 && !_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync(new RecordPosition(streamMessage), token) .ConfigureAwait(false); } }, async(_, reason, exception) => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync( new SubscriptionDropped(reason, exception), _messagePumpCancellation.Token) .ConfigureAwait(false); } }, prefetchJsonData: false, name: "RoadRegistry.Editor.ProjectionHost.EventProcessor"); break; case RecordPosition record: try { logger.LogInformation("Recording position of {MessageType} at {Position}.", record.Message.Type, record.Message.Position); await using (var recordContext = dbContextFactory()) { await recordContext .UpdateProjectionState( RoadRegistryEditorProjectionHost, record.Message.Position, _messagePumpCancellation.Token) .ConfigureAwait(false); await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } } catch (Exception exception) { logger.LogError(exception, exception.Message); } break; case ProcessStreamMessage process: try { logger.LogInformation("Processing {MessageType} at {Position}", process.Message.Type, process.Message.Position); var envelope = envelopeFactory.Create(process.Message); var handlers = resolver(envelope); await using (var processContext = dbContextFactory()) { foreach (var handler in handlers) { await handler .Handler(processContext, envelope, _messagePumpCancellation.Token) .ConfigureAwait(false); } await processContext.UpdateProjectionState( RoadRegistryEditorProjectionHost, process.Message.Position, _messagePumpCancellation.Token).ConfigureAwait(false); await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } process.Complete(); } catch (Exception exception) { logger.LogError(exception, exception.Message); // how are we going to recover from this? do we even need to recover from this? // prediction: it's going to be a serialization error, a data quality error, or a bug process.Fault(exception); } break; case SubscriptionDropped dropped: if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a stream store error"); await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a subscriber error"); if (dropped.Exception != null && dropped.Exception is SqlException sqlException && sqlException.Number == -2 /* timeout */) { await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } } break; } } } } catch (TaskCanceledException) { logger.LogInformation("EventProcessor message pump is exiting due to cancellation"); } catch (OperationCanceledException) { logger.LogInformation("EventProcessor message pump is exiting due to cancellation"); } catch (Exception exception) { logger.LogError(exception, "EventProcessor message pump is exiting due to a bug"); } finally { subscription?.Dispose(); } }, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default); }
/// <summary> /// Initializes a new instance of the <see cref="Graph"/> class. /// </summary> /// <param name="store">Stream store</param> /// <param name="serializer">Metadata serializer</param> public Graph(IStreamStore store, ISerializer <IEvent> serializer) { _store = store; _serializer = serializer; Task.Run(() => store.SubscribeToAll(Position.Start - 1, MessageReceived)); }
public EventProcessor( IStreamStore streamStore, IEventProcessorPositionStore positionStore, AcceptStreamMessageFilter filter, EventHandlerDispatcher dispatcher, Scheduler scheduler, ILogger <EventProcessor> logger) { if (streamStore == null) { throw new ArgumentNullException(nameof(streamStore)); } if (positionStore == null) { throw new ArgumentNullException(nameof(positionStore)); } if (filter == null) { throw new ArgumentNullException(nameof(filter)); } if (dispatcher == null) { throw new ArgumentNullException(nameof(dispatcher)); } _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _messagePumpCancellation = new CancellationTokenSource(); _messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false, AllowSynchronousContinuations = false }); _messagePump = Task.Factory.StartNew(async() => { IAllStreamSubscription subscription = null; try { logger.LogInformation("EventProcessor message pump entered ..."); while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false)) { while (_messageChannel.Reader.TryRead(out var message)) { switch (message) { case Subscribe _: logger.LogInformation("Subscribing ..."); subscription?.Dispose(); var position = await positionStore .ReadPosition(RoadNetworkArchiveEventQueue, _messagePumpCancellation.Token) .ConfigureAwait(false); logger.LogInformation("Subscribing as of {0}", position ?? -1L); subscription = streamStore.SubscribeToAll( position, async(_, streamMessage, token) => { if (filter(streamMessage)) { var command = new ProcessStreamMessage(streamMessage); await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false); await command.Completion.ConfigureAwait(false); } else if (streamMessage.Position % RecordPositionThreshold == 0 && !_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync(new RecordPosition(streamMessage), token) .ConfigureAwait(false); } else { logger.LogInformation("Skipping {MessageType} at {Position}", streamMessage.Type, streamMessage.Position); } }, async(_, reason, exception) => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync(new SubscriptionDropped(reason, exception), _messagePumpCancellation.Token) .ConfigureAwait(false); } }, prefetchJsonData: false, name: "RoadRegistry.BackOffice.EventHost.EventProcessor"); break; case RecordPosition record: try { logger.LogInformation("Recording position of {MessageType} at {Position}.", record.Message.Type, record.Message.Position); await positionStore .WritePosition( RoadNetworkArchiveEventQueue, record.Message.Position, _messagePumpCancellation.Token) .ConfigureAwait(false); } catch (Exception exception) { logger.LogError(exception, exception.Message); } break; case ProcessStreamMessage process: try { logger.LogInformation("Processing {MessageType} at {Position}", process.Message.Type, process.Message.Position); var body = JsonConvert.DeserializeObject( await process.Message.GetJsonData(_messagePumpCancellation.Token).ConfigureAwait(false), EventMapping.GetEventType(process.Message.Type), SerializerSettings); var @event = new Event(body).WithMessageId(process.Message.MessageId); await dispatcher(@event, _messagePumpCancellation.Token).ConfigureAwait(false); await positionStore .WritePosition( RoadNetworkArchiveEventQueue, process.Message.Position, _messagePumpCancellation.Token) .ConfigureAwait(false); process.Complete(); } catch (Exception exception) { logger.LogError(exception, exception.Message); // how are we going to recover from this? do we even need to recover from this? // prediction: it's going to be a serialization error, a data quality error, or a bug process.Fault(exception); } break; case SubscriptionDropped dropped: if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a stream store error"); await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a subscriber error"); if (dropped.Exception != null && dropped.Exception is SqlException sqlException && sqlException.Number == -2 /* timeout */) { await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Subscribe(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } } break; } } } } catch (TaskCanceledException) { if (logger.IsEnabled(LogLevel.Information)) { logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation."); } } catch (OperationCanceledException) { if (logger.IsEnabled(LogLevel.Information)) { logger.Log(LogLevel.Information, "EventProcessor message pump is exiting due to cancellation."); } } catch (Exception exception) { logger.LogError(exception, "EventProcessor message pump is exiting due to a bug."); } finally { subscription?.Dispose(); } }, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default); }
public void Subscribe() { _subscription = _store.SubscribeToAll(_lastReadPosition, OnEvent, OnSubscriptionDropped, OnCatchUpStatus, true, _projection.SchemaIdentifier.Name); _startedCatchingUpAt = DateTimeOffset.Now; }
public Consumer(int id, IStreamStore store, IScheduler scheduler) { Id = id; Store = store ?? throw new ArgumentNullException(nameof(store)); Scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); MessagePumpCancellation = new CancellationTokenSource(); Mailbox = new BufferBlock <object>(new DataflowBlockOptions { BoundedCapacity = int.MaxValue, MaxMessagesPerTask = 1, CancellationToken = MessagePumpCancellation.Token }); MessagePump = Task.Run(async() => { try { var random = new Random(); long?position = null; IAllStreamSubscription subscription = null; while (!MessagePumpCancellation.Token.IsCancellationRequested) { var message = await Mailbox.ReceiveAsync(MessagePumpCancellation.Token).ConfigureAwait(false); switch (message) { case SubscribeToAll subscribe: Console.WriteLine("[{0}]SubscribeToAll", Id); position = subscribe.ContinueAfter; if (subscription != null) { subscription.Dispose(); } subscription = Store.SubscribeToAll( subscribe.ContinueAfter, async(_, received, token) => { var completion = new TaskCompletionSource <bool>(); await Mailbox.SendAsync(new MessageReceived { Position = received.Position, Completion = completion }, MessagePumpCancellation.Token).ConfigureAwait(false); await completion.Task.ConfigureAwait(false); }, (_, reason, exception) => Mailbox.SendAsync(new SubscriptionDropped { Reason = reason, Exception = exception }) ); break; case MessageReceived received: if (position.HasValue && position.Value > received.Position) { Console.WriteLine("Received message at position {0} after message at position {1}.", received.Position, position.Value); } position = received.Position; received.Completion.TrySetResult(true); //signal that the next message can be pushed into the mailbox break; case SubscriptionDropped dropped: if (!MessagePumpCancellation.IsCancellationRequested) { if (dropped.Exception == null) { Console.WriteLine("[{0}]Subscription dropped because {1}", Id, dropped.Reason); } else { Console.WriteLine("[{0}]Subscription dropped because {1}:{2}", Id, dropped.Reason, dropped.Exception); } await Scheduler .ScheduleTellOnceAsync( () => Mailbox.Post(new SubscribeToAll { ContinueAfter = position }), TimeSpan.FromMilliseconds(random.Next(100, 5000)), // consume resubscribes in between 100 and 5000ms MessagePumpCancellation.Token) .ConfigureAwait(false); position = null; } break; } } } catch (OperationCanceledException) { } }, MessagePumpCancellation.Token); }
public SubscriptToAllExample(IStreamStore streamStore) { _streamStore = streamStore; _subscription = streamStore.SubscribeToAll(null, StreamMessageReceived, SubscriptionDropped, IsCaughtUp); }
public Subscriber(IStreamStore store, StreamId targetStream) { if (store == null) { throw new ArgumentNullException(nameof(store)); } MessagePumpCancellation = new CancellationTokenSource(); Mailbox = new BufferBlock <object>( new DataflowBlockOptions { BoundedCapacity = int.MaxValue, MaxMessagesPerTask = 1, CancellationToken = MessagePumpCancellation.Token } ); Disposed = Mailbox .Completion .ContinueWith( task => { MessagePumpCancellation?.Dispose(); }, TaskContinuationOptions.ExecuteSynchronously); MessagePump = new Lazy <Task>( () => Task.Run(async() => { long?position = null; var targetStreamVersion = ExpectedVersion.NoStream; var page = await store.ReadStreamBackwards(targetStream, StreamVersion.End, 1, MessagePumpCancellation.Token); if (page.Status == PageReadStatus.Success) { targetStreamVersion = page.LastStreamVersion; if (page.Messages.Length == 1) { position = JsonConvert .DeserializeObject <MetaDataView>(page.Messages[0].JsonMetadata) .CausedByPosition; } } var subscription = store.SubscribeToAll( position, async(_, received, token) => { if (received.StreamId != targetStream) // skip your own messages { var msg = new Messages.AllStreamSubscriptionReceivedMessage { Position = received.Position, StreamId = received.StreamId, StreamVersion = received.StreamVersion, MessageId = received.MessageId, Type = received.Type, JsonData = await received.GetJsonData(token), JsonMetaData = received.JsonMetadata, CreatedUtc = received.CreatedUtc }; await Mailbox.SendAsync(msg, token); } }, (_, reason, exception) => Mailbox.Post(new Messages.AllStreamSubscriptionDropped { Reason = reason, Exception = exception })); while (!MessagePumpCancellation.IsCancellationRequested) { var message = await Mailbox.ReceiveAsync(MessagePumpCancellation.Token); switch (message) { case Messages.AllStreamSubscriptionReceivedMessage received: var appendResult = await store.AppendToStream( targetStream, targetStreamVersion, new [] { new NewStreamMessage( received.MessageId, received.Type, received.JsonData, received.JsonMetaData) }, MessagePumpCancellation.Token); targetStreamVersion = appendResult.CurrentVersion; break; case Messages.AllStreamSubscriptionDropped dropped: break; } } }, MessagePumpCancellation.Token), LazyThreadSafetyMode.ExecutionAndPublication); }