public async Task <ReadAllPage> ReadAllForwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData = true, CancellationToken cancellationToken = new CancellationToken()) { var visibilityDate = DateTime.UtcNow - _guaranteedDelay; var page = await _store .ReadAllForwards(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken) .NotOnCapturedContext(); if (page.Messages.Any()) { var messageDate = page.Messages.Select(e => DateTime.SpecifyKind(e.CreatedUtc, DateTimeKind.Utc)).Max(); if (messageDate > visibilityDate) { await Task.Delay(messageDate - visibilityDate, cancellationToken).NotOnCapturedContext(); await _store .ReadAllForwards(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken) .NotOnCapturedContext(); } } return(page); }
public async Task <long> GetManyPagesAsync(IStreamStore store, int chunksCount, int batchSize) { long start = 0; //var events = new List<StreamMessage>(); for (int i = 0; i < chunksCount; i++) { var stopwatch = Stopwatch.StartNew(); var timesSlept = 0; bool moreThanPage = false; while (!moreThanPage) { var page = await store.ReadAllForwards(start, batchSize); if (page.IsEnd) { Thread.Sleep(10); timesSlept++; } else { start = page.NextPosition; moreThanPage = true; Console.WriteLine($"Page start from {start} read. Slept {timesSlept} times by 10ms.Elapsed {stopwatch.Elapsed}"); } } } return(start); }
protected override async Task <List <IDomainEvent> > LoadAllChangeLogs() { var changeLog = new List <IDomainEvent>(); var page = await _eventStore.ReadAllForwards(0, int.MaxValue); foreach (var message in page.Messages) { var json = await message.GetJsonData(); var evt = (IDomainEvent)JsonConvert.DeserializeObject(json, SqlStreamStoreOutbox.SerializerSettings); changeLog.Add(evt); } return(changeLog); }
public async Task <Fact[]> RetrieveFacts(long fromPositionExclusive) { var results = new List <Fact>(); var page = await _streamStore.ReadAllForwards(fromPositionExclusive < 0?Position.Start : fromPositionExclusive, 10); results.AddRange(page.Messages.Where(m => m.Position != fromPositionExclusive).Select(MapToFact)); while (!page.IsEnd) { page = await page.ReadNext(); results.AddRange(page.Messages.Where(m => m.Position != fromPositionExclusive).Select(MapToFact)); } return(results.ToArray()); }
private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize) { int count = 0; var stopwatch = Stopwatch.StartNew(); try { // var position = Position.Start; var part = readPageSize / 5; do { var position = Math.Max(await streamStore.ReadHeadPosition(ct) - part, 0); var page = await streamStore.ReadAllForwards(position, readPageSize, prefetchJsonData : false, cancellationToken : ct); count += page.Messages.Length; if (page.Messages.Length < 2) { continue; } Output.WriteLine($"< from {position}"); for (var index = 1; index < page.Messages.Length; index++) { if (page.Messages[index].Position != page.Messages[index - 1].Position + 1) { Output.WriteLine( $"< Gap found {page.Messages[index - 1].Position} and {page.Messages[index].Position}"); break; } } } while(!ct.IsCancellationRequested); } catch (Exception ex) when(!(ex is OperationCanceledException)) { Output.WriteLine(ex.ToString()); } finally { stopwatch.Stop(); var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0); Output.WriteLine(""); Output.WriteLine($"< {count} messages read {stopwatch.Elapsed} ({rate} m/s)"); } }
private static async Task WriteActualGaps(CancellationToken ct, IStreamStore streamStore) { var stopwatch = Stopwatch.StartNew(); var count = 0; Output.WriteLine("Actual gaps:"); var page = await streamStore.ReadAllForwards(Position.Start, 73, false, ct); count += page.Messages.Length; var prevPosition = page.Messages[0].Position; for (int i = 1; i < page.Messages.Length; i++) { if (prevPosition + 1 != page.Messages[i].Position) { Output.WriteLine($"- {prevPosition} : {page.Messages[i].Position}"); } prevPosition = page.Messages[i].Position; } while (!page.IsEnd) { page = await page.ReadNext(ct); count += page.Messages.Length; for (int i = 0; i < page.Messages.Length; i++) { if (prevPosition + 1 != page.Messages[i].Position) { Output.WriteLine($"- {prevPosition} : {page.Messages[i].Position}"); } prevPosition = page.Messages[i].Position; } } stopwatch.Stop(); var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0); Output.WriteLine(""); Output.WriteLine($"< {count} messages read {stopwatch.Elapsed} ({rate} m/s)"); }
private async Task <RecordedEvent[]> ReadThens(long position) { var recorded = new List <RecordedEvent>(); var page = await _store.ReadAllForwards(position, 1024); foreach (var then in page.Messages) { recorded.Add( new RecordedEvent( new StreamName(then.StreamId), JsonConvert.DeserializeObject( await then.GetJsonData(), _mapping.GetEventType(then.Type), _settings ) ) ); } while (!page.IsEnd) { page = await page.ReadNext(); foreach (var then in page.Messages) { recorded.Add( new RecordedEvent( new StreamName(then.StreamId), JsonConvert.DeserializeObject( await then.GetJsonData(), _mapping.GetEventType(then.Type), _settings ) ) ); } } return(recorded.ToArray()); }
private async Task <IReadOnlyCollection <RecordedEvent> > ReadThens(long position, CancellationToken cancellationToken) { var recorded = new List <RecordedEvent>(); var page = await _store.ReadAllForwards(position, 1024, cancellationToken : cancellationToken); foreach (var then in page.Messages) { recorded.Add( new RecordedEvent( new StreamName(then.StreamId), JsonSerializer.Deserialize( await then.GetJsonData(cancellationToken), _messageTypeResolver(then.Type), _options ) ) ); } while (!page.IsEnd) { page = await page.ReadNext(cancellationToken); foreach (var then in page.Messages) { recorded.Add( new RecordedEvent( new StreamName(then.StreamId), JsonSerializer.Deserialize( await then.GetJsonData(cancellationToken), _messageTypeResolver(then.Type), _options ) ) ); } } return(recorded.ToArray()); }
public async Task <List <IEvent> > GetEventsFromStreamAsync(string stream, int?startFromVersion, CancellationToken cancellationToken = default) { var stopWatch = new Stopwatch(); stopWatch.Start(); const int _PAGE_SIZE = 10; var page = await _streamStore.ReadAllForwards(Position.Start, _PAGE_SIZE, cancellationToken : cancellationToken); var messages = new List <StreamMessage>(page.Messages); while (!page.IsEnd) //should not take more than 20 iterations. { page = await page.ReadNext(cancellationToken); messages.AddRange(page.Messages); } var result = new List <IEvent>(); foreach (var sm in messages) { var metadata = _serDes.Deserialize <EventMetadata>(sm.JsonMetadata); var eventType = metadata.GetEventType(); var data = await sm.GetJsonData(cancellationToken); var @event = _serDes.Deserialize(data, eventType) as IEvent; result.Add(@event); } stopWatch.Stop(); _logger.LogDebug("SqlStreamStore.GetEventsFromStreamAsync for {Stream} took {ElapsedMilliseconds} ms", stream, stopWatch.ElapsedMilliseconds); return(result); }
public EventProcessor( IStreamStore streamStore, AcceptStreamMessageFilter filter, EnvelopeFactory envelopeFactory, ConnectedProjectionHandlerResolver <EditorContext> resolver, Func <EditorContext> dbContextFactory, Scheduler scheduler, ILogger <EventProcessor> logger) { if (streamStore == null) { throw new ArgumentNullException(nameof(streamStore)); } if (filter == null) { throw new ArgumentNullException(nameof(filter)); } if (envelopeFactory == null) { throw new ArgumentNullException(nameof(envelopeFactory)); } if (resolver == null) { throw new ArgumentNullException(nameof(resolver)); } if (dbContextFactory == null) { throw new ArgumentNullException(nameof(dbContextFactory)); } _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _messagePumpCancellation = new CancellationTokenSource(); _messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false, AllowSynchronousContinuations = false }); _messagePump = Task.Factory.StartNew(async() => { IAllStreamSubscription subscription = null; try { logger.LogInformation("EventProcessor message pump entered ..."); while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false)) { while (_messageChannel.Reader.TryRead(out var message)) { switch (message) { case Resume _: logger.LogInformation("Resuming ..."); await using (var resumeContext = dbContextFactory()) { var projection = await resumeContext.ProjectionStates .SingleOrDefaultAsync( item => item.Name == RoadRegistryEditorProjectionHost, _messagePumpCancellation.Token) .ConfigureAwait(false); var after = projection?.Position; var head = await streamStore.ReadHeadPosition(); if (head == Position.Start || (after.HasValue ? head - after.Value <= CatchUpThreshold : head - CatchUpThreshold <= 0)) { await _messageChannel.Writer .WriteAsync(new Subscribe(after), _messagePumpCancellation.Token) .ConfigureAwait(false); } else { await _messageChannel.Writer .WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token) .ConfigureAwait(false); } } break; case CatchUp catchUp: logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L); var observedMessageCount = 0; var catchUpPosition = catchUp.AfterPosition ?? Position.Start; var context = dbContextFactory(); var page = await streamStore .ReadAllForwards( catchUpPosition, catchUp.BatchSize, true, _messagePumpCancellation.Token) .ConfigureAwait(false); while (!page.IsEnd) { foreach (var streamMessage in page.Messages) { if (catchUp.AfterPosition.HasValue && streamMessage.Position == catchUp.AfterPosition.Value) { continue; // skip already processed message } if (filter(streamMessage)) { logger.LogInformation("Catching up on {MessageType} at {Position}", streamMessage.Type, streamMessage.Position); var envelope = envelopeFactory.Create(streamMessage); var handlers = resolver(envelope); foreach (var handler in handlers) { await handler .Handler(context, envelope, _messagePumpCancellation.Token) .ConfigureAwait(false); } } observedMessageCount++; catchUpPosition = streamMessage.Position; if (observedMessageCount % CatchUpBatchSize == 0) { logger.LogInformation( "Flushing catch up position of {0} and persisting changes ...", catchUpPosition); await context .UpdateProjectionState( RoadRegistryEditorProjectionHost, catchUpPosition, _messagePumpCancellation.Token) .ConfigureAwait(false); await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); await context.DisposeAsync().ConfigureAwait(false); context = dbContextFactory(); observedMessageCount = 0; } } page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false); } if (observedMessageCount > 0) // case where we just read the last page and pending work in memory needs to be flushed { logger.LogInformation( "Flushing catch up position of {Position} and persisting changes ...", catchUpPosition); await context .UpdateProjectionState( RoadRegistryEditorProjectionHost, catchUpPosition, _messagePumpCancellation.Token) .ConfigureAwait(false); await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } await context.DisposeAsync().ConfigureAwait(false); //switch to subscription as of the last page await _messageChannel.Writer .WriteAsync( new Subscribe(catchUpPosition), _messagePumpCancellation.Token) .ConfigureAwait(false); break; case Subscribe subscribe: logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L); subscription?.Dispose(); subscription = streamStore.SubscribeToAll( subscribe.AfterPosition, async(_, streamMessage, token) => { if (filter(streamMessage)) { logger.LogInformation("Observing {0} at {1}", streamMessage.Type, streamMessage.Position); var command = new ProcessStreamMessage(streamMessage); await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false); await command.Completion.ConfigureAwait(false); } else if (streamMessage.Position % RecordPositionThreshold == 0 && !_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync(new RecordPosition(streamMessage), token) .ConfigureAwait(false); } }, async(_, reason, exception) => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer .WriteAsync( new SubscriptionDropped(reason, exception), _messagePumpCancellation.Token) .ConfigureAwait(false); } }, prefetchJsonData: false, name: "RoadRegistry.Editor.ProjectionHost.EventProcessor"); break; case RecordPosition record: try { logger.LogInformation("Recording position of {MessageType} at {Position}.", record.Message.Type, record.Message.Position); await using (var recordContext = dbContextFactory()) { await recordContext .UpdateProjectionState( RoadRegistryEditorProjectionHost, record.Message.Position, _messagePumpCancellation.Token) .ConfigureAwait(false); await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } } catch (Exception exception) { logger.LogError(exception, exception.Message); } break; case ProcessStreamMessage process: try { logger.LogInformation("Processing {MessageType} at {Position}", process.Message.Type, process.Message.Position); var envelope = envelopeFactory.Create(process.Message); var handlers = resolver(envelope); await using (var processContext = dbContextFactory()) { foreach (var handler in handlers) { await handler .Handler(processContext, envelope, _messagePumpCancellation.Token) .ConfigureAwait(false); } await processContext.UpdateProjectionState( RoadRegistryEditorProjectionHost, process.Message.Position, _messagePumpCancellation.Token).ConfigureAwait(false); await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false); } process.Complete(); } catch (Exception exception) { logger.LogError(exception, exception.Message); // how are we going to recover from this? do we even need to recover from this? // prediction: it's going to be a serialization error, a data quality error, or a bug process.Fault(exception); } break; case SubscriptionDropped dropped: if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a stream store error"); await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError) { logger.LogError(dropped.Exception, "Subscription was dropped because of a subscriber error"); if (dropped.Exception != null && dropped.Exception is SqlException sqlException && sqlException.Number == -2 /* timeout */) { await scheduler.Schedule(async token => { if (!_messagePumpCancellation.IsCancellationRequested) { await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false); } }, ResubscribeAfter).ConfigureAwait(false); } } break; } } } } catch (TaskCanceledException) { logger.LogInformation("EventProcessor message pump is exiting due to cancellation"); } catch (OperationCanceledException) { logger.LogInformation("EventProcessor message pump is exiting due to cancellation"); } catch (Exception exception) { logger.LogError(exception, "EventProcessor message pump is exiting due to a bug"); } finally { subscription?.Dispose(); } }, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default); }
public Task <ReadAllPage> Invoke(IStreamStore streamStore, CancellationToken ct) => ReadDirection == Constants.ReadDirection.Forwards ? streamStore.ReadAllForwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct) : streamStore.ReadAllBackwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct);
public async Task <StreamMessage> Invoke(IStreamStore streamStore, CancellationToken ct) { var page = await streamStore.ReadAllForwards(Position, 1, true, ct); return(page.Messages.Where(m => m.Position == Position).FirstOrDefault()); }