Beispiel #1
0
        public async Task <ReadAllPage> ReadAllForwards(
            long fromPositionInclusive,
            int maxCount,
            bool prefetchJsonData = true,
            CancellationToken cancellationToken = new CancellationToken())
        {
            var visibilityDate = DateTime.UtcNow - _guaranteedDelay;
            var page           = await _store
                                 .ReadAllForwards(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken)
                                 .NotOnCapturedContext();

            if (page.Messages.Any())
            {
                var messageDate = page.Messages.Select(e => DateTime.SpecifyKind(e.CreatedUtc, DateTimeKind.Utc)).Max();
                if (messageDate > visibilityDate)
                {
                    await Task.Delay(messageDate - visibilityDate, cancellationToken).NotOnCapturedContext();

                    await _store
                    .ReadAllForwards(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken)
                    .NotOnCapturedContext();
                }
            }

            return(page);
        }
Beispiel #2
0
        public async Task <long> GetManyPagesAsync(IStreamStore store, int chunksCount, int batchSize)
        {
            long start = 0;

            //var events = new List<StreamMessage>();
            for (int i = 0; i < chunksCount; i++)
            {
                var  stopwatch    = Stopwatch.StartNew();
                var  timesSlept   = 0;
                bool moreThanPage = false;
                while (!moreThanPage)
                {
                    var page = await store.ReadAllForwards(start, batchSize);

                    if (page.IsEnd)
                    {
                        Thread.Sleep(10);
                        timesSlept++;
                    }
                    else
                    {
                        start        = page.NextPosition;
                        moreThanPage = true;
                        Console.WriteLine($"Page start from {start} read. Slept {timesSlept} times by 10ms.Elapsed {stopwatch.Elapsed}");
                    }
                }
            }

            return(start);
        }
        protected override async Task <List <IDomainEvent> > LoadAllChangeLogs()
        {
            var changeLog = new List <IDomainEvent>();
            var page      = await _eventStore.ReadAllForwards(0, int.MaxValue);

            foreach (var message in page.Messages)
            {
                var json = await message.GetJsonData();

                var evt = (IDomainEvent)JsonConvert.DeserializeObject(json, SqlStreamStoreOutbox.SerializerSettings);
                changeLog.Add(evt);
            }

            return(changeLog);
        }
        public async Task <Fact[]> RetrieveFacts(long fromPositionExclusive)
        {
            var results = new List <Fact>();
            var page    = await _streamStore.ReadAllForwards(fromPositionExclusive < 0?Position.Start : fromPositionExclusive, 10);

            results.AddRange(page.Messages.Where(m => m.Position != fromPositionExclusive).Select(MapToFact));
            while (!page.IsEnd)
            {
                page = await page.ReadNext();

                results.AddRange(page.Messages.Where(m => m.Position != fromPositionExclusive).Select(MapToFact));
            }

            return(results.ToArray());
        }
Beispiel #5
0
        private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize)
        {
            int count     = 0;
            var stopwatch = Stopwatch.StartNew();

            try
            {
                // var position = Position.Start;
                var part = readPageSize / 5;
                do
                {
                    var position = Math.Max(await streamStore.ReadHeadPosition(ct) - part, 0);
                    var page     = await streamStore.ReadAllForwards(position,
                                                                     readPageSize,
                                                                     prefetchJsonData : false,
                                                                     cancellationToken : ct);

                    count += page.Messages.Length;
                    if (page.Messages.Length < 2)
                    {
                        continue;
                    }
                    Output.WriteLine($"< from {position}");
                    for (var index = 1; index < page.Messages.Length; index++)
                    {
                        if (page.Messages[index].Position != page.Messages[index - 1].Position + 1)
                        {
                            Output.WriteLine(
                                $"< Gap found {page.Messages[index - 1].Position} and {page.Messages[index].Position}");
                            break;
                        }
                    }
                } while(!ct.IsCancellationRequested);
            }
            catch (Exception ex) when(!(ex is OperationCanceledException))
            {
                Output.WriteLine(ex.ToString());
            }
            finally
            {
                stopwatch.Stop();
                var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0);
                Output.WriteLine("");
                Output.WriteLine($"< {count} messages read {stopwatch.Elapsed} ({rate} m/s)");
            }
        }
Beispiel #6
0
        private static async Task WriteActualGaps(CancellationToken ct, IStreamStore streamStore)
        {
            var stopwatch = Stopwatch.StartNew();
            var count     = 0;

            Output.WriteLine("Actual gaps:");
            var page = await streamStore.ReadAllForwards(Position.Start, 73, false, ct);

            count += page.Messages.Length;
            var prevPosition = page.Messages[0].Position;

            for (int i = 1; i < page.Messages.Length; i++)
            {
                if (prevPosition + 1 != page.Messages[i].Position)
                {
                    Output.WriteLine($"- {prevPosition} : {page.Messages[i].Position}");
                }
                prevPosition = page.Messages[i].Position;
            }
            while (!page.IsEnd)
            {
                page = await page.ReadNext(ct);

                count += page.Messages.Length;
                for (int i = 0; i < page.Messages.Length; i++)
                {
                    if (prevPosition + 1 != page.Messages[i].Position)
                    {
                        Output.WriteLine($"- {prevPosition} : {page.Messages[i].Position}");
                    }
                    prevPosition = page.Messages[i].Position;
                }
            }
            stopwatch.Stop();
            var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0);

            Output.WriteLine("");
            Output.WriteLine($"< {count} messages read {stopwatch.Elapsed} ({rate} m/s)");
        }
Beispiel #7
0
        private async Task <RecordedEvent[]> ReadThens(long position)
        {
            var recorded = new List <RecordedEvent>();
            var page     = await _store.ReadAllForwards(position, 1024);

            foreach (var then in page.Messages)
            {
                recorded.Add(
                    new RecordedEvent(
                        new StreamName(then.StreamId),
                        JsonConvert.DeserializeObject(
                            await then.GetJsonData(),
                            _mapping.GetEventType(then.Type),
                            _settings
                            )
                        )
                    );
            }
            while (!page.IsEnd)
            {
                page = await page.ReadNext();

                foreach (var then in page.Messages)
                {
                    recorded.Add(
                        new RecordedEvent(
                            new StreamName(then.StreamId),
                            JsonConvert.DeserializeObject(
                                await then.GetJsonData(),
                                _mapping.GetEventType(then.Type),
                                _settings
                                )
                            )
                        );
                }
            }
            return(recorded.ToArray());
        }
Beispiel #8
0
        private async Task <IReadOnlyCollection <RecordedEvent> > ReadThens(long position, CancellationToken cancellationToken)
        {
            var recorded = new List <RecordedEvent>();
            var page     = await _store.ReadAllForwards(position, 1024, cancellationToken : cancellationToken);

            foreach (var then in page.Messages)
            {
                recorded.Add(
                    new RecordedEvent(
                        new StreamName(then.StreamId),
                        JsonSerializer.Deserialize(
                            await then.GetJsonData(cancellationToken),
                            _messageTypeResolver(then.Type),
                            _options
                            )
                        )
                    );
            }
            while (!page.IsEnd)
            {
                page = await page.ReadNext(cancellationToken);

                foreach (var then in page.Messages)
                {
                    recorded.Add(
                        new RecordedEvent(
                            new StreamName(then.StreamId),
                            JsonSerializer.Deserialize(
                                await then.GetJsonData(cancellationToken),
                                _messageTypeResolver(then.Type),
                                _options
                                )
                            )
                        );
                }
            }
            return(recorded.ToArray());
        }
Beispiel #9
0
        public async Task <List <IEvent> > GetEventsFromStreamAsync(string stream, int?startFromVersion, CancellationToken cancellationToken = default)
        {
            var stopWatch = new Stopwatch();

            stopWatch.Start();

            const int _PAGE_SIZE = 10;

            var page = await _streamStore.ReadAllForwards(Position.Start, _PAGE_SIZE, cancellationToken : cancellationToken);

            var messages = new List <StreamMessage>(page.Messages);

            while (!page.IsEnd) //should not take more than 20 iterations.
            {
                page = await page.ReadNext(cancellationToken);

                messages.AddRange(page.Messages);
            }

            var result = new List <IEvent>();

            foreach (var sm in messages)
            {
                var metadata  = _serDes.Deserialize <EventMetadata>(sm.JsonMetadata);
                var eventType = metadata.GetEventType();
                var data      = await sm.GetJsonData(cancellationToken);

                var @event = _serDes.Deserialize(data, eventType) as IEvent;
                result.Add(@event);
            }

            stopWatch.Stop();
            _logger.LogDebug("SqlStreamStore.GetEventsFromStreamAsync for {Stream} took {ElapsedMilliseconds} ms", stream, stopWatch.ElapsedMilliseconds);

            return(result);
        }
        public EventProcessor(
            IStreamStore streamStore,
            AcceptStreamMessageFilter filter,
            EnvelopeFactory envelopeFactory,
            ConnectedProjectionHandlerResolver <EditorContext> resolver,
            Func <EditorContext> dbContextFactory,
            Scheduler scheduler,
            ILogger <EventProcessor> logger)
        {
            if (streamStore == null)
            {
                throw new ArgumentNullException(nameof(streamStore));
            }
            if (filter == null)
            {
                throw new ArgumentNullException(nameof(filter));
            }
            if (envelopeFactory == null)
            {
                throw new ArgumentNullException(nameof(envelopeFactory));
            }
            if (resolver == null)
            {
                throw new ArgumentNullException(nameof(resolver));
            }
            if (dbContextFactory == null)
            {
                throw new ArgumentNullException(nameof(dbContextFactory));
            }

            _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
            _logger    = logger ?? throw new ArgumentNullException(nameof(logger));

            _messagePumpCancellation = new CancellationTokenSource();
            _messageChannel          = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
            {
                SingleReader = true,
                SingleWriter = false,
                AllowSynchronousContinuations = false
            });
            _messagePump = Task.Factory.StartNew(async() =>
            {
                IAllStreamSubscription subscription = null;
                try
                {
                    logger.LogInformation("EventProcessor message pump entered ...");
                    while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
                    {
                        while (_messageChannel.Reader.TryRead(out var message))
                        {
                            switch (message)
                            {
                            case Resume _:
                                logger.LogInformation("Resuming ...");
                                await using (var resumeContext = dbContextFactory())
                                {
                                    var projection =
                                        await resumeContext.ProjectionStates
                                        .SingleOrDefaultAsync(
                                            item => item.Name == RoadRegistryEditorProjectionHost,
                                            _messagePumpCancellation.Token)
                                        .ConfigureAwait(false);
                                    var after = projection?.Position;
                                    var head  = await streamStore.ReadHeadPosition();
                                    if (head == Position.Start || (after.HasValue
                                            ? head - after.Value <= CatchUpThreshold
                                            : head - CatchUpThreshold <= 0))
                                    {
                                        await _messageChannel.Writer
                                        .WriteAsync(new Subscribe(after), _messagePumpCancellation.Token)
                                        .ConfigureAwait(false);
                                    }
                                    else
                                    {
                                        await _messageChannel.Writer
                                        .WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token)
                                        .ConfigureAwait(false);
                                    }
                                }

                                break;

                            case CatchUp catchUp:
                                logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L);
                                var observedMessageCount = 0;
                                var catchUpPosition      = catchUp.AfterPosition ?? Position.Start;
                                var context = dbContextFactory();
                                var page    = await streamStore
                                              .ReadAllForwards(
                                    catchUpPosition,
                                    catchUp.BatchSize,
                                    true,
                                    _messagePumpCancellation.Token)
                                              .ConfigureAwait(false);

                                while (!page.IsEnd)
                                {
                                    foreach (var streamMessage in page.Messages)
                                    {
                                        if (catchUp.AfterPosition.HasValue &&
                                            streamMessage.Position == catchUp.AfterPosition.Value)
                                        {
                                            continue;     // skip already processed message
                                        }

                                        if (filter(streamMessage))
                                        {
                                            logger.LogInformation("Catching up on {MessageType} at {Position}",
                                                                  streamMessage.Type, streamMessage.Position);
                                            var envelope = envelopeFactory.Create(streamMessage);
                                            var handlers = resolver(envelope);
                                            foreach (var handler in handlers)
                                            {
                                                await handler
                                                .Handler(context, envelope, _messagePumpCancellation.Token)
                                                .ConfigureAwait(false);
                                            }
                                        }

                                        observedMessageCount++;
                                        catchUpPosition = streamMessage.Position;

                                        if (observedMessageCount % CatchUpBatchSize == 0)
                                        {
                                            logger.LogInformation(
                                                "Flushing catch up position of {0} and persisting changes ...",
                                                catchUpPosition);
                                            await context
                                            .UpdateProjectionState(
                                                RoadRegistryEditorProjectionHost,
                                                catchUpPosition,
                                                _messagePumpCancellation.Token)
                                            .ConfigureAwait(false);
                                            await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
                                            await context.DisposeAsync().ConfigureAwait(false);

                                            context = dbContextFactory();
                                            observedMessageCount = 0;
                                        }
                                    }

                                    page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false);
                                }

                                if (observedMessageCount > 0)     // case where we just read the last page and pending work in memory needs to be flushed
                                {
                                    logger.LogInformation(
                                        "Flushing catch up position of {Position} and persisting changes ...",
                                        catchUpPosition);
                                    await context
                                    .UpdateProjectionState(
                                        RoadRegistryEditorProjectionHost,
                                        catchUpPosition,
                                        _messagePumpCancellation.Token)
                                    .ConfigureAwait(false);
                                    await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
                                }

                                await context.DisposeAsync().ConfigureAwait(false);

                                //switch to subscription as of the last page
                                await _messageChannel.Writer
                                .WriteAsync(
                                    new Subscribe(catchUpPosition),
                                    _messagePumpCancellation.Token)
                                .ConfigureAwait(false);
                                break;

                            case Subscribe subscribe:
                                logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L);
                                subscription?.Dispose();
                                subscription = streamStore.SubscribeToAll(
                                    subscribe.AfterPosition, async(_, streamMessage, token) =>
                                {
                                    if (filter(streamMessage))
                                    {
                                        logger.LogInformation("Observing {0} at {1}", streamMessage.Type,
                                                              streamMessage.Position);
                                        var command = new ProcessStreamMessage(streamMessage);
                                        await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false);
                                        await command.Completion.ConfigureAwait(false);
                                    }
                                    else if (streamMessage.Position % RecordPositionThreshold == 0 &&
                                             !_messagePumpCancellation.IsCancellationRequested)
                                    {
                                        await _messageChannel.Writer
                                        .WriteAsync(new RecordPosition(streamMessage), token)
                                        .ConfigureAwait(false);
                                    }
                                }, async(_, reason, exception) =>
                                {
                                    if (!_messagePumpCancellation.IsCancellationRequested)
                                    {
                                        await _messageChannel.Writer
                                        .WriteAsync(
                                            new SubscriptionDropped(reason, exception),
                                            _messagePumpCancellation.Token)
                                        .ConfigureAwait(false);
                                    }
                                },
                                    prefetchJsonData: false,
                                    name: "RoadRegistry.Editor.ProjectionHost.EventProcessor");

                                break;

                            case RecordPosition record:
                                try
                                {
                                    logger.LogInformation("Recording position of {MessageType} at {Position}.",
                                                          record.Message.Type, record.Message.Position);

                                    await using (var recordContext = dbContextFactory())
                                    {
                                        await recordContext
                                        .UpdateProjectionState(
                                            RoadRegistryEditorProjectionHost,
                                            record.Message.Position, _messagePumpCancellation.Token)
                                        .ConfigureAwait(false);
                                        await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
                                    }
                                }
                                catch (Exception exception)
                                {
                                    logger.LogError(exception, exception.Message);
                                }

                                break;

                            case ProcessStreamMessage process:
                                try
                                {
                                    logger.LogInformation("Processing {MessageType} at {Position}",
                                                          process.Message.Type, process.Message.Position);

                                    var envelope = envelopeFactory.Create(process.Message);
                                    var handlers = resolver(envelope);
                                    await using (var processContext = dbContextFactory())
                                    {
                                        foreach (var handler in handlers)
                                        {
                                            await handler
                                            .Handler(processContext, envelope, _messagePumpCancellation.Token)
                                            .ConfigureAwait(false);
                                        }

                                        await processContext.UpdateProjectionState(
                                            RoadRegistryEditorProjectionHost,
                                            process.Message.Position,
                                            _messagePumpCancellation.Token).ConfigureAwait(false);
                                        await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
                                    }

                                    process.Complete();
                                }
                                catch (Exception exception)
                                {
                                    logger.LogError(exception, exception.Message);

                                    // how are we going to recover from this? do we even need to recover from this?
                                    // prediction: it's going to be a serialization error, a data quality error, or a bug

                                    process.Fault(exception);
                                }

                                break;

                            case SubscriptionDropped dropped:
                                if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
                                {
                                    logger.LogError(dropped.Exception,
                                                    "Subscription was dropped because of a stream store error");
                                    await scheduler.Schedule(async token =>
                                    {
                                        if (!_messagePumpCancellation.IsCancellationRequested)
                                        {
                                            await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
                                        }
                                    }, ResubscribeAfter).ConfigureAwait(false);
                                }
                                else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
                                {
                                    logger.LogError(dropped.Exception,
                                                    "Subscription was dropped because of a subscriber error");

                                    if (dropped.Exception != null &&
                                        dropped.Exception is SqlException sqlException &&
                                        sqlException.Number == -2 /* timeout */)
                                    {
                                        await scheduler.Schedule(async token =>
                                        {
                                            if (!_messagePumpCancellation.IsCancellationRequested)
                                            {
                                                await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
                                            }
                                        }, ResubscribeAfter).ConfigureAwait(false);
                                    }
                                }

                                break;
                            }
                        }
                    }
                }
                catch (TaskCanceledException)
                {
                    logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
                }
                catch (OperationCanceledException)
                {
                    logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
                }
                catch (Exception exception)
                {
                    logger.LogError(exception, "EventProcessor message pump is exiting due to a bug");
                }
                finally
                {
                    subscription?.Dispose();
                }
            }, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
        }
Beispiel #11
0
 public Task <ReadAllPage> Invoke(IStreamStore streamStore, CancellationToken ct)
 => ReadDirection == Constants.ReadDirection.Forwards
         ? streamStore.ReadAllForwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct)
         : streamStore.ReadAllBackwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct);
        public async Task <StreamMessage> Invoke(IStreamStore streamStore, CancellationToken ct)
        {
            var page = await streamStore.ReadAllForwards(Position, 1, true, ct);

            return(page.Messages.Where(m => m.Position == Position).FirstOrDefault());
        }