public async Task <ReadAllPage> ReadAllForwards(
long fromPositionInclusive,
int maxCount,
bool prefetchJsonData = true,
CancellationToken cancellationToken = new CancellationToken())
{
var visibilityDate = DateTime.UtcNow - _guaranteedDelay;
var page = await _store
.ReadAllForwards(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken)
.NotOnCapturedContext();
if (page.Messages.Any())
{
var messageDate = page.Messages.Select(e => DateTime.SpecifyKind(e.CreatedUtc, DateTimeKind.Utc)).Max();
if (messageDate > visibilityDate)
{
await Task.Delay(messageDate - visibilityDate, cancellationToken).NotOnCapturedContext();
await _store
.ReadAllForwards(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken)
.NotOnCapturedContext();
}
}
return(page);
}
public async Task <long> GetManyPagesAsync(IStreamStore store, int chunksCount, int batchSize)
{
long start = 0;
//var events = new List<StreamMessage>();
for (int i = 0; i < chunksCount; i++)
{
var stopwatch = Stopwatch.StartNew();
var timesSlept = 0;
bool moreThanPage = false;
while (!moreThanPage)
{
var page = await store.ReadAllForwards(start, batchSize);
if (page.IsEnd)
{
Thread.Sleep(10);
timesSlept++;
}
else
{
start = page.NextPosition;
moreThanPage = true;
Console.WriteLine($"Page start from {start} read. Slept {timesSlept} times by 10ms.Elapsed {stopwatch.Elapsed}");
}
}
}
return(start);
}
protected override async Task <List <IDomainEvent> > LoadAllChangeLogs()
{
var changeLog = new List <IDomainEvent>();
var page = await _eventStore.ReadAllForwards(0, int.MaxValue);
foreach (var message in page.Messages)
{
var json = await message.GetJsonData();
var evt = (IDomainEvent)JsonConvert.DeserializeObject(json, SqlStreamStoreOutbox.SerializerSettings);
changeLog.Add(evt);
}
return(changeLog);
}
public async Task <Fact[]> RetrieveFacts(long fromPositionExclusive)
{
var results = new List <Fact>();
var page = await _streamStore.ReadAllForwards(fromPositionExclusive < 0?Position.Start : fromPositionExclusive, 10);
results.AddRange(page.Messages.Where(m => m.Position != fromPositionExclusive).Select(MapToFact));
while (!page.IsEnd)
{
page = await page.ReadNext();
results.AddRange(page.Messages.Where(m => m.Position != fromPositionExclusive).Select(MapToFact));
}
return(results.ToArray());
}
private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize)
{
int count = 0;
var stopwatch = Stopwatch.StartNew();
try
{
// var position = Position.Start;
var part = readPageSize / 5;
do
{
var position = Math.Max(await streamStore.ReadHeadPosition(ct) - part, 0);
var page = await streamStore.ReadAllForwards(position,
readPageSize,
prefetchJsonData : false,
cancellationToken : ct);
count += page.Messages.Length;
if (page.Messages.Length < 2)
{
continue;
}
Output.WriteLine($"< from {position}");
for (var index = 1; index < page.Messages.Length; index++)
{
if (page.Messages[index].Position != page.Messages[index - 1].Position + 1)
{
Output.WriteLine(
$"< Gap found {page.Messages[index - 1].Position} and {page.Messages[index].Position}");
break;
}
}
} while(!ct.IsCancellationRequested);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Output.WriteLine(ex.ToString());
}
finally
{
stopwatch.Stop();
var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0);
Output.WriteLine("");
Output.WriteLine($"< {count} messages read {stopwatch.Elapsed} ({rate} m/s)");
}
}
private static async Task WriteActualGaps(CancellationToken ct, IStreamStore streamStore)
{
var stopwatch = Stopwatch.StartNew();
var count = 0;
Output.WriteLine("Actual gaps:");
var page = await streamStore.ReadAllForwards(Position.Start, 73, false, ct);
count += page.Messages.Length;
var prevPosition = page.Messages[0].Position;
for (int i = 1; i < page.Messages.Length; i++)
{
if (prevPosition + 1 != page.Messages[i].Position)
{
Output.WriteLine($"- {prevPosition} : {page.Messages[i].Position}");
}
prevPosition = page.Messages[i].Position;
}
while (!page.IsEnd)
{
page = await page.ReadNext(ct);
count += page.Messages.Length;
for (int i = 0; i < page.Messages.Length; i++)
{
if (prevPosition + 1 != page.Messages[i].Position)
{
Output.WriteLine($"- {prevPosition} : {page.Messages[i].Position}");
}
prevPosition = page.Messages[i].Position;
}
}
stopwatch.Stop();
var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0);
Output.WriteLine("");
Output.WriteLine($"< {count} messages read {stopwatch.Elapsed} ({rate} m/s)");
}
private async Task <RecordedEvent[]> ReadThens(long position)
{
var recorded = new List <RecordedEvent>();
var page = await _store.ReadAllForwards(position, 1024);
foreach (var then in page.Messages)
{
recorded.Add(
new RecordedEvent(
new StreamName(then.StreamId),
JsonConvert.DeserializeObject(
await then.GetJsonData(),
_mapping.GetEventType(then.Type),
_settings
)
)
);
}
while (!page.IsEnd)
{
page = await page.ReadNext();
foreach (var then in page.Messages)
{
recorded.Add(
new RecordedEvent(
new StreamName(then.StreamId),
JsonConvert.DeserializeObject(
await then.GetJsonData(),
_mapping.GetEventType(then.Type),
_settings
)
)
);
}
}
return(recorded.ToArray());
}
private async Task <IReadOnlyCollection <RecordedEvent> > ReadThens(long position, CancellationToken cancellationToken)
{
var recorded = new List <RecordedEvent>();
var page = await _store.ReadAllForwards(position, 1024, cancellationToken : cancellationToken);
foreach (var then in page.Messages)
{
recorded.Add(
new RecordedEvent(
new StreamName(then.StreamId),
JsonSerializer.Deserialize(
await then.GetJsonData(cancellationToken),
_messageTypeResolver(then.Type),
_options
)
)
);
}
while (!page.IsEnd)
{
page = await page.ReadNext(cancellationToken);
foreach (var then in page.Messages)
{
recorded.Add(
new RecordedEvent(
new StreamName(then.StreamId),
JsonSerializer.Deserialize(
await then.GetJsonData(cancellationToken),
_messageTypeResolver(then.Type),
_options
)
)
);
}
}
return(recorded.ToArray());
}
public async Task <List <IEvent> > GetEventsFromStreamAsync(string stream, int?startFromVersion, CancellationToken cancellationToken = default)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
const int _PAGE_SIZE = 10;
var page = await _streamStore.ReadAllForwards(Position.Start, _PAGE_SIZE, cancellationToken : cancellationToken);
var messages = new List <StreamMessage>(page.Messages);
while (!page.IsEnd) //should not take more than 20 iterations.
{
page = await page.ReadNext(cancellationToken);
messages.AddRange(page.Messages);
}
var result = new List <IEvent>();
foreach (var sm in messages)
{
var metadata = _serDes.Deserialize <EventMetadata>(sm.JsonMetadata);
var eventType = metadata.GetEventType();
var data = await sm.GetJsonData(cancellationToken);
var @event = _serDes.Deserialize(data, eventType) as IEvent;
result.Add(@event);
}
stopWatch.Stop();
_logger.LogDebug("SqlStreamStore.GetEventsFromStreamAsync for {Stream} took {ElapsedMilliseconds} ms", stream, stopWatch.ElapsedMilliseconds);
return(result);
}
public EventProcessor(
IStreamStore streamStore,
AcceptStreamMessageFilter filter,
EnvelopeFactory envelopeFactory,
ConnectedProjectionHandlerResolver <EditorContext> resolver,
Func <EditorContext> dbContextFactory,
Scheduler scheduler,
ILogger <EventProcessor> logger)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
if (envelopeFactory == null)
{
throw new ArgumentNullException(nameof(envelopeFactory));
}
if (resolver == null)
{
throw new ArgumentNullException(nameof(resolver));
}
if (dbContextFactory == null)
{
throw new ArgumentNullException(nameof(dbContextFactory));
}
_scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_messagePumpCancellation = new CancellationTokenSource();
_messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = false
});
_messagePump = Task.Factory.StartNew(async() =>
{
IAllStreamSubscription subscription = null;
try
{
logger.LogInformation("EventProcessor message pump entered ...");
while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
{
while (_messageChannel.Reader.TryRead(out var message))
{
switch (message)
{
case Resume _:
logger.LogInformation("Resuming ...");
await using (var resumeContext = dbContextFactory())
{
var projection =
await resumeContext.ProjectionStates
.SingleOrDefaultAsync(
item => item.Name == RoadRegistryEditorProjectionHost,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
var after = projection?.Position;
var head = await streamStore.ReadHeadPosition();
if (head == Position.Start || (after.HasValue
? head - after.Value <= CatchUpThreshold
: head - CatchUpThreshold <= 0))
{
await _messageChannel.Writer
.WriteAsync(new Subscribe(after), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
else
{
await _messageChannel.Writer
.WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
break;
case CatchUp catchUp:
logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L);
var observedMessageCount = 0;
var catchUpPosition = catchUp.AfterPosition ?? Position.Start;
var context = dbContextFactory();
var page = await streamStore
.ReadAllForwards(
catchUpPosition,
catchUp.BatchSize,
true,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
while (!page.IsEnd)
{
foreach (var streamMessage in page.Messages)
{
if (catchUp.AfterPosition.HasValue &&
streamMessage.Position == catchUp.AfterPosition.Value)
{
continue; // skip already processed message
}
if (filter(streamMessage))
{
logger.LogInformation("Catching up on {MessageType} at {Position}",
streamMessage.Type, streamMessage.Position);
var envelope = envelopeFactory.Create(streamMessage);
var handlers = resolver(envelope);
foreach (var handler in handlers)
{
await handler
.Handler(context, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
observedMessageCount++;
catchUpPosition = streamMessage.Position;
if (observedMessageCount % CatchUpBatchSize == 0)
{
logger.LogInformation(
"Flushing catch up position of {0} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
await context.DisposeAsync().ConfigureAwait(false);
context = dbContextFactory();
observedMessageCount = 0;
}
}
page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false);
}
if (observedMessageCount > 0) // case where we just read the last page and pending work in memory needs to be flushed
{
logger.LogInformation(
"Flushing catch up position of {Position} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
await context.DisposeAsync().ConfigureAwait(false);
//switch to subscription as of the last page
await _messageChannel.Writer
.WriteAsync(
new Subscribe(catchUpPosition),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
break;
case Subscribe subscribe:
logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L);
subscription?.Dispose();
subscription = streamStore.SubscribeToAll(
subscribe.AfterPosition, async(_, streamMessage, token) =>
{
if (filter(streamMessage))
{
logger.LogInformation("Observing {0} at {1}", streamMessage.Type,
streamMessage.Position);
var command = new ProcessStreamMessage(streamMessage);
await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false);
await command.Completion.ConfigureAwait(false);
}
else if (streamMessage.Position % RecordPositionThreshold == 0 &&
!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new RecordPosition(streamMessage), token)
.ConfigureAwait(false);
}
}, async(_, reason, exception) =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(
new SubscriptionDropped(reason, exception),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
}
},
prefetchJsonData: false,
name: "RoadRegistry.Editor.ProjectionHost.EventProcessor");
break;
case RecordPosition record:
try
{
logger.LogInformation("Recording position of {MessageType} at {Position}.",
record.Message.Type, record.Message.Position);
await using (var recordContext = dbContextFactory())
{
await recordContext
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
record.Message.Position, _messagePumpCancellation.Token)
.ConfigureAwait(false);
await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
}
break;
case ProcessStreamMessage process:
try
{
logger.LogInformation("Processing {MessageType} at {Position}",
process.Message.Type, process.Message.Position);
var envelope = envelopeFactory.Create(process.Message);
var handlers = resolver(envelope);
await using (var processContext = dbContextFactory())
{
foreach (var handler in handlers)
{
await handler
.Handler(processContext, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
await processContext.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
process.Message.Position,
_messagePumpCancellation.Token).ConfigureAwait(false);
await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
process.Complete();
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
// how are we going to recover from this? do we even need to recover from this?
// prediction: it's going to be a serialization error, a data quality error, or a bug
process.Fault(exception);
}
break;
case SubscriptionDropped dropped:
if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a stream store error");
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a subscriber error");
if (dropped.Exception != null &&
dropped.Exception is SqlException sqlException &&
sqlException.Number == -2 /* timeout */)
{
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
}
break;
}
}
}
}
catch (TaskCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (OperationCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (Exception exception)
{
logger.LogError(exception, "EventProcessor message pump is exiting due to a bug");
}
finally
{
subscription?.Dispose();
}
}, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
}
public Task <ReadAllPage> Invoke(IStreamStore streamStore, CancellationToken ct)
=> ReadDirection == Constants.ReadDirection.Forwards
? streamStore.ReadAllForwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct)
: streamStore.ReadAllBackwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct);
public async Task <StreamMessage> Invoke(IStreamStore streamStore, CancellationToken ct)
{
var page = await streamStore.ReadAllForwards(Position, 1, true, ct);
return(page.Messages.Where(m => m.Position == Position).FirstOrDefault());
}
