public async Task Bomb()
{
var current = await _streamStore.ReadHeadPosition();
var amountOfEvents = MaxAmountOfEvents - (current + 1);
while (amountOfEvents > 0)
{
var newBatch = amountOfEvents > 2500 ? 2500: (int)amountOfEvents;
var newEvents = Enumerable.Range(0, newBatch).Select((_, i) =>
new
{
Index = i,
Message = new NewStreamMessage(Guid.NewGuid(), "RandomEvent", JsonSerializer.Serialize(new Event
{
Random = Rand.NextDouble()
}))
}).ToArray();
var tasks = newEvents
.GroupBy(x => x.Index / 50)
.Select(x => x.Select(y => y.Message).ToArray())
.Select(x =>
_streamStore.AppendToStream($"{StreamName}/{Guid.NewGuid()}", ExpectedVersion.Any, x));
await Task.WhenAll(tasks);
amountOfEvents -= newBatch;
Console.WriteLine($"Still need to add: {amountOfEvents} events");
}
}
public static async Task WaitUntilAvailable(this IStreamStore store, ILogger <Program> logger, CancellationToken cancellationToken = default)
{
if (store is MsSqlStreamStore)
{
var watch = Stopwatch.StartNew();
var exit = false;
while (!exit)
{
try
{
if (logger.IsEnabled(LogLevel.Information))
{
logger.LogInformation($"Waiting until sql stream store becomes available ... ({watch.Elapsed:c})");
}
await store.ReadHeadPosition(cancellationToken);
exit = true;
}
catch (Exception exception)
{
if (logger.IsEnabled(LogLevel.Warning))
{
logger.LogWarning(exception, "Encountered an exception while waiting for sql stream store to become available.");
}
await Task.Delay(1000, cancellationToken);
}
}
}
}
public async Task <IActionResult> Delete(
[FromServices] AddressCrabEditClient editClient,
[FromServices] IStreamStore streamStore,
[FromServices] LegacyContext context,
[FromRoute] string lokaleIdentificator,
CancellationToken cancellationToken)
{
// TODO: Turn this into proper VBR API Validation
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
var persistentLocalId = int.Parse(lokaleIdentificator);
// todo: should become position from bus.Dispatch
var position = await streamStore.ReadHeadPosition(cancellationToken);
var addressId = context
.CrabIdToPersistentLocalIds
.SingleOrDefault(item => item.PersistentLocalId == persistentLocalId);
CrabEditResponse deleteResponse = addressId != null
? await editClient.Delete(addressId, cancellationToken)
: CrabEditResponse.NothingExecuted;
return(AcceptedWithPosition(
position,
deleteResponse.ExecutionTime));
}
/// <inheritdoc />
public async Task Wait()
{
var position = await _store.ReadHeadPosition();
try
{
await _position.Timeout(Configuration.Timeout).FirstAsync(p => p == position);
}
catch
{
await Populate();
}
}
private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize)
{
int count = 0;
var stopwatch = Stopwatch.StartNew();
try
{
// var position = Position.Start;
var part = readPageSize / 5;
do
{
var position = Math.Max(await streamStore.ReadHeadPosition(ct) - part, 0);
var page = await streamStore.ReadAllForwards(position,
readPageSize,
prefetchJsonData : false,
cancellationToken : ct);
count += page.Messages.Length;
if (page.Messages.Length < 2)
{
continue;
}
Output.WriteLine($"< from {position}");
for (var index = 1; index < page.Messages.Length; index++)
{
if (page.Messages[index].Position != page.Messages[index - 1].Position + 1)
{
Output.WriteLine(
$"< Gap found {page.Messages[index - 1].Position} and {page.Messages[index].Position}");
break;
}
}
} while(!ct.IsCancellationRequested);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
Output.WriteLine(ex.ToString());
}
finally
{
stopwatch.Stop();
var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0);
Output.WriteLine("");
Output.WriteLine($"< {count} messages read {stopwatch.Elapsed} ({rate} m/s)");
}
}
public async Task <IActionResult> Get(
[FromServices] IStreamStore streamStore,
CancellationToken cancellationToken)
{
var registeredConnectedProjections = _projectionManager
.GetRegisteredProjections()
.ToList();
var responses = await CreateProjectionResponses(registeredConnectedProjections, cancellationToken);
var streamPosition = await streamStore.ReadHeadPosition(cancellationToken);
return(Ok(new ProjectionResponseList(responses, _baseUri)
{
StreamPosition = streamPosition
}));
}
public async Task <IActionResult> Correct(
[FromServices] AddressCrabEditClient editClient,
[FromServices] LegacyContext context,
[FromServices] IStreamStore streamStore,
[FromRoute] string lokaleIdentificator,
[FromBody] CorrectAddressRequest request,
CancellationToken cancellationToken)
{
// TODO: Turn this into proper VBR API Validation
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
var persistentLocalId = int.Parse(lokaleIdentificator);
// todo: should become position from bus.Dispatch
var position = await streamStore.ReadHeadPosition(cancellationToken);
var addressId = context
.CrabIdToPersistentLocalIds
.Single(item => item.PersistentLocalId == persistentLocalId);
CrabEditClientResult crabEditResult;
if (addressId.HouseNumberId.HasValue)
{
crabEditResult = await editClient.CorrectHouseNumber(addressId.HouseNumberId.Value, request, cancellationToken);
}
else if (addressId.SubaddressId.HasValue)
{
crabEditResult = await editClient.CorrectSubaddress(addressId.SubaddressId.Value, request, cancellationToken);
}
else
{
throw new InvalidOperationException();
}
return(AcceptedWithPosition(
position,
crabEditResult.ExecutionTime));
}
public async Task <long> PersistFacts(Fact[] facts)
{
var factsByAggregate = facts.GroupBy(x => x.Identifier);
AppendResult result = null;
foreach (var aggregateWithEvents in factsByAggregate)
{
result = await _streamStore.AppendToStream(
aggregateWithEvents.Key,
ExpectedVersion.Any,
aggregateWithEvents
.Select(e => new NewStreamMessage(
Guid.NewGuid(),
_eventMapping.GetEventName(e.Event.GetType()),
_eventSerializer.SerializeObject(e.Event))).ToArray());
}
return(result?.CurrentPosition ?? await _streamStore.ReadHeadPosition());
}
public EventProcessor(
IStreamStore streamStore,
AcceptStreamMessageFilter filter,
EnvelopeFactory envelopeFactory,
ConnectedProjectionHandlerResolver <EditorContext> resolver,
Func <EditorContext> dbContextFactory,
Scheduler scheduler,
ILogger <EventProcessor> logger)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
if (filter == null)
{
throw new ArgumentNullException(nameof(filter));
}
if (envelopeFactory == null)
{
throw new ArgumentNullException(nameof(envelopeFactory));
}
if (resolver == null)
{
throw new ArgumentNullException(nameof(resolver));
}
if (dbContextFactory == null)
{
throw new ArgumentNullException(nameof(dbContextFactory));
}
_scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_messagePumpCancellation = new CancellationTokenSource();
_messageChannel = Channel.CreateUnbounded <object>(new UnboundedChannelOptions
{
SingleReader = true,
SingleWriter = false,
AllowSynchronousContinuations = false
});
_messagePump = Task.Factory.StartNew(async() =>
{
IAllStreamSubscription subscription = null;
try
{
logger.LogInformation("EventProcessor message pump entered ...");
while (await _messageChannel.Reader.WaitToReadAsync(_messagePumpCancellation.Token).ConfigureAwait(false))
{
while (_messageChannel.Reader.TryRead(out var message))
{
switch (message)
{
case Resume _:
logger.LogInformation("Resuming ...");
await using (var resumeContext = dbContextFactory())
{
var projection =
await resumeContext.ProjectionStates
.SingleOrDefaultAsync(
item => item.Name == RoadRegistryEditorProjectionHost,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
var after = projection?.Position;
var head = await streamStore.ReadHeadPosition();
if (head == Position.Start || (after.HasValue
? head - after.Value <= CatchUpThreshold
: head - CatchUpThreshold <= 0))
{
await _messageChannel.Writer
.WriteAsync(new Subscribe(after), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
else
{
await _messageChannel.Writer
.WriteAsync(new CatchUp(after, CatchUpBatchSize), _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
break;
case CatchUp catchUp:
logger.LogInformation("Catching up as of {Position}", catchUp.AfterPosition ?? -1L);
var observedMessageCount = 0;
var catchUpPosition = catchUp.AfterPosition ?? Position.Start;
var context = dbContextFactory();
var page = await streamStore
.ReadAllForwards(
catchUpPosition,
catchUp.BatchSize,
true,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
while (!page.IsEnd)
{
foreach (var streamMessage in page.Messages)
{
if (catchUp.AfterPosition.HasValue &&
streamMessage.Position == catchUp.AfterPosition.Value)
{
continue; // skip already processed message
}
if (filter(streamMessage))
{
logger.LogInformation("Catching up on {MessageType} at {Position}",
streamMessage.Type, streamMessage.Position);
var envelope = envelopeFactory.Create(streamMessage);
var handlers = resolver(envelope);
foreach (var handler in handlers)
{
await handler
.Handler(context, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
}
observedMessageCount++;
catchUpPosition = streamMessage.Position;
if (observedMessageCount % CatchUpBatchSize == 0)
{
logger.LogInformation(
"Flushing catch up position of {0} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
await context.DisposeAsync().ConfigureAwait(false);
context = dbContextFactory();
observedMessageCount = 0;
}
}
page = await page.ReadNext(_messagePumpCancellation.Token).ConfigureAwait(false);
}
if (observedMessageCount > 0) // case where we just read the last page and pending work in memory needs to be flushed
{
logger.LogInformation(
"Flushing catch up position of {Position} and persisting changes ...",
catchUpPosition);
await context
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
catchUpPosition,
_messagePumpCancellation.Token)
.ConfigureAwait(false);
await context.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
await context.DisposeAsync().ConfigureAwait(false);
//switch to subscription as of the last page
await _messageChannel.Writer
.WriteAsync(
new Subscribe(catchUpPosition),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
break;
case Subscribe subscribe:
logger.LogInformation("Subscribing as of {0}", subscribe.AfterPosition ?? -1L);
subscription?.Dispose();
subscription = streamStore.SubscribeToAll(
subscribe.AfterPosition, async(_, streamMessage, token) =>
{
if (filter(streamMessage))
{
logger.LogInformation("Observing {0} at {1}", streamMessage.Type,
streamMessage.Position);
var command = new ProcessStreamMessage(streamMessage);
await _messageChannel.Writer.WriteAsync(command, token).ConfigureAwait(false);
await command.Completion.ConfigureAwait(false);
}
else if (streamMessage.Position % RecordPositionThreshold == 0 &&
!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(new RecordPosition(streamMessage), token)
.ConfigureAwait(false);
}
}, async(_, reason, exception) =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer
.WriteAsync(
new SubscriptionDropped(reason, exception),
_messagePumpCancellation.Token)
.ConfigureAwait(false);
}
},
prefetchJsonData: false,
name: "RoadRegistry.Editor.ProjectionHost.EventProcessor");
break;
case RecordPosition record:
try
{
logger.LogInformation("Recording position of {MessageType} at {Position}.",
record.Message.Type, record.Message.Position);
await using (var recordContext = dbContextFactory())
{
await recordContext
.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
record.Message.Position, _messagePumpCancellation.Token)
.ConfigureAwait(false);
await recordContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
}
break;
case ProcessStreamMessage process:
try
{
logger.LogInformation("Processing {MessageType} at {Position}",
process.Message.Type, process.Message.Position);
var envelope = envelopeFactory.Create(process.Message);
var handlers = resolver(envelope);
await using (var processContext = dbContextFactory())
{
foreach (var handler in handlers)
{
await handler
.Handler(processContext, envelope, _messagePumpCancellation.Token)
.ConfigureAwait(false);
}
await processContext.UpdateProjectionState(
RoadRegistryEditorProjectionHost,
process.Message.Position,
_messagePumpCancellation.Token).ConfigureAwait(false);
await processContext.SaveChangesAsync(_messagePumpCancellation.Token).ConfigureAwait(false);
}
process.Complete();
}
catch (Exception exception)
{
logger.LogError(exception, exception.Message);
// how are we going to recover from this? do we even need to recover from this?
// prediction: it's going to be a serialization error, a data quality error, or a bug
process.Fault(exception);
}
break;
case SubscriptionDropped dropped:
if (dropped.Reason == SubscriptionDroppedReason.StreamStoreError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a stream store error");
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
else if (dropped.Reason == SubscriptionDroppedReason.SubscriberError)
{
logger.LogError(dropped.Exception,
"Subscription was dropped because of a subscriber error");
if (dropped.Exception != null &&
dropped.Exception is SqlException sqlException &&
sqlException.Number == -2 /* timeout */)
{
await scheduler.Schedule(async token =>
{
if (!_messagePumpCancellation.IsCancellationRequested)
{
await _messageChannel.Writer.WriteAsync(new Resume(), token).ConfigureAwait(false);
}
}, ResubscribeAfter).ConfigureAwait(false);
}
}
break;
}
}
}
}
catch (TaskCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (OperationCanceledException)
{
logger.LogInformation("EventProcessor message pump is exiting due to cancellation");
}
catch (Exception exception)
{
logger.LogError(exception, "EventProcessor message pump is exiting due to a bug");
}
finally
{
subscription?.Dispose();
}
}, _messagePumpCancellation.Token, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
}
public long StoreVersion()
{
var headPosition = streamStore.ReadHeadPosition().GetAwaiter().GetResult();
return(headPosition);
}
public Task <long> ReadHeadPosition(CancellationToken cancellationToken = new CancellationToken())
{
return(_store.ReadHeadPosition(cancellationToken));
}
