private void CreateSubscription(IReadonlyStreamStore streamStore, Func <Owned <TContext> > contextFactory, long?position)
{
_subscriptions.Add(
streamStore.SubscribeToAll(
name: RunnerName,
continueAfterPosition: position,
streamMessageReceived: async(subscription, message, cancellationToken) =>
{
_logger.LogTrace(
"[{Latency}] [POS {Position}] [{StreamId}] [{Type}]",
DateTime.UtcNow - message.CreatedUtc, // This is not very precise since we could have differing clocks, and should be seen as merely informational
message.Position,
message.StreamId,
message.Type);
using (var context = contextFactory().Value)
{
await context.UpdateProjectionState(RunnerName, message.Position, cancellationToken);
await _projector.ProjectAsync(context, _envelopeFactory.Create(message), cancellationToken);
await context.SaveChangesAsync(cancellationToken);
}
},
subscriptionDropped: async(subscription, reason, exception) =>
{
_logger.LogWarning(exception, "Subscription {SubscriptionName} was dropped. Reason: {Reason}", subscription.Name, reason);
await Task.Delay(TimeSpan.FromSeconds(5));
// When Handle throws, we will no longer be subscribed (and no subsequent subscribe will be attempted).
Handle(streamStore, contextFactory);
}));
}
public static IApplicationBuilder UseHealthProbe(
this IApplicationBuilder builder, IReadonlyStreamStore streamStore)
=> builder
.Use(GetAndHeadOnly)
.UseRouter(router => router
.MapMiddlewareGet("ready", inner => inner.Use(Ready(streamStore)))
.MapMiddlewareGet("live", inner => inner.Use(Live)));
public AllStreamSubscription(
long?continueAfterPosition,
IReadonlyStreamStore readonlyStreamStore,
IObservable <Unit> streamStoreAppendedNotification,
AllStreamMessageReceived streamMessageReceived,
AllSubscriptionDropped subscriptionDropped,
HasCaughtUp hasCaughtUp,
bool prefetchJsonData,
string name)
{
FromPosition = continueAfterPosition;
LastPosition = continueAfterPosition;
_nextPosition = continueAfterPosition + 1 ?? Position.Start;
_readonlyStreamStore = readonlyStreamStore;
_streamMessageReceived = streamMessageReceived;
_prefetchJsonData = prefetchJsonData;
_subscriptionDropped = subscriptionDropped ?? ((_, __, ___) => { });
_hasCaughtUp = hasCaughtUp ?? (_ => { });
Name = string.IsNullOrWhiteSpace(name) ? Guid.NewGuid().ToString() : name;
readonlyStreamStore.OnDispose += ReadonlyStreamStoreOnOnDispose;
_notification = streamStoreAppendedNotification.Subscribe(_ =>
{
_streamStoreNotification.Set();
});
Task.Run(PullAndPush);
s_logger.Info(
"AllStream subscription created {name} continuing after position {position}",
Name,
continueAfterPosition?.ToString() ?? "<null>");
}
public StreamSubscription(
string streamId,
int?continueAfterVersion,
IReadonlyStreamStore readonlyStreamStore,
IObservable <Unit> streamStoreAppendedNotification,
StreamMessageReceived streamMessageReceived,
SubscriptionDropped subscriptionDropped,
HasCaughtUp hasCaughtUp,
bool prefectchJsonData,
string name)
{
StreamId = streamId;
_continueAfterVersion = continueAfterVersion;
_readonlyStreamStore = readonlyStreamStore;
_streamMessageReceived = streamMessageReceived;
_prefectchJsonData = prefectchJsonData;
_subscriptionDropped = subscriptionDropped ?? ((_, __, ___) => { });
_hasCaughtUp = hasCaughtUp ?? ((_) => { });
Name = string.IsNullOrWhiteSpace(name) ? Guid.NewGuid().ToString() : name;
readonlyStreamStore.OnDispose += ReadonlyStreamStoreOnOnDispose;
_notification = streamStoreAppendedNotification.Subscribe(_ =>
{
_streamStoreNotification.Set();
});
Task.Run(PullAndPush);
s_logger.Info($"Stream subscription created {Name} continuing after version " +
$"{continueAfterVersion?.ToString() ?? "<null>"}.");
}
public AllStreamResource(IReadonlyStreamStore streamStore)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
_streamStore = streamStore;
}
public ConnectedProjectionsStreamStoreSubscription(
IReadonlyStreamStore streamStore,
IConnectedProjectionsCommandBus commandBus,
ILoggerFactory loggerFactory)
{
_streamStore = streamStore ?? throw new ArgumentNullException(nameof(streamStore));
_commandBus = commandBus ?? throw new ArgumentNullException(nameof(commandBus));
_logger = loggerFactory?.CreateLogger <ConnectedProjectionsStreamStoreSubscription>() ?? throw new ArgumentNullException(nameof(loggerFactory));
}
public PollingStreamStoreNotifier(IReadonlyStreamStore readonlyStreamStore, int interval = 1000)
{
_readonlyStreamStore = readonlyStreamStore;
_timer = new Timer(interval)
{
AutoReset = false
};
_timer.Elapsed += (_, __) => Poll().SwallowException();
}
public MetadataMaxAgeCache(IReadonlyStreamStore store, TimeSpan expiration, int maxSize, GetUtcNow getUtcNow)
{
Ensure.That(store, nameof(store)).IsNotNull();
Ensure.That(maxSize).IsGte(0);
Ensure.That(getUtcNow).IsNotNull();
_store = store;
_expiration = expiration;
_maxSize = maxSize;
_getUtcNow = getUtcNow;
}
/// <summary>
/// Reads messages from all streams backwards.
/// </summary>
/// <param name="readonlyStreamStore">
/// The stream store instance.
/// </param>
/// <param name="fromPositionInclusive">
/// The position to start reading from. Use <see cref="Position.Start"/> to start from the beginning.
/// Note: messages that have expired will be filtered out.
/// </param>
/// <param name="maxCount">
/// The maximum number of messages to read (int.MaxValue is a bad idea).
/// </param>
/// <param name="cancellationToken">
/// The cancellation instruction.
/// </param>
/// <returns>
/// An <see cref="ReadAllPage"/> presenting the result of the read. If all messages read have expired
/// then the message collection MAY be empty.
/// </returns>
public static Task <ReadAllPage> ReadAllBackwards(
this IReadonlyStreamStore readonlyStreamStore,
long fromPositionInclusive,
int maxCount,
CancellationToken cancellationToken = default(CancellationToken))
{
return(readonlyStreamStore.ReadAllBackwards(
fromPositionInclusive,
maxCount,
cancellationToken: cancellationToken));
}
public DefaultCatchUpStreamGapStrategy(
ILoggerFactory loggerFactory,
IStreamGapStrategyConfigurationSettings settings,
IReadonlyStreamStore streamStore,
IClock clock)
{
Settings = settings ?? throw new ArgumentNullException(nameof(settings));
_streamStore = streamStore ?? throw new ArgumentNullException(nameof(streamStore));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_logger = loggerFactory?.CreateLogger <DefaultCatchUpStreamGapStrategy>() ?? throw new ArgumentNullException(nameof(loggerFactory));
}
private async Task <ReadAllPage> ReadPages(
IReadonlyStreamStore streamStore,
long?position,
CancellationToken cancellationToken)
{
return(await streamStore.ReadAllForwards(
position + 1 ?? Position.Start,
_settings.CatchUpPageSize,
prefetchJsonData : true,
cancellationToken));
}
public ConnectedProjectionCatchUp <TContext> CreateCatchUp(
IReadonlyStreamStore streamStore,
IConnectedProjectionsCommandBus commandBus,
IStreamGapStrategy catchUpStreamGapStrategy,
ILogger logger)
=> new ConnectedProjectionCatchUp <TContext>(
this,
_settings,
streamStore,
commandBus,
catchUpStreamGapStrategy,
logger);
public AllStreamSubscription(
long?fromPosition,
IReadonlyStreamStore readonlyStreamStore,
IObservable <Unit> streamStoreAppendedNotification,
StreamMessageReceived streamMessageReceived,
SubscriptionDropped subscriptionDropped = null,
string name = null)
: base(readonlyStreamStore, streamStoreAppendedNotification, streamMessageReceived, subscriptionDropped, name)
{
FromPosition = fromPosition;
LastPosition = fromPosition;
_nextPosition = fromPosition + 1 ?? Position.Start;
}
protected SubscriptionBase(
IReadonlyStreamStore readonlyStreamStore,
IObservable <Unit> streamStoreAppendedNotification,
StreamMessageReceived streamMessageReceived,
SubscriptionDropped subscriptionDropped = null,
string name = null)
{
ReadonlyStreamStore = readonlyStreamStore;
StreamStoreAppendedNotification = streamStoreAppendedNotification;
StreamMessageReceived = streamMessageReceived;
Name = string.IsNullOrWhiteSpace(name) ? Guid.NewGuid().ToString() : name;
SubscriptionDropped = subscriptionDropped ?? ((_, __) => { });
}
/// <summary>
/// Reads messages from a stream forwards.
/// </summary>
/// <param name="readonlyStreamStore">
/// The stream store instance.
/// </param>
/// <param name="streamId">
/// The stream ID to read.
/// </param>
/// <param name="fromVersionInclusive">
/// The version of the stream to start reading from. Use <see cref="StreamVersion.Start"/> to read from
/// the start.
/// </param>
/// <param name="maxCount">
/// The maximum number of messages to read (int.MaxValue is a bad idea).
/// </param>
/// <param name="cancellationToken">
/// The cancellation instruction.
/// </param>
/// <returns>
/// An <see cref="ReadStreamPage"/> represent the result of the operation. If all the messages read
/// have expired then the message collection MAY be empty.
/// </returns>
public static Task <ReadStreamPage> ReadStreamForwards(
this IReadonlyStreamStore readonlyStreamStore,
string streamId,
int fromVersionInclusive,
int maxCount,
CancellationToken cancellationToken = default(CancellationToken))
{
return(readonlyStreamStore.ReadStreamForwards(
streamId,
fromVersionInclusive,
maxCount,
cancellationToken: cancellationToken));
}
public StreamSubscription(
string streamId,
int startVersion,
IReadonlyStreamStore readonlyStreamStore,
IObservable <Unit> streamStoreAppendedNotification,
StreamMessageReceived streamMessageReceived,
SubscriptionDropped subscriptionDropped,
string name = null)
: base(readonlyStreamStore, streamStoreAppendedNotification, streamMessageReceived, subscriptionDropped, name)
{
_streamId = streamId;
_nextVersion = startVersion;
_lastVersion = startVersion - 1;
}
private static MidFunc Ready(IReadonlyStreamStore streamStore) => async(context, next) =>
{
try
{
await streamStore.ReadHeadPosition(context.RequestAborted);
}
catch
{
context.Response.StatusCode = 503;
return;
}
context.Response.StatusCode = 204;
};
public ConnectedProjectionCatchUp(
ConnectedProjectionName name,
IReadonlyStreamStore streamStore,
Func <Owned <TContext> > contextFactory,
ConnectedProjectionMessageHandler <TContext> messageHandler,
IConnectedProjectionsCommandBus commandBus,
ILogger logger)
{
_runnerName = name ?? throw new ArgumentNullException(nameof(name));
_streamStore = streamStore ?? throw new ArgumentNullException(nameof(streamStore));
_contextFactory = contextFactory ?? throw new ArgumentNullException(nameof(contextFactory));
_messageHandler = messageHandler ?? throw new ArgumentNullException(nameof(messageHandler));
_commandBus = commandBus ?? throw new ArgumentNullException(nameof(commandBus));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public ConnectedProjectionsCatchUpRunner(
IRegisteredProjections registeredProjections,
IReadonlyStreamStore streamStore,
IConnectedProjectionsCommandBus commandBus,
ILoggerFactory loggerFactory)
{
_projectionCatchUps = new Dictionary <ConnectedProjectionName, CancellationTokenSource>();
_registeredProjections = registeredProjections ?? throw new ArgumentNullException(nameof(registeredProjections));
_registeredProjections.IsCatchingUp = IsCatchingUp;
_streamStore = streamStore ?? throw new ArgumentNullException(nameof(streamStore));
_commandBus = commandBus ?? throw new ArgumentNullException(nameof(commandBus));
_logger = loggerFactory?.CreateLogger <ConnectedProjectionsCatchUpRunner>() ?? throw new ArgumentNullException(nameof(loggerFactory));
}
public ConnectedProjectionCatchUp(
IConnectedProjection <TContext> projection,
IConnectedProjectionCatchUpSettings settings,
IReadonlyStreamStore streamStore,
IConnectedProjectionsCommandBus commandBus,
IStreamGapStrategy catchUpStreamGapStrategy,
ILogger logger)
{
_projection = projection ?? throw new ArgumentNullException(nameof(projection));
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
_streamStore = streamStore ?? throw new ArgumentNullException(nameof(streamStore));
_commandBus = commandBus ?? throw new ArgumentNullException(nameof(commandBus));
_catchUpStreamGapStrategy = catchUpStreamGapStrategy ?? throw new ArgumentNullException(nameof(catchUpStreamGapStrategy));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public static MidFunc UseStreamStore(IReadonlyStreamStore streamStore)
{
var streams = new StreamResource(streamStore);
var builder = new AppBuilder()
.MapWhen(context => context.Request.Path.Value?.Split('/')?.Length == 3,
inner => inner.Use(GetStreamMessage(streams)))
.MapWhen(context => context.Request.Path.Value?.Length > 1, inner => inner.Use(GetStream(streams)));
return(next =>
{
builder.Run(ctx => next(ctx.Environment));
return builder.Build();
});
}
public void Handle(IReadonlyStreamStore streamStore, Func <Owned <TContext> > contextFactory)
{
CleanUp();
long?position;
using (var context = contextFactory().Value)
{
position = context
.ProjectionStates
.SingleOrDefault(p => p.Name == RunnerName)
?.Position;
}
_tasks.Add(
Task.Factory.StartNew(
() => CreateSubscription(streamStore, contextFactory, position),
TaskCreationOptions.LongRunning));
}
public static MidFunc UseSqlStreamStoreHal(IReadonlyStreamStore streamStore)
{
if (streamStore == null)
{
throw new ArgumentNullException(nameof(streamStore));
}
var builder = new AppBuilder()
.Use(AccessControl)
.Use(Index)
.Map("/stream", inner => inner.Use(AllStreamMiddleware.UseStreamStore(streamStore)))
.Map("/streams", inner => inner.Use(StreamMiddleware.UseStreamStore(streamStore)));
return(next =>
{
builder.Run(ctx => next(ctx.Environment));
return builder.Build();
});
}
public static MidFunc UseStreamStore(IReadonlyStreamStore streamStore)
{
var allStream = new AllStreamResource(streamStore);
var builder = new AppBuilder()
.MapWhen(context => !context.Request.Path.HasValue, inner => inner.Use(GetStream(allStream)))
.MapWhen(context =>
{
long _;
return(long.TryParse(context.Request.Path.Value?.Remove(0, 1), out _));
},
inner => inner.Use(GetStreamMessage(allStream)));
return(next =>
{
builder.Run(ctx => next(ctx.Environment));
return builder.Build();
});
}
public PollingStreamStoreNotifier(IReadonlyStreamStore readonlyStreamStore, int interval = 1000)
: this(readonlyStreamStore.ReadHeadPosition, interval)
{
}
private async Task CatchUpAsync(IReadonlyStreamStore streamStore, Func <Owned <TContext> > contextFactory, long?position, CancellationToken cancellationToken)
{
long?positionOfLastMessageOnPage = null;
try
{
_logger.LogDebug(
"CatchupThreshold ({CatchupThreshold}) exceeded, catching up with paging (CatchupPageSize: {CatchupPageSize})",
CatchupThreshold,
CatchupPageSize);
var fromPositionInclusive = position ?? global::SqlStreamStore.Streams.Position.Start;
var page = await streamStore.ReadAllForwards(fromPositionInclusive, CatchupPageSize, true, cancellationToken);
_logger.LogDebug(
"Processing page of {PageSize} starting at POS {FromPosition}",
page.Messages.Length,
page.FromPosition);
using (var context = contextFactory().Value)
{
foreach (var message in page.Messages)
{
if (message.Position == fromPositionInclusive && message.Position != global::SqlStreamStore.Streams.Position.Start)
{
continue; // because fromPosition should be exclusive
}
positionOfLastMessageOnPage = message.Position;
_logger.LogTrace(
"[{Latency}] [POS {Position}] [{StreamId}] [{Type}]",
DateTime.UtcNow - message.CreatedUtc, // This is not very precise since we could have differing clocks, and should be seen as merely informational
message.Position,
message.StreamId,
message.Type);
if (_envelopeFactory.TryCreate(message, out var envelope))
{
await _projector.ProjectAsync(context, envelope, cancellationToken);
}
else
{
_logger.LogDebug(
"Skipping message {Type} at position {Position} in stream {Stream}@{Version} because it does not appear in the event mapping",
message.Type,
message.Position,
message.StreamId,
message.StreamVersion);
}
}
if (positionOfLastMessageOnPage.HasValue)
{
await context.UpdateProjectionState(RunnerName, positionOfLastMessageOnPage.Value, cancellationToken);
}
await context.SaveChangesAsync(cancellationToken);
}
while (!page.IsEnd)
{
page = await page.ReadNext(cancellationToken);
_logger.LogDebug(
"Processing page of {PageSize} starting at POS {FromPosition}",
page.Messages.Length,
page.FromPosition);
using (var context = contextFactory().Value)
{
positionOfLastMessageOnPage = null;
foreach (var message in page.Messages)
{
positionOfLastMessageOnPage = message.Position;
_logger.LogTrace(
"[{Latency}] [POS {Position}] [{StreamId}] [{Type}]",
DateTime.UtcNow - message.CreatedUtc, // This is not very precise since we could have differing clocks, and should be seen as merely informational
message.Position,
message.StreamId,
message.Type);
if (_envelopeFactory.TryCreate(message, out var envelope))
{
await _projector.ProjectAsync(context, envelope, cancellationToken);
}
else
{
_logger.LogDebug(
"Skipping message {Type} at position {Position} in stream {Stream}@{Version} because it does not appear in the event mapping",
message.Type,
message.Position,
message.StreamId,
message.StreamVersion);
}
}
if (positionOfLastMessageOnPage.HasValue)
{
await context.UpdateProjectionState(RunnerName, positionOfLastMessageOnPage.Value, cancellationToken);
}
await context.SaveChangesAsync(cancellationToken);
}
}
await StartAsyncInternal(streamStore, contextFactory, cancellationToken); // subscribe when done catching up
}
catch (Exception exception)
{
_logger.LogWarning(
exception,
"{RunnerName} catching up failed because an exception was thrown when handling the message at {Position}.",
RunnerName,
positionOfLastMessageOnPage ?? -1L);
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
await StartAsyncInternal(streamStore, contextFactory, cancellationToken);
}
}
private async Task StartAsyncInternal(
IReadonlyStreamStore streamStore,
Func <Owned <TContext> > contextFactory,
CancellationToken cancellationToken)
{
CleanUp();
// discover runner position
long?position;
using (var context = contextFactory().Value)
{
var dbPosition = await context
.ProjectionStates
.SingleOrDefaultAsync(p => p.Name == RunnerName, cancellationToken);
position = dbPosition?.Position;
}
// discover head position
var head = await streamStore.ReadHeadPosition(cancellationToken);
// determine whether to play catch up or start subscribing
var shouldCatchUp = position.HasValue
? head - position.Value > CatchupThreshold
: head + 1 > CatchupThreshold; // head+1 because first position is 0
if (shouldCatchUp)
{
_tasks.Add(CatchUpAsync(streamStore, contextFactory, position, cancellationToken));
}
else
{
_subscriptions.Add(
streamStore.SubscribeToAll(
name: RunnerName,
continueAfterPosition: position,
streamMessageReceived: async(subscription, message, ct) =>
{
_logger.LogTrace(
"[{Latency}] [POS {Position}] [{StreamId}] [{Type}]",
DateTime.UtcNow - message.CreatedUtc, // This is not very precise since we could have differing clocks, and should be seen as merely informational
message.Position,
message.StreamId,
message.Type);
if (_envelopeFactory.TryCreate(message, out var envelope))
{
using (var context = contextFactory().Value)
{
await context.UpdateProjectionState(RunnerName, message.Position, cancellationToken);
await _projector.ProjectAsync(context, envelope, cancellationToken);
await context.SaveChangesAsync(cancellationToken);
}
}
else
{
_logger.LogDebug(
"Skipping message {Type} at position {Position} in stream {Stream}@{Version} because it does not appear in the event mapping",
message.Type,
message.Position,
message.StreamId,
message.StreamVersion);
}
},
subscriptionDropped: async(subscription, reason, exception) =>
{
_logger.LogWarning(exception, "Subscription {SubscriptionName} was dropped. Reason: {Reason}", subscription.Name, reason);
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
await StartAsyncInternal(streamStore, contextFactory, cancellationToken);
}));
}
}
// async handling with catchup
public async Task StartAsync(IReadonlyStreamStore streamStore, Func <Owned <TContext> > contextFactory, CancellationToken cancellationToken = default)
{
cancellationToken.Register(CleanUp);
await StartAsyncInternal(streamStore, contextFactory, cancellationToken);
}
public ConnectedProjectionCatchUp <FakeProjectionContext> CreateCatchUp(
IReadonlyStreamStore streamStore,
IConnectedProjectionsCommandBus commandBus,
IStreamGapStrategy catchUpStreamGapStrategy,
ILogger logger)
=> throw new NotImplementedException();
public StreamLogger(IReadonlyStreamStore store)
{
_store = store;
}
