public StatsSubscriber(EventStoreClient eventStoreClient, string subscriptionId,
ICheckpointStore checkpointStore, IEventSerializer eventSerializer,
IEnumerable <IEventHandler> eventHandlers, ILoggerFactory loggerFactory = null,
IEventFilter eventFilter = null, SubscriptionGapMeasure measure = null) : base(eventStoreClient,
subscriptionId, checkpointStore, eventSerializer, eventHandlers, loggerFactory, eventFilter, measure)
{
}
public ProjectionManager(IEventStoreConnection connection, ICheckpointStore checkpointStore,
params IProjection[] projections)
{
_connection = connection;
_checkpointStore = checkpointStore;
_projections = projections;
}
/// <summary>
/// Creates EventStoreDB catch-up subscription service for a given stream
/// </summary>
/// <param name="eventStoreClient">EventStoreDB gRPC client instance</param>
/// <param name="streamName">Name of the stream to receive events from</param>
/// <param name="subscriptionId">Subscription ID</param>
/// <param name="checkpointStore">Checkpoint store instance</param>
/// <param name="eventSerializer">Event serializer instance</param>
/// <param name="eventHandlers">Collection of event handlers</param>
/// <param name="loggerFactory">Optional: logger factory</param>
/// <param name="measure">Optional: gap measurement for metrics</param>
/// <param name="throwOnError"></param>
public StreamSubscription(
EventStoreClient eventStoreClient,
string streamName,
string subscriptionId,
ICheckpointStore checkpointStore,
IEnumerable <IEventHandler> eventHandlers,
IEventSerializer?eventSerializer = null,
ILoggerFactory?loggerFactory = null,
ISubscriptionGapMeasure?measure = null,
bool throwOnError = false
) : this(
eventStoreClient,
new StreamSubscriptionOptions {
StreamName = streamName,
SubscriptionId = subscriptionId,
ThrowOnError = throwOnError
},
checkpointStore,
eventHandlers,
eventSerializer,
loggerFactory,
measure
)
{
}
public async Task CheckpointStoreBasicTest()
{
ICheckpointStore checkpointStore = CheckpointStore.Create(new StoreProvider(new InMemoryDbStoreProvider()));
for (long i = 0; i < 10; i++)
{
var checkpointData = new CheckpointData(i);
await checkpointStore.SetCheckpointDataAsync($"Endpoint{i}", checkpointData, CancellationToken.None);
}
IDictionary <string, CheckpointData> allCheckpointData = await checkpointStore.GetAllCheckpointDataAsync(CancellationToken.None);
Assert.Equal(10, allCheckpointData.Count);
long counter = 0;
foreach (KeyValuePair <string, CheckpointData> checkpointValue in allCheckpointData)
{
Assert.Equal(counter, checkpointValue.Value.Offset);
Assert.Equal($"Endpoint{counter}", checkpointValue.Key);
counter++;
}
for (long i = 0; i < 10; i++)
{
CheckpointData checkpointData = await checkpointStore.GetCheckpointDataAsync($"Endpoint{i}", CancellationToken.None);
Assert.NotNull(checkpointData);
Assert.Equal(i, checkpointData.Offset);
}
}
/// <summary>
/// Creates EventStoreDB catch-up subscription service for a given stream
/// </summary>
/// <param name="client"></param>
/// <param name="checkpointStore">Checkpoint store instance</param>
/// <param name="options">Subscription options</param>
/// <param name="consumePipe"></param>
public StreamSubscription(
EventStoreClient client,
StreamSubscriptionOptions options,
ICheckpointStore checkpointStore,
ConsumePipe consumePipe
) : base(client, options, checkpointStore, consumePipe)
=> Ensure.NotEmptyString(options.StreamName);
public RebuildTask(ILogger log,
IDependencyResolver ioc,
IHistoryReader historyReader,
IVersioningRepository versioningRepository,
ICheckpointStore checkpoints,
CancellationToken?pause = null)
{
if (log == null)
{
throw new ArgumentNullException("log");
}
if (ioc == null)
{
throw new ArgumentNullException("ioc");
}
if (historyReader == null)
{
throw new ArgumentNullException("historyReader");
}
if (versioningRepository == null)
{
throw new ArgumentNullException("versioningRepository");
}
if (checkpoints == null)
{
throw new ArgumentNullException("checkpoints");
}
this.log = log;
this.ioc = ioc;
this.historyReader = historyReader;
this.versioningRepository = versioningRepository;
this.checkpoints = checkpoints;
this.running = pause ?? CancellationToken.None;
}
/// <summary>
/// Creates EventStoreDB catch-up subscription service for $all
/// </summary>
/// <param name="eventStoreClient"></param>
/// <param name="options"></param>
/// <param name="checkpointStore">Checkpoint store instance</param>
/// <param name="consumePipe"></param>
public AllStreamSubscription(
EventStoreClient eventStoreClient,
AllStreamSubscriptionOptions options,
ICheckpointStore checkpointStore,
ConsumePipe consumePipe
) : base(eventStoreClient, options, checkpointStore, consumePipe)
{
}
public CommitDispatcher(ICommandBus commandBus, IEventBus eventBus, ILogger logger, ICheckpointStore checkpoints)
{
this.commandBus = commandBus;
this.eventBus = eventBus;
this.logger = logger;
this.checkpoints = checkpoints;
eventBusPublish = new GenericMethodCaller(eventBus, "Publish");
commandBusPublish = new GenericMethodCaller(commandBus, "Publish");
}
public CommitDispatcher(ICommandBus commandBus, IEventBus eventBus, ILogger logger, ICheckpointStore checkpoints)
{
this.commandBus = commandBus;
this.eventBus = eventBus;
this.logger = logger;
this.checkpoints = checkpoints;
eventBusPublish = new GenericMethodCaller(eventBus, "Publish");
commandBusPublish = new GenericMethodCaller(commandBus, "Publish");
}
MasterCheckpointer(string id, ICheckpointStore store, long offset)
{
this.Id = Preconditions.CheckNotNull(id);
this.store = Preconditions.CheckNotNull(store);
this.Offset = offset;
this.Proposed = offset;
this.closed = new AtomicBoolean(false);
this.childCheckpointers = new AtomicReference <ImmutableDictionary <string, ICheckpointer> >(ImmutableDictionary <string, ICheckpointer> .Empty);
this.sync = new AsyncLock();
}
Checkpointer(string id, ICheckpointStore store, CheckpointData checkpointData)
{
this.Id = Preconditions.CheckNotNull(id);
this.store = Preconditions.CheckNotNull(store);
this.Offset = checkpointData.Offset;
this.LastFailedRevivalTime = checkpointData.LastFailedRevivalTime;
this.UnhealthySince = checkpointData.UnhealthySince;
this.Proposed = checkpointData.Offset;
this.closed = new AtomicBoolean(false);
}
public ProjectionManager(
IEventStoreConnection connection, ICheckpointStore checkpointStore, ISerializer serializer,
TypeMapper typeMapper, Projection[] projections)
{
_connection = connection ?? throw new ArgumentNullException(nameof(connection));
_checkpointStore = checkpointStore ?? throw new ArgumentNullException(nameof(checkpointStore));
_serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
_typeMapper = typeMapper ?? throw new ArgumentNullException(nameof(typeMapper));
_projections = projections ?? throw new ArgumentNullException(nameof(projections));
}
protected EventStoreCatchUpSubscriptionBase(
EventStoreClient eventStoreClient,
T options,
ICheckpointStore checkpointStore,
ConsumePipe consumePipe
) : base(eventStoreClient, options, ConfigurePipe(consumePipe, options))
{
CheckpointStore = Ensure.NotNull(checkpointStore);
CheckpointCommitHandler = new CheckpointCommitHandler(options.SubscriptionId, checkpointStore, 10);
}
public MessageStore(IStoreProvider storeProvider, ICheckpointStore checkpointStore, TimeSpan timeToLive, bool checkEntireQueueOnCleanup = false)
{
this.storeProvider = Preconditions.CheckNotNull(storeProvider);
this.messageEntityStore = this.storeProvider.GetEntityStore <string, MessageWrapper>(Constants.MessageStorePartitionKey);
this.endpointSequentialStores = new ConcurrentDictionary <string, ISequentialStore <MessageRef> >();
this.timeToLive = timeToLive;
this.checkpointStore = Preconditions.CheckNotNull(checkpointStore, nameof(checkpointStore));
this.messagesCleaner = new CleanupProcessor(this, checkEntireQueueOnCleanup);
Events.MessageStoreCreated();
}
public SubscriptionManager(
IEventStoreConnection connection,
ICheckpointStore checkpointStore,
string name,
params ISubscription[] subscriptions)
{
_connection = connection;
_checkpointStore = checkpointStore;
_name = name;
_subscriptions = subscriptions;
}
protected CatchupSubscriber(
IEventStoreConnection connection,
ICheckpointStore checkpointStore,
IMediator mediator,
IServiceLocator serviceLocator)
{
_connection = connection;
_checkpointStore = checkpointStore;
_mediator = mediator;
_serviceLocator = serviceLocator;
}
public SubscriptionManager(
IEventStoreConnection connection,
ICheckpointStore checkpointStore,
string subscriptionName,
params EventHandler[] eventHandlers)
{
_connection = connection;
_checkpointStore = checkpointStore;
_subscriptionName = subscriptionName;
_eventHandlers = eventHandlers;
}
public ReplicatorService(
IEventReader reader,
SinkPipeOptions sinkOptions,
PreparePipelineOptions prepareOptions,
ICheckpointStore checkpointStore
)
{
_reader = reader;
_sinkOptions = sinkOptions;
_prepareOptions = prepareOptions;
_checkpointStore = checkpointStore;
}
public async Task CleanupTestCheckpointed(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20);
ICheckpointStore checkpointStore = result.checkpointStore;
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 0);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 0);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None);
await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(100));
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
}
}
async Task <(IMessageStore, ICheckpointStore)> GetMessageStore(int ttlSecs = 300)
{
var dbStoreProvider = new InMemoryDbStoreProvider();
IStoreProvider storeProvider = new StoreProvider(dbStoreProvider);
ICheckpointStore checkpointStore = CheckpointStore.Create(storeProvider);
IMessageStore messageStore = new MessageStore(storeProvider, checkpointStore, TimeSpan.FromSeconds(ttlSecs));
await messageStore.AddEndpoint("module1");
await messageStore.AddEndpoint("module2");
return(messageStore, checkpointStore);
}
public MongoProjectionService(
EventStoreClient eventStoreClient,
ICheckpointStore checkpointStore,
string checkpointId,
params IEventHandler[] projections
)
{
_eventStoreClient = eventStoreClient;
_checkpointStore = checkpointStore;
_checkpointId = checkpointId;
_projections = projections;
}
// EventStoreAllCatchUpSubscription _subscription;
public SubscriptionIntegrator(IEventStoreConnection connection
, ICheckpointStore checkpointStore
, string subscriptionName
, IOnewayAsyncMessenger <EventEnvelope> messenger
, IEventDeserializer deserializer)
{
_connection = connection;
_checkpointStore = checkpointStore;
_subscriptionName = subscriptionName;
_messenger = messenger;
_deserializer = deserializer;
}
internal ProjectionManager(
IEventStoreConnection connection, ICheckpointStore checkpointStore, ISerializer serializer,
TypeMapper typeMapper, Projection[] projections, int?maxLiveQueueSize, int?readBatchSize)
{
_connection = connection ?? throw new ArgumentNullException(nameof(connection));
_checkpointStore = checkpointStore ?? throw new ArgumentNullException(nameof(checkpointStore));
_serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
_typeMapper = typeMapper ?? throw new ArgumentNullException(nameof(typeMapper));
_projections = projections ?? throw new ArgumentNullException(nameof(projections));
_maxLiveQueueSize = maxLiveQueueSize ?? 10000;
_readBatchSize = readBatchSize ?? 500;
}
protected EsdbSubscriptionService(
EventStoreClient eventStoreClient,
string subscriptionId,
ICheckpointStore checkpointStore,
IEventSerializer eventSerializer,
IEnumerable <IEventHandler> eventHandlers,
ILoggerFactory?loggerFactory = null,
SubscriptionGapMeasure?measure = null
) : base(subscriptionId, checkpointStore, eventSerializer, eventHandlers, loggerFactory, measure)
{
EventStoreClient = Ensure.NotNull(eventStoreClient, nameof(eventStoreClient));
}
Checkpointer(string id, ICheckpointStore store, CheckpointData checkpointData, string endpointId, uint priority)
{
this.Id = Preconditions.CheckNotNull(id);
this.store = Preconditions.CheckNotNull(store);
this.Offset = checkpointData.Offset;
this.LastFailedRevivalTime = checkpointData.LastFailedRevivalTime;
this.UnhealthySince = checkpointData.UnhealthySince;
this.Proposed = checkpointData.Offset;
this.closed = new AtomicBoolean(false);
this.EndpointId = endpointId;
this.Priority = priority.ToString();
}
protected EventStoreSubscriptionService(
EventStoreClient eventStoreClient,
EventStoreSubscriptionOptions options,
ICheckpointStore checkpointStore,
IEnumerable <IEventHandler> eventHandlers,
IEventSerializer?eventSerializer = null,
ILoggerFactory?loggerFactory = null,
ISubscriptionGapMeasure?measure = null
) : base(options, checkpointStore, eventHandlers, eventSerializer, loggerFactory, measure)
{
EventStoreClient = Ensure.NotNull(eventStoreClient, nameof(eventStoreClient));
}
public static async Task <Checkpointer> CreateAsync(string id, ICheckpointStore store)
{
Preconditions.CheckNotNull(id);
Preconditions.CheckNotNull(store);
Events.CreateStart(id);
CheckpointData checkpointData = await store.GetCheckpointDataAsync(id, CancellationToken.None);
var checkpointer = new Checkpointer(id, store, checkpointData);
Events.CreateFinished(checkpointer);
return(checkpointer);
}
async Task <(IMessageStore, ICheckpointStore, InMemoryDbStore)> GetMessageStore(bool checkEntireQueueOnCleanup, int ttlSecs = 300, int messageCleanupIntervalSecs = 30)
{
var dbStoreProvider = new InMemoryDbStoreProvider();
IStoreProvider storeProvider = new StoreProvider(dbStoreProvider);
InMemoryDbStore inMemoryDbStore = dbStoreProvider.GetDbStore("messages") as InMemoryDbStore;
ICheckpointStore checkpointStore = CheckpointStore.Create(storeProvider);
IMessageStore messageStore = new MessageStore(storeProvider, checkpointStore, TimeSpan.FromSeconds(ttlSecs), checkEntireQueueOnCleanup, messageCleanupIntervalSecs);
await messageStore.AddEndpoint("module1");
await messageStore.AddEndpoint("module2");
return(messageStore, checkpointStore, inMemoryDbStore);
}
public TransactionalAllStreamSubscriptionService(
EventStoreClient eventStoreClient,
EventStoreSubscriptionOptions options,
ICheckpointStore checkpointStore,
IEnumerable <IEventHandler> eventHandlers,
IEventSerializer?eventSerializer = null,
ILoggerFactory?loggerFactory = null,
ISubscriptionGapMeasure?measure = null) : base(eventStoreClient,
options, checkpointStore, eventHandlers, eventSerializer,
loggerFactory, measure)
{
_log = loggerFactory.CreateLogger(GetType());
}
public ReaderPipe(
IEventReader reader, ICheckpointStore checkpointStore, Func <PrepareContext, ValueTask> send
)
{
ILog log = LogProvider.GetCurrentClassLogger();
_pipe = Pipe.New <ReaderContext>(
cfg => {
cfg.UseConcurrencyLimit(1);
cfg.UseRetry(
retry => {
retry.Incremental(
100,
TimeSpan.Zero,
TimeSpan.FromMilliseconds(100)
);
retry.ConnectRetryObserver(new LoggingRetryObserver());
}
);
cfg.UseLog();
cfg.UseExecuteAsync(Reader);
}
);
async Task Reader(ReaderContext ctx)
{
try {
var start = await checkpointStore.LoadCheckpoint(ctx.CancellationToken).ConfigureAwait(false);
log.Info("Reading from {Position}", start);
await reader.ReadEvents(
start,
async read => {
ReplicationMetrics.ReadingPosition.Set(read.Position.EventPosition);
await send(new PrepareContext(read, ctx.CancellationToken)).ConfigureAwait(false);
},
ctx.CancellationToken
).ConfigureAwait(false);
}
catch (OperationCanceledException) {
// it's ok
}
finally {
log.Info("Reader stopped");
}
}
}
public async Task CleanupTestCheckpointed()
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20);
ICheckpointStore checkpointStore = result.checkpointStore;
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None);
await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(100));
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(0, batch.Count());
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Equal(0, batch.Count());
}
}
public RebuildTask(ILogger log,
IDependencyResolver ioc,
IHistoryReader historyReader,
IVersioningRepository versioningRepository,
ICheckpointStore checkpoints,
CancellationToken? pause = null)
{
if (log == null) throw new ArgumentNullException("log");
if (ioc == null) throw new ArgumentNullException("ioc");
if (historyReader == null) throw new ArgumentNullException("historyReader");
if (versioningRepository == null) throw new ArgumentNullException("versioningRepository");
if (checkpoints == null) throw new ArgumentNullException("checkpoints");
this.log = log;
this.ioc = ioc;
this.historyReader = historyReader;
this.versioningRepository = versioningRepository;
this.checkpoints = checkpoints;
this.running = pause ?? CancellationToken.None;
}
public UtilityTasks(ILogger log,
IDependencyResolver ioc,
IStoreEvents storeEvents,
IHistoryReader historyReader,
IVersioningRepository versioningRepository,
ICheckpointStore checkpoints,
IPersistHelper persistHelper,
IRepository repository,
string rmConnectionString)
{
this.log = log;
this.ioc = ioc;
this.storeEvents = storeEvents;
this.historyReader = historyReader;
this.versioningRepository = versioningRepository;
this.checkpoints = checkpoints;
this.rmConnectionString = rmConnectionString;
this.persistHelper = persistHelper;
this.repository = repository;
}
