public StatsSubscriber(EventStoreClient eventStoreClient, string subscriptionId, ICheckpointStore checkpointStore, IEventSerializer eventSerializer, IEnumerable <IEventHandler> eventHandlers, ILoggerFactory loggerFactory = null, IEventFilter eventFilter = null, SubscriptionGapMeasure measure = null) : base(eventStoreClient, subscriptionId, checkpointStore, eventSerializer, eventHandlers, loggerFactory, eventFilter, measure) { }
public ProjectionManager(IEventStoreConnection connection, ICheckpointStore checkpointStore, params IProjection[] projections) { _connection = connection; _checkpointStore = checkpointStore; _projections = projections; }
/// <summary> /// Creates EventStoreDB catch-up subscription service for a given stream /// </summary> /// <param name="eventStoreClient">EventStoreDB gRPC client instance</param> /// <param name="streamName">Name of the stream to receive events from</param> /// <param name="subscriptionId">Subscription ID</param> /// <param name="checkpointStore">Checkpoint store instance</param> /// <param name="eventSerializer">Event serializer instance</param> /// <param name="eventHandlers">Collection of event handlers</param> /// <param name="loggerFactory">Optional: logger factory</param> /// <param name="measure">Optional: gap measurement for metrics</param> /// <param name="throwOnError"></param> public StreamSubscription( EventStoreClient eventStoreClient, string streamName, string subscriptionId, ICheckpointStore checkpointStore, IEnumerable <IEventHandler> eventHandlers, IEventSerializer?eventSerializer = null, ILoggerFactory?loggerFactory = null, ISubscriptionGapMeasure?measure = null, bool throwOnError = false ) : this( eventStoreClient, new StreamSubscriptionOptions { StreamName = streamName, SubscriptionId = subscriptionId, ThrowOnError = throwOnError }, checkpointStore, eventHandlers, eventSerializer, loggerFactory, measure ) { }
public async Task CheckpointStoreBasicTest() { ICheckpointStore checkpointStore = CheckpointStore.Create(new StoreProvider(new InMemoryDbStoreProvider())); for (long i = 0; i < 10; i++) { var checkpointData = new CheckpointData(i); await checkpointStore.SetCheckpointDataAsync($"Endpoint{i}", checkpointData, CancellationToken.None); } IDictionary <string, CheckpointData> allCheckpointData = await checkpointStore.GetAllCheckpointDataAsync(CancellationToken.None); Assert.Equal(10, allCheckpointData.Count); long counter = 0; foreach (KeyValuePair <string, CheckpointData> checkpointValue in allCheckpointData) { Assert.Equal(counter, checkpointValue.Value.Offset); Assert.Equal($"Endpoint{counter}", checkpointValue.Key); counter++; } for (long i = 0; i < 10; i++) { CheckpointData checkpointData = await checkpointStore.GetCheckpointDataAsync($"Endpoint{i}", CancellationToken.None); Assert.NotNull(checkpointData); Assert.Equal(i, checkpointData.Offset); } }
/// <summary> /// Creates EventStoreDB catch-up subscription service for a given stream /// </summary> /// <param name="client"></param> /// <param name="checkpointStore">Checkpoint store instance</param> /// <param name="options">Subscription options</param> /// <param name="consumePipe"></param> public StreamSubscription( EventStoreClient client, StreamSubscriptionOptions options, ICheckpointStore checkpointStore, ConsumePipe consumePipe ) : base(client, options, checkpointStore, consumePipe) => Ensure.NotEmptyString(options.StreamName);
public RebuildTask(ILogger log, IDependencyResolver ioc, IHistoryReader historyReader, IVersioningRepository versioningRepository, ICheckpointStore checkpoints, CancellationToken?pause = null) { if (log == null) { throw new ArgumentNullException("log"); } if (ioc == null) { throw new ArgumentNullException("ioc"); } if (historyReader == null) { throw new ArgumentNullException("historyReader"); } if (versioningRepository == null) { throw new ArgumentNullException("versioningRepository"); } if (checkpoints == null) { throw new ArgumentNullException("checkpoints"); } this.log = log; this.ioc = ioc; this.historyReader = historyReader; this.versioningRepository = versioningRepository; this.checkpoints = checkpoints; this.running = pause ?? CancellationToken.None; }
/// <summary> /// Creates EventStoreDB catch-up subscription service for $all /// </summary> /// <param name="eventStoreClient"></param> /// <param name="options"></param> /// <param name="checkpointStore">Checkpoint store instance</param> /// <param name="consumePipe"></param> public AllStreamSubscription( EventStoreClient eventStoreClient, AllStreamSubscriptionOptions options, ICheckpointStore checkpointStore, ConsumePipe consumePipe ) : base(eventStoreClient, options, checkpointStore, consumePipe) { }
public CommitDispatcher(ICommandBus commandBus, IEventBus eventBus, ILogger logger, ICheckpointStore checkpoints) { this.commandBus = commandBus; this.eventBus = eventBus; this.logger = logger; this.checkpoints = checkpoints; eventBusPublish = new GenericMethodCaller(eventBus, "Publish"); commandBusPublish = new GenericMethodCaller(commandBus, "Publish"); }
MasterCheckpointer(string id, ICheckpointStore store, long offset) { this.Id = Preconditions.CheckNotNull(id); this.store = Preconditions.CheckNotNull(store); this.Offset = offset; this.Proposed = offset; this.closed = new AtomicBoolean(false); this.childCheckpointers = new AtomicReference <ImmutableDictionary <string, ICheckpointer> >(ImmutableDictionary <string, ICheckpointer> .Empty); this.sync = new AsyncLock(); }
Checkpointer(string id, ICheckpointStore store, CheckpointData checkpointData) { this.Id = Preconditions.CheckNotNull(id); this.store = Preconditions.CheckNotNull(store); this.Offset = checkpointData.Offset; this.LastFailedRevivalTime = checkpointData.LastFailedRevivalTime; this.UnhealthySince = checkpointData.UnhealthySince; this.Proposed = checkpointData.Offset; this.closed = new AtomicBoolean(false); }
public ProjectionManager( IEventStoreConnection connection, ICheckpointStore checkpointStore, ISerializer serializer, TypeMapper typeMapper, Projection[] projections) { _connection = connection ?? throw new ArgumentNullException(nameof(connection)); _checkpointStore = checkpointStore ?? throw new ArgumentNullException(nameof(checkpointStore)); _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); _typeMapper = typeMapper ?? throw new ArgumentNullException(nameof(typeMapper)); _projections = projections ?? throw new ArgumentNullException(nameof(projections)); }
protected EventStoreCatchUpSubscriptionBase( EventStoreClient eventStoreClient, T options, ICheckpointStore checkpointStore, ConsumePipe consumePipe ) : base(eventStoreClient, options, ConfigurePipe(consumePipe, options)) { CheckpointStore = Ensure.NotNull(checkpointStore); CheckpointCommitHandler = new CheckpointCommitHandler(options.SubscriptionId, checkpointStore, 10); }
public MessageStore(IStoreProvider storeProvider, ICheckpointStore checkpointStore, TimeSpan timeToLive, bool checkEntireQueueOnCleanup = false) { this.storeProvider = Preconditions.CheckNotNull(storeProvider); this.messageEntityStore = this.storeProvider.GetEntityStore <string, MessageWrapper>(Constants.MessageStorePartitionKey); this.endpointSequentialStores = new ConcurrentDictionary <string, ISequentialStore <MessageRef> >(); this.timeToLive = timeToLive; this.checkpointStore = Preconditions.CheckNotNull(checkpointStore, nameof(checkpointStore)); this.messagesCleaner = new CleanupProcessor(this, checkEntireQueueOnCleanup); Events.MessageStoreCreated(); }
public SubscriptionManager( IEventStoreConnection connection, ICheckpointStore checkpointStore, string name, params ISubscription[] subscriptions) { _connection = connection; _checkpointStore = checkpointStore; _name = name; _subscriptions = subscriptions; }
protected CatchupSubscriber( IEventStoreConnection connection, ICheckpointStore checkpointStore, IMediator mediator, IServiceLocator serviceLocator) { _connection = connection; _checkpointStore = checkpointStore; _mediator = mediator; _serviceLocator = serviceLocator; }
public SubscriptionManager( IEventStoreConnection connection, ICheckpointStore checkpointStore, string subscriptionName, params EventHandler[] eventHandlers) { _connection = connection; _checkpointStore = checkpointStore; _subscriptionName = subscriptionName; _eventHandlers = eventHandlers; }
public ReplicatorService( IEventReader reader, SinkPipeOptions sinkOptions, PreparePipelineOptions prepareOptions, ICheckpointStore checkpointStore ) { _reader = reader; _sinkOptions = sinkOptions; _prepareOptions = prepareOptions; _checkpointStore = checkpointStore; }
public async Task CleanupTestCheckpointed(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20); ICheckpointStore checkpointStore = result.checkpointStore; using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 0); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 0); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None); await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None); await Task.Delay(TimeSpan.FromSeconds(100)); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Empty(batch); } }
async Task <(IMessageStore, ICheckpointStore)> GetMessageStore(int ttlSecs = 300) { var dbStoreProvider = new InMemoryDbStoreProvider(); IStoreProvider storeProvider = new StoreProvider(dbStoreProvider); ICheckpointStore checkpointStore = CheckpointStore.Create(storeProvider); IMessageStore messageStore = new MessageStore(storeProvider, checkpointStore, TimeSpan.FromSeconds(ttlSecs)); await messageStore.AddEndpoint("module1"); await messageStore.AddEndpoint("module2"); return(messageStore, checkpointStore); }
public MongoProjectionService( EventStoreClient eventStoreClient, ICheckpointStore checkpointStore, string checkpointId, params IEventHandler[] projections ) { _eventStoreClient = eventStoreClient; _checkpointStore = checkpointStore; _checkpointId = checkpointId; _projections = projections; }
// EventStoreAllCatchUpSubscription _subscription; public SubscriptionIntegrator(IEventStoreConnection connection , ICheckpointStore checkpointStore , string subscriptionName , IOnewayAsyncMessenger <EventEnvelope> messenger , IEventDeserializer deserializer) { _connection = connection; _checkpointStore = checkpointStore; _subscriptionName = subscriptionName; _messenger = messenger; _deserializer = deserializer; }
internal ProjectionManager( IEventStoreConnection connection, ICheckpointStore checkpointStore, ISerializer serializer, TypeMapper typeMapper, Projection[] projections, int?maxLiveQueueSize, int?readBatchSize) { _connection = connection ?? throw new ArgumentNullException(nameof(connection)); _checkpointStore = checkpointStore ?? throw new ArgumentNullException(nameof(checkpointStore)); _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); _typeMapper = typeMapper ?? throw new ArgumentNullException(nameof(typeMapper)); _projections = projections ?? throw new ArgumentNullException(nameof(projections)); _maxLiveQueueSize = maxLiveQueueSize ?? 10000; _readBatchSize = readBatchSize ?? 500; }
protected EsdbSubscriptionService( EventStoreClient eventStoreClient, string subscriptionId, ICheckpointStore checkpointStore, IEventSerializer eventSerializer, IEnumerable <IEventHandler> eventHandlers, ILoggerFactory?loggerFactory = null, SubscriptionGapMeasure?measure = null ) : base(subscriptionId, checkpointStore, eventSerializer, eventHandlers, loggerFactory, measure) { EventStoreClient = Ensure.NotNull(eventStoreClient, nameof(eventStoreClient)); }
Checkpointer(string id, ICheckpointStore store, CheckpointData checkpointData, string endpointId, uint priority) { this.Id = Preconditions.CheckNotNull(id); this.store = Preconditions.CheckNotNull(store); this.Offset = checkpointData.Offset; this.LastFailedRevivalTime = checkpointData.LastFailedRevivalTime; this.UnhealthySince = checkpointData.UnhealthySince; this.Proposed = checkpointData.Offset; this.closed = new AtomicBoolean(false); this.EndpointId = endpointId; this.Priority = priority.ToString(); }
protected EventStoreSubscriptionService( EventStoreClient eventStoreClient, EventStoreSubscriptionOptions options, ICheckpointStore checkpointStore, IEnumerable <IEventHandler> eventHandlers, IEventSerializer?eventSerializer = null, ILoggerFactory?loggerFactory = null, ISubscriptionGapMeasure?measure = null ) : base(options, checkpointStore, eventHandlers, eventSerializer, loggerFactory, measure) { EventStoreClient = Ensure.NotNull(eventStoreClient, nameof(eventStoreClient)); }
public static async Task <Checkpointer> CreateAsync(string id, ICheckpointStore store) { Preconditions.CheckNotNull(id); Preconditions.CheckNotNull(store); Events.CreateStart(id); CheckpointData checkpointData = await store.GetCheckpointDataAsync(id, CancellationToken.None); var checkpointer = new Checkpointer(id, store, checkpointData); Events.CreateFinished(checkpointer); return(checkpointer); }
async Task <(IMessageStore, ICheckpointStore, InMemoryDbStore)> GetMessageStore(bool checkEntireQueueOnCleanup, int ttlSecs = 300, int messageCleanupIntervalSecs = 30) { var dbStoreProvider = new InMemoryDbStoreProvider(); IStoreProvider storeProvider = new StoreProvider(dbStoreProvider); InMemoryDbStore inMemoryDbStore = dbStoreProvider.GetDbStore("messages") as InMemoryDbStore; ICheckpointStore checkpointStore = CheckpointStore.Create(storeProvider); IMessageStore messageStore = new MessageStore(storeProvider, checkpointStore, TimeSpan.FromSeconds(ttlSecs), checkEntireQueueOnCleanup, messageCleanupIntervalSecs); await messageStore.AddEndpoint("module1"); await messageStore.AddEndpoint("module2"); return(messageStore, checkpointStore, inMemoryDbStore); }
public TransactionalAllStreamSubscriptionService( EventStoreClient eventStoreClient, EventStoreSubscriptionOptions options, ICheckpointStore checkpointStore, IEnumerable <IEventHandler> eventHandlers, IEventSerializer?eventSerializer = null, ILoggerFactory?loggerFactory = null, ISubscriptionGapMeasure?measure = null) : base(eventStoreClient, options, checkpointStore, eventHandlers, eventSerializer, loggerFactory, measure) { _log = loggerFactory.CreateLogger(GetType()); }
public ReaderPipe( IEventReader reader, ICheckpointStore checkpointStore, Func <PrepareContext, ValueTask> send ) { ILog log = LogProvider.GetCurrentClassLogger(); _pipe = Pipe.New <ReaderContext>( cfg => { cfg.UseConcurrencyLimit(1); cfg.UseRetry( retry => { retry.Incremental( 100, TimeSpan.Zero, TimeSpan.FromMilliseconds(100) ); retry.ConnectRetryObserver(new LoggingRetryObserver()); } ); cfg.UseLog(); cfg.UseExecuteAsync(Reader); } ); async Task Reader(ReaderContext ctx) { try { var start = await checkpointStore.LoadCheckpoint(ctx.CancellationToken).ConfigureAwait(false); log.Info("Reading from {Position}", start); await reader.ReadEvents( start, async read => { ReplicationMetrics.ReadingPosition.Set(read.Position.EventPosition); await send(new PrepareContext(read, ctx.CancellationToken)).ConfigureAwait(false); }, ctx.CancellationToken ).ConfigureAwait(false); } catch (OperationCanceledException) { // it's ok } finally { log.Info("Reader stopped"); } } }
public async Task CleanupTestCheckpointed() { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20); ICheckpointStore checkpointStore = result.checkpointStore; using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(i / 2, offset); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None); await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None); await Task.Delay(TimeSpan.FromSeconds(100)); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(0, batch.Count()); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Equal(0, batch.Count()); } }
public RebuildTask(ILogger log, IDependencyResolver ioc, IHistoryReader historyReader, IVersioningRepository versioningRepository, ICheckpointStore checkpoints, CancellationToken? pause = null) { if (log == null) throw new ArgumentNullException("log"); if (ioc == null) throw new ArgumentNullException("ioc"); if (historyReader == null) throw new ArgumentNullException("historyReader"); if (versioningRepository == null) throw new ArgumentNullException("versioningRepository"); if (checkpoints == null) throw new ArgumentNullException("checkpoints"); this.log = log; this.ioc = ioc; this.historyReader = historyReader; this.versioningRepository = versioningRepository; this.checkpoints = checkpoints; this.running = pause ?? CancellationToken.None; }
public UtilityTasks(ILogger log, IDependencyResolver ioc, IStoreEvents storeEvents, IHistoryReader historyReader, IVersioningRepository versioningRepository, ICheckpointStore checkpoints, IPersistHelper persistHelper, IRepository repository, string rmConnectionString) { this.log = log; this.ioc = ioc; this.storeEvents = storeEvents; this.historyReader = historyReader; this.versioningRepository = versioningRepository; this.checkpoints = checkpoints; this.rmConnectionString = rmConnectionString; this.persistHelper = persistHelper; this.repository = repository; }