public async Task CreateAccountEventIsPublishedToBus() { MassTransitDispatcher massTransitDispatcher = new MassTransitDispatcher(bus); PollingClient pollingClient = new PollingClient(store.Advanced, 100); IObserveCommits commitObserver = pollingClient.ObserveFrom(null); AccountDenormalizer denormalizer = new AccountDenormalizer(); bus.Subscribe(denormalizer); using (PollingHook pollingHook = new PollingHook(commitObserver)) { using (var subscription = commitObserver.Subscribe(massTransitDispatcher)) { commitObserver.PollNow(); commitObserver.Start(); Guid accountID = Guid.NewGuid(); string name = Guid.NewGuid().ToString(); string twitter = Guid.NewGuid().ToString(); System.Diagnostics.Debug.Print(string.Format("Creating account {0}", accountID)); client.CreateNewAccount(accountID, name, twitter); DateTime timeoutEnd = DateTime.Now.AddSeconds(10); while (denormalizer.AccountName != name && DateTime.Now < timeoutEnd) { await Task.Delay(100); } denormalizer.AccountName.Should().Be(name); } } }
protected override void Context() { for (int c = 1; c <= ParallelWriters; c++) { var client = new AcceptanceTestMongoPersistenceFactory().Build(); if (c == 1) { client.Drop(); client.Initialize(); } _writers.Add(client); } _observer = new Observer(); var reader = new AcceptanceTestMongoPersistenceFactory().Build(); _client = new PollingClient(reader, PollingInterval); _observeCommits = _client.ObserveFrom(null); _subscription = _observeCommits.Subscribe(_observer); _observeCommits.Start(); }
public void CreateAccountEventIsPublishedToBus() { using (MassTransitDispatcher massTransitDispatcher = new MassTransitDispatcher(bus)) { PollingClient pollingClient = new PollingClient(store.Advanced, 100); IObserveCommits commitObserver = pollingClient.ObserveFrom(null); IEventHandler <SimpleAggregateCreated> denormalizer = A.Fake <IEventHandler <SimpleAggregateCreated> >(); AutoResetEvent are = new AutoResetEvent(false); A.CallTo(() => denormalizer.Handle(A <SimpleAggregateCreated> .Ignored)).Invokes(() => are.Set()); bus.Subscribe(denormalizer); using (PollingHook pollingHook = new PollingHook(commitObserver)) { using (var subscription = commitObserver.Subscribe(massTransitDispatcher)) { commitObserver.PollNow(); commitObserver.Start(); Guid aggregateID = Guid.NewGuid(); SimpleAggregate aggregate = new SimpleAggregate(aggregateID, DateTime.Now); repository.Save(aggregate, Guid.NewGuid(), (o) => { }); are.WaitOne(10000).Should().BeTrue("event should be dispatched and recieved within timeout"); } } } }
public void testPolling() { PollingClient client = new PollingClient(SUB_PORT); try { TopicPoller poller = client.subscribe(REMOTE_HOST, REMOTE_PORT, REMOTE_TABLE_NAME, 0); int count = 0; bool started = false; long start = DateTime.Now.Ticks; long last = DateTime.Now.Ticks; while (true) { List <IMessage> messages = poller.poll(TIMEOUT); if (messages == null || messages.Count == 0) { start = DateTime.Now.Ticks; continue; } int messageCount = messages.Count; if (messageCount > 0 && !started) { started = true; start = DateTime.Now.Ticks; } count += messageCount; foreach (IMessage message in messages) { string symbol = message.getEntity(0).getString(); string price = message.getEntity(1).getString(); string size = message.getEntity(2).getString(); string ex = message.getEntity(3).getString(); } if (messageCount > 0) { if (((BasicInt)messages.Last().getEntity(4)).getValue() == -1) { break; } } long now = DateTime.Now.Ticks; if (now - last >= 1000) { long batchEnd = DateTime.Now.Ticks; Console.WriteLine(count + " messages took " + ((batchEnd - start) / 1000.0) + " ms, throghput: " + count / ((batchEnd - start) / 1000000.0) + " messages/s"); last = now; } } long end = DateTime.Now.Ticks; Console.WriteLine(count + " messages took " + ((end - start) / 1000.0) + " ms, throghput: " + count / ((end - start) / 1000000.0) + " messages/s"); System.Environment.Exit(0); } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.Write(ex.StackTrace); Assert.Fail(ex.Message); } }
public IObserveCommits Construct() { var pollingClient = new PollingClient(_eventStore.Advanced); var checkpoint = _checkpointTracker.GetCheckpoint(); var subscription = pollingClient.ObserveFrom(checkpoint); var started = DateTime.UtcNow; var liveNotification = subscription .SkipWhile(commit => commit.CommitStamp < started) .Cast <object>() .Merge(subscription.Throttle(TimeSpan.FromSeconds(5)).Select(_ => new SubscriptionIsLive())) .Take(1) .Select(_ => new SubscriptionIsLive()); var subscriptionWithLiveNotification = subscription .Cast <object>() .Merge(liveNotification); foreach (var commitObserver in _commitObservers) { subscriptionWithLiveNotification.Subscribe(commitObserver); } if (checkpoint == NullCheckpointToken.Value) { subscription.Start(); } return(subscription); }
protected override void Context() { base.Context(); StoreEvents.Advanced.CommitSingle(); _observeCommits = PollingClient.ObserveFrom(); _commitObserved = _observeCommits.FirstAsync().ToTask(); }
protected override void Context() { base.Context(); StoreEvents.Advanced.CommitSingle(); _observeCommits = PollingClient.ObserveFrom(); _twoCommitsObserved = _observeCommits.Take(2).ToTask(); }
public void testPollingUnSubscribe() { PollingClient client = new PollingClient(SUB_PORT); try { client.subscribe(REMOTE_HOST, REMOTE_PORT, REMOTE_TABLE_NAME, 0); } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.Write(ex.StackTrace); Assert.Fail(ex.Message); } PollingClient client1 = new PollingClient(SUB_PORT); try { client1.unsubscribe(REMOTE_HOST, REMOTE_PORT, REMOTE_TABLE_NAME); } catch (Exception e) { Assert.Fail(e.Message); } }
protected override void Context() { base.Context(); StoreEvents.Advanced.CommitMany(4, null, "bucket_2"); StoreEvents.Advanced.CommitMany(4, null, "bucket_1"); _observeCommits = PollingClient.ObserveFrom(); _commitObserved = _observeCommits.FirstAsync().ToTask(); }
protected override void Context() { base.Context(); StoreEvents.Advanced.CommitSingle(); _observeCommits1 = PollingClient.ObserveFrom(); _observeCommits1Complete = _observeCommits1.Take(5).ToTask(); _observeCommits2Complete = _observeCommits1.Take(10).ToTask(); }
public async Task polling_client_should_not_miss_data(int parallelism, bool autopolling) { _logger.LogDebug("Starting with {Parallelism} workers and Autopolling {Autopolling}", parallelism, autopolling); var sequenceChecker = new StrictSequenceChecker($"Workers {parallelism} autopolling {autopolling}"); var poller = new PollingClient(Store, 0, sequenceChecker, this.LoggerFactory) { PollingIntervalMilliseconds = 0, HoleDetectionTimeout = 1000 }; if (autopolling) { poller.Start(); _logger.LogDebug("Started Polling"); } const int range = 1000; var producer = new ActionBlock <int>(async i => { await Store.AppendAsync("p", i, "demo", "op#" + i).ConfigureAwait(false); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = parallelism }); _logger.LogDebug("Started pushing data: {elements} elements", range); foreach (var i in Enumerable.Range(1, range)) { Assert.True(await producer.SendAsync(i).ConfigureAwait(false)); } producer.Complete(); await producer.Completion.ConfigureAwait(false); _logger.LogDebug("Data pushed"); if (autopolling) { _logger.LogDebug("Stopping poller"); await poller.Stop(); _logger.LogDebug("Poller stopped"); } // read to end _logger.LogDebug("Polling to end"); var timeout = new CancellationTokenSource(60000); await poller.Poll(timeout.Token).ConfigureAwait(false); _logger.LogDebug("Polling to end - done"); Assert.True(poller.Position == sequenceChecker.Position, "Sequence " + sequenceChecker.Position + " != Position " + poller.Position); Assert.True(range == poller.Position, "Poller @" + poller.Position); Assert.True(range == sequenceChecker.Position, "Sequence @" + sequenceChecker.Position); }
public void Add(PollingClient pollingClient) { lock (sync) { pollingClients.Add(pollingClient); } pollingClient.Start(); }
private void StartSubscribing(IObserver <ICommit> observer, string checkPointToken = null, int interval = 5000) { var pc = new PollingClient(_store.Advanced, interval); var commitObserver = pc.ObserveFrom(checkPointToken); _pipeLineHook.Subscribe(commitObserver); commitObserver.Subscribe(observer); commitObserver.Start(); }
private void Polling(IStoreEvents store) { var pollingClient = new PollingClient(store.Advanced, 5000); var observeCommits = pollingClient.ObserveFrom("0"); var eventStreamObserver = new EventStreamObserver(); observeCommits.Subscribe(eventStreamObserver); GlobalTimer.Timer.Start(); observeCommits.Start(); }
protected override void Because() { PollingClient.Start(); Task.Factory.StartNew(() => { for (int i = 0; i < 15; i++) { StoreEvents.Advanced.CommitSingle(); } }); }
public SampleApp(IPersistence store, string name, bool useSnapshots, bool quiet, bool fast) { _quiet = quiet; _name = name; _rooms = 32; _storeProfile = new ProfileDecorator(store); _streams = new StreamsFactory(_storeProfile); _aggregateFactory = new DefaultAggregateFactory(); var network = fast ? (INetworkSimulator) new NoNetworkLatencySimulator() : (INetworkSimulator) new ReliableNetworkSimulator(10, 50); _appProjections = new AppProjections(network, quiet); _poller = new PollingClient(_storeProfile, 0, _appProjections, this._loggerFactory); if (useSnapshots) { _cloneProfiler = new TaskProfilingInfo("Cloning state"); var inMemoryPersistence = new InMemoryPersistence(new InMemoryPersistenceOptions { CloneFunc = CloneSnapshot }); _snapshotProfile = new ProfileDecorator(inMemoryPersistence); _snapshots = new DefaultSnapshotStore(_snapshotProfile); } _unboundedOptions = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded, BoundedCapacity = DataflowBlockOptions.Unbounded, EnsureOrdered = false, MaxMessagesPerTask = DataflowBlockOptions.Unbounded }; _boundedOptions = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount * 8, BoundedCapacity = 500, EnsureOrdered = true }; if (store is MongoPersistence) { _unboundedOptions.MaxDegreeOfParallelism = Environment.ProcessorCount * 4; _unboundedOptions.BoundedCapacity = 2000; } }
public IObserveCommits Construct() { var pollingClient = new PollingClient(eventStore.Advanced); var checkpoint = checkpointRepo.LoadCheckpoint(); IObserveCommits subscription = pollingClient.ObserveFrom(checkpoint); foreach (var commitObserver in commitObservers) { subscription.Subscribe(commitObserver); } return(subscription); }
public void Start() { var start = _tracker.LoadCheckpoint(); var client = new PollingClient(_eventStore.Advanced, 1000); var dispatcher = new CommitsDispatcher(_tracker, Logger); _observer = client.ObserveFrom(start); _subscription = _observer.Subscribe(dispatcher); _observer.Start(); Logger.InfoFormat("Projection engine started from {0}", start); }
/// <summary> /// Create a poller, start the poller and return starting position of the poller. /// </summary> /// <param name="poller"></param> /// <param name="pollerId"></param> /// <returns></returns> private Int64 CreatePollerAndStart(ref PollingClient poller, Int32 pollerId, Func <AtomicDispatchChunk, Task <Boolean> > dispatchToTplFunction) { if (poller != null) { poller.Stop(); } var startingPosition = TryGetStartingPoint(pollerId); Logger.InfoFormat("AtomicProjectionEngine: Starting poller id {0} from position {1}", pollerId, startingPosition); var subscription = new JarvisFrameworkLambdaSubscription(c => dispatchToTplFunction(new AtomicDispatchChunk(pollerId, c))); poller = new PollingClient(_persistence, startingPosition, subscription, _nStoreLoggerFactory); poller.Start(); return(startingPosition); }
public async Task should_read_from_position(long start, long expected) { await Store.AppendAsync("a", 1, "1").ConfigureAwait(false); await Store.AppendAsync("a", 2, "2").ConfigureAwait(false); await Store.AppendAsync("a", 3, "3").ConfigureAwait(false); var recorder = new AllPartitionsRecorder(); var client = new PollingClient(Store, start, recorder, LoggerFactory); await client.Poll(5000).ConfigureAwait(false); Assert.Equal(expected, recorder.Length); }
public void Init() { //checkpoint token, can be persisted so we don't need to read the EventStore from scratch string checkpointToken = null; var pollingClient = new PollingClient(Program.StoreEvents.Advanced); _observer = pollingClient.ObserveFromBucket(InvoicesBucketId, checkpointToken); _observer.Subscribe(new InvoicesObserver()); //init the projection on startup _observer.PollNow(); //start a long running task that will poll the event store periodicaly _observerTask = _observer.Start(); }
/// <summary> /// Create a poller, start the poller and return starting position of the poller. /// </summary> /// <param name="poller"></param> /// <param name="pollerId"></param> /// <returns></returns> private Int64 CreatePollerAndStart(ref PollingClient poller, Int32 pollerId, Func <AtomicDispatchChunk, Task <Boolean> > dispatchToTplFunction) { if (poller != null) { poller.Stop(); } //The default value to start is zero, it is a situation that should never happen, if it happens this poller has no projection to dispatch. var startingPosition = TryGetStartingPoint(pollerId, 0); Logger.InfoFormat("AtomicProjectionEngine: Starting poller id {0} from position {1}", pollerId, startingPosition); var subscription = new JarvisFrameworkLambdaSubscription(c => dispatchToTplFunction(new AtomicDispatchChunk(pollerId, c))); poller = new PollingClient(_persistence, startingPosition, subscription, _nStoreLoggerFactory); poller.Start(); return(startingPosition); }
public void Configure( Int64 checkpointTokenFrom, Int32 bufferSize) { _logger.InfoFormat("CommitPollingClient {0}: Configured starting from {1} buffer {2}", _id, checkpointTokenFrom, bufferSize); _bufferSize = bufferSize; _lastDispatchedPosition = checkpointTokenFrom - 1; LastException = null; //prepare single poller thread. CreateTplChain(); _innerSubscription = new JarvisFrameworkLambdaSubscription(DispatchChunk); _innerClient = new PollingClient( _persistence, checkpointTokenFrom, _innerSubscription, _factory); RegisterHealthChecks(_id); }
public async Task TyingItTogether() { var denormalizer = new AccountDenormalizer(); bus.Subscribe(denormalizer); bus.Subscribe(new KaChingNotifier()); bus.Subscribe(new OmgSadnessNotifier()); var massTransitDispatcher = new MassTransitDispatcher(bus); var pollingClient = new PollingClient(store.Advanced, 100); var commitObserver = pollingClient.ObserveFrom(null); using (PollingHook pollingHook = new PollingHook(commitObserver)) { using (var subscription = commitObserver.Subscribe(massTransitDispatcher)) { commitObserver.PollNow(); commitObserver.Start(); Guid accountID = Guid.NewGuid(); string name = Guid.NewGuid().ToString(); string twitter = Guid.NewGuid().ToString(); client.CreateNewAccount(accountID, name, twitter); client.CloseAccount(accountID); DateTime timeoutEnd = DateTime.Now.AddSeconds(10); while ((denormalizer.AccountName != name || denormalizer.IsActive) && DateTime.Now < timeoutEnd) { await Task.Delay(100); } denormalizer.AccountName.Should().Be(name); denormalizer.IsActive.Should().Be(false); store.OpenStream(accountID, 0, int.MaxValue).CommittedEvents.Count.Should().Be(2); } } }
public DomainRuntime( IPersistence persistence, IAggregateFactory aggregateFactory, ISnapshotStore snapshots, ChunkProcessor processor) { _persistence = persistence; _aggregateFactory = aggregateFactory; _snapshots = snapshots; _streamsFactory = new StreamsFactory(persistence); if (processor != null) { _pollingClient = new PollingClient( persistence, 0, // <----- TODO: read from state? new LambdaSubscription(processor), NStoreNullLoggerFactory.Instance ); _pollingClient.Start(); } }
// [InlineData(3, 3)] @@TODO enable tombstone! public async Task poller_should_skip_missing_chunks(long missing, long expected) { await Store.AppendAsync("a", 1, "1").ConfigureAwait(false); await Store.AppendAsync("a", 2, "2").ConfigureAwait(false); await Store.AppendAsync("a", 3, "3").ConfigureAwait(false); await Store.DeleteAsync("a", missing, missing).ConfigureAwait(false); var recored = new AllPartitionsRecorder(); var poller = new PollingClient(Store, 0, recored, this.LoggerFactory) { HoleDetectionTimeout = 100 }; var cts = new CancellationTokenSource(20000); await poller.Poll(cts.Token).ConfigureAwait(false); await poller.Poll(cts.Token).ConfigureAwait(false); Assert.Equal(expected, poller.Position); }
private static void Main() { using (var store = WireupEventStore()) { var client = new PollingClient(store.Advanced); string checkpointToken = LoadCheckpoint(); using (IObserveCommits observeCommits = client.ObserveFrom(checkpointToken)) using (observeCommits.Subscribe(commit => { // Project the commit etc Console.WriteLine(Resources.CommitInfo, commit.BucketId, commit.StreamId, commit.CommitSequence); // Track the most recent checkpoint checkpointToken = commit.CheckpointToken; })) { observeCommits.Start(); Console.WriteLine(Resources.PressAnyKey); Console.ReadKey(); SaveCheckpoint(checkpointToken); } } }
protected override void Because() { PollingClient.Start(); }
protected override void Cleanup() { PollingClient.Dispose(); }
private readonly INStoreLoggerFactory _loggerFactory = new ConsoleLoggerFactory(); // NStoreNullLoggerFactory.Instance; public ProjectionEngine(IPersistence persistence, INetworkSimulator networkSimulator) { _pollingClient = new PollingClient(persistence, 0, new Projections(networkSimulator), _loggerFactory); }