public async Task CreateAccountEventIsPublishedToBus()
{
MassTransitDispatcher massTransitDispatcher = new MassTransitDispatcher(bus);
PollingClient pollingClient = new PollingClient(store.Advanced, 100);
IObserveCommits commitObserver = pollingClient.ObserveFrom(null);
AccountDenormalizer denormalizer = new AccountDenormalizer();
bus.Subscribe(denormalizer);
using (PollingHook pollingHook = new PollingHook(commitObserver))
{
using (var subscription = commitObserver.Subscribe(massTransitDispatcher))
{
commitObserver.PollNow();
commitObserver.Start();
Guid accountID = Guid.NewGuid();
string name = Guid.NewGuid().ToString();
string twitter = Guid.NewGuid().ToString();
System.Diagnostics.Debug.Print(string.Format("Creating account {0}", accountID));
client.CreateNewAccount(accountID, name, twitter);
DateTime timeoutEnd = DateTime.Now.AddSeconds(10);
while (denormalizer.AccountName != name && DateTime.Now < timeoutEnd)
{
await Task.Delay(100);
}
denormalizer.AccountName.Should().Be(name);
}
}
}
protected override void Context()
{
for (int c = 1; c <= ParallelWriters; c++)
{
var client = new AcceptanceTestMongoPersistenceFactory().Build();
if (c == 1)
{
client.Drop();
client.Initialize();
}
_writers.Add(client);
}
_observer = new Observer();
var reader = new AcceptanceTestMongoPersistenceFactory().Build();
_client = new PollingClient(reader, PollingInterval);
_observeCommits = _client.ObserveFrom(null);
_subscription = _observeCommits.Subscribe(_observer);
_observeCommits.Start();
}
public void CreateAccountEventIsPublishedToBus()
{
using (MassTransitDispatcher massTransitDispatcher = new MassTransitDispatcher(bus))
{
PollingClient pollingClient = new PollingClient(store.Advanced, 100);
IObserveCommits commitObserver = pollingClient.ObserveFrom(null);
IEventHandler <SimpleAggregateCreated> denormalizer = A.Fake <IEventHandler <SimpleAggregateCreated> >();
AutoResetEvent are = new AutoResetEvent(false);
A.CallTo(() => denormalizer.Handle(A <SimpleAggregateCreated> .Ignored)).Invokes(() => are.Set());
bus.Subscribe(denormalizer);
using (PollingHook pollingHook = new PollingHook(commitObserver))
{
using (var subscription = commitObserver.Subscribe(massTransitDispatcher))
{
commitObserver.PollNow();
commitObserver.Start();
Guid aggregateID = Guid.NewGuid();
SimpleAggregate aggregate = new SimpleAggregate(aggregateID, DateTime.Now);
repository.Save(aggregate, Guid.NewGuid(), (o) => { });
are.WaitOne(10000).Should().BeTrue("event should be dispatched and recieved within timeout");
}
}
}
}
public void testPolling()
{
PollingClient client = new PollingClient(SUB_PORT);
try
{
TopicPoller poller = client.subscribe(REMOTE_HOST, REMOTE_PORT, REMOTE_TABLE_NAME, 0);
int count = 0;
bool started = false;
long start = DateTime.Now.Ticks;
long last = DateTime.Now.Ticks;
while (true)
{
List <IMessage> messages = poller.poll(TIMEOUT);
if (messages == null || messages.Count == 0)
{
start = DateTime.Now.Ticks;
continue;
}
int messageCount = messages.Count;
if (messageCount > 0 && !started)
{
started = true;
start = DateTime.Now.Ticks;
}
count += messageCount;
foreach (IMessage message in messages)
{
string symbol = message.getEntity(0).getString();
string price = message.getEntity(1).getString();
string size = message.getEntity(2).getString();
string ex = message.getEntity(3).getString();
}
if (messageCount > 0)
{
if (((BasicInt)messages.Last().getEntity(4)).getValue() == -1)
{
break;
}
}
long now = DateTime.Now.Ticks;
if (now - last >= 1000)
{
long batchEnd = DateTime.Now.Ticks;
Console.WriteLine(count + " messages took " + ((batchEnd - start) / 1000.0) + " ms, throghput: " + count / ((batchEnd - start) / 1000000.0) + " messages/s");
last = now;
}
}
long end = DateTime.Now.Ticks;
Console.WriteLine(count + " messages took " + ((end - start) / 1000.0) + " ms, throghput: " + count / ((end - start) / 1000000.0) + " messages/s");
System.Environment.Exit(0);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.Write(ex.StackTrace);
Assert.Fail(ex.Message);
}
}
public IObserveCommits Construct()
{
var pollingClient = new PollingClient(_eventStore.Advanced);
var checkpoint = _checkpointTracker.GetCheckpoint();
var subscription = pollingClient.ObserveFrom(checkpoint);
var started = DateTime.UtcNow;
var liveNotification = subscription
.SkipWhile(commit => commit.CommitStamp < started)
.Cast <object>()
.Merge(subscription.Throttle(TimeSpan.FromSeconds(5)).Select(_ => new SubscriptionIsLive()))
.Take(1)
.Select(_ => new SubscriptionIsLive());
var subscriptionWithLiveNotification = subscription
.Cast <object>()
.Merge(liveNotification);
foreach (var commitObserver in _commitObservers)
{
subscriptionWithLiveNotification.Subscribe(commitObserver);
}
if (checkpoint == NullCheckpointToken.Value)
{
subscription.Start();
}
return(subscription);
}
protected override void Context()
{
base.Context();
StoreEvents.Advanced.CommitSingle();
_observeCommits = PollingClient.ObserveFrom();
_commitObserved = _observeCommits.FirstAsync().ToTask();
}
protected override void Context()
{
base.Context();
StoreEvents.Advanced.CommitSingle();
_observeCommits = PollingClient.ObserveFrom();
_twoCommitsObserved = _observeCommits.Take(2).ToTask();
}
public void testPollingUnSubscribe()
{
PollingClient client = new PollingClient(SUB_PORT);
try
{
client.subscribe(REMOTE_HOST, REMOTE_PORT, REMOTE_TABLE_NAME, 0);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.Write(ex.StackTrace);
Assert.Fail(ex.Message);
}
PollingClient client1 = new PollingClient(SUB_PORT);
try
{
client1.unsubscribe(REMOTE_HOST, REMOTE_PORT, REMOTE_TABLE_NAME);
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
protected override void Context()
{
base.Context();
StoreEvents.Advanced.CommitMany(4, null, "bucket_2");
StoreEvents.Advanced.CommitMany(4, null, "bucket_1");
_observeCommits = PollingClient.ObserveFrom();
_commitObserved = _observeCommits.FirstAsync().ToTask();
}
protected override void Context()
{
base.Context();
StoreEvents.Advanced.CommitSingle();
_observeCommits1 = PollingClient.ObserveFrom();
_observeCommits1Complete = _observeCommits1.Take(5).ToTask();
_observeCommits2Complete = _observeCommits1.Take(10).ToTask();
}
public async Task polling_client_should_not_miss_data(int parallelism, bool autopolling)
{
_logger.LogDebug("Starting with {Parallelism} workers and Autopolling {Autopolling}", parallelism, autopolling);
var sequenceChecker = new StrictSequenceChecker($"Workers {parallelism} autopolling {autopolling}");
var poller = new PollingClient(Store, 0, sequenceChecker, this.LoggerFactory)
{
PollingIntervalMilliseconds = 0,
HoleDetectionTimeout = 1000
};
if (autopolling)
{
poller.Start();
_logger.LogDebug("Started Polling");
}
const int range = 1000;
var producer = new ActionBlock <int>(async i =>
{
await Store.AppendAsync("p", i, "demo", "op#" + i).ConfigureAwait(false);
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelism
});
_logger.LogDebug("Started pushing data: {elements} elements", range);
foreach (var i in Enumerable.Range(1, range))
{
Assert.True(await producer.SendAsync(i).ConfigureAwait(false));
}
producer.Complete();
await producer.Completion.ConfigureAwait(false);
_logger.LogDebug("Data pushed");
if (autopolling)
{
_logger.LogDebug("Stopping poller");
await poller.Stop();
_logger.LogDebug("Poller stopped");
}
// read to end
_logger.LogDebug("Polling to end");
var timeout = new CancellationTokenSource(60000);
await poller.Poll(timeout.Token).ConfigureAwait(false);
_logger.LogDebug("Polling to end - done");
Assert.True(poller.Position == sequenceChecker.Position, "Sequence " + sequenceChecker.Position + " != Position " + poller.Position);
Assert.True(range == poller.Position, "Poller @" + poller.Position);
Assert.True(range == sequenceChecker.Position, "Sequence @" + sequenceChecker.Position);
}
public void Add(PollingClient pollingClient)
{
lock (sync)
{
pollingClients.Add(pollingClient);
}
pollingClient.Start();
}
private void StartSubscribing(IObserver <ICommit> observer, string checkPointToken = null, int interval = 5000)
{
var pc = new PollingClient(_store.Advanced, interval);
var commitObserver = pc.ObserveFrom(checkPointToken);
_pipeLineHook.Subscribe(commitObserver);
commitObserver.Subscribe(observer);
commitObserver.Start();
}
private void Polling(IStoreEvents store)
{
var pollingClient = new PollingClient(store.Advanced, 5000);
var observeCommits = pollingClient.ObserveFrom("0");
var eventStreamObserver = new EventStreamObserver();
observeCommits.Subscribe(eventStreamObserver);
GlobalTimer.Timer.Start();
observeCommits.Start();
}
protected override void Because()
{
PollingClient.Start();
Task.Factory.StartNew(() =>
{
for (int i = 0; i < 15; i++)
{
StoreEvents.Advanced.CommitSingle();
}
});
}
public SampleApp(IPersistence store, string name, bool useSnapshots, bool quiet, bool fast)
{
_quiet = quiet;
_name = name;
_rooms = 32;
_storeProfile = new ProfileDecorator(store);
_streams = new StreamsFactory(_storeProfile);
_aggregateFactory = new DefaultAggregateFactory();
var network = fast
? (INetworkSimulator) new NoNetworkLatencySimulator()
: (INetworkSimulator) new ReliableNetworkSimulator(10, 50);
_appProjections = new AppProjections(network, quiet);
_poller = new PollingClient(_storeProfile, 0, _appProjections, this._loggerFactory);
if (useSnapshots)
{
_cloneProfiler = new TaskProfilingInfo("Cloning state");
var inMemoryPersistence = new InMemoryPersistence(new InMemoryPersistenceOptions
{
CloneFunc = CloneSnapshot
});
_snapshotProfile = new ProfileDecorator(inMemoryPersistence);
_snapshots = new DefaultSnapshotStore(_snapshotProfile);
}
_unboundedOptions = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded,
BoundedCapacity = DataflowBlockOptions.Unbounded,
EnsureOrdered = false,
MaxMessagesPerTask = DataflowBlockOptions.Unbounded
};
_boundedOptions = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 8,
BoundedCapacity = 500,
EnsureOrdered = true
};
if (store is MongoPersistence)
{
_unboundedOptions.MaxDegreeOfParallelism = Environment.ProcessorCount * 4;
_unboundedOptions.BoundedCapacity = 2000;
}
}
public IObserveCommits Construct()
{
var pollingClient = new PollingClient(eventStore.Advanced);
var checkpoint = checkpointRepo.LoadCheckpoint();
IObserveCommits subscription = pollingClient.ObserveFrom(checkpoint);
foreach (var commitObserver in commitObservers)
{
subscription.Subscribe(commitObserver);
}
return(subscription);
}
public void Start()
{
var start = _tracker.LoadCheckpoint();
var client = new PollingClient(_eventStore.Advanced, 1000);
var dispatcher = new CommitsDispatcher(_tracker, Logger);
_observer = client.ObserveFrom(start);
_subscription = _observer.Subscribe(dispatcher);
_observer.Start();
Logger.InfoFormat("Projection engine started from {0}", start);
}
/// <summary>
/// Create a poller, start the poller and return starting position of the poller.
/// </summary>
/// <param name="poller"></param>
/// <param name="pollerId"></param>
/// <returns></returns>
private Int64 CreatePollerAndStart(ref PollingClient poller, Int32 pollerId, Func <AtomicDispatchChunk, Task <Boolean> > dispatchToTplFunction)
{
if (poller != null)
{
poller.Stop();
}
var startingPosition = TryGetStartingPoint(pollerId);
Logger.InfoFormat("AtomicProjectionEngine: Starting poller id {0} from position {1}", pollerId, startingPosition);
var subscription = new JarvisFrameworkLambdaSubscription(c => dispatchToTplFunction(new AtomicDispatchChunk(pollerId, c)));
poller = new PollingClient(_persistence, startingPosition, subscription, _nStoreLoggerFactory);
poller.Start();
return(startingPosition);
}
public async Task should_read_from_position(long start, long expected)
{
await Store.AppendAsync("a", 1, "1").ConfigureAwait(false);
await Store.AppendAsync("a", 2, "2").ConfigureAwait(false);
await Store.AppendAsync("a", 3, "3").ConfigureAwait(false);
var recorder = new AllPartitionsRecorder();
var client = new PollingClient(Store, start, recorder, LoggerFactory);
await client.Poll(5000).ConfigureAwait(false);
Assert.Equal(expected, recorder.Length);
}
public void Init()
{
//checkpoint token, can be persisted so we don't need to read the EventStore from scratch
string checkpointToken = null;
var pollingClient = new PollingClient(Program.StoreEvents.Advanced);
_observer = pollingClient.ObserveFromBucket(InvoicesBucketId, checkpointToken);
_observer.Subscribe(new InvoicesObserver());
//init the projection on startup
_observer.PollNow();
//start a long running task that will poll the event store periodicaly
_observerTask = _observer.Start();
}
/// <summary>
/// Create a poller, start the poller and return starting position of the poller.
/// </summary>
/// <param name="poller"></param>
/// <param name="pollerId"></param>
/// <returns></returns>
private Int64 CreatePollerAndStart(ref PollingClient poller, Int32 pollerId, Func <AtomicDispatchChunk, Task <Boolean> > dispatchToTplFunction)
{
if (poller != null)
{
poller.Stop();
}
//The default value to start is zero, it is a situation that should never happen, if it happens this poller has no projection to dispatch.
var startingPosition = TryGetStartingPoint(pollerId, 0);
Logger.InfoFormat("AtomicProjectionEngine: Starting poller id {0} from position {1}", pollerId, startingPosition);
var subscription = new JarvisFrameworkLambdaSubscription(c => dispatchToTplFunction(new AtomicDispatchChunk(pollerId, c)));
poller = new PollingClient(_persistence, startingPosition, subscription, _nStoreLoggerFactory);
poller.Start();
return(startingPosition);
}
public void Configure(
Int64 checkpointTokenFrom,
Int32 bufferSize)
{
_logger.InfoFormat("CommitPollingClient {0}: Configured starting from {1} buffer {2}", _id, checkpointTokenFrom, bufferSize);
_bufferSize = bufferSize;
_lastDispatchedPosition = checkpointTokenFrom - 1;
LastException = null;
//prepare single poller thread.
CreateTplChain();
_innerSubscription = new JarvisFrameworkLambdaSubscription(DispatchChunk);
_innerClient = new PollingClient(
_persistence,
checkpointTokenFrom,
_innerSubscription,
_factory);
RegisterHealthChecks(_id);
}
public async Task TyingItTogether()
{
var denormalizer = new AccountDenormalizer();
bus.Subscribe(denormalizer);
bus.Subscribe(new KaChingNotifier());
bus.Subscribe(new OmgSadnessNotifier());
var massTransitDispatcher = new MassTransitDispatcher(bus);
var pollingClient = new PollingClient(store.Advanced, 100);
var commitObserver = pollingClient.ObserveFrom(null);
using (PollingHook pollingHook = new PollingHook(commitObserver))
{
using (var subscription = commitObserver.Subscribe(massTransitDispatcher))
{
commitObserver.PollNow();
commitObserver.Start();
Guid accountID = Guid.NewGuid();
string name = Guid.NewGuid().ToString();
string twitter = Guid.NewGuid().ToString();
client.CreateNewAccount(accountID, name, twitter);
client.CloseAccount(accountID);
DateTime timeoutEnd = DateTime.Now.AddSeconds(10);
while ((denormalizer.AccountName != name ||
denormalizer.IsActive) &&
DateTime.Now < timeoutEnd)
{
await Task.Delay(100);
}
denormalizer.AccountName.Should().Be(name);
denormalizer.IsActive.Should().Be(false);
store.OpenStream(accountID, 0, int.MaxValue).CommittedEvents.Count.Should().Be(2);
}
}
}
public DomainRuntime(
IPersistence persistence,
IAggregateFactory aggregateFactory,
ISnapshotStore snapshots,
ChunkProcessor processor)
{
_persistence = persistence;
_aggregateFactory = aggregateFactory;
_snapshots = snapshots;
_streamsFactory = new StreamsFactory(persistence);
if (processor != null)
{
_pollingClient = new PollingClient(
persistence,
0, // <----- TODO: read from state?
new LambdaSubscription(processor),
NStoreNullLoggerFactory.Instance
);
_pollingClient.Start();
}
}
// [InlineData(3, 3)] @@TODO enable tombstone!
public async Task poller_should_skip_missing_chunks(long missing, long expected)
{
await Store.AppendAsync("a", 1, "1").ConfigureAwait(false);
await Store.AppendAsync("a", 2, "2").ConfigureAwait(false);
await Store.AppendAsync("a", 3, "3").ConfigureAwait(false);
await Store.DeleteAsync("a", missing, missing).ConfigureAwait(false);
var recored = new AllPartitionsRecorder();
var poller = new PollingClient(Store, 0, recored, this.LoggerFactory)
{
HoleDetectionTimeout = 100
};
var cts = new CancellationTokenSource(20000);
await poller.Poll(cts.Token).ConfigureAwait(false);
await poller.Poll(cts.Token).ConfigureAwait(false);
Assert.Equal(expected, poller.Position);
}
private static void Main()
{
using (var store = WireupEventStore())
{
var client = new PollingClient(store.Advanced);
string checkpointToken = LoadCheckpoint();
using (IObserveCommits observeCommits = client.ObserveFrom(checkpointToken))
using (observeCommits.Subscribe(commit =>
{
// Project the commit etc
Console.WriteLine(Resources.CommitInfo, commit.BucketId, commit.StreamId, commit.CommitSequence);
// Track the most recent checkpoint
checkpointToken = commit.CheckpointToken;
}))
{
observeCommits.Start();
Console.WriteLine(Resources.PressAnyKey);
Console.ReadKey();
SaveCheckpoint(checkpointToken);
}
}
}
protected override void Because()
{
PollingClient.Start();
}
protected override void Cleanup()
{
PollingClient.Dispose();
}
private readonly INStoreLoggerFactory _loggerFactory = new ConsoleLoggerFactory(); // NStoreNullLoggerFactory.Instance;
public ProjectionEngine(IPersistence persistence, INetworkSimulator networkSimulator)
{
_pollingClient = new PollingClient(persistence, 0, new Projections(networkSimulator), _loggerFactory);
}
