public async Task Distributed_catchups_can_store_a_cursor_per_partition() { var cursorStore = new InMemoryProjectionStore<ICursor<int>>(_ => Cursor.New<int>()); var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet<string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .DistributeAmong(partitions, batchSize: 73, cursorPerPartition: cursorStore.Trace().AsHandler()); catchup.Subscribe(aggregator); await catchup.RunUntilCaughtUp(); cursorStore.Count().Should().Be(26); Enumerable.Range(1, 26).ToList().ForEach(i => { cursorStore.Should().Contain(c => c.Position == i*100); }); }
public async Task WithinPartition_by_range_correctly_partitions_queries_by_int() { var id = Guid.NewGuid().ToString(); await WriteEvents(i => new Event { SequenceNumber = i, Id = id }); var stream = Stream.PartitionedByRange <Event, int, int>(async(q, p) => { using (var db = new AlluvialSqlTestsDbContext()) { return(await db.Events .Where(e => e.Id == id) .WithinPartition(e => e.SequenceNumber, p) .ToArrayAsync()); } }); var catchup = stream.DistributeAmong(Partition.ByRange(0, 100).Among(5)); var store = new InMemoryProjectionStore <int>(); catchup.Subscribe(async(count, events) => events.Count, store); await catchup.RunSingleBatch(); store.Select(x => x).Should().BeEquivalentTo(new[] { 20, 20, 20, 20, 20 }); }
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup() { var store = new InMemoryProjectionStore<Projection<HashSet<string>, int>>(); var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet<string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .Trace() .DistributeAmong(partitions, batchSize: 15); catchup.Subscribe(aggregator, store.Trace()); await catchup.RunUntilCaughtUp(); partitions.ToList() .ForEach(partition => store.Should() .ContainSingle(projection => projection.Value.Count() == 100 && projection.Value.All(p => p.IsWithinPartition(partition)))); }
public async Task When_projections_are_cursors_then_catchup_does_not_replay_previously_seen_events() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var eventsAggregated = new List <IDomainEvent>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 100); catchup.Subscribe(new BalanceProjector() .Trace((p, es) => eventsAggregated.AddRange(es)), projectionStore); await catchup.RunUntilCaughtUp(); var streamId = streamIds.First(); store.WriteEvents(streamId, 100m); var cursor = await catchup.RunUntilCaughtUp(); cursor.Position .Should() .Be("101"); var balanceProjection = await projectionStore.Get(streamId); balanceProjection.Balance.Should().Be(101); balanceProjection.CursorPosition.Should().Be(2); eventsAggregated.Count.Should().Be(101); }
public async Task WithinPartition_by_range_correctly_partitions_queries_by_int() { var id = Guid.NewGuid().ToString(); await WriteEvents(i => new Event { SequenceNumber = i, Id = id }); var stream = Stream.PartitionedByRange<Event, int, int>(async (q, p) => { using (var db = new AlluvialSqlTestsDbContext()) { return await db.Events .Where(e => e.Id == id) .WithinPartition(e => e.SequenceNumber, p) .ToArrayAsync(); } }); var catchup = stream.DistributeAmong(Partition.ByRange(0, 100).Among(5)); var store = new InMemoryProjectionStore<int>(); catchup.Subscribe(async (count, events) => events.Count, store); await catchup.RunSingleBatch(); store.Select(x => x).Should().BeEquivalentTo(new[] { 20, 20, 20, 20, 20 }); }
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var count = 0; var projector = new BalanceProjector() .Pipeline(async(projection, batch, next) => { Interlocked.Increment(ref count); Console.WriteLine(count); if (count < 49) { Throw(); } await next(projection, batch); }).Trace(); var catchup = StreamCatchup.Create(stream.Trace(), batchSize: 50); catchup.Subscribe(projector, projectionStore.AsHandler(), onError: e => e.Continue()); await catchup.RunUntilCaughtUp(); projectionStore.Count().Should().Be(1); projectionStore.Single().CursorPosition.Should().Be(1); }
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup() { var store = new InMemoryProjectionStore <Projection <HashSet <string>, int> >(); var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet <string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .Trace() .DistributeAmong(partitions, batchSize: 15); catchup.Subscribe(aggregator, store.Trace()); await catchup.RunUntilCaughtUp(); partitions.ToList() .ForEach(partition => store.Should() .ContainSingle(projection => projection.Value.Count() == 100 && projection.Value.All(p => p.IsWithinPartition(partition)))); }
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var projector = new BalanceProjector() .Pipeline(async(projection, batch, next) => { if (projectionStore.Count() >= 30) { throw new Exception("oops"); } await next(projection, batch); }); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50); catchup.Subscribe(projector, projectionStore); Action runSingleBatch = () => catchup.RunSingleBatch().Wait(); runSingleBatch.ShouldThrow <Exception>() .And .Message .Should() .Contain("oops"); }
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup() { var count = 0; var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var projector = new BalanceProjector() .Pipeline(async(projection, batch, next) => { Interlocked.Increment(ref count); if (count < 20) { throw new Exception("oops"); } await next(projection, batch); }).Trace(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(), batchSize: 50); catchup.Subscribe(projector, projectionStore.AsHandler(), onError: e => e.Continue()); await catchup.RunSingleBatch(); projectionStore.Count().Should().Be(31); }
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written_to_existing_streams() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 5); catchup.Subscribe(new BalanceProjector(), projectionStore); using (catchup.Poll(TimeSpan.FromMilliseconds(10))) { // write more events Task.Run(async() => { foreach (var streamId in streamIds.Take(20)) { store.WriteEvents(streamId); await Task.Delay(1); } Console.WriteLine("wrote 20 more events"); }); await Wait.Until(() => { var sum = projectionStore.Sum(b => b.Balance); Console.WriteLine("sum is " + sum); return(sum >= 120); }); } }
public async Task When_a_partition_is_queried_then_the_cursor_is_updated() { var partitions = new[] { StreamQuery.Partition(0, 500), StreamQuery.Partition(500, 1000) }; var store = new InMemoryProjectionStore<Projection<int, int>>(); var aggregator = Aggregator.CreateFor<int, int>((p, i) => p.Value += i.Sum()); await Task.WhenAll(partitions.Select(async partition => { var stream = await partitioner.GetStream(partition); Console.WriteLine(stream); var catchup = StreamCatchup.Create(stream); catchup.Subscribe(aggregator, store); await catchup.RunSingleBatch(); })); store.Should() .ContainSingle(p => p.CursorPosition == 500) .And .ContainSingle(p => p.CursorPosition == 1000); }
public async Task Distributed_catchups_can_store_a_cursor_per_partition() { var cursorStore = new InMemoryProjectionStore <ICursor <int> >(_ => Cursor.New <int>()); var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet <string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .DistributeAmong(partitions, batchSize: 73, cursorPerPartition: cursorStore.Trace().AsHandler()); catchup.Subscribe(aggregator); await catchup.RunUntilCaughtUp(); cursorStore.Count().Should().Be(26); Enumerable.Range(1, 26).ToList().ForEach(i => { cursorStore.Should().Contain(c => c.Position == i * 100); }); }
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); store.WriteEvents(streamId, 100); var projector = new BalanceProjector() .Pipeline(async(projection, batch, next) => { Throw(); await next(projection, batch); }); var catchup = StreamCatchup.Create(stream, batchSize: 50); catchup.Subscribe(projector, projectionStore); Action runSingleBatch = () => catchup.RunSingleBatch().Wait(); runSingleBatch.ShouldThrow <Exception>() .And .Message .Should() .Contain("oops"); }
public async Task By_default_ProjectionStore_Trace_writes_projections_during_Put_to_trace_output() { var store = new InMemoryProjectionStore <BalanceProjection>(); await store.Trace().Put("the-stream-id", new BalanceProjection()); traceListener.Messages .Should() .Contain("[Put] Projection(IDomainEvent,Int32): null @ cursor 0 for stream the-stream-id"); }
public async Task When_one_projection_fails_its_cursor_is_not_advanced_while_other_projections_cursors_are_advanced() { var projections = new InMemoryProjectionStore <BalanceProjection>(); // first catch up all the projections var stream = streamSource.StreamPerAggregate().Trace(); var catchup = StreamCatchup.All(stream); var initialSubscription = catchup.Subscribe(new BalanceProjector(), projections); await catchup.RunUntilCaughtUp(); initialSubscription.Dispose(); // write some additional events var streamIdsWithoutErrors = streamIds.Take(5).ToList(); var streamIdsWithErrors = streamIds.Skip(5).Take(5).ToList(); foreach (var streamId in streamIdsWithoutErrors.Concat(streamIdsWithErrors)) { store.WriteEvents(streamId, howMany: 10); } // subscribe a flaky projector catchup.Subscribe(new BalanceProjector() .Pipeline(async(projection, batch, next) => { var aggregateId = batch.Select(i => i.AggregateId).First(); if (streamIdsWithErrors.Contains(aggregateId)) { throw new Exception("oops"); } await next(projection, batch); }), projections.AsHandler(), e => e.Continue()); await catchup.RunSingleBatch(); var projectionsWithoutErrors = streamIdsWithoutErrors.Select( id => projections.Get(id).Result); var projectionsWithErrors = streamIdsWithErrors.Select( id => projections.Get(id).Result); foreach (var projection in projectionsWithoutErrors) { projection.CursorPosition.Should().Be(11); projection.Balance.Should().Be(11); } foreach (var projection in projectionsWithErrors) { projection.CursorPosition.Should().Be(1); projection.Balance.Should().Be(1); } }
public async Task GuidQueryPartitioner_partitions_guids_fairly() { var totalNumberOfGuids = 1000; var numberOfPartitions = 50; var partitions = Partition.AllGuids() .Among(numberOfPartitions); var guids = Enumerable.Range(1, totalNumberOfGuids).Select(_ => Guid.NewGuid()).ToArray(); var partitioned = Stream.Partitioned <Guid, int, Guid>( async(q, p) => guids.Where(g => g.IsWithinPartition(p)), advanceCursor: (q, b) => q.Cursor.AdvanceTo(totalNumberOfGuids)); var aggregator = Aggregator.Create <Projection <HashSet <Guid>, int>, Guid>((p, b) => { if (p.Value == null) { p.Value = new HashSet <Guid>(); } foreach (var guid in b) { p.Value.Add(guid); } }); var store = new InMemoryProjectionStore <Projection <HashSet <Guid>, int> >(); await Task.WhenAll(partitions.Select(async partition => { var stream = await partitioned.GetStream(partition); var catchup = StreamCatchup.Create(stream, batchSize: int.MaxValue); catchup.Subscribe(aggregator, store); await catchup.RunSingleBatch(); var projection = await store.Get(stream.Id); Console.WriteLine(partition + ": " + projection.Value.Count); })); var approximateGuidsPerPartition = totalNumberOfGuids / numberOfPartitions; var tolerance = (int)(totalNumberOfGuids * .12); store.Sum(p => p.Value.Count).Should().Be(totalNumberOfGuids); store.ToList().ForEach(projection => { projection.Value .Count .Should() .BeInRange(approximateGuidsPerPartition - tolerance, approximateGuidsPerPartition + tolerance); }); }
public async Task When_multiple_projectors_are_subscribed_then_data_that_both_projections_have_seen_is_not_requeried() { var streamId = Guid.NewGuid().ToString(); var queriedEvents = new ConcurrentBag <IDomainEvent>(); var balanceProjections = new InMemoryProjectionStore <BalanceProjection>(); await balanceProjections.Put(streamId, new BalanceProjection { CursorPosition = 2 }); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate() .Map(ss => ss.Select(s => s.Trace(onResults: (q, b) => { foreach (var e in b) { queriedEvents.Add(e); } }))), Cursor.New("100"), batchSize: 1); catchup.Subscribe(new BalanceProjector(), balanceProjections); store.WriteEvents(streamId); // "101" - 1 store.WriteEvents(streamId); // "102" - 2 store.WriteEvents(streamId); // "103" - 3 await catchup.RunSingleBatch(); queriedEvents.Count .Should() .Be(1, "the first two events should be skipped because of the starting cursor position"); queriedEvents.Should() .ContainSingle(e => e.StreamRevision == 3, "only the most recent event should be queried"); var accountHistoryProjections = new InMemoryProjectionStore <AccountHistoryProjection>(); await accountHistoryProjections.Put(streamId, new AccountHistoryProjection { CursorPosition = 2 }); catchup.Subscribe(new AccountHistoryProjector(), accountHistoryProjections); store.WriteEvents(streamId); await catchup.RunSingleBatch(); queriedEvents.Select(e => e.StreamRevision) .ShouldBeEquivalentTo(new[] { 3, 4 }, "event 3 needs to be repeated because the newly-subscribed aggregator hasn't seen it yet"); }
public async Task When_one_projection_fails_its_cursor_is_not_advanced_while_other_projections_cursors_are_advanced() { var projections = new InMemoryProjectionStore<BalanceProjection>(); // first catch up all the projections var stream = streamSource.StreamPerAggregate().Trace(); var catchup = StreamCatchup.All(stream); var initialSubscription = catchup.Subscribe(new BalanceProjector(), projections); await catchup.RunUntilCaughtUp(); initialSubscription.Dispose(); // write some additional events var streamIdsWithoutErrors = streamIds.Take(5).ToList(); var streamIdsWithErrors = streamIds.Skip(5).Take(5).ToList(); foreach (var streamId in streamIdsWithoutErrors.Concat(streamIdsWithErrors)) { store.WriteEvents(streamId, howMany: 10); } // subscribe a flaky projector catchup.Subscribe(new BalanceProjector() .Pipeline(async (projection, batch, next) => { var aggregateId = batch.Select(i => i.AggregateId).First(); if (streamIdsWithErrors.Contains(aggregateId)) { throw new Exception("oops"); } await next(projection, batch); }), projections.AsHandler(), e => e.Continue()); await catchup.RunSingleBatch(); var projectionsWithoutErrors = streamIdsWithoutErrors.Select( id => projections.Get(id).Result); var projectionsWithErrors = streamIdsWithErrors.Select( id => projections.Get(id).Result); foreach (var projection in projectionsWithoutErrors) { projection.CursorPosition.Should().Be(11); projection.Balance.Should().Be(11); } foreach (var projection in projectionsWithErrors) { projection.CursorPosition.Should().Be(1); projection.Balance.Should().Be(1); } }
public async Task Catchup_upstream_batch_size_can_be_specified() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 20); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Count() .Should() .Be(20); }
public async Task Catchup_batch_size_can_be_specified() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); store.WriteEvents(streamId, howMany: 50); var catchup = StreamCatchup.Create(stream, batchCount: 20); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Single() .Balance .Should() .Be(20); }
public async Task Catchup_upstream_batch_size_can_be_specified() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 20); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Count() .Should() .Be(20); }
public async Task Catchup_can_traverse_all_events() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); store.WriteEvents(streamId, 999); var catchup = StreamCatchup.Create(stream); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(1000); projectionStore.Count() .Should() .Be(1); }
public async Task Catchup_can_use_a_sequence_of_keys_to_traverse_all_aggregates() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate()); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(100); projectionStore.Count() .Should() .Be(100); }
public async Task Catchup_can_use_a_sequence_of_keys_to_traverse_all_aggregates() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate()); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(100); projectionStore.Count() .Should() .Be(100); }
public async Task Catchup_batch_size_can_be_specified() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); store.WriteEvents(streamId, howMany: 50); var catchup = StreamCatchup.Create(stream, batchSize: 20); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Single() .Balance .Should() .Be(20); }
public async Task Catchup_can_traverse_all_events() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); store.WriteEvents(streamId, 999); var catchup = StreamCatchup.Create(stream); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(1000); projectionStore.Count() .Should() .Be(1); }
public async Task When_one_batch_is_running_a_second_caller_to_RunSingleBatch_can_await_the_completion_of_the_same_batch() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.Create(stream, batchSize: 1); catchup.Subscribe(new BalanceProjector() .Pipeline(async (projection, batch, next) => { await Task.Delay(500); await next(projection, batch); }), projectionStore); var cursor1 = catchup.RunSingleBatch(); var cursor2 = catchup.RunSingleBatch(); (await cursor1).Should() .BeSameAs(await cursor2); }
public async Task An_initial_cursor_can_be_specified() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); store.WriteEvents(streamId, howMany: 999); var catchup = StreamCatchup.Create(stream, initialCursor: Cursor.New(800), batchSize: 1000); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(200); }
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); store.WriteEvents(streamId, howMany: 999); var catchup = StreamCatchup.Create(stream, batchSize: 10); catchup.Subscribe(new BalanceProjector(), projectionStore); TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception); await catchup.RunUntilCaughtUp(); projectionStore.Single() .Balance .Should() .Be(1000); }
public async Task When_one_batch_is_running_a_second_call_to_RunSingleBatch_will_not_do_anything() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.Create(stream, batchCount: 1); catchup.Subscribe(new BalanceProjector() .Pipeline(async (projection, batch, next) => { await Task.Delay(500); await next(projection, batch); }), projectionStore); await Task.WhenAll(catchup.RunSingleBatch(), catchup.RunSingleBatch()); projectionStore.Count() .Should() .Be(1); }
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 10); catchup.Subscribe(new BalanceProjector(), projectionStore); TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception); await catchup.RunUntilCaughtUp(); projectionStore.Sum(b => b.Balance) .Should() .Be(100); projectionStore.Count() .Should() .Be(100); }
public async Task When_one_batch_is_running_a_second_caller_to_RunSingleBatch_can_await_the_completion_of_the_same_batch() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.Create(stream, batchSize: 1); catchup.Subscribe(new BalanceProjector() .Pipeline(async(projection, batch, next) => { await Task.Delay(500); await next(projection, batch); }), projectionStore); var cursor1 = catchup.RunSingleBatch(); var cursor2 = catchup.RunSingleBatch(); (await cursor1).Should() .BeSameAs(await cursor2); }
public async Task When_one_batch_is_running_a_second_call_to_RunSingleBatch_will_not_do_anything() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.Create(stream, batchSize: 1); catchup.Subscribe(new BalanceProjector() .Pipeline(async(projection, batch, next) => { await Task.Delay(500); await next(projection, batch); }), projectionStore); await Task.WhenAll(catchup.RunSingleBatch(), catchup.RunSingleBatch()); projectionStore.Count() .Should() .Be(1); }
public async Task Stream_traversal_can_continue_from_upstream_cursor_that_was_returned_by_RunSingleBatch() { var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50); catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore <BalanceProjection>()); var cursor = await catchup.RunSingleBatch(); var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), cursor); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Count() .Should() .Be(50); }
public async Task Catchup_query_cursor_resumes_from_last_position() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(), batchSize: 50); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(50); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(100); }
public async Task Catchup_starting_cursor_can_be_specified() { var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50); catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore <BalanceProjection>()); var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var updatedStreams = streamSource.StreamPerAggregate(); var cursor = updatedStreams.NewCursor(); cursor.AdvanceTo("50"); catchup = StreamCatchup.All(updatedStreams, cursor); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Count() .Should() .Be(50); }
public async Task WithinPartition_by_range_correctly_partitions_queries_by_string() { var guid = Guid.NewGuid(); var partitions = new[] { Partition.ByRange("", "mm"), Partition.ByRange("mm", "zz") }; Values.AtoZ() .ToList() .ForEach(c => WriteEvents(i => new Event { SequenceNumber = i, Guid = guid, Id = c + " " + Guid.NewGuid() }, 10).Wait()); var stream = Stream.PartitionedByRange <Event, int, string>(async(q, p) => { using (var db = new AlluvialSqlTestsDbContext()) { return(await db.Events .Where(e => e.Guid == guid) .WithinPartition(e => e.Id, p) .ToArrayAsync()); } }); var catchup = stream.DistributeAmong(partitions); var store = new InMemoryProjectionStore <int>(); catchup.Subscribe(async(count, events) => events.Count, store); await catchup.RunSingleBatch(); store.Select(x => x).Should().BeEquivalentTo(new[] { 130, 130 }); }
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.Create(stream, batchSize: 50); catchup.Subscribe(new BalanceProjector().Trace(), projectionStore); using (catchup.Poll(TimeSpan.FromMilliseconds(10))) { using (Background.Loop(_ => store.WriteEvents(streamId, howMany: 2), .1)) { await Wait.Until(() => { var sum = projectionStore.Sum(b => b.Balance); Console.WriteLine("sum is " + sum); return(sum >= 500); }); } } }
public async Task Catchup_query_cursor_resumes_from_last_position() { var projectionStore = new InMemoryProjectionStore <BalanceProjection>(); store.WriteEvents(streamId, howMany: 999); var catchup = StreamCatchup.Create(stream, batchSize: 500); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(500); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(1000); }
public async Task When_an_aggregation_fails_then_the_projection_is_not_updated() { var projections = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace()); // subscribe a flaky projector catchup.Subscribe(new BalanceProjector() .Pipeline(async (projection, batch, next) => { bool shouldThrow = true; // declared outside to nudge the compiler into inferring the correct .Pipeline overload if (shouldThrow) { throw new Exception("oops"); } await next(projection, batch); }), projections.AsHandler(), e => e.Continue()); await catchup.RunSingleBatch(); projections.Count().Should().Be(0); }
public async Task When_a_partition_is_queried_then_the_cursor_is_updated() { var partitions = Partition.ByRange(0, 1000).Among(2); var store = new InMemoryProjectionStore <Projection <int, int> >(); var aggregator = Aggregator.Create <Projection <int, int>, int>((p, i) => p.Value += i.Sum()); await Task.WhenAll(partitions.Select(async partition => { var stream = await partitionedStream.GetStream(partition); Console.WriteLine(stream); var catchup = StreamCatchup.Create(stream); catchup.Subscribe(aggregator, store); await catchup.RunSingleBatch(); })); store.Should() .ContainSingle(p => p.CursorPosition == 500) .And .ContainSingle(p => p.CursorPosition == 1000); }
public async Task When_an_aggregation_fails_then_the_projection_is_not_updated() { var projections = new InMemoryProjectionStore <BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace()); // subscribe a flaky projector catchup.Subscribe(new BalanceProjector() .Pipeline(async(projection, batch, next) => { bool shouldThrow = true; // declared outside to nudge the compiler into inferring the correct .Pipeline overload if (shouldThrow) { throw new Exception("oops"); } await next(projection, batch); }), projections.AsHandler(), e => e.Continue()); await catchup.RunSingleBatch(); projections.Count().Should().Be(0); }
public async Task Catchup_query_cursor_resumes_from_last_position() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); store.WriteEvents(streamId, howMany: 999); var catchup = StreamCatchup.Create(stream, batchCount: 500); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(500); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(1000); }
public async Task Catchup_starting_cursor_can_be_specified() { var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50); catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore<BalanceProjection>()); var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var updatedStreams = streamSource.StreamPerAggregate(); var cursor = updatedStreams.NewCursor(); cursor.AdvanceTo("50"); catchup = StreamCatchup.All(updatedStreams, cursor); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Count() .Should() .Be(50); }
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup() { var count = 0; var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var projector = new BalanceProjector() .Pipeline(async (projection, batch, next) => { Interlocked.Increment(ref count); if (count < 20) { throw new Exception("oops"); } await next(projection, batch); }).Trace(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(), batchSize: 50); catchup.Subscribe(projector, projectionStore.AsHandler(), onError: e => e.Continue()); await catchup.RunSingleBatch(); projectionStore.Count().Should().Be(31); }
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var projector = new BalanceProjector() .Pipeline(async (projection, batch, next) => { if (projectionStore.Count() >= 30) { throw new Exception("oops"); } await next(projection, batch); }); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50); catchup.Subscribe(projector, projectionStore); Action runSingleBatch = () => catchup.RunSingleBatch().Wait(); runSingleBatch.ShouldThrow<Exception>() .And .Message .Should() .Contain("oops"); }
public async Task When_multiple_projectors_are_subscribed_then_data_that_both_projections_have_seen_is_not_requeried() { var queriedEvents = new ConcurrentBag<IDomainEvent>(); var balanceProjections = new InMemoryProjectionStore<BalanceProjection>(); await balanceProjections.Put(streamId, new BalanceProjection { CursorPosition = 2 }); var catchup = StreamCatchup.Create(stream.Trace(onResults: (q, b) => { foreach (var e in b) { queriedEvents.Add(e); } }), batchCount: 10); catchup.Subscribe(new BalanceProjector(), balanceProjections); store.WriteEvents(streamId, howMany: 2); await catchup.RunSingleBatch(); queriedEvents.Count .Should() .Be(1, "the first two events should be skipped because of the starting cursor position"); queriedEvents.Should() .ContainSingle(e => e.StreamRevision == 3, "only the most recent event should be queried"); var accountHistoryProjections = new InMemoryProjectionStore<AccountHistoryProjection>(); await accountHistoryProjections.Put(streamId, new AccountHistoryProjection { CursorPosition = 2 }); catchup.Subscribe(new AccountHistoryProjector(), accountHistoryProjections); store.WriteEvents(streamId); await catchup.RunSingleBatch(); queriedEvents.Select(e => e.StreamRevision) .ShouldBeEquivalentTo(new[] { 3, 3, 4 }, "event 3 needs to be repeated because the newly-subscribed aggregator hasn't seen it yet"); }
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor() { var partitions = Enumerable.Range(0, 9) .Select(i => StreamQuery.Partition(i*100, (i + 1)*100)) .ToArray(); var store = new InMemoryProjectionStore<Projection<HashSet<int>, int>>(); var aggregator = Aggregator.CreateFor<HashSet<int>, int>((p, xs) => { if (p.Value == null) { p.Value = new HashSet<int>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); // set up 10 competing catchups for (var i = 0; i < 10; i++) { var distributor = new InMemoryStreamQueryDistributor( partitions.Select(p => new LeasableResource(p.ToString(), TimeSpan.FromSeconds(10))).ToArray(), "") .Trace(); distributor.OnReceive(async lease => { var partition = partitions.Single(p => p.ToString() == lease.LeasableResource.Name); var catchup = StreamCatchup.Create(await partitioner.GetStream(partition)); catchup.Subscribe(aggregator, store.Trace()); await catchup.RunSingleBatch(); }); distributor.Start(); disposables.Add(distributor); } partitions.ToList() .ForEach(partition => store.Should() .ContainSingle(p => p.Value.Count() == 100 && p.Value.OrderBy(i => i).Last() == partition.UpperBoundInclusive)); }
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.Create(stream, batchCount: 50); catchup.Subscribe(new BalanceProjector().Trace(), projectionStore); using (catchup.Poll(TimeSpan.FromMilliseconds(10))) { using (Background.Loop(_ => store.WriteEvents(streamId, howMany: 2), .1)) { await Wait.Until(() => { var sum = projectionStore.Sum(b => b.Balance); Console.WriteLine("sum is " + sum); return sum >= 500; }); } } }
public async Task When_projections_are_cursors_then_catchup_does_not_replay_previously_seen_events() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var eventsAggregated = new List<IDomainEvent>(); var catchup = StreamCatchup.Create(stream, batchCount: 100); catchup.Subscribe(new BalanceProjector().Trace((p, es) => eventsAggregated.AddRange(es)), projectionStore); await catchup.RunUntilCaughtUp(); store.WriteEvents(streamId, howMany: 100); var cursor = await catchup.RunUntilCaughtUp(); cursor.Position .Should() .Be(101); var balanceProjection = await projectionStore.Get(streamId); balanceProjection.Balance.Should().Be(101); balanceProjection.CursorPosition.Should().Be(101); eventsAggregated.Count.Should().Be(101); }
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); store.WriteEvents(streamId, howMany: 999); var catchup = StreamCatchup.Create(stream, batchCount: 10); catchup.Subscribe(new BalanceProjector(), projectionStore); TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception); await catchup.RunUntilCaughtUp(); projectionStore.Single() .Balance .Should() .Be(1000); }
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written_to_existing_streams() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 5); catchup.Subscribe(new BalanceProjector(), projectionStore); using (catchup.Poll(TimeSpan.FromMilliseconds(10))) { // write more events Task.Run(async () => { foreach (var streamId in streamIds.Take(20)) { store.WriteEvents(streamId); await Task.Delay(1); } Console.WriteLine("wrote 20 more events"); }); await Wait.Until(() => { var sum = projectionStore.Sum(b => b.Balance); Console.WriteLine("sum is " + sum); return sum >= 120; }); } }
public async Task GuidQueryPartitioner_partitions_guids_fairly() { var totalNumberOfGuids = 1000; var numberOfPartitions = 50; var partitions = StreamQuery.Partition( Guid.Empty, Guid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff")) .Among(numberOfPartitions); var guids = Enumerable.Range(1, totalNumberOfGuids).Select(_ => Guid.NewGuid()).ToArray(); var partitioner = Stream.Partition<Guid, int, Guid>( async (q, p) => guids.Where(g => new SqlGuid(g).CompareTo(new SqlGuid(p.LowerBoundExclusive)) > 0 && new SqlGuid(g).CompareTo(new SqlGuid(p.UpperBoundInclusive)) <= 0), advanceCursor: (q, b) => q.Cursor.AdvanceTo(totalNumberOfGuids)); var aggregator = Aggregator.Create<Projection<HashSet<Guid>, int>, Guid>((p, b) => { if (p.Value == null) { p.Value = new HashSet<Guid>(); } foreach (var guid in b) { p.Value.Add(guid); } }); var store = new InMemoryProjectionStore<Projection<HashSet<Guid>, int>>(); await Task.WhenAll(partitions.Select(async partition => { var stream = await partitioner.GetStream(partition); var catchup = StreamCatchup.Create(stream, batchCount: int.MaxValue); catchup.Subscribe(aggregator, store); await catchup.RunSingleBatch(); await Task.Delay(500); var projection = await store.Get(stream.Id); Console.WriteLine(partition + ": " + projection.Value.Count); })); var approximateGuidsPerPartition = totalNumberOfGuids/numberOfPartitions; var tolerance = (int) (totalNumberOfGuids*.12); Console.WriteLine("\nMissing guids: "); foreach (var guid in guids.Where(g => !store.Any(p => p.Value.Contains(g)))) { Console.WriteLine(" " + guid); } store.Sum(p => p.Value.Count).Should().Be(totalNumberOfGuids); store.ToList().ForEach(projection => { projection.Value .Count .Should() .BeInRange(approximateGuidsPerPartition - tolerance, approximateGuidsPerPartition + tolerance); }); }
public async Task WithinPartition_by_range_correctly_partitions_queries_by_string() { var guid = Guid.NewGuid(); var partitions = new[] { Partition.ByRange("", "mm"), Partition.ByRange("mm", "zz") }; Values.AtoZ() .ToList() .ForEach(c => WriteEvents(i => new Event { SequenceNumber = i, Guid = guid, Id = c + " " + Guid.NewGuid() }, 10).Wait()); var stream = Stream.PartitionedByRange<Event, int, string>(async (q, p) => { using (var db = new AlluvialSqlTestsDbContext()) { return await db.Events .Where(e => e.Guid == guid) .WithinPartition(e => e.Id, p) .ToArrayAsync(); } }); var catchup = stream.DistributeAmong(partitions); var store = new InMemoryProjectionStore<int>(); catchup.Subscribe(async (count, events) => events.Count, store); await catchup.RunSingleBatch(); store.Select(x => x).Should().BeEquivalentTo(new[] { 130, 130 }); }
public async Task Catchup_query_cursor_resumes_from_last_position() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(), batchSize: 50); catchup.Subscribe(new BalanceProjector(), projectionStore); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(50); await catchup.RunSingleBatch(); projectionStore.Sum(b => b.Balance) .Should() .Be(100); }
public async Task By_default_ProjectionStore_Trace_writes_projections_during_Put_to_trace_output() { var store = new InMemoryProjectionStore<BalanceProjection>(); await store.Trace().Put("the-stream-id", new BalanceProjection()); traceListener.Messages .Should() .Contain("[Put] Projection(IDomainEvent,Int32): null @ cursor 0 for stream the-stream-id"); }
public async Task GuidQueryPartitioner_partitions_guids_fairly() { var totalNumberOfGuids = 1000; var numberOfPartitions = 50; var partitions = Partition.AllGuids() .Among(numberOfPartitions); var guids = Enumerable.Range(1, totalNumberOfGuids).Select(_ => Guid.NewGuid()).ToArray(); var partitioned = Stream.Partitioned<Guid, int, Guid>( async (q, p) => guids.Where(g => g.IsWithinPartition(p)), advanceCursor: (q, b) => q.Cursor.AdvanceTo(totalNumberOfGuids)); var aggregator = Aggregator.Create<Projection<HashSet<Guid>, int>, Guid>((p, b) => { if (p.Value == null) { p.Value = new HashSet<Guid>(); } foreach (var guid in b) { p.Value.Add(guid); } }); var store = new InMemoryProjectionStore<Projection<HashSet<Guid>, int>>(); await Task.WhenAll(partitions.Select(async partition => { var stream = await partitioned.GetStream(partition); var catchup = StreamCatchup.Create(stream, batchSize: int.MaxValue); catchup.Subscribe(aggregator, store); await catchup.RunSingleBatch(); var projection = await store.Get(stream.Id); Console.WriteLine(partition + ": " + projection.Value.Count); })); var approximateGuidsPerPartition = totalNumberOfGuids/numberOfPartitions; var tolerance = (int) (totalNumberOfGuids*.12); store.Sum(p => p.Value.Count).Should().Be(totalNumberOfGuids); store.ToList().ForEach(projection => { projection.Value .Count .Should() .BeInRange(approximateGuidsPerPartition - tolerance, approximateGuidsPerPartition + tolerance); }); }
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); store.WriteEvents(streamId, 100); var projector = new BalanceProjector() .Pipeline(async (projection, batch, next) => { Throw(); await next(projection, batch); }); var catchup = StreamCatchup.Create(stream, batchCount: 50); catchup.Subscribe(projector, projectionStore); Action runSingleBatch = () => catchup.RunSingleBatch().Wait(); runSingleBatch.ShouldThrow<Exception>() .And .Message .Should() .Contain("oops"); }
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var count = 0; var projector = new BalanceProjector() .Pipeline(async (projection, batch, next) => { Interlocked.Increment(ref count); Console.WriteLine(count); if (count < 49) { Throw(); } await next(projection, batch); }).Trace(); var catchup = StreamCatchup.Create(stream.Trace(), batchCount: 50); catchup.Subscribe(projector, projectionStore.AsHandler(), onError: e => e.Continue()); await catchup.RunUntilCaughtUp(); projectionStore.Count().Should().Be(1); projectionStore.Single().CursorPosition.Should().Be(1); }
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results() { var projectionStore = new InMemoryProjectionStore<BalanceProjection>(); var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 10); catchup.Subscribe(new BalanceProjector(), projectionStore); TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception); await catchup.RunUntilCaughtUp(); projectionStore.Sum(b => b.Balance) .Should() .Be(100); projectionStore.Count() .Should() .Be(100); }