public async Task Distributed_catchups_can_store_a_cursor_per_partition()
{
var cursorStore = new InMemoryProjectionStore<ICursor<int>>(_ => Cursor.New<int>());
var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet<string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.DistributeAmong(partitions,
batchSize: 73,
cursorPerPartition: cursorStore.Trace().AsHandler());
catchup.Subscribe(aggregator);
await catchup.RunUntilCaughtUp();
cursorStore.Count().Should().Be(26);
Enumerable.Range(1, 26).ToList().ForEach(i =>
{
cursorStore.Should().Contain(c => c.Position == i*100);
});
}
public async Task WithinPartition_by_range_correctly_partitions_queries_by_int()
{
var id = Guid.NewGuid().ToString();
await WriteEvents(i => new Event
{
SequenceNumber = i,
Id = id
});
var stream = Stream.PartitionedByRange <Event, int, int>(async(q, p) =>
{
using (var db = new AlluvialSqlTestsDbContext())
{
return(await db.Events
.Where(e => e.Id == id)
.WithinPartition(e => e.SequenceNumber, p)
.ToArrayAsync());
}
});
var catchup = stream.DistributeAmong(Partition.ByRange(0, 100).Among(5));
var store = new InMemoryProjectionStore <int>();
catchup.Subscribe(async(count, events) => events.Count, store);
await catchup.RunSingleBatch();
store.Select(x => x).Should().BeEquivalentTo(new[] { 20, 20, 20, 20, 20 });
}
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup()
{
var store = new InMemoryProjectionStore<Projection<HashSet<string>, int>>();
var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet<string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.Trace()
.DistributeAmong(partitions, batchSize: 15);
catchup.Subscribe(aggregator, store.Trace());
await catchup.RunUntilCaughtUp();
partitions.ToList()
.ForEach(partition =>
store.Should()
.ContainSingle(projection =>
projection.Value.Count() == 100 &&
projection.Value.All(p => p.IsWithinPartition(partition))));
}
public async Task When_projections_are_cursors_then_catchup_does_not_replay_previously_seen_events()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var eventsAggregated = new List <IDomainEvent>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 100);
catchup.Subscribe(new BalanceProjector()
.Trace((p, es) => eventsAggregated.AddRange(es)), projectionStore);
await catchup.RunUntilCaughtUp();
var streamId = streamIds.First();
store.WriteEvents(streamId, 100m);
var cursor = await catchup.RunUntilCaughtUp();
cursor.Position
.Should()
.Be("101");
var balanceProjection = await projectionStore.Get(streamId);
balanceProjection.Balance.Should().Be(101);
balanceProjection.CursorPosition.Should().Be(2);
eventsAggregated.Count.Should().Be(101);
}
public async Task WithinPartition_by_range_correctly_partitions_queries_by_int()
{
var id = Guid.NewGuid().ToString();
await WriteEvents(i => new Event
{
SequenceNumber = i,
Id = id
});
var stream = Stream.PartitionedByRange<Event, int, int>(async (q, p) =>
{
using (var db = new AlluvialSqlTestsDbContext())
{
return await db.Events
.Where(e => e.Id == id)
.WithinPartition(e => e.SequenceNumber, p)
.ToArrayAsync();
}
});
var catchup = stream.DistributeAmong(Partition.ByRange(0, 100).Among(5));
var store = new InMemoryProjectionStore<int>();
catchup.Subscribe(async (count, events) => events.Count, store);
await catchup.RunSingleBatch();
store.Select(x => x).Should().BeEquivalentTo(new[] { 20, 20, 20, 20, 20 });
}
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var count = 0;
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
Interlocked.Increment(ref count);
Console.WriteLine(count);
if (count < 49)
{
Throw();
}
await next(projection, batch);
}).Trace();
var catchup = StreamCatchup.Create(stream.Trace(),
batchSize: 50);
catchup.Subscribe(projector,
projectionStore.AsHandler(),
onError: e => e.Continue());
await catchup.RunUntilCaughtUp();
projectionStore.Count().Should().Be(1);
projectionStore.Single().CursorPosition.Should().Be(1);
}
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup()
{
var store = new InMemoryProjectionStore <Projection <HashSet <string>, int> >();
var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet <string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.Trace()
.DistributeAmong(partitions, batchSize: 15);
catchup.Subscribe(aggregator, store.Trace());
await catchup.RunUntilCaughtUp();
partitions.ToList()
.ForEach(partition =>
store.Should()
.ContainSingle(projection =>
projection.Value.Count() == 100 &&
projection.Value.All(p => p.IsWithinPartition(partition))));
}
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
if (projectionStore.Count() >= 30)
{
throw new Exception("oops");
}
await next(projection, batch);
});
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(projector, projectionStore);
Action runSingleBatch = () => catchup.RunSingleBatch().Wait();
runSingleBatch.ShouldThrow <Exception>()
.And
.Message
.Should()
.Contain("oops");
}
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup()
{
var count = 0;
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
Interlocked.Increment(ref count);
if (count < 20)
{
throw new Exception("oops");
}
await next(projection, batch);
}).Trace();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(),
batchSize: 50);
catchup.Subscribe(projector,
projectionStore.AsHandler(),
onError: e => e.Continue());
await catchup.RunSingleBatch();
projectionStore.Count().Should().Be(31);
}
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written_to_existing_streams()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 5);
catchup.Subscribe(new BalanceProjector(), projectionStore);
using (catchup.Poll(TimeSpan.FromMilliseconds(10)))
{
// write more events
Task.Run(async() =>
{
foreach (var streamId in streamIds.Take(20))
{
store.WriteEvents(streamId);
await Task.Delay(1);
}
Console.WriteLine("wrote 20 more events");
});
await Wait.Until(() =>
{
var sum = projectionStore.Sum(b => b.Balance);
Console.WriteLine("sum is " + sum);
return(sum >= 120);
});
}
}
public async Task When_a_partition_is_queried_then_the_cursor_is_updated()
{
var partitions = new[]
{
StreamQuery.Partition(0, 500),
StreamQuery.Partition(500, 1000)
};
var store = new InMemoryProjectionStore<Projection<int, int>>();
var aggregator = Aggregator.CreateFor<int, int>((p, i) => p.Value += i.Sum());
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitioner.GetStream(partition);
Console.WriteLine(stream);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
}));
store.Should()
.ContainSingle(p => p.CursorPosition == 500)
.And
.ContainSingle(p => p.CursorPosition == 1000);
}
public async Task Distributed_catchups_can_store_a_cursor_per_partition()
{
var cursorStore = new InMemoryProjectionStore <ICursor <int> >(_ => Cursor.New <int>());
var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet <string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.DistributeAmong(partitions,
batchSize: 73,
cursorPerPartition: cursorStore.Trace().AsHandler());
catchup.Subscribe(aggregator);
await catchup.RunUntilCaughtUp();
cursorStore.Count().Should().Be(26);
Enumerable.Range(1, 26).ToList().ForEach(i =>
{
cursorStore.Should().Contain(c => c.Position == i * 100);
});
}
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, 100);
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
Throw();
await next(projection, batch);
});
var catchup = StreamCatchup.Create(stream, batchSize: 50);
catchup.Subscribe(projector, projectionStore);
Action runSingleBatch = () => catchup.RunSingleBatch().Wait();
runSingleBatch.ShouldThrow <Exception>()
.And
.Message
.Should()
.Contain("oops");
}
public async Task By_default_ProjectionStore_Trace_writes_projections_during_Put_to_trace_output()
{
var store = new InMemoryProjectionStore <BalanceProjection>();
await store.Trace().Put("the-stream-id", new BalanceProjection());
traceListener.Messages
.Should()
.Contain("[Put] Projection(IDomainEvent,Int32): null @ cursor 0 for stream the-stream-id");
}
public async Task When_one_projection_fails_its_cursor_is_not_advanced_while_other_projections_cursors_are_advanced()
{
var projections = new InMemoryProjectionStore <BalanceProjection>();
// first catch up all the projections
var stream = streamSource.StreamPerAggregate().Trace();
var catchup = StreamCatchup.All(stream);
var initialSubscription = catchup.Subscribe(new BalanceProjector(), projections);
await catchup.RunUntilCaughtUp();
initialSubscription.Dispose();
// write some additional events
var streamIdsWithoutErrors = streamIds.Take(5).ToList();
var streamIdsWithErrors = streamIds.Skip(5).Take(5).ToList();
foreach (var streamId in streamIdsWithoutErrors.Concat(streamIdsWithErrors))
{
store.WriteEvents(streamId, howMany: 10);
}
// subscribe a flaky projector
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
var aggregateId = batch.Select(i => i.AggregateId).First();
if (streamIdsWithErrors.Contains(aggregateId))
{
throw new Exception("oops");
}
await next(projection, batch);
}),
projections.AsHandler(),
e => e.Continue());
await catchup.RunSingleBatch();
var projectionsWithoutErrors = streamIdsWithoutErrors.Select(
id => projections.Get(id).Result);
var projectionsWithErrors = streamIdsWithErrors.Select(
id => projections.Get(id).Result);
foreach (var projection in projectionsWithoutErrors)
{
projection.CursorPosition.Should().Be(11);
projection.Balance.Should().Be(11);
}
foreach (var projection in projectionsWithErrors)
{
projection.CursorPosition.Should().Be(1);
projection.Balance.Should().Be(1);
}
}
public async Task GuidQueryPartitioner_partitions_guids_fairly()
{
var totalNumberOfGuids = 1000;
var numberOfPartitions = 50;
var partitions = Partition.AllGuids()
.Among(numberOfPartitions);
var guids = Enumerable.Range(1, totalNumberOfGuids).Select(_ => Guid.NewGuid()).ToArray();
var partitioned = Stream.Partitioned <Guid, int, Guid>(
async(q, p) =>
guids.Where(g => g.IsWithinPartition(p)),
advanceCursor: (q, b) => q.Cursor.AdvanceTo(totalNumberOfGuids));
var aggregator = Aggregator.Create <Projection <HashSet <Guid>, int>, Guid>((p, b) =>
{
if (p.Value == null)
{
p.Value = new HashSet <Guid>();
}
foreach (var guid in b)
{
p.Value.Add(guid);
}
});
var store = new InMemoryProjectionStore <Projection <HashSet <Guid>, int> >();
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitioned.GetStream(partition);
var catchup = StreamCatchup.Create(stream, batchSize: int.MaxValue);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
var projection = await store.Get(stream.Id);
Console.WriteLine(partition + ": " + projection.Value.Count);
}));
var approximateGuidsPerPartition = totalNumberOfGuids / numberOfPartitions;
var tolerance = (int)(totalNumberOfGuids * .12);
store.Sum(p => p.Value.Count).Should().Be(totalNumberOfGuids);
store.ToList().ForEach(projection =>
{
projection.Value
.Count
.Should()
.BeInRange(approximateGuidsPerPartition - tolerance,
approximateGuidsPerPartition + tolerance);
});
}
public async Task When_multiple_projectors_are_subscribed_then_data_that_both_projections_have_seen_is_not_requeried()
{
var streamId = Guid.NewGuid().ToString();
var queriedEvents = new ConcurrentBag <IDomainEvent>();
var balanceProjections = new InMemoryProjectionStore <BalanceProjection>();
await balanceProjections.Put(streamId, new BalanceProjection
{
CursorPosition = 2
});
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate()
.Map(ss => ss.Select(s => s.Trace(onResults: (q, b) =>
{
foreach (var e in b)
{
queriedEvents.Add(e);
}
}))),
Cursor.New("100"),
batchSize: 1);
catchup.Subscribe(new BalanceProjector(), balanceProjections);
store.WriteEvents(streamId); // "101" - 1
store.WriteEvents(streamId); // "102" - 2
store.WriteEvents(streamId); // "103" - 3
await catchup.RunSingleBatch();
queriedEvents.Count
.Should()
.Be(1,
"the first two events should be skipped because of the starting cursor position");
queriedEvents.Should()
.ContainSingle(e => e.StreamRevision == 3,
"only the most recent event should be queried");
var accountHistoryProjections = new InMemoryProjectionStore <AccountHistoryProjection>();
await accountHistoryProjections.Put(streamId, new AccountHistoryProjection
{
CursorPosition = 2
});
catchup.Subscribe(new AccountHistoryProjector(), accountHistoryProjections);
store.WriteEvents(streamId);
await catchup.RunSingleBatch();
queriedEvents.Select(e => e.StreamRevision)
.ShouldBeEquivalentTo(new[] { 3, 4 },
"event 3 needs to be repeated because the newly-subscribed aggregator hasn't seen it yet");
}
public async Task When_one_projection_fails_its_cursor_is_not_advanced_while_other_projections_cursors_are_advanced()
{
var projections = new InMemoryProjectionStore<BalanceProjection>();
// first catch up all the projections
var stream = streamSource.StreamPerAggregate().Trace();
var catchup = StreamCatchup.All(stream);
var initialSubscription = catchup.Subscribe(new BalanceProjector(), projections);
await catchup.RunUntilCaughtUp();
initialSubscription.Dispose();
// write some additional events
var streamIdsWithoutErrors = streamIds.Take(5).ToList();
var streamIdsWithErrors = streamIds.Skip(5).Take(5).ToList();
foreach (var streamId in streamIdsWithoutErrors.Concat(streamIdsWithErrors))
{
store.WriteEvents(streamId, howMany: 10);
}
// subscribe a flaky projector
catchup.Subscribe(new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
var aggregateId = batch.Select(i => i.AggregateId).First();
if (streamIdsWithErrors.Contains(aggregateId))
{
throw new Exception("oops");
}
await next(projection, batch);
}),
projections.AsHandler(),
e => e.Continue());
await catchup.RunSingleBatch();
var projectionsWithoutErrors = streamIdsWithoutErrors.Select(
id => projections.Get(id).Result);
var projectionsWithErrors = streamIdsWithErrors.Select(
id => projections.Get(id).Result);
foreach (var projection in projectionsWithoutErrors)
{
projection.CursorPosition.Should().Be(11);
projection.Balance.Should().Be(11);
}
foreach (var projection in projectionsWithErrors)
{
projection.CursorPosition.Should().Be(1);
projection.Balance.Should().Be(1);
}
}
public async Task Catchup_upstream_batch_size_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 20);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(20);
}
public async Task Catchup_batch_size_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
store.WriteEvents(streamId, howMany: 50);
var catchup = StreamCatchup.Create(stream, batchCount: 20);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Single()
.Balance
.Should()
.Be(20);
}
public async Task Catchup_upstream_batch_size_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 20);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(20);
}
public async Task Catchup_can_traverse_all_events()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
store.WriteEvents(streamId, 999);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(1000);
projectionStore.Count()
.Should()
.Be(1);
}
public async Task Catchup_can_use_a_sequence_of_keys_to_traverse_all_aggregates()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate());
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
projectionStore.Count()
.Should()
.Be(100);
}
public async Task Catchup_can_use_a_sequence_of_keys_to_traverse_all_aggregates()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate());
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
projectionStore.Count()
.Should()
.Be(100);
}
public async Task Catchup_batch_size_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, howMany: 50);
var catchup = StreamCatchup.Create(stream, batchSize: 20);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Single()
.Balance
.Should()
.Be(20);
}
public async Task Catchup_can_traverse_all_events()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, 999);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(1000);
projectionStore.Count()
.Should()
.Be(1);
}
public async Task When_one_batch_is_running_a_second_caller_to_RunSingleBatch_can_await_the_completion_of_the_same_batch()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchSize: 1);
catchup.Subscribe(new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
await Task.Delay(500);
await next(projection, batch);
}), projectionStore);
var cursor1 = catchup.RunSingleBatch();
var cursor2 = catchup.RunSingleBatch();
(await cursor1).Should()
.BeSameAs(await cursor2);
}
public async Task An_initial_cursor_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, howMany: 999);
var catchup = StreamCatchup.Create(stream,
initialCursor: Cursor.New(800),
batchSize: 1000);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(200);
}
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, howMany: 999);
var catchup = StreamCatchup.Create(stream, batchSize: 10);
catchup.Subscribe(new BalanceProjector(), projectionStore);
TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception);
await catchup.RunUntilCaughtUp();
projectionStore.Single()
.Balance
.Should()
.Be(1000);
}
public async Task When_one_batch_is_running_a_second_call_to_RunSingleBatch_will_not_do_anything()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchCount: 1);
catchup.Subscribe(new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
await Task.Delay(500);
await next(projection, batch);
}), projectionStore);
await Task.WhenAll(catchup.RunSingleBatch(), catchup.RunSingleBatch());
projectionStore.Count()
.Should()
.Be(1);
}
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 10);
catchup.Subscribe(new BalanceProjector(), projectionStore);
TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception);
await catchup.RunUntilCaughtUp();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
projectionStore.Count()
.Should()
.Be(100);
}
public async Task When_one_batch_is_running_a_second_caller_to_RunSingleBatch_can_await_the_completion_of_the_same_batch()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchSize: 1);
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
await Task.Delay(500);
await next(projection, batch);
}), projectionStore);
var cursor1 = catchup.RunSingleBatch();
var cursor2 = catchup.RunSingleBatch();
(await cursor1).Should()
.BeSameAs(await cursor2);
}
public async Task When_one_batch_is_running_a_second_call_to_RunSingleBatch_will_not_do_anything()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchSize: 1);
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
await Task.Delay(500);
await next(projection, batch);
}), projectionStore);
await Task.WhenAll(catchup.RunSingleBatch(), catchup.RunSingleBatch());
projectionStore.Count()
.Should()
.Be(1);
}
public async Task Stream_traversal_can_continue_from_upstream_cursor_that_was_returned_by_RunSingleBatch()
{
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore <BalanceProjection>());
var cursor = await catchup.RunSingleBatch();
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), cursor);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(50);
}
public async Task Catchup_query_cursor_resumes_from_last_position()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(50);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
}
public async Task Catchup_starting_cursor_can_be_specified()
{
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore <BalanceProjection>());
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var updatedStreams = streamSource.StreamPerAggregate();
var cursor = updatedStreams.NewCursor();
cursor.AdvanceTo("50");
catchup = StreamCatchup.All(updatedStreams, cursor);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(50);
}
public async Task WithinPartition_by_range_correctly_partitions_queries_by_string()
{
var guid = Guid.NewGuid();
var partitions = new[]
{
Partition.ByRange("", "mm"),
Partition.ByRange("mm", "zz")
};
Values.AtoZ()
.ToList()
.ForEach(c =>
WriteEvents(i => new Event
{
SequenceNumber = i,
Guid = guid,
Id = c + " " + Guid.NewGuid()
}, 10).Wait());
var stream = Stream.PartitionedByRange <Event, int, string>(async(q, p) =>
{
using (var db = new AlluvialSqlTestsDbContext())
{
return(await db.Events
.Where(e => e.Guid == guid)
.WithinPartition(e => e.Id, p)
.ToArrayAsync());
}
});
var catchup = stream.DistributeAmong(partitions);
var store = new InMemoryProjectionStore <int>();
catchup.Subscribe(async(count, events) => events.Count, store);
await catchup.RunSingleBatch();
store.Select(x => x).Should().BeEquivalentTo(new[] { 130, 130 });
}
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchSize: 50);
catchup.Subscribe(new BalanceProjector().Trace(), projectionStore);
using (catchup.Poll(TimeSpan.FromMilliseconds(10)))
{
using (Background.Loop(_ => store.WriteEvents(streamId, howMany: 2), .1))
{
await Wait.Until(() =>
{
var sum = projectionStore.Sum(b => b.Balance);
Console.WriteLine("sum is " + sum);
return(sum >= 500);
});
}
}
}
public async Task Catchup_query_cursor_resumes_from_last_position()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, howMany: 999);
var catchup = StreamCatchup.Create(stream, batchSize: 500);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(500);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(1000);
}
public async Task When_an_aggregation_fails_then_the_projection_is_not_updated()
{
var projections = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace());
// subscribe a flaky projector
catchup.Subscribe(new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
bool shouldThrow = true; // declared outside to nudge the compiler into inferring the correct .Pipeline overload
if (shouldThrow)
{
throw new Exception("oops");
}
await next(projection, batch);
}),
projections.AsHandler(),
e => e.Continue());
await catchup.RunSingleBatch();
projections.Count().Should().Be(0);
}
public async Task When_a_partition_is_queried_then_the_cursor_is_updated()
{
var partitions = Partition.ByRange(0, 1000).Among(2);
var store = new InMemoryProjectionStore <Projection <int, int> >();
var aggregator = Aggregator.Create <Projection <int, int>, int>((p, i) => p.Value += i.Sum());
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitionedStream.GetStream(partition);
Console.WriteLine(stream);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
}));
store.Should()
.ContainSingle(p => p.CursorPosition == 500)
.And
.ContainSingle(p => p.CursorPosition == 1000);
}
public async Task When_an_aggregation_fails_then_the_projection_is_not_updated()
{
var projections = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace());
// subscribe a flaky projector
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
bool shouldThrow = true; // declared outside to nudge the compiler into inferring the correct .Pipeline overload
if (shouldThrow)
{
throw new Exception("oops");
}
await next(projection, batch);
}),
projections.AsHandler(),
e => e.Continue());
await catchup.RunSingleBatch();
projections.Count().Should().Be(0);
}
public async Task Catchup_query_cursor_resumes_from_last_position()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
store.WriteEvents(streamId, howMany: 999);
var catchup = StreamCatchup.Create(stream, batchCount: 500);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(500);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(1000);
}
public async Task Catchup_starting_cursor_can_be_specified()
{
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore<BalanceProjection>());
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var updatedStreams = streamSource.StreamPerAggregate();
var cursor = updatedStreams.NewCursor();
cursor.AdvanceTo("50");
catchup = StreamCatchup.All(updatedStreams, cursor);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(50);
}
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup()
{
var count = 0;
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var projector = new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
Interlocked.Increment(ref count);
if (count < 20)
{
throw new Exception("oops");
}
await next(projection, batch);
}).Trace();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(),
batchSize: 50);
catchup.Subscribe(projector,
projectionStore.AsHandler(),
onError: e => e.Continue());
await catchup.RunSingleBatch();
projectionStore.Count().Should().Be(31);
}
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var projector = new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
if (projectionStore.Count() >= 30)
{
throw new Exception("oops");
}
await next(projection, batch);
});
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(projector, projectionStore);
Action runSingleBatch = () => catchup.RunSingleBatch().Wait();
runSingleBatch.ShouldThrow<Exception>()
.And
.Message
.Should()
.Contain("oops");
}
public async Task When_multiple_projectors_are_subscribed_then_data_that_both_projections_have_seen_is_not_requeried()
{
var queriedEvents = new ConcurrentBag<IDomainEvent>();
var balanceProjections = new InMemoryProjectionStore<BalanceProjection>();
await balanceProjections.Put(streamId, new BalanceProjection
{
CursorPosition = 2
});
var catchup = StreamCatchup.Create(stream.Trace(onResults: (q, b) =>
{
foreach (var e in b)
{
queriedEvents.Add(e);
}
}), batchCount: 10);
catchup.Subscribe(new BalanceProjector(), balanceProjections);
store.WriteEvents(streamId, howMany: 2);
await catchup.RunSingleBatch();
queriedEvents.Count
.Should()
.Be(1,
"the first two events should be skipped because of the starting cursor position");
queriedEvents.Should()
.ContainSingle(e => e.StreamRevision == 3,
"only the most recent event should be queried");
var accountHistoryProjections = new InMemoryProjectionStore<AccountHistoryProjection>();
await accountHistoryProjections.Put(streamId, new AccountHistoryProjection
{
CursorPosition = 2
});
catchup.Subscribe(new AccountHistoryProjector(), accountHistoryProjections);
store.WriteEvents(streamId);
await catchup.RunSingleBatch();
queriedEvents.Select(e => e.StreamRevision)
.ShouldBeEquivalentTo(new[] { 3, 3, 4 },
"event 3 needs to be repeated because the newly-subscribed aggregator hasn't seen it yet");
}
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor()
{
var partitions = Enumerable.Range(0, 9)
.Select(i => StreamQuery.Partition(i*100, (i + 1)*100))
.ToArray();
var store = new InMemoryProjectionStore<Projection<HashSet<int>, int>>();
var aggregator = Aggregator.CreateFor<HashSet<int>, int>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet<int>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
// set up 10 competing catchups
for (var i = 0; i < 10; i++)
{
var distributor = new InMemoryStreamQueryDistributor(
partitions.Select(p => new LeasableResource(p.ToString(), TimeSpan.FromSeconds(10))).ToArray(), "")
.Trace();
distributor.OnReceive(async lease =>
{
var partition = partitions.Single(p => p.ToString() == lease.LeasableResource.Name);
var catchup = StreamCatchup.Create(await partitioner.GetStream(partition));
catchup.Subscribe(aggregator, store.Trace());
await catchup.RunSingleBatch();
});
distributor.Start();
disposables.Add(distributor);
}
partitions.ToList()
.ForEach(partition =>
store.Should()
.ContainSingle(p =>
p.Value.Count() == 100 &&
p.Value.OrderBy(i => i).Last() == partition.UpperBoundInclusive));
}
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchCount: 50);
catchup.Subscribe(new BalanceProjector().Trace(), projectionStore);
using (catchup.Poll(TimeSpan.FromMilliseconds(10)))
{
using (Background.Loop(_ => store.WriteEvents(streamId, howMany: 2), .1))
{
await Wait.Until(() =>
{
var sum = projectionStore.Sum(b => b.Balance);
Console.WriteLine("sum is " + sum);
return sum >= 500;
});
}
}
}
public async Task When_projections_are_cursors_then_catchup_does_not_replay_previously_seen_events()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var eventsAggregated = new List<IDomainEvent>();
var catchup = StreamCatchup.Create(stream, batchCount: 100);
catchup.Subscribe(new BalanceProjector().Trace((p, es) => eventsAggregated.AddRange(es)), projectionStore);
await catchup.RunUntilCaughtUp();
store.WriteEvents(streamId, howMany: 100);
var cursor = await catchup.RunUntilCaughtUp();
cursor.Position
.Should()
.Be(101);
var balanceProjection = await projectionStore.Get(streamId);
balanceProjection.Balance.Should().Be(101);
balanceProjection.CursorPosition.Should().Be(101);
eventsAggregated.Count.Should().Be(101);
}
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
store.WriteEvents(streamId, howMany: 999);
var catchup = StreamCatchup.Create(stream, batchCount: 10);
catchup.Subscribe(new BalanceProjector(), projectionStore);
TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception);
await catchup.RunUntilCaughtUp();
projectionStore.Single()
.Balance
.Should()
.Be(1000);
}
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written_to_existing_streams()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 5);
catchup.Subscribe(new BalanceProjector(), projectionStore);
using (catchup.Poll(TimeSpan.FromMilliseconds(10)))
{
// write more events
Task.Run(async () =>
{
foreach (var streamId in streamIds.Take(20))
{
store.WriteEvents(streamId);
await Task.Delay(1);
}
Console.WriteLine("wrote 20 more events");
});
await Wait.Until(() =>
{
var sum = projectionStore.Sum(b => b.Balance);
Console.WriteLine("sum is " + sum);
return sum >= 120;
});
}
}
public async Task GuidQueryPartitioner_partitions_guids_fairly()
{
var totalNumberOfGuids = 1000;
var numberOfPartitions = 50;
var partitions = StreamQuery.Partition(
Guid.Empty,
Guid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff"))
.Among(numberOfPartitions);
var guids = Enumerable.Range(1, totalNumberOfGuids).Select(_ => Guid.NewGuid()).ToArray();
var partitioner = Stream.Partition<Guid, int, Guid>(
async (q, p) =>
guids.Where(g => new SqlGuid(g).CompareTo(new SqlGuid(p.LowerBoundExclusive)) > 0 &&
new SqlGuid(g).CompareTo(new SqlGuid(p.UpperBoundInclusive)) <= 0),
advanceCursor: (q, b) => q.Cursor.AdvanceTo(totalNumberOfGuids));
var aggregator = Aggregator.Create<Projection<HashSet<Guid>, int>, Guid>((p, b) =>
{
if (p.Value == null)
{
p.Value = new HashSet<Guid>();
}
foreach (var guid in b)
{
p.Value.Add(guid);
}
});
var store = new InMemoryProjectionStore<Projection<HashSet<Guid>, int>>();
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitioner.GetStream(partition);
var catchup = StreamCatchup.Create(stream, batchCount: int.MaxValue);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
await Task.Delay(500);
var projection = await store.Get(stream.Id);
Console.WriteLine(partition + ": " + projection.Value.Count);
}));
var approximateGuidsPerPartition = totalNumberOfGuids/numberOfPartitions;
var tolerance = (int) (totalNumberOfGuids*.12);
Console.WriteLine("\nMissing guids: ");
foreach (var guid in guids.Where(g => !store.Any(p => p.Value.Contains(g))))
{
Console.WriteLine(" " + guid);
}
store.Sum(p => p.Value.Count).Should().Be(totalNumberOfGuids);
store.ToList().ForEach(projection =>
{
projection.Value
.Count
.Should()
.BeInRange(approximateGuidsPerPartition - tolerance,
approximateGuidsPerPartition + tolerance);
});
}
public async Task WithinPartition_by_range_correctly_partitions_queries_by_string()
{
var guid = Guid.NewGuid();
var partitions = new[]
{
Partition.ByRange("", "mm"),
Partition.ByRange("mm", "zz")
};
Values.AtoZ()
.ToList()
.ForEach(c =>
WriteEvents(i => new Event
{
SequenceNumber = i,
Guid = guid,
Id = c + " " + Guid.NewGuid()
}, 10).Wait());
var stream = Stream.PartitionedByRange<Event, int, string>(async (q, p) =>
{
using (var db = new AlluvialSqlTestsDbContext())
{
return await db.Events
.Where(e => e.Guid == guid)
.WithinPartition(e => e.Id, p)
.ToArrayAsync();
}
});
var catchup = stream.DistributeAmong(partitions);
var store = new InMemoryProjectionStore<int>();
catchup.Subscribe(async (count, events) => events.Count, store);
await catchup.RunSingleBatch();
store.Select(x => x).Should().BeEquivalentTo(new[] { 130, 130 });
}
public async Task Catchup_query_cursor_resumes_from_last_position()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(50);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
}
public async Task By_default_ProjectionStore_Trace_writes_projections_during_Put_to_trace_output()
{
var store = new InMemoryProjectionStore<BalanceProjection>();
await store.Trace().Put("the-stream-id", new BalanceProjection());
traceListener.Messages
.Should()
.Contain("[Put] Projection(IDomainEvent,Int32): null @ cursor 0 for stream the-stream-id");
}
public async Task GuidQueryPartitioner_partitions_guids_fairly()
{
var totalNumberOfGuids = 1000;
var numberOfPartitions = 50;
var partitions = Partition.AllGuids()
.Among(numberOfPartitions);
var guids = Enumerable.Range(1, totalNumberOfGuids).Select(_ => Guid.NewGuid()).ToArray();
var partitioned = Stream.Partitioned<Guid, int, Guid>(
async (q, p) =>
guids.Where(g => g.IsWithinPartition(p)),
advanceCursor: (q, b) => q.Cursor.AdvanceTo(totalNumberOfGuids));
var aggregator = Aggregator.Create<Projection<HashSet<Guid>, int>, Guid>((p, b) =>
{
if (p.Value == null)
{
p.Value = new HashSet<Guid>();
}
foreach (var guid in b)
{
p.Value.Add(guid);
}
});
var store = new InMemoryProjectionStore<Projection<HashSet<Guid>, int>>();
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitioned.GetStream(partition);
var catchup = StreamCatchup.Create(stream, batchSize: int.MaxValue);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
var projection = await store.Get(stream.Id);
Console.WriteLine(partition + ": " + projection.Value.Count);
}));
var approximateGuidsPerPartition = totalNumberOfGuids/numberOfPartitions;
var tolerance = (int) (totalNumberOfGuids*.12);
store.Sum(p => p.Value.Count).Should().Be(totalNumberOfGuids);
store.ToList().ForEach(projection =>
{
projection.Value
.Count
.Should()
.BeInRange(approximateGuidsPerPartition - tolerance,
approximateGuidsPerPartition + tolerance);
});
}
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
store.WriteEvents(streamId, 100);
var projector = new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
Throw();
await next(projection, batch);
});
var catchup = StreamCatchup.Create(stream, batchCount: 50);
catchup.Subscribe(projector, projectionStore);
Action runSingleBatch = () => catchup.RunSingleBatch().Wait();
runSingleBatch.ShouldThrow<Exception>()
.And
.Message
.Should()
.Contain("oops");
}
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var count = 0;
var projector = new BalanceProjector()
.Pipeline(async (projection, batch, next) =>
{
Interlocked.Increment(ref count);
Console.WriteLine(count);
if (count < 49)
{
Throw();
}
await next(projection, batch);
}).Trace();
var catchup = StreamCatchup.Create(stream.Trace(),
batchCount: 50);
catchup.Subscribe(projector,
projectionStore.AsHandler(),
onError: e => e.Continue());
await catchup.RunUntilCaughtUp();
projectionStore.Count().Should().Be(1);
projectionStore.Single().CursorPosition.Should().Be(1);
}
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results()
{
var projectionStore = new InMemoryProjectionStore<BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 10);
catchup.Subscribe(new BalanceProjector(), projectionStore);
TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception);
await catchup.RunUntilCaughtUp();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
projectionStore.Count()
.Should()
.Be(100);
}
