public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var count = 0;
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
Interlocked.Increment(ref count);
Console.WriteLine(count);
if (count < 49)
{
Throw();
}
await next(projection, batch);
}).Trace();
var catchup = StreamCatchup.Create(stream.Trace(),
batchSize: 50);
catchup.Subscribe(projector,
projectionStore.AsHandler(),
onError: e => e.Continue());
await catchup.RunUntilCaughtUp();
projectionStore.Count().Should().Be(1);
projectionStore.Single().CursorPosition.Should().Be(1);
}
public async Task When_advancing_the_cursor_in_a_multi_stream_catchup_throws_then_the_exception_is_surfaced_to_OnError()
{
var streams = Enumerable.Range(1, 24)
.AsStream()
.IntoMany(async(i, from, to) => Stream.Create <string, int>(
query: async q => Enumerable.Range(from, to).Select(ii => ii.ToString()),
advanceCursor: (q, b) => { throw new Exception("oops!"); }));
var error = default(StreamCatchupError <Projection <string, int> >);
var catchup = StreamCatchup.All(streams);
catchup.Subscribe <Projection <string, int>, string, int>(async(sum, batch) => new Projection <string, int>(),
(streamId, use) => use(null),
onError: e =>
{
error = e;
e.Continue();
});
await catchup.RunSingleBatch();
error.Should().NotBeNull();
error.Exception.Message.Should().Contain("oops");
}
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, 100);
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
Throw();
await next(projection, batch);
});
var catchup = StreamCatchup.Create(stream, batchSize: 50);
catchup.Subscribe(projector, projectionStore);
Action runSingleBatch = () => catchup.RunSingleBatch().Wait();
runSingleBatch.ShouldThrow <Exception>()
.And
.Message
.Should()
.Contain("oops");
}
public async Task Getting_and_storing_projections_and_cursors_can_operate_transactionally_via_a_closure()
{
BalanceProjection finalProjection = null;
var catchup = StreamCatchup.Create(stream);
FetchAndSaveProjection <BalanceProjection> fetchAndSaveProjection = (async(id, callAggregatorPipeline) =>
{
using (var transaction = new TransactionScope())
{
// e.g. get the projection / cursor from the store
var proj = new BalanceProjection
{
CursorPosition = 5
};
proj = await callAggregatorPipeline(proj);
finalProjection = proj;
// save the projection / cursor back to the store
transaction.Complete();
}
});
catchup.Subscribe(new BalanceProjector(), fetchAndSaveProjection);
store.WriteEvents(streamId, amount: 100m, howMany: 5);
await catchup.RunSingleBatch();
finalProjection.Balance.Should().Be(100m);
}
public async Task Catchup_can_query_streams_such_that_repeated_data_is_not_queried()
{
var queriedEvents = new ConcurrentBag <IDomainEvent>();
var catchup = StreamCatchup.Create(stream.Trace(onResults: (q, b) =>
{
foreach (var e in b)
{
queriedEvents.Add(e);
}
}), batchSize: 1);
catchup.Subscribe(new BalanceProjector());
store.WriteEvents(streamId);
await catchup.RunSingleBatch();
queriedEvents.Select(e => e.StreamRevision)
.ShouldBeEquivalentTo(new[] { 1 });
store.WriteEvents(streamId);
await catchup.RunSingleBatch();
queriedEvents.Select(e => e.StreamRevision)
.ShouldBeEquivalentTo(new[] { 1, 2 });
store.WriteEvents(streamId);
await catchup.RunSingleBatch();
queriedEvents.Select(e => e.StreamRevision)
.ShouldBeEquivalentTo(new[] { 1, 2, 3 });
}
public async Task RunSingleBatch_throws_when_an_aggregator_throws_an_exception()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
if (projectionStore.Count() >= 30)
{
throw new Exception("oops");
}
await next(projection, batch);
});
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(projector, projectionStore);
Action runSingleBatch = () => catchup.RunSingleBatch().Wait();
runSingleBatch.ShouldThrow <Exception>()
.And
.Message
.Should()
.Contain("oops");
}
public async Task OnError_Continue_prevents_aggregator_exceptions_from_stopping_catchup()
{
var count = 0;
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var projector = new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
Interlocked.Increment(ref count);
if (count < 20)
{
throw new Exception("oops");
}
await next(projection, batch);
}).Trace();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(),
batchSize: 50);
catchup.Subscribe(projector,
projectionStore.AsHandler(),
onError: e => e.Continue());
await catchup.RunSingleBatch();
projectionStore.Count().Should().Be(31);
}
public async Task Catchup_Poll_keeps_projections_updated_as_new_events_are_written_to_existing_streams()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 5);
catchup.Subscribe(new BalanceProjector(), projectionStore);
using (catchup.Poll(TimeSpan.FromMilliseconds(10)))
{
// write more events
Task.Run(async() =>
{
foreach (var streamId in streamIds.Take(20))
{
store.WriteEvents(streamId);
await Task.Delay(1);
}
Console.WriteLine("wrote 20 more events");
});
await Wait.Until(() =>
{
var sum = projectionStore.Sum(b => b.Balance);
Console.WriteLine("sum is " + sum);
return(sum >= 120);
});
}
}
public async Task When_projections_are_cursors_then_catchup_does_not_replay_previously_seen_events()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var eventsAggregated = new List <IDomainEvent>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 100);
catchup.Subscribe(new BalanceProjector()
.Trace((p, es) => eventsAggregated.AddRange(es)), projectionStore);
await catchup.RunUntilCaughtUp();
var streamId = streamIds.First();
store.WriteEvents(streamId, 100m);
var cursor = await catchup.RunUntilCaughtUp();
cursor.Position
.Should()
.Be("101");
var balanceProjection = await projectionStore.Get(streamId);
balanceProjection.Balance.Should().Be(101);
balanceProjection.CursorPosition.Should().Be(2);
eventsAggregated.Count.Should().Be(101);
}
public async Task When_advancing_the_cursor_throws_then_an_exception_is_thrown()
{
var values = Enumerable.Range(1, 20);
var stream = Stream.Create <int, int>(
query: q => values.Skip(q.Cursor.Position)
.Take(q.BatchSize ?? 1000),
advanceCursor: (q, b) =>
{
Throw();
});
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe <Projection <int, int>, int, int>(async(sum, batch) =>
{
sum.Value += batch.Count;
return(sum);
});
Action runSingleBatch = () => catchup.RunSingleBatch().Wait();
runSingleBatch.ShouldThrow <Exception>()
.And
.Message
.Should()
.Contain("oops!");
}
public async Task One_stream_can_transparently_delegate_to_another()
{
var upstream = NEventStoreStream.AllEvents(store);
store.WriteEvents(i => new AccountOpened(), 100);
var projection = new Projection <int, string>();
var dependentStream = Stream.Create <int, string>(
async q =>
{
var mapped = upstream.Map(e => new[] { e.Count() });
var batch = await mapped.Fetch(q);
return(batch);
});
var catchup = StreamCatchup.Create(dependentStream, batchSize: 50);
FetchAndSaveProjection <Projection <int, string> > manageProjection = async(id, aggregate) =>
{
await aggregate(projection);
};
catchup.Subscribe(async(p, b) =>
{
p.Value += b.Sum();
return(p);
}, manageProjection);
await catchup.RunSingleBatch();
Console.WriteLine(projection.ToLogString());
projection.Value.Should().Be(50);
projection.CursorPosition.Should().Be("50");
}
public async Task When_advancing_the_cursor_in_a_single_stream_catchup_throws_then_the_exception_is_surfaced_to_OnError()
{
var stream = Stream.Create <int, int>(q => Enumerable.Range(1, 100).Skip(q.Cursor.Position),
advanceCursor: (q, b) =>
{
throw new Exception("oops!");
});
var error = default(StreamCatchupError <Projection <int, int> >);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe <Projection <int, int>, int, int>(async(sum, batch) =>
{
sum.Value += batch.Count;
return(sum);
},
(streamId, use) => use(new Projection <int, int>()),
onError: e =>
{
error = e;
e.Continue();
});
await catchup.RunSingleBatch();
error.Should().NotBeNull();
error.Exception.Message.Should().Contain("oops");
}
public async Task A_stream_can_be_derived_from_an_index_projection_in_order_to_perform_a_reduce_operation()
{
store.WriteEvents(i => new AccountOpened
{
AccountType = i % 2 == 0
? BankAccountType.Checking
: BankAccountType.Savings
}, howMany: 100);
var eventsByAggregate = streamSource.StreamPerAggregate()
.Trace()
.Map(ss => ss.Select(s => s.Trace()));
var indexCatchup = StreamCatchup.All(eventsByAggregate, batchSize: 1);
var index = new Projection <ConcurrentBag <AccountOpened>, string>
{
Value = new ConcurrentBag <AccountOpened>()
};
// subscribe a catchup to the updates stream to build up an index
indexCatchup.Subscribe(
Aggregator.Create <Projection <ConcurrentBag <AccountOpened> >, IDomainEvent>(async(p, events) =>
{
foreach (var e in events.OfType <AccountOpened>()
.Where(e => e.AccountType == BankAccountType.Savings))
{
p.Value.Add(e);
}
return(p);
}).Trace(),
async(streamId, aggregate) => await aggregate(index));
// create a catchup over the index
var savingsAccounts = Stream.Create <IDomainEvent, string>(
"Savings accounts",
async q => index.Value.SkipWhile(v => q.Cursor.HasReached(v.CheckpointToken))
.Take(q.BatchSize ?? 1000));
var savingsAccountsCatchup = StreamCatchup.Create(savingsAccounts);
var numberOfSavingsAccounts = new Projection <int, int>();
savingsAccountsCatchup.Subscribe <Projection <int, int>, IDomainEvent, string>(
manageProjection: async(streamId, aggregate) =>
{
numberOfSavingsAccounts = await aggregate(numberOfSavingsAccounts);
},
aggregate: async(c, es) =>
{
c.Value += es.Count;
return(c);
});
using (indexCatchup.Poll(TimeSpan.FromMilliseconds(100)))
using (savingsAccountsCatchup.Poll(TimeSpan.FromMilliseconds(50)))
{
await Wait.Until(() => numberOfSavingsAccounts.Value >= 50);
}
}
public async Task When_one_projection_fails_its_cursor_is_not_advanced_while_other_projections_cursors_are_advanced()
{
var projections = new InMemoryProjectionStore <BalanceProjection>();
// first catch up all the projections
var stream = streamSource.StreamPerAggregate().Trace();
var catchup = StreamCatchup.All(stream);
var initialSubscription = catchup.Subscribe(new BalanceProjector(), projections);
await catchup.RunUntilCaughtUp();
initialSubscription.Dispose();
// write some additional events
var streamIdsWithoutErrors = streamIds.Take(5).ToList();
var streamIdsWithErrors = streamIds.Skip(5).Take(5).ToList();
foreach (var streamId in streamIdsWithoutErrors.Concat(streamIdsWithErrors))
{
store.WriteEvents(streamId, howMany: 10);
}
// subscribe a flaky projector
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
var aggregateId = batch.Select(i => i.AggregateId).First();
if (streamIdsWithErrors.Contains(aggregateId))
{
throw new Exception("oops");
}
await next(projection, batch);
}),
projections.AsHandler(),
e => e.Continue());
await catchup.RunSingleBatch();
var projectionsWithoutErrors = streamIdsWithoutErrors.Select(
id => projections.Get(id).Result);
var projectionsWithErrors = streamIdsWithErrors.Select(
id => projections.Get(id).Result);
foreach (var projection in projectionsWithoutErrors)
{
projection.CursorPosition.Should().Be(11);
projection.Balance.Should().Be(11);
}
foreach (var projection in projectionsWithErrors)
{
projection.CursorPosition.Should().Be(1);
projection.Balance.Should().Be(1);
}
}
public async Task GuidQueryPartitioner_partitions_guids_fairly()
{
var totalNumberOfGuids = 1000;
var numberOfPartitions = 50;
var partitions = Partition.AllGuids()
.Among(numberOfPartitions);
var guids = Enumerable.Range(1, totalNumberOfGuids).Select(_ => Guid.NewGuid()).ToArray();
var partitioned = Stream.Partitioned <Guid, int, Guid>(
async(q, p) =>
guids.Where(g => g.IsWithinPartition(p)),
advanceCursor: (q, b) => q.Cursor.AdvanceTo(totalNumberOfGuids));
var aggregator = Aggregator.Create <Projection <HashSet <Guid>, int>, Guid>((p, b) =>
{
if (p.Value == null)
{
p.Value = new HashSet <Guid>();
}
foreach (var guid in b)
{
p.Value.Add(guid);
}
});
var store = new InMemoryProjectionStore <Projection <HashSet <Guid>, int> >();
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitioned.GetStream(partition);
var catchup = StreamCatchup.Create(stream, batchSize: int.MaxValue);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
var projection = await store.Get(stream.Id);
Console.WriteLine(partition + ": " + projection.Value.Count);
}));
var approximateGuidsPerPartition = totalNumberOfGuids / numberOfPartitions;
var tolerance = (int)(totalNumberOfGuids * .12);
store.Sum(p => p.Value.Count).Should().Be(totalNumberOfGuids);
store.ToList().ForEach(projection =>
{
projection.Value
.Count
.Should()
.BeInRange(approximateGuidsPerPartition - tolerance,
approximateGuidsPerPartition + tolerance);
});
}
public async Task When_multiple_projectors_are_subscribed_then_data_that_both_projections_have_seen_is_not_requeried()
{
var streamId = Guid.NewGuid().ToString();
var queriedEvents = new ConcurrentBag <IDomainEvent>();
var balanceProjections = new InMemoryProjectionStore <BalanceProjection>();
await balanceProjections.Put(streamId, new BalanceProjection
{
CursorPosition = 2
});
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate()
.Map(ss => ss.Select(s => s.Trace(onResults: (q, b) =>
{
foreach (var e in b)
{
queriedEvents.Add(e);
}
}))),
Cursor.New("100"),
batchSize: 1);
catchup.Subscribe(new BalanceProjector(), balanceProjections);
store.WriteEvents(streamId); // "101" - 1
store.WriteEvents(streamId); // "102" - 2
store.WriteEvents(streamId); // "103" - 3
await catchup.RunSingleBatch();
queriedEvents.Count
.Should()
.Be(1,
"the first two events should be skipped because of the starting cursor position");
queriedEvents.Should()
.ContainSingle(e => e.StreamRevision == 3,
"only the most recent event should be queried");
var accountHistoryProjections = new InMemoryProjectionStore <AccountHistoryProjection>();
await accountHistoryProjections.Put(streamId, new AccountHistoryProjection
{
CursorPosition = 2
});
catchup.Subscribe(new AccountHistoryProjector(), accountHistoryProjections);
store.WriteEvents(streamId);
await catchup.RunSingleBatch();
queriedEvents.Select(e => e.StreamRevision)
.ShouldBeEquivalentTo(new[] { 3, 4 },
"event 3 needs to be repeated because the newly-subscribed aggregator hasn't seen it yet");
}
public async Task Catchup_upstream_batch_size_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 20);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(20);
}
public async Task Catchup_cursor_retrieval_can_be_specified_using_catchup_configuration()
{
ICursor <string> storedCursor = Cursor.New("3");
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(),
batchSize: 1,
manageCursor: async(id, use) => { await use(storedCursor); });
catchup.Subscribe(new BalanceProjector());
var cursor = await catchup.RunSingleBatch();
storedCursor.Position.Should().Be("4");
cursor.Position.Should().Be("4");
}
public async Task Catchup_batch_size_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, howMany: 50);
var catchup = StreamCatchup.Create(stream, batchSize: 20);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Single()
.Balance
.Should()
.Be(20);
}
public async Task Catchup_can_use_a_sequence_of_keys_to_traverse_all_aggregates()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate());
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
projectionStore.Count()
.Should()
.Be(100);
}
public async Task Catchup_can_traverse_all_events()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, 999);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(1000);
projectionStore.Count()
.Should()
.Be(1);
}
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, howMany: 999);
var catchup = StreamCatchup.Create(stream, batchSize: 10);
catchup.Subscribe(new BalanceProjector(), projectionStore);
TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception);
await catchup.RunUntilCaughtUp();
projectionStore.Single()
.Balance
.Should()
.Be(1000);
}
public async Task An_initial_cursor_can_be_specified()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, howMany: 999);
var catchup = StreamCatchup.Create(stream,
initialCursor: Cursor.New(800),
batchSize: 1000);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(200);
}
public async Task Stream_traversal_can_continue_from_upstream_cursor_that_was_returned_by_RunSingleBatch()
{
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore <BalanceProjection>());
var cursor = await catchup.RunSingleBatch();
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), cursor);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(50);
}
public async Task When_one_batch_is_running_a_second_caller_to_RunSingleBatch_can_await_the_completion_of_the_same_batch()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchSize: 1);
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
await Task.Delay(500);
await next(projection, batch);
}), projectionStore);
var cursor1 = catchup.RunSingleBatch();
var cursor2 = catchup.RunSingleBatch();
(await cursor1).Should()
.BeSameAs(await cursor2);
}
public async Task When_one_batch_is_running_a_second_call_to_RunSingleBatch_will_not_do_anything()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.Create(stream, batchSize: 1);
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
await Task.Delay(500);
await next(projection, batch);
}), projectionStore);
await Task.WhenAll(catchup.RunSingleBatch(), catchup.RunSingleBatch());
projectionStore.Count()
.Should()
.Be(1);
}
public async Task Catchup_RunUntilCaughtUp_runs_until_the_stream_has_no_more_results()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 10);
catchup.Subscribe(new BalanceProjector(), projectionStore);
TaskScheduler.UnobservedTaskException += (sender, args) => Console.WriteLine(args.Exception);
await catchup.RunUntilCaughtUp();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
projectionStore.Count()
.Should()
.Be(100);
}
public async Task Catchup_cursor_storage_can_be_specified_using_catchup_configuration()
{
ICursor <string> storedCursor = null;
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(),
batchSize: 10,
manageCursor: async(id, use) =>
{
var c = streamSource.StreamPerAggregate().NewCursor();
await use(c);
storedCursor = c;
});
catchup.Subscribe(new BalanceProjector());
var cursor = await catchup.RunSingleBatch();
cursor.Position.Should().Be("10");
storedCursor.Position.Should().Be("10");
}
public async Task Catchup_starting_cursor_can_be_specified()
{
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), new InMemoryProjectionStore <BalanceProjection>());
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var updatedStreams = streamSource.StreamPerAggregate();
var cursor = updatedStreams.NewCursor();
cursor.AdvanceTo("50");
catchup = StreamCatchup.All(updatedStreams, cursor);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Count()
.Should()
.Be(50);
}
public async Task Catchup_query_cursor_resumes_from_last_position()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate().Trace(), batchSize: 50);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(50);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(100);
}
