public async Task Distributed_catchups_can_store_a_cursor_per_partition() { var cursorStore = new InMemoryProjectionStore <ICursor <int> >(_ => Cursor.New <int>()); var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet <string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .DistributeAmong(partitions, batchSize: 73, cursorPerPartition: cursorStore.Trace().AsHandler()); catchup.Subscribe(aggregator); await catchup.RunUntilCaughtUp(); cursorStore.Count().Should().Be(26); Enumerable.Range(1, 26).ToList().ForEach(i => { cursorStore.Should().Contain(c => c.Position == i * 100); }); }
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup() { var store = new InMemoryProjectionStore <Projection <HashSet <string>, int> >(); var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet <string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .Trace() .DistributeAmong(partitions, batchSize: 15); catchup.Subscribe(aggregator, store.Trace()); await catchup.RunUntilCaughtUp(); partitions.ToList() .ForEach(partition => store.Should() .ContainSingle(projection => projection.Value.Count() == 100 && projection.Value.All(p => p.IsWithinPartition(partition)))); }
public async Task When_a_partition_is_queried_then_the_cursor_is_updated() { var partitions = new[] { StreamQuery.Partition(0, 500), StreamQuery.Partition(500, 1000) }; var store = new InMemoryProjectionStore<Projection<int, int>>(); var aggregator = Aggregator.CreateFor<int, int>((p, i) => p.Value += i.Sum()); await Task.WhenAll(partitions.Select(async partition => { var stream = await partitioner.GetStream(partition); Console.WriteLine(stream); var catchup = StreamCatchup.Create(stream); catchup.Subscribe(aggregator, store); await catchup.RunSingleBatch(); })); store.Should() .ContainSingle(p => p.CursorPosition == 500) .And .ContainSingle(p => p.CursorPosition == 1000); }
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup() { var store = new InMemoryProjectionStore<Projection<HashSet<string>, int>>(); var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet<string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .Trace() .DistributeAmong(partitions, batchSize: 15); catchup.Subscribe(aggregator, store.Trace()); await catchup.RunUntilCaughtUp(); partitions.ToList() .ForEach(partition => store.Should() .ContainSingle(projection => projection.Value.Count() == 100 && projection.Value.All(p => p.IsWithinPartition(partition)))); }
public async Task When_a_partition_is_queried_then_the_cursor_is_updated() { var partitions = Partition.ByRange(0, 1000).Among(2); var store = new InMemoryProjectionStore <Projection <int, int> >(); var aggregator = Aggregator.Create <Projection <int, int>, int>((p, i) => p.Value += i.Sum()); await Task.WhenAll(partitions.Select(async partition => { var stream = await partitionedStream.GetStream(partition); Console.WriteLine(stream); var catchup = StreamCatchup.Create(stream); catchup.Subscribe(aggregator, store); await catchup.RunSingleBatch(); })); store.Should() .ContainSingle(p => p.CursorPosition == 500) .And .ContainSingle(p => p.CursorPosition == 1000); }
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor() { var partitions = Enumerable.Range(0, 9) .Select(i => StreamQuery.Partition(i*100, (i + 1)*100)) .ToArray(); var store = new InMemoryProjectionStore<Projection<HashSet<int>, int>>(); var aggregator = Aggregator.CreateFor<HashSet<int>, int>((p, xs) => { if (p.Value == null) { p.Value = new HashSet<int>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); // set up 10 competing catchups for (var i = 0; i < 10; i++) { var distributor = new InMemoryStreamQueryDistributor( partitions.Select(p => new LeasableResource(p.ToString(), TimeSpan.FromSeconds(10))).ToArray(), "") .Trace(); distributor.OnReceive(async lease => { var partition = partitions.Single(p => p.ToString() == lease.LeasableResource.Name); var catchup = StreamCatchup.Create(await partitioner.GetStream(partition)); catchup.Subscribe(aggregator, store.Trace()); await catchup.RunSingleBatch(); }); distributor.Start(); disposables.Add(distributor); } partitions.ToList() .ForEach(partition => store.Should() .ContainSingle(p => p.Value.Count() == 100 && p.Value.OrderBy(i => i).Last() == partition.UpperBoundInclusive)); }
public async Task Distributed_catchups_can_store_a_cursor_per_partition() { var cursorStore = new InMemoryProjectionStore<ICursor<int>>(_ => Cursor.New<int>()); var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) => { if (p.Value == null) { p.Value = new HashSet<string>(); } foreach (var x in xs) { p.Value.Add(x); } }).Trace(); var catchup = partitionedStream .DistributeAmong(partitions, batchSize: 73, cursorPerPartition: cursorStore.Trace().AsHandler()); catchup.Subscribe(aggregator); await catchup.RunUntilCaughtUp(); cursorStore.Count().Should().Be(26); Enumerable.Range(1, 26).ToList().ForEach(i => { cursorStore.Should().Contain(c => c.Position == i*100); }); }