public async Task Distributed_catchups_can_store_a_cursor_per_partition()
{
var cursorStore = new InMemoryProjectionStore <ICursor <int> >(_ => Cursor.New <int>());
var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet <string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.DistributeAmong(partitions,
batchSize: 73,
cursorPerPartition: cursorStore.Trace().AsHandler());
catchup.Subscribe(aggregator);
await catchup.RunUntilCaughtUp();
cursorStore.Count().Should().Be(26);
Enumerable.Range(1, 26).ToList().ForEach(i =>
{
cursorStore.Should().Contain(c => c.Position == i * 100);
});
}
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup()
{
var store = new InMemoryProjectionStore <Projection <HashSet <string>, int> >();
var aggregator = Aggregator.Create <Projection <HashSet <string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet <string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.Trace()
.DistributeAmong(partitions, batchSize: 15);
catchup.Subscribe(aggregator, store.Trace());
await catchup.RunUntilCaughtUp();
partitions.ToList()
.ForEach(partition =>
store.Should()
.ContainSingle(projection =>
projection.Value.Count() == 100 &&
projection.Value.All(p => p.IsWithinPartition(partition))));
}
public async Task When_a_partition_is_queried_then_the_cursor_is_updated()
{
var partitions = new[]
{
StreamQuery.Partition(0, 500),
StreamQuery.Partition(500, 1000)
};
var store = new InMemoryProjectionStore<Projection<int, int>>();
var aggregator = Aggregator.CreateFor<int, int>((p, i) => p.Value += i.Sum());
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitioner.GetStream(partition);
Console.WriteLine(stream);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
}));
store.Should()
.ContainSingle(p => p.CursorPosition == 500)
.And
.ContainSingle(p => p.CursorPosition == 1000);
}
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor_catchup()
{
var store = new InMemoryProjectionStore<Projection<HashSet<string>, int>>();
var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet<string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.Trace()
.DistributeAmong(partitions, batchSize: 15);
catchup.Subscribe(aggregator, store.Trace());
await catchup.RunUntilCaughtUp();
partitions.ToList()
.ForEach(partition =>
store.Should()
.ContainSingle(projection =>
projection.Value.Count() == 100 &&
projection.Value.All(p => p.IsWithinPartition(partition))));
}
public async Task When_a_partition_is_queried_then_the_cursor_is_updated()
{
var partitions = Partition.ByRange(0, 1000).Among(2);
var store = new InMemoryProjectionStore <Projection <int, int> >();
var aggregator = Aggregator.Create <Projection <int, int>, int>((p, i) => p.Value += i.Sum());
await Task.WhenAll(partitions.Select(async partition =>
{
var stream = await partitionedStream.GetStream(partition);
Console.WriteLine(stream);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(aggregator, store);
await catchup.RunSingleBatch();
}));
store.Should()
.ContainSingle(p => p.CursorPosition == 500)
.And
.ContainSingle(p => p.CursorPosition == 1000);
}
public async Task Competing_catchups_can_lease_a_partition_using_a_distributor()
{
var partitions = Enumerable.Range(0, 9)
.Select(i => StreamQuery.Partition(i*100, (i + 1)*100))
.ToArray();
var store = new InMemoryProjectionStore<Projection<HashSet<int>, int>>();
var aggregator = Aggregator.CreateFor<HashSet<int>, int>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet<int>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
// set up 10 competing catchups
for (var i = 0; i < 10; i++)
{
var distributor = new InMemoryStreamQueryDistributor(
partitions.Select(p => new LeasableResource(p.ToString(), TimeSpan.FromSeconds(10))).ToArray(), "")
.Trace();
distributor.OnReceive(async lease =>
{
var partition = partitions.Single(p => p.ToString() == lease.LeasableResource.Name);
var catchup = StreamCatchup.Create(await partitioner.GetStream(partition));
catchup.Subscribe(aggregator, store.Trace());
await catchup.RunSingleBatch();
});
distributor.Start();
disposables.Add(distributor);
}
partitions.ToList()
.ForEach(partition =>
store.Should()
.ContainSingle(p =>
p.Value.Count() == 100 &&
p.Value.OrderBy(i => i).Last() == partition.UpperBoundInclusive));
}
public async Task Distributed_catchups_can_store_a_cursor_per_partition()
{
var cursorStore = new InMemoryProjectionStore<ICursor<int>>(_ => Cursor.New<int>());
var aggregator = Aggregator.Create<Projection<HashSet<string>, int>, string>((p, xs) =>
{
if (p.Value == null)
{
p.Value = new HashSet<string>();
}
foreach (var x in xs)
{
p.Value.Add(x);
}
}).Trace();
var catchup = partitionedStream
.DistributeAmong(partitions,
batchSize: 73,
cursorPerPartition: cursorStore.Trace().AsHandler());
catchup.Subscribe(aggregator);
await catchup.RunUntilCaughtUp();
cursorStore.Count().Should().Be(26);
Enumerable.Range(1, 26).ToList().ForEach(i =>
{
cursorStore.Should().Contain(c => c.Position == i*100);
});
}
