public async Task TryDoingIt()
{
var numberOfMessages = 60;
var positionsStorage = new InMemPositionsStorage();
var topic = GetNewTopic(numberOfPartitions: 4);
var locks = new InMemExclusiveLockBandit();
var stopEverything = Using(new CancellationTokenSource());
var stopSecondConsumer = Using(new CancellationTokenSource());
var cancellationToken = stopEverything.Token;
cancellationToken.Register(() => stopSecondConsumer.Cancel());
var receivedEvents = new ConcurrentQueue <MyEvent>();
// start producer that produces an event every second
var producer = StartProducer(cancellationToken, topic, numberOfMessages);
// when producer has finished producing, give everyone 10 more seconds to complete what they're doing
producer.ContinueWith(_ => stopEverything.CancelAfter(TimeSpan.FromSeconds(10)), cancellationToken);
// wait short while, start consumer
await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken);
var consumer1 = StartConsumer(cancellationToken, "CONSUMER 1", topic, positionsStorage, receivedEvents, locks);
// wait a while and start another consumer
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
var consumer2 = StartConsumer(stopSecondConsumer.Token, "CONSUMER 2", topic, positionsStorage, receivedEvents, locks);
// stop the second consumer after 20 seconds, so the first goes back to being the only one there
stopSecondConsumer.CancelAfter(TimeSpan.FromSeconds(20));
// let things run, but ensure that we die if things don't stop by themselves within 1 minute
stopEverything.CancelAfter(TimeSpan.FromMinutes(1));
// wait for all tasks
await Task.WhenAll(producer, consumer1, consumer2);
// check stuff
Assert.That(receivedEvents.Count, Is.GreaterThanOrEqualTo(numberOfMessages), $@"Got the following messages:
{string.Join(Environment.NewLine, receivedEvents.OrderBy(e => e.Number).Select(e => $" {e.Number}: {e.Text}"))}
");
Assert.That(receivedEvents.Select(e => e.Text).Distinct().Count(), Is.EqualTo(numberOfMessages), $@"Got the following messages:
{string.Join(Environment.NewLine, receivedEvents.OrderBy(e => e.Number).Select(e => $" {e.Number}: {e.Text}"))}
");
}
protected override void SetUp()
{
_positionsStorage = new InMemPositionsStorage();
_topic = GetNewTopic();
_producer = Configure
.Producer(c => c.UseKafka(KafkaTestConfig.Address))
.Logging(l => l.UseSerilog())
.Create();
Using(_producer);
}
public static void StoreInMemory(this StandardConfigurer <IPositionManager> configurer, InMemPositionsStorage positionsStorage = null)
{
var registrar = StandardConfigurer.Open(configurer);
registrar.Register(_ => new InMemPositionsManager(positionsStorage ?? new InMemPositionsStorage()));
}
public InMemPositionsManager(InMemPositionsStorage positionsStorage)
{
_positionsStorage = positionsStorage ?? throw new ArgumentNullException(nameof(positionsStorage));
}
static Task StartConsumer(CancellationToken cancellationToken, string name, string topic, InMemPositionsStorage positionsStorage, ConcurrentQueue <MyEvent> receivedEvents, InMemExclusiveLockBandit locks)
{
return(StartTask(cancellationToken, name, async() =>
{
var consumer = Configure.Consumer("default", c =>
{
c.UseKafka(KafkaTestConfig.Address)
.OnPartitionsAssigned(async(context, partitions) =>
{
var tasks = partitions
.Select(async partition =>
{
var key = GetLockKey(partition);
var grabbedLock = await locks.GrabLock(key);
context.SetItem(key, grabbedLock);
return grabbedLock;
})
.ToList();
await Task.WhenAll(tasks);
})
.OnPartitionsRevoked(async(context, partitions) =>
{
var keys = partitions.Select(GetLockKey).ToList();
foreach (var lockToRelease in keys.Select(context.GetItem <IDisposable>))
{
lockToRelease?.Dispose();
}
})
;
})
.Topics(t => t.Subscribe(topic))
.Serialization(s => s.UseNewtonsoftJson())
.Handle(async(messages, _, _) =>
{
foreach (var message in messages)
{
if (message.Body is MyEvent myEvent)
{
Console.WriteLine($"{name} got event {myEvent.Number}");
receivedEvents.Enqueue(myEvent);
}
}
})
.Positions(s => s.StoreInMemory(positionsStorage))
.Start();
using (consumer)
{
cancellationToken.WaitHandle.WaitOne();
}
}));
public async Task ConsumerCanPickUpWhereItLeftOff()
{
var receivedStrings = new ConcurrentQueue <string>();
var topic = BrokerFactory.GetNewTopic();
var producer = BrokerFactory.ConfigureProducer()
.Topics(m => m.Map <string>(topic))
.Create();
Using(producer);
IDisposable CreateConsumer(InMemPositionsStorage storage)
{
return(BrokerFactory.ConfigureConsumer("default-group")
.Handle(async(messages, token) =>
{
var strings = messages.Select(m => m.Body).Cast <string>();
receivedStrings.Enqueue(strings);
})
.Topics(t => t.Subscribe(topic))
.Positions(p => p.StoreInMemory(storage))
.Start());
}
var positionsStorage = new InMemPositionsStorage();
const string partitionKey = "same-every-time";
using (CreateConsumer(positionsStorage))
{
await producer.Send("HEJ", partitionKey : partitionKey);
await producer.Send("MED", partitionKey : partitionKey);
await producer.Send("DIG", partitionKey : partitionKey);
string GetFailureDetailsFunction() => $@"Got these strings:
{receivedStrings.ToPrettyJson()}";
await receivedStrings.WaitOrDie(
completionExpression : q => q.Count == 3,
failExpression : q => q.Count > 3,
failureDetailsFunction : GetFailureDetailsFunction
);
}
Console.WriteLine($@"Got these positions after FIRST run:
{string.Join(Environment.NewLine, positionsStorage.GetAll(topic).Select(position => $" {position}"))}
");
using (CreateConsumer(positionsStorage))
{
await producer.Send("MIN", partitionKey : partitionKey);
await producer.Send("SØDE", partitionKey : partitionKey);
await producer.Send("VEN", partitionKey : partitionKey);
await receivedStrings.WaitOrDie(q => q.Count == 6, failExpression : q => q.Count > 6);
// additional delay to be absolutely sure that no additional messages arrive after this point
await Task.Delay(TimeSpan.FromSeconds(1));
}
Console.WriteLine($@"Got these positions after SECOND run:
{string.Join(Environment.NewLine, positionsStorage.GetAll(topic).Select(position => $" {position}"))}
");
Assert.That(receivedStrings.Count, Is.EqualTo(6), $@"Queue did not contain 6 strings as expected:
{receivedStrings.ToPrettyJson()}");
var expectedReceivedStrings = new[]
{
"HEJ",
"MED",
"DIG",
"MIN",
"SØDE",
"VEN",
};
Assert.That(receivedStrings, Is.EqualTo(expectedReceivedStrings), $@"
Expected
{string.Join(", ", expectedReceivedStrings)}
but got
{string.Join(", ", receivedStrings)}
");
}