/// <summary>
/// Returns the total number of messages difference between this set of offsets, and the given set.
/// </summary>
/// <param name="priorOffsets">The earlier offsets. If the passed offsets are larger than this instance offsets, the result will be negative.</param>
/// <returns></returns>
public long MessagesSince(TopicPartitionOffsets priorOffsets)
{
if (_offsets.Count != priorOffsets._offsets.Count || Topic != priorOffsets.Topic)
{
throw new ArgumentException("priorOffsets does not match Topic or Number of Partitions of this set of offsets.");
}
return(_offsets.Keys.Select(p => _offsets[p] - priorOffsets._offsets[p]).Sum());
}
public StartAndStopAtExplicitOffsets(TopicPartitionOffsets startingOffsets, TopicPartitionOffsets stoppingOffsets)
{
_startingOffsets = startingOffsets;
_stoppingOffsets = stoppingOffsets;
}
public StartAndStopAtExplicitOffsets(IEnumerable<Tuple<int, long>> startingOffsets, IEnumerable<Tuple<int, long>> stoppingOffsets)
{
_startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets);
_stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets);
}
public StartAndStopAtExplicitOffsets(Dictionary<int, long> startingOffsets, Dictionary<int, long> stoppingOffsets)
{
_startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets);
_stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets);
}
public async void ExplicitOffset()
{
kafka4net.Tracing.EtwTrace.Marker("ExplicitOffset");
// create new topic with 3 partitions
var topic = "part33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic,3,3);
// fill it out with 10K messages
const int count = 10*1000;
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producer.ConnectAsync();
var sentMessagesObservable = Observable.FromEvent<Message[]>(evtHandler => producer.OnSuccess += evtHandler, evtHandler => { })
.SelectMany(msgs=>msgs)
.Take(count)
.TakeUntil(DateTime.Now.AddSeconds(10))
.ToList();
_log.Info("Sending data");
Enumerable.Range(1, count).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
var sentMsgs = await sentMessagesObservable;
_log.Info("Producer sent {0} messages.", sentMsgs.Count);
_log.Debug("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
var offsetFetchCluster = new Cluster(_seed2Addresses);
await offsetFetchCluster.ConnectAsync();
// consume tail-300 for each partition
await Task.Delay(TimeSpan.FromSeconds(1));
var offsets = new TopicPartitionOffsets(
topic, (await offsetFetchCluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd))
.GetPartitionsOffset.Select(kv=>new KeyValuePair<int,long>(kv.Key,kv.Value-300)));
_log.Info("Sum of offsets {0}. Raw: {1}",offsets.Partitions.Sum(p=>offsets.NextOffset(p)), offsets);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, offsets));
var messages = consumer.OnMessageArrived.
GroupBy(m => m.Partition).Replay();
messages.Connect();
await consumer.IsConnected;
var consumerSubscription = messages.Subscribe(p => p.Take(10).Subscribe(
m => _log.Debug("Got message {0}/{1}", m.Partition, BitConverter.ToInt32(m.Value, 0)),
e => _log.Error("Error", e),
() => _log.Debug("Complete part {0}", p.Key)
));
// wait for 3 partitions to arrrive and every partition to read at least 100 messages
await messages.Select(g => g.Take(100)).Take(3).ToTask();
consumerSubscription.Dispose();
consumer.Dispose();
kafka4net.Tracing.EtwTrace.Marker("/ExplicitOffset");
}
public async void InvalidOffsetShouldLogErrorAndStopFetching()
{
var count = 100;
var topic = "test11."+_rnd.Next();
await FillOutQueue(topic, count);
var badPartitionMap = new Dictionary<int, long>{{0, -1}};
var offsets = new TopicPartitionOffsets(topic, badPartitionMap);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, offsets));
await consumer.OnMessageArrived.Take(count);
await consumer.IsConnected;
_log.Info("Done");
}
public async void SaveOffsetsAndResumeConsuming()
{
kafka4net.Tracing.EtwTrace.Marker("SaveOffsetsAndResumeConsuming");
var sentEvents = new Subject<Message>();
var topic = "part12." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 5, 2);
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
producer.OnSuccess += e => e.ForEach(sentEvents.OnNext);
await producer.ConnectAsync();
// send 100 messages
Enumerable.Range(1, 100).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
_log.Info("Waiting for 100 sent messages");
sentEvents.Subscribe(msg => _log.Debug("Sent {0}", BitConverter.ToInt32(msg.Value, 0)));
await sentEvents.Take(100).ToTask();
var offsets1 = await producer.Cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicStart);
_log.Info("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
// now consume the "first" 50. Stop, save offsets, and restart.
var consumer1 = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, offsets1));
var receivedEvents = new List<int>(100);
_log.Info("Consuming first half of messages.");
await consumer1.OnMessageArrived
.Do(msg =>
{
var value = BitConverter.ToInt32(msg.Value, 0);
_log.Info("Consumer1 Received value {0} from partition {1} at offset {2}", value, msg.Partition, msg.Offset);
receivedEvents.Add(value);
offsets1.UpdateOffset(msg.Partition, msg.Offset);
})
.Take(50);
//await consumer1.IsConnected;
_log.Info("Closing first consumer");
consumer1.Dispose();
// now serialize the offsets.
var offsetBytes = offsets1.WriteOffsets();
// load a new set of offsets, and a new consumer
var offsets2 = new TopicPartitionOffsets(offsetBytes);
var consumer2 = new Consumer(new ConsumerConfiguration(_seed2Addresses, offsets2.Topic, offsets2));
await consumer2.OnMessageArrived
.Do(msg =>
{
var value = BitConverter.ToInt32(msg.Value, 0);
_log.Info("Consumer2 Received value {0} from partition {1} at offset {2}", value, msg.Partition, msg.Offset);
receivedEvents.Add(value);
offsets2.UpdateOffset(msg.Partition, msg.Offset);
})
.Take(50);
//await consumer2.IsConnected;
_log.Info("Closing second consumer");
consumer2.Dispose();
Assert.AreEqual(100, receivedEvents.Distinct().Count());
Assert.AreEqual(100, receivedEvents.Count);
kafka4net.Tracing.EtwTrace.Marker("/SaveOffsetsAndResumeConsuming");
}
public void TopicPartitionOffsetsSerializeAndDeSerialize()
{
kafka4net.Tracing.EtwTrace.Marker("TopicPartitionOffsetsSerializeAndDeSerialize");
var offsets1 = new TopicPartitionOffsets("test");
for (int i = 0; i < 50; i++)
{
offsets1.UpdateOffset(i,_rnd.Next());
}
// save bytes
var offsetBytes = offsets1.WriteOffsets();
var offsets2 = new TopicPartitionOffsets(offsetBytes);
for (int i = 0; i < 50; i++)
{
Assert.AreEqual(offsets1.NextOffset(i),offsets2.NextOffset(i));
}
kafka4net.Tracing.EtwTrace.Marker("/TopicPartitionOffsetsSerializeAndDeSerialize");
}
public async void StartAndStopAtExplicitOffset()
{
kafka4net.Tracing.EtwTrace.Marker("StartAndStopAtExplicitOffset");
// create new topic with 3 partitions
var topic = "part33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 3, 3);
// fill it out with 10K messages
const int count = 10 * 1000;
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producer.ConnectAsync();
var sentMessagesObservable = Observable.FromEvent<Message[]>(evtHandler => producer.OnSuccess += evtHandler, evtHandler => { })
.SelectMany(msgs => msgs)
.Take(count)
.TakeUntil(DateTime.Now.AddSeconds(10))
.ToList();
_log.Info("Sending data");
Enumerable.Range(1, count).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
var sentMsgs = await sentMessagesObservable;
_log.Info("Producer sent {0} messages.", sentMsgs.Count);
_log.Debug("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
var offsetFetchCluster = new Cluster(_seed2Addresses);
await offsetFetchCluster.ConnectAsync();
await Task.Delay(TimeSpan.FromSeconds(1));
var offsets = (await offsetFetchCluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd));
_log.Info("Sum of offsets {0}. Raw: {1}", offsets.Partitions.Sum(p => offsets.NextOffset(p)), offsets);
// consume first 300 for each partition
var offsetStarts = new TopicPartitionOffsets(topic, offsets.GetPartitionsOffset.ToDictionary(pair => pair.Key, pair => pair.Value > 300 ? 300 : pair.Value));
var offsetStops = new TopicPartitionOffsets(topic, offsets.GetPartitionsOffset.ToDictionary(pair => pair.Key, pair => pair.Value > 600 ? 600 : pair.Value));
var numMessages = offsetStops.MessagesSince(offsetStarts);
var startStopProvider = new StartAndStopAtExplicitOffsets(offsetStarts, offsetStops);
_log.Info("Attempting to consume {0} messages and stop at {1}", numMessages, offsetStops);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, startStopProvider, stopPosition: startStopProvider));
var messages = await consumer.OnMessageArrived.ToList();
consumer.Dispose();
Assert.AreEqual(numMessages, messages.Count);
kafka4net.Tracing.EtwTrace.Marker("/StartAndStopAtExplicitOffset");
}
public StartAndStopAtExplicitOffsets(TopicPartitionOffsets startingOffsets, TopicPartitionOffsets stoppingOffsets)
{
_startingOffsets = startingOffsets;
_stoppingOffsets = stoppingOffsets;
}
public StartAndStopAtExplicitOffsets(IEnumerable <Tuple <int, long> > startingOffsets, IEnumerable <Tuple <int, long> > stoppingOffsets)
{
_startingOffsets = new TopicPartitionOffsets("__xxx__", startingOffsets);
_stoppingOffsets = new TopicPartitionOffsets("__xxx__", stoppingOffsets);
}
public StartAndStopAtExplicitOffsets(Dictionary <int, long> startingOffsets, Dictionary <int, long> stoppingOffsets)
{
_startingOffsets = new TopicPartitionOffsets("__xxx__", startingOffsets);
_stoppingOffsets = new TopicPartitionOffsets("__xxx__", stoppingOffsets);
}
/// <summary>For every patition, resolve offsets and build TopicPartition object</summary>
private async Task<IEnumerable<TopicPartition>> BuildTopicPartitionsAsync()
{
// if they didn't specify explicit locations, initialize them here.
var startPositionProvider = Configuration.StartPosition;
if (startPositionProvider.StartLocation != ConsumerLocation.SpecifiedLocations)
{
// no offsets provided. Need to issue an offset request to get start/end locations and use them for consuming
var partitions = await _cluster.FetchPartitionOffsetsAsync(Topic, startPositionProvider.StartLocation);
if (_log.IsDebugEnabled)
_log.Debug("Consumer for topic {0} got time->offset resolved for location {1}. parts: [{2}]",
Topic, startPositionProvider,
string.Join(",", partitions.Partitions.OrderBy(p => p).Select(p => string.Format("{0}:{1}", p, partitions.NextOffset(p)))));
IStartPositionProvider origProvider = startPositionProvider;
// the new explicit offsets provider should use only the partitions included in the original one.
startPositionProvider = new TopicPartitionOffsets(partitions.Topic, partitions.GetPartitionsOffset.Where(kv=> origProvider.ShouldConsumePartition(kv.Key)));
}
// we now have specified locations to start from, just get the partition metadata, and build the TopicPartitions
var partitionMeta = await _cluster.GetOrFetchMetaForTopicAsync(Topic);
return partitionMeta
// only new partitions we don't already have in our dictionary
.Where(pm => !_topicPartitions.ContainsKey(pm.Id))
// only partitions we are "told" to.
.Where(pm => startPositionProvider.ShouldConsumePartition(pm.Id))
.Select(part =>
{
var tp = new TopicPartition(_cluster, Topic, part.Id, startPositionProvider.GetStartOffset(part.Id));
_topicPartitions.Add(tp.PartitionId, tp);
return tp;
});
}
/// <summary>
/// Returns the total number of messages difference between this set of offsets, and the given set.
/// </summary>
/// <param name="priorOffsets">The earlier offsets. If the passed offsets are larger than this instance offsets, the result will be negative.</param>
/// <returns></returns>
public long MessagesSince(TopicPartitionOffsets priorOffsets)
{
if (_offsets.Count != priorOffsets._offsets.Count || Topic != priorOffsets.Topic)
throw new ArgumentException("priorOffsets does not match Topic or Number of Partitions of this set of offsets.");
return _offsets.Keys.Select(p => _offsets[p] - priorOffsets._offsets[p]).Sum();
}