public Producer(TopicPartitionOffset topicPartitionOffset, string boostrapServers)
{
this._topicPartitionOffset = topicPartitionOffset;
this._boostrapServers = boostrapServers;
}
private void OnPartitionEof(object sender, TopicPartitionOffset e)
{
Console.WriteLine($"EOF Topic= {e.Topic} Offset= {e.Offset}");
}
public void StoreOffset(TopicPartitionOffset offset)
{
// TODO
throw new NotImplementedException();
}
/// <inheritdoc cref="IConsumer{TKey,TValue}.Assign(TopicPartitionOffset)" />
public void Assign(TopicPartitionOffset partition) => throw new NotSupportedException();
public Consumer(TopicPartitionOffset topicPartitionOffset, ConsumerConfig config)
{
_topicPartitionOffset = topicPartitionOffset;
_config = config;
}
private void OnEndReached(object sender, TopicPartitionOffset offset)
{
_handler.OnEndReached(offset.Topic, offset.Partition, offset.Offset);
}
public void Seek(TopicPartitionOffset tpo)
{
cluster.Seek(this, tpo);
}
public void StoreOffset(TopicPartitionOffset offset)
{
}
public ValueTask OnPartitionEndReached(TopicPartitionOffset offset)
{
_commitController.Commit(offset);
return(new ValueTask());
}
public void Seek(TopicPartitionOffset tpo)
{
_consumer.Seek(tpo);
}
public void StoreOffset(TopicPartitionOffset offset)
{
_consumer.StoreOffset(offset);
}
public void Assign(TopicPartitionOffset partition)
{
_consumer.Assign(partition);
}
public static void Consumer_Poll_Error(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Consumer_Poll_Error");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
TopicPartitionOffset firstProduced = null;
using (var producer = new Producer(producerConfig))
{
var keyData = Encoding.UTF8.GetBytes("key");
firstProduced = producer.ProduceAsync(singlePartitionTopic, new Message {
Key = keyData
}).Result.TopicPartitionOffset;
var valData = Encoding.UTF8.GetBytes("val");
producer.ProduceAsync(singlePartitionTopic, new Message {
Value = valData
});
producer.Flush(TimeSpan.FromSeconds(10));
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000,
EnablePartitionEof = true
};
// test key deserialization error behavior
using (var consumer = new Consumer <Null, string>(consumerConfig))
{
int msgCnt = 0;
int errCnt = 0;
consumer.OnPartitionsAssigned += (_, partitions) =>
{
Assert.Single(partitions);
Assert.Equal(firstProduced.TopicPartition, partitions[0]);
consumer.Assign(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
};
consumer.OnPartitionsRevoked += (_, partitions)
=> consumer.Unassign();
consumer.Subscribe(singlePartitionTopic);
while (true)
{
try
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
msgCnt += 1;
}
catch (ConsumeException e)
{
errCnt += 1;
Assert.Equal(ErrorCode.Local_KeyDeserialization, e.Error.Code);
Assert.Equal(firstProduced.Offset.Value, e.ConsumerRecord.Offset.Value);
}
}
Assert.Equal(1, msgCnt);
Assert.Equal(1, errCnt);
consumer.Close();
}
// test value deserialization error behavior
using (var consumer = new Consumer <string, Null>(consumerConfig))
{
int msgCnt = 0;
int errCnt = 0;
consumer.OnPartitionsAssigned += (_, partitions) =>
{
Assert.Single(partitions);
Assert.Equal(firstProduced.TopicPartition, partitions[0]);
consumer.Assign(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset)));
};
consumer.OnPartitionsRevoked += (_, partitions)
=> consumer.Unassign();
consumer.Subscribe(singlePartitionTopic);
while (true)
{
try
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
msgCnt += 1;
}
catch (ConsumeException e)
{
errCnt += 1;
Assert.Equal(ErrorCode.Local_ValueDeserialization, e.Error.Code);
Assert.Equal(firstProduced.Offset.Value + 1, e.ConsumerRecord.Offset.Value);
}
}
Assert.Equal(1, msgCnt);
Assert.Equal(1, errCnt);
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Poll_Error");
}
private KafkaBatchContainer(string streamNamespace, IEnumerable <string> events, TopicPartitionOffset topicPartitionOffset)
{
_events = events;
TopicPartitionOffset = topicPartitionOffset;
StreamNamespace = streamNamespace;
}
public async ValueTask OnPartitionEndReached(TopicPartitionOffset offset)
{
await Commit().ConfigureAwait(false);
}
public async Task OnPartitionEndReached(TopicPartitionOffset offset)
{
await _commitController.Commit(offset);
}
public void Properties()
{
var tpo = new TopicPartitionOffset("mytopic", 42, 107);
Assert.Equal(tpo.TopicPartition, new TopicPartition("mytopic", 42));
}
public static TopicPartitionOffset AddOffset([NotNull] this TopicPartitionOffset topicPartitionOffset, int addOffset) => new TopicPartitionOffset(topicPartitionOffset.TopicPartition, topicPartitionOffset.Offset + addOffset);
public void Seek(TopicPartitionOffset tpo) => _innerConsumer.Seek(tpo);
public static bool IsSpecial(this TopicPartitionOffset tpo) => tpo.Offset.IsSpecial;
public void Assign(TopicPartitionOffset partition)
=> Assign(new List <TopicPartitionOffset> {
partition
});
public Task Commit(TopicPartitionOffset offset)
{
return(Consumer.CommitAsync(new List <TopicPartitionOffset> {
offset
}));
}
public Task OnPartitionEndReached(TopicPartitionOffset offset)
{
return(Commit(offset));
}
protected virtual void OnPartitionEndReached(object sender, TopicPartitionOffset offset)
{
Log.DebugFormat("Group [{0}]: Reached end of topic: {1} and partition: {2}, next message will be at offset: {3}", Group, offset.Topic, offset.Partition, offset.Offset);
}
public void Assign(TopicPartitionOffset partition)
{
// TODO
}
private void OnConsumerOnOnPartitionEof(object _, TopicPartitionOffset end)
{
LogManager.Instance.Info($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
}
public void Seek(TopicPartitionOffset tpo)
{
// TODO
throw new NotImplementedException();
}
private static void assertCloseToNow_byte(Consumer <byte[], byte[]> consumer, TopicPartitionOffset tpo)
{
consumer.Assign(new List <TopicPartitionOffset>()
{
tpo
});
var cr = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(cr.Message);
Assert.Equal(TimestampType.CreateTime, cr.Message.Timestamp.Type);
Assert.True(Math.Abs((cr.Message.Timestamp.UtcDateTime - DateTime.UtcNow).TotalSeconds) < 120);
}
public TopicPartitionOffsetError(TopicPartitionOffset tpo, Error error)
: this(tpo.Topic, tpo.Partition, tpo.Offset, error)
{
}
public Producer(string topicName, int partition, string boostrapServers)
{
this._boostrapServers = boostrapServers;
this._topicPartitionOffset = new TopicPartitionOffset(topicName, new Partition(partition), new Offset(0));
}
