/// <summary>
/// Initializes a new instance of the <see cref="KafkaReceiver"/> class.
/// </summary>
/// <param name="name">The name of the receiver.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="groupId">
/// Client group id string. All clients sharing the same group.id belong to the same group.
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="enableAutoOffsetStore">
/// Whether to automatically store offset of last message provided to application.
/// </param>
/// <param name="autoOffsetReset">
/// Action to take when there is no initial offset in offset store or the desired
/// offset is out of range: 'smallest','earliest' - automatically reset the offset
/// to the smallest offset, 'largest','latest' - automatically reset the offset to
/// the largest offset, 'error' - trigger an error which is retrieved by consuming
/// messages and checking 'message->err'.
/// </param>
public KafkaReceiver(string name, string topic, string groupId, string bootstrapServers,
bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = Confluent.Kafka.AutoOffsetReset.Latest)
: base(name)
{
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
GroupId = groupId ?? throw new ArgumentNullException(nameof(groupId));
BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers));
EnableAutoOffsetStore = enableAutoOffsetStore;
AutoOffsetReset = autoOffsetReset;
var config = GetConsumerConfig(groupId, bootstrapServers, enableAutoOffsetStore, autoOffsetReset);
var builder = new ConsumerBuilder <string, byte[]>(config);
builder.SetErrorHandler(OnError);
_consumer = new Lazy <IConsumer <string, byte[]> >(() => builder.Build());
_pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages)
{
IsBackground = true
});
_trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling)
{
IsBackground = true
});
}
public void SimpleConsume(string topic, AutoOffsetReset offset, Action <ConsumeResult <Ignore, string> > messageHandler)
{
var consumerConfig = new ConsumerConfig {
GroupId = this.UniqueServiceId,
BootstrapServers = this.Brokers,
AutoOffsetReset = offset
};
using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) {
consumer.Subscribe(topic);
try {
while (true)
{
try {
var consumeResult = consumer.Consume();
messageHandler(consumeResult);
} catch (ConsumeException e) {
Console.WriteLine($"Error occurred: {e.Error.Reason}");
}
}
} catch (OperationCanceledException) {
// Ensure the consumer leaves the group cleanly and final offsets are committed.
consumer.Close();
}
}
}
public void SetAutoOffsetReset(AutoOffsetReset autoOffsetReset)
{
QueryParameters[RestApi.Parameters.QueryParameters.AutoOffsetResetPropertyName] =
autoOffsetReset.ToKSqlValue();
QueryStreamParameters[QueryStreamParameters.AutoOffsetResetPropertyName] =
autoOffsetReset.ToKSqlValue();
}
private void Handle(string fromServer, string srcTopic, string group,
AutoOffsetReset autoOffsetReset, CancellationToken cancellationToken)
{
var channel = Channel.CreateUnbounded <Message <byte[], byte[]> >();
var source = new KafkaSource(fromServer, srcTopic, group, autoOffsetReset);
var consumerTask = _kafkaService.StarConsumer(source.Topic, source.GetConsumerConfig(), channel.Writer,
cancellationToken);
var printTask = _kafkaService.StartPrintMessages(channel.Reader, cancellationToken);
}
public KafkaMessageConsumer()
{
_bootstrapServers = "";
_groupId = "dotnet-kafka-client";
_autoOffsetReset = AutoOffsetReset.Earliest;
_topics = new List <String>()
{
"test-topic"
};
_cts = new CancellationTokenSource();
Console.CancelKeyPress += new ConsoleCancelEventHandler(CancelKeyPressHandler);
}
public Dictionary <string, object> GetConsumerConfig()
{
return(new Dictionary <string, object>
{
{ "auto.commit.interval.ms", AutoCommitInterval.GetValueOrDefault(5000) },
{ "auto.offset.reset", AutoOffsetReset.GetValueOrDefault(Configuration.AutoOffsetReset.Earliest).ToString().ToLower() },
{ "bootstrap.servers", string.Join(",", BootstrapServers) },
{ "client.id", ClientId ?? $"{Assembly.GetEntryAssembly().FullName.ToLower()}-{Environment.MachineName.ToLower()}" },
{ "enable.auto.commit", EnableAutoCommit.GetValueOrDefault(false) },
{ "group.id", GroupId ?? $"{Assembly.GetEntryAssembly().GetName().Name.ToLower()}" },
{ "heartbeat.interval.ms", HeartbeatMs.GetValueOrDefault(3000) },
{ "request.timeout.ms", RequestTimeoutMs.GetValueOrDefault(30000) },
{ "session.timeout.ms", SessionTimeoutMs.GetValueOrDefault(10000) },
{ "debug", DebugContexts ?? "generic" },
{ "log_level", (int)LogLevel.GetValueOrDefault(Configuration.LogLevel.LOG_INFO) }
});
}
/// <summary>
/// Replays messages that were created from <paramref name="start"/> to <paramref name=
/// "end"/>, invoking the <paramref name="callback"/> delegate for each message. If
/// <paramref name="end"/> is null, then messages that were created from <paramref name=
/// "start"/> to the current UTC time are replayed.
/// </summary>
/// <param name="start">The start time.</param>
/// <param name="end">
/// The end time, or <see langword="null"/> to use the current time as the end time.
/// </param>
/// <param name="callback">The delegate to invoke for each replayed message.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="enableAutoOffsetStore">
/// Whether to automatically store offset of each message replayed.
/// </param>
/// <param name="autoOffsetReset">
/// Action to take when there is no initial offset in offset store or the desired
/// offset is out of range: 'smallest','earliest' - automatically reset the offset
/// to the smallest offset, 'largest','latest' - automatically reset the offset to
/// the largest offset, 'error' - trigger an error which is retrieved by consuming
/// messages and checking 'message->err'.
/// </param>
/// <exception cref="ArgumentException">
/// If <paramref name="end"/> is earlier than <paramref name="start"/>, or if <paramref
/// name="end"/> is null and <paramref name="start"/> is after the current UTC time.
/// </exception>
/// <exception cref="ArgumentNullException">
/// If <paramref name="callback"/> is null, or <paramref name="topic"/> is null or empty,
/// or <paramref name="bootstrapServers"/> is null or empty.
/// </exception>
public async Task Replay(DateTime start, DateTime?end, Func <IReceiverMessage, Task> callback,
string topic, string bootstrapServers, bool enableAutoOffsetStore = false,
AutoOffsetReset autoOffsetReset = AutoOffsetReset.Latest)
{
if (end.HasValue)
{
if (end.Value.Kind != DateTimeKind.Utc)
{
end = end.Value.ToUniversalTime();
}
if (end.Value < start)
{
throw new ArgumentException("Cannot be earlier than 'start' parameter.", nameof(end));
}
}
else
{
end = GetUtcNow();
if (end.Value < start)
{
throw new ArgumentException("Cannot be later than DateTime.UtcNow when 'end' parameter is null.", nameof(start));
}
}
if (callback is null)
{
throw new ArgumentNullException(nameof(callback));
}
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentNullException(nameof(topic), "Cannot be null or empty.");
}
if (string.IsNullOrEmpty(bootstrapServers))
{
throw new ArgumentNullException(nameof(bootstrapServers), "Cannot be null or empty.");
}
var consumer = GetConsumer(bootstrapServers, enableAutoOffsetStore, autoOffsetReset);
var startTimestamps = GetStartTimestamps(topic, bootstrapServers, start);
var startOffsets = consumer.OffsetsForTimes(startTimestamps, TimeSpan.FromSeconds(5));
await Replay(consumer, startOffsets, end.Value, callback, enableAutoOffsetStore);
}
private void Handle(string fromServer, string srcTopic, string group,
AutoOffsetReset autoOffsetReset, string toServer, string dstTopic,
CancellationToken cancellationToken = default)
{
var channel = Channel.CreateUnbounded <Message <byte[], byte[]> >();
dstTopic = dstTopic.IsEmpty()
? srcTopic
: dstTopic;
var source = new KafkaSource(fromServer, srcTopic, group, autoOffsetReset);
var destination = new KafkaDestination(toServer, dstTopic);
var consumerTask = _kafkaService.StarConsumer(source.Topic, source.GetConsumerConfig(), channel.Writer,
cancellationToken);
var producerTask = _kafkaService.StartProducer(destination.Topic, destination.GetProducerConfig(), channel.Reader,
cancellationToken);
}
private Task <IEnumerable <Message> > GetMessagesAsync(string topic, AutoOffsetReset offsetResetSetting)
{
List <Message> result = new List <Message>();
CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
try
{
ConsumerConfig consumerConfig = new ConsumerConfig()
{
BootstrapServers = _connectionConfiguration.BootstrapServers,
GroupId = Guid.NewGuid().ToString() + DateTime.Now.Ticks,
AutoOffsetReset = offsetResetSetting,
EnableAutoCommit = false
};
using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig).Build())
{
consumer.Subscribe(topic);
while (!cts.IsCancellationRequested)
{
var cr = consumer.Consume(cts.Token);
result.Add(new Message()
{
Id = cr.Offset.Value,
Data = cr.Message.Value,
Topic = cr.Topic,
PartitionOffset = cr.TopicPartitionOffset.TopicPartition.Partition.Value
});
}
}
}
catch (OperationCanceledException oce)
{
// cancellation token canceled mean that we have no more message coming in, return
_logger?.Error($"Operation canceled, probably due to time out which mean there's no message coming in", oce);
}
catch (Exception ex)
{
_logger?.Error(ex);
throw;
}
return(Task.FromResult((IEnumerable <Message>)result));
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaReceiver"/> class.
/// </summary>
/// <param name="name">The name of the receiver.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="groupId">
/// Client group id string. All clients sharing the same group.id belong to the same group.
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="enableAutoOffsetStore">
/// Whether to automatically store offset of last message provided to application.
/// </param>
/// <param name="autoOffsetReset">
/// Action to take when there is no initial offset in offset store or the desired
/// offset is out of range: 'smallest','earliest' - automatically reset the offset
/// to the smallest offset, 'largest','latest' - automatically reset the offset to
/// the largest offset, 'error' - trigger an error which is retrieved by consuming
/// messages and checking 'message->err'.
/// </param>
/// <param name="replayEngine">
/// The <see cref="IReplayEngine"/> used to replay messages. If <see langword="null"/>,
/// then a <see cref="DefaultReplayEngine"/> is used.
/// </param>
public KafkaReceiver(string name, string topic, string groupId, string bootstrapServers,
bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = AutoOffsetReset.Latest,
IReplayEngine replayEngine = null)
: base(name)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentNullException(nameof(topic));
}
if (string.IsNullOrEmpty(groupId))
{
throw new ArgumentNullException(nameof(groupId));
}
if (string.IsNullOrEmpty(bootstrapServers))
{
throw new ArgumentNullException(nameof(bootstrapServers));
}
Topic = topic;
GroupId = groupId;
BootstrapServers = bootstrapServers;
EnableAutoOffsetStore = enableAutoOffsetStore;
AutoOffsetReset = autoOffsetReset;
ReplayEngine = replayEngine ?? _defaultReplayEngine.Value;
var config = GetConsumerConfig(GroupId, BootstrapServers, EnableAutoOffsetStore, AutoOffsetReset);
var builder = new ConsumerBuilder <string, byte[]>(config);
builder.SetErrorHandler(OnError);
_consumer = new Lazy <IConsumer <string, byte[]> >(() => builder.Build());
_pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages)
{
IsBackground = true
});
_trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling)
{
IsBackground = true
});
}
/// <summary>
/// Initializes a new instance of the <see cref="Subscription"/> class.
/// </summary>
/// <param name="dataType">Type of the data.</param>
/// <param name="name">The name. Defaults to the data type's full name.</param>
/// <param name="channelName">The channel name. Defaults to the data type's full name.</param>
/// <param name="routingKey">The routing key. Defaults to the data type's full name.</param>
/// <param name="groupId">What is the id of the consumer group that this consumer belongs to; will not process the same partition as others in group</param>
/// <param name="bufferSize">The number of messages to buffer at any one time, also the number of messages to retrieve at once. Min of 1 Max of 10</param>
/// <param name="noOfPerformers">The no of threads reading this channel.</param>
/// <param name="timeoutInMilliseconds">The timeout in milliseconds.</param>
/// <param name="requeueCount">The number of times you want to requeue a message before dropping it.</param>
/// <param name="requeueDelayInMilliseconds">The number of milliseconds to delay the delivery of a requeue message for.</param>
/// <param name="unacceptableMessageLimit">The number of unacceptable messages to handle, before stopping reading from the channel.</param>
/// <param name="offsetDefault">Where should we begin processing if we cannot find a stored offset</param>
/// <param name="commitBatchSize">How often should we commit offsets?</param>
/// <param name="sessionTimeoutMs">What is the heartbeat interval for this consumer, after which Kafka will assume dead and rebalance the consumer group</param>
/// <param name="maxPollIntervalMs">How often does the consumer poll for a message to be considered alive, after which Kafka will assume dead and rebalance</param>
/// <param name="sweepUncommittedOffsetsIntervalMs">How often do we commit offsets that have yet to be saved</param>
/// <param name="isolationLevel">Should we read messages that are not on all replicas? May cause duplicates.</param>
/// <param name="isAsync">Is this channel read asynchronously</param>
/// <param name="numOfPartitions">How many partitions should this topic have - used if we create the topic</param>
/// <param name="replicationFactor">How many copies of each partition should we have across our broker's nodes - used if we create the topic</param> /// <param name="channelFactory">The channel factory to create channels for Consumer.</param>
/// <param name="makeChannels">Should we make channels if they don't exist, defaults to creating</param>
/// <param name="emptyChannelDelay">How long to pause when a channel is empty in milliseconds</param>
/// <param name="channelFailureDelay">How long to pause when there is a channel failure in milliseconds</param>
public KafkaSubscription(
Type dataType,
SubscriptionName name = null,
ChannelName channelName = null,
RoutingKey routingKey = null,
string groupId = null,
int bufferSize = 1,
int noOfPerformers = 1,
int timeoutInMilliseconds = 300,
int requeueCount = -1,
int requeueDelayInMilliseconds = 0,
int unacceptableMessageLimit = 0,
AutoOffsetReset offsetDefault = AutoOffsetReset.Earliest,
long commitBatchSize = 10,
int sessionTimeoutMs = 10000,
int maxPollIntervalMs = 300000,
int sweepUncommittedOffsetsIntervalMs = 30000,
IsolationLevel isolationLevel = IsolationLevel.ReadCommitted,
bool isAsync = false,
int numOfPartitions = 1,
short replicationFactor = 1,
IAmAChannelFactory channelFactory = null,
OnMissingChannel makeChannels = OnMissingChannel.Create,
int emptyChannelDelay = 500,
int channelFailureDelay = 1000)
: base(dataType, name, channelName, routingKey, bufferSize, noOfPerformers, timeoutInMilliseconds, requeueCount,
requeueDelayInMilliseconds, unacceptableMessageLimit, isAsync, channelFactory, makeChannels, emptyChannelDelay, channelFailureDelay)
{
CommitBatchSize = commitBatchSize;
GroupId = groupId;
IsolationLevel = isolationLevel;
MaxPollIntervalMs = maxPollIntervalMs;
SweepUncommittedOffsetsIntervalMs = sweepUncommittedOffsetsIntervalMs;
OffsetDefault = offsetDefault;
SessionTimeoutMs = sessionTimeoutMs;
NumPartitions = numOfPartitions;
ReplicationFactor = replicationFactor;
}
public ConsumeResult <Ignore, string> ManualConsume(string topic, AutoOffsetReset offset)
{
Console.WriteLine("Starting Manual Consume");
var consumerConfig = new ConsumerConfig {
GroupId = this.UniqueServiceId,
BootstrapServers = this.Brokers,
AutoOffsetReset = offset
};
using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) {
consumer.Subscribe(topic);
try {
ConsumeResult <Ignore, string> consumeResult = consumer.Consume();
consumer.Close();
return(consumeResult);
} catch (OperationCanceledException) {
consumer.Close();
return(null);
}
}
}
public ConsumerConfig ConsumerConfig(string groupID, AutoOffsetReset autoOffsetReset = AutoOffsetReset.Earliest)
{
switch (SecurityProtocol)
{
case SecurityProtocol.Ssl:
return(new ConsumerConfig
{
GroupId = groupID,
BootstrapServers = NextAddress(ServerAddress) + ":" + Port,
SslCaLocation = this.SslCaLocation,
SecurityProtocol = this.SecurityProtocol,
AutoOffsetReset = autoOffsetReset
});
case SecurityProtocol.SaslSsl:
case SecurityProtocol.SaslPlaintext:
return(new ConsumerConfig
{
GroupId = groupID,
BootstrapServers = NextAddress(ServerAddress) + ":" + Port,
SecurityProtocol = this.SecurityProtocol,
SaslMechanism = this.SaslMechanism,
SaslUsername = this.SaslUsername,
SaslPassword = this.SaslPassword,
AutoOffsetReset = autoOffsetReset
});
default:
return(new ConsumerConfig
{
GroupId = groupID,
BootstrapServers = NextAddress(ServerAddress) + ":" + Port,
SecurityProtocol = this.SecurityProtocol,
AutoOffsetReset = autoOffsetReset
});
}
}
public KafkaMessageConsumer(
KafkaMessagingGatewayConfiguration configuration,
RoutingKey routingKey,
string groupId,
AutoOffsetReset offsetDefault = AutoOffsetReset.Earliest,
int sessionTimeoutMs = 10000,
int maxPollIntervalMs = 10000,
IsolationLevel isolationLevel = IsolationLevel.ReadCommitted,
long commitBatchSize = 10,
int sweepUncommittedOffsetsIntervalMs = 30000,
int readCommittedOffsetsTimeoutMs = 5000,
int numPartitions = 1,
short replicationFactor = 1,
int topicFindTimeoutMs = 10000,
OnMissingChannel makeChannels = OnMissingChannel.Create
)
{
if (configuration is null)
{
throw new ConfigurationException("You must set a KafkaMessaginGatewayConfiguration to connect to a broker");
}
if (routingKey is null)
{
throw new ConfigurationException("You must set a RoutingKey as the Topic for the consumer");
}
if (groupId is null)
{
throw new ConfigurationException("You must set a GroupId for the consumer");
}
Topic = routingKey;
_clientConfig = new ClientConfig
{
BootstrapServers = string.Join(",", configuration.BootStrapServers),
ClientId = configuration.Name,
Debug = configuration.Debug,
SaslMechanism = configuration.SaslMechanisms.HasValue ? (Confluent.Kafka.SaslMechanism?)((int)configuration.SaslMechanisms.Value) : null,
SaslKerberosPrincipal = configuration.SaslKerberosPrincipal,
SaslUsername = configuration.SaslUsername,
SaslPassword = configuration.SaslPassword,
SecurityProtocol = configuration.SecurityProtocol.HasValue ? (Confluent.Kafka.SecurityProtocol?)((int)configuration.SecurityProtocol.Value) : null,
SslCaLocation = configuration.SslCaLocation
};
_consumerConfig = new ConsumerConfig(_clientConfig)
{
GroupId = groupId,
ClientId = configuration.Name,
AutoOffsetReset = offsetDefault,
BootstrapServers = string.Join(",", configuration.BootStrapServers),
SessionTimeoutMs = sessionTimeoutMs,
MaxPollIntervalMs = maxPollIntervalMs,
EnablePartitionEof = true,
AllowAutoCreateTopics = false, //We will do this explicit always so as to allow us to set parameters for the topic
IsolationLevel = isolationLevel,
//We commit the last offset for acknowledged requests when a batch of records has been processed.
EnableAutoOffsetStore = false,
EnableAutoCommit = false,
// https://www.confluent.io/blog/cooperative-rebalancing-in-kafka-streams-consumer-ksqldb/
PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky,
};
_maxBatchSize = commitBatchSize;
_sweepUncommittedInterval = TimeSpan.FromMilliseconds(sweepUncommittedOffsetsIntervalMs);
_readCommittedOffsetsTimeoutMs = readCommittedOffsetsTimeoutMs;
_consumer = new ConsumerBuilder <string, string>(_consumerConfig)
.SetPartitionsAssignedHandler((consumer, list) =>
{
var partitions = list.Select(p => $"{p.Topic} : {p.Partition.Value}");
_logger.Value.InfoFormat("Parition Added {0}", String.Join(",", partitions));
_partitions.AddRange(list);
})
.SetPartitionsRevokedHandler((consumer, list) =>
{
_consumer.Commit(list);
var revokedPartitions = list.Select(tpo => $"{tpo.Topic} : {tpo.Partition}").ToList();
_logger.Value.InfoFormat("Partitions for consumer revoked {0}", string.Join(",", revokedPartitions));
_partitions = _partitions.Where(tp => list.All(tpo => tpo.TopicPartition != tp)).ToList();
})
.SetPartitionsLostHandler((consumer, list) =>
{
var lostPartitions = list.Select(tpo => $"{tpo.Topic} : {tpo.Partition}").ToList();
_logger.Value.InfoFormat("Partitions for consumer lost {0}", string.Join(",", lostPartitions));
_partitions = _partitions.Where(tp => list.All(tpo => tpo.TopicPartition != tp)).ToList();
})
.SetErrorHandler((consumer, error) =>
{
_logger.Value.Error($"Code: {error.Code}, Reason: {error.Reason}, Fatal: {error.IsFatal}");
})
.Build();
_logger.Value.InfoFormat($"Kakfa consumer subscribing to {Topic}");
_consumer.Subscribe(new [] { Topic.Value });
_creator = new KafkaMessageCreator();
MakeChannels = makeChannels;
Topic = routingKey;
NumPartitions = numPartitions;
ReplicationFactor = replicationFactor;
TopicFindTimeoutMs = topicFindTimeoutMs;
EnsureTopic();
}
public static string ToKSqlValue(this AutoOffsetReset value)
{
return(value.ToString().ToLower());
}
public ConsumerConfig GetConsumerConfig(string bootstrapServers, string groupId, AutoOffsetReset autoOffset = AutoOffsetReset.Latest)
{
return(new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = groupId,
EnableAutoCommit = true,
AutoCommitIntervalMs = 5000,
AutoOffsetReset = autoOffset
});
}
/// <summary>
/// Replays messages that were created from <paramref name="start"/> to <paramref name=
/// "end"/>, invoking the <paramref name="callback"/> delegate for each message. If
/// <paramref name="end"/> is null, then messages that were created from <paramref name=
/// "start"/> to the current UTC time are replayed.
/// </summary>
/// <param name="start">The start time.</param>
/// <param name="end">
/// The end time, or <see langword="null"/> to use the current time as the end time.
/// </param>
/// <param name="callback">The delegate to invoke for each replayed message.</param>
/// <param name="topic">
/// The topic to subscribe to. A regex can be specified to subscribe to the set of
/// all matching topics (which is updated as topics are added / removed from the
/// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="enableAutoOffsetStore">
/// Whether to automatically store offset of each message replayed.
/// </param>
/// <param name="autoOffsetReset">
/// Action to take when there is no initial offset in offset store or the desired
/// offset is out of range: 'smallest','earliest' - automatically reset the offset
/// to the smallest offset, 'largest','latest' - automatically reset the offset to
/// the largest offset, 'error' - trigger an error which is retrieved by consuming
/// messages and checking 'message->err'.
/// </param>
/// <exception cref="ArgumentException">
/// If <paramref name="end"/> is earlier than <paramref name="start"/>, or if <paramref
/// name="end"/> is null and <paramref name="start"/> is after the current UTC time.
/// </exception>
/// <exception cref="ArgumentNullException">
/// If <paramref name="callback"/> is null, or <paramref name="topic"/> is null or empty,
/// or <paramref name="bootstrapServers"/> is null or empty.
/// </exception>
public static Task Replay(DateTime start, DateTime?end, Func <IReceiverMessage, Task> callback,
string topic, string bootstrapServers, bool enableAutoOffsetStore = false,
AutoOffsetReset autoOffsetReset = AutoOffsetReset.Latest) =>
_defaultReplayEngine.Value.Replay(start, end, callback, topic, bootstrapServers, enableAutoOffsetStore, autoOffsetReset);
/// <summary>
/// Gets a consumer.
/// </summary>
protected virtual IConsumer <string, byte[]> GetConsumer(string bootstrapServers,
bool enableAutoOffsetStore, AutoOffsetReset autoOffsetReset)
{
var config = GetConsumerConfig(ReplayGroupId, bootstrapServers, enableAutoOffsetStore, autoOffsetReset);
var builder = new ConsumerBuilder <string, byte[]>(config);
return(builder.Build());
}
internal static ConsumerConfig GetConsumerConfig(string groupId, string bootstrapServers, bool enableAutoOffsetStore, AutoOffsetReset autoOffsetReset) =>
new ConsumerConfig
{
GroupId = groupId,
BootstrapServers = bootstrapServers,
EnableAutoOffsetStore = enableAutoOffsetStore,
AutoOffsetReset = autoOffsetReset
};
public ConsumerConnectionBuilder <TKey, TValue> AutoOffSetReset(AutoOffsetReset autoOffSetReset)
{
this.configs.AutoOffsetReset = autoOffSetReset;
return(this);
}
/// <summary>
/// Determines what to do when there is no initial offset in Apache Kafka® or if the current offset doesn't exist on the server. The default value in ksqlDB is latest, which means all Kafka topics are read from the latest available offset.
/// </summary>
/// <typeparam name="TSource">The type of the elements of source.</typeparam>
/// <param name="source">The sequence to take elements from.</param>
/// <param name="autoOffsetReset">Earliest or the latest offset.</param>
/// <returns>The original sequence.</returns>
public static IQbservable <TSource> WithOffsetResetPolicy <TSource>(this IQbservable <TSource> source, AutoOffsetReset autoOffsetReset)
{
return(source.Provider.CreateQuery <TSource>(
Expression.Call(
null,
WithOffsetResetPolicyTResult(typeof(TSource)),
source.Expression, Expression.Constant(autoOffsetReset)
)));
}
public KafkaMessagingConsumerConfiguration()
{
OffsetDefault = AutoOffsetReset.Earliest;
}
