/// <summary> /// Initializes a new instance of the <see cref="KafkaReceiver"/> class. /// </summary> /// <param name="name">The name of the receiver.</param> /// <param name="topic"> /// The topic to subscribe to. A regex can be specified to subscribe to the set of /// all matching topics (which is updated as topics are added / removed from the /// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex /// </param> /// <param name="groupId"> /// Client group id string. All clients sharing the same group.id belong to the same group. /// </param> /// <param name="bootstrapServers"> /// List of brokers as a CSV list of broker host or host:port. /// </param> /// <param name="enableAutoOffsetStore"> /// Whether to automatically store offset of last message provided to application. /// </param> /// <param name="autoOffsetReset"> /// Action to take when there is no initial offset in offset store or the desired /// offset is out of range: 'smallest','earliest' - automatically reset the offset /// to the smallest offset, 'largest','latest' - automatically reset the offset to /// the largest offset, 'error' - trigger an error which is retrieved by consuming /// messages and checking 'message->err'. /// </param> public KafkaReceiver(string name, string topic, string groupId, string bootstrapServers, bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = Confluent.Kafka.AutoOffsetReset.Latest) : base(name) { Topic = topic ?? throw new ArgumentNullException(nameof(topic)); GroupId = groupId ?? throw new ArgumentNullException(nameof(groupId)); BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers)); EnableAutoOffsetStore = enableAutoOffsetStore; AutoOffsetReset = autoOffsetReset; var config = GetConsumerConfig(groupId, bootstrapServers, enableAutoOffsetStore, autoOffsetReset); var builder = new ConsumerBuilder <string, byte[]>(config); builder.SetErrorHandler(OnError); _consumer = new Lazy <IConsumer <string, byte[]> >(() => builder.Build()); _pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages) { IsBackground = true }); _trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling) { IsBackground = true }); }
public void SimpleConsume(string topic, AutoOffsetReset offset, Action <ConsumeResult <Ignore, string> > messageHandler) { var consumerConfig = new ConsumerConfig { GroupId = this.UniqueServiceId, BootstrapServers = this.Brokers, AutoOffsetReset = offset }; using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) { consumer.Subscribe(topic); try { while (true) { try { var consumeResult = consumer.Consume(); messageHandler(consumeResult); } catch (ConsumeException e) { Console.WriteLine($"Error occurred: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. consumer.Close(); } } }
public void SetAutoOffsetReset(AutoOffsetReset autoOffsetReset) { QueryParameters[RestApi.Parameters.QueryParameters.AutoOffsetResetPropertyName] = autoOffsetReset.ToKSqlValue(); QueryStreamParameters[QueryStreamParameters.AutoOffsetResetPropertyName] = autoOffsetReset.ToKSqlValue(); }
private void Handle(string fromServer, string srcTopic, string group, AutoOffsetReset autoOffsetReset, CancellationToken cancellationToken) { var channel = Channel.CreateUnbounded <Message <byte[], byte[]> >(); var source = new KafkaSource(fromServer, srcTopic, group, autoOffsetReset); var consumerTask = _kafkaService.StarConsumer(source.Topic, source.GetConsumerConfig(), channel.Writer, cancellationToken); var printTask = _kafkaService.StartPrintMessages(channel.Reader, cancellationToken); }
public KafkaMessageConsumer() { _bootstrapServers = ""; _groupId = "dotnet-kafka-client"; _autoOffsetReset = AutoOffsetReset.Earliest; _topics = new List <String>() { "test-topic" }; _cts = new CancellationTokenSource(); Console.CancelKeyPress += new ConsoleCancelEventHandler(CancelKeyPressHandler); }
public Dictionary <string, object> GetConsumerConfig() { return(new Dictionary <string, object> { { "auto.commit.interval.ms", AutoCommitInterval.GetValueOrDefault(5000) }, { "auto.offset.reset", AutoOffsetReset.GetValueOrDefault(Configuration.AutoOffsetReset.Earliest).ToString().ToLower() }, { "bootstrap.servers", string.Join(",", BootstrapServers) }, { "client.id", ClientId ?? $"{Assembly.GetEntryAssembly().FullName.ToLower()}-{Environment.MachineName.ToLower()}" }, { "enable.auto.commit", EnableAutoCommit.GetValueOrDefault(false) }, { "group.id", GroupId ?? $"{Assembly.GetEntryAssembly().GetName().Name.ToLower()}" }, { "heartbeat.interval.ms", HeartbeatMs.GetValueOrDefault(3000) }, { "request.timeout.ms", RequestTimeoutMs.GetValueOrDefault(30000) }, { "session.timeout.ms", SessionTimeoutMs.GetValueOrDefault(10000) }, { "debug", DebugContexts ?? "generic" }, { "log_level", (int)LogLevel.GetValueOrDefault(Configuration.LogLevel.LOG_INFO) } }); }
/// <summary> /// Replays messages that were created from <paramref name="start"/> to <paramref name= /// "end"/>, invoking the <paramref name="callback"/> delegate for each message. If /// <paramref name="end"/> is null, then messages that were created from <paramref name= /// "start"/> to the current UTC time are replayed. /// </summary> /// <param name="start">The start time.</param> /// <param name="end"> /// The end time, or <see langword="null"/> to use the current time as the end time. /// </param> /// <param name="callback">The delegate to invoke for each replayed message.</param> /// <param name="topic"> /// The topic to subscribe to. A regex can be specified to subscribe to the set of /// all matching topics (which is updated as topics are added / removed from the /// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex /// </param> /// <param name="bootstrapServers"> /// List of brokers as a CSV list of broker host or host:port. /// </param> /// <param name="enableAutoOffsetStore"> /// Whether to automatically store offset of each message replayed. /// </param> /// <param name="autoOffsetReset"> /// Action to take when there is no initial offset in offset store or the desired /// offset is out of range: 'smallest','earliest' - automatically reset the offset /// to the smallest offset, 'largest','latest' - automatically reset the offset to /// the largest offset, 'error' - trigger an error which is retrieved by consuming /// messages and checking 'message->err'. /// </param> /// <exception cref="ArgumentException"> /// If <paramref name="end"/> is earlier than <paramref name="start"/>, or if <paramref /// name="end"/> is null and <paramref name="start"/> is after the current UTC time. /// </exception> /// <exception cref="ArgumentNullException"> /// If <paramref name="callback"/> is null, or <paramref name="topic"/> is null or empty, /// or <paramref name="bootstrapServers"/> is null or empty. /// </exception> public async Task Replay(DateTime start, DateTime?end, Func <IReceiverMessage, Task> callback, string topic, string bootstrapServers, bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = AutoOffsetReset.Latest) { if (end.HasValue) { if (end.Value.Kind != DateTimeKind.Utc) { end = end.Value.ToUniversalTime(); } if (end.Value < start) { throw new ArgumentException("Cannot be earlier than 'start' parameter.", nameof(end)); } } else { end = GetUtcNow(); if (end.Value < start) { throw new ArgumentException("Cannot be later than DateTime.UtcNow when 'end' parameter is null.", nameof(start)); } } if (callback is null) { throw new ArgumentNullException(nameof(callback)); } if (string.IsNullOrEmpty(topic)) { throw new ArgumentNullException(nameof(topic), "Cannot be null or empty."); } if (string.IsNullOrEmpty(bootstrapServers)) { throw new ArgumentNullException(nameof(bootstrapServers), "Cannot be null or empty."); } var consumer = GetConsumer(bootstrapServers, enableAutoOffsetStore, autoOffsetReset); var startTimestamps = GetStartTimestamps(topic, bootstrapServers, start); var startOffsets = consumer.OffsetsForTimes(startTimestamps, TimeSpan.FromSeconds(5)); await Replay(consumer, startOffsets, end.Value, callback, enableAutoOffsetStore); }
private void Handle(string fromServer, string srcTopic, string group, AutoOffsetReset autoOffsetReset, string toServer, string dstTopic, CancellationToken cancellationToken = default) { var channel = Channel.CreateUnbounded <Message <byte[], byte[]> >(); dstTopic = dstTopic.IsEmpty() ? srcTopic : dstTopic; var source = new KafkaSource(fromServer, srcTopic, group, autoOffsetReset); var destination = new KafkaDestination(toServer, dstTopic); var consumerTask = _kafkaService.StarConsumer(source.Topic, source.GetConsumerConfig(), channel.Writer, cancellationToken); var producerTask = _kafkaService.StartProducer(destination.Topic, destination.GetProducerConfig(), channel.Reader, cancellationToken); }
private Task <IEnumerable <Message> > GetMessagesAsync(string topic, AutoOffsetReset offsetResetSetting) { List <Message> result = new List <Message>(); CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); try { ConsumerConfig consumerConfig = new ConsumerConfig() { BootstrapServers = _connectionConfiguration.BootstrapServers, GroupId = Guid.NewGuid().ToString() + DateTime.Now.Ticks, AutoOffsetReset = offsetResetSetting, EnableAutoCommit = false }; using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig).Build()) { consumer.Subscribe(topic); while (!cts.IsCancellationRequested) { var cr = consumer.Consume(cts.Token); result.Add(new Message() { Id = cr.Offset.Value, Data = cr.Message.Value, Topic = cr.Topic, PartitionOffset = cr.TopicPartitionOffset.TopicPartition.Partition.Value }); } } } catch (OperationCanceledException oce) { // cancellation token canceled mean that we have no more message coming in, return _logger?.Error($"Operation canceled, probably due to time out which mean there's no message coming in", oce); } catch (Exception ex) { _logger?.Error(ex); throw; } return(Task.FromResult((IEnumerable <Message>)result)); }
/// <summary> /// Initializes a new instance of the <see cref="KafkaReceiver"/> class. /// </summary> /// <param name="name">The name of the receiver.</param> /// <param name="topic"> /// The topic to subscribe to. A regex can be specified to subscribe to the set of /// all matching topics (which is updated as topics are added / removed from the /// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex /// </param> /// <param name="groupId"> /// Client group id string. All clients sharing the same group.id belong to the same group. /// </param> /// <param name="bootstrapServers"> /// List of brokers as a CSV list of broker host or host:port. /// </param> /// <param name="enableAutoOffsetStore"> /// Whether to automatically store offset of last message provided to application. /// </param> /// <param name="autoOffsetReset"> /// Action to take when there is no initial offset in offset store or the desired /// offset is out of range: 'smallest','earliest' - automatically reset the offset /// to the smallest offset, 'largest','latest' - automatically reset the offset to /// the largest offset, 'error' - trigger an error which is retrieved by consuming /// messages and checking 'message->err'. /// </param> /// <param name="replayEngine"> /// The <see cref="IReplayEngine"/> used to replay messages. If <see langword="null"/>, /// then a <see cref="DefaultReplayEngine"/> is used. /// </param> public KafkaReceiver(string name, string topic, string groupId, string bootstrapServers, bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = AutoOffsetReset.Latest, IReplayEngine replayEngine = null) : base(name) { if (string.IsNullOrEmpty(topic)) { throw new ArgumentNullException(nameof(topic)); } if (string.IsNullOrEmpty(groupId)) { throw new ArgumentNullException(nameof(groupId)); } if (string.IsNullOrEmpty(bootstrapServers)) { throw new ArgumentNullException(nameof(bootstrapServers)); } Topic = topic; GroupId = groupId; BootstrapServers = bootstrapServers; EnableAutoOffsetStore = enableAutoOffsetStore; AutoOffsetReset = autoOffsetReset; ReplayEngine = replayEngine ?? _defaultReplayEngine.Value; var config = GetConsumerConfig(GroupId, BootstrapServers, EnableAutoOffsetStore, AutoOffsetReset); var builder = new ConsumerBuilder <string, byte[]>(config); builder.SetErrorHandler(OnError); _consumer = new Lazy <IConsumer <string, byte[]> >(() => builder.Build()); _pollingThread = new Lazy <Thread>(() => new Thread(PollForMessages) { IsBackground = true }); _trackingThread = new Lazy <Thread>(() => new Thread(TrackMessageHandling) { IsBackground = true }); }
/// <summary> /// Initializes a new instance of the <see cref="Subscription"/> class. /// </summary> /// <param name="dataType">Type of the data.</param> /// <param name="name">The name. Defaults to the data type's full name.</param> /// <param name="channelName">The channel name. Defaults to the data type's full name.</param> /// <param name="routingKey">The routing key. Defaults to the data type's full name.</param> /// <param name="groupId">What is the id of the consumer group that this consumer belongs to; will not process the same partition as others in group</param> /// <param name="bufferSize">The number of messages to buffer at any one time, also the number of messages to retrieve at once. Min of 1 Max of 10</param> /// <param name="noOfPerformers">The no of threads reading this channel.</param> /// <param name="timeoutInMilliseconds">The timeout in milliseconds.</param> /// <param name="requeueCount">The number of times you want to requeue a message before dropping it.</param> /// <param name="requeueDelayInMilliseconds">The number of milliseconds to delay the delivery of a requeue message for.</param> /// <param name="unacceptableMessageLimit">The number of unacceptable messages to handle, before stopping reading from the channel.</param> /// <param name="offsetDefault">Where should we begin processing if we cannot find a stored offset</param> /// <param name="commitBatchSize">How often should we commit offsets?</param> /// <param name="sessionTimeoutMs">What is the heartbeat interval for this consumer, after which Kafka will assume dead and rebalance the consumer group</param> /// <param name="maxPollIntervalMs">How often does the consumer poll for a message to be considered alive, after which Kafka will assume dead and rebalance</param> /// <param name="sweepUncommittedOffsetsIntervalMs">How often do we commit offsets that have yet to be saved</param> /// <param name="isolationLevel">Should we read messages that are not on all replicas? May cause duplicates.</param> /// <param name="isAsync">Is this channel read asynchronously</param> /// <param name="numOfPartitions">How many partitions should this topic have - used if we create the topic</param> /// <param name="replicationFactor">How many copies of each partition should we have across our broker's nodes - used if we create the topic</param> /// <param name="channelFactory">The channel factory to create channels for Consumer.</param> /// <param name="makeChannels">Should we make channels if they don't exist, defaults to creating</param> /// <param name="emptyChannelDelay">How long to pause when a channel is empty in milliseconds</param> /// <param name="channelFailureDelay">How long to pause when there is a channel failure in milliseconds</param> public KafkaSubscription( Type dataType, SubscriptionName name = null, ChannelName channelName = null, RoutingKey routingKey = null, string groupId = null, int bufferSize = 1, int noOfPerformers = 1, int timeoutInMilliseconds = 300, int requeueCount = -1, int requeueDelayInMilliseconds = 0, int unacceptableMessageLimit = 0, AutoOffsetReset offsetDefault = AutoOffsetReset.Earliest, long commitBatchSize = 10, int sessionTimeoutMs = 10000, int maxPollIntervalMs = 300000, int sweepUncommittedOffsetsIntervalMs = 30000, IsolationLevel isolationLevel = IsolationLevel.ReadCommitted, bool isAsync = false, int numOfPartitions = 1, short replicationFactor = 1, IAmAChannelFactory channelFactory = null, OnMissingChannel makeChannels = OnMissingChannel.Create, int emptyChannelDelay = 500, int channelFailureDelay = 1000) : base(dataType, name, channelName, routingKey, bufferSize, noOfPerformers, timeoutInMilliseconds, requeueCount, requeueDelayInMilliseconds, unacceptableMessageLimit, isAsync, channelFactory, makeChannels, emptyChannelDelay, channelFailureDelay) { CommitBatchSize = commitBatchSize; GroupId = groupId; IsolationLevel = isolationLevel; MaxPollIntervalMs = maxPollIntervalMs; SweepUncommittedOffsetsIntervalMs = sweepUncommittedOffsetsIntervalMs; OffsetDefault = offsetDefault; SessionTimeoutMs = sessionTimeoutMs; NumPartitions = numOfPartitions; ReplicationFactor = replicationFactor; }
public ConsumeResult <Ignore, string> ManualConsume(string topic, AutoOffsetReset offset) { Console.WriteLine("Starting Manual Consume"); var consumerConfig = new ConsumerConfig { GroupId = this.UniqueServiceId, BootstrapServers = this.Brokers, AutoOffsetReset = offset }; using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) { consumer.Subscribe(topic); try { ConsumeResult <Ignore, string> consumeResult = consumer.Consume(); consumer.Close(); return(consumeResult); } catch (OperationCanceledException) { consumer.Close(); return(null); } } }
public ConsumerConfig ConsumerConfig(string groupID, AutoOffsetReset autoOffsetReset = AutoOffsetReset.Earliest) { switch (SecurityProtocol) { case SecurityProtocol.Ssl: return(new ConsumerConfig { GroupId = groupID, BootstrapServers = NextAddress(ServerAddress) + ":" + Port, SslCaLocation = this.SslCaLocation, SecurityProtocol = this.SecurityProtocol, AutoOffsetReset = autoOffsetReset }); case SecurityProtocol.SaslSsl: case SecurityProtocol.SaslPlaintext: return(new ConsumerConfig { GroupId = groupID, BootstrapServers = NextAddress(ServerAddress) + ":" + Port, SecurityProtocol = this.SecurityProtocol, SaslMechanism = this.SaslMechanism, SaslUsername = this.SaslUsername, SaslPassword = this.SaslPassword, AutoOffsetReset = autoOffsetReset }); default: return(new ConsumerConfig { GroupId = groupID, BootstrapServers = NextAddress(ServerAddress) + ":" + Port, SecurityProtocol = this.SecurityProtocol, AutoOffsetReset = autoOffsetReset }); } }
public KafkaMessageConsumer( KafkaMessagingGatewayConfiguration configuration, RoutingKey routingKey, string groupId, AutoOffsetReset offsetDefault = AutoOffsetReset.Earliest, int sessionTimeoutMs = 10000, int maxPollIntervalMs = 10000, IsolationLevel isolationLevel = IsolationLevel.ReadCommitted, long commitBatchSize = 10, int sweepUncommittedOffsetsIntervalMs = 30000, int readCommittedOffsetsTimeoutMs = 5000, int numPartitions = 1, short replicationFactor = 1, int topicFindTimeoutMs = 10000, OnMissingChannel makeChannels = OnMissingChannel.Create ) { if (configuration is null) { throw new ConfigurationException("You must set a KafkaMessaginGatewayConfiguration to connect to a broker"); } if (routingKey is null) { throw new ConfigurationException("You must set a RoutingKey as the Topic for the consumer"); } if (groupId is null) { throw new ConfigurationException("You must set a GroupId for the consumer"); } Topic = routingKey; _clientConfig = new ClientConfig { BootstrapServers = string.Join(",", configuration.BootStrapServers), ClientId = configuration.Name, Debug = configuration.Debug, SaslMechanism = configuration.SaslMechanisms.HasValue ? (Confluent.Kafka.SaslMechanism?)((int)configuration.SaslMechanisms.Value) : null, SaslKerberosPrincipal = configuration.SaslKerberosPrincipal, SaslUsername = configuration.SaslUsername, SaslPassword = configuration.SaslPassword, SecurityProtocol = configuration.SecurityProtocol.HasValue ? (Confluent.Kafka.SecurityProtocol?)((int)configuration.SecurityProtocol.Value) : null, SslCaLocation = configuration.SslCaLocation }; _consumerConfig = new ConsumerConfig(_clientConfig) { GroupId = groupId, ClientId = configuration.Name, AutoOffsetReset = offsetDefault, BootstrapServers = string.Join(",", configuration.BootStrapServers), SessionTimeoutMs = sessionTimeoutMs, MaxPollIntervalMs = maxPollIntervalMs, EnablePartitionEof = true, AllowAutoCreateTopics = false, //We will do this explicit always so as to allow us to set parameters for the topic IsolationLevel = isolationLevel, //We commit the last offset for acknowledged requests when a batch of records has been processed. EnableAutoOffsetStore = false, EnableAutoCommit = false, // https://www.confluent.io/blog/cooperative-rebalancing-in-kafka-streams-consumer-ksqldb/ PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky, }; _maxBatchSize = commitBatchSize; _sweepUncommittedInterval = TimeSpan.FromMilliseconds(sweepUncommittedOffsetsIntervalMs); _readCommittedOffsetsTimeoutMs = readCommittedOffsetsTimeoutMs; _consumer = new ConsumerBuilder <string, string>(_consumerConfig) .SetPartitionsAssignedHandler((consumer, list) => { var partitions = list.Select(p => $"{p.Topic} : {p.Partition.Value}"); _logger.Value.InfoFormat("Parition Added {0}", String.Join(",", partitions)); _partitions.AddRange(list); }) .SetPartitionsRevokedHandler((consumer, list) => { _consumer.Commit(list); var revokedPartitions = list.Select(tpo => $"{tpo.Topic} : {tpo.Partition}").ToList(); _logger.Value.InfoFormat("Partitions for consumer revoked {0}", string.Join(",", revokedPartitions)); _partitions = _partitions.Where(tp => list.All(tpo => tpo.TopicPartition != tp)).ToList(); }) .SetPartitionsLostHandler((consumer, list) => { var lostPartitions = list.Select(tpo => $"{tpo.Topic} : {tpo.Partition}").ToList(); _logger.Value.InfoFormat("Partitions for consumer lost {0}", string.Join(",", lostPartitions)); _partitions = _partitions.Where(tp => list.All(tpo => tpo.TopicPartition != tp)).ToList(); }) .SetErrorHandler((consumer, error) => { _logger.Value.Error($"Code: {error.Code}, Reason: {error.Reason}, Fatal: {error.IsFatal}"); }) .Build(); _logger.Value.InfoFormat($"Kakfa consumer subscribing to {Topic}"); _consumer.Subscribe(new [] { Topic.Value }); _creator = new KafkaMessageCreator(); MakeChannels = makeChannels; Topic = routingKey; NumPartitions = numPartitions; ReplicationFactor = replicationFactor; TopicFindTimeoutMs = topicFindTimeoutMs; EnsureTopic(); }
public static string ToKSqlValue(this AutoOffsetReset value) { return(value.ToString().ToLower()); }
public ConsumerConfig GetConsumerConfig(string bootstrapServers, string groupId, AutoOffsetReset autoOffset = AutoOffsetReset.Latest) { return(new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = groupId, EnableAutoCommit = true, AutoCommitIntervalMs = 5000, AutoOffsetReset = autoOffset }); }
/// <summary> /// Replays messages that were created from <paramref name="start"/> to <paramref name= /// "end"/>, invoking the <paramref name="callback"/> delegate for each message. If /// <paramref name="end"/> is null, then messages that were created from <paramref name= /// "start"/> to the current UTC time are replayed. /// </summary> /// <param name="start">The start time.</param> /// <param name="end"> /// The end time, or <see langword="null"/> to use the current time as the end time. /// </param> /// <param name="callback">The delegate to invoke for each replayed message.</param> /// <param name="topic"> /// The topic to subscribe to. A regex can be specified to subscribe to the set of /// all matching topics (which is updated as topics are added / removed from the /// cluster). A regex must be front anchored to be recognized as a regex. e.g. ^myregex /// </param> /// <param name="bootstrapServers"> /// List of brokers as a CSV list of broker host or host:port. /// </param> /// <param name="enableAutoOffsetStore"> /// Whether to automatically store offset of each message replayed. /// </param> /// <param name="autoOffsetReset"> /// Action to take when there is no initial offset in offset store or the desired /// offset is out of range: 'smallest','earliest' - automatically reset the offset /// to the smallest offset, 'largest','latest' - automatically reset the offset to /// the largest offset, 'error' - trigger an error which is retrieved by consuming /// messages and checking 'message->err'. /// </param> /// <exception cref="ArgumentException"> /// If <paramref name="end"/> is earlier than <paramref name="start"/>, or if <paramref /// name="end"/> is null and <paramref name="start"/> is after the current UTC time. /// </exception> /// <exception cref="ArgumentNullException"> /// If <paramref name="callback"/> is null, or <paramref name="topic"/> is null or empty, /// or <paramref name="bootstrapServers"/> is null or empty. /// </exception> public static Task Replay(DateTime start, DateTime?end, Func <IReceiverMessage, Task> callback, string topic, string bootstrapServers, bool enableAutoOffsetStore = false, AutoOffsetReset autoOffsetReset = AutoOffsetReset.Latest) => _defaultReplayEngine.Value.Replay(start, end, callback, topic, bootstrapServers, enableAutoOffsetStore, autoOffsetReset);
/// <summary> /// Gets a consumer. /// </summary> protected virtual IConsumer <string, byte[]> GetConsumer(string bootstrapServers, bool enableAutoOffsetStore, AutoOffsetReset autoOffsetReset) { var config = GetConsumerConfig(ReplayGroupId, bootstrapServers, enableAutoOffsetStore, autoOffsetReset); var builder = new ConsumerBuilder <string, byte[]>(config); return(builder.Build()); }
internal static ConsumerConfig GetConsumerConfig(string groupId, string bootstrapServers, bool enableAutoOffsetStore, AutoOffsetReset autoOffsetReset) => new ConsumerConfig { GroupId = groupId, BootstrapServers = bootstrapServers, EnableAutoOffsetStore = enableAutoOffsetStore, AutoOffsetReset = autoOffsetReset };
public ConsumerConnectionBuilder <TKey, TValue> AutoOffSetReset(AutoOffsetReset autoOffSetReset) { this.configs.AutoOffsetReset = autoOffSetReset; return(this); }
/// <summary> /// Determines what to do when there is no initial offset in Apache Kafka® or if the current offset doesn't exist on the server. The default value in ksqlDB is latest, which means all Kafka topics are read from the latest available offset. /// </summary> /// <typeparam name="TSource">The type of the elements of source.</typeparam> /// <param name="source">The sequence to take elements from.</param> /// <param name="autoOffsetReset">Earliest or the latest offset.</param> /// <returns>The original sequence.</returns> public static IQbservable <TSource> WithOffsetResetPolicy <TSource>(this IQbservable <TSource> source, AutoOffsetReset autoOffsetReset) { return(source.Provider.CreateQuery <TSource>( Expression.Call( null, WithOffsetResetPolicyTResult(typeof(TSource)), source.Expression, Expression.Constant(autoOffsetReset) ))); }
public KafkaMessagingConsumerConfiguration() { OffsetDefault = AutoOffsetReset.Earliest; }