public void ZkAwareProducerSends1Message() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); var mockPartitioner = new MockAlwaysZeroPartitioner(); using (var producer = new Producer<string, Message>(prodConfig, mockPartitioner, new DefaultEncoder())) { var producerData = new ProducerData<string, Message>( CurrentTestTopic, "somekey", new List<Message> { originalMessage }); producer.Send(producerData); while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfiguration( multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); IConsumer consumer = new Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null & response.Messages.Count() > 0) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); } }
/// <summary> /// Initializes a new instance of the <see cref="Consumer"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> public Consumer(ConsumerConfiguration config) { Guard.NotNull(config, "config"); this.config = config; this.host = config.Broker.Host; this.port = config.Broker.Port; }
public static long GetCurrentKafkaOffset(string topic, string address, int port, int partition) { var request = new OffsetRequest(topic, partition, DateTime.Now.AddDays(-5).Ticks, 10); var consumerConfig = new ConsumerConfiguration(address, port); IConsumer consumer = new Consumer(consumerConfig, address, port); IList<long> list = consumer.GetOffsetsBefore(request); return list.Sum(); }
/// <summary> /// Initializes a new instance of the <see cref="Consumer"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> /// <param name="host"></param> /// <param name="port"></param> public Consumer(ConsumerConfiguration config, string host, int port) { Guard.NotNull(config, "config"); this.config = config; this.host = host; this.port = port; }
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos) { this.name = name; this.zkClient = zkClient; this.config = config; this.broker = broker; this.partitionTopicInfos = partitionTopicInfos; this.simpleConsumer = new Consumer(this.config, broker.Host, broker.Port); }
/// <summary> /// Initializes a new instance of the <see cref="ZookeeperConsumerConnector"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. At the minimum, need to specify the group ID /// of the consumer and the ZooKeeper connection string. /// </param> /// <param name="enableFetcher"> /// Indicates whether fetchers should be enabled /// </param> public ZookeeperConsumerConnector(ConsumerConfiguration config, bool enableFetcher) { this.config = config; this.enableFetcher = enableFetcher; this.ConnectZk(); this.CreateFetcher(); if (this.config.AutoCommit) { Logger.InfoFormat(CultureInfo.CurrentCulture, "starting auto committer every {0} ms", this.config.AutoCommitInterval); scheduler.ScheduleWithRate(this.AutoCommit, this.config.AutoCommitInterval, this.config.AutoCommitInterval); } }
/// <summary> /// Initializes a new instance of the <see cref="Consumer"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> public Consumer(ConsumerConfiguration config) { Guard.NotNull(config, "config"); this.config = config; this.host = config.Broker.Host; this.port = config.Broker.Port; this.connection = new KafkaConnection( this.host, this.port, this.config.BufferSize, this.config.SendTimeout, this.config.ReceiveTimeout, this.config.ReconnectInterval); this.CreatedTimeInUTC = DateTime.UtcNow.Ticks; }
internal ZKRebalancerListener( ConsumerConfiguration config, string consumerIdString, IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry, IZooKeeperClient zkClient, ZookeeperConsumerConnector zkConsumerConnector, IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues, Fetcher fetcher, object syncLock) { this.syncLock = syncLock; this.consumerIdString = consumerIdString; this.config = config; this.topicRegistry = topicRegistry; this.zkClient = zkClient; this.dirs = new ZKGroupDirs(config.GroupId); this.zkConsumerConnector = zkConsumerConnector; this.queues = queues; this.fetcher = fetcher; }
public ConsumerConfiguration(ConsumerConfiguration cosumerConfigTemplate, BrokerConfiguration brokerConfiguration) { this.Broker = brokerConfiguration; this.NumberOfTries = cosumerConfigTemplate.NumberOfTries; this.Timeout = cosumerConfigTemplate.Timeout; this.AutoOffsetReset = cosumerConfigTemplate.AutoOffsetReset; this.AutoCommit = cosumerConfigTemplate.AutoCommit; this.AutoCommitInterval = cosumerConfigTemplate.AutoCommitInterval; this.FetchSize = cosumerConfigTemplate.FetchSize; this.MaxFetchFactor = cosumerConfigTemplate.MaxFetchFactor; this.BackOffIncrement = cosumerConfigTemplate.BackOffIncrement; this.ConsumerId = cosumerConfigTemplate.ConsumerId; this.ReconnectInterval = cosumerConfigTemplate.ReconnectInterval; this.ShutdownTimeout = cosumerConfigTemplate.ShutdownTimeout; this.MaxFetchBufferLength = cosumerConfigTemplate.MaxFetchBufferLength; this.SendTimeout = cosumerConfigTemplate.SendTimeout; this.ReceiveTimeout = cosumerConfigTemplate.ReceiveTimeout; this.BufferSize = cosumerConfigTemplate.BufferSize; this.Verbose = cosumerConfigTemplate.Verbose; this.ConsumeGroupRebalanceRetryIntervalMs = cosumerConfigTemplate.ConsumeGroupRebalanceRetryIntervalMs; this.ConsumeGroupFindNewLeaderSleepIntervalMs = cosumerConfigTemplate.ConsumeGroupFindNewLeaderSleepIntervalMs; }
/// <summary> /// Initializes a new instance of the <see cref="Fetcher"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> /// <param name="zkClient"> /// The wrapper above ZooKeeper client. /// </param> public Fetcher(ConsumerConfiguration config, IZooKeeperClient zkClient) { this.config = config; this.zkClient = zkClient; }
internal ConsumerGroupHelperUnit(int threadID, ConsumeGroupHelperOptions cg, AutoResetEvent e, int c) { this.resetEvent = e; this.cgOptions = cg; this.ThreadID = threadID; this.Count = c; configSettings = new ConsumerConfiguration { AutoOffsetReset = OffsetRequest.SmallestTime, AutoCommit = false, GroupId = cgOptions.ConsumerGroupName, ConsumerId = cgOptions.ConsumerId + "_Thread_" + threadID.ToString(), Timeout = cgOptions.Timeout, ZooKeeper = new ZooKeeperConfiguration(cgOptions.Zookeeper, 30000, 4000, 8000), BufferSize = cgOptions.BufferSize, FetchSize = cgOptions.FetchSize, MaxFetchBufferLength = cgOptions.MaxFetchBufferLength// cgOptions.FetchSize * (10~40) / cgOptions.MessageSize, }; if (cgOptions.CancellationTimeoutMs != KafkaNETExampleConstants.DefaultCancellationTimeoutMs) configSettings.Timeout = -1; }
/// <summary> /// clear the leader and configurations dependent on the leader /// </summary> public void ClearLeader() { if (this.producer != null) { this.producer.Dispose(); this.producer = null; } if (this.consumer != null) { this.consumer.Dispose(); this.consumer = null; } this.producerConf = null; this.consumerConf = null; this.leaders.Clear(); if (this.PartitionsMetadata != null) { this.PartitionsMetadata = null; } }
public void ProducerSends3Messages() { var prodConfig = this.ConfigBasedSyncProdConfig; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1")); var originalMessage2 = new Message(Encoding.UTF8.GetBytes("TestData2")); var originalMessage3 = new Message(Encoding.UTF8.GetBytes("TestData3")); var originalMessageList = new List<Message> { originalMessage1, originalMessage2, originalMessage3 }; var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets( new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); using (var producer = new Producer(prodConfig)) { var producerData = new ProducerData<string, Message>(CurrentTestTopic, originalMessageList); producer.Send(producerData); } Thread.Sleep(waitSingle); while ( !multipleBrokersHelper.CheckIfAnyBrokerHasChanged( new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfiguration( multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); IConsumer consumer = new Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 2) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(3, response.Messages.Count()); Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString()); Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString()); Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString()); }
public static long GetCurrentKafkaOffset(string topic, ConsumerConfiguration clientConfig) { return GetCurrentKafkaOffset(topic, clientConfig.Broker.Host, clientConfig.Broker.Port); }
/// <summary> /// Initializes a new instance of the <see cref="Consumer"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> /// <param name="host"></param> /// <param name="port"></param> public Consumer(ConsumerConfiguration config, string host, int port) { Guard.NotNull(config, "config"); this.config = config; this.host = host; this.port = port; this.connection = new KafkaConnection( this.host, this.port, this.config.BufferSize, this.config.SendTimeout, this.config.ReceiveTimeout, this.config.ReconnectInterval); }
public ClusterHealthChecker(ConsumerConfiguration config) { this.config = config; this.ConnectZk(); }
public ConsumerOffsetChecker(ConsumerConfiguration config) { this.config = config; this.ConnectZk(); }
public static PullResponse PullMessage( ConsumerConfiguration consumerConfig, BrokerConfiguration leaderBrokerConfig, int correlationID, string topic, int partitionIndex, long fetchOffset, KafkaClientHelperConfiguration helperConfiguration, out long offsetNew) { offsetNew = -1; PullResponse result = null; int payloadCount = 0; // at least retry once int maxRetry = 1; int retryCount = 0; string s = string.Empty; bool success = false; while (!success && retryCount < maxRetry) { try { var requestMap = new Dictionary<string, List<PartitionFetchInfo>>(); requestMap.Add( topic, new List<PartitionFetchInfo>() { new PartitionFetchInfo( partitionIndex, fetchOffset, helperConfiguration.FetchSize) }); using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port)) { var response = consumer.Fetch(new FetchRequest( correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id helperConfiguration.MaxWaitTime, helperConfiguration.MinWaitBytes, requestMap)); if (response == null) { throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } var partitionData = response.PartitionData(topic, partitionIndex); if (partitionData == null) { throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode) { s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return null; } if (partitionData.Error != ErrorMapping.NoError) { s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return null; } var messages = partitionData.MessageSet.Messages; s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); success = true; result = new PullResponse(partitionData); if (null != messages && messages.Count() > 0) { payloadCount = messages.Count(); long lastOffset = messages.Last().Offset; if ((payloadCount + fetchOffset) != (lastOffset + 1)) { s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry); } offsetNew = messages.Last().Offset + 1; } return result; } } catch (Exception) { if (retryCount >= maxRetry) { throw; } } finally { retryCount++; } } // end of while loop return result; }