private static void PublisherWithPartition(string topic, int value) { using (var producer = new Producer <Null, string>(GetConfig())) { try { var key = value % 5; var partition = new Partition(key); var topicPartition = new TopicPartition(topic, partition); var message = new Message <Null, string> { Value = $"value-{value}" }; var dr = producer.ProduceAsync(topicPartition, message).GetAwaiter().GetResult(); Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'"); } catch (KafkaException e) { Console.WriteLine($"Delivery failed: {e.Error.Reason}"); } } }
public StreamTask(string threadId, TaskId id, TopicPartition partition, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer) : base(id, partition, processorTopology, consumer, configuration) { this.threadId = threadId; this.kafkaSupplier = kafkaSupplier; this.consumedOffsets = new Dictionary <TopicPartition, long>(); // eos enabled if (producer == null) { this.producer = CreateEOSProducer(); InitializeTransaction(); eosEnabled = true; } else { this.producer = producer; } this.collector = new RecordCollector(logPrefix); collector.Init(ref this.producer); var sourceTimestampExtractor = (processorTopology.GetSourceProcessor(id.Topic) as ISourceProcessor).Extractor; Context = new ProcessorContext(configuration, stateMgr).UseRecordCollector(collector); processor = processorTopology.GetSourceProcessor(partition.Topic); queue = new RecordQueue <ConsumeResult <byte[], byte[]> >( 100, logPrefix, $"record-queue-{id.Topic}-{id.Partition}", sourceTimestampExtractor == null ? configuration.DefaultTimestampExtractor : sourceTimestampExtractor); }
internal DeliveryReport <byte[], byte[]> Produce(TopicPartition topicPartition, Message <byte[], byte[]> message) { DeliveryReport <byte[], byte[]> r = new DeliveryReport <byte[], byte[]>(); r.Status = PersistenceStatus.NotPersisted; CreateTopic(topicPartition.Topic); if (topics[topicPartition.Topic].PartitionNumber > topicPartition.Partition) { topics[topicPartition.Topic].AddMessage(message.Key, message.Value, topicPartition.Partition); r.Status = PersistenceStatus.Persisted; } else { topics[topicPartition.Topic].CreateNewPartitions(topicPartition.Partition); topics[topicPartition.Topic].AddMessage(message.Key, message.Value, topicPartition.Partition); r.Status = PersistenceStatus.Persisted; } r.Message = message; r.Partition = topicPartition.Partition; r.Topic = topicPartition.Topic; r.Timestamp = new Timestamp(DateTime.Now); r.Error = new Error(ErrorCode.NoError); r.Status = PersistenceStatus.Persisted; // TODO r.Offset return(r); }
public KafkaSink( string bootstrapServers, int batchSizeLimit, int period, SecurityProtocol securityProtocol, SaslMechanism saslMechanism, string topic, string saslUsername, string saslPassword, string sslCaLocation) : base(batchSizeLimit, TimeSpan.FromSeconds(period)) { var config = new ProducerConfig() .SetValue("ApiVersionFallbackMs", 0) .SetValue("EnableDeliveryReports", false) .LoadFromEnvironmentVariables() .SetValue("BootstrapServers", bootstrapServers) .SetValue("SecurityProtocol", securityProtocol) .SetValue("SaslMechanism", saslMechanism) .SetValue("SslCaLocation", string.IsNullOrEmpty(sslCaLocation) ? null : Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), sslCaLocation)) .SetValue("SaslUsername", saslUsername) .SetValue("SaslPassword", saslPassword); producer = new ProducerBuilder <Null, byte[]>(config) .Build(); formatter = new Formatting.Json.JsonFormatter(renderMessage: true); this.topic = new TopicPartition(topic, Partition.Any); }
public void Constuctor() { var tp = new TopicPartition("mytopic", 42); Assert.Equal(tp.Topic, "mytopic"); Assert.Equal(tp.Partition, 42); }
public async Task Custom_partition_event_handling_Should_work() { int elementsCount = 100; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var topicPartition1 = new TopicPartition(topic1, 0); await GivenInitializedTopic(topicPartition1); await ProduceStrings(new TopicPartition(topic1, 0), Enumerable.Range(1, elementsCount), ProducerSettings); var consumerSettings = CreateConsumerSettings <string>(group1); var customHandler = new CustomEventsHandler(); var(control, probe) = CreateProbe(consumerSettings, Subscriptions.Topics(topic1).WithPartitionEventsHandler(customHandler)); probe.Request(elementsCount); foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString())) { probe.ExpectNext(i, TimeSpan.FromSeconds(10)); } var shutdown = control.Shutdown(); await AwaitConditionAsync(() => shutdown.IsCompleted); customHandler.AssignmentEventsCounter.Current.Should().BeGreaterThan(0); customHandler.StopEventsCounter.Current.Should().BeGreaterThan(0); }
/// <summary> /// 异步发送消息 /// </summary> /// <param name="kafkaMessage"></param> /// <returns></returns> public async Task <DeliveryResult> PublishAsync(KafkaMessage kafkaMessage) { var topic = string.IsNullOrEmpty(kafkaMessage.Topic) ? DefaultTopic : kafkaMessage.Topic; if (string.IsNullOrEmpty(topic)) { throw new ArgumentException("topic can not be empty", nameof(kafkaMessage.Topic)); } var key = string.IsNullOrEmpty(kafkaMessage.Key) ? DefaultKey : kafkaMessage.Key; if (string.IsNullOrEmpty(key)) { throw new ArgumentException("key can not be empty", nameof(kafkaMessage.Key)); } var producer = RentProducer(); DeliveryResult <string, object> deliveryResult; if (kafkaMessage.Partition == null) { deliveryResult = await producer.ProduceAsync(topic, new Message <string, object>() { Key = key, Value = kafkaMessage.Message }); } else { var topicPartition = new TopicPartition(topic, new Partition(kafkaMessage.Partition.Value)); deliveryResult = await producer.ProduceAsync(topicPartition, new Message <string, object>() { Key = key, Value = kafkaMessage.Message }); } producer.Flush(new TimeSpan(0, 0, 0, 0, FlushTimeOut)); ReturnProducer(producer); return(new DeliveryResult(deliveryResult)); }
public void WithNullMaterialize() { // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore var config = new StreamConfig <StringSerDes, StringSerDes>(); var serdes = new StringSerDes(); config.ApplicationId = "test-count"; var builder = new StreamBuilder(); Materialized <string, long, IKeyValueStore <Bytes, byte[]> > m = null; builder .Table <string, string>("topic") .GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v)) .Count(m); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null); task.GroupMetadata = consumer as SyncConsumer; task.InitializeStateStores(); task.InitializeTopology(); Assert.AreEqual(2, task.Context.States.StateStoreNames.Count()); var nameStore1 = task.Context.States.StateStoreNames.ElementAt(0); var nameStore2 = task.Context.States.StateStoreNames.ElementAt(1); Assert.IsNotNull(nameStore1); Assert.IsNotNull(nameStore2); Assert.AreNotEqual(string.Empty, nameStore1); Assert.AreNotEqual(string.Empty, nameStore2); var store1 = task.GetStore(nameStore1); var store2 = task.GetStore(nameStore2); Assert.IsInstanceOf <TimestampedKeyValueStore <string, string> >(store1); Assert.IsInstanceOf <TimestampedKeyValueStore <string, long> >(store2); Assert.AreEqual(0, (store1 as TimestampedKeyValueStore <string, string>).ApproximateNumEntries()); Assert.AreEqual(0, (store2 as TimestampedKeyValueStore <string, long>).ApproximateNumEntries()); }
public void AddRecords(TopicPartition partition, IEnumerable <ConsumeResult <byte[], byte[]> > records) { foreach (var r in records) { queue.AddRecord(r); } // TODO : NO PAUSE FOR MOMENT //if (queue.MaxSize <= queue.Size) // consumer.Pause(new List<TopicPartition> { partition }); int newQueueSize = queue.Size; if (log.IsDebugEnabled) { log.Debug($"{logPrefix}Added records into the buffered queue of partition {partition}, new queue size is {newQueueSize}"); } //// if after adding these records, its partition queue's buffered size has been //// increased beyond the threshold, we can then pause the consumption for this partition //if (newQueueSize > maxBufferedSize) //{ // consumer.pause(singleton(partition)); //} }
public void Begin() { config = new StreamConfig(); config.ApplicationId = $"unit-test-rocksdb-w"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, null, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbWindowStore( new RocksDbSegmentedBytesStore("test-w-store", (long)defaultRetention.TotalMilliseconds, 5000, new RocksDbWindowKeySchema()), (long)defaultSize.TotalMilliseconds); store.Init(context, store); }
public void Begin() { config = new StreamConfig(); config.ApplicationId = $"unit-test-rocksdb-kv"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, null, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbKeyValueStore("test-store"); store.Init(context, store); }
public KafkaConsumerProcessorTest() { _topicPartition = new TopicPartition("topic-a", 0); var consumerSettings = new ConsumerSettings { MessageType = typeof(SomeMessage), Topic = _topicPartition.Topic, ConsumerType = typeof(SomeMessageConsumer), ConsumerMode = ConsumerMode.Consumer }; consumerSettings.SetGroup("group-a"); var massageBusMock = new MessageBusMock(); massageBusMock.BusSettings.Consumers.Add(consumerSettings); massageBusMock.DependencyResolverMock.Setup(x => x.Resolve(typeof(SomeMessageConsumer))).Returns(_consumer); byte[] MessageValueProvider(Message m) => m.Value; var consumerInstancePoolMock = new Mock <ConsumerInstancePoolMessageProcessor <Message> >(consumerSettings, massageBusMock.Bus, (Func <Message, byte[]>)MessageValueProvider, null); _messageQueueWorkerMock = new Mock <MessageQueueWorker <Message> >(consumerInstancePoolMock.Object, _checkpointTrigger.Object); _subject = new KafkaConsumerProcessor(consumerSettings, _topicPartition, _commitControllerMock.Object, massageBusMock.Bus, _messageQueueWorkerMock.Object); }
private void AddToOffsetDict(TopicPartition topicPartition, TopicPartitionOffset TopicPartitionOffset) { _currentOffsets.AddOrUpdate(topicPartition, TopicPartitionOffset, (key, oldValue) => { return(TopicPartitionOffset.Offset > oldValue.Offset ? TopicPartitionOffset : oldValue); }); }
public async Task <ProduceResult> Publish(MessageValue mv) { try { using (var p = new ProducerBuilder <Null, string>(_config).Build()) { Console.WriteLine($"Produce message {mv.Value}"); var random = new Random(); TopicPartition tp = new TopicPartition(this._topicName, new Partition(random.Next(0, 5))); var dr = await p.ProduceAsync(tp, new Message <Null, string> { Value = mv.Value }); return(new ProduceResult() { TopicPartitionOffset = dr.TopicPartitionOffset.Offset.Value, TopicPartition = dr.TopicPartition.Partition.Value }); } } catch (System.Exception e) { Console.Write($"Error: {e.Message}"); } return(new ProduceResult()); }
public async Task <string> SendMessage(string topic, string message, bool display, int key) { var msg = new Message <string, string> { Key = key.ToString(), Value = message }; DeliveryResult <string, string> delRep; if (key > 1) { var p = new Partition(key); var tp = new TopicPartition(topic, p); delRep = await producer.ProduceAsync(tp, msg); } else { delRep = await producer.ProduceAsync(topic, msg); } var topicOffset = delRep.TopicPartitionOffset; if (display) { Console.WriteLine($"Delivered '{delRep.Value}' to: {topicOffset}"); } return(message); }
public void NoteProcessed(TopicPartition topicPartition, long offset) { var logContext = $"OffsetTracker.NoteProcessed for topicPartition='{topicPartition}', offset={offset}"; _logger.LogDebug($"{logContext}"); Fetch(topicPartition).NoteProcessed(offset); }
/// <summary> /// Смещает offset в конец раздела, если разница между текущим и максимальным offset-ом больше 10. /// Это происходит если продьсер пишет в топик, а консьюмер еще не читает. /// </summary> private void SetHightOffset() { foreach (var topic in _topics) { try { var blockTime = TimeSpan.FromSeconds(1); var topicPartition = new TopicPartition(topic, 0); var currentOffset = _consumer.Position(new List <TopicPartition> { topicPartition }).FirstOrDefault()?.Offset.Value; var hightOffset = _consumer.QueryWatermarkOffsets(topicPartition, blockTime).High; if ((currentOffset == -1001) || (hightOffset - currentOffset) > 10) { var topicPartitionOffsets = new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, hightOffset) }; _consumer.CommitAsync(topicPartitionOffsets).Wait(blockTime); } } catch (KafkaException ex) { //LOG Console.WriteLine(ex); } } }
public KafkaMessage(string topic, int partition, string key, string message) { TopicPartition = new TopicPartition(topic, partition); Message = new Message <string, string> { Key = key, Value = message }; }
public async Task PlainSource_should_resume_stage_if_broker_unavailable() { var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var topicPartition1 = new TopicPartition(topic1, 0); await GivenInitializedTopic(topicPartition1); var config = ConsumerSettings <Null, string> .Create(Sys, null, null) .WithBootstrapServers("localhost:10092") .WithGroupId(group1); var regex = new Regex("\\[localhost:10092\\/bootstrap: Connect to [a-zA-Z0-9#:.*]* failed:"); var logProbe = CreateTestProbe(); Sys.EventStream.Subscribe <Info>(logProbe.Ref); var(control, probe) = CreateProbe(config, Subscriptions.Assignment(topicPartition1)); probe.Request(1); AwaitAssert(() => { var info = logProbe.ExpectMsg <Info>(); regex.IsMatch(info.Message.ToString() ?? "").Should().BeTrue(); info.Message.ToString().Should().Contain("[Resume]"); }); //AwaitCondition(() => control.IsShutdown.IsCompleted, TimeSpan.FromSeconds(10)); }
public void ToStringTest() { var tp = new TopicPartition("mytopic", 42); Assert.True(tp.ToString().Contains(tp.Topic)); Assert.True(tp.ToString().Contains(tp.Partition.ToString())); }
public static WatermarkOffsets BuscarOffsetsDisponiveis(this ConsumidorAbstrato ob, string topico, string consumerGroup) { ob.ConsumerConfig.GroupId = consumerGroup; using (var consumer = new ConsumerBuilder <string, string>(ob.ConsumerConfig).Build()) { consumer.Subscribe(topico); var assignment = consumer.Assignment; while (assignment.Count == 0) { Thread.Sleep(50); //ha um bug no driver para .net, as vezes é preciso aguardar a informação chegar assignment = consumer.Assignment; } var topicPartition = new TopicPartition(assignment[0].Topic, assignment[0].Partition); var wmo = consumer.GetWatermarkOffsets(topicPartition); while (wmo.High.Value == -1001) //ha um bug no driver para .net, as vezes é preciso aguardar a informação chegar { wmo = consumer.GetWatermarkOffsets(topicPartition); Thread.Sleep(50); } return(wmo); } }
public async Task PlainSource_consumes_messages_from_KafkaProducer_with_subscribe_to_topic() { int elementsCount = 100; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var topicPartition1 = new TopicPartition(topic1, 0); await GivenInitializedTopic(topicPartition1); await ProduceStrings(new TopicPartition(topic1, 0), Enumerable.Range(1, elementsCount), ProducerSettings); var consumerSettings = CreateConsumerSettings <string>(group1); var(control, probe) = CreateProbe(consumerSettings, Subscriptions.Topics(topic1)); probe.Request(elementsCount); foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString())) { probe.ExpectNext(i, TimeSpan.FromSeconds(10)); } var shutdown = control.Shutdown(); AwaitCondition(() => shutdown.IsCompleted); }
/** * This is the primary function of <TwitchCommandConsumer> and is to be run as a thread */ public void TwiConThread() { Console.WriteLine("DiscordMessageConsumerThread Start"); Active = true; //objs: Kafka Consumer client // //consumer - the Kafka consumer client //topicp - the topic partition from <consumer> to consume from var consumer = new ConsumerBuilder <string, string>(config).Build(); var topicp = new TopicPartition(topic, 0); consumer.Assign(topicp); while (Active) { try { //var: consumerresult //The result of checking for the next new message on the Kafka server var consumerresult = consumer.Consume(canceltoken); if (!consumerresult.IsPartitionEOF) { messageQueue.Enqueue(consumerresult.Key + " " + consumerresult.Value); } else { Thread.Sleep(100); } } catch (System.OperationCanceledException e) { } } }
internal WatermarkOffsets GetWatermarkOffsets(TopicPartition topicPartition) { var topic = topics[topicPartition.Topic]; var p = topic.GetPartition(topicPartition.Partition); return(new WatermarkOffsets(new Offset(p.LowOffset), new Offset(p.HighOffset))); }
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token) { this.configuration = configuration; this.configuration.ClientId = clientId; this.topicConfiguration = topicConfiguration; this.token = token; builder = topologyBuilder; this.supplier = supplier ?? new SyncKafkaSupplier(); producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer; foreach (var sourceTopic in builder.GetSourceTopics().Union(builder.GetGlobalTopics())) { var part = new TopicPartition(sourceTopic, 0); var taskId = builder.GetTaskIdFromPartition(part); if (partitionsByTaskId.ContainsKey(taskId)) { partitionsByTaskId[taskId].Add(part); } else { partitionsByTaskId.Add(taskId, new List <TopicPartition> { part }); } } }
/// <summary> /// Handles the subscription of a new TopicPartition to this fetcher. /// Keeps track of the subscribed partitions in order to not fetch messages if the FlowControlState is Closed. /// </summary> /// <param name="topicPartition"></param> /// <returns></returns> public IDisposable Subscribe(TopicPartition topicPartition) { _topicPartitions.Add(topicPartition); EtwTrace.Log.FetcherPartitionSubscribed(_id, topicPartition.PartitionId); // cleanup var topicPartitionCleanup = Disposable.Create(() => _topicPartitions.Remove(topicPartition)); var receivedMessagesSubscriptionCleanup = ReceivedMessages.Where(rm => rm.Topic == topicPartition.Topic && rm.Partition == topicPartition.PartitionId) .Subscribe(topicPartition); var flowControlCleanup = topicPartition.FlowControl. // we need to wake up from waiting loop any time flow control hits low watermark and becomes enabled Where(enabled => enabled). Subscribe(_ => _wakeupSignal.OnNext(true)); var cleanup = new CompositeDisposable { topicPartitionCleanup, receivedMessagesSubscriptionCleanup, flowControlCleanup }; if (_log.IsDebugEnabled) { cleanup.Add(Disposable.Create(() => _log.Debug("Fetcher #{0} {1} topicPartition is unsubscribing", _id, topicPartition))); } _log.Debug("Fetcher #{0} added {1}", _id, topicPartition); return cleanup; }
public KafkaTopicScalerTest() { consumer = new Mock <IConsumer <string, byte[]> >(); partition0 = new TopicPartition(TopicName, new Partition(0)); partition1 = new TopicPartition(TopicName, new Partition(1)); partition2 = new TopicPartition(TopicName, new Partition(2)); partition3 = new TopicPartition(TopicName, new Partition(3)); partitions = new List <TopicPartition> { partition0, partition1, partition2, partition3 }; topicScaler = new KafkaTopicScalerForTest <string, byte[]>( TopicName, "consumer-group-test", "testfunction", consumer.Object, new AdminClientConfig(), NullLogger.Instance); topicScaler.WithPartitions(partitions); }
static void Pause() { using (var consumer = new Consumer <Ignore, string>(GetConfig())) { try { var assignment = consumer.Assignment; var partition = new Partition(1); var topicPartition = new TopicPartition(TOPIC, partition); consumer.Pause(new List <TopicPartition> { topicPartition }); //consumer.Position(new List<TopicPartition> { topicPartition }); //var pos = consumer.Position(new List<TopicPartition> { topicPartition }).First(); //consumer.Seek(new TopicPartitionOffset(topicPartition, new Offset(50))); } catch (Exception ex) { throw; } consumer.Close(); } }
public async Task CommitableSource_consumes_messages_from_Producer_without_commits() { int elementsCount = 100; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var topicPartition1 = new TopicPartition(topic1, 0); await GivenInitializedTopic(topicPartition1); await Source .From(Enumerable.Range(1, elementsCount)) .Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var consumerSettings = CreateConsumerSettings <string>(group1); var probe = KafkaConsumer .CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1)) .Where(c => !c.Record.Value.Equals(InitialMsg)) .Select(c => c.Record.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(elementsCount); foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString())) { probe.ExpectNext(i, TimeSpan.FromSeconds(10)); } probe.Cancel(); }
public async Task Consumir(string topico, Offset offset) { ConsumerConfig.GroupId = ConsumerGroup; ConsumerConfig.EnableAutoCommit = false; using (var consumer = new ConsumerBuilder <string, string>(ConsumerConfig).Build()) { consumer.Subscribe(topico); var assignment = consumer.Assignment; while (assignment.Count == 0) { Thread.Sleep(50); assignment = consumer.Assignment; } var topicPartition = new TopicPartition(assignment[0].Topic, assignment[0].Partition); consumer.Assign(topicPartition); await this.SeekAsync(consumer, assignment[0].Topic, assignment[0].Partition, offset); var cr = consumer.Consume(); Console.WriteLine($"Consumido registro via offset {cr.Offset} da partição {cr.Partition} com chave '{cr.Message.Key}' e valor '{cr.Message.Value}'"); } }
private ConsumeResult <byte[], byte[]> ConsumeInternal(TimeSpan timeout) { DateTime dt = DateTime.Now; ConsumeResult <byte[], byte[]> result = null; foreach (var kp in offsets) { if (timeout != TimeSpan.Zero && (dt + timeout) < DateTime.Now) { break; } var tp = new TopicPartition(kp.Key, 0); if (producer != null && ((partitionsState.ContainsKey(tp) && !partitionsState[tp]) || !partitionsState.ContainsKey(tp))) { var messages = producer.GetHistory(kp.Key).ToArray(); if (messages.Length > kp.Value.OffsetConsumed) { result = new ConsumeResult <byte[], byte[]> { Offset = kp.Value.OffsetConsumed, Topic = kp.Key, Partition = 0, Message = messages[kp.Value.OffsetConsumed] }; ++kp.Value.OffsetConsumed; return(result); } } } return(result); }
internal void OnTopicPartitionComplete(TopicPartition topicPartition) { // called back from TopicPartition when its subscription is completed. // Remove from our dictionary, and if all TopicPartitions are removed, call our own OnComplete. _topicPartitions.Remove(topicPartition.PartitionId); if (_topicPartitions.Count == 0) { // If we call OnCompleted right now, any last message that may be being sent will not process. Just tell the scheduler to do it in a moment. _cluster.Scheduler.Schedule(() => OnMessageArrivedInput.OnCompleted()); } }
/// <summary>For every patition, resolve offsets and build TopicPartition object</summary> private async Task<IEnumerable<TopicPartition>> BuildTopicPartitionsAsync() { // if they didn't specify explicit locations, initialize them here. var startPositionProvider = Configuration.StartPosition; if (startPositionProvider.StartLocation != ConsumerLocation.SpecifiedLocations) { // no offsets provided. Need to issue an offset request to get start/end locations and use them for consuming var partitions = await _cluster.FetchPartitionOffsetsAsync(Topic, startPositionProvider.StartLocation); if (_log.IsDebugEnabled) _log.Debug("Consumer for topic {0} got time->offset resolved for location {1}. parts: [{2}]", Topic, startPositionProvider, string.Join(",", partitions.Partitions.OrderBy(p => p).Select(p => string.Format("{0}:{1}", p, partitions.NextOffset(p))))); IStartPositionProvider origProvider = startPositionProvider; // the new explicit offsets provider should use only the partitions included in the original one. startPositionProvider = new TopicPartitionOffsets(partitions.Topic, partitions.GetPartitionsOffset.Where(kv=> origProvider.ShouldConsumePartition(kv.Key))); } // we now have specified locations to start from, just get the partition metadata, and build the TopicPartitions var partitionMeta = await _cluster.GetOrFetchMetaForTopicAsync(Topic); return partitionMeta // only new partitions we don't already have in our dictionary .Where(pm => !_topicPartitions.ContainsKey(pm.Id)) // only partitions we are "told" to. .Where(pm => startPositionProvider.ShouldConsumePartition(pm.Id)) .Select(part => { var tp = new TopicPartition(_cluster, Topic, part.Id, startPositionProvider.GetStartOffset(part.Id)); _topicPartitions.Add(tp.PartitionId, tp); return tp; }); }