public void TestInvalidPartition() { var producerDataList = new List <KeyedMessage <string, Message> >(); producerDataList.Add(new KeyedMessage <string, Message>("topic1", "key1", new Message(Encoding.UTF8.GetBytes("msg1")))); var config = new ProducerConfig(); config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props); // form expected partitions metadata var topic1Metadata = this.GetTopicMetadata("topic1", 0, 0, "localhost", 9092); var topicPartitionInfos = new Dictionary <string, TopicMetadata> { { "topic1", topic1Metadata } }; var producerPool = new ProducerPool(config); var handler = new DefaultEventHandler <string, string>( config, new NegativePartitioner(), null, null, producerPool, topicPartitionInfos); try { handler.PartitionAndCollate(producerDataList); } catch { Assert.False(true, "Should not throw any exception"); } }
public void TestNoBroker() { var config = new ProducerConfig(); config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props); // create topic metadata with 0 partitions var topic1Metadata = new TopicMetadata("topic1", new List <PartitionMetadata>()); var topicPartitionInfos = new Dictionary <string, TopicMetadata> { { "topic1", topic1Metadata } }; var producerPool = new ProducerPool(config); var producerDataList = new List <KeyedMessage <string, string> >(); producerDataList.Add(new KeyedMessage <string, string>("topic1", "msg1")); var handler = new DefaultEventHandler <string, string>( config, null, new StringEncoder(), new StringEncoder(), producerPool, topicPartitionInfos); try { handler.Handle(producerDataList); Assert.True(false, "Should fail with FailedToSendMessageException"); } catch (FailedToSendMessageException) { // expted } }
public void TestSerializeEvents() { var produceData = TestUtils.GetMsgStrings(5).Select(m => new KeyedMessage <string, string>("topic1", m)).ToList(); var config = new ProducerConfig(); config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props); // form expected partitions metadata var topic1Metadata = this.GetTopicMetadata("topic1", 0, 0, "localhost", 9092); var topicPartitionInfos = new Dictionary <string, TopicMetadata> { { "topic1", topic1Metadata } }; var producerPool = new ProducerPool(config); var handler = new DefaultEventHandler <string, string>( config, null, new StringEncoder(), new StringEncoder(), producerPool, topicPartitionInfos); var serializedData = handler.Serialize(produceData); var deserializedData = serializedData.Select( d => new KeyedMessage <string, string>(d.Topic, Util.ReadString(d.Message.Payload))).ToList(); TestUtils.CheckEquals(produceData.GetEnumerator(), deserializedData.GetEnumerator()); }
public override Task PublishManyFromOutboxAsync(IEnumerable <OutgoingEventInfo> outgoingEvents, OutboxConfig outboxConfig) { var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName); var outgoingEventArray = outgoingEvents.ToArray(); foreach (var outgoingEvent in outgoingEventArray) { var messageId = outgoingEvent.Id.ToString("N"); var headers = new Headers { { "messageId", System.Text.Encoding.UTF8.GetBytes(messageId) } }; producer.Produce( AbpKafkaEventBusOptions.TopicName, new Message <string, byte[]> { Key = outgoingEvent.EventName, Value = outgoingEvent.EventData, Headers = headers }); } return(Task.CompletedTask); }
private Task <DeliveryResult <string, byte[]> > PublishAsync( string topicName, string eventName, byte[] body, Headers headers, Dictionary <string, object> headersArguments) { var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName); return(PublishAsync(producer, topicName, eventName, body, headers, headersArguments)); }
public override async Task PublishAsync(Type eventType, object eventData) { var eventName = EventNameAttribute.GetNameOrDefault(eventType); var body = Serializer.Serialize(eventData); var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName); await producer.ProduceAsync( AbpKafkaEventBusOptions.TopicName, new Message <string, byte[]> { Key = eventName, Value = body }); }
private async Task PublishAsync(string topicName, string eventName, byte[] body, Headers headers, Dictionary <string, object> headersArguments) { var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName); SetEventMessageHeaders(headers, headersArguments); await producer.ProduceAsync( topicName, new Message <string, byte[]> { Key = eventName, Value = body, Headers = headers }); }
private async Task PublishAsync(string topicName, Type eventType, object eventData, Headers headers, Dictionary <string, object> headersArguments) { var eventName = EventNameAttribute.GetNameOrDefault(eventType); var body = Serializer.Serialize(eventData); var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName); SetEventMessageHeaders(headers, headersArguments); await producer.ProduceAsync( topicName, new Message <string, byte[]> { Key = eventName, Value = body, Headers = headers }); }
private Task <DeliveryResult <string, byte[]> > PublishAsync( string topicName, string eventName, byte[] body, Headers headers) { var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName); return(producer.ProduceAsync( topicName, new Message <string, byte[]> { Key = eventName, Value = body, Headers = headers })); }
/// <summary> /// Used by the producer to send a metadata request since it has access to the ProducerConfig /// </summary> /// <param name="topics">The topics for which the metadata needs to be fetched</param> /// <param name="brokers">The brokers in the cluster as configured on the client</param> /// <param name="producerConfig">The producer's config</param> /// <param name="correlationId">topic metadata response</param> /// <returns></returns> public static TopicMetadataResponse FetchTopicMetadata( ISet <string> topics, IList <Broker> brokers, ProducerConfig producerConfig, int correlationId) { var fetchMetaDataSucceeded = false; var i = 0; var topicMetadataRequest = new TopicMetadataRequest( TopicMetadataRequest.CurrentVersion, correlationId, producerConfig.ClientId, topics.ToList()); TopicMetadataResponse topicMetadataResponse = null; Exception t = null; // shuffle the list of brokers before sending metadata requests so that most requests don't get routed to the same broker var shuffledBrokers = brokers.Shuffle(); while (i < shuffledBrokers.Count() && !fetchMetaDataSucceeded) { var producer = ProducerPool.CreateSyncProducer(producerConfig, shuffledBrokers[i]); Logger.InfoFormat("Fetching metadata from broker {0} with correlation id {1} for {2} topic(s) {3}", shuffledBrokers[i], correlationId, topics.Count, string.Join(",", topics)); try { topicMetadataResponse = producer.Send(topicMetadataRequest); fetchMetaDataSucceeded = true; } catch (Exception e) { Logger.Warn(string.Format("Fetching topic metadata with correlation id {0} for topic [{1}] from broker [{2}] failed", correlationId, topics, shuffledBrokers[i]), e); t = e; } finally { i++; producer.Dispose(); } } if (!fetchMetaDataSucceeded) { throw new KafkaException( string.Format( "fetching topic metadata for topics [{0}] from broker [{1}] failed", string.Join(",", topics), string.Join(", ", shuffledBrokers)), t); } Logger.DebugFormat("Successfully fetched metadata for {0} topic(s) {1}", topics.Count(), string.Join(",", topics)); return(topicMetadataResponse); }
public void TestRandomPartitioner() { var config = new ProducerConfig(); config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props); // create topic metadata with 0 partitions var topic1Metadata = this.GetTopicMetadata("topic1", 0, 0, "localhost", 9092); var topic2Metadata = this.GetTopicMetadata("topic2", 0, 0, "localhost", 9092); var topicPartitionInfos = new Dictionary <string, TopicMetadata> { { "topic1", topic1Metadata }, { "topic2", topic2Metadata } }; var producerPool = new ProducerPool(config); var handler = new DefaultEventHandler <string, string>( config, null, null, null, producerPool, topicPartitionInfos); var producerDataList = new List <KeyedMessage <string, Message> >(); producerDataList.Add(new KeyedMessage <string, Message>("topic1", new Message(Encoding.UTF8.GetBytes("msg1")))); producerDataList.Add(new KeyedMessage <string, Message>("topic2", new Message(Encoding.UTF8.GetBytes("msg2")))); producerDataList.Add(new KeyedMessage <string, Message>("topic1", new Message(Encoding.UTF8.GetBytes("msg3")))); var partitionedDataOpt = handler.PartitionAndCollate(producerDataList); if (partitionedDataOpt != null) { foreach (var brokerAndData in partitionedDataOpt) { var dataPerBroker = brokerAndData.Value; foreach (var topicPartitionData in dataPerBroker) { var partitionId = topicPartitionData.Key.Partiton; Assert.True(partitionId == 0); } } } else { Assert.False(true, "Failed to collate requests by topic, partition"); } }
public DefaultEventHandler(ProducerConfig config, IPartitioner partitioner, IEncoder <TValue> encoder, IEncoder <TKey> keyEncoder, ProducerPool producerPool, Dictionary <string, TopicMetadata> topicPartitionInfos = null) { this.Config = config; this.partitioner = partitioner; this.encoder = encoder; this.keyEncoder = keyEncoder; this.producerPool = producerPool; if (topicPartitionInfos != null) { this.topicPartitionInfos = topicPartitionInfos; } this.brokerPartitionInfo = new BrokerPartitionInfo(this.Config, this.producerPool, this.topicPartitionInfos); this.topicMetadataRefreshInterval = TimeSpan.FromMilliseconds(config.TopicMetadataRefreshIntervalMs); this.producerStats = ProducerStatsRegistry.GetProducerStats(config.ClientId); this.producerTopicStats = ProducerTopicStatsRegistry.GetProducerTopicStats(config.ClientId); }
public void TestPartitionAndCollateEvents() { var producerDataList = new List <KeyedMessage <int, Message> >(); // use bogus key and partition key override for some messages producerDataList.Add(new KeyedMessage <int, Message>("topic1", 0, new Message(Encoding.UTF8.GetBytes("msg1")))); producerDataList.Add(new KeyedMessage <int, Message>("topic2", -99, 1, new Message(Encoding.UTF8.GetBytes("msg2")))); producerDataList.Add(new KeyedMessage <int, Message>("topic1", 2, new Message(Encoding.UTF8.GetBytes("msg3")))); producerDataList.Add(new KeyedMessage <int, Message>("topic1", -101, 3, new Message(Encoding.UTF8.GetBytes("msg4")))); producerDataList.Add(new KeyedMessage <int, Message>("topic2", 4, new Message(Encoding.UTF8.GetBytes("msg5")))); var broker1 = new Broker(0, "localhost", 9092); var broker2 = new Broker(1, "localhost", 9093); // form expected partitions metadata var partition1Metadata = new PartitionMetadata(0, broker1, new List <Broker> { broker1, broker2 }); var partition2Metadata = new PartitionMetadata(1, broker2, new List <Broker> { broker1, broker2 }); var topic1Metadata = new TopicMetadata( "topic1", new List <PartitionMetadata> { partition1Metadata, partition2Metadata }); var topic2Metadata = new TopicMetadata( "topic2", new List <PartitionMetadata> { partition1Metadata, partition2Metadata }); var topicPartitionInfos = new Dictionary <string, TopicMetadata> { { "topic1", topic1Metadata }, { "topic2", topic2Metadata } }; var intPartitioner = new IntPartitioner(); var config = new ProducerConfig(); config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props); var producerPool = new ProducerPool(config); var handler = new DefaultEventHandler <int, string>( config, intPartitioner, null, new IntEncoder(), producerPool, topicPartitionInfos); var topic1Broker1Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic1", 0, new Message( Encoding.UTF8.GetBytes("msg1"))), new KeyedMessage <int, Message>( "topic1", 2, new Message( Encoding.UTF8.GetBytes("msg3"))) }; var topic1Broker2Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic1", -101, 3, new Message( Encoding.UTF8.GetBytes("msg4"))) }; var topic2Broker1Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic2", 4, new Message( Encoding.UTF8.GetBytes("msg5"))) }; var topic2Broker2Data = new List <KeyedMessage <int, Message> > { new KeyedMessage <int, Message>( "topic2", -99, 1, new Message( Encoding.UTF8.GetBytes("msg2"))) }; var expectedResult = new Dictionary <int, Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > > { { 0, new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > { { new TopicAndPartition("topic1", 0), topic1Broker1Data }, { new TopicAndPartition("topic2", 0), topic2Broker1Data } } }, { 1, new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > { { new TopicAndPartition("topic1", 1), topic1Broker2Data }, { new TopicAndPartition("topic2", 1), topic2Broker2Data } } }, }; var actualResut = handler.PartitionAndCollate(producerDataList); Assert.Equal(expectedResult.Count, actualResut.Count); Assert.True(expectedResult.Keys.SequenceEqual(actualResut.Keys)); foreach (var key in expectedResult.Keys) { var exptectedInnerDict = expectedResult[key]; var actualInnerDict = actualResut[key]; Assert.Equal(exptectedInnerDict.Count, actualInnerDict.Count); foreach (var topicAndPartition in exptectedInnerDict.Keys) { var exptectedKeyedMsgs = exptectedInnerDict[topicAndPartition]; var actualKeyedMsgs = actualInnerDict[topicAndPartition]; Assert.True(exptectedKeyedMsgs.SequenceEqual(actualKeyedMsgs)); } } }