public void TestInvalidPartition()
{
var producerDataList = new List <KeyedMessage <string, Message> >();
producerDataList.Add(new KeyedMessage <string, Message>("topic1", "key1", new Message(Encoding.UTF8.GetBytes("msg1"))));
var config = new ProducerConfig();
config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props);
// form expected partitions metadata
var topic1Metadata = this.GetTopicMetadata("topic1", 0, 0, "localhost", 9092);
var topicPartitionInfos = new Dictionary <string, TopicMetadata> {
{ "topic1", topic1Metadata }
};
var producerPool = new ProducerPool(config);
var handler = new DefaultEventHandler <string, string>(
config, new NegativePartitioner(), null, null, producerPool, topicPartitionInfos);
try
{
handler.PartitionAndCollate(producerDataList);
}
catch
{
Assert.False(true, "Should not throw any exception");
}
}
public void TestNoBroker()
{
var config = new ProducerConfig();
config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props);
// create topic metadata with 0 partitions
var topic1Metadata = new TopicMetadata("topic1", new List <PartitionMetadata>());
var topicPartitionInfos = new Dictionary <string, TopicMetadata> {
{ "topic1", topic1Metadata }
};
var producerPool = new ProducerPool(config);
var producerDataList = new List <KeyedMessage <string, string> >();
producerDataList.Add(new KeyedMessage <string, string>("topic1", "msg1"));
var handler = new DefaultEventHandler <string, string>(
config, null, new StringEncoder(), new StringEncoder(), producerPool, topicPartitionInfos);
try
{
handler.Handle(producerDataList);
Assert.True(false, "Should fail with FailedToSendMessageException");
}
catch (FailedToSendMessageException)
{
// expted
}
}
public void TestSerializeEvents()
{
var produceData =
TestUtils.GetMsgStrings(5).Select(m => new KeyedMessage <string, string>("topic1", m)).ToList();
var config = new ProducerConfig();
config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props);
// form expected partitions metadata
var topic1Metadata = this.GetTopicMetadata("topic1", 0, 0, "localhost", 9092);
var topicPartitionInfos = new Dictionary <string, TopicMetadata> {
{ "topic1", topic1Metadata }
};
var producerPool = new ProducerPool(config);
var handler = new DefaultEventHandler <string, string>(
config, null, new StringEncoder(), new StringEncoder(), producerPool, topicPartitionInfos);
var serializedData = handler.Serialize(produceData);
var deserializedData =
serializedData.Select(
d => new KeyedMessage <string, string>(d.Topic, Util.ReadString(d.Message.Payload))).ToList();
TestUtils.CheckEquals(produceData.GetEnumerator(), deserializedData.GetEnumerator());
}
public override Task PublishManyFromOutboxAsync(IEnumerable <OutgoingEventInfo> outgoingEvents, OutboxConfig outboxConfig)
{
var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName);
var outgoingEventArray = outgoingEvents.ToArray();
foreach (var outgoingEvent in outgoingEventArray)
{
var messageId = outgoingEvent.Id.ToString("N");
var headers = new Headers
{
{ "messageId", System.Text.Encoding.UTF8.GetBytes(messageId) }
};
producer.Produce(
AbpKafkaEventBusOptions.TopicName,
new Message <string, byte[]>
{
Key = outgoingEvent.EventName,
Value = outgoingEvent.EventData,
Headers = headers
});
}
return(Task.CompletedTask);
}
private Task <DeliveryResult <string, byte[]> > PublishAsync(
string topicName,
string eventName,
byte[] body,
Headers headers,
Dictionary <string, object> headersArguments)
{
var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName);
return(PublishAsync(producer, topicName, eventName, body, headers, headersArguments));
}
public override async Task PublishAsync(Type eventType, object eventData)
{
var eventName = EventNameAttribute.GetNameOrDefault(eventType);
var body = Serializer.Serialize(eventData);
var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName);
await producer.ProduceAsync(
AbpKafkaEventBusOptions.TopicName,
new Message <string, byte[]>
{
Key = eventName, Value = body
});
}
private async Task PublishAsync(string topicName, string eventName, byte[] body, Headers headers, Dictionary <string, object> headersArguments)
{
var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName);
SetEventMessageHeaders(headers, headersArguments);
await producer.ProduceAsync(
topicName,
new Message <string, byte[]>
{
Key = eventName,
Value = body,
Headers = headers
});
}
private async Task PublishAsync(string topicName, Type eventType, object eventData, Headers headers, Dictionary <string, object> headersArguments)
{
var eventName = EventNameAttribute.GetNameOrDefault(eventType);
var body = Serializer.Serialize(eventData);
var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName);
SetEventMessageHeaders(headers, headersArguments);
await producer.ProduceAsync(
topicName,
new Message <string, byte[]>
{
Key = eventName, Value = body, Headers = headers
});
}
private Task <DeliveryResult <string, byte[]> > PublishAsync(
string topicName,
string eventName,
byte[] body,
Headers headers)
{
var producer = ProducerPool.Get(AbpKafkaEventBusOptions.ConnectionName);
return(producer.ProduceAsync(
topicName,
new Message <string, byte[]>
{
Key = eventName,
Value = body,
Headers = headers
}));
}
/// <summary>
/// Used by the producer to send a metadata request since it has access to the ProducerConfig
/// </summary>
/// <param name="topics">The topics for which the metadata needs to be fetched</param>
/// <param name="brokers">The brokers in the cluster as configured on the client</param>
/// <param name="producerConfig">The producer's config</param>
/// <param name="correlationId">topic metadata response</param>
/// <returns></returns>
public static TopicMetadataResponse FetchTopicMetadata(
ISet <string> topics, IList <Broker> brokers, ProducerConfig producerConfig, int correlationId)
{
var fetchMetaDataSucceeded = false;
var i = 0;
var topicMetadataRequest = new TopicMetadataRequest(
TopicMetadataRequest.CurrentVersion, correlationId, producerConfig.ClientId, topics.ToList());
TopicMetadataResponse topicMetadataResponse = null;
Exception t = null;
// shuffle the list of brokers before sending metadata requests so that most requests don't get routed to the same broker
var shuffledBrokers = brokers.Shuffle();
while (i < shuffledBrokers.Count() && !fetchMetaDataSucceeded)
{
var producer = ProducerPool.CreateSyncProducer(producerConfig, shuffledBrokers[i]);
Logger.InfoFormat("Fetching metadata from broker {0} with correlation id {1} for {2} topic(s) {3}", shuffledBrokers[i], correlationId, topics.Count, string.Join(",", topics));
try
{
topicMetadataResponse = producer.Send(topicMetadataRequest);
fetchMetaDataSucceeded = true;
}
catch (Exception e)
{
Logger.Warn(string.Format("Fetching topic metadata with correlation id {0} for topic [{1}] from broker [{2}] failed", correlationId, topics, shuffledBrokers[i]), e);
t = e;
}
finally
{
i++;
producer.Dispose();
}
}
if (!fetchMetaDataSucceeded)
{
throw new KafkaException(
string.Format(
"fetching topic metadata for topics [{0}] from broker [{1}] failed", string.Join(",", topics), string.Join(", ", shuffledBrokers)),
t);
}
Logger.DebugFormat("Successfully fetched metadata for {0} topic(s) {1}", topics.Count(), string.Join(",", topics));
return(topicMetadataResponse);
}
public void TestRandomPartitioner()
{
var config = new ProducerConfig();
config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props);
// create topic metadata with 0 partitions
var topic1Metadata = this.GetTopicMetadata("topic1", 0, 0, "localhost", 9092);
var topic2Metadata = this.GetTopicMetadata("topic2", 0, 0, "localhost", 9092);
var topicPartitionInfos = new Dictionary <string, TopicMetadata>
{
{ "topic1", topic1Metadata },
{ "topic2", topic2Metadata }
};
var producerPool = new ProducerPool(config);
var handler = new DefaultEventHandler <string, string>(
config, null, null, null, producerPool, topicPartitionInfos);
var producerDataList = new List <KeyedMessage <string, Message> >();
producerDataList.Add(new KeyedMessage <string, Message>("topic1", new Message(Encoding.UTF8.GetBytes("msg1"))));
producerDataList.Add(new KeyedMessage <string, Message>("topic2", new Message(Encoding.UTF8.GetBytes("msg2"))));
producerDataList.Add(new KeyedMessage <string, Message>("topic1", new Message(Encoding.UTF8.GetBytes("msg3"))));
var partitionedDataOpt = handler.PartitionAndCollate(producerDataList);
if (partitionedDataOpt != null)
{
foreach (var brokerAndData in partitionedDataOpt)
{
var dataPerBroker = brokerAndData.Value;
foreach (var topicPartitionData in dataPerBroker)
{
var partitionId = topicPartitionData.Key.Partiton;
Assert.True(partitionId == 0);
}
}
}
else
{
Assert.False(true, "Failed to collate requests by topic, partition");
}
}
public DefaultEventHandler(ProducerConfig config, IPartitioner partitioner, IEncoder <TValue> encoder, IEncoder <TKey> keyEncoder, ProducerPool producerPool, Dictionary <string, TopicMetadata> topicPartitionInfos = null)
{
this.Config = config;
this.partitioner = partitioner;
this.encoder = encoder;
this.keyEncoder = keyEncoder;
this.producerPool = producerPool;
if (topicPartitionInfos != null)
{
this.topicPartitionInfos = topicPartitionInfos;
}
this.brokerPartitionInfo = new BrokerPartitionInfo(this.Config, this.producerPool, this.topicPartitionInfos);
this.topicMetadataRefreshInterval = TimeSpan.FromMilliseconds(config.TopicMetadataRefreshIntervalMs);
this.producerStats = ProducerStatsRegistry.GetProducerStats(config.ClientId);
this.producerTopicStats = ProducerTopicStatsRegistry.GetProducerTopicStats(config.ClientId);
}
public void TestPartitionAndCollateEvents()
{
var producerDataList = new List <KeyedMessage <int, Message> >();
// use bogus key and partition key override for some messages
producerDataList.Add(new KeyedMessage <int, Message>("topic1", 0, new Message(Encoding.UTF8.GetBytes("msg1"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic2", -99, 1, new Message(Encoding.UTF8.GetBytes("msg2"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic1", 2, new Message(Encoding.UTF8.GetBytes("msg3"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic1", -101, 3, new Message(Encoding.UTF8.GetBytes("msg4"))));
producerDataList.Add(new KeyedMessage <int, Message>("topic2", 4, new Message(Encoding.UTF8.GetBytes("msg5"))));
var broker1 = new Broker(0, "localhost", 9092);
var broker2 = new Broker(1, "localhost", 9093);
// form expected partitions metadata
var partition1Metadata = new PartitionMetadata(0, broker1, new List <Broker> {
broker1, broker2
});
var partition2Metadata = new PartitionMetadata(1, broker2, new List <Broker> {
broker1, broker2
});
var topic1Metadata = new TopicMetadata(
"topic1", new List <PartitionMetadata> {
partition1Metadata, partition2Metadata
});
var topic2Metadata = new TopicMetadata(
"topic2", new List <PartitionMetadata> {
partition1Metadata, partition2Metadata
});
var topicPartitionInfos = new Dictionary <string, TopicMetadata>
{
{ "topic1", topic1Metadata },
{ "topic2", topic2Metadata }
};
var intPartitioner = new IntPartitioner();
var config = new ProducerConfig();
config.Brokers = TestUtils.GetBrokerListFromConfigs(this.props);
var producerPool = new ProducerPool(config);
var handler = new DefaultEventHandler <int, string>(
config, intPartitioner, null, new IntEncoder(), producerPool, topicPartitionInfos);
var topic1Broker1Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic1",
0,
new Message(
Encoding.UTF8.GetBytes("msg1"))),
new KeyedMessage <int, Message>(
"topic1",
2,
new Message(
Encoding.UTF8.GetBytes("msg3")))
};
var topic1Broker2Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic1",
-101,
3,
new Message(
Encoding.UTF8.GetBytes("msg4")))
};
var topic2Broker1Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic2",
4,
new Message(
Encoding.UTF8.GetBytes("msg5")))
};
var topic2Broker2Data = new List <KeyedMessage <int, Message> >
{
new KeyedMessage <int, Message>(
"topic2",
-99,
1,
new Message(
Encoding.UTF8.GetBytes("msg2")))
};
var expectedResult = new Dictionary <int, Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > > >
{
{ 0,
new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > >
{
{ new TopicAndPartition("topic1", 0), topic1Broker1Data },
{ new TopicAndPartition("topic2", 0), topic2Broker1Data }
} },
{ 1,
new Dictionary <TopicAndPartition, List <KeyedMessage <int, Message> > >
{
{ new TopicAndPartition("topic1", 1), topic1Broker2Data },
{ new TopicAndPartition("topic2", 1), topic2Broker2Data }
} },
};
var actualResut = handler.PartitionAndCollate(producerDataList);
Assert.Equal(expectedResult.Count, actualResut.Count);
Assert.True(expectedResult.Keys.SequenceEqual(actualResut.Keys));
foreach (var key in expectedResult.Keys)
{
var exptectedInnerDict = expectedResult[key];
var actualInnerDict = actualResut[key];
Assert.Equal(exptectedInnerDict.Count, actualInnerDict.Count);
foreach (var topicAndPartition in exptectedInnerDict.Keys)
{
var exptectedKeyedMsgs = exptectedInnerDict[topicAndPartition];
var actualKeyedMsgs = actualInnerDict[topicAndPartition];
Assert.True(exptectedKeyedMsgs.SequenceEqual(actualKeyedMsgs));
}
}
}
