/// <summary>
/// Initializes a new instance of the <see cref="Producer" /> class.
/// </summary>
/// <param name="config">The config object.</param>
/// <remarks>
/// Can be used when all config parameters will be specified through the config object
/// and will be instantiated via reflection
/// </remarks>
public Producer(ProducerConfiguration config)
: base(config)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class.
/// </summary>
/// <param name="config">The config.</param>
/// <param name="callback">The callback invoked when new broker is added.</param>
public ZKBrokerPartitionInfo(ProducerConfiguration config, Action <int, string, int> callback)
: this(new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer, config.ZooKeeper.ZkConnectionTimeoutMs), callback)
{
this.callback = callback;
}
public Producer <TKey, TData> RefreshMetadataAndRecreateProducerOfOnePartition(short versionId, string clientId, int correlationId, string topic, int partitionId, bool forceRefreshMetadata, bool forceRecreateEvenHostPortSame, ProducerConfiguration producerConfigTemplate, bool randomReturnIfProducerOfTargetPartionNotExists)
{
Logger.InfoFormat("RefreshMetadataAndRecreateProducerWithPartition == enter: Topic:{0} partitionId:{1} forceRefreshMetadata:{2} forceRecreateEvenHostPortSame:{3} randomReturnIfProducerOfTargetPartionNotExists:{4} ", topic, partitionId, forceRefreshMetadata, forceRecreateEvenHostPortSame, randomReturnIfProducerOfTargetPartionNotExists);
TopicMetadata topicMetadata = null;
if (forceRefreshMetadata)
{
topicMetadata = RefreshMetadata(versionId, clientId, correlationId, topic, forceRefreshMetadata);
}
if (!this.TopicMetadataPartitionsLeaders[topic].ContainsKey(partitionId))
{
throw new NoLeaderForPartitionException(string.Format("No leader for topic {0} parition {1} ", topic, partitionId));
}
Tuple <Broker, BrokerConfiguration> value = this.TopicMetadataPartitionsLeaders[topic][partitionId];
CreateProducerOfOnePartition(topic, partitionId, value.Item2, producerConfigTemplate, forceRecreateEvenHostPortSame);
return(GetProducerOfPartition(topic, partitionId, randomReturnIfProducerOfTargetPartionNotExists));
}
private void CreateProducerOfOnePartition(string topic, int partitionId, BrokerConfiguration broker, ProducerConfiguration producerConfigTemplate, bool forceRecreateEvenHostPortSame)
{
Logger.InfoFormat("CreateProducer == enter: Topic:{0} partitionId:{1} broker:{2} forceRecreateEvenHostPortSame:{3} ", topic, partitionId, broker, forceRecreateEvenHostPortSame);
//Explicitly set partitionID
ProducerConfiguration producerConfig = new ProducerConfiguration(producerConfigTemplate, new List <BrokerConfiguration> {
broker
}, partitionId)
{
ForceToPartition = partitionId
};
if (!this.TopicPartitionsLeaderProducers.ContainsKey(topic))
{
this.TopicPartitionsLeaderProducers.TryAdd(topic, new ConcurrentDictionary <int, Producer <TKey, TData> >());
}
ConcurrentDictionary <int, Producer <TKey, TData> > dictPartitionLeaderProducersOfOneTopic = this.TopicPartitionsLeaderProducers[topic];
bool needRecreate = true;
Producer <TKey, TData> oldProducer = null;
if (dictPartitionLeaderProducersOfOneTopic.TryGetValue(partitionId, out oldProducer))
{
if (oldProducer.Config.Brokers.Any() &&
oldProducer.Config.Brokers[0].Equals(producerConfig.Brokers[0]))
{
needRecreate = false;
}
}
if (forceRecreateEvenHostPortSame)
{
needRecreate = true;
}
if (needRecreate)
{
lock (GetProduceLockOfTopic(topic, partitionId))
{
if (dictPartitionLeaderProducersOfOneTopic.TryGetValue(partitionId, out oldProducer))
{
if (oldProducer.Config.Brokers.Any() &&
oldProducer.Config.Brokers[0].Equals(producerConfig.Brokers[0]))
{
needRecreate = false;
}
}
if (forceRecreateEvenHostPortSame)
{
needRecreate = true;
}
if (!needRecreate)
{
Logger.InfoFormat("CreateProducer == Add producer SKIP after got lock for topic {0} partition {1} since leader {2} not changed. Maybe created by other thread. END.", topic, partitionId, producerConfig.Brokers[0]);
return;
}
bool removeOldProducer = false;
if (oldProducer != null)
{
oldProducer.Dispose();
removeOldProducer = dictPartitionLeaderProducersOfOneTopic.TryRemove(partitionId, out oldProducer);
Logger.InfoFormat("CreateProducer == Remove producer for topic {0} partition {1} leader {2} removeOldProducer:{3} ", topic, partitionId, oldProducer.Config.Brokers[0], removeOldProducer);
}
Producer <TKey, TData> producer = new Producer <TKey, TData>(producerConfig);
bool addNewProducer = dictPartitionLeaderProducersOfOneTopic.TryAdd(partitionId, producer);
Logger.InfoFormat("CreateProducer == Add producer for topic {0} partition {1} leader {2} SyncProducerOfOneBroker:{3} removeOldProducer:{4} addNewProducer:{5} END.", topic, partitionId, broker, producerConfig.SyncProducerOfOneBroker, removeOldProducer, addNewProducer);
}
}
else
{
Logger.InfoFormat("CreateProducer == Add producer SKIP for topic {0} partition {1} since leader {2} not changed. END.", topic, partitionId, producerConfig.Brokers[0]);
}
}
public int InitializeProducerPoolForTopic(short versionId, string clientId, int correlationId, string topic, bool forceRefreshMetadata, ProducerConfiguration producerConfigTemplate, bool forceRecreateEvenHostPortSame)
{
Logger.InfoFormat("InitializeProducerPoolForTopic == enter: Topic:{0} forceRefreshMetadata:{1} forceRecreateEvenHostPortSame:{2} ", topic, forceRefreshMetadata, forceRecreateEvenHostPortSame);
TopicMetadata topicMetadata = null;
if (forceRefreshMetadata)
{
topicMetadata = RefreshMetadata(versionId, clientId, correlationId, topic, forceRefreshMetadata);
}
Dictionary <int, Tuple <Broker, BrokerConfiguration> > partitionLeaders = this.TopicMetadataPartitionsLeaders[topic];
//TODO: but some times the partition maybe has no leader
//For use partitioner calss. Totally only create one producer.
if (string.IsNullOrEmpty(this.Config.PartitionerClass))
{
foreach (KeyValuePair <int, Tuple <Broker, BrokerConfiguration> > kv in partitionLeaders)
{
CreateProducerOfOnePartition(topic, kv.Key, kv.Value.Item2, producerConfigTemplate, forceRecreateEvenHostPortSame);
}
Logger.InfoFormat("InitializeProducerPoolForTopic == exit: Topic:{0} forceRefreshMetadata:{1} forceRecreateEvenHostPortSame:{2} this.TopicPartitionsLeaderProducers[topic].Count:{3}", topic, forceRefreshMetadata, forceRecreateEvenHostPortSame, this.TopicPartitionsLeaderProducers[topic].Count);
return(this.TopicPartitionsLeaderProducers[topic].Count);
}
else
{
ProducerConfiguration producerConfig = new ProducerConfiguration(producerConfigTemplate);
producerConfig.ZooKeeper = this.Config.ZookeeperConfig;
producerConfig.PartitionerClass = this.Config.PartitionerClass;
Producer <TKey, TData> oldProducer = null;
if (TopicProducersWithPartitionerClass.TryGetValue(topic, out oldProducer))
{
bool removeOldProducer = false;
if (oldProducer != null)
{
oldProducer.Dispose();
removeOldProducer = TopicProducersWithPartitionerClass.TryRemove(topic, out oldProducer);
Logger.InfoFormat("InitializeProducerPoolForTopic == Remove producer from TopicProducersWithPartitionerClass for topic {0} removeOldProducer:{1} ", topic, removeOldProducer);
}
}
Producer <TKey, TData> producer = new Producer <TKey, TData>(producerConfig);
bool addNewProducer = TopicProducersWithPartitionerClass.TryAdd(topic, producer);
Logger.InfoFormat("InitializeProducerPoolForTopic == Add producer TopicProducersWithPartitionerClass for topic {0} SyncProducerOfOneBroker:{1} addNewProducer:{2} END.", topic, producerConfig.SyncProducerOfOneBroker, addNewProducer);
return(addNewProducer ? 1 : 0);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ConfigBrokerPartitionInfo"/> class.
/// </summary>
/// <param name="config">The config.</param>
public ConfigBrokerPartitionInfo(ProducerConfiguration config)
{
Guard.NotNull(config, "config");
this.config = config;
this.InitializeBrokers();
}
public async Task TestProducerConsumer()
{
var cancel = new CancellationTokenSource();
var directoryEndpoint = "http://localhost:8080";
var producer1Endpoint = "tcp://localhost:8181";
var producer1HeartbeatEndpoint = "tcp://localhost:8282";
IWebHost host = null;
new Task(() =>
{
host = new WebHostBuilder()
.UseKestrel()
.UseUrls(directoryEndpoint)
.UseStartup <DirectoryStartup>()
.Build();
host.Run();
}, cancel.Token).Start();
await Task.Delay(500);
var directory = RestService.For <IDirectory>(directoryEndpoint);
var configurationProducer1 = new ProducerConfiguration()
{
IsTest = true,
Endpoint = producer1Endpoint,
HeartbeatEnpoint = producer1HeartbeatEndpoint,
Id = Guid.NewGuid()
};
var producer1 = new AccidentProducer(configurationProducer1, directory, new JsonSerializerSettings());
producer1.Start();
await Task.Delay(500);
var configurationConsumer1 = new ConsumerConfiguration <AccidentEvent>()
{
Topic = "Paris.Business",
Id = Guid.NewGuid()
};
var consumedEvents = new List <AccidentEvent>();
var consumer = new Consumer <AccidentEvent>(configurationConsumer1, directory, new JsonSerializerSettings());
consumer.GetSubscription()
.Subscribe(ev =>
{
consumedEvents.Add(ev);
});
consumer.Start();
await Task.Delay(500);
cancel.Cancel();
await host.StopAsync();
producer1.Stop();
consumer.Stop();
Assert.IsTrue(consumedEvents.Count > 0);
}
public async Task TestE2E()
{
var cancel = new CancellationTokenSource();
var directoryEndpoint = "http://localhost:8080";
var producer1Endpoint = "tcp://localhost:8181";
var producer1HeartbeatEndpoint = "tcp://localhost:8282";
var consumerEndpoint = "tcp://localhost:8383";
var consumerHeartbeatEndpoint = "tcp://localhost:8484";
var producer2Endpoint = "tcp://localhost:8585";
var producer2HeartbeatEndpoint = "tcp://localhost:8686";
//create directory
IWebHost host = null;
new Task(() =>
{
host = new WebHostBuilder()
.UseKestrel()
.UseUrls(directoryEndpoint)
.UseStartup <DirectoryStartup>()
.Build();
host.Run();
}, cancel.Token).Start();
await Task.Delay(500);
var directory = RestService.For <IDirectory>(directoryEndpoint);
//create producers
var configurationProducer1 = new ProducerConfiguration()
{
IsTest = true,
Endpoint = producer1Endpoint,
HeartbeatEnpoint = producer1HeartbeatEndpoint,
Id = Guid.NewGuid()
};
var configurationProducer2 = new ProducerConfiguration()
{
IsTest = true,
Endpoint = producer2Endpoint,
HeartbeatEnpoint = producer2HeartbeatEndpoint,
Id = Guid.NewGuid()
};
var producer1 = new AccidentProducer(configurationProducer1, directory, new JsonSerializerSettings());
var producer2 = new AccidentProducer(configurationProducer2, directory, new JsonSerializerSettings());
//start only one producer
producer1.Start();
await Task.Delay(500);
var configurationConsumer1 = new ConsumerConfiguration <AccidentEvent>()
{
Topic = "Paris.Business",
Id = Guid.NewGuid(),
Endpoint = consumerEndpoint,
HeartbeatEndpoint = consumerHeartbeatEndpoint
};
var consumedEvents = new List <AccidentEvent>();
var consumer = new Consumer <AccidentEvent>(configurationConsumer1, directory, new JsonSerializerSettings());
consumer.GetSubscription()
.Subscribe(ev =>
{
consumedEvents.Add(ev);
});
//start consumer
consumer.Start();
await Task.Delay(1000);
//the consumer should have fetch and subscribe to a producer
var stateOfTheWorld = await directory.GetStateOfTheWorld();
var currentEventCount = consumedEvents.Count;
//the producer shouold have register to the registry
var producer = stateOfTheWorld.First();
Assert.AreEqual(ProducerState.Alive, producer.State);
//at least an event should have match the filter
Assert.Greater(currentEventCount, 0);
Assert.AreEqual(1, stateOfTheWorld.Count());
//memorize current event count
var eventCount = consumedEvents.Count;
//kill the producer
producer1.Stop();
await Task.Delay(1000);
//the directory should have heartbeat the consumer, the consumer should not have consume any more event
stateOfTheWorld = await directory.GetStateOfTheWorld();
producer = stateOfTheWorld.First();
Assert.AreEqual(ProducerState.NotResponding, producer.State);
Assert.AreEqual(eventCount, consumedEvents.Count);
//start the second producer
producer2.Start();
await Task.Delay(1000);
//the directory shoud have register the new producer, the consumer should have subscribe to the new consumer
stateOfTheWorld = await directory.GetStateOfTheWorld();
Assert.AreEqual(2, stateOfTheWorld.Count());
Assert.Greater(consumedEvents.Count, currentEventCount);
cancel.Cancel();
producer2.Stop();
await host.StopAsync();
consumer.Stop();
}
private static Symbol[] GetCapabilities(ProducerConfiguration configuration)
{
return(configuration.RoutingType.HasValue
? new[] { configuration.RoutingType.Value.GetRoutingCapability() }
: null);
}
