private async ValueTask OnTopicsExtended(ICollection <string> topicsExtended)
{
if (topicsExtended.Count == 0 || !topicsExtended.Contains(Topic))
{
return;
}
var topicName = TopicName.Get(Topic);
var result = await _lookup.Ask <AskResponse>(new GetPartitionedTopicMetadata(topicName));
if (result.Failed)
{
throw result.Exception;
}
var metadata = result.ConvertTo <PartitionedTopicMetadata>();
var topics = GetPartitionsForTopic(topicName, metadata).ToList();
var oldPartitionNumber = _topicMetadata.NumPartitions();
var currentPartitionNumber = topics.Count;
if (_log.IsDebugEnabled)
{
_log.Debug($"[{Topic}] partitions number. old: {oldPartitionNumber}, new: {currentPartitionNumber}");
}
if (oldPartitionNumber == currentPartitionNumber)
{
return;
}
else if (oldPartitionNumber < currentPartitionNumber)
{
var newPartitions = topics.GetRange(oldPartitionNumber, currentPartitionNumber);
foreach (var partitionName in newPartitions)
{
var producerId = await _generator.Ask <long>(NewProducerId.Instance);
var partitionIndex = TopicName.GetPartitionIndex(partitionName);
var producer = _context.ActorOf(Props.Create(() => new ProducerActor <T>(producerId, Client, _lookup, _cnxPool, _generator, partitionName, Conf, partitionIndex, Schema, Interceptors, ClientConfiguration)));
var co = await producer.Ask <AskResponse>(Connect.Instance, ClientConfiguration.OperationTimeout);
if (!co.Failed)
{
_producers.Add(producer);
var routee = Routee.FromActorRef(producer);
_router.Tell(new AddRoutee(routee));
}
}
if (_log.IsDebugEnabled)
{
_log.Debug($"[{Topic}] success create producers for extended partitions. old: {oldPartitionNumber}, new: {currentPartitionNumber}");
}
_topicMetadata = new TopicMetadata(currentPartitionNumber);
}
else
{
_log.Error($"[{Topic}] not support shrink topic partitions. old: {oldPartitionNumber}, new: {currentPartitionNumber}");
}
}
public PartitionedProducer(IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, string topic, ProducerConfigurationData conf, int numPartitions, ISchema <T> schema, ProducerInterceptors <T> interceptors, ClientConfigurationData clientConfiguration) : base(client, lookup, cnxPool, topic, conf, schema, interceptors, clientConfiguration)
{
_cnxPool = cnxPool;
_lookup = lookup;
_self = Self;
_producers = new List <IActorRef>();
_generator = idGenerator;
_context = Context;
_producers = new List <IActorRef>(numPartitions);
_topicMetadata = new TopicMetadata(numPartitions);
_stats = clientConfiguration.StatsIntervalSeconds > 0 ? new ProducerStatsRecorder(Context.System, "PartitionedProducer", topic, conf.MaxPendingMessages) : null;
_log = Context.GetLogger();
var maxPendingMessages = Math.Min(conf.MaxPendingMessages, conf.MaxPendingMessagesAcrossPartitions / numPartitions);
conf.MaxPendingMessages = maxPendingMessages;
switch (conf.MessageRoutingMode)
{
case MessageRoutingMode.ConsistentHashingMode:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new ConsistentHashingGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
case MessageRoutingMode.BroadcastMode:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new BroadcastGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
case MessageRoutingMode.RandomMode:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new RandomGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
default:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new RoundRobinGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
}
// start track and auto subscribe partition increasement
if (conf.AutoUpdatePartitions)
{
_partitionsAutoUpdateTimeout = _context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimeSpan.FromSeconds(conf.AutoUpdatePartitionsIntervalSeconds), TimeSpan.FromSeconds(conf.AutoUpdatePartitionsIntervalSeconds), Self, ExtendTopics.Instance, ActorRefs.NoSender);
}
Receive <Flush>(_ => {
Flush();
});
ReceiveAsync <Connect>(async _ =>
{
await Start().ConfigureAwait(false);
});
Receive <TriggerFlush>(_ => {
TriggerFlush();
});
ReceiveAsync <ExtendTopics>(async _ =>
{
var t = topic;
await OnTopicsExtended(new List <string> {
t
});
});
ReceiveAsync <InternalSend <T> >(async m =>
{
try
{
//get excepyion vai out
await InternalSend(m.Message);
}
catch (Exception ex)
{
Sender.Tell(ex);
_log.Error(ex.ToString());
}
});
Receive <InternalSendWithTxn <T> >(m =>
{
try
{
InternalSendWithTxn(m.Message, m.Txn);
}
catch (Exception ex)
{
Sender.Tell(ex);
_log.Error(ex.ToString());
}
});
ReceiveAny(any => _router.Forward(any));
}
public abstract int ChoosePartition <T>(IMessage <T> msg, TopicMetadata metadata);
