protected ProducerActorBase(IActorRef client, IActorRef lookup, IActorRef cnxPool, string topic, ProducerConfigurationData conf, ISchema <T> schema, ProducerInterceptors <T> interceptors, ClientConfigurationData configurationData)
{
ClientConfiguration = configurationData;
Client = client;
_topic = topic;
if (conf.BatchingEnabled && conf.AckReceivedListerner == null)
{
conf.AckReceivedListerner = (acked) =>
{
Context.System.Log.Info($"AckReceived(ledger-id:{acked.LedgerId}, entery-id:{acked.EntryId}, sequence-id:{acked.SequenceId}, highest-sequence-id:{acked.HighestSequenceId})");
};
}
Conf = conf;
Schema = schema;
Interceptors = interceptors;
SchemaCache = new Dictionary <SchemaHash, byte[]>();
if (!conf.MultiSchema)
{
_multiSchemaMode = MultiSchemaMode.Disabled;
}
var pName = ProducerName().GetAwaiter().GetResult();
State = new HandlerState(lookup, cnxPool, topic, Context.System, pName);
}
public Producer(IActorRef producer, ISchema <T> schema, ProducerConfigurationData conf, TimeSpan opTimeout)
{
_producerActor = producer;
_schema = schema;
_conf = conf;
_operationTimeout = opTimeout;
}
public ProducerContainer(IActorRef producer, ProducerConfigurationData configuration, int maxMessageSize, ActorSystem system)
{
Producer = producer;
Configuration = configuration;
MaxMessageSize = maxMessageSize;
System = system;
}
public TypedMessageBuilder(IActorRef producer, ISchema <T> schema, User.Transaction txn, ProducerConfigurationData conf)
{
_metadata = new MessageMetadata();
_conf = conf;
_producer = producer;
_schema = schema;
_content = ReadOnlySequence <byte> .Empty;
_txn = txn;
}
private async ValueTask <Producer <T> > CreateProducer <T>(ProducerConfigurationData conf, ISchema <T> schema, ProducerInterceptors <T> interceptors)
{
if (conf == null)
{
throw new PulsarClientException.InvalidConfigurationException("Producer configuration undefined");
}
if (schema is AutoConsumeSchema)
{
throw new PulsarClientException.InvalidConfigurationException("AutoConsumeSchema is only used by consumers to detect schemas automatically");
}
var state = await _client.Ask <int>(GetClientState.Instance).ConfigureAwait(false);
if (state != 0)
{
throw new PulsarClientException.AlreadyClosedException($"Client already closed : state = {state}");
}
var topic = conf.TopicName;
if (!TopicName.IsValid(topic))
{
throw new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'");
}
if (schema is AutoProduceBytesSchema <T> autoProduceBytesSchema)
{
if (autoProduceBytesSchema.SchemaInitialized())
{
return(await CreateProducer(topic, conf, schema, interceptors).ConfigureAwait(false));
}
else
{
var schem = await _lookup.Ask <AskResponse>(new GetSchema(TopicName.Get(conf.TopicName))).ConfigureAwait(false);
if (schem.Failed)
{
throw schem.Exception;
}
var sc = schem.ConvertTo <GetSchemaInfoResponse>();
if (sc.SchemaInfo != null)
{
autoProduceBytesSchema.Schema = (ISchema <T>) ISchema <T> .GetSchema(sc.SchemaInfo);
}
else
{
autoProduceBytesSchema.Schema = (ISchema <T>)ISchema <T> .Bytes;
}
return(await CreateProducer(topic, conf, schema, interceptors).ConfigureAwait(false));
}
}
return(await CreateProducer(topic, conf, schema, interceptors).ConfigureAwait(false));
}
private void Init(ProducerConfigurationData conf)
{
try
{
_log.Info($"Starting Pulsar producer perf with config: {JsonSerializer.Serialize(conf, new JsonSerializerOptions{WriteIndented = true})}");
//log.info("Pulsar client config: {}", W.withoutAttribute("authentication").writeValueAsString(pulsarClient.Configuration));
}
catch (IOException e)
{
_log.Error("Failed to dump config info", e);
}
_oldTime = DateTime.Now.Millisecond;
StatTimeout = _system.Scheduler.Advanced.ScheduleOnceCancelable(TimeSpan.FromSeconds(_statsIntervalSeconds), StatsAction);
}
private async ValueTask <Producer <T> > CreateProducer <T>(string topic, ProducerConfigurationData conf, ISchema <T> schema, ProducerInterceptors <T> interceptors)
{
var metadata = await GetPartitionedTopicMetadata(topic).ConfigureAwait(false);
if (_actorSystem.Log.IsDebugEnabled)
{
_actorSystem.Log.Debug($"[{topic}] Received topic metadata. partitions: {metadata.Partitions}");
}
if (metadata.Partitions > 0)
{
var partitionActor = _actorSystem.ActorOf(Props.Create(() => new PartitionedProducer <T>(_client, _lookup, _cnxPool, _generator, topic, conf, metadata.Partitions, schema, interceptors, _clientConfigurationData)));
var co = await partitionActor.Ask <AskResponse>(Connect.Instance, _clientConfigurationData.OperationTimeout);
if (co.Failed)
{
await partitionActor.GracefulStop(TimeSpan.FromSeconds(5));
throw co.Exception;
}
_client.Tell(new AddProducer(partitionActor));
return(new Producer <T>(partitionActor, schema, conf, _clientConfigurationData.OperationTimeout));
}
else
{
var producerId = await _generator.Ask <long>(NewProducerId.Instance).ConfigureAwait(false);
var producer = _actorSystem.ActorOf(Props.Create(() => new ProducerActor <T>(producerId, _client, _lookup, _cnxPool, _generator, topic, conf, -1, schema, interceptors, _clientConfigurationData)));
var co = await producer.Ask <AskResponse>(Connect.Instance, _clientConfigurationData.OperationTimeout);
if (co.Failed)
{
await producer.GracefulStop(TimeSpan.FromSeconds(5));
throw co.Exception;
}
_client.Tell(new AddProducer(producer));
return(new Producer <T>(producer, schema, conf, _clientConfigurationData.OperationTimeout));
}
}
public ProducerStatsRecorder(long statsIntervalSeconds, ProducerConfigurationData conf, ActorSystem system, string producerName, string topic, long pendingQueueSize)
{
_system = system;
_producerName = producerName;
_topic = topic;
_pendingQueueSize = pendingQueueSize;
_log = system.Log;
Dec.NumberDecimalSeparator = "0.000";
ThroughputFormat.NumberDecimalSeparator = "0.00";
_statsIntervalSeconds = statsIntervalSeconds;
_numMsgsSent = new StripedLongAdder();
_numBytesSent = new StripedLongAdder();
_numSendFailed = new StripedLongAdder();
_numAcksReceived = new StripedLongAdder();
_totalMsgsSent = new StripedLongAdder();
_totalBytesSent = new StripedLongAdder();
_totalSendFailed = new StripedLongAdder();
_totalAcksReceived = new StripedLongAdder();
Init(conf);
}
public PartitionedProducer(IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, string topic, ProducerConfigurationData conf, int numPartitions, ISchema <T> schema, ProducerInterceptors <T> interceptors, ClientConfigurationData clientConfiguration) : base(client, lookup, cnxPool, topic, conf, schema, interceptors, clientConfiguration)
{
_cnxPool = cnxPool;
_lookup = lookup;
_self = Self;
_producers = new List <IActorRef>();
_generator = idGenerator;
_context = Context;
_producers = new List <IActorRef>(numPartitions);
_topicMetadata = new TopicMetadata(numPartitions);
_stats = clientConfiguration.StatsIntervalSeconds > 0 ? new ProducerStatsRecorder(Context.System, "PartitionedProducer", topic, conf.MaxPendingMessages) : null;
_log = Context.GetLogger();
var maxPendingMessages = Math.Min(conf.MaxPendingMessages, conf.MaxPendingMessagesAcrossPartitions / numPartitions);
conf.MaxPendingMessages = maxPendingMessages;
switch (conf.MessageRoutingMode)
{
case MessageRoutingMode.ConsistentHashingMode:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new ConsistentHashingGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
case MessageRoutingMode.BroadcastMode:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new BroadcastGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
case MessageRoutingMode.RandomMode:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new RandomGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
default:
_router = Context.System.ActorOf(Props.Empty.WithRouter(new RoundRobinGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}");
break;
}
// start track and auto subscribe partition increasement
if (conf.AutoUpdatePartitions)
{
_partitionsAutoUpdateTimeout = _context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimeSpan.FromSeconds(conf.AutoUpdatePartitionsIntervalSeconds), TimeSpan.FromSeconds(conf.AutoUpdatePartitionsIntervalSeconds), Self, ExtendTopics.Instance, ActorRefs.NoSender);
}
Receive <Flush>(_ => {
Flush();
});
ReceiveAsync <Connect>(async _ =>
{
await Start().ConfigureAwait(false);
});
Receive <TriggerFlush>(_ => {
TriggerFlush();
});
ReceiveAsync <ExtendTopics>(async _ =>
{
var t = topic;
await OnTopicsExtended(new List <string> {
t
});
});
ReceiveAsync <InternalSend <T> >(async m =>
{
try
{
//get excepyion vai out
await InternalSend(m.Message);
}
catch (Exception ex)
{
Sender.Tell(ex);
_log.Error(ex.ToString());
}
});
Receive <InternalSendWithTxn <T> >(m =>
{
try
{
InternalSendWithTxn(m.Message, m.Txn);
}
catch (Exception ex)
{
Sender.Tell(ex);
_log.Error(ex.ToString());
}
});
ReceiveAny(any => _router.Forward(any));
}
public TypedMessageBuilder(IActorRef producer, ISchema <T> schema, ProducerConfigurationData conf) : this(producer, schema, null, conf)
{
}
private async ValueTask <Producer <T> > CreateProducer <T>(ProducerConfigurationData conf, ISchema <T> schema)
{
return(await CreateProducer(conf, schema, null).ConfigureAwait(false));
}
private async ValueTask <Producer <byte[]> > CreateProducer(ProducerConfigurationData conf)
{
return(await CreateProducer(conf, ISchema <object> .Bytes, null).ConfigureAwait(false));
}
