protected ProducerActorBase(IActorRef client, IActorRef lookup, IActorRef cnxPool, string topic, ProducerConfigurationData conf, ISchema <T> schema, ProducerInterceptors <T> interceptors, ClientConfigurationData configurationData) { ClientConfiguration = configurationData; Client = client; _topic = topic; if (conf.BatchingEnabled && conf.AckReceivedListerner == null) { conf.AckReceivedListerner = (acked) => { Context.System.Log.Info($"AckReceived(ledger-id:{acked.LedgerId}, entery-id:{acked.EntryId}, sequence-id:{acked.SequenceId}, highest-sequence-id:{acked.HighestSequenceId})"); }; } Conf = conf; Schema = schema; Interceptors = interceptors; SchemaCache = new Dictionary <SchemaHash, byte[]>(); if (!conf.MultiSchema) { _multiSchemaMode = MultiSchemaMode.Disabled; } var pName = ProducerName().GetAwaiter().GetResult(); State = new HandlerState(lookup, cnxPool, topic, Context.System, pName); }
public Producer(IActorRef producer, ISchema <T> schema, ProducerConfigurationData conf, TimeSpan opTimeout) { _producerActor = producer; _schema = schema; _conf = conf; _operationTimeout = opTimeout; }
public ProducerContainer(IActorRef producer, ProducerConfigurationData configuration, int maxMessageSize, ActorSystem system) { Producer = producer; Configuration = configuration; MaxMessageSize = maxMessageSize; System = system; }
public TypedMessageBuilder(IActorRef producer, ISchema <T> schema, User.Transaction txn, ProducerConfigurationData conf) { _metadata = new MessageMetadata(); _conf = conf; _producer = producer; _schema = schema; _content = ReadOnlySequence <byte> .Empty; _txn = txn; }
private async ValueTask <Producer <T> > CreateProducer <T>(ProducerConfigurationData conf, ISchema <T> schema, ProducerInterceptors <T> interceptors) { if (conf == null) { throw new PulsarClientException.InvalidConfigurationException("Producer configuration undefined"); } if (schema is AutoConsumeSchema) { throw new PulsarClientException.InvalidConfigurationException("AutoConsumeSchema is only used by consumers to detect schemas automatically"); } var state = await _client.Ask <int>(GetClientState.Instance).ConfigureAwait(false); if (state != 0) { throw new PulsarClientException.AlreadyClosedException($"Client already closed : state = {state}"); } var topic = conf.TopicName; if (!TopicName.IsValid(topic)) { throw new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'"); } if (schema is AutoProduceBytesSchema <T> autoProduceBytesSchema) { if (autoProduceBytesSchema.SchemaInitialized()) { return(await CreateProducer(topic, conf, schema, interceptors).ConfigureAwait(false)); } else { var schem = await _lookup.Ask <AskResponse>(new GetSchema(TopicName.Get(conf.TopicName))).ConfigureAwait(false); if (schem.Failed) { throw schem.Exception; } var sc = schem.ConvertTo <GetSchemaInfoResponse>(); if (sc.SchemaInfo != null) { autoProduceBytesSchema.Schema = (ISchema <T>) ISchema <T> .GetSchema(sc.SchemaInfo); } else { autoProduceBytesSchema.Schema = (ISchema <T>)ISchema <T> .Bytes; } return(await CreateProducer(topic, conf, schema, interceptors).ConfigureAwait(false)); } } return(await CreateProducer(topic, conf, schema, interceptors).ConfigureAwait(false)); }
private void Init(ProducerConfigurationData conf) { try { _log.Info($"Starting Pulsar producer perf with config: {JsonSerializer.Serialize(conf, new JsonSerializerOptions{WriteIndented = true})}"); //log.info("Pulsar client config: {}", W.withoutAttribute("authentication").writeValueAsString(pulsarClient.Configuration)); } catch (IOException e) { _log.Error("Failed to dump config info", e); } _oldTime = DateTime.Now.Millisecond; StatTimeout = _system.Scheduler.Advanced.ScheduleOnceCancelable(TimeSpan.FromSeconds(_statsIntervalSeconds), StatsAction); }
private async ValueTask <Producer <T> > CreateProducer <T>(string topic, ProducerConfigurationData conf, ISchema <T> schema, ProducerInterceptors <T> interceptors) { var metadata = await GetPartitionedTopicMetadata(topic).ConfigureAwait(false); if (_actorSystem.Log.IsDebugEnabled) { _actorSystem.Log.Debug($"[{topic}] Received topic metadata. partitions: {metadata.Partitions}"); } if (metadata.Partitions > 0) { var partitionActor = _actorSystem.ActorOf(Props.Create(() => new PartitionedProducer <T>(_client, _lookup, _cnxPool, _generator, topic, conf, metadata.Partitions, schema, interceptors, _clientConfigurationData))); var co = await partitionActor.Ask <AskResponse>(Connect.Instance, _clientConfigurationData.OperationTimeout); if (co.Failed) { await partitionActor.GracefulStop(TimeSpan.FromSeconds(5)); throw co.Exception; } _client.Tell(new AddProducer(partitionActor)); return(new Producer <T>(partitionActor, schema, conf, _clientConfigurationData.OperationTimeout)); } else { var producerId = await _generator.Ask <long>(NewProducerId.Instance).ConfigureAwait(false); var producer = _actorSystem.ActorOf(Props.Create(() => new ProducerActor <T>(producerId, _client, _lookup, _cnxPool, _generator, topic, conf, -1, schema, interceptors, _clientConfigurationData))); var co = await producer.Ask <AskResponse>(Connect.Instance, _clientConfigurationData.OperationTimeout); if (co.Failed) { await producer.GracefulStop(TimeSpan.FromSeconds(5)); throw co.Exception; } _client.Tell(new AddProducer(producer)); return(new Producer <T>(producer, schema, conf, _clientConfigurationData.OperationTimeout)); } }
public ProducerStatsRecorder(long statsIntervalSeconds, ProducerConfigurationData conf, ActorSystem system, string producerName, string topic, long pendingQueueSize) { _system = system; _producerName = producerName; _topic = topic; _pendingQueueSize = pendingQueueSize; _log = system.Log; Dec.NumberDecimalSeparator = "0.000"; ThroughputFormat.NumberDecimalSeparator = "0.00"; _statsIntervalSeconds = statsIntervalSeconds; _numMsgsSent = new StripedLongAdder(); _numBytesSent = new StripedLongAdder(); _numSendFailed = new StripedLongAdder(); _numAcksReceived = new StripedLongAdder(); _totalMsgsSent = new StripedLongAdder(); _totalBytesSent = new StripedLongAdder(); _totalSendFailed = new StripedLongAdder(); _totalAcksReceived = new StripedLongAdder(); Init(conf); }
public PartitionedProducer(IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, string topic, ProducerConfigurationData conf, int numPartitions, ISchema <T> schema, ProducerInterceptors <T> interceptors, ClientConfigurationData clientConfiguration) : base(client, lookup, cnxPool, topic, conf, schema, interceptors, clientConfiguration) { _cnxPool = cnxPool; _lookup = lookup; _self = Self; _producers = new List <IActorRef>(); _generator = idGenerator; _context = Context; _producers = new List <IActorRef>(numPartitions); _topicMetadata = new TopicMetadata(numPartitions); _stats = clientConfiguration.StatsIntervalSeconds > 0 ? new ProducerStatsRecorder(Context.System, "PartitionedProducer", topic, conf.MaxPendingMessages) : null; _log = Context.GetLogger(); var maxPendingMessages = Math.Min(conf.MaxPendingMessages, conf.MaxPendingMessagesAcrossPartitions / numPartitions); conf.MaxPendingMessages = maxPendingMessages; switch (conf.MessageRoutingMode) { case MessageRoutingMode.ConsistentHashingMode: _router = Context.System.ActorOf(Props.Empty.WithRouter(new ConsistentHashingGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}"); break; case MessageRoutingMode.BroadcastMode: _router = Context.System.ActorOf(Props.Empty.WithRouter(new BroadcastGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}"); break; case MessageRoutingMode.RandomMode: _router = Context.System.ActorOf(Props.Empty.WithRouter(new RandomGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}"); break; default: _router = Context.System.ActorOf(Props.Empty.WithRouter(new RoundRobinGroup()), $"Partition{DateTimeHelper.CurrentUnixTimeMillis()}"); break; } // start track and auto subscribe partition increasement if (conf.AutoUpdatePartitions) { _partitionsAutoUpdateTimeout = _context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimeSpan.FromSeconds(conf.AutoUpdatePartitionsIntervalSeconds), TimeSpan.FromSeconds(conf.AutoUpdatePartitionsIntervalSeconds), Self, ExtendTopics.Instance, ActorRefs.NoSender); } Receive <Flush>(_ => { Flush(); }); ReceiveAsync <Connect>(async _ => { await Start().ConfigureAwait(false); }); Receive <TriggerFlush>(_ => { TriggerFlush(); }); ReceiveAsync <ExtendTopics>(async _ => { var t = topic; await OnTopicsExtended(new List <string> { t }); }); ReceiveAsync <InternalSend <T> >(async m => { try { //get excepyion vai out await InternalSend(m.Message); } catch (Exception ex) { Sender.Tell(ex); _log.Error(ex.ToString()); } }); Receive <InternalSendWithTxn <T> >(m => { try { InternalSendWithTxn(m.Message, m.Txn); } catch (Exception ex) { Sender.Tell(ex); _log.Error(ex.ToString()); } }); ReceiveAny(any => _router.Forward(any)); }
public TypedMessageBuilder(IActorRef producer, ISchema <T> schema, ProducerConfigurationData conf) : this(producer, schema, null, conf) { }
private async ValueTask <Producer <T> > CreateProducer <T>(ProducerConfigurationData conf, ISchema <T> schema) { return(await CreateProducer(conf, schema, null).ConfigureAwait(false)); }
private async ValueTask <Producer <byte[]> > CreateProducer(ProducerConfigurationData conf) { return(await CreateProducer(conf, ISchema <object> .Bytes, null).ConfigureAwait(false)); }