private ReaderConfigurationData <T> PrepareConsumerConfiguration(ReaderConfigurationData <T> readerConfiguration, string topic, MessageId messageId, int permits)
{
readerConfiguration.TopicName = topic;
readerConfiguration.StartMessageId = messageId;
readerConfiguration.ReceiverQueueSize = permits;
return(readerConfiguration);
}
public Reader(IActorRef stateActor, IActorRef consumer, ISchema <T> schema, ReaderConfigurationData <T> conf)
{
_stateActor = stateActor;
_readerActor = consumer;
_schema = schema;
_conf = conf;
}
public PulsarSourceActor(ClientConfigurationData client, ReaderConfigurationData <T> readerConfiguration, IActorRef clientActor, IActorRef lookup, IActorRef cnxPool, IActorRef generator, long fromOffset, long toOffset, bool isLive, ISchema <T> schema)
{
_scheduler = Context.System.Scheduler.Advanced;
_toOffset = toOffset;
_parent = Context.Parent;
_lastEventMessageOffset = fromOffset;
var topicName = TopicName.Get(readerConfiguration.TopicName);
IActorRef stateA = Context.ActorOf(Props.Create(() => new ConsumerStateActor()), $"StateActor{Guid.NewGuid()}");
var subscription = "player-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10);
if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix))
{
subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription;
}
ConsumerConfigurationData <T> consumerConfiguration = new ConsumerConfigurationData <T>();
consumerConfiguration.TopicNames.Add(readerConfiguration.TopicName);
consumerConfiguration.SubscriptionName = subscription;
consumerConfiguration.SubscriptionType = SubType.Exclusive;
consumerConfiguration.SubscriptionMode = SubscriptionMode.NonDurable;
consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize;
consumerConfiguration.ReadCompacted = readerConfiguration.ReadCompacted;
consumerConfiguration.StartMessageId = readerConfiguration.StartMessageId;
if (readerConfiguration.ReaderName != null)
{
consumerConfiguration.ConsumerName = readerConfiguration.ReaderName;
}
if (readerConfiguration.ResetIncludeHead)
{
consumerConfiguration.ResetIncludeHead = true;
}
consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction;
if (readerConfiguration.CryptoKeyReader != null)
{
consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader;
}
if (readerConfiguration.KeyHashRanges != null)
{
consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray());
}
var partitionIdx = TopicName.GetPartitionIndex(readerConfiguration.TopicName);
var consumerId = generator.Ask <long>(NewConsumerId.Instance).GetAwaiter().GetResult();
_child = Context.ActorOf(Props.Create(() => new ConsumerActor <T>(consumerId, stateA, clientActor, lookup, cnxPool, generator, readerConfiguration.TopicName, consumerConfiguration, Context.System.Scheduler.Advanced, partitionIdx, true, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, schema, true, client)));
_child.Tell(Connect.Instance);
if (isLive)
{
LiveConsume();
}
else
{
Consume();
}
}
private async ValueTask <Reader <T> > CreateSingleTopicReader <T>(ReaderConfigurationData <T> conf, ISchema <T> schema)
{
var topic = conf.TopicName;
try
{
var metadata = await GetPartitionedTopicMetadata(topic).ConfigureAwait(false);
if (_actorSystem.Log.IsDebugEnabled)
{
_actorSystem.Log.Debug($"[{topic}] Received topic metadata. partitions: {metadata.Partitions}");
}
if (metadata.Partitions > 0 && MultiTopicsConsumer <T> .IsIllegalMultiTopicsMessageId(conf.StartMessageId))
{
throw new PulsarClientException("The partitioned topic startMessageId is illegal");
}
IActorRef reader;
var stateA = _actorSystem.ActorOf(Props.Create(() => new ConsumerStateActor()), $"StateActor{Guid.NewGuid()}");
if (metadata.Partitions > 0)
{
reader = _actorSystem.ActorOf(Props.Create(() => new MultiTopicsReader <T>(stateA, _client, _lookup, _cnxPool, _generator, conf, _actorSystem.Scheduler.Advanced, schema, _clientConfigurationData)));
var response = await reader.Ask <AskResponse>(new Subscribe(topic, metadata.Partitions)).ConfigureAwait(false);
if (response.Failed)
{
await reader.GracefulStop(TimeSpan.FromSeconds(5));
throw response.Exception;
}
_client.Tell(new AddConsumer(reader));
return(new Reader <T>(stateA, reader, schema, conf));
}
else
{
var consumerId = await _generator.Ask <long>(NewConsumerId.Instance).ConfigureAwait(false);
reader = _actorSystem.ActorOf(Props.Create(() => new ReaderActor <T>(consumerId, stateA, _client, _lookup, _cnxPool, _generator, conf, _actorSystem.Scheduler.Advanced, schema, _clientConfigurationData)));
var response = await reader.Ask <AskResponse>(Connect.Instance).ConfigureAwait(false);
if (response.Failed)
{
await reader.GracefulStop(TimeSpan.FromSeconds(5));
throw response.Exception;
}
_client.Tell(new AddConsumer(reader));
return(new Reader <T>(stateA, reader, schema, conf));
}
}
catch (Exception ex)
{
_actorSystem.Log.Warning($"[{topic}] Failed to get create topic reader: {ex}");
throw;
}
}
public ReplayTopic(ReaderConfigurationData <T> readerConfigurationData, string adminUrl, long @from, long to, long max, Tag tag, bool tagged, SourceType source)
{
ReaderConfigurationData = readerConfigurationData;
From = @from;
To = to;
Max = max;
Tag = tag;
Tagged = tagged;
Source = source;
AdminUrl = adminUrl;
}
internal StartReplayTopic(ClientConfigurationData clientConfigurationData, ReaderConfigurationData <T> readerConfigurationData, string adminUrl, long @from, long to, long max, Tag tag, bool tagged, SourceType source)
{
ClientConfigurationData = clientConfigurationData;
ReaderConfigurationData = readerConfigurationData;
From = @from;
To = to;
Max = max;
Tag = tag;
Tagged = tagged;
Source = source;
AdminUrl = adminUrl;
}
public EventsByTopic(string tenant, string ns, string topic, long fromMessageId, long toMessageId, string adminUrl, ReaderConfigurationData <T> configuration, ClientConfigurationData clientConfiguration)
{
Tenant = tenant;
Namespace = ns;
Topic = topic;
FromMessageId = fromMessageId;
ToMessageId = toMessageId;
Source = SourceType.Pulsar;
AdminUrl = adminUrl;
Configuration = configuration;
ClientConfiguration = clientConfiguration;
}
private async ValueTask <Reader <T> > CreateMultiTopicReader <T>(ReaderConfigurationData <T> conf, ISchema <T> schema)
{
var stateA = _actorSystem.ActorOf(Props.Create(() => new ConsumerStateActor()), $"StateActor{Guid.NewGuid()}");
var reader = _actorSystem.ActorOf(Props.Create(() => new MultiTopicsReader <T>(stateA, _client, _lookup, _cnxPool, _generator, conf, _actorSystem.Scheduler.Advanced, schema, _clientConfigurationData)));
var response = await reader.Ask <AskResponse>(new SubscribeAndCreateTopicsIfDoesNotExist(conf.TopicNames.ToList(), true)).ConfigureAwait(false);
if (response.Failed)
{
await reader.GracefulStop(TimeSpan.FromSeconds(5));
throw response.Exception;
}
_client.Tell(new AddConsumer(reader));
return(new Reader <T>(stateA, reader, schema, conf));
}
public ReaderActor(long consumerId, IActorRef stateActor, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, ReaderConfigurationData <T> readerConfiguration, IAdvancedScheduler listenerExecutor, ISchema <T> schema, ClientConfigurationData clientConfigurationData)
{
_generator = idGenerator;
var subscription = "reader-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10);
if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix))
{
subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription;
}
ConsumerConfigurationData <T> consumerConfiguration = new ConsumerConfigurationData <T>();
consumerConfiguration.TopicNames.Add(readerConfiguration.TopicName);
consumerConfiguration.SubscriptionName = subscription;
consumerConfiguration.SubscriptionType = SubType.Exclusive;
consumerConfiguration.SubscriptionMode = SubscriptionMode.NonDurable;
consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize;
consumerConfiguration.ReadCompacted = readerConfiguration.ReadCompacted;
// Reader doesn't need any batch receiving behaviours
// disable the batch receive timer for the ConsumerImpl instance wrapped by the ReaderImpl
consumerConfiguration.BatchReceivePolicy = _disabledBatchReceivePolicy;
if (readerConfiguration.StartMessageId != null)
{
consumerConfiguration.StartMessageId = (BatchMessageId)readerConfiguration.StartMessageId;
}
if (readerConfiguration.ReaderName != null)
{
consumerConfiguration.ConsumerName = readerConfiguration.ReaderName;
}
if (readerConfiguration.ResetIncludeHead)
{
consumerConfiguration.ResetIncludeHead = true;
}
if (readerConfiguration.ReaderListener != null)
{
var readerListener = readerConfiguration.ReaderListener;
consumerConfiguration.MessageListener = new MessageListenerAnonymousInnerClass(Self, readerListener);
}
consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction;
if (readerConfiguration.CryptoKeyReader != null)
{
consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader;
}
if (readerConfiguration.KeyHashRanges != null)
{
consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray());
}
int partitionIdx = TopicName.GetPartitionIndex(readerConfiguration.TopicName);
if (consumerConfiguration.ReceiverQueueSize == 0)
{
_consumer = Context.ActorOf(Props.Create(() => new ZeroQueueConsumer <T>(consumerId, stateActor, client, lookup, cnxPool, _generator, readerConfiguration.TopicName, consumerConfiguration, listenerExecutor, partitionIdx, false, readerConfiguration.StartMessageId, schema, true, clientConfigurationData)));
}
else
{
_consumer = Context.ActorOf(Props.Create(() => new ConsumerActor <T>(consumerId, stateActor, client, lookup, cnxPool, _generator, readerConfiguration.TopicName, consumerConfiguration, listenerExecutor, partitionIdx, false, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, schema, true, clientConfigurationData)));
}
Receive <HasReachedEndOfTopic>(m => {
_consumer.Tell(m, Sender);
});
Receive <AcknowledgeCumulativeMessage <T> > (m => {
_consumer.Tell(m, Sender);
});
Receive <Messages.Consumer.Receive> (m => {
_consumer.Tell(m, Sender);
});
Receive <Connect> (m => {
_consumer.Tell(m, Sender);
});
Receive <MessageProcessed <T> > (m => {
_consumer.Tell(m, Sender);
});
Receive <HasMessageAvailable> (m => {
_consumer.Tell(m, Sender);
});
Receive <GetTopic> (m => {
_consumer.Tell(m, Sender);
});
Receive <IsConnected> (m => {
_consumer.Tell(m, Sender);
});
Receive <SeekMessageId> (m => {
_consumer.Tell(m, Sender);
});
Receive <SeekTimestamp> (m => {
_consumer.Tell(m, Sender);
});
}
public static Props Prop(ClientConfigurationData client, ReaderConfigurationData <T> readerConfiguration, IActorRef clientActor, IActorRef lookup, IActorRef cnxPool, IActorRef generator, long fromOffset, long toOffset, bool isLive, Tag tag, ISchema <T> schema)
{
return(Props.Create(() => new PulsarTaggedSourceActor <T>(client, readerConfiguration, clientActor, lookup, cnxPool, generator, fromOffset, toOffset, isLive, tag, schema)));
}
public MultiTopicsReader(IActorRef state, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, ReaderConfigurationData <T> readerConfiguration, IAdvancedScheduler listenerExecutor, ISchema <T> schema, ClientConfigurationData clientConfigurationData)
{
_generator = idGenerator;
var subscription = "multiTopicsReader-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10);
if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix))
{
subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription;
}
var consumerConfiguration = new ConsumerConfigurationData <T>();
foreach (var topic in readerConfiguration.TopicNames)
{
consumerConfiguration.TopicNames.Add(topic);
}
consumerConfiguration.SubscriptionName = subscription;
consumerConfiguration.SubscriptionType = SubType.Exclusive;
consumerConfiguration.SubscriptionMode = SubscriptionMode.NonDurable;
consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize;
consumerConfiguration.ReadCompacted = readerConfiguration.ReadCompacted;
if (readerConfiguration.ReaderListener != null)
{
var readerListener = readerConfiguration.ReaderListener;
consumerConfiguration.MessageListener = new MessageListenerAnonymousInnerClass(Self, readerListener);
}
if (readerConfiguration.StartMessageId != null)
{
consumerConfiguration.StartMessageId = (BatchMessageId)readerConfiguration.StartMessageId;
}
if (readerConfiguration.ReaderName != null)
{
consumerConfiguration.ConsumerName = readerConfiguration.ReaderName;
}
if (readerConfiguration.ResetIncludeHead)
{
consumerConfiguration.ResetIncludeHead = true;
}
consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction;
if (readerConfiguration.CryptoKeyReader != null)
{
consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader;
}
if (readerConfiguration.KeyHashRanges != null)
{
consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray());
}
_consumer = Context.ActorOf(Props.Create(() => new MultiTopicsConsumer <T>(state, client, lookup, cnxPool, _generator, consumerConfiguration, listenerExecutor, schema, true, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, clientConfigurationData)));
ReceiveAsync <SubscribeAndCreateTopicsIfDoesNotExist>(async subs =>
{
_sender = Sender;
await SubscribeAndCreateTopics(subs);
});
ReceiveAsync <Subscribe>(async sub =>
{
_sender = Sender;
await SubscribeToTopic(sub);
});
Receive <HasReachedEndOfTopic>(m => {
_consumer.Tell(m, Sender);
});
Receive <AcknowledgeCumulativeMessage <T> >(m => {
_consumer.Tell(m, Sender);
});
Receive <MessageProcessed <T> >(m => {
_consumer.Tell(m, Sender);
});
Receive <Messages.Consumer.Receive>(m => {
_consumer.Tell(m, Sender);
});
Receive <HasMessageAvailable>(m => {
_consumer.Tell(m, Sender);
});
Receive <GetTopic>(m => {
_consumer.Tell(m, Sender);
});
Receive <IsConnected>(m => {
_consumer.Tell(m, Sender);
});
Receive <SeekMessageId>(m => {
_consumer.Tell(m, Sender);
});
Receive <SeekTimestamp>(m => {
_consumer.Tell(m, Sender);
});
ReceiveAny(m => {
_consumer.Tell(m, Sender);
});
}
