public PulsarSourceActor(ClientConfigurationData client, ReaderConfigurationData <T> readerConfiguration, IActorRef clientActor, IActorRef lookup, IActorRef cnxPool, IActorRef generator, long fromOffset, long toOffset, bool isLive, ISchema <T> schema) { _scheduler = Context.System.Scheduler.Advanced; _toOffset = toOffset; _parent = Context.Parent; _lastEventMessageOffset = fromOffset; var topicName = TopicName.Get(readerConfiguration.TopicName); IActorRef stateA = Context.ActorOf(Props.Create(() => new ConsumerStateActor()), $"StateActor{Guid.NewGuid()}"); var subscription = "player-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10); if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix)) { subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription; } ConsumerConfigurationData <T> consumerConfiguration = new ConsumerConfigurationData <T>(); consumerConfiguration.TopicNames.Add(readerConfiguration.TopicName); consumerConfiguration.SubscriptionName = subscription; consumerConfiguration.SubscriptionType = SubType.Exclusive; consumerConfiguration.SubscriptionMode = SubscriptionMode.NonDurable; consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize; consumerConfiguration.ReadCompacted = readerConfiguration.ReadCompacted; consumerConfiguration.StartMessageId = readerConfiguration.StartMessageId; if (readerConfiguration.ReaderName != null) { consumerConfiguration.ConsumerName = readerConfiguration.ReaderName; } if (readerConfiguration.ResetIncludeHead) { consumerConfiguration.ResetIncludeHead = true; } consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction; if (readerConfiguration.CryptoKeyReader != null) { consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader; } if (readerConfiguration.KeyHashRanges != null) { consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray()); } var partitionIdx = TopicName.GetPartitionIndex(readerConfiguration.TopicName); var consumerId = generator.Ask <long>(NewConsumerId.Instance).GetAwaiter().GetResult(); _child = Context.ActorOf(Props.Create(() => new ConsumerActor <T>(consumerId, stateA, clientActor, lookup, cnxPool, generator, readerConfiguration.TopicName, consumerConfiguration, Context.System.Scheduler.Advanced, partitionIdx, true, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, schema, true, client))); _child.Tell(Connect.Instance); if (isLive) { LiveConsume(); } else { Consume(); } }
public void TestExclusiveHashRangeInvalid() { var policy = KeySharedPolicy.StickyHashRange(); Assert.Throws <System.ArgumentException>(() => policy.Validate()); policy.GetRanges(Range.Of(0, 9), Range.Of(0, 5)); Assert.Throws <System.ArgumentException>(() => policy.Validate()); }
public void TestAutoSplit() { KeySharedPolicy policy = KeySharedPolicy.AutoSplitHashRange(); Assert.Equal(2 << 15, policy.HashRangeTotal); policy.Validate(); }
public void TestExclusiveHashRange() { var policy = KeySharedPolicy.StickyHashRange(); Assert.Equal(2 << 15, policy.HashRangeTotal); policy.GetRanges(Range.Of(0, 1), Range.Of(1, 2)); Assert.Equal(2, policy.Ranges.Count); }
private Consumer <byte[]> CreateConsumer(string topic, string consumerSub, KeySharedPolicy keySharedPolicy = null) { var builder = new ConsumerConfigBuilder <byte[]>(); builder.Topic(topic); builder.SubscriptionName(consumerSub); builder.AckTimeout(TimeSpan.FromSeconds(10)); builder.ForceTopicCreation(true); if (keySharedPolicy != null) { builder.KeySharedPolicy(keySharedPolicy); } builder.SubscriptionType(CommandSubscribe.SubType.KeyShared); return(_client.NewConsumer(builder)); }
public ReaderActor(long consumerId, IActorRef stateActor, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, ReaderConfigurationData <T> readerConfiguration, IAdvancedScheduler listenerExecutor, ISchema <T> schema, ClientConfigurationData clientConfigurationData) { _generator = idGenerator; var subscription = "reader-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10); if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix)) { subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription; } ConsumerConfigurationData <T> consumerConfiguration = new ConsumerConfigurationData <T>(); consumerConfiguration.TopicNames.Add(readerConfiguration.TopicName); consumerConfiguration.SubscriptionName = subscription; consumerConfiguration.SubscriptionType = SubType.Exclusive; consumerConfiguration.SubscriptionMode = SubscriptionMode.NonDurable; consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize; consumerConfiguration.ReadCompacted = readerConfiguration.ReadCompacted; // Reader doesn't need any batch receiving behaviours // disable the batch receive timer for the ConsumerImpl instance wrapped by the ReaderImpl consumerConfiguration.BatchReceivePolicy = _disabledBatchReceivePolicy; if (readerConfiguration.StartMessageId != null) { consumerConfiguration.StartMessageId = (BatchMessageId)readerConfiguration.StartMessageId; } if (readerConfiguration.ReaderName != null) { consumerConfiguration.ConsumerName = readerConfiguration.ReaderName; } if (readerConfiguration.ResetIncludeHead) { consumerConfiguration.ResetIncludeHead = true; } if (readerConfiguration.ReaderListener != null) { var readerListener = readerConfiguration.ReaderListener; consumerConfiguration.MessageListener = new MessageListenerAnonymousInnerClass(Self, readerListener); } consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction; if (readerConfiguration.CryptoKeyReader != null) { consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader; } if (readerConfiguration.KeyHashRanges != null) { consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray()); } int partitionIdx = TopicName.GetPartitionIndex(readerConfiguration.TopicName); if (consumerConfiguration.ReceiverQueueSize == 0) { _consumer = Context.ActorOf(Props.Create(() => new ZeroQueueConsumer <T>(consumerId, stateActor, client, lookup, cnxPool, _generator, readerConfiguration.TopicName, consumerConfiguration, listenerExecutor, partitionIdx, false, readerConfiguration.StartMessageId, schema, true, clientConfigurationData))); } else { _consumer = Context.ActorOf(Props.Create(() => new ConsumerActor <T>(consumerId, stateActor, client, lookup, cnxPool, _generator, readerConfiguration.TopicName, consumerConfiguration, listenerExecutor, partitionIdx, false, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, schema, true, clientConfigurationData))); } Receive <HasReachedEndOfTopic>(m => { _consumer.Tell(m, Sender); }); Receive <AcknowledgeCumulativeMessage <T> > (m => { _consumer.Tell(m, Sender); }); Receive <Messages.Consumer.Receive> (m => { _consumer.Tell(m, Sender); }); Receive <Connect> (m => { _consumer.Tell(m, Sender); }); Receive <MessageProcessed <T> > (m => { _consumer.Tell(m, Sender); }); Receive <HasMessageAvailable> (m => { _consumer.Tell(m, Sender); }); Receive <GetTopic> (m => { _consumer.Tell(m, Sender); }); Receive <IsConnected> (m => { _consumer.Tell(m, Sender); }); Receive <SeekMessageId> (m => { _consumer.Tell(m, Sender); }); Receive <SeekTimestamp> (m => { _consumer.Tell(m, Sender); }); }
public static ReadOnlySequence <byte> NewSubscribe(string topic, string subscription, long consumerId, long requestId, CommandSubscribe.SubType subType, int priorityLevel, string consumerName, bool isDurable, MessageIdData startMessageId, IDictionary <string, string> metadata, bool readCompacted, bool isReplicated, CommandSubscribe.InitialPosition subscriptionInitialPosition, long startMessageRollbackDurationInSec, ISchemaInfo schemaInfo, bool createTopicIfDoesNotExist, KeySharedPolicy keySharedPolicy) { var subscribe = new CommandSubscribe { Topic = topic, Subscription = subscription, subType = subType, ConsumerId = (ulong)consumerId, ConsumerName = consumerName, RequestId = (ulong)requestId, PriorityLevel = priorityLevel, Durable = isDurable, ReadCompacted = readCompacted, initialPosition = subscriptionInitialPosition, ReplicateSubscriptionState = isReplicated, ForceTopicCreation = createTopicIfDoesNotExist }; if (keySharedPolicy != null) { var keySharedMeta = new KeySharedMeta { allowOutOfOrderDelivery = keySharedPolicy.AllowOutOfOrderDelivery, keySharedMode = ConvertKeySharedMode(keySharedPolicy.KeySharedMode) }; if (keySharedPolicy is KeySharedPolicy.KeySharedPolicySticky sticky) { var ranges = sticky.GetRanges().Ranges; foreach (var range in ranges) { keySharedMeta.hashRanges.Add(new IntRange { Start = range.Start, End = range.End }); } } subscribe.keySharedMeta = keySharedMeta; } if (startMessageId != null) { subscribe.StartMessageId = startMessageId; } if (startMessageRollbackDurationInSec > 0) { subscribe.StartMessageRollbackDurationSec = (ulong)startMessageRollbackDurationInSec; } subscribe.Metadatas.AddRange(CommandUtils.ToKeyValueList(metadata)); if (schemaInfo != null) { var schema = GetSchema(schemaInfo); subscribe.Schema = schema; } return(Serializer.Serialize(subscribe.ToBaseCommand())); }
public MultiTopicsReader(IActorRef state, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, ReaderConfigurationData <T> readerConfiguration, IAdvancedScheduler listenerExecutor, ISchema <T> schema, ClientConfigurationData clientConfigurationData) { _generator = idGenerator; var subscription = "multiTopicsReader-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10); if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix)) { subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription; } var consumerConfiguration = new ConsumerConfigurationData <T>(); foreach (var topic in readerConfiguration.TopicNames) { consumerConfiguration.TopicNames.Add(topic); } consumerConfiguration.SubscriptionName = subscription; consumerConfiguration.SubscriptionType = SubType.Exclusive; consumerConfiguration.SubscriptionMode = SubscriptionMode.NonDurable; consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize; consumerConfiguration.ReadCompacted = readerConfiguration.ReadCompacted; if (readerConfiguration.ReaderListener != null) { var readerListener = readerConfiguration.ReaderListener; consumerConfiguration.MessageListener = new MessageListenerAnonymousInnerClass(Self, readerListener); } if (readerConfiguration.StartMessageId != null) { consumerConfiguration.StartMessageId = (BatchMessageId)readerConfiguration.StartMessageId; } if (readerConfiguration.ReaderName != null) { consumerConfiguration.ConsumerName = readerConfiguration.ReaderName; } if (readerConfiguration.ResetIncludeHead) { consumerConfiguration.ResetIncludeHead = true; } consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction; if (readerConfiguration.CryptoKeyReader != null) { consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader; } if (readerConfiguration.KeyHashRanges != null) { consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray()); } _consumer = Context.ActorOf(Props.Create(() => new MultiTopicsConsumer <T>(state, client, lookup, cnxPool, _generator, consumerConfiguration, listenerExecutor, schema, true, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, clientConfigurationData))); ReceiveAsync <SubscribeAndCreateTopicsIfDoesNotExist>(async subs => { _sender = Sender; await SubscribeAndCreateTopics(subs); }); ReceiveAsync <Subscribe>(async sub => { _sender = Sender; await SubscribeToTopic(sub); }); Receive <HasReachedEndOfTopic>(m => { _consumer.Tell(m, Sender); }); Receive <AcknowledgeCumulativeMessage <T> >(m => { _consumer.Tell(m, Sender); }); Receive <MessageProcessed <T> >(m => { _consumer.Tell(m, Sender); }); Receive <Messages.Consumer.Receive>(m => { _consumer.Tell(m, Sender); }); Receive <HasMessageAvailable>(m => { _consumer.Tell(m, Sender); }); Receive <GetTopic>(m => { _consumer.Tell(m, Sender); }); Receive <IsConnected>(m => { _consumer.Tell(m, Sender); }); Receive <SeekMessageId>(m => { _consumer.Tell(m, Sender); }); Receive <SeekTimestamp>(m => { _consumer.Tell(m, Sender); }); ReceiveAny(m => { _consumer.Tell(m, Sender); }); }
public ConsumerConfigBuilder <T> KeySharedPolicy(KeySharedPolicy keySharedPolicy) { keySharedPolicy.Validate(); _conf.KeySharedPolicy = keySharedPolicy; return(this); }