Example #1
0
        public PulsarSourceActor(ClientConfigurationData client, ReaderConfigurationData <T> readerConfiguration, IActorRef clientActor, IActorRef lookup, IActorRef cnxPool, IActorRef generator, long fromOffset, long toOffset, bool isLive, ISchema <T> schema)
        {
            _scheduler = Context.System.Scheduler.Advanced;
            _toOffset  = toOffset;
            _parent    = Context.Parent;
            _lastEventMessageOffset = fromOffset;
            var       topicName    = TopicName.Get(readerConfiguration.TopicName);
            IActorRef stateA       = Context.ActorOf(Props.Create(() => new ConsumerStateActor()), $"StateActor{Guid.NewGuid()}");
            var       subscription = "player-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10);

            if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix))
            {
                subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription;
            }

            ConsumerConfigurationData <T> consumerConfiguration = new ConsumerConfigurationData <T>();

            consumerConfiguration.TopicNames.Add(readerConfiguration.TopicName);
            consumerConfiguration.SubscriptionName  = subscription;
            consumerConfiguration.SubscriptionType  = SubType.Exclusive;
            consumerConfiguration.SubscriptionMode  = SubscriptionMode.NonDurable;
            consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize;
            consumerConfiguration.ReadCompacted     = readerConfiguration.ReadCompacted;
            consumerConfiguration.StartMessageId    = readerConfiguration.StartMessageId;

            if (readerConfiguration.ReaderName != null)
            {
                consumerConfiguration.ConsumerName = readerConfiguration.ReaderName;
            }

            if (readerConfiguration.ResetIncludeHead)
            {
                consumerConfiguration.ResetIncludeHead = true;
            }

            consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction;
            if (readerConfiguration.CryptoKeyReader != null)
            {
                consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader;
            }

            if (readerConfiguration.KeyHashRanges != null)
            {
                consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray());
            }

            var partitionIdx = TopicName.GetPartitionIndex(readerConfiguration.TopicName);
            var consumerId   = generator.Ask <long>(NewConsumerId.Instance).GetAwaiter().GetResult();

            _child = Context.ActorOf(Props.Create(() => new ConsumerActor <T>(consumerId, stateA, clientActor, lookup, cnxPool, generator, readerConfiguration.TopicName, consumerConfiguration, Context.System.Scheduler.Advanced, partitionIdx, true, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, schema, true, client)));
            _child.Tell(Connect.Instance);
            if (isLive)
            {
                LiveConsume();
            }
            else
            {
                Consume();
            }
        }
 protected BaseJournalDaoWithReadMessages(IAdvancedScheduler ec,
                                          IMaterializer mat, AkkaPersistenceDataConnectionFactory connectionFactory)
 {
     this.ec            = ec;
     this.mat           = mat;
     _connectionFactory = connectionFactory;
 }
Example #3
0
 public ByteArrayJournalDao(IAdvancedScheduler sched, IMaterializer mat,
                            AkkaPersistenceDataConnectionFactory connection,
                            JournalConfig journalConfig,
                            Akka.Serialization.Serialization serializer, ILoggingAdapter logger) : base(sched, mat,
                                                                                                        connection, journalConfig,
                                                                                                        new ByteArrayJournalSerializer(journalConfig, serializer,
                                                                                                                                       journalConfig.PluginConfig.TagSeparator), logger)
 {
 }
Example #4
0
        public ConsumerActorBase(IActorRef stateActor, IActorRef lookup, IActorRef connectionPool, string topic, ConsumerConfigurationData <T> conf, int receiverQueueSize, IAdvancedScheduler listenerExecutor, ISchema <T> schema)
        {
            if (conf.Interceptors != null && conf.Interceptors.Count > 0)
            {
                Interceptors = new ConsumerInterceptors <T>(Context.System, conf.Interceptors);
            }

            StateActor            = stateActor;
            _topic                = topic;
            _consumerName         = conf.ConsumerName ?? Utility.ConsumerName.GenerateRandomName();
            State                 = new HandlerState(lookup, connectionPool, topic, Context.System, _consumerName);
            _log                  = Context.GetLogger();
            MaxReceiverQueueSize  = receiverQueueSize;
            _subscription         = conf.SubscriptionName;
            Conf                  = conf;
            Listener              = conf.MessageListener;
            ConsumerEventListener = conf.ConsumerEventListener;

            IncomingMessages = new BufferBlock <IMessage <T> >();
            UnAckedChunckedMessageIdSequenceMap = Context.ActorOf(Tracker.UnAckedChunckedMessageIdSequenceMap.Prop());

            ListenerExecutor = listenerExecutor;
            Schema           = schema;

            if (conf.BatchReceivePolicy != null)
            {
                var userBatchReceivePolicy = conf.BatchReceivePolicy;
                if (userBatchReceivePolicy.MaxNumMessages > MaxReceiverQueueSize)
                {
                    BatchReceivePolicy = new BatchReceivePolicy.Builder().MaxNumMessages(MaxReceiverQueueSize).MaxNumBytes(userBatchReceivePolicy.MaxNumBytes).Timeout((int)userBatchReceivePolicy.TimeoutMs).Build();
                    _log.Warning($"BatchReceivePolicy maxNumMessages: {userBatchReceivePolicy.MaxNumMessages} is greater than maxReceiverQueueSize: {MaxReceiverQueueSize}, reset to maxReceiverQueueSize. batchReceivePolicy: {BatchReceivePolicy}");
                }
                else if (userBatchReceivePolicy.MaxNumMessages <= 0 && userBatchReceivePolicy.MaxNumBytes <= 0)
                {
                    BatchReceivePolicy = new BatchReceivePolicy.Builder().MaxNumMessages(BatchReceivePolicy.DefaultPolicy.MaxNumMessages).MaxNumBytes(BatchReceivePolicy.DefaultPolicy.MaxNumBytes).Timeout((int)userBatchReceivePolicy.TimeoutMs).Build();
                    _log.Warning("BatchReceivePolicy maxNumMessages: {} or maxNumBytes: {} is less than 0. " + "Reset to DEFAULT_POLICY. batchReceivePolicy: {}", userBatchReceivePolicy.MaxNumMessages, userBatchReceivePolicy.MaxNumBytes, BatchReceivePolicy.ToString());
                }
                else
                {
                    BatchReceivePolicy = conf.BatchReceivePolicy;
                }
            }
            else
            {
                BatchReceivePolicy = BatchReceivePolicy.DefaultPolicy;
            }

            if (BatchReceivePolicy.TimeoutMs > 0)
            {
                //BatchReceiveTimeout = ListenerExecutor.ScheduleOnceCancelable(TimeSpan.FromMilliseconds(TimeUnit.MILLISECONDS.ToMilliseconds(BatchReceivePolicy.TimeoutMs)), PendingBatchReceiveTask);
            }
            _stateUpdater = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5), Self, SendState.Instance, ActorRefs.NoSender);
        }
Example #5
0
 public PrestoSourceActor(BufferBlock <IEventEnvelope> buffer, bool isLive, IPrestoEventSourceMessage message)
 {
     _buffer             = buffer;
     _self               = Self;
     _log                = Context.GetLogger();
     _scheduler          = Context.System.Scheduler.Advanced;
     _topicName          = TopicName.Get(message.Topic);
     _message            = message;
     _toMessageId        = message.ToMessageId;
     _lastEventMessageId = _message.FromMessageId;
     _partitionIndex     = ((MessageId)MessageIdUtils.GetMessageId(message.FromMessageId)).PartitionIndex;
     _queryRange         = message.ToMessageId - message.FromMessageId;
     FirstQuery(isLive);
 }
Example #6
0
        public ReaderActor(long consumerId, IActorRef stateActor, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, ReaderConfigurationData <T> readerConfiguration, IAdvancedScheduler listenerExecutor, ISchema <T> schema, ClientConfigurationData clientConfigurationData)
        {
            _generator = idGenerator;
            var subscription = "reader-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10);

            if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix))
            {
                subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription;
            }

            ConsumerConfigurationData <T> consumerConfiguration = new ConsumerConfigurationData <T>();

            consumerConfiguration.TopicNames.Add(readerConfiguration.TopicName);
            consumerConfiguration.SubscriptionName  = subscription;
            consumerConfiguration.SubscriptionType  = SubType.Exclusive;
            consumerConfiguration.SubscriptionMode  = SubscriptionMode.NonDurable;
            consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize;
            consumerConfiguration.ReadCompacted     = readerConfiguration.ReadCompacted;

            // Reader doesn't need any batch receiving behaviours
            // disable the batch receive timer for the ConsumerImpl instance wrapped by the ReaderImpl
            consumerConfiguration.BatchReceivePolicy = _disabledBatchReceivePolicy;

            if (readerConfiguration.StartMessageId != null)
            {
                consumerConfiguration.StartMessageId = (BatchMessageId)readerConfiguration.StartMessageId;
            }


            if (readerConfiguration.ReaderName != null)
            {
                consumerConfiguration.ConsumerName = readerConfiguration.ReaderName;
            }

            if (readerConfiguration.ResetIncludeHead)
            {
                consumerConfiguration.ResetIncludeHead = true;
            }

            if (readerConfiguration.ReaderListener != null)
            {
                var readerListener = readerConfiguration.ReaderListener;
                consumerConfiguration.MessageListener = new MessageListenerAnonymousInnerClass(Self, readerListener);
            }

            consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction;
            if (readerConfiguration.CryptoKeyReader != null)
            {
                consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader;
            }

            if (readerConfiguration.KeyHashRanges != null)
            {
                consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray());
            }

            int partitionIdx = TopicName.GetPartitionIndex(readerConfiguration.TopicName);

            if (consumerConfiguration.ReceiverQueueSize == 0)
            {
                _consumer = Context.ActorOf(Props.Create(() => new ZeroQueueConsumer <T>(consumerId, stateActor, client, lookup, cnxPool, _generator, readerConfiguration.TopicName, consumerConfiguration, listenerExecutor, partitionIdx, false, readerConfiguration.StartMessageId, schema, true, clientConfigurationData)));
            }
            else
            {
                _consumer = Context.ActorOf(Props.Create(() => new ConsumerActor <T>(consumerId, stateActor, client, lookup, cnxPool, _generator, readerConfiguration.TopicName, consumerConfiguration, listenerExecutor, partitionIdx, false, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, schema, true, clientConfigurationData)));
            }
            Receive <HasReachedEndOfTopic>(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <AcknowledgeCumulativeMessage <T> > (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <Messages.Consumer.Receive> (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <Connect> (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <MessageProcessed <T> > (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <HasMessageAvailable> (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <GetTopic> (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <IsConnected> (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <SeekMessageId> (m => {
                _consumer.Tell(m, Sender);
            });
            Receive <SeekTimestamp> (m => {
                _consumer.Tell(m, Sender);
            });
        }
Example #7
0
        public MultiTopicsReader(IActorRef state, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, ReaderConfigurationData <T> readerConfiguration, IAdvancedScheduler listenerExecutor, ISchema <T> schema, ClientConfigurationData clientConfigurationData)
        {
            _generator = idGenerator;
            var subscription = "multiTopicsReader-" + ConsumerName.Sha1Hex(Guid.NewGuid().ToString()).Substring(0, 10);

            if (!string.IsNullOrWhiteSpace(readerConfiguration.SubscriptionRolePrefix))
            {
                subscription = readerConfiguration.SubscriptionRolePrefix + "-" + subscription;
            }
            var consumerConfiguration = new ConsumerConfigurationData <T>();

            foreach (var topic in readerConfiguration.TopicNames)
            {
                consumerConfiguration.TopicNames.Add(topic);
            }

            consumerConfiguration.SubscriptionName  = subscription;
            consumerConfiguration.SubscriptionType  = SubType.Exclusive;
            consumerConfiguration.SubscriptionMode  = SubscriptionMode.NonDurable;
            consumerConfiguration.ReceiverQueueSize = readerConfiguration.ReceiverQueueSize;
            consumerConfiguration.ReadCompacted     = readerConfiguration.ReadCompacted;

            if (readerConfiguration.ReaderListener != null)
            {
                var readerListener = readerConfiguration.ReaderListener;
                consumerConfiguration.MessageListener = new MessageListenerAnonymousInnerClass(Self, readerListener);
            }
            if (readerConfiguration.StartMessageId != null)
            {
                consumerConfiguration.StartMessageId = (BatchMessageId)readerConfiguration.StartMessageId;
            }

            if (readerConfiguration.ReaderName != null)
            {
                consumerConfiguration.ConsumerName = readerConfiguration.ReaderName;
            }
            if (readerConfiguration.ResetIncludeHead)
            {
                consumerConfiguration.ResetIncludeHead = true;
            }
            consumerConfiguration.CryptoFailureAction = readerConfiguration.CryptoFailureAction;
            if (readerConfiguration.CryptoKeyReader != null)
            {
                consumerConfiguration.CryptoKeyReader = readerConfiguration.CryptoKeyReader;
            }
            if (readerConfiguration.KeyHashRanges != null)
            {
                consumerConfiguration.KeySharedPolicy = KeySharedPolicy.StickyHashRange().GetRanges(readerConfiguration.KeyHashRanges.ToArray());
            }
            _consumer = Context.ActorOf(Props.Create(() => new MultiTopicsConsumer <T>(state, client, lookup, cnxPool, _generator, consumerConfiguration, listenerExecutor, schema, true, readerConfiguration.StartMessageId, readerConfiguration.StartMessageFromRollbackDurationInSec, clientConfigurationData)));

            ReceiveAsync <SubscribeAndCreateTopicsIfDoesNotExist>(async subs =>
            {
                _sender = Sender;
                await SubscribeAndCreateTopics(subs);
            });
            ReceiveAsync <Subscribe>(async sub =>
            {
                _sender = Sender;
                await SubscribeToTopic(sub);
            });
            Receive <HasReachedEndOfTopic>(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <AcknowledgeCumulativeMessage <T> >(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <MessageProcessed <T> >(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <Messages.Consumer.Receive>(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <HasMessageAvailable>(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <GetTopic>(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <IsConnected>(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <SeekMessageId>(m => {
                _consumer.Tell(m, Sender);
            });
            Receive <SeekTimestamp>(m => {
                _consumer.Tell(m, Sender);
            });
            ReceiveAny(m => {
                _consumer.Tell(m, Sender);
            });
        }
Example #8
0
 internal ZeroQueueConsumer(long consumerId, IActorRef stateActor, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef idGenerator, string topic, ConsumerConfigurationData <T> conf, IAdvancedScheduler listenerExecutor, int partitionIndex, bool hasParentConsumer, IMessageId startMessageId, ISchema <T> schema, bool createTopicIfDoesNotExist, ClientConfigurationData clientConfiguration) : base(consumerId, stateActor, client, lookup, cnxPool, idGenerator, topic, conf, listenerExecutor, partitionIndex, hasParentConsumer, startMessageId, 0, schema, createTopicIfDoesNotExist, clientConfiguration)
 {
 }
 public ByteArrayReadJournalDao(IAdvancedScheduler ec, IMaterializer mat, AkkaPersistenceDataConnectionFactory connectionFactory, ReadJournalConfig readJournalConfig, FlowPersistentReprSerializer <JournalRow> serializer) : base(ec, mat, connectionFactory, readJournalConfig, serializer)
 {
 }