Example #1
0
 /// <summary>
 /// Construct a Producer class.
 /// </summary>
 /// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
 /// <param name="maximumAsyncQueue">The maximum async calls allowed before blocking new requests.  -1 indicates unlimited.</param>
 /// <remarks>
 /// The maximumAsyncQueue parameter provides a mechanism for blocking an async request return if the amount of requests queue is 
 /// over a certain limit.  This is usefull if a client is trying to push a large stream of documents through the producer and
 /// wants to block downstream if the producer is overloaded.
 /// 
 /// A message will start its timeout countdown as soon as it is added to the producer async queue.  If there are a large number of 
 /// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
 /// attempted to send to Kafka before a timeout exception is thrown.
 /// </remarks>
 public Producer(IBrokerRouter brokerRouter, int maximumAsyncQueue = -1)
 {
     _router = brokerRouter;
     _metadataQueries = new MetadataQueries(_router);
     _maximumAsyncQueue = maximumAsyncQueue == -1 ? int.MaxValue : maximumAsyncQueue;
     _sendSemaphore = new SemaphoreSlim(_maximumAsyncQueue, _maximumAsyncQueue);
 }
Example #2
0
        public Consumer(ConsumerOptions options, params OffsetPosition[] positions)
        {
            _options = options;
            _fetchResponseQueue = new BlockingCollection<Message>(_options.ConsumerBufferSize);
            _metadataQueries = new MetadataQueries(_options.Router);

            SetOffsetPosition(positions);
        }
Example #3
0
        public Consumer(ConsumerOptions options, params OffsetPosition[] positions)
        {
            _options = options;
            _fetchResponseQueue = new BlockingCollection<Message>(_options.ConsumerBufferSize);
            _metadataQueries = new MetadataQueries(_options.Router);

            //TODO this is wrong, we should only query once and then react only to errors or socket exceptions
            //this timer will periodically look for new partitions and automatically add them to the consuming queue
            //using the same whitelist logic
            _topicPartitionQueryTimer = new ScheduledTimer()
                .Do(RefreshTopicPartitions)
                .Every(TimeSpan.FromMilliseconds(_options.TopicPartitionQueryTimeMs))
                .StartingAt(DateTime.Now);

            SetOffsetPosition(positions);
        }
Example #4
0
        /// <summary>
        /// Construct a Producer class.
        /// </summary>
        /// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
        /// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests.  -1 indicates unlimited.</param>
        /// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
        /// <remarks>
        /// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
        /// by blocking the caller requesting the async call.  This affectively puts an upper limit on the amount of times a caller can 
        /// call SendMessageAsync before the caller is blocked.
        /// 
        /// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
        /// overwhelmed and the buffer starts to fill up.  This is an inaccurate limiting memory use as the amount of memory actually used is 
        /// dependant on the general message size being buffered.
        /// 
        /// A message will start its timeout countdown as soon as it is added to the producer async queue.  If there are a large number of 
        /// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
        /// attempted to send to Kafka before a timeout exception is thrown.
        /// </remarks>
        public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
        {
            BrokerRouter = brokerRouter;
            _maximumAsyncRequests = maximumAsyncRequests;
            _metadataQueries = new MetadataQueries(BrokerRouter);
            _nagleBlockingCollection = new NagleBlockingCollection<TopicMessage>(maximumMessageBuffer);
            _semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);

            BatchSize = DefaultBatchSize;
            BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);

            _postTask = Task.Run(async () =>
            {
                await BatchSendAsync().ConfigureAwait(false);
                //TODO add log for ending the sending thread.
            });

        }
        /// <summary>
        /// Construct a Producer class.
        /// </summary>
        /// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
        /// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests.  -1 indicates unlimited.</param>
        /// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
        /// <remarks>
        /// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
        /// by blocking the caller requesting the async call.  This affectively puts an upper limit on the amount of times a caller can
        /// call SendMessageAsync before the caller is blocked.
        ///
        /// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
        /// overwhelmed and the buffer starts to fill up.  This is an inaccurate limiting memory use as the amount of memory actually used is
        /// dependant on the general message size being buffered.
        ///
        /// A message will start its timeout countdown as soon as it is added to the producer async queue.  If there are a large number of
        /// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
        /// attempted to send to Kafka before a timeout exception is thrown.
        /// </remarks>
        public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
        {
            BrokerRouter = brokerRouter;
            _protocolGateway = new ProtocolGateway(BrokerRouter);
            _maximumAsyncRequests = maximumAsyncRequests;
            _metadataQueries = new MetadataQueries(BrokerRouter);
            _asyncCollection = new AsyncCollection<TopicMessage>();
            _semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);

            BatchSize = DefaultBatchSize;
            BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);

            _postTask = Task.Run(() =>
            {
                BatchSendAsync();
                BrokerRouter.Log.InfoFormat("ending the sending thread");
            });
        }