/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncQueue">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <remarks>
/// The maximumAsyncQueue parameter provides a mechanism for blocking an async request return if the amount of requests queue is
/// over a certain limit. This is usefull if a client is trying to push a large stream of documents through the producer and
/// wants to block downstream if the producer is overloaded.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncQueue = -1)
{
_router = brokerRouter;
_metadataQueries = new MetadataQueries(_router);
_maximumAsyncQueue = maximumAsyncQueue == -1 ? int.MaxValue : maximumAsyncQueue;
_sendSemaphore = new SemaphoreSlim(_maximumAsyncQueue, _maximumAsyncQueue);
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncQueue">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <remarks>
/// The maximumAsyncQueue parameter provides a mechanism for blocking an async request return if the amount of requests queue is
/// over a certain limit. This is usefull if a client is trying to push a large stream of documents through the producer and
/// wants to block downstream if the producer is overloaded.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncQueue = -1)
{
_router = brokerRouter;
_metadataQueries = new MetadataQueries(_router);
_maximumAsyncQueue = maximumAsyncQueue == -1 ? int.MaxValue : maximumAsyncQueue;
_sendSemaphore = new SemaphoreSlim(_maximumAsyncQueue, _maximumAsyncQueue);
}
internal protected ConsumerBase(ConsumerOptions options, params OffsetPosition[] positions)
{
_options = options;
_metadataQueries = new MetadataQueries(_options.Router);
SetOffsetPosition(positions);
}
public Consumer(ConsumerOptions options, params OffsetPosition[] positions)
{
_options = options;
_fetchResponseQueue = new BlockingCollection<Message>(_options.ConsumerBufferSize);
_metadataQueries = new MetadataQueries(_options.Router);
SetOffsetPosition(positions);
}
public Consumer(ConsumerOptions options, params OffsetPosition[] positions)
{
_options = options;
_fetchResponseQueue = new BlockingCollection <Message>(_options.ConsumerBufferSize);
_metadataQueries = new MetadataQueries(_options.Router);
_disposeTask = new TaskCompletionSource <int>();
SetOffsetPosition(positions);
}
public Consumer(ConsumerOptions options, params OffsetPosition[] positions)
{
_options = options;
_fetchResponseQueue = new BlockingCollection <Message>(_options.ConsumerBufferSize);
_metadataQueries = new MetadataQueries(_options.Router);
//TODO this is wrong, we should only query once and then react only to errors or socket exceptions
//this timer will periodically look for new partitions and automatically add them to the consuming queue
//using the same whitelist logic
_topicPartitionQueryTimer = new ScheduledTimer()
.Do(RefreshTopicPartitions)
.Every(TimeSpan.FromMilliseconds(_options.TopicPartitionQueryTimeMs))
.StartingAt(DateTime.Now);
SetOffsetPosition(positions);
}
public Consumer(ConsumerOptions options, params OffsetPosition[] positions)
{
_options = options;
_fetchResponseQueue = new BlockingCollection<Message>(_options.ConsumerBufferSize);
_metadataQueries = new MetadataQueries(_options.Router);
//TODO this is wrong, we should only query once and then react only to errors or socket exceptions
//this timer will periodically look for new partitions and automatically add them to the consuming queue
//using the same whitelist logic
_topicPartitionQueryTimer = new ScheduledTimer()
.Do(RefreshTopicPartitions)
.Every(TimeSpan.FromMilliseconds(_options.TopicPartitionQueryTimeMs))
.StartingAt(DateTime.Now);
SetOffsetPosition(positions);
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_asyncCollection = new AsyncCollection <TopicMessage>();
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(async() =>
{
await BatchSendAsync().ConfigureAwait(false);
//TODO add log for ending the sending thread.
});
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_protocolGateway = new ProtocolGateway(BrokerRouter);
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_asyncCollection = new AsyncCollection <TopicMessage>();
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(() =>
{
BatchSendAsync();
BrokerRouter.Log.InfoFormat("ending the sending thread");
});
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_nagleBlockingCollection = new NagleBlockingCollection<TopicMessage>(maximumMessageBuffer);
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(async () =>
{
await BatchSendAsync().ConfigureAwait(false);
//TODO add log for ending the sending thread.
});
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_protocolGateway = new ProtocolGateway(BrokerRouter);
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_asyncCollection = new AsyncCollection<TopicMessage>();
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(() =>
{
BatchSendAsync();
BrokerRouter.Log.InfoFormat("ending the sending thread");
});
}