/// <summary>
/// Subscription is performed asynchronously.
/// </summary>
/// <param name="seedBrokers">Comma separated list of seed brokers. Port numbers are optional.
/// <example>192.168.56.10,192.168.56.20:8081,broker3.local.net:8181</example>
/// </param>
/// <param name="topic"></param>
/// <param name="startPosition"></param>
/// <param name="maxWaitTimeMs"></param>
/// <param name="minBytesPerFetch"></param>
/// <param name="maxBytesPerFetch"></param>
/// <param name="highWatermark"></param>
/// <param name="useFlowControl">
/// If set to true, subscriber must call consumers Ack function to keep data flowing.
/// Is used to prevent out of memory errors when subscriber is slow and data velocity is high
/// (re-reading the log from beginning for example).
/// Make sure that subscriber consumes more than lowWatermark, or data might stop flowing because driver would
/// wait for subscriber to drain until lowWatermark and subscriber would wait for more data until continue processing
/// (could happen when buffering is used).
/// </param>
/// <param name="lowWatermark"></param>
/// <param name="stopPosition"></param>
/// <param name="scheduler">Driver will schedule outgoing messages events using this scheduler. By default it is
/// EventLoopScheduler. Be careful if you want to redefine it. Concurrent scheduler can rearrange order of messages
/// within the same partition!</param>
public ConsumerConfiguration(
string seedBrokers,
string topic,
IStartPositionProvider startPosition,
int maxWaitTimeMs = 500,
int minBytesPerFetch = 1,
int maxBytesPerFetch = 256 *1024,
int lowWatermark = 500,
int highWatermark = 2000,
bool useFlowControl = false,
IStopPositionProvider stopPosition = null,
IScheduler scheduler = null)
{
LowWatermark = lowWatermark;
HighWatermark = highWatermark;
UseFlowControl = useFlowControl;
if (lowWatermark < 0)
{
throw new ArgumentException("Can not be negative", "lowWatermark");
}
if (highWatermark < 0)
{
throw new ArgumentException("Can not be negative", "highWatermark");
}
if (highWatermark < lowWatermark)
{
throw new InvalidOperationException("highWatermark must be greater than lowWatermark");
}
SeedBrokers = seedBrokers;
StartPosition = startPosition;
Topic = topic;
MaxWaitTimeMs = maxWaitTimeMs;
MinBytesPerFetch = minBytesPerFetch;
MaxBytesPerFetch = maxBytesPerFetch;
StopPosition = stopPosition ?? new StopPositionNever();
OutgoingScheduler = scheduler ?? new EventLoopScheduler(ts => new Thread(ts)
{
IsBackground = true
});
}
/// <summary>For every patition, resolve offsets and build TopicPartition object</summary>
private async Task <IEnumerable <TopicPartition> > BuildTopicPartitionsAsync()
{
// if they didn't specify explicit locations, initialize them here.
var startPositionProvider = Configuration.StartPosition;
if (startPositionProvider.StartLocation != ConsumerLocation.SpecifiedLocations)
{
// no offsets provided. Need to issue an offset request to get start/end locations and use them for consuming
var partitions = await _cluster.FetchPartitionOffsetsAsync(Topic, startPositionProvider.StartLocation);
if (_log.IsDebugEnabled)
{
_log.Debug("Consumer for topic {0} got time->offset resolved for location {1}. parts: [{2}]",
Topic, startPositionProvider,
string.Join(",", partitions.Partitions.OrderBy(p => p).Select(p => string.Format("{0}:{1}", p, partitions.NextOffset(p)))));
}
IStartPositionProvider origProvider = startPositionProvider;
// the new explicit offsets provider should use only the partitions included in the original one.
startPositionProvider = new TopicPartitionOffsets(partitions.Topic, partitions.GetPartitionsOffset.Where(kv => origProvider.ShouldConsumePartition(kv.Key)));
}
// we now have specified locations to start from, just get the partition metadata, and build the TopicPartitions
var partitionMeta = await _cluster.GetOrFetchMetaForTopicAsync(Topic);
return(partitionMeta
// only new partitions we don't already have in our dictionary
.Where(pm => !_topicPartitions.ContainsKey(pm.Id))
// only partitions we are "told" to.
.Where(pm => startPositionProvider.ShouldConsumePartition(pm.Id))
.Select(part =>
{
var tp = new TopicPartition(_cluster, Topic, part.Id, startPositionProvider.GetStartOffset(part.Id));
_topicPartitions.Add(tp.PartitionId, tp);
return tp;
}));
}
/// <summary>
/// Subscription is performed asynchronously.
/// </summary>
/// <param name="seedBrokers">Comma separated list of seed brokers. Port numbers are optional.
/// <example>192.168.56.10,192.168.56.20:8081,broker3.local.net:8181</example>
/// </param>
/// <param name="topic"></param>
/// <param name="startPosition"></param>
/// <param name="maxWaitTimeMs"></param>
/// <param name="minBytesPerFetch"></param>
/// <param name="maxBytesPerFetch"></param>
/// <param name="highWatermark"></param>
/// <param name="useFlowControl">
/// If set to true, subscriber must call consumers Ack function to keep data flowing.
/// Is used to prevent out of memory errors when subscriber is slow and data velocity is high
/// (re-reading the log from beginning for example).
/// Make sure that subscriber consumes more than lowWatermark, or data might stop flowing because driver would
/// wait for subscriber to drain until lowWatermark and subscriber would wait for more data until continue processing
/// (could happen when buffering is used).
/// </param>
/// <param name="lowWatermark"></param>
/// <param name="stopPosition"></param>
/// <param name="scheduler">Driver will schedule outgoing messages events using this scheduler. By default it is
/// EventLoopScheduler. Be careful if you want to redefine it. Concurrent scheduler can rearrange order of messages
/// within the same partition!</param>
public ConsumerConfiguration(
string seedBrokers,
string topic,
IStartPositionProvider startPosition,
int maxWaitTimeMs=500,
int minBytesPerFetch = 1,
int maxBytesPerFetch=256*1024,
int lowWatermark = 500,
int highWatermark = 2000,
bool useFlowControl = false,
IStopPositionProvider stopPosition = null,
IScheduler scheduler = null)
{
LowWatermark = lowWatermark;
HighWatermark = highWatermark;
UseFlowControl = useFlowControl;
if(lowWatermark < 0)
throw new ArgumentException("Can not be negative", "lowWatermark");
if (highWatermark < 0)
throw new ArgumentException("Can not be negative", "highWatermark");
if(highWatermark < lowWatermark)
throw new InvalidOperationException("highWatermark must be greater than lowWatermark");
SeedBrokers = seedBrokers;
StartPosition = startPosition;
Topic = topic;
MaxWaitTimeMs = maxWaitTimeMs;
MinBytesPerFetch = minBytesPerFetch;
MaxBytesPerFetch = maxBytesPerFetch;
StopPosition = stopPosition ?? new StopPositionNever();
OutgoingScheduler = scheduler ?? new EventLoopScheduler(ts => new Thread(ts) { IsBackground = true});
}
