/// <summary>
/// Gets an observable sequence of any changes to the Fetcher for a specific topic/partition.
/// </summary>
/// <param name="topic"></param>
/// <param name="partitionId"></param>
/// <param name="consumerConfiguration"></param>
/// <returns></returns>
internal IObservable <Fetcher> GetFetcherChanges(string topic, int partitionId, ConsumerConfiguration consumerConfiguration)
{
return(PartitionStateChanges
.Where(t => t.Topic == topic && t.PartitionId == partitionId)
.Select(state =>
{
// if the state is not ready, return NULL for the fetcher.
if (!state.ErrorCode.IsSuccess())
{
return (Fetcher)null;
}
// get or create the correct Fetcher for this topic/partition
var broker = FindBrokerMetaForPartitionId(state.Topic, state.PartitionId);
var fetcher = _activeFetchers.GetOrAdd(broker, b =>
{
// all new fetchers need to be "watched" for errors.
var f = new Fetcher(this, b, _protocol, consumerConfiguration, _cancel.Token);
// subscribe to error and complete notifications, and remove from active fetchers
f.ReceivedMessages.Subscribe(_ => { },
err =>
{
_log.Warn("Received error from fetcher {0}. Removing from active fetchers.", f);
Fetcher ef;
_activeFetchers.TryRemove(broker, out ef);
},
() =>
{
_log.Info("Received complete from fetcher {0}. Removing from active fetchers.", f);
Fetcher ef;
_activeFetchers.TryRemove(broker, out ef);
});
return f;
});
return fetcher;
})
.Do(f => _log.Debug("GetFetcherChanges returning {1} fetcher {0}", f, f == null ? "null" : "new"),
ex => _log.Error(ex, "GetFetcherChages saw ERROR returning new fetcher."),
() => _log.Error("GetFetcherChanges saw COMPLETE from returning new fetcher.")));
}
public async void SpeedTest()
{
int _pageViewBatchSize = 10 * 1000;
TimeSpan _batchTime = TimeSpan.FromSeconds(10);
var conf = new ConsumerConfiguration("kafkadev-01.lv.ntent.com",
"vsw.avrodto.addelivery.activitylogging.pageview",
new StartPositionTopicStart(),
useFlowControl: true,
highWatermark: _pageViewBatchSize * 5,
lowWatermark: _pageViewBatchSize * 2
);
var consumer = new Consumer(conf);
var count = 0L;
var lastCount = count;
var msg = consumer.OnMessageArrived.Publish().RefCount();
msg.
Buffer(_batchTime, _pageViewBatchSize).
Subscribe(_ =>
{
Interlocked.Add(ref count, _.Count);
consumer.Ack(_.Count);
//consumer.Ack();
});
var interval = TimeSpan.FromSeconds(5);
var lastTime = DateTime.Now;
Observable.Interval(interval).Subscribe(_ => {
var c = count;
var now = DateTime.Now;
var time = now - lastTime;
var speed = (c - lastCount)/time.TotalSeconds;
lastCount = c;
lastTime = now;
Console.WriteLine("{0}msg/sec #{1}", speed, c);
});
await consumer.IsConnected;
await msg.Timeout(TimeSpan.FromSeconds(30));
Console.WriteLine("Complete {0}", count);
}
/// <summary>
/// Create a new consumer using the specified configuration. See @ConsumerConfiguration
/// </summary>
/// <param name="consumerConfig"></param>
public Consumer(ConsumerConfiguration consumerConfig)
{
Configuration = consumerConfig;
_cluster = new Cluster(consumerConfig.SeedBrokers);
_cluster.OnThreadHang += e => OnMessageArrivedInput.OnError(e);
// Low/high watermark implementation
FlowControl = _flowControlInput.
Scan(1, (last, current) =>
{
if (current < Configuration.LowWatermark)
return 1;
if (current > Configuration.HighWatermark)
return -1;
return last; // While in between watermarks, carry over previous on/off state
}).
DistinctUntilChanged().
Select(i => i > 0).
Do(f =>FlowControlEnabled = f).
Do(f => EtwTrace.Log.ConsumerFlowControl(f ? 1 : 0));
var onMessage = Observable.Create<ReceivedMessage>(observer =>
{
if (Interlocked.CompareExchange(ref _haveSubscriber, 1, 0) == 1)
throw new InvalidOperationException("Only one subscriber is allowed. Use OnMessageArrived.Publish().RefCount()");
// Relay messages from partition to consumer's output
// Ensure that only single subscriber is allowed because it is important to count
// outstanding messaged consumed by user
OnMessageArrivedInput.Subscribe(observer);
//
// It is not possible to wait for completion of partition resolution process, so start it asynchronously.
// This means that OnMessageArrived.Subscribe() will complete when consumer is not actually connected yet.
//
Task.Run(async () =>
{
try
{
await _cluster.ConnectAsync();
await SubscribeClient();
EtwTrace.Log.ConsumerStarted(GetHashCode(), Topic);
_connectionComplete.TrySetResult(true);
// check that we actually got any partitions subscribed
if (_topicPartitions.Count == 0)
OnMessageArrivedInput.OnCompleted();
}
catch (Exception e)
{
_connectionComplete.TrySetException(e);
OnMessageArrivedInput.OnError(e);
}
});
// upon unsubscribe
return Disposable.Create(() => _partitionsSubscription.Values.ForEach(s=>s.Dispose()));
});
if (Configuration.UseFlowControl) {
// Increment counter of messages sent for processing
onMessage = onMessage.Do(msg =>
{
var count = Interlocked.Increment(ref _outstandingMessageProcessingCount);
_flowControlInput.OnNext(count);
});
}
// handle stop condition
onMessage = onMessage.Do(message =>
{
// check if this partition is done per the condition passed in configuration. If so, unsubscribe it.
bool partitionDone = (Configuration.StopPosition.IsPartitionConsumingComplete(message));
IDisposable partitionSubscription;
if (partitionDone && _partitionsSubscription.TryGetValue(message.Partition, out partitionSubscription))
{
_partitionsSubscription.Remove(message.Partition);
// calling Dispose here will cause the OnTopicPartitionComplete method to be called when it is completed.
partitionSubscription.Dispose();
}
});
OnMessageArrived = onMessage.
// isolate driver from user code misbehave
ObserveOn(Configuration.OutgoingScheduler);
// If permanent error within any single partition, fail the whole consumer (intentionally).
// Do not try to keep going (failfast principle).
_cluster.PartitionStateChanges.
Where(s => s.ErrorCode.IsPermanentFailure()).
Subscribe(state => OnMessageArrivedInput.OnError(new PartitionFailedException(state.Topic, state.PartitionId, state.ErrorCode)));
}
/// <summary>
/// Gets an observable sequence of any changes to the Fetcher for a specific topic/partition.
/// </summary>
/// <param name="topic"></param>
/// <param name="partitionId"></param>
/// <param name="consumerConfiguration"></param>
/// <returns></returns>
internal IObservable<Fetcher> GetFetcherChanges(string topic, int partitionId, ConsumerConfiguration consumerConfiguration)
{
return PartitionStateChanges
.Where(t => t.Topic == topic && t.PartitionId == partitionId)
.Select(state =>
{
// if the state is not ready, return NULL for the fetcher.
if (!state.ErrorCode.IsSuccess())
return (Fetcher)null;
// get or create the correct Fetcher for this topic/partition
var broker = FindBrokerMetaForPartitionId(state.Topic, state.PartitionId);
var fetcher = _activeFetchers.GetOrAdd(broker, b =>
{
// all new fetchers need to be "watched" for errors.
var f = new Fetcher(this, b, _protocol, consumerConfiguration, _cancel.Token);
// subscribe to error and complete notifications, and remove from active fetchers
f.ReceivedMessages.Subscribe(_ => { },
err =>
{
_log.Warn("Received error from fetcher {0}. Removing from active fetchers.", f);
Fetcher ef;
_activeFetchers.TryRemove(broker, out ef);
},
() =>
{
_log.Info("Received complete from fetcher {0}. Removing from active fetchers.", f);
Fetcher ef;
_activeFetchers.TryRemove(broker, out ef);
});
return f;
});
return fetcher;
})
.Do(f => _log.Debug("GetFetcherChanges returning {1} fetcher {0}", f, f == null ? "null" : "new"),
ex => _log.Error(ex, "GetFetcherChages saw ERROR returning new fetcher."),
() => _log.Error("GetFetcherChanges saw COMPLETE from returning new fetcher."));
}
/// <summary>
/// Create a new consumer using the specified configuration. See @ConsumerConfiguration
/// </summary>
/// <param name="consumerConfig"></param>
public Consumer(ConsumerConfiguration consumerConfig)
{
Configuration = consumerConfig;
_cluster = new Cluster(consumerConfig.SeedBrokers);
_cluster.OnThreadHang += e => OnMessageArrivedInput.OnError(e);
// Low/high watermark implementation
FlowControl = _flowControlInput.
Scan(1, (last, current) =>
{
if (current < Configuration.LowWatermark)
{
return(1);
}
if (current > Configuration.HighWatermark)
{
return(-1);
}
return(last); // While in between watermarks, carry over previous on/off state
}).
DistinctUntilChanged().
Select(i => i > 0).
Do(f => FlowControlEnabled = f).
Do(f => EtwTrace.Log.ConsumerFlowControl(f ? 1 : 0));
var onMessage = Observable.Create <ReceivedMessage>(observer =>
{
if (Interlocked.CompareExchange(ref _haveSubscriber, 1, 0) == 1)
{
throw new InvalidOperationException("Only one subscriber is allowed. Use OnMessageArrived.Publish().RefCount()");
}
// Relay messages from partition to consumer's output
// Ensure that only single subscriber is allowed because it is important to count
// outstanding messaged consumed by user
OnMessageArrivedInput.Subscribe(observer);
//
// It is not possible to wait for completion of partition resolution process, so start it asynchronously.
// This means that OnMessageArrived.Subscribe() will complete when consumer is not actually connected yet.
//
Task.Run(async() =>
{
try
{
await _cluster.ConnectAsync();
await SubscribeClient();
EtwTrace.Log.ConsumerStarted(GetHashCode(), Topic);
_connectionComplete.TrySetResult(true);
// check that we actually got any partitions subscribed
if (_topicPartitions.Count == 0)
{
OnMessageArrivedInput.OnCompleted();
}
}
catch (Exception e)
{
_connectionComplete.TrySetException(e);
OnMessageArrivedInput.OnError(e);
}
});
// upon unsubscribe
return(Disposable.Create(() => _partitionsSubscription.Values.ForEach(s => s.Dispose())));
});
if (Configuration.UseFlowControl)
{
// Increment counter of messages sent for processing
onMessage = onMessage.Do(msg =>
{
var count = Interlocked.Increment(ref _outstandingMessageProcessingCount);
_flowControlInput.OnNext(count);
});
}
// handle stop condition
onMessage = onMessage.Do(message =>
{
// check if this partition is done per the condition passed in configuration. If so, unsubscribe it.
bool partitionDone = (Configuration.StopPosition.IsPartitionConsumingComplete(message));
IDisposable partitionSubscription;
if (partitionDone && _partitionsSubscription.TryGetValue(message.Partition, out partitionSubscription))
{
_partitionsSubscription.Remove(message.Partition);
// calling Dispose here will cause the OnTopicPartitionComplete method to be called when it is completed.
partitionSubscription.Dispose();
}
});
OnMessageArrived = onMessage.
// isolate driver from user code misbehave
ObserveOn(Configuration.OutgoingScheduler);
// If permanent error within any single partition, fail the whole consumer (intentionally).
// Do not try to keep going (failfast principle).
_cluster.PartitionStateChanges.
Where(s => s.ErrorCode.IsPermanentFailure()).
Subscribe(state => OnMessageArrivedInput.OnError(new PartitionFailedException(state.Topic, state.PartitionId, state.ErrorCode)));
}