public Fetcher(Cluster cluster, BrokerMeta broker, Protocol protocol, ConsumerConfiguration consumerConfig, CancellationToken cancel)
{
_cluster = cluster;
_broker = broker;
_protocol = protocol;
_cancel = cancel;
_consumerConfig = consumerConfig;
_fetchResponses = FetchLoop().Publish().RefCount();
BuildReceivedMessages();
_cancel.Register(() => _wakeupSignal.OnNext(true));
if(_log.IsDebugEnabled)
_log.Debug("Created new fetcher #{0} for broker: {1}", _id, _broker);
EtwTrace.Log.FetcherStart(_id, consumerConfig.Topic);
}
public PartitionRecoveryMonitor(Cluster cluster, Protocol protocol, CancellationToken cancel)
{
_protocol = protocol;
_cancel = cancel;
_id = Interlocked.Increment(ref _idGenerator);
_log.Debug("Created PartitionRecoveryMonitor {0}", this);
EtwTrace.Log.RecoveryMonitor_Create(_id);
// listen for new brokers
_subscriptionsDisposable.Add(cluster.NewBrokers.Subscribe(
broker =>
{
// check and add this broker
if (_brokers.ContainsKey(broker.NodeId)) return;
_recoveryTasks.Add(RecoveryLoop(broker)
.ContinueWith(t =>
{
_log.Debug("Recovery loop task for broker {0} completed with status {1}", broker.ToString(), t.Status);
_subscriptionsDisposable.Dispose();
}));
_brokers.Add(broker.NodeId, broker);
},
ex => _log.Error("Error thrown in NewBrokers subscription!")
));
//
// listen for any topic status changes and keep our "failed" list updated
//
_subscriptionsDisposable.Add(cluster.PartitionStateChanges.Subscribe(
state =>
{
var key = new Tuple<string, int>(state.Topic, state.PartitionId);
// check if it is failed or recovered, and remove or add to our failed list.
if (state.ErrorCode.IsSuccess())
{
if (_failedList.ContainsKey(key))
{
_log.Debug("#{0} Partition {1}-{2} is recovered. Removing from failed list.", _id, state.Topic, state.PartitionId);
EtwTrace.Log.RecoveryMonitor_PartitionRecovered(_id, state.Topic, state.PartitionId);
_failedList.Remove(key);
}
}
else
{
if (!_failedList.ContainsKey(key))
{
_log.Debug("#{0} Partition {1}-{2} is in error state {3}. Adding to failed list.", _id, state.Topic, state.PartitionId, state.ErrorCode);
EtwTrace.Log.RecoveryMonitor_PartitionFailed(_id, state.Topic, state.PartitionId, (int)state.ErrorCode);
_failedList.Add(key, state.ErrorCode);
}
else
{
_log.Debug("#{0} Partition {1}-{2} is updated but still errored. Updating ErrorCode in failed list.", _id, state.Topic, state.PartitionId);
EtwTrace.Log.RecoveryMonitor_PartitionFailedAgain(_id, state.Topic, state.PartitionId, (int)state.ErrorCode);
_failedList[key] = state.ErrorCode;
}
}
},
ex => _log.Error(ex, "#{0} Error thrown in PartitionStateChanges subscription!", _id)));
}
/// <summary>
/// ConnectAsync and fetch list of brokers and metadata.
/// </summary>
/// <param name="seedBrokers">Comma separated list of seed brokers. Port numbers are optional.
/// <example>192.168.56.10,192.168.56.20:8081,broker3.local.net:8181</example>
/// </param>
public Cluster(string seedBrokers) : this()
{
_seedBrokers = seedBrokers;
_protocol = new Protocol(this);
_state = ClusterState.Disconnected;
}
