public KafkaConsumerInstance(IZookeeperConnection zkConnect, ConsumerOptions options)
{
_consumerConnector = zkConnect.CreateConsumerConnector(options);
_consumerConnector.Rebalanced += OnRebalanced;
_consumerConnector.ZookeeperDisconnected += OnZookeeperDisconnected;
_consumerConnector.ZookeeperSessionExpired += OnZookeeperSessionExpired;
}
public JobConsumerMessageConnector()
{
_submitJobConsumerConnector = ConsumerConnectorCache <SubmitJobConsumer <TJob> > .Connector;
_startJobConsumerConnector = ConsumerConnectorCache <StartJobConsumer <TJob> > .Connector;
_finalizeJobConsumerConnector = ConsumerConnectorCache <FinalizeJobConsumer <TJob> > .Connector;
_superviseJobConsumerConnector = ConsumerConnectorCache <SuperviseJobConsumer> .Connector;
}
private void ShutdownInternal(bool force = false)
{
Logger.Info($"Shutdown received for consumer {_group} on {_topic} with force:{force}");
if (!_running)
{
return;
}
lock (Lock)
{
if (!_running)
{
return;
}
if (_forceShutdown)
{
return;
}
_forceShutdown = force;
var c = _consumer;
_consumer = null;
c?.Dispose();
_threadList.Clear();
}
}
public ConsumerSubscriptionBuilder(IConsumerFactory <TConsumer> consumerFactory,
ReferenceFactory referenceFactory)
{
_consumerFactory = consumerFactory;
_referenceFactory = referenceFactory;
_consumerConnector = ConsumerConnectorCache.GetConsumerConnector <TConsumer>();
}
public static ConnectHandle ConnectConsumer <T>(this IConsumePipeConnector filter, Func <T> factoryMethod, params IPipeSpecification <ConsumerConsumeContext <T> >[] specifications)
where T : class
{
var consumerFactory = new DelegateConsumerFactory <T>(factoryMethod);
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector <T>();
return(connector.ConnectConsumer(filter, consumerFactory, specifications));
}
public static ConnectHandle ConnectConsumer <T>(this IConsumePipeConnector filter,
params IPipeSpecification <ConsumerConsumeContext <T> >[] pipeSpecifications)
where T : class, new()
{
var consumerFactory = new DefaultConstructorConsumerFactory <T>();
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector <T>();
return(connector.ConnectConsumer(filter, consumerFactory, pipeSpecifications));
}
/// <summary>
/// Subscribe a component type to the pipeline that is resolved from the container for each message
/// </summary>
/// <typeparam name="TComponent"></typeparam>
/// <param name="pipeline">The pipeline to configure</param>
/// <returns></returns>
public static UnsubscribeAction ConnectConsumer <TComponent>(this IInboundMessagePipeline pipeline)
where TComponent : class, new()
{
return(pipeline.Configure(x =>
{
var consumerFactory = new DelegateConsumerFactory <TComponent>(() => new TComponent());
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector <TComponent>();
return connector.Connect(x, consumerFactory);
}));
}
/// <summary>
/// Subscribe a component type to the pipeline that is resolved from the container for each message
/// </summary>
/// <typeparam name="TConsumer"></typeparam>
/// <param name="pipeline">The pipeline to configure</param>
/// <param name="consumerFactory"></param>
/// <returns></returns>
public static UnsubscribeAction ConnectConsumer <TConsumer>(this IInboundMessagePipeline pipeline,
Func <TConsumer> consumerFactory)
where TConsumer : class
{
return(pipeline.Configure(x =>
{
var factory = new DelegateConsumerFactory <TConsumer>(consumerFactory);
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector <TConsumer>();
return connector.Connect(x, factory);
}));
}
public static UnsubscribeAction SubscribeConsumer <TConsumer>([NotNull] this IServiceBus bus,
[NotNull] IConsumerFactory <TConsumer>
consumerFactory)
where TConsumer : class, IConsumer
{
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Subscribing Consumer: {0} (using supplied consumer factory)", typeof(TConsumer));
}
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector <TConsumer>();
return(bus.Configure(x => connector.Connect(x, consumerFactory)));
}
public static UnsubscribeAction SubscribeConsumer <TConsumer>([NotNull] this IServiceBus bus)
where TConsumer : class, IConsumer, new()
{
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Subscribing Consumer: {0} (using default consumer factory)", typeof(TConsumer));
}
var delegateConsumerFactory = new DelegateConsumerFactory <TConsumer>(() => new TConsumer());
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector <TConsumer>();
return(bus.Configure(x => connector.Connect(x, delegateConsumerFactory)));
}
public static UnsubscribeAction SubscribeConsumer([NotNull] this IServiceBus bus, [NotNull] Type consumerType,
[NotNull] Func <Type, object> consumerFactory)
{
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Subscribing Consumer: {0} (by type, using object consumer factory)", consumerType);
}
object factory = FastActivator.Create(typeof(ObjectConsumerFactory <>), new[] { consumerType },
new object[] { consumerFactory });
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector(consumerType);
return(bus.Configure(x => connector.FastInvoke <IConsumerConnector, UnsubscribeAction>("Connect", x, factory)));
}
public static UnsubscribeAction SubscribeInterceptingConsumer <TConsumer>([NotNull] this IServiceBus bus,
[NotNull] Func <TConsumer> consumerFactory, [NotNull] Action <Action> interceptor)
where TConsumer : class, IConsumer
{
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Subscribing Consumer: {0} (using delegate consumer factory)", typeof(TConsumer));
}
var delegateConsumerFactory = new DelegateConsumerFactory <TConsumer>(consumerFactory);
var interceptingConsumerFactory = new InterceptingConsumerFactory <TConsumer>(delegateConsumerFactory,
interceptor);
IConsumerConnector connector = ConsumerConnectorCache.GetConsumerConnector <TConsumer>();
return(bus.Configure(x => connector.Connect(x, interceptingConsumerFactory)));
}
public JobConsumerMessageConnector()
{
_submitJobConsumerConnector = ConsumerConnectorCache <SubmitJobConsumer <TJob> > .Connector;
_startJobConsumerConnector = ConsumerConnectorCache <StartJobConsumer <TJob> > .Connector;
_cancelJobConsumerConnector = ConsumerConnectorCache <CancelJobConsumer <TJob> > .Connector;
}
public void Start(
Action <IEnumerable <ConsumedMessage> > dataSubscriber,
Action <Exception> errorSubscriber = null,
Action closeAction = null)
{
if (_running)
{
return;
}
_restart = false;
do
{
if (_restart)
{
Task.Delay(RestartMsDelay).Wait();
}
if (_forceShutdown)
{
break;
}
Task[] tasks;
lock (Lock)
{
Logger.Info($"Starting batched consumer {_group} for {_topic}.");
_restart = false;
try
{
_consumer = _connection.CreateConsumerConnector(new ConsumerOptions
{
GroupName = _group,
AutoCommit = false,
AutoOffsetReset = Offset.Earliest
});
_consumer.ZookeeperSessionExpired += (sender, args) =>
{
OnZookeeperSessionExpired();
Logger.WarnFormat($"Zookeeper session expired. Shutting down consumer {_group} for {_topic} to restart...");
Restart();
};
_consumer.ZookeeperDisconnected += (sender, args) =>
{
OnZookeeperDisconnected();
Logger.WarnFormat($"Zookeeper disconnected. Shutting down consumer {_group} for {_topic} to restart...");
Restart();
};
_consumer.Rebalanced += (sender, args) => OnRebalanced(args);
var streams = _consumer.CreateMessageStreams(
new Dictionary <string, int> {
{ _topic, _threads }
},
new DefaultDecoder());
tasks = streams[_topic]
.Select(s => StartConsumer(s, dataSubscriber, errorSubscriber))
.ToArray();
Logger.Info($"Consumer {_consumer.ConsumerId} started with {tasks.Length} threads.");
_running = true;
}
catch (Exception ex)
{
Logger.Error($"Exception starting consumer {_group} for {_topic}. Restarting...", ex);
Restart();
continue;
}
}
Task.WaitAll(tasks);
Logger.InfoFormat("Consumer {0} for {1} shut down.", _group, _topic);
} while (_restart);
closeAction?.Invoke();
_running = false;
}
