// ---------------------------------- consume --------------------------------------
public IDisposable Consume(IEnumerable <QueueConsumerPair> queueConsumerPairs, Action <IConsumerConfiguration> configure)
{
Preconditions.CheckNotNull(queueConsumerPairs, nameof(queueConsumerPairs));
Preconditions.CheckNotNull(configure, "configure");
if (disposed)
{
throw new MessageBusException("This bus has been disposed");
}
var queueOnMessages = queueConsumerPairs.Select(x =>
{
var onMessage = x.OnMessage;
if (onMessage == null)
{
var handlerCollection = handlerCollectionFactory.CreateHandlerCollection(x.Queue);
x.AddHandlers(handlerCollection);
onMessage = (body, properties, messageReceivedInfo) =>
{
var deserializedMessage = messageSerializationStrategy.DeserializeMessage(properties, body);
var handler = handlerCollection.GetHandler(deserializedMessage.MessageType);
return(handler(deserializedMessage, messageReceivedInfo));
};
}
return(Tuple.Create(x.Queue, onMessage));
}).ToList();
var consumerConfiguration = new ConsumerConfiguration(connectionConfiguration.PrefetchCount);
configure(consumerConfiguration);
var consumer = consumerFactory.CreateConsumer(queueOnMessages, connection, consumerConfiguration);
return(consumer.StartConsuming());
}
public void SubscribeAsync <T>(string subscriptionId, Func <T, Task> onMessage)
{
if (onMessage == null)
{
throw new ArgumentNullException("onMessage");
}
string queueName = GetQueueName <T>(subscriptionId);
string exchangeName = GetExchangeName <T>();
string topic = GetTopic <T>();
Action subscribeAction = () =>
{
var channel = connection.CreateModel();
modelList.Add(channel);
DeclarePublishExchange(channel, exchangeName);
channel.BasicQos(0, _prefetchCount, false);
var queue = channel.QueueDeclare(
queueName, // queue
true, // durable
false, // exclusive
false, // autoDelete
null); // arguments
channel.QueueBind(
queue, // queue
exchangeName, // exchange
topic); // routingKey
var consumer = consumerFactory.CreateConsumer(channel,
(consumerTag, deliveryTag, redelivered, exchange, routingKey, properties, body) =>
{
CheckMessageType <T>(properties);
var message = serializer.BytesToMessage <T>(body);
return(onMessage(message));
});
channel.BasicConsume(
queueName, // queue
noAck, // noAck
consumer.ConsumerTag, // consumerTag
consumer); // consumer
};
AddSubscriptionAction(subscribeAction);
}
public ISubscription SubscribeAsync <T>(Func <T, TMessageContext, Task> subscribeMethod, SubscriptionConfiguration config)
{
var routingKey = _config.RouteWithGlobalId
? $"{config.RoutingKey}.#"
: config.RoutingKey;
var topologyTask = _topologyProvider.BindQueueAsync(config.Queue, config.Exchange, routingKey);
var channelTask = _channelFactory.CreateChannelAsync();
var subscriberTask = Task
.WhenAll(topologyTask, channelTask)
.ContinueWith(t =>
{
var consumer = _consumerFactory.CreateConsumer(config, channelTask.Result);
consumer.OnMessageAsync = (o, args) => _errorHandling.ExecuteAsync(() =>
{
var body = _serializer.Deserialize <T>(args.Body);
var context = _contextProvider.ExtractContext(args.BasicProperties.Headers[PropertyHeaders.Context]);
_contextEnhancer.WireUpContextFeatures(context, consumer, args);
return(subscribeMethod(body, context));
}, exception => _errorHandling.OnSubscriberExceptionAsync(consumer, config, args, exception));
consumer.Model.BasicConsume(config.Queue.FullQueueName, config.NoAck, consumer);
_logger.LogDebug($"Setting up a consumer on channel '{channelTask.Result.ChannelNumber}' for queue {config.Queue.QueueName} with NoAck set to {config.NoAck}.");
return(new Subscription(consumer, config.Queue.QueueName));
});
Task.WaitAll(subscriberTask);
_subscriptions.Add(subscriberTask.Result);
return(subscriberTask.Result);
}
public ISubscription SubscribeAsync <T>(Func <T, Task> subscribeMethod, SubscriptionConfiguration config)
{
var routingKey = config.RoutingKey;
var topologyTask = _topologyProvider.BindQueueAsync(config.Queue, config.Exchange, routingKey);
var channelTask = _channelFactory.CreateChannelAsync();
var subscriberTask = Task
.WhenAll(topologyTask, channelTask)
.ContinueWith(t =>
{
if (topologyTask.IsFaulted)
{
throw topologyTask.Exception ?? new Exception("Topology Task Faulted");
}
var consumer = _consumerFactory.CreateConsumer(config, channelTask.Result);
consumer.OnMessageAsync = async(o, args) =>
{
var body = _serializer.Deserialize <T>(args.Body);
await subscribeMethod(body);
};
consumer.Model.BasicConsume(config.Queue.FullQueueName, config.NoAck, consumer);
_logger.LogDebug($"Setting up a consumer on channel '{channelTask.Result.ChannelNumber}' for queue {config.Queue.QueueName} with NoAck set to {config.NoAck}.");
return(new Subscription(consumer, config.Queue.FullQueueName));
});
Task.WaitAll(subscriberTask);
_subscriptions.Add(subscriberTask.Result);
return(subscriberTask.Result);
}
protected AbstractKafkaPipeline(ILogger <AbstractKafkaPipeline <TConsumerKey, TConsumerValue, TPipe> > logger,
IConsumerFactory <TConsumerKey, TConsumerValue> consumerFactory,
IServiceProvider serviceProvider)
: base(serviceProvider)
{
Logger = logger;
Consumer = consumerFactory.CreateConsumer();
}
public virtual void Subscribe(IQueue queue, Func <Byte[], MessageProperties, MessageReceivedInfo, Task> onMessage)
{
if (queue == null)
{
throw new ArgumentNullException("queue");
}
if (onMessage == null)
{
throw new ArgumentNullException("onMessage");
}
if (disposed)
{
throw new EasyNetQException("This bus has been disposed");
}
var subscriptionAction = new SubscriptionAction(queue.IsSingleUse);
subscriptionAction.Action = () =>
{
var channel = connection.CreateModel();
channel.ModelShutdown += (model, reason) => logger.DebugWrite("Model Shutdown for queue: '{0}'", queue.Name);
queue.Visit(new TopologyBuilder(channel));
channel.BasicQos(0, connectionConfiguration.PrefetchCount, false);
var consumer = consumerFactory.CreateConsumer(subscriptionAction, channel, queue.IsSingleUse,
(consumerTag, deliveryTag, redelivered, exchange, routingKey, properties, body) =>
{
var messageRecievedInfo = new MessageReceivedInfo
{
ConsumerTag = consumerTag,
DeliverTag = deliveryTag,
Redelivered = redelivered,
Exchange = exchange,
RoutingKey = routingKey
};
var messsageProperties = new MessageProperties(properties);
return(onMessage(body, messsageProperties, messageRecievedInfo));
});
channel.BasicConsume(
queue.Name, // queue
NoAck, // noAck
consumer.ConsumerTag, // consumerTag
consumer); // consumer
logger.DebugWrite("Declared Consumer. queue='{0}', prefetchcount={1}",
queue.Name,
connectionConfiguration.PrefetchCount);
};
AddSubscriptionAction(subscriptionAction);
}
public ISubscription RespondAsync <TRequest, TResponse>(Func <TRequest, TMessageContext, Task <TResponse> > onMessage, ResponderConfiguration cfg)
{
var routingKey = _config.RouteWithGlobalId ? $"{cfg.RoutingKey}.#" : cfg.RoutingKey;
var topologyTask = _topologyProvider.BindQueueAsync(cfg.Queue, cfg.Exchange, routingKey);
var channelTask = _channelFactory.CreateChannelAsync();
var respondTask = Task.WhenAll(topologyTask, channelTask)
.ContinueWith(t =>
{
if (topologyTask.IsFaulted)
{
throw topologyTask.Exception ?? new Exception("Topology Task Fauled");
}
var consumer = _consumerFactory.CreateConsumer(cfg, channelTask.Result);
consumer.OnMessageAsync = (o, args) => _errorHandling.ExecuteAsync(() =>
{
var body = _serializer.Deserialize <TRequest>(args.Body);
var context = _contextProvider.ExtractContext(args.BasicProperties.Headers[PropertyHeaders.Context]);
_contextEnhancer.WireUpContextFeatures(context, consumer, args);
return(onMessage(body, context)
.ContinueWith(tResponse =>
{
if (tResponse.IsFaulted)
{
throw tResponse.Exception ?? new Exception();
}
if (consumer.AcknowledgedTags.Contains(args.DeliveryTag))
{
return;
}
if (tResponse.Result == null)
{
return;
}
_logger.LogDebug($"Sending response to request with correlation '{args.BasicProperties.CorrelationId}'.");
consumer.Model.BasicPublish(
exchange: string.Empty,
routingKey: args.BasicProperties.ReplyTo,
basicProperties: _propertyProvider.GetProperties <TResponse>(p => p.CorrelationId = args.BasicProperties.CorrelationId),
body: _serializer.Serialize(tResponse.Result)
);
}));
}, exception => _errorHandling.OnResponseHandlerExceptionAsync(consumer, cfg, args, exception));
consumer.Model.BasicConsume(cfg.Queue.QueueName, cfg.NoAck, consumer);
return(new Subscription(consumer, cfg.Queue.QueueName));
});
Task.WaitAll(respondTask);
_subscriptions.Add(respondTask.Result);
return(respondTask.Result);
}
public override async Task Execute(HorseClient client, HorseMessage message, object model)
{
TModel t = (TModel)model;
Exception exception = null;
IConsumerFactory consumerFactory = null;
try
{
if (_consumer != null)
{
await Consume(_consumer, message, t, client);
}
else if (_consumerFactoryCreator != null)
{
consumerFactory = _consumerFactoryCreator();
object consumerObject = await consumerFactory.CreateConsumer(_consumerType);
IDirectConsumer <TModel> consumer = (IDirectConsumer <TModel>)consumerObject;
await Consume(consumer, message, t, client);
}
else
{
throw new ArgumentNullException("There is no consumer defined");
}
if (SendAck)
{
await client.SendAck(message);
}
}
catch (Exception e)
{
if (SendNack)
{
await SendNegativeAck(message, client, e);
}
await SendExceptions(message, client, e);
exception = e;
}
finally
{
if (consumerFactory != null)
{
consumerFactory.Consumed(exception);
}
}
}
public virtual IDisposable Consume(IQueue queue, Func <Byte[], MessageProperties, MessageReceivedInfo, Task> onMessage)
{
Preconditions.CheckNotNull(queue, "queue");
Preconditions.CheckNotNull(onMessage, "onMessage");
if (disposed)
{
throw new EasyNetQException("This bus has been disposed");
}
var consumer = consumerFactory.CreateConsumer(queue, onMessage, connection);
return(consumer.StartConsuming());
}
private Task <IRawConsumer> GetOrCreateConsumerAsync(IConsumerConfiguration cfg)
{
return(_channelFactory
.GetChannelAsync()
.ContinueWith(tChannel =>
{
var consumerCs = new ConsumerCompletionSource();
if (_consumerCompletionSources.TryAdd(tChannel.Result, consumerCs))
{
var newConsumer = _consumerFactory.CreateConsumer(cfg, tChannel.Result);
WireUpConsumer(newConsumer, cfg);
consumerCs.ConsumerQueues.TryAdd(
key: cfg.Queue.FullQueueName,
value: newConsumer.Model.BasicConsume(cfg.Queue.FullQueueName, cfg.NoAck, newConsumer)
);
_currentConsumer = consumerCs;
_logger.LogInformation($"Created consumer on channel '{tChannel.Result.ChannelNumber}' that consumes messages from '{cfg.Queue.FullQueueName}'.");
consumerCs.TrySetResult(newConsumer);
return consumerCs.Task;
}
_logger.LogDebug($"Consumer for channel '{tChannel.Result.ChannelNumber}' exists. Using it.");
consumerCs = _consumerCompletionSources[tChannel.Result];
if (consumerCs.ConsumerQueues.ContainsKey(cfg.Queue.FullQueueName))
{
return consumerCs.Task;
}
return consumerCs.Task.ContinueWith(t =>
{
lock (consumerCs.Consumer)
{
if (consumerCs.ConsumerQueues.ContainsKey(cfg.Queue.FullQueueName))
{
return t.Result;
}
consumerCs.ConsumerQueues.TryAdd(
key: cfg.Queue.FullQueueName,
value: consumerCs.Consumer.Model.BasicConsume(cfg.Queue.FullQueueName, cfg.NoAck, consumerCs.Consumer)
);
_logger.LogDebug($"Existign consumer for channel '{tChannel.Result.ChannelNumber}' consumes '{cfg.Queue.FullQueueName}'.");
return t.Result;
}
});
})
.Unwrap());
}
public virtual IDisposable Consume(IQueue queue, Func <byte[], MessageProperties, MessageReceivedInfo, Task> onMessage, Action <IConsumerConfiguration> configure)
{
Preconditions.CheckNotNull(queue, "queue");
Preconditions.CheckNotNull(onMessage, "onMessage");
Preconditions.CheckNotNull(configure, "configure");
if (disposed)
{
throw new EasyNetQException("This bus has been disposed");
}
var consumerConfiguration = new ConsumerConfiguration(connectionConfiguration.PrefetchCount);
configure(consumerConfiguration);
var consumer = consumerFactory.CreateConsumer(queue, (body, properties, receviedInfo) =>
{
var rawMessage = produceConsumeInterceptor.OnConsume(new RawMessage(properties, body));
return(onMessage(rawMessage.Body, rawMessage.Properties, receviedInfo));
}, connection, consumerConfiguration);
return(consumer.StartConsuming());
}
public KafkaOrchestratorBase(IConsumerFactory consumerFactory)
{
_consumer = consumerFactory.CreateConsumer(Dns.GetHostName());
_cancellationTokenSource = new CancellationTokenSource();
}
public virtual void Consume(IQueue queue, Func <Byte[], MessageProperties, MessageReceivedInfo, Task> onMessage)
{
Preconditions.CheckNotNull(queue, "queue");
Preconditions.CheckNotNull(onMessage, "onMessage");
if (disposed)
{
throw new EasyNetQException("This bus has been disposed");
}
var newConsumerTag = conventions.ConsumerTagConvention();
var subscriptionAction = new SubscriptionAction(newConsumerTag, logger, queue.IsSingleUse, queue.IsExclusive);
subscriptionAction.Action = (isNewConnection) =>
{
// recreate channel if current channel is no longer open or connection was dropped and reconnected (to survive server restart)
if (subscriptionAction.Channel == null || subscriptionAction.Channel.IsOpen == false || isNewConnection)
{
subscriptionAction.Channel = CreateChannel(queue);
}
var channel = subscriptionAction.Channel;
channel.BasicQos(0, connectionConfiguration.PrefetchCount, false);
var consumer = consumerFactory.CreateConsumer(subscriptionAction, channel, queue.IsSingleUse,
(consumerTag, deliveryTag, redelivered, exchange, routingKey, properties, body) =>
{
var messageRecievedInfo = new MessageReceivedInfo
{
ConsumerTag = consumerTag,
DeliverTag = deliveryTag,
Redelivered = redelivered,
Exchange = exchange,
RoutingKey = routingKey
};
var messsageProperties = new MessageProperties(properties);
return(onMessage(body, messsageProperties, messageRecievedInfo));
});
var cancelNotifications = consumer as IConsumerCancelNotifications;
if (cancelNotifications != null)
{
cancelNotifications.BasicCancel += OnBasicCancel;
}
channel.BasicConsume(
queue.Name, // queue
NoAck, // noAck
consumer.ConsumerTag, // consumerTag
consumer); // consumer
logger.DebugWrite("Declared Consumer. queue='{0}', consumer tag='{1}' prefetchcount={2}",
queue.Name,
consumer.ConsumerTag,
connectionConfiguration.PrefetchCount);
};
AddSubscriptionAction(subscriptionAction);
}
public override async Task Execute(HorseClient client, HorseMessage message, object model)
{
Exception exception = null;
IConsumerFactory consumerFactory = null;
bool respond = false;
try
{
TRequest requestModel = (TRequest)model;
IHorseRequestHandler <TRequest, TResponse> handler;
if (_handler != null)
{
handler = _handler;
}
else if (_handlerFactoryCreator != null)
{
consumerFactory = _handlerFactoryCreator();
object consumerObject = await consumerFactory.CreateConsumer(_handlerType);
handler = (IHorseRequestHandler <TRequest, TResponse>)consumerObject;
}
else
{
throw new ArgumentNullException("There is no consumer defined");
}
try
{
TResponse responseModel = await Handle(handler, requestModel, message, client);
HorseResultCode code = responseModel is null ? HorseResultCode.NoContent : HorseResultCode.Ok;
HorseMessage responseMessage = message.CreateResponse(code);
if (responseModel != null)
{
responseMessage.Serialize(responseModel, client.JsonSerializer);
}
respond = true;
await client.SendAsync(responseMessage);
}
catch (Exception e)
{
ErrorResponse errorModel = await handler.OnError(e, requestModel, message, client);
if (errorModel.ResultCode == HorseResultCode.Ok)
{
errorModel.ResultCode = HorseResultCode.Failed;
}
HorseMessage responseMessage = message.CreateResponse(errorModel.ResultCode);
if (!string.IsNullOrEmpty(errorModel.Reason))
{
responseMessage.SetStringContent(errorModel.Reason);
}
respond = true;
await client.SendAsync(responseMessage);
throw;
}
}
catch (Exception e)
{
if (!respond)
{
try
{
HorseMessage response = message.CreateResponse(HorseResultCode.InternalServerError);
await client.SendAsync(response);
}
catch
{
}
}
await SendExceptions(message, client, e);
exception = e;
}
finally
{
if (consumerFactory != null)
{
consumerFactory.Consumed(exception);
}
}
}
public override Task OnConnectedAsync()
{
Consumer consumerToRegister;
string clientConnectionId = Context.ConnectionId;
var headers = Context.GetHttpContext().Request.Headers;
// authorization tokens
// TODO: Implement token validation
string tenantToken = headers["x-andyx-tenant-authoriziation"];
string componentToken = headers["x-andyx-component-authoriziation"];
string tenant = headers["x-andyx-tenant"].ToString();
string product = headers["x-andyx-product"].ToString();
string component = headers["x-andyx-component"].ToString();
string topic = headers["x-andyx-topic"].ToString();
bool isPersistent = bool.Parse(headers["x-andyx-topic-is-persistent"]);
string consumerName = headers["x-andyx-consumer"].ToString();
SubscriptionType subscriptionType = (SubscriptionType)Enum.Parse(typeof(SubscriptionType), headers["x-andyx-consumer-type"].ToString());
InitialPosition initialPosition = (InitialPosition)Enum.Parse(typeof(InitialPosition), headers["x-andyx-consumer-initial-position"].ToString());
logger.LogInformation($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} requested connection");
// check if the consumer is already connected
var connectedTenant = tenantRepository.GetTenant(tenant);
if (connectedTenant == null)
{
logger.LogInformation($"Consumer '{consumerName}' failed to connect, tenant '{tenant}' does not exists");
return(OnDisconnectedAsync(new Exception($"There is no tenant registered with this name '{tenant}'")));
}
// check tenant token validation
bool isTenantTokenValidated = tenantRepository.ValidateTenantToken(tenant, tenantToken);
if (isTenantTokenValidated != true)
{
logger.LogInformation($"Consumer '{consumerName}' failed to connect, access is forbidden. Not authorized");
return(OnDisconnectedAsync(new Exception($"Consumer '{consumerName}' failed to connect, access is forbidden")));
}
var connectedProduct = tenantRepository.GetProduct(tenant, product);
if (connectedProduct == null)
{
if (connectedTenant.Settings.AllowProductCreation != true)
{
logger.LogInformation($"Consumer '{consumerName}' failed to connect, tenant '{tenant}' does not allow to create new product");
return(OnDisconnectedAsync(new Exception($"There is no product registered with this name '{product}'. Tenant '{tenant}' does not allow to create new product")));
}
var productDetails = tenantFactory.CreateProduct(product);
tenantRepository.AddProduct(tenant, product, productDetails);
storageHubService.CreateProductAsync(tenant, productDetails);
}
else
{
storageHubService.UpdateProductAsync(tenant, connectedProduct);
}
var connectedComponent = tenantRepository.GetComponent(tenant, product, component);
if (connectedComponent == null)
{
var componentDetails = tenantFactory.CreateComponent(component);
tenantRepository.AddComponent(tenant, product, component, componentDetails);
storageHubService.CreateComponentAsync(tenant, product, componentDetails);
}
else
{
// check component token validation
bool isComponentTokenValidated = tenantRepository.ValidateComponentToken(tenant, product, component, componentToken, consumerName, true);
if (isComponentTokenValidated != true)
{
logger.LogInformation($"Consumer '{consumerName}' failed to connect, access is forbidden. Not authorized, check component token");
return(OnDisconnectedAsync(new Exception($"Consumer '{consumerName}' failed to connect, access is forbidden, check component token")));
}
storageHubService.UpdateComponentAsync(tenant, product, connectedComponent);
}
var connectedTopic = tenantRepository.GetTopic(tenant, product, component, topic);
if (connectedTopic == null)
{
connectedComponent = tenantRepository.GetComponent(tenant, product, component);
if (connectedComponent.Settings.AllowTopicCreation != true)
{
logger.LogInformation($"Component '{component}' does not allow to create a new topic {topic} at '{tenant}/{product}/{component}'. To allow creating update property AllowTopicCreation at component.");
return(OnDisconnectedAsync(new Exception($"Component '{component}' does not allow to create a new topic {topic} at '{tenant}/{product}/{component}'. To allow creating update property AllowTopicCreation at component.")));
}
var topicDetails = tenantFactory.CreateTopic(topic, isPersistent);
tenantRepository.AddTopic(tenant, product, component, topic, topicDetails);
storageHubService.CreateTopicAsync(tenant, product, component, topicDetails);
}
else
{
storageHubService.UpdateTopicAsync(tenant, product, component, connectedTopic);
}
string consumerIdOnRepo = $"{tenant}{product}{component}{topic}|{consumerName}";
var consumerConencted = consumerHubRepository.GetConsumerById(consumerIdOnRepo);
if (consumerConencted != null)
{
if (subscriptionType == SubscriptionType.Exclusive)
{
logger.LogWarning($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} is already connected");
return(OnDisconnectedAsync(new Exception($"There is a consumer with name '{consumerName}' and with type 'EXCLUSIVE' is connected to this node")));
}
if (subscriptionType == SubscriptionType.Failover)
{
if (consumerConencted.Connections.Count >= 2)
{
logger.LogWarning($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} is already connected with 2 instances");
return(OnDisconnectedAsync(new Exception($"There are two consumers with name '{consumerName}' and with type 'Failover' are connected to this node")));
}
}
}
consumerToRegister = consumerFactory.CreateConsumer(tenant, product, component, topic, consumerName, subscriptionType, initialPosition);
consumerHubRepository.AddConsumer(consumerIdOnRepo, consumerToRegister);
consumerHubRepository.AddConsumerConnection(consumerIdOnRepo, clientConnectionId);
storageHubService.ConnectConsumerAsync(consumerToRegister);
Clients.Caller.ConsumerConnected(new Model.Consumers.Events.ConsumerConnectedDetails()
{
Id = consumerToRegister.Id,
Tenant = tenant,
Product = product,
Component = component,
Topic = topic,
ConsumerName = consumerName,
SubscriptionType = subscriptionType,
InitialPosition = initialPosition
});
// if consumer is not persistent, do not store the message, just allow streaming
if (isPersistent == true)
{
// Sent not acknoledged messages to this consumer (for exclusive and for the first shared/failover consumer connected)
if (subscriptionType == SubscriptionType.Exclusive)
{
storageHubService.RequestUnacknowledgedMessagesConsumer(consumerToRegister);
}
if (subscriptionType == SubscriptionType.Shared || subscriptionType == SubscriptionType.Failover)
{
if (consumerConencted == null)
{
storageHubService.RequestUnacknowledgedMessagesConsumer(consumerToRegister);
}
}
}
logger.LogInformation($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} is connected");
return(base.OnConnectedAsync());
}
