// ---------------------------------- consume -------------------------------------- public IDisposable Consume(IEnumerable <QueueConsumerPair> queueConsumerPairs, Action <IConsumerConfiguration> configure) { Preconditions.CheckNotNull(queueConsumerPairs, nameof(queueConsumerPairs)); Preconditions.CheckNotNull(configure, "configure"); if (disposed) { throw new MessageBusException("This bus has been disposed"); } var queueOnMessages = queueConsumerPairs.Select(x => { var onMessage = x.OnMessage; if (onMessage == null) { var handlerCollection = handlerCollectionFactory.CreateHandlerCollection(x.Queue); x.AddHandlers(handlerCollection); onMessage = (body, properties, messageReceivedInfo) => { var deserializedMessage = messageSerializationStrategy.DeserializeMessage(properties, body); var handler = handlerCollection.GetHandler(deserializedMessage.MessageType); return(handler(deserializedMessage, messageReceivedInfo)); }; } return(Tuple.Create(x.Queue, onMessage)); }).ToList(); var consumerConfiguration = new ConsumerConfiguration(connectionConfiguration.PrefetchCount); configure(consumerConfiguration); var consumer = consumerFactory.CreateConsumer(queueOnMessages, connection, consumerConfiguration); return(consumer.StartConsuming()); }
public void SubscribeAsync <T>(string subscriptionId, Func <T, Task> onMessage) { if (onMessage == null) { throw new ArgumentNullException("onMessage"); } string queueName = GetQueueName <T>(subscriptionId); string exchangeName = GetExchangeName <T>(); string topic = GetTopic <T>(); Action subscribeAction = () => { var channel = connection.CreateModel(); modelList.Add(channel); DeclarePublishExchange(channel, exchangeName); channel.BasicQos(0, _prefetchCount, false); var queue = channel.QueueDeclare( queueName, // queue true, // durable false, // exclusive false, // autoDelete null); // arguments channel.QueueBind( queue, // queue exchangeName, // exchange topic); // routingKey var consumer = consumerFactory.CreateConsumer(channel, (consumerTag, deliveryTag, redelivered, exchange, routingKey, properties, body) => { CheckMessageType <T>(properties); var message = serializer.BytesToMessage <T>(body); return(onMessage(message)); }); channel.BasicConsume( queueName, // queue noAck, // noAck consumer.ConsumerTag, // consumerTag consumer); // consumer }; AddSubscriptionAction(subscribeAction); }
public ISubscription SubscribeAsync <T>(Func <T, TMessageContext, Task> subscribeMethod, SubscriptionConfiguration config) { var routingKey = _config.RouteWithGlobalId ? $"{config.RoutingKey}.#" : config.RoutingKey; var topologyTask = _topologyProvider.BindQueueAsync(config.Queue, config.Exchange, routingKey); var channelTask = _channelFactory.CreateChannelAsync(); var subscriberTask = Task .WhenAll(topologyTask, channelTask) .ContinueWith(t => { var consumer = _consumerFactory.CreateConsumer(config, channelTask.Result); consumer.OnMessageAsync = (o, args) => _errorHandling.ExecuteAsync(() => { var body = _serializer.Deserialize <T>(args.Body); var context = _contextProvider.ExtractContext(args.BasicProperties.Headers[PropertyHeaders.Context]); _contextEnhancer.WireUpContextFeatures(context, consumer, args); return(subscribeMethod(body, context)); }, exception => _errorHandling.OnSubscriberExceptionAsync(consumer, config, args, exception)); consumer.Model.BasicConsume(config.Queue.FullQueueName, config.NoAck, consumer); _logger.LogDebug($"Setting up a consumer on channel '{channelTask.Result.ChannelNumber}' for queue {config.Queue.QueueName} with NoAck set to {config.NoAck}."); return(new Subscription(consumer, config.Queue.QueueName)); }); Task.WaitAll(subscriberTask); _subscriptions.Add(subscriberTask.Result); return(subscriberTask.Result); }
public ISubscription SubscribeAsync <T>(Func <T, Task> subscribeMethod, SubscriptionConfiguration config) { var routingKey = config.RoutingKey; var topologyTask = _topologyProvider.BindQueueAsync(config.Queue, config.Exchange, routingKey); var channelTask = _channelFactory.CreateChannelAsync(); var subscriberTask = Task .WhenAll(topologyTask, channelTask) .ContinueWith(t => { if (topologyTask.IsFaulted) { throw topologyTask.Exception ?? new Exception("Topology Task Faulted"); } var consumer = _consumerFactory.CreateConsumer(config, channelTask.Result); consumer.OnMessageAsync = async(o, args) => { var body = _serializer.Deserialize <T>(args.Body); await subscribeMethod(body); }; consumer.Model.BasicConsume(config.Queue.FullQueueName, config.NoAck, consumer); _logger.LogDebug($"Setting up a consumer on channel '{channelTask.Result.ChannelNumber}' for queue {config.Queue.QueueName} with NoAck set to {config.NoAck}."); return(new Subscription(consumer, config.Queue.FullQueueName)); }); Task.WaitAll(subscriberTask); _subscriptions.Add(subscriberTask.Result); return(subscriberTask.Result); }
protected AbstractKafkaPipeline(ILogger <AbstractKafkaPipeline <TConsumerKey, TConsumerValue, TPipe> > logger, IConsumerFactory <TConsumerKey, TConsumerValue> consumerFactory, IServiceProvider serviceProvider) : base(serviceProvider) { Logger = logger; Consumer = consumerFactory.CreateConsumer(); }
public virtual void Subscribe(IQueue queue, Func <Byte[], MessageProperties, MessageReceivedInfo, Task> onMessage) { if (queue == null) { throw new ArgumentNullException("queue"); } if (onMessage == null) { throw new ArgumentNullException("onMessage"); } if (disposed) { throw new EasyNetQException("This bus has been disposed"); } var subscriptionAction = new SubscriptionAction(queue.IsSingleUse); subscriptionAction.Action = () => { var channel = connection.CreateModel(); channel.ModelShutdown += (model, reason) => logger.DebugWrite("Model Shutdown for queue: '{0}'", queue.Name); queue.Visit(new TopologyBuilder(channel)); channel.BasicQos(0, connectionConfiguration.PrefetchCount, false); var consumer = consumerFactory.CreateConsumer(subscriptionAction, channel, queue.IsSingleUse, (consumerTag, deliveryTag, redelivered, exchange, routingKey, properties, body) => { var messageRecievedInfo = new MessageReceivedInfo { ConsumerTag = consumerTag, DeliverTag = deliveryTag, Redelivered = redelivered, Exchange = exchange, RoutingKey = routingKey }; var messsageProperties = new MessageProperties(properties); return(onMessage(body, messsageProperties, messageRecievedInfo)); }); channel.BasicConsume( queue.Name, // queue NoAck, // noAck consumer.ConsumerTag, // consumerTag consumer); // consumer logger.DebugWrite("Declared Consumer. queue='{0}', prefetchcount={1}", queue.Name, connectionConfiguration.PrefetchCount); }; AddSubscriptionAction(subscriptionAction); }
public ISubscription RespondAsync <TRequest, TResponse>(Func <TRequest, TMessageContext, Task <TResponse> > onMessage, ResponderConfiguration cfg) { var routingKey = _config.RouteWithGlobalId ? $"{cfg.RoutingKey}.#" : cfg.RoutingKey; var topologyTask = _topologyProvider.BindQueueAsync(cfg.Queue, cfg.Exchange, routingKey); var channelTask = _channelFactory.CreateChannelAsync(); var respondTask = Task.WhenAll(topologyTask, channelTask) .ContinueWith(t => { if (topologyTask.IsFaulted) { throw topologyTask.Exception ?? new Exception("Topology Task Fauled"); } var consumer = _consumerFactory.CreateConsumer(cfg, channelTask.Result); consumer.OnMessageAsync = (o, args) => _errorHandling.ExecuteAsync(() => { var body = _serializer.Deserialize <TRequest>(args.Body); var context = _contextProvider.ExtractContext(args.BasicProperties.Headers[PropertyHeaders.Context]); _contextEnhancer.WireUpContextFeatures(context, consumer, args); return(onMessage(body, context) .ContinueWith(tResponse => { if (tResponse.IsFaulted) { throw tResponse.Exception ?? new Exception(); } if (consumer.AcknowledgedTags.Contains(args.DeliveryTag)) { return; } if (tResponse.Result == null) { return; } _logger.LogDebug($"Sending response to request with correlation '{args.BasicProperties.CorrelationId}'."); consumer.Model.BasicPublish( exchange: string.Empty, routingKey: args.BasicProperties.ReplyTo, basicProperties: _propertyProvider.GetProperties <TResponse>(p => p.CorrelationId = args.BasicProperties.CorrelationId), body: _serializer.Serialize(tResponse.Result) ); })); }, exception => _errorHandling.OnResponseHandlerExceptionAsync(consumer, cfg, args, exception)); consumer.Model.BasicConsume(cfg.Queue.QueueName, cfg.NoAck, consumer); return(new Subscription(consumer, cfg.Queue.QueueName)); }); Task.WaitAll(respondTask); _subscriptions.Add(respondTask.Result); return(respondTask.Result); }
public override async Task Execute(HorseClient client, HorseMessage message, object model) { TModel t = (TModel)model; Exception exception = null; IConsumerFactory consumerFactory = null; try { if (_consumer != null) { await Consume(_consumer, message, t, client); } else if (_consumerFactoryCreator != null) { consumerFactory = _consumerFactoryCreator(); object consumerObject = await consumerFactory.CreateConsumer(_consumerType); IDirectConsumer <TModel> consumer = (IDirectConsumer <TModel>)consumerObject; await Consume(consumer, message, t, client); } else { throw new ArgumentNullException("There is no consumer defined"); } if (SendAck) { await client.SendAck(message); } } catch (Exception e) { if (SendNack) { await SendNegativeAck(message, client, e); } await SendExceptions(message, client, e); exception = e; } finally { if (consumerFactory != null) { consumerFactory.Consumed(exception); } } }
public virtual IDisposable Consume(IQueue queue, Func <Byte[], MessageProperties, MessageReceivedInfo, Task> onMessage) { Preconditions.CheckNotNull(queue, "queue"); Preconditions.CheckNotNull(onMessage, "onMessage"); if (disposed) { throw new EasyNetQException("This bus has been disposed"); } var consumer = consumerFactory.CreateConsumer(queue, onMessage, connection); return(consumer.StartConsuming()); }
private Task <IRawConsumer> GetOrCreateConsumerAsync(IConsumerConfiguration cfg) { return(_channelFactory .GetChannelAsync() .ContinueWith(tChannel => { var consumerCs = new ConsumerCompletionSource(); if (_consumerCompletionSources.TryAdd(tChannel.Result, consumerCs)) { var newConsumer = _consumerFactory.CreateConsumer(cfg, tChannel.Result); WireUpConsumer(newConsumer, cfg); consumerCs.ConsumerQueues.TryAdd( key: cfg.Queue.FullQueueName, value: newConsumer.Model.BasicConsume(cfg.Queue.FullQueueName, cfg.NoAck, newConsumer) ); _currentConsumer = consumerCs; _logger.LogInformation($"Created consumer on channel '{tChannel.Result.ChannelNumber}' that consumes messages from '{cfg.Queue.FullQueueName}'."); consumerCs.TrySetResult(newConsumer); return consumerCs.Task; } _logger.LogDebug($"Consumer for channel '{tChannel.Result.ChannelNumber}' exists. Using it."); consumerCs = _consumerCompletionSources[tChannel.Result]; if (consumerCs.ConsumerQueues.ContainsKey(cfg.Queue.FullQueueName)) { return consumerCs.Task; } return consumerCs.Task.ContinueWith(t => { lock (consumerCs.Consumer) { if (consumerCs.ConsumerQueues.ContainsKey(cfg.Queue.FullQueueName)) { return t.Result; } consumerCs.ConsumerQueues.TryAdd( key: cfg.Queue.FullQueueName, value: consumerCs.Consumer.Model.BasicConsume(cfg.Queue.FullQueueName, cfg.NoAck, consumerCs.Consumer) ); _logger.LogDebug($"Existign consumer for channel '{tChannel.Result.ChannelNumber}' consumes '{cfg.Queue.FullQueueName}'."); return t.Result; } }); }) .Unwrap()); }
public virtual IDisposable Consume(IQueue queue, Func <byte[], MessageProperties, MessageReceivedInfo, Task> onMessage, Action <IConsumerConfiguration> configure) { Preconditions.CheckNotNull(queue, "queue"); Preconditions.CheckNotNull(onMessage, "onMessage"); Preconditions.CheckNotNull(configure, "configure"); if (disposed) { throw new EasyNetQException("This bus has been disposed"); } var consumerConfiguration = new ConsumerConfiguration(connectionConfiguration.PrefetchCount); configure(consumerConfiguration); var consumer = consumerFactory.CreateConsumer(queue, (body, properties, receviedInfo) => { var rawMessage = produceConsumeInterceptor.OnConsume(new RawMessage(properties, body)); return(onMessage(rawMessage.Body, rawMessage.Properties, receviedInfo)); }, connection, consumerConfiguration); return(consumer.StartConsuming()); }
public KafkaOrchestratorBase(IConsumerFactory consumerFactory) { _consumer = consumerFactory.CreateConsumer(Dns.GetHostName()); _cancellationTokenSource = new CancellationTokenSource(); }
public virtual void Consume(IQueue queue, Func <Byte[], MessageProperties, MessageReceivedInfo, Task> onMessage) { Preconditions.CheckNotNull(queue, "queue"); Preconditions.CheckNotNull(onMessage, "onMessage"); if (disposed) { throw new EasyNetQException("This bus has been disposed"); } var newConsumerTag = conventions.ConsumerTagConvention(); var subscriptionAction = new SubscriptionAction(newConsumerTag, logger, queue.IsSingleUse, queue.IsExclusive); subscriptionAction.Action = (isNewConnection) => { // recreate channel if current channel is no longer open or connection was dropped and reconnected (to survive server restart) if (subscriptionAction.Channel == null || subscriptionAction.Channel.IsOpen == false || isNewConnection) { subscriptionAction.Channel = CreateChannel(queue); } var channel = subscriptionAction.Channel; channel.BasicQos(0, connectionConfiguration.PrefetchCount, false); var consumer = consumerFactory.CreateConsumer(subscriptionAction, channel, queue.IsSingleUse, (consumerTag, deliveryTag, redelivered, exchange, routingKey, properties, body) => { var messageRecievedInfo = new MessageReceivedInfo { ConsumerTag = consumerTag, DeliverTag = deliveryTag, Redelivered = redelivered, Exchange = exchange, RoutingKey = routingKey }; var messsageProperties = new MessageProperties(properties); return(onMessage(body, messsageProperties, messageRecievedInfo)); }); var cancelNotifications = consumer as IConsumerCancelNotifications; if (cancelNotifications != null) { cancelNotifications.BasicCancel += OnBasicCancel; } channel.BasicConsume( queue.Name, // queue NoAck, // noAck consumer.ConsumerTag, // consumerTag consumer); // consumer logger.DebugWrite("Declared Consumer. queue='{0}', consumer tag='{1}' prefetchcount={2}", queue.Name, consumer.ConsumerTag, connectionConfiguration.PrefetchCount); }; AddSubscriptionAction(subscriptionAction); }
public override async Task Execute(HorseClient client, HorseMessage message, object model) { Exception exception = null; IConsumerFactory consumerFactory = null; bool respond = false; try { TRequest requestModel = (TRequest)model; IHorseRequestHandler <TRequest, TResponse> handler; if (_handler != null) { handler = _handler; } else if (_handlerFactoryCreator != null) { consumerFactory = _handlerFactoryCreator(); object consumerObject = await consumerFactory.CreateConsumer(_handlerType); handler = (IHorseRequestHandler <TRequest, TResponse>)consumerObject; } else { throw new ArgumentNullException("There is no consumer defined"); } try { TResponse responseModel = await Handle(handler, requestModel, message, client); HorseResultCode code = responseModel is null ? HorseResultCode.NoContent : HorseResultCode.Ok; HorseMessage responseMessage = message.CreateResponse(code); if (responseModel != null) { responseMessage.Serialize(responseModel, client.JsonSerializer); } respond = true; await client.SendAsync(responseMessage); } catch (Exception e) { ErrorResponse errorModel = await handler.OnError(e, requestModel, message, client); if (errorModel.ResultCode == HorseResultCode.Ok) { errorModel.ResultCode = HorseResultCode.Failed; } HorseMessage responseMessage = message.CreateResponse(errorModel.ResultCode); if (!string.IsNullOrEmpty(errorModel.Reason)) { responseMessage.SetStringContent(errorModel.Reason); } respond = true; await client.SendAsync(responseMessage); throw; } } catch (Exception e) { if (!respond) { try { HorseMessage response = message.CreateResponse(HorseResultCode.InternalServerError); await client.SendAsync(response); } catch { } } await SendExceptions(message, client, e); exception = e; } finally { if (consumerFactory != null) { consumerFactory.Consumed(exception); } } }
public override Task OnConnectedAsync() { Consumer consumerToRegister; string clientConnectionId = Context.ConnectionId; var headers = Context.GetHttpContext().Request.Headers; // authorization tokens // TODO: Implement token validation string tenantToken = headers["x-andyx-tenant-authoriziation"]; string componentToken = headers["x-andyx-component-authoriziation"]; string tenant = headers["x-andyx-tenant"].ToString(); string product = headers["x-andyx-product"].ToString(); string component = headers["x-andyx-component"].ToString(); string topic = headers["x-andyx-topic"].ToString(); bool isPersistent = bool.Parse(headers["x-andyx-topic-is-persistent"]); string consumerName = headers["x-andyx-consumer"].ToString(); SubscriptionType subscriptionType = (SubscriptionType)Enum.Parse(typeof(SubscriptionType), headers["x-andyx-consumer-type"].ToString()); InitialPosition initialPosition = (InitialPosition)Enum.Parse(typeof(InitialPosition), headers["x-andyx-consumer-initial-position"].ToString()); logger.LogInformation($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} requested connection"); // check if the consumer is already connected var connectedTenant = tenantRepository.GetTenant(tenant); if (connectedTenant == null) { logger.LogInformation($"Consumer '{consumerName}' failed to connect, tenant '{tenant}' does not exists"); return(OnDisconnectedAsync(new Exception($"There is no tenant registered with this name '{tenant}'"))); } // check tenant token validation bool isTenantTokenValidated = tenantRepository.ValidateTenantToken(tenant, tenantToken); if (isTenantTokenValidated != true) { logger.LogInformation($"Consumer '{consumerName}' failed to connect, access is forbidden. Not authorized"); return(OnDisconnectedAsync(new Exception($"Consumer '{consumerName}' failed to connect, access is forbidden"))); } var connectedProduct = tenantRepository.GetProduct(tenant, product); if (connectedProduct == null) { if (connectedTenant.Settings.AllowProductCreation != true) { logger.LogInformation($"Consumer '{consumerName}' failed to connect, tenant '{tenant}' does not allow to create new product"); return(OnDisconnectedAsync(new Exception($"There is no product registered with this name '{product}'. Tenant '{tenant}' does not allow to create new product"))); } var productDetails = tenantFactory.CreateProduct(product); tenantRepository.AddProduct(tenant, product, productDetails); storageHubService.CreateProductAsync(tenant, productDetails); } else { storageHubService.UpdateProductAsync(tenant, connectedProduct); } var connectedComponent = tenantRepository.GetComponent(tenant, product, component); if (connectedComponent == null) { var componentDetails = tenantFactory.CreateComponent(component); tenantRepository.AddComponent(tenant, product, component, componentDetails); storageHubService.CreateComponentAsync(tenant, product, componentDetails); } else { // check component token validation bool isComponentTokenValidated = tenantRepository.ValidateComponentToken(tenant, product, component, componentToken, consumerName, true); if (isComponentTokenValidated != true) { logger.LogInformation($"Consumer '{consumerName}' failed to connect, access is forbidden. Not authorized, check component token"); return(OnDisconnectedAsync(new Exception($"Consumer '{consumerName}' failed to connect, access is forbidden, check component token"))); } storageHubService.UpdateComponentAsync(tenant, product, connectedComponent); } var connectedTopic = tenantRepository.GetTopic(tenant, product, component, topic); if (connectedTopic == null) { connectedComponent = tenantRepository.GetComponent(tenant, product, component); if (connectedComponent.Settings.AllowTopicCreation != true) { logger.LogInformation($"Component '{component}' does not allow to create a new topic {topic} at '{tenant}/{product}/{component}'. To allow creating update property AllowTopicCreation at component."); return(OnDisconnectedAsync(new Exception($"Component '{component}' does not allow to create a new topic {topic} at '{tenant}/{product}/{component}'. To allow creating update property AllowTopicCreation at component."))); } var topicDetails = tenantFactory.CreateTopic(topic, isPersistent); tenantRepository.AddTopic(tenant, product, component, topic, topicDetails); storageHubService.CreateTopicAsync(tenant, product, component, topicDetails); } else { storageHubService.UpdateTopicAsync(tenant, product, component, connectedTopic); } string consumerIdOnRepo = $"{tenant}{product}{component}{topic}|{consumerName}"; var consumerConencted = consumerHubRepository.GetConsumerById(consumerIdOnRepo); if (consumerConencted != null) { if (subscriptionType == SubscriptionType.Exclusive) { logger.LogWarning($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} is already connected"); return(OnDisconnectedAsync(new Exception($"There is a consumer with name '{consumerName}' and with type 'EXCLUSIVE' is connected to this node"))); } if (subscriptionType == SubscriptionType.Failover) { if (consumerConencted.Connections.Count >= 2) { logger.LogWarning($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} is already connected with 2 instances"); return(OnDisconnectedAsync(new Exception($"There are two consumers with name '{consumerName}' and with type 'Failover' are connected to this node"))); } } } consumerToRegister = consumerFactory.CreateConsumer(tenant, product, component, topic, consumerName, subscriptionType, initialPosition); consumerHubRepository.AddConsumer(consumerIdOnRepo, consumerToRegister); consumerHubRepository.AddConsumerConnection(consumerIdOnRepo, clientConnectionId); storageHubService.ConnectConsumerAsync(consumerToRegister); Clients.Caller.ConsumerConnected(new Model.Consumers.Events.ConsumerConnectedDetails() { Id = consumerToRegister.Id, Tenant = tenant, Product = product, Component = component, Topic = topic, ConsumerName = consumerName, SubscriptionType = subscriptionType, InitialPosition = initialPosition }); // if consumer is not persistent, do not store the message, just allow streaming if (isPersistent == true) { // Sent not acknoledged messages to this consumer (for exclusive and for the first shared/failover consumer connected) if (subscriptionType == SubscriptionType.Exclusive) { storageHubService.RequestUnacknowledgedMessagesConsumer(consumerToRegister); } if (subscriptionType == SubscriptionType.Shared || subscriptionType == SubscriptionType.Failover) { if (consumerConencted == null) { storageHubService.RequestUnacknowledgedMessagesConsumer(consumerToRegister); } } } logger.LogInformation($"Consumer '{consumerName}' and subscription type '{subscriptionType}' at {tenant}/{product}/{component}/{topic} is connected"); return(base.OnConnectedAsync()); }