public Consumer(string groupName, ConsumerSetting setting) { if (groupName == null) { throw new ArgumentNullException("groupName"); } GroupName = groupName; Setting = setting ?? new ConsumerSetting(); _lockObject = new object(); _subscriptionTopics = new Dictionary<string, HashSet<string>>(); _topicQueuesDict = new ConcurrentDictionary<string, IList<MessageQueue>>(); _pullRequestQueue = new BlockingCollection<PullRequest>(new ConcurrentQueue<PullRequest>()); _pullRequestDict = new ConcurrentDictionary<string, PullRequest>(); _messageRetryQueue = new BlockingCollection<ConsumingMessage>(new ConcurrentQueue<ConsumingMessage>()); _taskFactory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(Setting.ConsumeThreadMaxCount)); _remotingClient = new SocketRemotingClient(Setting.BrokerAddress, Setting.SocketSetting, Setting.LocalAddress); _adminRemotingClient = new SocketRemotingClient(Setting.BrokerAdminAddress, Setting.SocketSetting, Setting.LocalAdminAddress); _binarySerializer = ObjectContainer.Resolve<IBinarySerializer>(); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _allocateMessageQueueStragegy = ObjectContainer.Resolve<IAllocateMessageQueueStrategy>(); _executePullRequestWorker = new Worker("ExecutePullRequest", ExecutePullRequest); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); _remotingClient.RegisterConnectionEventListener(new ConnectionEventListener(this)); }
/// <summary>Parameterized constructor. /// </summary> /// <param name="eventHandlerTypeCodeProvider"></param> /// <param name="eventHandlerProvider"></param> /// <param name="eventPublishInfoStore"></param> /// <param name="eventHandleInfoStore"></param> /// <param name="eventHandleInfoCache"></param> /// <param name="actionExecutionService"></param> /// <param name="loggerFactory"></param> public DefaultEventProcessor( IEventHandlerTypeCodeProvider eventHandlerTypeCodeProvider, IEventHandlerProvider eventHandlerProvider, IEventPublishInfoStore eventPublishInfoStore, IEventHandleInfoStore eventHandleInfoStore, IEventHandleInfoCache eventHandleInfoCache, IActionExecutionService actionExecutionService, ILoggerFactory loggerFactory) { _eventHandlerTypeCodeProvider = eventHandlerTypeCodeProvider; _eventHandlerProvider = eventHandlerProvider; _eventPublishInfoStore = eventPublishInfoStore; _eventHandleInfoStore = eventHandleInfoStore; _eventHandleInfoCache = eventHandleInfoCache; _actionExecutionService = actionExecutionService; _logger = loggerFactory.Create(GetType().Name); _queueList = new List<BlockingCollection<EventProcessingContext>>(); for (var index = 0; index < WorkerCount; index++) { _queueList.Add(new BlockingCollection<EventProcessingContext>(new ConcurrentQueue<EventProcessingContext>())); } _workerList = new List<Worker>(); for (var index = 0; index < WorkerCount; index++) { var queue = _queueList[index]; var worker = new Worker(() => { DispatchEventsToHandlers(queue.Take()); }); _workerList.Add(worker); worker.Start(); } }
/// <summary>Parameterized constructor. /// </summary> /// <param name="loggerFactory"></param> public DefaultActionExecutionService(ILoggerFactory loggerFactory) { _logger = loggerFactory.Create(GetType().Name); _actionQueue = new BlockingCollection<ActionInfo>(new ConcurrentQueue<ActionInfo>()); _worker = new Worker(TryTakeAndExecuteAction, DefaultPeriod); _worker.Start(); }
public Consumer(string groupName, ConsumerSetting setting) { if (groupName == null) { throw new ArgumentNullException("groupName"); } GroupName = groupName; Setting = setting ?? new ConsumerSetting(); _lockObject = new object(); _subscriptionTopics = new Dictionary<string, HashSet<string>>(); _topicQueuesDict = new ConcurrentDictionary<string, IList<MessageQueue>>(); _pullRequestDict = new ConcurrentDictionary<string, PullRequest>(); _remotingClient = new SocketRemotingClient(Setting.BrokerAddress, Setting.SocketSetting, Setting.LocalAddress); _adminRemotingClient = new SocketRemotingClient(Setting.BrokerAdminAddress, Setting.SocketSetting, Setting.LocalAdminAddress); _binarySerializer = ObjectContainer.Resolve<IBinarySerializer>(); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _allocateMessageQueueStragegy = ObjectContainer.Resolve<IAllocateMessageQueueStrategy>(); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); _remotingClient.RegisterConnectionEventListener(new ConnectionEventListener(this)); if (Setting.MessageHandleMode == MessageHandleMode.Sequential) { _consumingMessageQueue = new BlockingCollection<ConsumingMessage>(); _consumeMessageWorker = new Worker("ConsumeMessage", () => HandleMessage(_consumingMessageQueue.Take())); } _messageRetryQueue = new BlockingCollection<ConsumingMessage>(); }
public SuspendedPullRequestManager() { _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _messageService = ObjectContainer.Resolve<IMessageService>(); _worker = new Worker(() => { var notifyItem = _notifyQueue.Take(); NotifyMessageArrived(BuildKey(notifyItem.Topic, notifyItem.QueueId), notifyItem.QueueOffset); }); }
public SocketRemotingClient(string address, int port) { _address = address; _port = port; _clientSocket = new ClientSocket(); _responseFutureDict = new ConcurrentDictionary<long, ResponseFuture>(); _responseMessageQueue = new BlockingCollection<byte[]>(new ConcurrentQueue<byte[]>()); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _processResponseMessageWorker = new Worker(ProcessResponseMessage); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().Name); _clientSocket.Connect(address, port); }
public CommandResultProcessor(IPEndPoint bindingAddress) { _remotingServer = new SocketRemotingServer("CommandResultProcessor.RemotingServer", bindingAddress); _commandTaskDict = new ConcurrentDictionary<string, CommandTaskCompletionSource>(); _commandExecutedMessageLocalQueue = new BlockingCollection<CommandResult>(new ConcurrentQueue<CommandResult>()); _domainEventHandledMessageLocalQueue = new BlockingCollection<DomainEventHandledMessage>(new ConcurrentQueue<DomainEventHandledMessage>()); _commandExecutedMessageWorker = new Worker("ProcessExecutedCommandMessage", () => ProcessExecutedCommandMessage(_commandExecutedMessageLocalQueue.Take())); _domainEventHandledMessageWorker = new Worker("ProcessDomainEventHandledMessage", () => ProcessDomainEventHandledMessage(_domainEventHandledMessageLocalQueue.Take())); _jsonSerializer = ObjectContainer.Resolve<IJsonSerializer>(); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); BindingAddress = bindingAddress; }
public SuspendedPullRequestManager() { _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _queueStore = ObjectContainer.Resolve<IQueueStore>(); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); _notifyQueue = new BlockingCollection<NotifyItem>(new ConcurrentQueue<NotifyItem>()); _notifyMessageArrivedWorker = new Worker("NotifyMessageArrived", () => { var notifyItem = _notifyQueue.Take(); if (notifyItem == null) return; NotifyMessageArrived(notifyItem.Topic, notifyItem.QueueId, notifyItem.QueueOffset); }); }
public SocketRemotingClient(IPEndPoint serverEndPoint, RemotingClientSetting setting = null, ISocketClientEventListener eventListener = null) { _sync = new object(); _serverEndPoint = serverEndPoint; _eventListener = eventListener; _setting = setting ?? new RemotingClientSetting(); _tcpClient = new TcpSocketClient(_setting.LocalEndPoint, serverEndPoint, ReceiveReplyMessage, this); _responseFutureDict = new ConcurrentDictionary<long, ResponseFuture>(); _replyMessageQueue = new BlockingCollection<byte[]>(new ConcurrentQueue<byte[]>()); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _worker = new Worker("SocketRemotingClient.HandleReplyMessage", HandleReplyMessage); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); }
public SuspendedPullRequestManager() { _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _queueStore = ObjectContainer.Resolve<IQueueStore>(); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); _notifyQueue = new BlockingCollection<NotifyItem>(new ConcurrentQueue<NotifyItem>()); _taskFactory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(Environment.ProcessorCount)); _checkBlockingPullRequestTaskName = string.Format("{0}.CheckBlockingPullRequest", this.GetType().Name); _notifyMessageArrivedWorker = new Worker(string.Format("{0}.NotifyMessageArrived", this.GetType().Name), () => { var notifyItem = _notifyQueue.Take(); if (notifyItem == null) return; NotifyMessageArrived(notifyItem.Topic, notifyItem.QueueId, notifyItem.QueueOffset); }); }
public SocketRemotingClient(EndPoint serverEndPoint, EndPoint localEndPoint = null) { _serverEndPoint = serverEndPoint; _localEndPoint = localEndPoint; _clientSocket = new ClientSocket(serverEndPoint, localEndPoint, ReceiveReplyMessage); _responseFutureDict = new ConcurrentDictionary<long, ResponseFuture>(); _replyMessageQueue = new BlockingCollection<byte[]>(new ConcurrentQueue<byte[]>()); _responseHandlerDict = new Dictionary<int, IResponseHandler>(); _connectionEventListeners = new List<IConnectionEventListener>(); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _worker = new Worker("SocketRemotingClient.HandleReplyMessage", HandleReplyMessage); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); RegisterConnectionEventListener(new ConnectionEventListener(this)); }
public SuspendedPullRequestManager(BrokerController brokerController) { _brokerController = brokerController; _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _messageService = ObjectContainer.Resolve<IMessageService>(); if (_brokerController.Setting.NotifyWhenMessageArrived) { _notifyMessageArrivedWorker = new Worker("SuspendedPullRequestManager.NotifyMessageArrived", () => { var notifyItem = _notifyQueue.Take(); if (notifyItem == null) return; _queueNotifyOffsetDict[BuildKeyPrefix(notifyItem.Topic, notifyItem.QueueId)] = notifyItem.QueueOffset; NotifyMessageArrived(notifyItem.Topic, notifyItem.QueueId, notifyItem.QueueOffset); }); } }
public void Start() { _queueRequestDict.Clear(); StopCheckBlockingPullRequestTask(); StopNotifyMessageArrivedWorker(); _notifyQueue = new BlockingCollection<NotifyItem>(new ConcurrentQueue<NotifyItem>()); _taskFactory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(BrokerController.Instance.Setting.NotifyMessageArrivedThreadMaxCount)); if (BrokerController.Instance.Setting.NotifyWhenMessageArrived) { _notifyMessageArrivedWorker = new Worker("SuspendedPullRequestManager.NotifyMessageArrived", () => { var notifyItem = _notifyQueue.Take(); if (notifyItem == null) return; NotifyMessageArrived(notifyItem.Topic, notifyItem.QueueId, notifyItem.QueueOffset); }); } StartCheckBlockingPullRequestTask(); StartNotifyMessageArrivedWorker(); }
public PullRequest( string consumerId, string groupName, MessageQueue messageQueue, SocketRemotingClient remotingClient, MessageHandleMode messageHandleMode, IMessageHandler messageHandler, IOffsetStore offsetStore, PullRequestSetting setting) { ConsumerId = consumerId; GroupName = groupName; MessageQueue = messageQueue; ProcessQueue = new ProcessQueue(); _queueOffset = -1; _remotingClient = remotingClient; _setting = setting; _messageHandleMode = messageHandleMode; _messageHandler = messageHandler; _offsetStore = offsetStore; _messageQueue = new BlockingCollection<WrappedMessage>(new ConcurrentQueue<WrappedMessage>()); _handlingMessageDict = new ConcurrentDictionary<long, WrappedMessage>(); _pullMessageWorker = new Worker(() => { try { PullMessage(); } catch (Exception ex) { if (!_stoped) { _logger.Error(string.Format("[{0}]: PullMessage has unknown exception. PullRequest: {1}.", ConsumerId, this), ex); } } }); _handleMessageWorker = new Worker(HandleMessage); _binarySerializer = ObjectContainer.Resolve<IBinarySerializer>(); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().Name); }
public PullMessageService(Consumer consumer, ClientService clientService) { _consumer = consumer; _clientService = clientService; _clientId = clientService.GetClientId(); _binarySerializer = ObjectContainer.Resolve<IBinarySerializer>(); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); if (consumer.Setting.AutoPull) { if (consumer.Setting.MessageHandleMode == MessageHandleMode.Sequential) { _consumingMessageQueue = new BlockingCollection<ConsumingMessage>(); _consumeMessageWorker = new Worker("ConsumeMessage", () => HandleMessage(_consumingMessageQueue.Take())); } _messageRetryQueue = new BlockingCollection<ConsumingMessage>(); } else { _pulledMessageQueue = new BlockingCollection<QueueMessage>(); } }
public void Start(Action<ReceiveContext> messageReceivedCallback) { if (_isRunning) return; _isRunning = true; _listenNewClientWorker = new Worker("ServerSocket.AcceptNewClient", () => { if (!_isRunning) return; _messageReceivedCallback = messageReceivedCallback; _newClientSocketSignal.Reset(); try { _socket.BeginAccept((asyncResult) => { if (!_isRunning) return; var clientSocket = _socket.EndAccept(asyncResult); var socketInfo = new SocketInfo(clientSocket); NotifyNewSocketAccepted(socketInfo); _newClientSocketSignal.Set(); if (!_isRunning) return; _socketService.ReceiveMessage(socketInfo, receivedMessage => { var receiveContext = new ReceiveContext(socketInfo, receivedMessage, context => { _socketService.SendMessage(context.ReplySocketInfo, context.ReplyMessage, sendResult => { }); }); _messageReceivedCallback(receiveContext); }); }, _socket); } catch (SocketException socketException) { _logger.Error(string.Format("Socket accept exception, ErrorCode:{0}", socketException.SocketErrorCode), socketException); } catch (Exception ex) { _logger.Error("Unknown socket accept exception.", ex); } if (_isRunning) { _newClientSocketSignal.WaitOne(); } }); _listenNewClientWorker.Start(); }
/// <summary>Parameterized costructor. /// </summary> /// <param name="commandExecutor"></param> public ProcessingCommandProcessor() { _queue = new BlockingCollection<ProcessingCommand>(new ConcurrentQueue<ProcessingCommand>()); _worker = new Worker(() => _commandExecutor.Execute(_queue.Take())); }
public Consumer(string id, string groupName, ConsumerSetting setting) { if (id == null) { throw new ArgumentNullException("id"); } if (groupName == null) { throw new ArgumentNullException("groupName"); } Id = id; GroupName = groupName; Setting = setting ?? new ConsumerSetting(); _lockObject = new object(); _subscriptionTopics = new List<string>(); _topicQueuesDict = new ConcurrentDictionary<string, IList<MessageQueue>>(); _pullRequestQueue = new BlockingCollection<PullRequest>(new ConcurrentQueue<PullRequest>()); _pullRequestDict = new ConcurrentDictionary<string, PullRequest>(); _consumingMessageQueue = new BlockingCollection<ConsumingMessage>(new ConcurrentQueue<ConsumingMessage>()); _messageRetryQueue = new BlockingCollection<ConsumingMessage>(new ConcurrentQueue<ConsumingMessage>()); _handlingMessageDict = new ConcurrentDictionary<long, ConsumingMessage>(); _taskIds = new List<int>(); _taskFactory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(Setting.ConsumeThreadMaxCount)); _remotingClient = new SocketRemotingClient(Setting.BrokerConsumerIPEndPoint, null, this); _binarySerializer = ObjectContainer.Resolve<IBinarySerializer>(); _scheduleService = ObjectContainer.Resolve<IScheduleService>(); _allocateMessageQueueStragegy = ObjectContainer.Resolve<IAllocateMessageQueueStrategy>(); _executePullRequestWorker = new Worker("Consumer.ExecutePullRequest", ExecutePullRequest); _handleMessageWorker = new Worker("Consumer.HandleMessage", HandleMessage); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(GetType().FullName); _waitSocketConnectHandle = new AutoResetEvent(false); }
static void Main(string[] args) { try { Configuration.Instance.UseLog4Net() .RegisterMessageContextType(typeof(IFramework.MessageQueue.EQueue.MessageFormat.MessageContext)) .InitliaizeEQueue(5000, 5001, 5000) .CommandHandlerProviderBuild(null, "CommandHandlers"); var consumerSetting = new ConsumerSetting(); consumerSetting.MessageHandleMode = MessageHandleMode.Sequential; consumerSetting.BrokerPort = 5001; var producerSetting = new ProducerSetting(); var producerPort = producerSetting.BrokerPort = 5000; var eventHandlerProvider = IoCFactory.Resolve<IHandlerProvider>("AsyncDomainEventSubscriber"); IMessageConsumer domainEventSubscriber = new DomainEventSubscriber("domainEventSubscriber1", consumerSetting, "DomainEventSubscriber", "domainevent", eventHandlerProvider); domainEventSubscriber.Start(); IoCFactory.Instance.CurrentContainer.RegisterInstance("DomainEventConsumer", domainEventSubscriber); IEventPublisher eventPublisher = new EventPublisher("EventPublisher", "domainevent", producerSetting); IoCFactory.Instance.CurrentContainer.RegisterInstance(typeof(IEventPublisher), eventPublisher, new ContainerControlledLifetimeManager()); var commandHandlerProvider = IoCFactory.Resolve<ICommandHandlerProvider>(); var commandConsumer1 = new CommandConsumer("consumer1", consumerSetting, "CommandConsumerGroup", "Command", consumerSetting.BrokerAddress, producerPort, commandHandlerProvider); var commandConsumer2 = new CommandConsumer("consumer2", consumerSetting, "CommandConsumerGroup", "Command", consumerSetting.BrokerAddress, producerPort, commandHandlerProvider); var commandConsumer3 = new CommandConsumer("consumer3", consumerSetting, "CommandConsumerGroup", "Command", consumerSetting.BrokerAddress, producerPort, commandHandlerProvider); var commandConsumer4 = new CommandConsumer("consumer4", consumerSetting, "CommandConsumerGroup", "Command", consumerSetting.BrokerAddress, producerPort, commandHandlerProvider); commandConsumer1.Start(); commandConsumer2.Start(); commandConsumer3.Start(); commandConsumer4.Start(); commandBus = new CommandBus("CommandBus", commandHandlerProvider, IoCFactory.Resolve<ILinearCommandManager>(), consumerSetting.BrokerAddress, producerPort, consumerSetting, "CommandBus", "Reply", "Command", true); IoCFactory.Instance.CurrentContainer.RegisterInstance(typeof(ICommandBus), commandBus, new ContainerControlledLifetimeManager()); commandBus.Start(); //Below to wait for consumer balance. var scheduleService = ObjectContainer.Resolve<IScheduleService>(); var waitHandle = new ManualResetEvent(false); var taskId = scheduleService.ScheduleTask("consumer logs", () => { var bAllocatedQueueIds = (commandBus as CommandBus).Consumer.GetCurrentQueues().Select(x => x.QueueId); var c1AllocatedQueueIds = commandConsumer1.Consumer.GetCurrentQueues().Select(x => x.QueueId); var c2AllocatedQueueIds = commandConsumer2.Consumer.GetCurrentQueues().Select(x => x.QueueId); var c3AllocatedQueueIds = commandConsumer3.Consumer.GetCurrentQueues().Select(x => x.QueueId); var c4AllocatedQueueIds = commandConsumer4.Consumer.GetCurrentQueues().Select(x => x.QueueId); var eAllocatedQueueIds = (domainEventSubscriber as DomainEventSubscriber).Consumer.GetCurrentQueues().Select(x => x.QueueId); Console.WriteLine(string.Format("Consumer message queue allocation result:bus:{0}, eventSubscriber:{1} c1:{2}, c2:{3}, c3:{4}, c4:{5}", string.Join(",", bAllocatedQueueIds), string.Join(",", eAllocatedQueueIds), string.Join(",", c1AllocatedQueueIds), string.Join(",", c2AllocatedQueueIds), string.Join(",", c3AllocatedQueueIds), string.Join(",", c4AllocatedQueueIds))); if (eAllocatedQueueIds.Count() == 4 && bAllocatedQueueIds.Count() == 4 && c1AllocatedQueueIds.Count() == 1 && c2AllocatedQueueIds.Count() == 1 && c3AllocatedQueueIds.Count() == 1 && c4AllocatedQueueIds.Count() == 1) { waitHandle.Set(); } }, 1000, 1000); waitHandle.WaitOne(); scheduleService.ShutdownTask(taskId); var worker = new Worker(commandBus); //worker.StartTest(100); while (true) { Console.WriteLine(CommandConsumer.GetConsumersStatus()); Console.WriteLine(domainEventSubscriber.GetStatus()); Console.WriteLine("please input batch count and rerun another test:"); var input = Console.ReadLine(); var batchCount = 0; if (int.TryParse(input, out batchCount)) { worker.StartTest(batchCount); } } } catch (Exception ex) { Console.WriteLine(ex.GetBaseException().Message, ex); } finally { Console.Read(); } }