public void CommitConsumeOffset(PullRequest pullRequest) { var consumedOffset = pullRequest.ProcessQueue.GetConsumedQueueOffset(); if (consumedOffset >= 0) { if (!pullRequest.ProcessQueue.TryUpdatePreviousConsumedQueueOffset(consumedOffset)) { return; } CommitConsumeOffset(pullRequest.MessageQueue, consumedOffset); } }
private void PersistOffset(PullRequest pullRequest) { try { if (Setting.MessageModel == MessageModel.BroadCasting) { ((ILocalOffsetStore)_offsetStore).PersistQueueOffset(GroupName, pullRequest.MessageQueue); } else if (Setting.MessageModel == MessageModel.Clustering) { var queueOffset = _offsetStore.GetQueueOffset(GroupName, pullRequest.MessageQueue); var request = new UpdateQueueOffsetRequest(GroupName, pullRequest.MessageQueue, queueOffset); var remotingRequest = new RemotingRequest((int)RequestCode.UpdateQueueOffsetRequest, _binarySerializer.Serialize(request)); _remotingClient.InvokeOneway(remotingRequest, 10000); } } catch (Exception ex) { _logger.Error(string.Format("[{0}]: PersistOffset has exception.", Id), ex); } }
private void SchedulePullRequest(PullRequest pullRequest) { Task.Factory.StartNew(ExecutePullRequest, pullRequest); }
private void ProcessPullResponse(PullRequest pullRequest, RemotingResponse remotingResponse) { if (remotingResponse == null) { _logger.ErrorFormat("Pull message response is null, pullRequest:{0}", pullRequest); SchedulePullRequest(pullRequest); return; } if (remotingResponse.Code == -1) { _logger.ErrorFormat("Pull message failed, pullRequest:{0}, errorMsg:{1}", pullRequest, Encoding.UTF8.GetString(remotingResponse.Body)); SchedulePullRequest(pullRequest); return; } if (remotingResponse.Code == (short)PullStatus.Found) { var messages = DecodeMessages(pullRequest, remotingResponse.Body); if (messages.Count() > 0) { var filterMessages = messages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags)); pullRequest.ProcessQueue.AddMessages(filterMessages); foreach (var message in filterMessages) { var consumingMessage = new ConsumingMessage(message, pullRequest.ProcessQueue); if (Setting.MessageHandleMode == MessageHandleMode.Sequential) { _consumingMessageQueue.Add(consumingMessage); } else { Task.Factory.StartNew(HandleMessage, consumingMessage); } } pullRequest.NextConsumeOffset = messages.Last().QueueOffset + 1; } } else if (remotingResponse.Code == (short)PullStatus.NextOffsetReset) { var newOffset = BitConverter.ToInt64(remotingResponse.Body, 0); var oldOffset = pullRequest.NextConsumeOffset; pullRequest.NextConsumeOffset = newOffset; _logger.InfoFormat("Reset queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldOffset, newOffset); } else if (remotingResponse.Code == (short)PullStatus.NoNewMessage) { if (_logger.IsDebugEnabled) { _logger.DebugFormat("No new message found, pullRequest:{0}", pullRequest); } } else if (remotingResponse.Code == (short)PullStatus.Ignored) { if (_logger.IsDebugEnabled) { _logger.DebugFormat("Pull request was ignored, pullRequest:{0}", pullRequest); } return; } else if (remotingResponse.Code == (short)PullStatus.BrokerIsCleaning) { Thread.Sleep(5000); } //Schedule the next pull request. SchedulePullRequest(pullRequest); }
private IEnumerable<QueueMessage> DecodeMessages(PullRequest pullRequest, byte[] buffer) { var messages = new List<QueueMessage>(); if (buffer == null || buffer.Length <= 4) { return messages; } try { var nextOffset = 0; var messageLength = MessageUtils.DecodeInt(buffer, nextOffset, out nextOffset); while (messageLength > 0) { var message = new QueueMessage(); var messageBytes = new byte[messageLength]; Buffer.BlockCopy(buffer, nextOffset, messageBytes, 0, messageLength); nextOffset += messageLength; message.ReadFrom(messageBytes); if (!message.IsValid()) { _logger.ErrorFormat("Invalid message, pullRequest: {0}", pullRequest); continue; } messages.Add(message); if (nextOffset >= buffer.Length) { break; } messageLength = MessageUtils.DecodeInt(buffer, nextOffset, out nextOffset); } } catch (Exception ex) { _logger.Error(string.Format("Decode pull return message has exception, pullRequest: {0}", pullRequest), ex); } return messages; }
private void ResetNextConsumeOffset(PullRequest pullRequest, long newOffset) { var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName); if (brokerConnection == null) { _logger.ErrorFormat("Reset nextConsumeOffset failed as broker is unavailable, pullRequest: {0}, newOffset: {1}", pullRequest, newOffset); return; } var remotingClient = brokerConnection.AdminRemotingClient; try { var oldOffset = pullRequest.NextConsumeOffset; pullRequest.NextConsumeOffset = newOffset; pullRequest.ProcessQueue.MarkAllConsumingMessageIgnored(); pullRequest.ProcessQueue.Reset(); var request = new UpdateQueueOffsetRequest(_consumer.GroupName, pullRequest.MessageQueue, newOffset - 1); var remotingRequest = new RemotingRequest((int)BrokerRequestCode.UpdateQueueConsumeOffsetRequest, _binarySerializer.Serialize(request)); remotingClient.InvokeOneway(remotingRequest); _logger.InfoFormat("Resetted nextConsumeOffset, [pullRequest:{0}, oldOffset:{1}, newOffset:{2}]", pullRequest, oldOffset, newOffset); } catch (Exception ex) { if (remotingClient.IsConnected) { _logger.Error(string.Format("Reset nextConsumeOffset failed, pullRequest: {0}, newOffset: {1}", pullRequest, newOffset), ex); } } }
private void ProcessPullResponse(PullRequest pullRequest, RemotingResponse remotingResponse, Action<IEnumerable<QueueMessage>> handlePulledMessageAction) { if (remotingResponse == null) { _logger.ErrorFormat("Pull message response is null, pullRequest:{0}", pullRequest); SchedulePullRequest(pullRequest); return; } if (remotingResponse.Code == -1) { _logger.ErrorFormat("Pull message failed, pullRequest:{0}, errorMsg:{1}", pullRequest, Encoding.UTF8.GetString(remotingResponse.Body)); SchedulePullRequest(pullRequest); return; } if (remotingResponse.Code == (short)PullStatus.Found) { var messages = DecodeMessages(pullRequest, remotingResponse.Body); if (messages.Count() > 0) { handlePulledMessageAction(messages); pullRequest.NextConsumeOffset = messages.Last().QueueOffset + 1; } } else if (remotingResponse.Code == (short)PullStatus.NextOffsetReset) { var newOffset = BitConverter.ToInt64(remotingResponse.Body, 0); ResetNextConsumeOffset(pullRequest, newOffset); } else if (remotingResponse.Code == (short)PullStatus.NoNewMessage) { //No new message to consume. } else if (remotingResponse.Code == (short)PullStatus.Ignored) { _logger.InfoFormat("Pull request was ignored, pullRequest:{0}", pullRequest); return; } else if (remotingResponse.Code == (short)PullStatus.BrokerIsCleaning) { Thread.Sleep(5000); } //Schedule the next pull request. SchedulePullRequest(pullRequest); }
private void UpdatePullRequestDict(string topic, IList<MessageQueue> messageQueues) { // Check message queues to remove var toRemovePullRequestKeys = new List<string>(); foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic)) { var key = pullRequest.MessageQueue.ToString(); if (!messageQueues.Any(x => x.ToString() == key)) { toRemovePullRequestKeys.Add(key); } } foreach (var pullRequestKey in toRemovePullRequestKeys) { PullRequest pullRequest; if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest)) { pullRequest.ProcessQueue.IsDropped = true; PersistOffset(pullRequest); _logger.InfoFormat("Dropped pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId); } } // Check message queues to add. foreach (var messageQueue in messageQueues) { var key = messageQueue.ToString(); PullRequest pullRequest; if (!_pullRequestDict.TryGetValue(key, out pullRequest)) { var request = new PullRequest(Id, GroupName, messageQueue, -1); if (_pullRequestDict.TryAdd(key, request)) { SchedulePullRequest(request); _logger.InfoFormat("Added pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, request.MessageQueue.Topic, request.MessageQueue.QueueId); } } } }
private void PullMessage(PullRequest pullRequest) { var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName); if (brokerConnection == null) { Task.Factory.StartDelayedTask(5 * 1000, () => SchedulePullRequest(pullRequest)); _logger.ErrorFormat("Pull message failed as the target broker connection not found, pullRequest:{0}", pullRequest); return; } var remotingClient = brokerConnection.RemotingClient; try { if (_consumer.Stopped) { return; } if (pullRequest.IsDropped) { return; } var messageCount = 0; var flowControlThreshold = 0; if (_consumer.Setting.AutoPull) { messageCount = pullRequest.ProcessQueue.GetMessageCount(); flowControlThreshold = _consumer.Setting.PullMessageFlowControlThreshold; } else { messageCount = _pulledMessageQueue.Count; flowControlThreshold = _consumer.Setting.ManualPullLocalMessageQueueMaxSize; } if (messageCount > flowControlThreshold) { var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds( messageCount, flowControlThreshold, _consumer.Setting.PullMessageFlowControlStepPercent, _consumer.Setting.PullMessageFlowControlStepWaitMilliseconds); Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest)); return; } var request = new PullMessageRequest { ConsumerId = _clientId, ConsumerGroup = _consumer.GroupName, MessageQueue = pullRequest.MessageQueue, Tags = string.Join("|", pullRequest.Tags), QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = _consumer.Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = _consumer.Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = _consumer.Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)BrokerRequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; remotingClient.InvokeAsync(remotingRequest, _consumer.Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_consumer.Stopped) { return; } if (pullRequest.IsDropped) { return; } if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result, pulledMessages => { var filterMessages = pulledMessages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags)); var consumingMessages = filterMessages.Select(x => new ConsumingMessage(x, pullRequest)).ToList(); if (_consumer.Setting.AutoPull) { pullRequest.ProcessQueue.AddMessages(consumingMessages); foreach (var consumingMessage in consumingMessages) { if (_consumer.Setting.MessageHandleMode == MessageHandleMode.Sequential) { _consumingMessageQueue.Add(consumingMessage); } else { Task.Factory.StartNew(HandleMessage, consumingMessage); } } } else { foreach (var consumingMessage in consumingMessages) { _pulledMessageQueue.Add(consumingMessage.Message); } } }); } catch (Exception ex) { if (_consumer.Stopped) { return; } if (pullRequest.IsDropped) { return; } if (remotingClient.IsConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_consumer.Stopped) { return; } if (pullRequest.IsDropped) { return; } if (remotingClient.IsConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
public ConsumingMessage(QueueMessage message, PullRequest pullRequest) { Message = message; PullRequest = pullRequest; Message.BrokerName = pullRequest.MessageQueue.BrokerName; }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } var messageCount = pullRequest.ProcessQueue.GetMessageCount(); var flowControlThreshold = Setting.PullMessageFlowControlThreshold; if (flowControlThreshold > 0 && messageCount >= flowControlThreshold) { var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds( messageCount, flowControlThreshold, Setting.PullMessageFlowControlStepPercent, Setting.PullMessageFlowControlStepWaitMilliseconds); Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest)); return; } var request = new PullMessageRequest { ConsumerId = GetConsumerId(), ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, Tags = string.Join("|", pullRequest.Tags), QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result); } catch (Exception ex) { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } if (_remotingClient.IsConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } if (_remotingClient.IsConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; var messageCount = pullRequest.ProcessQueue.GetMessageCount(); if (messageCount >= Setting.PullThresholdForQueue) { Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest)); if ((_flowControlTimes++ % 100) == 0) { _logger.DebugFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes); } return; } var request = new PullMessageRequest { ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds }; var data = _binarySerializer.Serialize(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; try { if (pullTask.Exception != null) { SchedulePullRequest(pullRequest); return; } var remotingResponse = pullTask.Result; var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body); if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0) { pullRequest.ProcessQueue.AddMessages(response.Messages); foreach (var message in response.Messages) { _consumingMessageQueue.Add(new ConsumingMessage(message, pullRequest.ProcessQueue)); } pullRequest.NextConsumeOffset += response.Messages.Count(); } else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset && response.NextOffset != null) { var oldConsumeOffset = pullRequest.NextConsumeOffset; pullRequest.NextConsumeOffset = response.NextOffset.Value; _logger.DebugFormat("Updated queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldConsumeOffset, pullRequest.NextConsumeOffset); } if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (remotingResponse.Code == (int)PullStatus.Ignored) return; SchedulePullRequest(pullRequest); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
private void UpdatePullRequestDict(KeyValuePair<string, HashSet<string>> pair, IList<MessageQueue> messageQueues) { var topic = pair.Key; var toRemovePullRequestKeys = new List<string>(); foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic)) { var key = pullRequest.MessageQueue.ToString(); if (!messageQueues.Any(x => x.ToString() == key)) { toRemovePullRequestKeys.Add(key); } } foreach (var pullRequestKey in toRemovePullRequestKeys) { PullRequest pullRequest; if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest)) { pullRequest.IsDropped = true; _commitConsumeOffsetService.CommitConsumeOffset(pullRequest); _logger.InfoFormat("Dropped pull request, consumerGroup: {0}, consumerId: {1}, queue: {2}, tags: {3}", _consumer.GroupName, _clientId, pullRequest.MessageQueue, string.Join("|", pullRequest.Tags)); } } foreach (var messageQueue in messageQueues) { var key = messageQueue.ToString(); PullRequest exist; if (!_pullRequestDict.TryGetValue(key, out exist)) { var pullRequest = new PullRequest(_clientId, _consumer.GroupName, messageQueue, -1, pair.Value); if (_pullRequestDict.TryAdd(key, pullRequest)) { _pullMessageService.SchedulePullRequest(pullRequest); _logger.InfoFormat("Added pull request, consumerGroup: {0}, consumerId: {1}, queue: {2}, tags: {3}", _consumer.GroupName, _clientId, pullRequest.MessageQueue, string.Join("|", pullRequest.Tags)); } } } }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; var messageCount = pullRequest.ProcessQueue.GetMessageCount(); if (messageCount >= Setting.PullThresholdForQueue) { Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest)); if ((_flowControlTimes++ % 100) == 0) { _logger.WarnFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes); } return; } var request = new PullMessageRequest { ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
private void SchedulePullRequest(PullRequest pullRequest) { _pullRequestQueue.Add(pullRequest); }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } var messageCount = pullRequest.ProcessQueue.GetMessageCount(); if (messageCount >= Setting.PullThresholdForQueue) { Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest)); if ((_flowControlTimes++ % 100) == 0) { _logger.WarnFormat("Pull message flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes); } return; } var request = new PullMessageRequest { ConsumerId = GetConsumerId(), ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result); } catch (Exception ex) { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } if (_remotingClient.IsConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) { return; } if (pullRequest.ProcessQueue.IsDropped) { return; } if (_remotingClient.IsConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
public void SchedulePullRequest(PullRequest pullRequest) { if (_consumer.Setting.AutoPull && _messageHandler == null) { _logger.Error("Schedule pullRequest is cancelled as the messageHandler is null."); return; } Task.Factory.StartNew(ExecutePullRequest, pullRequest); }
private void PullMessage(PullRequest pullRequest) { var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName); if (brokerConnection == null) { Task.Factory.StartDelayedTask(5 * 1000, () => SchedulePullRequest(pullRequest)); _logger.ErrorFormat("Pull message failed as the target broker connection not found, pullRequest:{0}", pullRequest); return; } var remotingClient = brokerConnection.RemotingClient; try { if (_consumer.Stopped) return; if (pullRequest.IsDropped) return; var messageCount = 0; var flowControlThreshold = 0; if (_consumer.Setting.AutoPull) { messageCount = pullRequest.ProcessQueue.GetMessageCount(); flowControlThreshold = _consumer.Setting.PullMessageFlowControlThreshold; } else { messageCount = _pulledMessageQueue.Count; flowControlThreshold = _consumer.Setting.ManualPullLocalMessageQueueMaxSize; } if (messageCount > flowControlThreshold) { var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds( messageCount, flowControlThreshold, _consumer.Setting.PullMessageFlowControlStepPercent, _consumer.Setting.PullMessageFlowControlStepWaitMilliseconds); Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest)); return; } var request = new PullMessageRequest { ConsumerId = _clientId, ConsumerGroup = _consumer.GroupName, MessageQueue = pullRequest.MessageQueue, Tags = string.Join("|", pullRequest.Tags), QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = _consumer.Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = _consumer.Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = _consumer.Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)BrokerRequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; remotingClient.InvokeAsync(remotingRequest, _consumer.Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_consumer.Stopped) return; if (pullRequest.IsDropped) return; if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result, pulledMessages => { var filterMessages = pulledMessages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags)); var consumingMessages = filterMessages.Select(x => new ConsumingMessage(x, pullRequest)).ToList(); if (_consumer.Setting.AutoPull) { pullRequest.ProcessQueue.AddMessages(consumingMessages); foreach (var consumingMessage in consumingMessages) { if (_consumer.Setting.MessageHandleMode == MessageHandleMode.Sequential) { _consumingMessageQueue.Add(consumingMessage); } else { Task.Factory.StartNew(HandleMessage, consumingMessage); } } } else { foreach (var consumingMessage in consumingMessages) { _pulledMessageQueue.Add(consumingMessage.Message); } } }); } catch (Exception ex) { if (_consumer.Stopped) return; if (pullRequest.IsDropped) return; if (remotingClient.IsConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_consumer.Stopped) return; if (pullRequest.IsDropped) return; if (remotingClient.IsConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
private void PersistOffset(PullRequest pullRequest) { try { var consumedMinQueueOffset = pullRequest.ProcessQueue.GetConsumedMinQueueOffset(); if (consumedMinQueueOffset >= 0) { if (!pullRequest.ProcessQueue.TryUpdatePreviousConsumedMinQueueOffset(consumedMinQueueOffset)) { return; } if (Setting.MessageModel == MessageModel.BroadCasting) { _localOffsetStore.PersistQueueOffset(GroupName, pullRequest.MessageQueue, consumedMinQueueOffset); } else if (Setting.MessageModel == MessageModel.Clustering) { var request = new UpdateQueueOffsetRequest(GroupName, pullRequest.MessageQueue, consumedMinQueueOffset); var remotingRequest = new RemotingRequest((int)RequestCode.UpdateQueueOffsetRequest, _binarySerializer.Serialize(request)); _remotingClient.InvokeOneway(remotingRequest, 10000); _logger.DebugFormat("Sent new queue offset to broker. group:{0}, consumerId:{1}, topic:{2}, queueId:{3}, offset:{4}", GroupName, Id, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, consumedMinQueueOffset); } } } catch (Exception ex) { _logger.Error(string.Format("PersistOffset has exception, consumerId:{0}, group:{1}, topic:{2}, queueId:{3}", Id, GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId), ex); } }
private void UpdatePullRequestDict(string topic, IList<MessageQueue> messageQueues) { // Check message queues to remove var toRemovePullRequestKeys = new List<string>(); foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic)) { var key = pullRequest.MessageQueue.ToString(); if (!messageQueues.Any(x => x.ToString() == key)) { toRemovePullRequestKeys.Add(key); } } foreach (var pullRequestKey in toRemovePullRequestKeys) { PullRequest pullRequest; if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest)) { pullRequest.Stop(); PersistRemovedMessageQueueOffset(pullRequest.MessageQueue); _logger.DebugFormat("[{0}]: removed pull request.[topic={1},queueId={2}]", Id, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId); } } // Check message queues to add. foreach (var messageQueue in messageQueues) { var key = messageQueue.ToString(); PullRequest pullRequest; if (!_pullRequestDict.TryGetValue(key, out pullRequest)) { var request = new PullRequest(Id, GroupName, messageQueue, _remotingClient, Setting.MessageHandleMode, _messageHandler, _offsetStore, Setting.PullRequestSetting); long nextOffset = ComputePullFromWhere(messageQueue); if (nextOffset >= 0) { request.NextOffset = nextOffset; if (_pullRequestDict.TryAdd(key, request)) { request.Start(); _logger.DebugFormat("[{0}]: added pull request.[topic={1},queueId={2}]", Id, request.MessageQueue.Topic, request.MessageQueue.QueueId); } } else { _logger.WarnFormat("[{0}]: the pull request {1} cannot be added as the nextOffset is < 0.", Id, request); } } } }
private void UpdatePullRequestDict(string topic, IList<MessageQueue> messageQueues) { // Check message queues to remove var toRemovePullRequestKeys = new List<string>(); foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic)) { var key = pullRequest.MessageQueue.ToString(); if (!messageQueues.Any(x => x.ToString() == key)) { toRemovePullRequestKeys.Add(key); } } foreach (var pullRequestKey in toRemovePullRequestKeys) { PullRequest pullRequest; if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest)) { pullRequest.Stop(); PersistOffset(pullRequest); _logger.DebugFormat("Removed pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId); } } // Check message queues to add. foreach (var messageQueue in messageQueues) { var key = messageQueue.ToString(); PullRequest pullRequest; if (!_pullRequestDict.TryGetValue(key, out pullRequest)) { var queueOffset = -1L; if (Setting.MessageModel == MessageModel.BroadCasting) { queueOffset = _localOffsetStore.GetQueueOffset(GroupName, messageQueue); } var request = new PullRequest(Id, GroupName, messageQueue, queueOffset, _remotingClient, Setting.MessageHandleMode, _messageHandler, Setting.PullRequestSetting); if (_pullRequestDict.TryAdd(key, request)) { request.Start(); _logger.DebugFormat("Added pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, request.MessageQueue.Topic, request.MessageQueue.QueueId); } } } }
private void PersistOffset(PullRequest pullRequest) { try { var consumedQueueOffset = pullRequest.ProcessQueue.GetConsumedQueueOffset(); if (consumedQueueOffset >= 0) { if (!pullRequest.ProcessQueue.TryUpdatePreviousConsumedQueueOffset(consumedQueueOffset)) { return; } var request = new UpdateQueueOffsetRequest(GroupName, pullRequest.MessageQueue, consumedQueueOffset); var remotingRequest = new RemotingRequest((int)RequestCode.UpdateQueueOffsetRequest, _binarySerializer.Serialize(request)); _adminRemotingClient.InvokeOneway(remotingRequest); if (_logger.IsDebugEnabled) { _logger.DebugFormat("Sent queue consume offset to broker. group: {0}, topic: {1}, queueId: {2}, offset: {3}", GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, consumedQueueOffset); } } } catch (Exception ex) { if (_remotingClient.IsConnected) { _logger.Error(string.Format("PersistOffset has exception, group: {0}, topic: {1}, queueId: {2}", GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId), ex); } } }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; var messageCount = pullRequest.ProcessQueue.GetMessageCount(); var flowControlThreshold = Setting.PullMessageFlowControlThreshold; if (flowControlThreshold > 0 && messageCount >= flowControlThreshold) { var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds( messageCount, flowControlThreshold, Setting.PullMessageFlowControlStepPercent, Setting.PullMessageFlowControlStepWaitMilliseconds); Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest)); return; } var request = new PullMessageRequest { ConsumerId = GetConsumerId(), ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, Tags = string.Join("|", pullRequest.Tags), QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_remotingClient.IsConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_remotingClient.IsConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
private void ProcessPullResponse(PullRequest pullRequest, RemotingResponse remotingResponse) { if (remotingResponse == null) { _logger.ErrorFormat("Pull message response is null, pullRequest:{0}", pullRequest); SchedulePullRequest(pullRequest); return; } if (remotingResponse.Code == -1) { _logger.ErrorFormat("Pull message failed, pullRequest:{0}, errorMsg:{1}", pullRequest, Encoding.UTF8.GetString(remotingResponse.Body)); SchedulePullRequest(pullRequest); return; } if (remotingResponse.Code == (int)PullStatus.Found) { var messages = _binarySerializer.Deserialize<IEnumerable<QueueMessage>>(remotingResponse.Body); if (messages.Count() > 0) { pullRequest.ProcessQueue.AddMessages(messages); foreach (var message in messages) { _consumingMessageQueue.Add(new ConsumingMessage(message, pullRequest.ProcessQueue)); } pullRequest.NextConsumeOffset = messages.Last().QueueOffset + 1; } } else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset) { var newOffset = BitConverter.ToInt64(remotingResponse.Body, 0); var oldOffset = pullRequest.NextConsumeOffset; pullRequest.NextConsumeOffset = newOffset; _logger.InfoFormat("Reset queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldOffset, newOffset); } else if (remotingResponse.Code == (int)PullStatus.NoNewMessage) { _logger.DebugFormat("No new message found, pullRequest:{0}", pullRequest); } else if (remotingResponse.Code == (int)PullStatus.Ignored) { _logger.InfoFormat("Pull request was ignored, pullRequest:{0}", pullRequest); return; } //Schedule the next pull request. SchedulePullRequest(pullRequest); }
private void UpdatePullRequestDict(KeyValuePair<string, HashSet<string>> subscriptionTopic, IList<MessageQueue> messageQueues) { // Check message queues to remove var toRemovePullRequestKeys = new List<string>(); foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == subscriptionTopic.Key)) { var key = pullRequest.MessageQueue.ToString(); if (!messageQueues.Any(x => x.ToString() == key)) { toRemovePullRequestKeys.Add(key); } } foreach (var pullRequestKey in toRemovePullRequestKeys) { PullRequest pullRequest; if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest)) { pullRequest.ProcessQueue.IsDropped = true; PersistOffset(pullRequest); _logger.InfoFormat("Dropped pull request, group: {0}, topic: {1}, queueId: {2}", GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId); } } // Check message queues to add. foreach (var messageQueue in messageQueues) { var key = messageQueue.ToString(); PullRequest pullRequest; if (!_pullRequestDict.TryGetValue(key, out pullRequest)) { var request = new PullRequest(GetConsumerId(), GroupName, messageQueue, -1, subscriptionTopic.Value); if (_pullRequestDict.TryAdd(key, request)) { SchedulePullRequest(request); _logger.InfoFormat("Added pull request, group: {0}, topic: {1}, queueId: {2}, tags: {3}", GroupName, request.MessageQueue.Topic, request.MessageQueue.QueueId, string.Join("|", request.Tags)); } } } }
public ConsumingMessage(QueueMessage message, PullRequest pullRequest) { Message = message; PullRequest = pullRequest; }