public void CommitConsumeOffset(PullRequest pullRequest)
{
var consumedOffset = pullRequest.ProcessQueue.GetConsumedQueueOffset();
if (consumedOffset >= 0)
{
if (!pullRequest.ProcessQueue.TryUpdatePreviousConsumedQueueOffset(consumedOffset))
{
return;
}
CommitConsumeOffset(pullRequest.MessageQueue, consumedOffset);
}
}
private void PersistOffset(PullRequest pullRequest)
{
try
{
if (Setting.MessageModel == MessageModel.BroadCasting)
{
((ILocalOffsetStore)_offsetStore).PersistQueueOffset(GroupName, pullRequest.MessageQueue);
}
else if (Setting.MessageModel == MessageModel.Clustering)
{
var queueOffset = _offsetStore.GetQueueOffset(GroupName, pullRequest.MessageQueue);
var request = new UpdateQueueOffsetRequest(GroupName, pullRequest.MessageQueue, queueOffset);
var remotingRequest = new RemotingRequest((int)RequestCode.UpdateQueueOffsetRequest, _binarySerializer.Serialize(request));
_remotingClient.InvokeOneway(remotingRequest, 10000);
}
}
catch (Exception ex)
{
_logger.Error(string.Format("[{0}]: PersistOffset has exception.", Id), ex);
}
}
private void SchedulePullRequest(PullRequest pullRequest)
{
Task.Factory.StartNew(ExecutePullRequest, pullRequest);
}
private void ProcessPullResponse(PullRequest pullRequest, RemotingResponse remotingResponse)
{
if (remotingResponse == null)
{
_logger.ErrorFormat("Pull message response is null, pullRequest:{0}", pullRequest);
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == -1)
{
_logger.ErrorFormat("Pull message failed, pullRequest:{0}, errorMsg:{1}", pullRequest, Encoding.UTF8.GetString(remotingResponse.Body));
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == (short)PullStatus.Found)
{
var messages = DecodeMessages(pullRequest, remotingResponse.Body);
if (messages.Count() > 0)
{
var filterMessages = messages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags));
pullRequest.ProcessQueue.AddMessages(filterMessages);
foreach (var message in filterMessages)
{
var consumingMessage = new ConsumingMessage(message, pullRequest.ProcessQueue);
if (Setting.MessageHandleMode == MessageHandleMode.Sequential)
{
_consumingMessageQueue.Add(consumingMessage);
}
else
{
Task.Factory.StartNew(HandleMessage, consumingMessage);
}
}
pullRequest.NextConsumeOffset = messages.Last().QueueOffset + 1;
}
}
else if (remotingResponse.Code == (short)PullStatus.NextOffsetReset)
{
var newOffset = BitConverter.ToInt64(remotingResponse.Body, 0);
var oldOffset = pullRequest.NextConsumeOffset;
pullRequest.NextConsumeOffset = newOffset;
_logger.InfoFormat("Reset queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldOffset, newOffset);
}
else if (remotingResponse.Code == (short)PullStatus.NoNewMessage)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("No new message found, pullRequest:{0}", pullRequest);
}
}
else if (remotingResponse.Code == (short)PullStatus.Ignored)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Pull request was ignored, pullRequest:{0}", pullRequest);
}
return;
}
else if (remotingResponse.Code == (short)PullStatus.BrokerIsCleaning)
{
Thread.Sleep(5000);
}
//Schedule the next pull request.
SchedulePullRequest(pullRequest);
}
private IEnumerable<QueueMessage> DecodeMessages(PullRequest pullRequest, byte[] buffer)
{
var messages = new List<QueueMessage>();
if (buffer == null || buffer.Length <= 4)
{
return messages;
}
try
{
var nextOffset = 0;
var messageLength = MessageUtils.DecodeInt(buffer, nextOffset, out nextOffset);
while (messageLength > 0)
{
var message = new QueueMessage();
var messageBytes = new byte[messageLength];
Buffer.BlockCopy(buffer, nextOffset, messageBytes, 0, messageLength);
nextOffset += messageLength;
message.ReadFrom(messageBytes);
if (!message.IsValid())
{
_logger.ErrorFormat("Invalid message, pullRequest: {0}", pullRequest);
continue;
}
messages.Add(message);
if (nextOffset >= buffer.Length)
{
break;
}
messageLength = MessageUtils.DecodeInt(buffer, nextOffset, out nextOffset);
}
}
catch (Exception ex)
{
_logger.Error(string.Format("Decode pull return message has exception, pullRequest: {0}", pullRequest), ex);
}
return messages;
}
private void ResetNextConsumeOffset(PullRequest pullRequest, long newOffset)
{
var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName);
if (brokerConnection == null)
{
_logger.ErrorFormat("Reset nextConsumeOffset failed as broker is unavailable, pullRequest: {0}, newOffset: {1}", pullRequest, newOffset);
return;
}
var remotingClient = brokerConnection.AdminRemotingClient;
try
{
var oldOffset = pullRequest.NextConsumeOffset;
pullRequest.NextConsumeOffset = newOffset;
pullRequest.ProcessQueue.MarkAllConsumingMessageIgnored();
pullRequest.ProcessQueue.Reset();
var request = new UpdateQueueOffsetRequest(_consumer.GroupName, pullRequest.MessageQueue, newOffset - 1);
var remotingRequest = new RemotingRequest((int)BrokerRequestCode.UpdateQueueConsumeOffsetRequest, _binarySerializer.Serialize(request));
remotingClient.InvokeOneway(remotingRequest);
_logger.InfoFormat("Resetted nextConsumeOffset, [pullRequest:{0}, oldOffset:{1}, newOffset:{2}]", pullRequest, oldOffset, newOffset);
}
catch (Exception ex)
{
if (remotingClient.IsConnected)
{
_logger.Error(string.Format("Reset nextConsumeOffset failed, pullRequest: {0}, newOffset: {1}", pullRequest, newOffset), ex);
}
}
}
private void ProcessPullResponse(PullRequest pullRequest, RemotingResponse remotingResponse, Action<IEnumerable<QueueMessage>> handlePulledMessageAction)
{
if (remotingResponse == null)
{
_logger.ErrorFormat("Pull message response is null, pullRequest:{0}", pullRequest);
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == -1)
{
_logger.ErrorFormat("Pull message failed, pullRequest:{0}, errorMsg:{1}", pullRequest, Encoding.UTF8.GetString(remotingResponse.Body));
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == (short)PullStatus.Found)
{
var messages = DecodeMessages(pullRequest, remotingResponse.Body);
if (messages.Count() > 0)
{
handlePulledMessageAction(messages);
pullRequest.NextConsumeOffset = messages.Last().QueueOffset + 1;
}
}
else if (remotingResponse.Code == (short)PullStatus.NextOffsetReset)
{
var newOffset = BitConverter.ToInt64(remotingResponse.Body, 0);
ResetNextConsumeOffset(pullRequest, newOffset);
}
else if (remotingResponse.Code == (short)PullStatus.NoNewMessage)
{
//No new message to consume.
}
else if (remotingResponse.Code == (short)PullStatus.Ignored)
{
_logger.InfoFormat("Pull request was ignored, pullRequest:{0}", pullRequest);
return;
}
else if (remotingResponse.Code == (short)PullStatus.BrokerIsCleaning)
{
Thread.Sleep(5000);
}
//Schedule the next pull request.
SchedulePullRequest(pullRequest);
}
private void UpdatePullRequestDict(string topic, IList<MessageQueue> messageQueues)
{
// Check message queues to remove
var toRemovePullRequestKeys = new List<string>();
foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic))
{
var key = pullRequest.MessageQueue.ToString();
if (!messageQueues.Any(x => x.ToString() == key))
{
toRemovePullRequestKeys.Add(key);
}
}
foreach (var pullRequestKey in toRemovePullRequestKeys)
{
PullRequest pullRequest;
if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest))
{
pullRequest.ProcessQueue.IsDropped = true;
PersistOffset(pullRequest);
_logger.InfoFormat("Dropped pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId);
}
}
// Check message queues to add.
foreach (var messageQueue in messageQueues)
{
var key = messageQueue.ToString();
PullRequest pullRequest;
if (!_pullRequestDict.TryGetValue(key, out pullRequest))
{
var request = new PullRequest(Id, GroupName, messageQueue, -1);
if (_pullRequestDict.TryAdd(key, request))
{
SchedulePullRequest(request);
_logger.InfoFormat("Added pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, request.MessageQueue.Topic, request.MessageQueue.QueueId);
}
}
}
}
private void PullMessage(PullRequest pullRequest)
{
var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName);
if (brokerConnection == null)
{
Task.Factory.StartDelayedTask(5 * 1000, () => SchedulePullRequest(pullRequest));
_logger.ErrorFormat("Pull message failed as the target broker connection not found, pullRequest:{0}", pullRequest);
return;
}
var remotingClient = brokerConnection.RemotingClient;
try
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
var messageCount = 0;
var flowControlThreshold = 0;
if (_consumer.Setting.AutoPull)
{
messageCount = pullRequest.ProcessQueue.GetMessageCount();
flowControlThreshold = _consumer.Setting.PullMessageFlowControlThreshold;
}
else
{
messageCount = _pulledMessageQueue.Count;
flowControlThreshold = _consumer.Setting.ManualPullLocalMessageQueueMaxSize;
}
if (messageCount > flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
_consumer.Setting.PullMessageFlowControlStepPercent,
_consumer.Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = _clientId,
ConsumerGroup = _consumer.GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = _consumer.Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = _consumer.Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = _consumer.Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)BrokerRequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
remotingClient.InvokeAsync(remotingRequest, _consumer.Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result, pulledMessages =>
{
var filterMessages = pulledMessages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags));
var consumingMessages = filterMessages.Select(x => new ConsumingMessage(x, pullRequest)).ToList();
if (_consumer.Setting.AutoPull)
{
pullRequest.ProcessQueue.AddMessages(consumingMessages);
foreach (var consumingMessage in consumingMessages)
{
if (_consumer.Setting.MessageHandleMode == MessageHandleMode.Sequential)
{
_consumingMessageQueue.Add(consumingMessage);
}
else
{
Task.Factory.StartNew(HandleMessage, consumingMessage);
}
}
}
else
{
foreach (var consumingMessage in consumingMessages)
{
_pulledMessageQueue.Add(consumingMessage.Message);
}
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
if (remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
if (remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void SchedulePullRequest(PullRequest pullRequest)
{
Task.Factory.StartNew(ExecutePullRequest, pullRequest);
}
public ConsumingMessage(QueueMessage message, PullRequest pullRequest)
{
Message = message;
PullRequest = pullRequest;
Message.BrokerName = pullRequest.MessageQueue.BrokerName;
}
private void ProcessPullResponse(PullRequest pullRequest, RemotingResponse remotingResponse)
{
if (remotingResponse == null)
{
_logger.ErrorFormat("Pull message response is null, pullRequest:{0}", pullRequest);
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == -1)
{
_logger.ErrorFormat("Pull message failed, pullRequest:{0}, errorMsg:{1}", pullRequest, Encoding.UTF8.GetString(remotingResponse.Body));
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == (short)PullStatus.Found)
{
var messages = DecodeMessages(pullRequest, remotingResponse.Body);
if (messages.Count() > 0)
{
var filterMessages = messages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags));
pullRequest.ProcessQueue.AddMessages(filterMessages);
foreach (var message in filterMessages)
{
var consumingMessage = new ConsumingMessage(message, pullRequest.ProcessQueue);
if (Setting.MessageHandleMode == MessageHandleMode.Sequential)
{
_consumingMessageQueue.Add(consumingMessage);
}
else
{
Task.Factory.StartNew(HandleMessage, consumingMessage);
}
}
pullRequest.NextConsumeOffset = messages.Last().QueueOffset + 1;
}
}
else if (remotingResponse.Code == (short)PullStatus.NextOffsetReset)
{
var newOffset = BitConverter.ToInt64(remotingResponse.Body, 0);
var oldOffset = pullRequest.NextConsumeOffset;
pullRequest.NextConsumeOffset = newOffset;
_logger.InfoFormat("Reset queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldOffset, newOffset);
}
else if (remotingResponse.Code == (short)PullStatus.NoNewMessage)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("No new message found, pullRequest:{0}", pullRequest);
}
}
else if (remotingResponse.Code == (short)PullStatus.Ignored)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Pull request was ignored, pullRequest:{0}", pullRequest);
}
return;
}
else if (remotingResponse.Code == (short)PullStatus.BrokerIsCleaning)
{
Thread.Sleep(5000);
}
//Schedule the next pull request.
SchedulePullRequest(pullRequest);
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
var flowControlThreshold = Setting.PullMessageFlowControlThreshold;
if (flowControlThreshold > 0 && messageCount >= flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
Setting.PullMessageFlowControlStepPercent,
Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = GetConsumerId(),
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
if (messageCount >= Setting.PullThresholdForQueue)
{
Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest));
if ((_flowControlTimes++ % 100) == 0)
{
_logger.DebugFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes);
}
return;
}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds
};
var data = _binarySerializer.Serialize(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
try
{
if (pullTask.Exception != null)
{
SchedulePullRequest(pullRequest);
return;
}
var remotingResponse = pullTask.Result;
var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body);
if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0)
{
pullRequest.ProcessQueue.AddMessages(response.Messages);
foreach (var message in response.Messages)
{
_consumingMessageQueue.Add(new ConsumingMessage(message, pullRequest.ProcessQueue));
}
pullRequest.NextConsumeOffset += response.Messages.Count();
}
else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset && response.NextOffset != null)
{
var oldConsumeOffset = pullRequest.NextConsumeOffset;
pullRequest.NextConsumeOffset = response.NextOffset.Value;
_logger.DebugFormat("Updated queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldConsumeOffset, pullRequest.NextConsumeOffset);
}
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (remotingResponse.Code == (int)PullStatus.Ignored) return;
SchedulePullRequest(pullRequest);
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void UpdatePullRequestDict(KeyValuePair<string, HashSet<string>> pair, IList<MessageQueue> messageQueues)
{
var topic = pair.Key;
var toRemovePullRequestKeys = new List<string>();
foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic))
{
var key = pullRequest.MessageQueue.ToString();
if (!messageQueues.Any(x => x.ToString() == key))
{
toRemovePullRequestKeys.Add(key);
}
}
foreach (var pullRequestKey in toRemovePullRequestKeys)
{
PullRequest pullRequest;
if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest))
{
pullRequest.IsDropped = true;
_commitConsumeOffsetService.CommitConsumeOffset(pullRequest);
_logger.InfoFormat("Dropped pull request, consumerGroup: {0}, consumerId: {1}, queue: {2}, tags: {3}",
_consumer.GroupName,
_clientId,
pullRequest.MessageQueue,
string.Join("|", pullRequest.Tags));
}
}
foreach (var messageQueue in messageQueues)
{
var key = messageQueue.ToString();
PullRequest exist;
if (!_pullRequestDict.TryGetValue(key, out exist))
{
var pullRequest = new PullRequest(_clientId, _consumer.GroupName, messageQueue, -1, pair.Value);
if (_pullRequestDict.TryAdd(key, pullRequest))
{
_pullMessageService.SchedulePullRequest(pullRequest);
_logger.InfoFormat("Added pull request, consumerGroup: {0}, consumerId: {1}, queue: {2}, tags: {3}",
_consumer.GroupName,
_clientId,
pullRequest.MessageQueue,
string.Join("|", pullRequest.Tags));
}
}
}
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
if (messageCount >= Setting.PullThresholdForQueue)
{
Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest));
if ((_flowControlTimes++ % 100) == 0)
{
_logger.WarnFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes);
}
return;
}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void SchedulePullRequest(PullRequest pullRequest)
{
_pullRequestQueue.Add(pullRequest);
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
if (messageCount >= Setting.PullThresholdForQueue)
{
Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest));
if ((_flowControlTimes++ % 100) == 0)
{
_logger.WarnFormat("Pull message flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes);
}
return;
}
var request = new PullMessageRequest
{
ConsumerId = GetConsumerId(),
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
public void SchedulePullRequest(PullRequest pullRequest)
{
if (_consumer.Setting.AutoPull && _messageHandler == null)
{
_logger.Error("Schedule pullRequest is cancelled as the messageHandler is null.");
return;
}
Task.Factory.StartNew(ExecutePullRequest, pullRequest);
}
private void SchedulePullRequest(PullRequest pullRequest)
{
_pullRequestQueue.Add(pullRequest);
}
private void PullMessage(PullRequest pullRequest)
{
var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName);
if (brokerConnection == null)
{
Task.Factory.StartDelayedTask(5 * 1000, () => SchedulePullRequest(pullRequest));
_logger.ErrorFormat("Pull message failed as the target broker connection not found, pullRequest:{0}", pullRequest);
return;
}
var remotingClient = brokerConnection.RemotingClient;
try
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
var messageCount = 0;
var flowControlThreshold = 0;
if (_consumer.Setting.AutoPull)
{
messageCount = pullRequest.ProcessQueue.GetMessageCount();
flowControlThreshold = _consumer.Setting.PullMessageFlowControlThreshold;
}
else
{
messageCount = _pulledMessageQueue.Count;
flowControlThreshold = _consumer.Setting.ManualPullLocalMessageQueueMaxSize;
}
if (messageCount > flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
_consumer.Setting.PullMessageFlowControlStepPercent,
_consumer.Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = _clientId,
ConsumerGroup = _consumer.GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = _consumer.Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = _consumer.Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = _consumer.Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)BrokerRequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
remotingClient.InvokeAsync(remotingRequest, _consumer.Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result, pulledMessages =>
{
var filterMessages = pulledMessages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags));
var consumingMessages = filterMessages.Select(x => new ConsumingMessage(x, pullRequest)).ToList();
if (_consumer.Setting.AutoPull)
{
pullRequest.ProcessQueue.AddMessages(consumingMessages);
foreach (var consumingMessage in consumingMessages)
{
if (_consumer.Setting.MessageHandleMode == MessageHandleMode.Sequential)
{
_consumingMessageQueue.Add(consumingMessage);
}
else
{
Task.Factory.StartNew(HandleMessage, consumingMessage);
}
}
}
else
{
foreach (var consumingMessage in consumingMessages)
{
_pulledMessageQueue.Add(consumingMessage.Message);
}
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
if (remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
if (remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void PersistOffset(PullRequest pullRequest)
{
try
{
var consumedMinQueueOffset = pullRequest.ProcessQueue.GetConsumedMinQueueOffset();
if (consumedMinQueueOffset >= 0)
{
if (!pullRequest.ProcessQueue.TryUpdatePreviousConsumedMinQueueOffset(consumedMinQueueOffset))
{
return;
}
if (Setting.MessageModel == MessageModel.BroadCasting)
{
_localOffsetStore.PersistQueueOffset(GroupName, pullRequest.MessageQueue, consumedMinQueueOffset);
}
else if (Setting.MessageModel == MessageModel.Clustering)
{
var request = new UpdateQueueOffsetRequest(GroupName, pullRequest.MessageQueue, consumedMinQueueOffset);
var remotingRequest = new RemotingRequest((int)RequestCode.UpdateQueueOffsetRequest, _binarySerializer.Serialize(request));
_remotingClient.InvokeOneway(remotingRequest, 10000);
_logger.DebugFormat("Sent new queue offset to broker. group:{0}, consumerId:{1}, topic:{2}, queueId:{3}, offset:{4}",
GroupName,
Id,
pullRequest.MessageQueue.Topic,
pullRequest.MessageQueue.QueueId,
consumedMinQueueOffset);
}
}
}
catch (Exception ex)
{
_logger.Error(string.Format("PersistOffset has exception, consumerId:{0}, group:{1}, topic:{2}, queueId:{3}", Id, GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId), ex);
}
}
private void UpdatePullRequestDict(string topic, IList<MessageQueue> messageQueues)
{
// Check message queues to remove
var toRemovePullRequestKeys = new List<string>();
foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic))
{
var key = pullRequest.MessageQueue.ToString();
if (!messageQueues.Any(x => x.ToString() == key))
{
toRemovePullRequestKeys.Add(key);
}
}
foreach (var pullRequestKey in toRemovePullRequestKeys)
{
PullRequest pullRequest;
if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest))
{
pullRequest.Stop();
PersistRemovedMessageQueueOffset(pullRequest.MessageQueue);
_logger.DebugFormat("[{0}]: removed pull request.[topic={1},queueId={2}]", Id, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId);
}
}
// Check message queues to add.
foreach (var messageQueue in messageQueues)
{
var key = messageQueue.ToString();
PullRequest pullRequest;
if (!_pullRequestDict.TryGetValue(key, out pullRequest))
{
var request = new PullRequest(Id, GroupName, messageQueue, _remotingClient, Setting.MessageHandleMode, _messageHandler, _offsetStore, Setting.PullRequestSetting);
long nextOffset = ComputePullFromWhere(messageQueue);
if (nextOffset >= 0)
{
request.NextOffset = nextOffset;
if (_pullRequestDict.TryAdd(key, request))
{
request.Start();
_logger.DebugFormat("[{0}]: added pull request.[topic={1},queueId={2}]", Id, request.MessageQueue.Topic, request.MessageQueue.QueueId);
}
}
else
{
_logger.WarnFormat("[{0}]: the pull request {1} cannot be added as the nextOffset is < 0.", Id, request);
}
}
}
}
private void UpdatePullRequestDict(string topic, IList<MessageQueue> messageQueues)
{
// Check message queues to remove
var toRemovePullRequestKeys = new List<string>();
foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == topic))
{
var key = pullRequest.MessageQueue.ToString();
if (!messageQueues.Any(x => x.ToString() == key))
{
toRemovePullRequestKeys.Add(key);
}
}
foreach (var pullRequestKey in toRemovePullRequestKeys)
{
PullRequest pullRequest;
if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest))
{
pullRequest.Stop();
PersistOffset(pullRequest);
_logger.DebugFormat("Removed pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId);
}
}
// Check message queues to add.
foreach (var messageQueue in messageQueues)
{
var key = messageQueue.ToString();
PullRequest pullRequest;
if (!_pullRequestDict.TryGetValue(key, out pullRequest))
{
var queueOffset = -1L;
if (Setting.MessageModel == MessageModel.BroadCasting)
{
queueOffset = _localOffsetStore.GetQueueOffset(GroupName, messageQueue);
}
var request = new PullRequest(Id, GroupName, messageQueue, queueOffset, _remotingClient, Setting.MessageHandleMode, _messageHandler, Setting.PullRequestSetting);
if (_pullRequestDict.TryAdd(key, request))
{
request.Start();
_logger.DebugFormat("Added pull request, consumerId:{0}, group:{1}, topic={2}, queueId={3}", Id, GroupName, request.MessageQueue.Topic, request.MessageQueue.QueueId);
}
}
}
}
private void PersistOffset(PullRequest pullRequest)
{
try
{
var consumedQueueOffset = pullRequest.ProcessQueue.GetConsumedQueueOffset();
if (consumedQueueOffset >= 0)
{
if (!pullRequest.ProcessQueue.TryUpdatePreviousConsumedQueueOffset(consumedQueueOffset))
{
return;
}
var request = new UpdateQueueOffsetRequest(GroupName, pullRequest.MessageQueue, consumedQueueOffset);
var remotingRequest = new RemotingRequest((int)RequestCode.UpdateQueueOffsetRequest, _binarySerializer.Serialize(request));
_adminRemotingClient.InvokeOneway(remotingRequest);
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Sent queue consume offset to broker. group: {0}, topic: {1}, queueId: {2}, offset: {3}",
GroupName,
pullRequest.MessageQueue.Topic,
pullRequest.MessageQueue.QueueId,
consumedQueueOffset);
}
}
}
catch (Exception ex)
{
if (_remotingClient.IsConnected)
{
_logger.Error(string.Format("PersistOffset has exception, group: {0}, topic: {1}, queueId: {2}", GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId), ex);
}
}
}
public ConsumingMessage(QueueMessage message, PullRequest pullRequest)
{
Message = message;
PullRequest = pullRequest;
Message.BrokerName = pullRequest.MessageQueue.BrokerName;
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
var flowControlThreshold = Setting.PullMessageFlowControlThreshold;
if (flowControlThreshold > 0 && messageCount >= flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
Setting.PullMessageFlowControlStepPercent,
Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = GetConsumerId(),
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void ProcessPullResponse(PullRequest pullRequest, RemotingResponse remotingResponse)
{
if (remotingResponse == null)
{
_logger.ErrorFormat("Pull message response is null, pullRequest:{0}", pullRequest);
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == -1)
{
_logger.ErrorFormat("Pull message failed, pullRequest:{0}, errorMsg:{1}", pullRequest, Encoding.UTF8.GetString(remotingResponse.Body));
SchedulePullRequest(pullRequest);
return;
}
if (remotingResponse.Code == (int)PullStatus.Found)
{
var messages = _binarySerializer.Deserialize<IEnumerable<QueueMessage>>(remotingResponse.Body);
if (messages.Count() > 0)
{
pullRequest.ProcessQueue.AddMessages(messages);
foreach (var message in messages)
{
_consumingMessageQueue.Add(new ConsumingMessage(message, pullRequest.ProcessQueue));
}
pullRequest.NextConsumeOffset = messages.Last().QueueOffset + 1;
}
}
else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset)
{
var newOffset = BitConverter.ToInt64(remotingResponse.Body, 0);
var oldOffset = pullRequest.NextConsumeOffset;
pullRequest.NextConsumeOffset = newOffset;
_logger.InfoFormat("Reset queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldOffset, newOffset);
}
else if (remotingResponse.Code == (int)PullStatus.NoNewMessage)
{
_logger.DebugFormat("No new message found, pullRequest:{0}", pullRequest);
}
else if (remotingResponse.Code == (int)PullStatus.Ignored)
{
_logger.InfoFormat("Pull request was ignored, pullRequest:{0}", pullRequest);
return;
}
//Schedule the next pull request.
SchedulePullRequest(pullRequest);
}
private void UpdatePullRequestDict(KeyValuePair<string, HashSet<string>> subscriptionTopic, IList<MessageQueue> messageQueues)
{
// Check message queues to remove
var toRemovePullRequestKeys = new List<string>();
foreach (var pullRequest in _pullRequestDict.Values.Where(x => x.MessageQueue.Topic == subscriptionTopic.Key))
{
var key = pullRequest.MessageQueue.ToString();
if (!messageQueues.Any(x => x.ToString() == key))
{
toRemovePullRequestKeys.Add(key);
}
}
foreach (var pullRequestKey in toRemovePullRequestKeys)
{
PullRequest pullRequest;
if (_pullRequestDict.TryRemove(pullRequestKey, out pullRequest))
{
pullRequest.ProcessQueue.IsDropped = true;
PersistOffset(pullRequest);
_logger.InfoFormat("Dropped pull request, group: {0}, topic: {1}, queueId: {2}", GroupName, pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId);
}
}
// Check message queues to add.
foreach (var messageQueue in messageQueues)
{
var key = messageQueue.ToString();
PullRequest pullRequest;
if (!_pullRequestDict.TryGetValue(key, out pullRequest))
{
var request = new PullRequest(GetConsumerId(), GroupName, messageQueue, -1, subscriptionTopic.Value);
if (_pullRequestDict.TryAdd(key, request))
{
SchedulePullRequest(request);
_logger.InfoFormat("Added pull request, group: {0}, topic: {1}, queueId: {2}, tags: {3}", GroupName, request.MessageQueue.Topic, request.MessageQueue.QueueId, string.Join("|", request.Tags));
}
}
}
}
public ConsumingMessage(QueueMessage message, PullRequest pullRequest)
{
Message = message;
PullRequest = pullRequest;
}