public static void WriteToStream(PullMessageRequest request, Stream stream)
{
using (var writer = new BinaryWriter(stream))
{
writer.Write(request.ConsumerGroup);
writer.Write(request.MessageQueue.Topic);
writer.Write(request.MessageQueue.QueueId);
writer.Write(request.QueueOffset);
writer.Write(request.PullMessageBatchSize);
writer.Write(request.SuspendPullRequestMilliseconds);
writer.Write((int)request.ConsumeFromWhere);
}
}
public static void WriteToStream(PullMessageRequest request, Stream stream)
{
using (var writer = new BinaryWriter(stream))
{
writer.Write(request.ConsumerGroup);
writer.Write(request.MessageQueue.Topic);
writer.Write(request.MessageQueue.QueueId);
writer.Write(request.QueueOffset);
writer.Write(request.PullMessageBatchSize);
writer.Write(request.SuspendPullRequestMilliseconds);
writer.Write((int)request.ConsumeFromWhere);
}
}
public static PullMessageRequest ReadFromStream(Stream stream)
{
using (var reader = new BinaryReader(stream))
{
var request = new PullMessageRequest();
request.ConsumerGroup = reader.ReadString();
request.MessageQueue = new MessageQueue(reader.ReadString(), reader.ReadInt32());
request.QueueOffset = reader.ReadInt64();
request.PullMessageBatchSize = reader.ReadInt32();
request.SuspendPullRequestMilliseconds = reader.ReadInt64();
request.ConsumeFromWhere = (ConsumeFromWhere)reader.ReadInt32();
return request;
}
}
public static PullMessageRequest ReadFromStream(Stream stream)
{
using (var reader = new BinaryReader(stream))
{
var request = new PullMessageRequest();
request.ConsumerGroup = reader.ReadString();
request.MessageQueue = new MessageQueue(reader.ReadString(), reader.ReadInt32());
request.QueueOffset = reader.ReadInt64();
request.PullMessageBatchSize = reader.ReadInt32();
request.SuspendPullRequestMilliseconds = reader.ReadInt64();
request.ConsumeFromWhere = (ConsumeFromWhere)reader.ReadInt32();
return(request);
}
}
public PullRequest(
long remotingRequestSequence,
PullMessageRequest pullMessageRequest,
IRequestHandlerContext requestHandlerContext,
DateTime suspendTime,
long suspendMilliseconds,
Action<PullRequest> newMessageArrivedAction,
Action<PullRequest> suspendTimeoutAction,
Action<PullRequest> replacedAction)
{
RemotingRequestSequence = remotingRequestSequence;
PullMessageRequest = pullMessageRequest;
RequestHandlerContext = requestHandlerContext;
SuspendTime = suspendTime;
SuspendMilliseconds = suspendMilliseconds;
NewMessageArrivedAction = newMessageArrivedAction;
SuspendTimeoutAction = suspendTimeoutAction;
ReplacedAction = replacedAction;
}
public PullRequest(
long remotingRequestSequence,
PullMessageRequest pullMessageRequest,
IRequestHandlerContext requestHandlerContext,
DateTime suspendStartTime,
long suspendMilliseconds,
Action<PullRequest> newMessageArrivedAction,
Action<PullRequest> timeoutAction,
Action<PullRequest> replacedAction)
{
Id = ObjectId.GenerateNewStringId();
RemotingRequestSequence = remotingRequestSequence;
PullMessageRequest = pullMessageRequest;
RequestHandlerContext = requestHandlerContext;
SuspendStartTime = suspendStartTime;
SuspendMilliseconds = suspendMilliseconds;
NewMessageArrivedAction = newMessageArrivedAction;
TimeoutAction = timeoutAction;
ReplacedAction = replacedAction;
}
public PullRequest(
RemotingRequest remotingRequest,
PullMessageRequest pullMessageRequest,
IRequestHandlerContext requestHandlerContext,
DateTime suspendStartTime,
long suspendMilliseconds,
Action<PullRequest> newMessageArrivedAction,
Action<PullRequest> timeoutAction,
Action<PullRequest> noNewMessageAction,
Action<PullRequest> replacedAction)
{
RemotingRequest = remotingRequest;
PullMessageRequest = pullMessageRequest;
RequestHandlerContext = requestHandlerContext;
SuspendStartTime = suspendStartTime;
SuspendMilliseconds = suspendMilliseconds;
NewMessageArrivedAction = newMessageArrivedAction;
TimeoutAction = timeoutAction;
NoNewMessageAction = noNewMessageAction;
ReplacedAction = replacedAction;
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
var flowControlThreshold = Setting.PullMessageFlowControlThreshold;
if (flowControlThreshold > 0 && messageCount >= flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
Setting.PullMessageFlowControlStepPercent,
Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = GetConsumerId(),
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private static byte[] SerializePullMessageRequest(PullMessageRequest request)
{
using (var stream = new MemoryStream())
{
PullMessageRequest.WriteToStream(request, stream);
return stream.ToArray();
}
}
private void PullMessage()
{
var messageCount = ProcessQueue.GetMessageCount();
if (messageCount >= _setting.PullThresholdForQueue)
{
Thread.Sleep(_setting.PullTimeDelayMillsWhenFlowControl);
if ((_flowControlTimes++ % 1000) == 0)
{
_logger.WarnFormat("Detect that the message local process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", this, messageCount, _flowControlTimes);
}
}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = MessageQueue,
QueueOffset = _queueOffset,
PullMessageBatchSize = _setting.PullMessageBatchSize
};
var data = _binarySerializer.Serialize(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
var remotingResponse = default(RemotingResponse);
try
{
remotingResponse = _remotingClient.InvokeSync(remotingRequest, _setting.PullRequestTimeoutMilliseconds);
}
catch (Exception ex)
{
if (!_stoped)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}, remotingRequest:{1}.", this, remotingRequest), ex);
}
return;
}
if (_stoped)
{
return;
}
var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body);
if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0)
{
_queueOffset += response.Messages.Count();
ProcessQueue.AddMessages(response.Messages);
response.Messages.ForEach(x => _messageQueue.Add(new WrappedMessage(MessageQueue, x, ProcessQueue)));
}
else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset && response.NextOffset != null)
{
_queueOffset = response.NextOffset.Value;
}
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
if (messageCount >= Setting.PullThresholdForQueue)
{
Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest));
if ((_flowControlTimes++ % 100) == 0)
{
_logger.WarnFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes);
}
return;
}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void PullMessage()
{
var messageCount = ProcessQueue.GetMessageCount();
//TODO, here the GetMessageSpan has bug when in parallel environment.
//var messageSpan = ProcessQueue.GetMessageSpan();
if (messageCount >= _setting.PullThresholdForQueue)
{
Thread.Sleep(_setting.PullTimeDelayMillsWhenFlowControl);
if ((_flowControlTimes1++ % 3000) == 0)
{
_logger.WarnFormat("[{0}]: the consumer message buffer is full, so do flow control, [messageCount={1},pullRequest={2},flowControlTimes={3}]", ConsumerId, messageCount, this, _flowControlTimes1);
}
}
//else if (messageSpan >= _setting.ConsumeMaxSpan)
//{
// Thread.Sleep(_setting.PullTimeDelayMillsWhenFlowControl);
// if ((flowControlTimes2++ % 3000) == 0)
// {
// _logger.WarnFormat("[{0}]: the consumer message span too long, so do flow control, [messageSpan={1},pullRequest={2},flowControlTimes={3}]", ConsumerId, messageSpan, this, flowControlTimes2);
// }
//}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = MessageQueue,
QueueOffset = NextOffset,
PullMessageBatchSize = _setting.PullMessageBatchSize
};
var data = _binarySerializer.Serialize(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
var remotingResponse = default(RemotingResponse);
try
{
remotingResponse = _remotingClient.InvokeSync(remotingRequest, _setting.PullRequestTimeoutMilliseconds);
}
catch (Exception ex)
{
if (!_stoped)
{
_logger.Error(string.Format("[{0}]: PullMessage has exception. RemotingRequest: {1}, PullRequest: {2}.", ConsumerId, remotingRequest, this), ex);
}
return;
}
if (_stoped)
{
return;
}
var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body);
if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0)
{
NextOffset += response.Messages.Count();
ProcessQueue.AddMessages(response.Messages);
response.Messages.ForEach(x => _messageQueue.Add(new WrappedMessage(MessageQueue, x, ProcessQueue)));
}
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
if (messageCount >= Setting.PullThresholdForQueue)
{
Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest));
if ((_flowControlTimes++ % 100) == 0)
{
_logger.DebugFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes);
}
return;
}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds
};
var data = _binarySerializer.Serialize(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
try
{
if (pullTask.Exception != null)
{
SchedulePullRequest(pullRequest);
return;
}
var remotingResponse = pullTask.Result;
var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body);
if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0)
{
pullRequest.ProcessQueue.AddMessages(response.Messages);
foreach (var message in response.Messages)
{
_consumingMessageQueue.Add(new ConsumingMessage(message, pullRequest.ProcessQueue));
}
pullRequest.NextConsumeOffset += response.Messages.Count();
}
else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset && response.NextOffset != null)
{
var oldConsumeOffset = pullRequest.NextConsumeOffset;
pullRequest.NextConsumeOffset = response.NextOffset.Value;
_logger.DebugFormat("Updated queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldConsumeOffset, pullRequest.NextConsumeOffset);
}
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (remotingResponse.Code == (int)PullStatus.Ignored) return;
SchedulePullRequest(pullRequest);
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped) return;
if (pullRequest.ProcessQueue.IsDropped) return;
if (_isBrokerServerConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
