private static PullMessageRequest DeserializePullMessageRequest(byte[] data)
{
using (var stream = new MemoryStream(data))
{
return(PullMessageRequest.ReadFromStream(stream));
}
}
private static byte[] SerializePullMessageRequest(PullMessageRequest request)
{
using (var stream = new MemoryStream())
{
PullMessageRequest.WriteToStream(request, stream);
return(stream.ToArray());
}
}
private void PullMessage()
{
var messageCount = ProcessQueue.GetMessageCount();
if (messageCount >= _setting.PullThresholdForQueue)
{
Thread.Sleep(_setting.PullTimeDelayMillsWhenFlowControl);
if ((_flowControlTimes++ % 1000) == 0)
{
_logger.WarnFormat("Detect that the message local process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", this, messageCount, _flowControlTimes);
}
}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = MessageQueue,
QueueOffset = _queueOffset,
PullMessageBatchSize = _setting.PullMessageBatchSize
};
var data = _binarySerializer.Serialize(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
var remotingResponse = default(RemotingResponse);
try
{
remotingResponse = _remotingClient.InvokeSync(remotingRequest, _setting.PullRequestTimeoutMilliseconds);
}
catch (Exception ex)
{
if (!_stoped)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}, remotingRequest:{1}.", this, remotingRequest), ex);
}
return;
}
if (_stoped)
{
return;
}
var response = _binarySerializer.Deserialize <PullMessageResponse>(remotingResponse.Body);
if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0)
{
_queueOffset += response.Messages.Count();
ProcessQueue.AddMessages(response.Messages);
response.Messages.ForEach(x => _messageQueue.Add(new WrappedMessage(MessageQueue, x, ProcessQueue)));
}
else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset && response.NextOffset != null)
{
_queueOffset = response.NextOffset.Value;
}
}
public void PullMessageRequest_TestData_returnNotNull()
{
var generator = new UsernameTokenGenerator("admin", "Dwapiatka25");
var address = "http://192.168.2.124/onvif/Events/Subscription?index=0";
var request = new PullMessageRequest(generator, address);
var xml = request.ToXML();
Debug.WriteLine(xml);
Assert.IsNotNull(xml);
}
public Task <Response> Pull(BrokerGroup group, PullMessageRequest request)
{
var client = CreateOrGetClient(group);
if (client == null)
{
return(ErrorTask(new NoWritableBrokerException($"Cannot find pullable broker for {request.Subject}/{request.Group}")));
}
var ackHandlerQueue = _ackHandlerQueueManager.GetOrCreate(request.Subject, request.Group, client.BrokerGroupName, request.IsBroadcast);
request.PullOffsetBegin = ackHandlerQueue.MinPullOffset;
request.PullOffsetEnd = ackHandlerQueue.MaxPullOffset;
var timeout = request.TimeoutMillis < 0 ? MinRoundtripMs : (request.TimeoutMillis + MinRoundtripMs);
return(client.Send(new Request(request), timeout));
}
private void WriteTags(PullMessageRequest request, byte[] buffer, Stream output)
{
ByteBufHelper.WriteInt16(_request.TagType, buffer, output);
var tags = request.Tags;
if (tags == null || tags.Length == 0)
{
ByteBufHelper.WriteByte((byte)0, output);
return;
}
ByteBufHelper.WriteByte((byte)tags.Length, output);
foreach (var tag in _request.Tags)
{
ByteBufHelper.WriteString(tag, buffer, output);
}
}
public PullRequest(
long remotingRequestSequence,
PullMessageRequest pullMessageRequest,
IRequestHandlerContext requestHandlerContext,
DateTime suspendStartTime,
long suspendMilliseconds,
Action <PullRequest> newMessageArrivedAction,
Action <PullRequest> timeoutAction,
Action <PullRequest> replacedAction)
{
RemotingRequestSequence = remotingRequestSequence;
PullMessageRequest = pullMessageRequest;
RequestHandlerContext = requestHandlerContext;
SuspendStartTime = suspendStartTime;
SuspendMilliseconds = suspendMilliseconds;
NewMessageArrivedAction = newMessageArrivedAction;
TimeoutAction = timeoutAction;
ReplacedAction = replacedAction;
}
public PullRequest(
RemotingRequest remotingRequest,
PullMessageRequest pullMessageRequest,
IRequestHandlerContext requestHandlerContext,
DateTime suspendStartTime,
long suspendMilliseconds,
Action <PullRequest> newMessageArrivedAction,
Action <PullRequest> timeoutAction,
Action <PullRequest> replacedAction)
{
Id = ObjectId.GenerateNewStringId();
RemotingRequest = remotingRequest;
PullMessageRequest = pullMessageRequest;
RequestHandlerContext = requestHandlerContext;
SuspendStartTime = suspendStartTime;
SuspendMilliseconds = suspendMilliseconds;
NewMessageArrivedAction = newMessageArrivedAction;
TimeoutAction = timeoutAction;
ReplacedAction = replacedAction;
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
if (messageCount >= Setting.PullThresholdForQueue)
{
Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest));
if ((_flowControlTimes++ % 100) == 0)
{
_logger.WarnFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes);
}
return;
}
var request = new PullMessageRequest
{
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void PullMessage(PullRequest pullRequest)
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
var messageCount = pullRequest.ProcessQueue.GetMessageCount();
var flowControlThreshold = Setting.PullMessageFlowControlThreshold;
if (flowControlThreshold > 0 && messageCount >= flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
Setting.PullMessageFlowControlStepPercent,
Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = GetConsumerId(),
ConsumerGroup = GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
_remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result);
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_stoped)
{
return;
}
if (pullRequest.ProcessQueue.IsDropped)
{
return;
}
if (_remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private void PullMessage(PullRequest pullRequest)
{
var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName);
if (brokerConnection == null)
{
Task.Factory.StartDelayedTask(5 * 1000, () => SchedulePullRequest(pullRequest));
_logger.ErrorFormat("Pull message failed as the target broker connection not found, pullRequest:{0}", pullRequest);
return;
}
var remotingClient = brokerConnection.RemotingClient;
try
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
var messageCount = 0;
var flowControlThreshold = 0;
if (_consumer.Setting.AutoPull)
{
messageCount = pullRequest.ProcessQueue.GetMessageCount();
flowControlThreshold = _consumer.Setting.PullMessageFlowControlThreshold;
}
else
{
messageCount = _pulledMessageQueue.Count;
flowControlThreshold = _consumer.Setting.ManualPullLocalMessageQueueMaxSize;
}
if (messageCount > flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
_consumer.Setting.PullMessageFlowControlStepPercent,
_consumer.Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = _clientId,
ConsumerGroup = _consumer.GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = _consumer.Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = _consumer.Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = _consumer.Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)BrokerRequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
remotingClient.InvokeAsync(remotingRequest, _consumer.Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result, pulledMessages =>
{
var filterMessages = pulledMessages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags));
var consumingMessages = filterMessages.Select(x => new ConsumingMessage(x, pullRequest)).ToList();
if (_consumer.Setting.AutoPull)
{
pullRequest.ProcessQueue.AddMessages(consumingMessages);
foreach (var consumingMessage in consumingMessages)
{
if (_consumer.Setting.MessageHandleMode == MessageHandleMode.Sequential)
{
_consumingMessageQueue.Add(consumingMessage);
}
else
{
Task.Factory.StartNew(HandleMessage, consumingMessage);
}
}
}
else
{
foreach (var consumingMessage in consumingMessages)
{
_pulledMessageQueue.Add(consumingMessage.Message);
}
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
if (remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped) return;
if (pullRequest.IsDropped) return;
if (remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
private static byte[] SerializePullMessageRequest(PullMessageRequest request)
{
using (var stream = new MemoryStream())
{
PullMessageRequest.WriteToStream(request, stream);
return stream.ToArray();
}
}
private void PullMessage(PullRequest pullRequest)
{
var brokerConnection = _clientService.GetBrokerConnection(pullRequest.MessageQueue.BrokerName);
if (brokerConnection == null)
{
Task.Factory.StartDelayedTask(5 * 1000, () => SchedulePullRequest(pullRequest));
_logger.ErrorFormat("Pull message failed as the target broker connection not found, pullRequest:{0}", pullRequest);
return;
}
var remotingClient = brokerConnection.RemotingClient;
try
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
var messageCount = 0;
var flowControlThreshold = 0;
if (_consumer.Setting.AutoPull)
{
messageCount = pullRequest.ProcessQueue.GetMessageCount();
flowControlThreshold = _consumer.Setting.PullMessageFlowControlThreshold;
}
else
{
messageCount = _pulledMessageQueue.Count;
flowControlThreshold = _consumer.Setting.ManualPullLocalMessageQueueMaxSize;
}
if (messageCount > flowControlThreshold)
{
var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds(
messageCount,
flowControlThreshold,
_consumer.Setting.PullMessageFlowControlStepPercent,
_consumer.Setting.PullMessageFlowControlStepWaitMilliseconds);
Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest));
return;
}
var request = new PullMessageRequest
{
ConsumerId = _clientId,
ConsumerGroup = _consumer.GroupName,
MessageQueue = pullRequest.MessageQueue,
Tags = string.Join("|", pullRequest.Tags),
QueueOffset = pullRequest.NextConsumeOffset,
PullMessageBatchSize = _consumer.Setting.PullMessageBatchSize,
SuspendPullRequestMilliseconds = _consumer.Setting.SuspendPullRequestMilliseconds,
ConsumeFromWhere = _consumer.Setting.ConsumeFromWhere
};
var data = SerializePullMessageRequest(request);
var remotingRequest = new RemotingRequest((int)BrokerRequestCode.PullMessage, data);
pullRequest.PullStartTime = DateTime.Now;
remotingClient.InvokeAsync(remotingRequest, _consumer.Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask =>
{
try
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
if (pullTask.Exception != null)
{
_logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception);
SchedulePullRequest(pullRequest);
return;
}
ProcessPullResponse(pullRequest, pullTask.Result, pulledMessages =>
{
var filterMessages = pulledMessages.Where(x => IsQueueMessageMatchTag(x, pullRequest.Tags));
var consumingMessages = filterMessages.Select(x => new ConsumingMessage(x, pullRequest)).ToList();
if (_consumer.Setting.AutoPull)
{
pullRequest.ProcessQueue.AddMessages(consumingMessages);
foreach (var consumingMessage in consumingMessages)
{
if (_consumer.Setting.MessageHandleMode == MessageHandleMode.Sequential)
{
_consumingMessageQueue.Add(consumingMessage);
}
else
{
Task.Factory.StartNew(HandleMessage, consumingMessage);
}
}
}
else
{
foreach (var consumingMessage in consumingMessages)
{
_pulledMessageQueue.Add(consumingMessage.Message);
}
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
if (remotingClient.IsConnected)
{
string remotingResponseBodyLength;
if (pullTask.Result != null)
{
remotingResponseBodyLength = pullTask.Result.Body.Length.ToString();
}
else
{
remotingResponseBodyLength = "pull message result is null.";
}
_logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex);
}
SchedulePullRequest(pullRequest);
}
});
}
catch (Exception ex)
{
if (_consumer.Stopped)
{
return;
}
if (pullRequest.IsDropped)
{
return;
}
if (remotingClient.IsConnected)
{
_logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex);
}
SchedulePullRequest(pullRequest);
}
}
public MesssagePullPayloadHolder(PullMessageRequest request)
{
_request = request;
}