public static void WriteToStream(PullMessageRequest request, Stream stream) { using (var writer = new BinaryWriter(stream)) { writer.Write(request.ConsumerGroup); writer.Write(request.MessageQueue.Topic); writer.Write(request.MessageQueue.QueueId); writer.Write(request.QueueOffset); writer.Write(request.PullMessageBatchSize); writer.Write(request.SuspendPullRequestMilliseconds); writer.Write((int)request.ConsumeFromWhere); } }
public static PullMessageRequest ReadFromStream(Stream stream) { using (var reader = new BinaryReader(stream)) { var request = new PullMessageRequest(); request.ConsumerGroup = reader.ReadString(); request.MessageQueue = new MessageQueue(reader.ReadString(), reader.ReadInt32()); request.QueueOffset = reader.ReadInt64(); request.PullMessageBatchSize = reader.ReadInt32(); request.SuspendPullRequestMilliseconds = reader.ReadInt64(); request.ConsumeFromWhere = (ConsumeFromWhere)reader.ReadInt32(); return request; } }
public static PullMessageRequest ReadFromStream(Stream stream) { using (var reader = new BinaryReader(stream)) { var request = new PullMessageRequest(); request.ConsumerGroup = reader.ReadString(); request.MessageQueue = new MessageQueue(reader.ReadString(), reader.ReadInt32()); request.QueueOffset = reader.ReadInt64(); request.PullMessageBatchSize = reader.ReadInt32(); request.SuspendPullRequestMilliseconds = reader.ReadInt64(); request.ConsumeFromWhere = (ConsumeFromWhere)reader.ReadInt32(); return(request); } }
public PullRequest( long remotingRequestSequence, PullMessageRequest pullMessageRequest, IRequestHandlerContext requestHandlerContext, DateTime suspendTime, long suspendMilliseconds, Action<PullRequest> newMessageArrivedAction, Action<PullRequest> suspendTimeoutAction, Action<PullRequest> replacedAction) { RemotingRequestSequence = remotingRequestSequence; PullMessageRequest = pullMessageRequest; RequestHandlerContext = requestHandlerContext; SuspendTime = suspendTime; SuspendMilliseconds = suspendMilliseconds; NewMessageArrivedAction = newMessageArrivedAction; SuspendTimeoutAction = suspendTimeoutAction; ReplacedAction = replacedAction; }
public PullRequest( long remotingRequestSequence, PullMessageRequest pullMessageRequest, IRequestHandlerContext requestHandlerContext, DateTime suspendStartTime, long suspendMilliseconds, Action<PullRequest> newMessageArrivedAction, Action<PullRequest> timeoutAction, Action<PullRequest> replacedAction) { Id = ObjectId.GenerateNewStringId(); RemotingRequestSequence = remotingRequestSequence; PullMessageRequest = pullMessageRequest; RequestHandlerContext = requestHandlerContext; SuspendStartTime = suspendStartTime; SuspendMilliseconds = suspendMilliseconds; NewMessageArrivedAction = newMessageArrivedAction; TimeoutAction = timeoutAction; ReplacedAction = replacedAction; }
public PullRequest( RemotingRequest remotingRequest, PullMessageRequest pullMessageRequest, IRequestHandlerContext requestHandlerContext, DateTime suspendStartTime, long suspendMilliseconds, Action<PullRequest> newMessageArrivedAction, Action<PullRequest> timeoutAction, Action<PullRequest> noNewMessageAction, Action<PullRequest> replacedAction) { RemotingRequest = remotingRequest; PullMessageRequest = pullMessageRequest; RequestHandlerContext = requestHandlerContext; SuspendStartTime = suspendStartTime; SuspendMilliseconds = suspendMilliseconds; NewMessageArrivedAction = newMessageArrivedAction; TimeoutAction = timeoutAction; NoNewMessageAction = noNewMessageAction; ReplacedAction = replacedAction; }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; var messageCount = pullRequest.ProcessQueue.GetMessageCount(); var flowControlThreshold = Setting.PullMessageFlowControlThreshold; if (flowControlThreshold > 0 && messageCount >= flowControlThreshold) { var milliseconds = FlowControlUtil.CalculateFlowControlTimeMilliseconds( messageCount, flowControlThreshold, Setting.PullMessageFlowControlStepPercent, Setting.PullMessageFlowControlStepWaitMilliseconds); Task.Factory.StartDelayedTask(milliseconds, () => SchedulePullRequest(pullRequest)); return; } var request = new PullMessageRequest { ConsumerId = GetConsumerId(), ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, Tags = string.Join("|", pullRequest.Tags), QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_remotingClient.IsConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_remotingClient.IsConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
private static byte[] SerializePullMessageRequest(PullMessageRequest request) { using (var stream = new MemoryStream()) { PullMessageRequest.WriteToStream(request, stream); return stream.ToArray(); } }
private void PullMessage() { var messageCount = ProcessQueue.GetMessageCount(); if (messageCount >= _setting.PullThresholdForQueue) { Thread.Sleep(_setting.PullTimeDelayMillsWhenFlowControl); if ((_flowControlTimes++ % 1000) == 0) { _logger.WarnFormat("Detect that the message local process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", this, messageCount, _flowControlTimes); } } var request = new PullMessageRequest { ConsumerGroup = GroupName, MessageQueue = MessageQueue, QueueOffset = _queueOffset, PullMessageBatchSize = _setting.PullMessageBatchSize }; var data = _binarySerializer.Serialize(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); var remotingResponse = default(RemotingResponse); try { remotingResponse = _remotingClient.InvokeSync(remotingRequest, _setting.PullRequestTimeoutMilliseconds); } catch (Exception ex) { if (!_stoped) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}, remotingRequest:{1}.", this, remotingRequest), ex); } return; } if (_stoped) { return; } var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body); if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0) { _queueOffset += response.Messages.Count(); ProcessQueue.AddMessages(response.Messages); response.Messages.ForEach(x => _messageQueue.Add(new WrappedMessage(MessageQueue, x, ProcessQueue))); } else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset && response.NextOffset != null) { _queueOffset = response.NextOffset.Value; } }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; var messageCount = pullRequest.ProcessQueue.GetMessageCount(); if (messageCount >= Setting.PullThresholdForQueue) { Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest)); if ((_flowControlTimes++ % 100) == 0) { _logger.WarnFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes); } return; } var request = new PullMessageRequest { ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds, ConsumeFromWhere = Setting.ConsumeFromWhere }; var data = SerializePullMessageRequest(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (pullTask.Exception != null) { _logger.Error(string.Format("Pull message failed, pullRequest:{0}", pullRequest), pullTask.Exception); SchedulePullRequest(pullRequest); return; } ProcessPullResponse(pullRequest, pullTask.Result); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { string remotingResponseBodyLength; if (pullTask.Result != null) { remotingResponseBodyLength = pullTask.Result.Body.Length.ToString(); } else { remotingResponseBodyLength = "pull message result is null."; } _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}, remotingResponseBodyLength:{1}", pullRequest, remotingResponseBodyLength), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }
private void PullMessage() { var messageCount = ProcessQueue.GetMessageCount(); //TODO, here the GetMessageSpan has bug when in parallel environment. //var messageSpan = ProcessQueue.GetMessageSpan(); if (messageCount >= _setting.PullThresholdForQueue) { Thread.Sleep(_setting.PullTimeDelayMillsWhenFlowControl); if ((_flowControlTimes1++ % 3000) == 0) { _logger.WarnFormat("[{0}]: the consumer message buffer is full, so do flow control, [messageCount={1},pullRequest={2},flowControlTimes={3}]", ConsumerId, messageCount, this, _flowControlTimes1); } } //else if (messageSpan >= _setting.ConsumeMaxSpan) //{ // Thread.Sleep(_setting.PullTimeDelayMillsWhenFlowControl); // if ((flowControlTimes2++ % 3000) == 0) // { // _logger.WarnFormat("[{0}]: the consumer message span too long, so do flow control, [messageSpan={1},pullRequest={2},flowControlTimes={3}]", ConsumerId, messageSpan, this, flowControlTimes2); // } //} var request = new PullMessageRequest { ConsumerGroup = GroupName, MessageQueue = MessageQueue, QueueOffset = NextOffset, PullMessageBatchSize = _setting.PullMessageBatchSize }; var data = _binarySerializer.Serialize(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); var remotingResponse = default(RemotingResponse); try { remotingResponse = _remotingClient.InvokeSync(remotingRequest, _setting.PullRequestTimeoutMilliseconds); } catch (Exception ex) { if (!_stoped) { _logger.Error(string.Format("[{0}]: PullMessage has exception. RemotingRequest: {1}, PullRequest: {2}.", ConsumerId, remotingRequest, this), ex); } return; } if (_stoped) { return; } var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body); if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0) { NextOffset += response.Messages.Count(); ProcessQueue.AddMessages(response.Messages); response.Messages.ForEach(x => _messageQueue.Add(new WrappedMessage(MessageQueue, x, ProcessQueue))); } }
private void PullMessage(PullRequest pullRequest) { try { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; var messageCount = pullRequest.ProcessQueue.GetMessageCount(); if (messageCount >= Setting.PullThresholdForQueue) { Task.Factory.StartDelayedTask(Setting.PullTimeDelayMillsWhenFlowControl, () => SchedulePullRequest(pullRequest)); if ((_flowControlTimes++ % 100) == 0) { _logger.DebugFormat("Detect that the message process queue has too many messages, so do flow control. pullRequest={0}, queueMessageCount={1}, flowControlTimes={2}", pullRequest, messageCount, _flowControlTimes); } return; } var request = new PullMessageRequest { ConsumerGroup = GroupName, MessageQueue = pullRequest.MessageQueue, QueueOffset = pullRequest.NextConsumeOffset, PullMessageBatchSize = Setting.PullMessageBatchSize, SuspendPullRequestMilliseconds = Setting.SuspendPullRequestMilliseconds }; var data = _binarySerializer.Serialize(request); var remotingRequest = new RemotingRequest((int)RequestCode.PullMessage, data); pullRequest.PullStartTime = DateTime.Now; _remotingClient.InvokeAsync(remotingRequest, Setting.PullRequestTimeoutMilliseconds).ContinueWith(pullTask => { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; try { if (pullTask.Exception != null) { SchedulePullRequest(pullRequest); return; } var remotingResponse = pullTask.Result; var response = _binarySerializer.Deserialize<PullMessageResponse>(remotingResponse.Body); if (remotingResponse.Code == (int)PullStatus.Found && response.Messages.Count() > 0) { pullRequest.ProcessQueue.AddMessages(response.Messages); foreach (var message in response.Messages) { _consumingMessageQueue.Add(new ConsumingMessage(message, pullRequest.ProcessQueue)); } pullRequest.NextConsumeOffset += response.Messages.Count(); } else if (remotingResponse.Code == (int)PullStatus.NextOffsetReset && response.NextOffset != null) { var oldConsumeOffset = pullRequest.NextConsumeOffset; pullRequest.NextConsumeOffset = response.NextOffset.Value; _logger.DebugFormat("Updated queue next consume offset. topic:{0}, queueId:{1}, old offset:{2}, new offset:{3}", pullRequest.MessageQueue.Topic, pullRequest.MessageQueue.QueueId, oldConsumeOffset, pullRequest.NextConsumeOffset); } if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (remotingResponse.Code == (int)PullStatus.Ignored) return; SchedulePullRequest(pullRequest); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { _logger.Error(string.Format("Process pull result has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }); } catch (Exception ex) { if (_stoped) return; if (pullRequest.ProcessQueue.IsDropped) return; if (_isBrokerServerConnected) { _logger.Error(string.Format("PullMessage has exception, pullRequest:{0}", pullRequest), ex); } SchedulePullRequest(pullRequest); } }