private void Consumer() { while (PQStatus == Status.RunIng) { try { var itemCount = PQ.Count; if (itemCount > 0) { var list = new List <Object>(); lock (PQ) { //启用了消费阀值 && 当前队列项数大于等于消费阀值 if (pqSetting.ConsumerItemCount > 0 && itemCount >= pqSetting.ConsumerItemCount) { for (var i = 0; i < pqSetting.ConsumerItemCount; i++) { var v = PQ.Dequeue().Value; if (v != null) { list.Add(v); } } } else if (itemCount < pqSetting.ConsumerItemCount) { for (var i = 0; i < itemCount; i++) { var v = PQ.Dequeue().Value; if (v != null) { list.Add(v); } } } } if (list.Count > 0) { var _watch = Stopwatch.StartNew(); YmatouLoggingService.Debug("开始消费,qNmame->{0},itemCount->{1}", pqSetting.QName, list.Count); try { //lock(list) pqSetting.ConsumeAction(list); } catch (QueueException e) { YmatouLoggingService.Debug(string.Format("PQService客户端消费错误,pqName->{0},msg->{1},stack->{2}", pqSetting.QName, e.Message, e.StackTrace)); } YmatouLoggingService.Debug("结束消费,qNmame->{0},itemCount->{1},runTime->{2} ms", pqSetting.QName, list.Count, _watch.ElapsedMilliseconds); _watch.Reset(); var _listQv = list.ConvertAll <PQValue <K, V> >(e => { return(e as PQValue <K, V>); }); if (_listQv != null && _listQv.Count > 0) { var consumerNotCount = _listQv.Where(e => e != null && e.ConsumerResult == QueueItemConsumerStats.NoConsumer); var consumerFailCount = _listQv.Where(e => e != null && e.ConsumerResult == QueueItemConsumerStats.Fail); var tmpConsumerNotCount = consumerNotCount.Count(); var tmpconsumerFailCount = consumerFailCount.Count(); var consumerOkCount = _listQv.Count - tmpConsumerNotCount - tmpconsumerFailCount; #if DEBUG YmatouLoggingService.Debug("消费结果:总共->{0},未消费->{1},成功->{2},失败->{3}", list.Count, tmpConsumerNotCount, consumerOkCount, tmpconsumerFailCount); #else QueueLogs.DebugFormat("消费结果:总共->{0},未消费->{1},成功->{2},失败->{3}", list.Count, tmpConsumerNotCount, consumerOkCount, tmpconsumerFailCount); #endif if (tmpConsumerNotCount > 0) { foreach (var o in consumerNotCount) { PQ.Enqueue(o.Key, o); } } if (tmpconsumerFailCount > 0) { if (pqSetting.CFS == ConsumerFailStrategy.Delet) { } else if (pqSetting.CFS == ConsumerFailStrategy.PersistenceEnqueue) { } else if (pqSetting.CFS == ConsumerFailStrategy.ClientHandleFail) { } else if (pqSetting.CFS == ConsumerFailStrategy.AgainEnqueue) { foreach (var o in consumerFailCount) { //消费失败次数累计 var tmpCount = o.ConsumerCount; Interlocked.Add(ref tmpCount, 1); o.ConsumerCount = tmpCount; if (o.ConsumerCount <= 3) { PQ.Enqueue(o.Key, o); } } } } } } } } catch (QueueException ex) { YmatouLoggingService.Error(string.Format("PQService 服务端错误,pqName->{0},msg->{1},stack->{2}", pqSetting.QName, ex.Message, ex.StackTrace)); } finally { Thread.Sleep(pqSetting.ScanningTime); } } }
private void Consumer() { while (BQStatus == Status.RunIng) { _consumerStats = true; var _consumerTime = _queueSettings.ConsumerTime; var _consumerItemCount = _queueSettings.ConsumerItemCount; //记录一次消费的状态 var _currentQueueStats = new QueueStats(); _currentQueueStats.QueueName = _queueSettings.QueueName; _currentQueueStats.ConsumerThreadName = Thread.CurrentThread.Name; var _listConsumerItem = new List <Object>(_consumerItemCount); try { var _queueCount = _queue.Count; _currentQueueStats.CurrentQueueCount = _queueCount; lock (_queue) { //批量消费策略 if (_queueCount > 0) { if (_queueCount >= _consumerItemCount) { for (var i = 0; i < _consumerItemCount; i++) { var v = _queue.Dequeue(); if (v != null) { _listConsumerItem.Add(v); } } } else { //TODO:生产不足策略:等待数据到来,或消费掉不足的所有数据 #if DEBUG YmatouLoggingService.Debug("生产数据不足:{0},{1},{2}", _queueSettings.QueueName, _consumerItemCount, _queueCount); #endif for (var i = 0; i < _queueCount; i++) { var v = _queue.Dequeue(); if (v != null) { _listConsumerItem.Add(v); } } } } } if (_listConsumerItem.Count > 0) { //单个消费 //var item = _queue.Dequeue(); try { //监控 var _watch = Stopwatch.StartNew(); //客户端消费数据 try { YmatouLoggingService.Debug("进行一次消费 {0}-> {1} 项", _queueSettings.QueueName, _listConsumerItem.Count); _queueSettings.ConsumerAction(_listConsumerItem); } catch (QueueException ce) { _currentQueueStats.ClientErrorMessage = ce.Message; _currentQueueStats.ClientErrorTime = DateTime.Now; YmatouLoggingService.Debug("客户端消费错误:{0},{1},{2}", _queueSettings.QueueName, ce.Message, ce.StackTrace); } catch (Exception e) { _currentQueueStats.ClientErrorMessage = e.Message; _currentQueueStats.ClientErrorTime = DateTime.Now; YmatouLoggingService.Debug("客户端消费错误:{0},{1},{2}", _queueSettings.QueueName, e.Message, e.StackTrace); } _currentQueueStats.ConsumerUseTime = _watch.ElapsedMilliseconds; _watch.Stop(); //一次消费完成,状态跟踪 //统计总共消费 //Interlocked.Add(ref _beenConsumedCount, _listConsumerItem.Count); var _listQv = _listConsumerItem.ConvertAll <QueueValue <T> >(e => { return(e as QueueValue <T>); }); if (_listQv.Count > 0) { _currentQueueStats.SuccessConsumedCount = _listQv.Count(e => e != null && e.ConsumerResult == QueueItemConsumerStats.Ok); _currentQueueStats.FailConsumedCount = _listQv.Count(e => e != null && e.ConsumerResult != QueueItemConsumerStats.Ok); _currentQueueStats.CurrentBeenConsumedCount = _listConsumerItem.Count; //一次消费完成,对消费失败的数据再处理 if (_queueSettings.ConsumerFS != ConsumerFailStrategy.ClientHandleFail) { if (_queueSettings.ConsumerFS == ConsumerFailStrategy.Delet) { } else if (_queueSettings.ConsumerFS == ConsumerFailStrategy.PersistenceEnqueue) { } else { var _listFailItem = _listQv.Where(e => e != null && e.ConsumerResult != QueueItemConsumerStats.Ok && e.ConsumerCount <= _consumerFailCount); foreach (var item in _listFailItem) { var _errorCount = item.ConsumerCount; Interlocked.Add(ref _errorCount, 1); item.ConsumerCount = _errorCount; if (item.ConsumerCount < _consumerFailCount) { _queue.Enqueue(item); } } } } } } catch (QueueException e) { #if !DEBUG QueueLogs.ErrorFormat("队列服务错误,{0},{1},{2}", _queueSettings.QueueName, e.Message, e.StackTrace); #endif _currentQueueStats.ErrorTime = DateTime.Now; _currentQueueStats.ErrorMessage = e.Message; Interlocked.Add(ref _errorCount, 1); _currentQueueStats.ConsumerErrorItemCount = _errorCount; var _listQv = _listConsumerItem.ConvertAll <QueueValue <T> >(_ => { return(_ as QueueValue <T>); }); foreach (var item in _listQv) { _queue.Enqueue(item); } } } } catch (QueueException ex) { #if !DEBUG QueueLogs.ErrorFormat("客户端消费错误2:{0},{1},{2}", _queueSettings.QueueName, ex.Message, ex.StackTrace); #endif _currentQueueStats.ErrorMessage += "" + ex.Message; _currentQueueStats.ErrorTime = DateTime.Now; } finally { _currentQueueStats.NextConsumerTime = DateTime.Now.Add(_consumerTime); #if DEBUGs BlockingBoundedQueueStatsHelp.QueueStatsLogs(_currentQueueStats); #endif Thread.Sleep(_consumerTime); } } }