public void DeleteQueue(string topic, int queueId) { lock (this) { var key = QueueKeyUtil.CreateQueueKey(topic, queueId); Queue queue; if (!_queueDict.TryGetValue(key, out queue)) { return; } //检查队列对Producer或Consumer是否可见,如果可见是不允许删除的 if (queue.Setting.ProducerVisible || queue.Setting.ConsumerVisible) { throw new Exception("Queue is visible to producer or consumer, cannot be delete."); } //检查是否有未消费完的消息 var minConsumedOffset = _consumeOffsetStore.GetMinConsumedOffset(topic, queueId); var queueCurrentOffset = queue.NextOffset - 1; if (minConsumedOffset < queueCurrentOffset) { throw new Exception(string.Format("Queue is not allowed to delete as there are messages haven't been consumed, not consumed messageCount: {0}", queueCurrentOffset - minConsumedOffset)); } //删除队列的消费进度信息 _consumeOffsetStore.DeleteConsumeOffset(queue.Key); //删除队列本身,包括所有的文件 queue.Delete(); //最后将队列从字典中移除 _queueDict.Remove(key); } }
public void UpdateConsumeOffset(string topic, int queueId, long offset, string group) { var queueOffsetDict = _groupConsumeOffsetsDict.GetOrAdd(group, k => { return new ConcurrentDictionary<string, long>(); }); var key = QueueKeyUtil.CreateQueueKey(topic, queueId); queueOffsetDict.AddOrUpdate(key, offset, (currentKey, oldOffset) => offset > oldOffset ? offset : oldOffset); }
public void SetConsumeNextOffset(string topic, int queueId, string group, long nextOffset) { var queueOffsetDict = _groupNextConsumeOffsetsDict.GetOrAdd(group, k => { return(new ConcurrentDictionary <string, long>()); }); var key = QueueKeyUtil.CreateQueueKey(topic, queueId); queueOffsetDict[key] = nextOffset; }
public void UpdateConsumeOffset(string topic, int queueId, long offset, string group) { var queueOffsetDict = _groupConsumeOffsetsDict.GetOrAdd(group, k => { return(new ConcurrentDictionary <string, long>()); }); var key = QueueKeyUtil.CreateQueueKey(topic, queueId); queueOffsetDict[key] = offset; }
public long GetQueueMinOffset(string topic, int queueId) { var key = QueueKeyUtil.CreateQueueKey(topic, queueId); Queue queue; if (_queueDict.TryGetValue(key, out queue)) { return(queue.GetMinQueueOffset()); } return(-1); }
public Queue GetQueue(string topic, int queueId) { var key = QueueKeyUtil.CreateQueueKey(topic, queueId); Queue queue; if (_queueDict.TryGetValue(key, out queue) && !queue.Setting.IsDeleted) { return(queue); } return(null); }
private void LoadQueue(string topic, int queueId) { var queue = new Queue(topic, queueId); queue.Load(); if (queue.Setting.IsDeleted) { return; } var key = QueueKeyUtil.CreateQueueKey(topic, queueId); _queueDict.TryAdd(key, queue); }
public Queue(string topic, int queueId) { Topic = topic; QueueId = queueId; Key = QueueKeyUtil.CreateQueueKey(topic, queueId); _jsonSerializer = ObjectContainer.Resolve <IJsonSerializer>(); _chunkManager = new ChunkManager(Key, BrokerController.Instance.Setting.QueueChunkConfig, Topic + @"\" + QueueId); _chunkWriter = new ChunkWriter(_chunkManager); _chunkReader = new ChunkReader(_chunkManager, _chunkWriter); _queueSettingFile = Path.Combine(_chunkManager.ChunkPath, QueueSettingFileName); _logger = ObjectContainer.Resolve <ILoggerFactory>().Create(this.GetType().FullName); }
public long GetConsumeOffset(string topic, int queueId, string group) { ConcurrentDictionary<string, long> queueOffsetDict; if (_groupConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict)) { long offset; var key = QueueKeyUtil.CreateQueueKey(topic, queueId); if (queueOffsetDict.TryGetValue(key, out offset)) { return offset; } } return -1L; }
private void SendHeartbeat() { try { var consumingQueues = _pullRequestDict.Values.ToList().Select(x => QueueKeyUtil.CreateQueueKey(x.MessageQueue.Topic, x.MessageQueue.QueueId)).ToList(); _remotingClient.InvokeOneway(new RemotingRequest( (int)RequestCode.ConsumerHeartbeat, _binarySerializer.Serialize(new ConsumerData(GetConsumerId(), GroupName, _subscriptionTopics.Keys, consumingQueues)))); } catch (Exception ex) { if (_remotingClient.IsConnected) { _logger.Error(string.Format("SendHeartbeat remoting request to broker has exception, group: {0}", GroupName), ex); } } }
public bool TryFetchNextConsumeOffset(string topic, int queueId, string group, out long nextOffset) { nextOffset = 0L; ConcurrentDictionary <string, long> queueOffsetDict; if (_groupNextConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict)) { long offset; var key = QueueKeyUtil.CreateQueueKey(topic, queueId); if (queueOffsetDict.TryRemove(key, out offset)) { nextOffset = offset; return(true); } } return(false); }
public IEnumerable <TopicConsumeInfo> QueryTopicConsumeInfos(string groupName, string topic) { var entryList = _groupConsumeOffsetsDict.Where(x => string.IsNullOrEmpty(groupName) || x.Key.Contains(groupName)); var topicConsumeInfoList = new List <TopicConsumeInfo>(); foreach (var entry in entryList) { foreach (var subEntry in entry.Value.Where(x => string.IsNullOrEmpty(topic) || QueueKeyUtil.ParseQueueKey(x.Key)[0].Contains(topic))) { var items = QueueKeyUtil.ParseQueueKey(subEntry.Key); topicConsumeInfoList.Add(new TopicConsumeInfo { ConsumerGroup = entry.Key, Topic = items[0], QueueId = int.Parse(items[1]), ConsumedOffset = subEntry.Value }); } } return(topicConsumeInfoList); }
public long GetMinConsumedOffset(string topic, int queueId) { var key = QueueKeyUtil.CreateQueueKey(topic, queueId); var minOffset = -1L; foreach (var queueOffsetDict in _groupConsumeOffsetsDict.Values) { long offset; if (queueOffsetDict.TryGetValue(key, out offset)) { if (minOffset == -1) { minOffset = offset; } else if (offset < minOffset) { minOffset = offset; } } } return(minOffset); }
public bool IsConsumerExistForQueue(string topic, int queueId) { var key = QueueKeyUtil.CreateQueueKey(topic, queueId); return(_consumerConsumingQueueDict.Values.Any(x => x.Any(y => y == key))); }
public Queue GetQueue(string topic, int queueId) { return(GetQueue(QueueKeyUtil.CreateQueueKey(topic, queueId))); }