public void DeleteQueue(string topic, int queueId) { lock (_lockObj) { var key = new QueueKey(topic, queueId); Queue queue; if (!_queueDict.TryGetValue(key, out queue)) { return; } //检查队列是否可删除 CheckQueueAllowToDelete(queue); //删除队列的消费进度信息 _consumeOffsetStore.DeleteConsumeOffset(queue.Key); //删除队列本身,包括所有的文件 queue.Delete(); //最后将队列从字典中移除 _queueDict.Remove(key); //如果当前Broker上一个队列都没有了,则清空整个Broker下的所有文件 if (_queueDict.IsEmpty) { BrokerController.Instance.Clean(); } } }
public void UpdateConsumeOffset(string topic, int queueId, long offset, string group) { var queueOffsetDict = _groupConsumeOffsetsDict.GetOrAdd(group, k => { return(new ConcurrentDictionary <QueueKey, long>()); }); var key = new QueueKey(topic, queueId); queueOffsetDict[key] = offset; }
private Queue GetQueue(QueueKey key) { Queue queue; if (_queueDict.TryGetValue(key, out queue) && !queue.Setting.IsDeleted) { return(queue); } return(null); }
public void SetConsumeNextOffset(string topic, int queueId, string group, long nextOffset) { var queueOffsetDict = _groupNextConsumeOffsetsDict.GetOrAdd(group, k => { return(new ConcurrentDictionary <QueueKey, long>()); }); var key = new QueueKey(topic, queueId); queueOffsetDict[key] = nextOffset; }
public long GetQueueMinOffset(string topic, int queueId) { var key = new QueueKey(topic, queueId); Queue queue; if (_queueDict.TryGetValue(key, out queue)) { return(queue.GetMinQueueOffset()); } return(-1); }
public long GetQueueCurrentOffset(string topic, int queueId) { var key = new QueueKey(topic, queueId); Queue queue; if (_queueDict.TryGetValue(key, out queue)) { return(queue.NextOffset - 1); } return(-1); }
public Queue(string topic,int queueId) { Topic = topic; QueueId = queueId; Key = new QueueKey(topic, queueId); _jsonSerializer = ObjectContainer.Resolve<IJsonSerializer>(); _chunkManager = new ChunkManager( "QueueChunk-" + Key.ToString(), BrokerController.Instance.Setting. }
public Queue(string topic, int queueId) { Topic = topic; QueueId = queueId; Key = new QueueKey(topic, queueId); _chunkManager = new ChunkManager("QueueChunk-" + Key.ToString(), BrokerService.Instance.Setting.QueueChunkConfig, BrokerService.Instance.Setting.IsMessageStoreMemoryMode, Topic + @"\" + QueueId); _chunkWriter = new ChunkWriter(_chunkManager); _chunkReader = new ChunkReader(_chunkManager, _chunkWriter); _queueSettingFile = Path.Combine(_chunkManager.ChunkPath, QueueSettingFileName); }
public void ResetQueue(string queueName, string urlPrefix) { if (_serverDelegationFeature is not null) { var key = new QueueKey(queueName, urlPrefix); if (_queues.TryGetValue(key, out var queueWeakRef) && queueWeakRef.TryGetTarget(out var queue)) { queue.Detatch(); Log.QueueReset(_logger, queueName, urlPrefix); } } }
public void DeleteConsumeOffset(QueueKey queueKey) { foreach (var dict in _groupConsumeOffsetsDict.Values) { var keys = dict.Keys.Where(x => x == queueKey); foreach (var key in keys) { dict.Remove(key); } } PersistConsumeOffsetInfo(); }
public Queue(string topic, int queueId) { Topic = topic; QueueId = queueId; Key = new QueueKey(topic, queueId); _jsonSerializer = ObjectContainer.Resolve<IJsonSerializer>(); _chunkManager = new ChunkManager("QueueChunk-" + Key.ToString(), BrokerController.Instance.Setting.QueueChunkConfig, BrokerController.Instance.Setting.IsMessageStoreMemoryMode, Topic + @"\" + QueueId); _chunkWriter = new ChunkWriter(_chunkManager); _chunkReader = new ChunkReader(_chunkManager, _chunkWriter); _queueSettingFile = Path.Combine(_chunkManager.ChunkPath, QueueSettingFileName); _logger = ObjectContainer.Resolve<ILoggerFactory>().Create(this.GetType().FullName); }
public Queue(string topic, int queueId) { Topic = topic; QueueId = queueId; Key = new QueueKey(topic, queueId); _jsonSerializer = ObjectContainer.Resolve <IJsonSerializer>(); _chunkManager = new ChunkManager(Key.ToString(), BrokerController.Instance.Setting.QueueChunkConfig, BrokerController.Instance.Setting.IsMessageStoreMemoryMode, Topic + @"\" + QueueId); _chunkWriter = new ChunkWriter(_chunkManager); _chunkReader = new ChunkReader(_chunkManager, _chunkWriter); _queueSettingFile = Path.Combine(_chunkManager.ChunkPath, QueueSettingFileName); _logger = ObjectContainer.Resolve <ILoggerFactory>().Create(this.GetType().FullName); }
private void LoadQueue(string topic, int queueId) { var queue = new Queue(topic, queueId); queue.Load(); if (queue.Setting.IsDeleted) { return; } var key = new QueueKey(topic, queueId); _queueDict.TryAdd(key, queue); }
public long GetConsumeOffset(string topic, int queueId, string group) { ConcurrentDictionary<QueueKey, long> queueOffsetDict; if (_groupConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict)) { long offset; var key = new QueueKey(topic, queueId); if (queueOffsetDict.TryGetValue(key, out offset)) { return offset; } } return -1L; }
public long GetConsumeOffset(string topic, int queueId, string group) { ConcurrentDictionary <QueueKey, long> queueOffsetDict; if (_groupConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict)) { long offset; var key = new QueueKey(topic, queueId); if (queueOffsetDict.TryGetValue(key, out offset)) { return(offset); } } return(-1L); }
public bool TryFetchNextConsumeOffset(string topic, int queueId, string group, out long nextOffset) { nextOffset = 0L; ConcurrentDictionary <QueueKey, long> queueOffsetDict; if (_groupNextConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict)) { long offset; var key = new QueueKey(topic, queueId); if (queueOffsetDict.TryRemove(key, out offset)) { nextOffset = offset; return(true); } } return(false); }
public long GetMinConsumedOffset(string topic, int queueId) { var key = new QueueKey(topic, queueId); var minOffset = -1L; foreach (var queueOffsetDict in _groupConsumeOffsetsDict.Values) { long offset; if (queueOffsetDict.TryGetValue(key, out offset)) { if (minOffset == -1) { minOffset = offset; } else if (offset < minOffset) { minOffset = offset; } } } return(minOffset); }
private ConcurrentDictionary <string, ConcurrentDictionary <QueueKey, long> > ConvertDictFrom(ConcurrentDictionary <string, ConcurrentDictionary <string, ConcurrentDictionary <int, long> > > source) { var toDict = new ConcurrentDictionary <string, ConcurrentDictionary <QueueKey, long> >(); foreach (var entry1 in source) { var key1 = entry1.Key; var dict = toDict.GetOrAdd(key1, x => new ConcurrentDictionary <QueueKey, long>()); foreach (var entry2 in entry1.Value) { var topic = entry2.Key; foreach (var entry3 in entry2.Value) { var queueId = entry3.Key; var offset = entry3.Value; var queueKey = new QueueKey(topic, queueId); dict.TryAdd(queueKey, offset); } } } return(toDict); }
public void UpdateConsumeOffset(string topic, int queueId, long offset, string group) { var queueOffsetDict = _groupConsumeOffsetsDict.GetOrAdd(group, k => { return new ConcurrentDictionary<QueueKey, long>(); }); var key = new QueueKey(topic, queueId); queueOffsetDict[key] = offset; }
private void AddOrUpdateRules(ClusterState cluster) { if (_serverDelegationFeature is null) { return; } // We support multiple destinations referencing the same queue but http.sys only // allows us to create one handle to the queue, so we keep track of queues two ways. // 1. We map destination => queue using a ConditionalWeakTable // This allows us to find the queue to delegate to when processing a request // 2. We map (queue name + url prefix) => queue using a WeakReference // This allows us to find the already created queue if more than one destination points to the same queue. // It also allows us to find the queue by name/url prefix to support reset. // // Using weak references means we ensure the queue exist as long as the referencing destinations exist. // Once all the destinations are gone, GC will eventually finalize the underlying SafeHandle to the http.sys // queue, which will clean up references in http.sys, allowing us to re-create it again later if needed. foreach (var destination in cluster.Destinations.Select(kvp => kvp.Value)) { var queueName = destination.GetHttpSysDelegationQueue(); var urlPrefix = destination.Model?.Config?.Address; if (queueName is not null && urlPrefix is not null) { var queueKey = new QueueKey(queueName, urlPrefix); if (!_queuesPerDestination.TryGetValue(destination, out var queue) || !queue.Equals(queueKey)) { if (!_queues.TryGetValue(queueKey, out var queueWeakRef) || !queueWeakRef.TryGetTarget(out queue)) { // Either the queue hasn't been created or it has been cleaned up. // Create a new one, and try to add it if someone else didn't beat us to it. queue = new DelegationQueue(queueName, urlPrefix); queueWeakRef = new WeakReference <DelegationQueue>(queue); queueWeakRef = _queues.AddOrUpdate( queueKey, (key, newValue) => newValue, (key, value, newValue) => value.TryGetTarget(out _) ? value : newValue, queueWeakRef); queueWeakRef.TryGetTarget(out queue); } if (queue is not null) { // We call this outside of the above if bock so that if previous // initialization failed, we will retry it for every new destination added. var queueState = queue.Initialize(_serverDelegationFeature); if (!queueState.IsInitialized) { Log.QueueInitFailed( _logger, destination.DestinationId, queueName, urlPrefix, queueState.InitializationException); } _queuesPerDestination.AddOrUpdate(destination, queue); } else { // This should never happen because we always create a new one above _queuesPerDestination.Remove(destination); Log.QueueInitFailed( _logger, destination.DestinationId, queueName, urlPrefix, new Exception("Delegation queue is null after adding a new one. This shouldn't happen.")); } } }
public long GetMinConsumedOffset(string topic, int queueId) { var key = new QueueKey(topic, queueId); var minOffset = -1L; foreach (var queueOffsetDict in _groupConsumeOffsetsDict.Values) { long offset; if (queueOffsetDict.TryGetValue(key, out offset)) { if (minOffset == -1) { minOffset = offset; } else if (offset < minOffset) { minOffset = offset; } } } return minOffset; }
public bool IsQueueExist(QueueKey queueKey) { return(GetQueue(queueKey) != null); }
private ConcurrentDictionary<string, ConcurrentDictionary<QueueKey, long>> ConvertDictFrom(ConcurrentDictionary<string, ConcurrentDictionary<string, ConcurrentDictionary<int, long>>> source) { var toDict = new ConcurrentDictionary<string, ConcurrentDictionary<QueueKey, long>>(); foreach (var entry1 in source) { var key1 = entry1.Key; var dict = toDict.GetOrAdd(key1, x => new ConcurrentDictionary<QueueKey, long>()); foreach (var entry2 in entry1.Value) { var topic = entry2.Key; foreach (var entry3 in entry2.Value) { var queueId = entry3.Key; var offset = entry3.Value; var queueKey = new QueueKey(topic, queueId); dict.TryAdd(queueKey, offset); } } } return toDict; }
public bool TryFetchNextConsumeOffset(string topic, int queueId, string group, out long nextOffset) { nextOffset = 0L; ConcurrentDictionary<QueueKey, long> queueOffsetDict; if (_groupNextConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict)) { long offset; var key = new QueueKey(topic, queueId); if (queueOffsetDict.TryRemove(key, out offset)) { nextOffset = offset; return true; } } return false; }
public void SetConsumeNextOffset(string topic, int queueId, string group, long nextOffset) { var queueOffsetDict = _groupNextConsumeOffsetsDict.GetOrAdd(group, k => { return new ConcurrentDictionary<QueueKey, long>(); }); var key = new QueueKey(topic, queueId); queueOffsetDict[key] = nextOffset; }
public bool IsConsumerExistForQueue(string topic, int queueId) { var key = new QueueKey(topic, queueId); return(_consumerConsumingQueueDict.Values.Any(x => x.Any(y => y == key))); }
public void DESFunctionRKTest() { QueueKey queueKey = new QueueKey(TestData.ExampleKey); DesMethods.DESFunctionRK(TestData.ExampleKey, queueKey.GetKey(0)); }
protected override void Execute(CodeActivityContext context) { // Get the connection string DBConnection ext = context.GetExtension <DBConnection>(); if (ext == null) { throw new InvalidProgramException("No connection string available"); } // Lookup the operator UserTasksDataContext dc = new UserTasksDataContext(ext.ConnectionString); OperatorConfig oc = dc.OperatorConfigs.SingleOrDefault(x => x.OperatorKey == OperatorKey.Get(context)); if (oc == null) { oc = new OperatorConfig(); oc.OperatorKey = OperatorKey.Get(context); oc.UnderEvaluation = false; oc.Frequency = 5; oc.NumberSinceLastEval = 0; dc.OperatorConfigs.InsertOnSubmit(oc); dc.SubmitChanges(); } // Determine the Queue info char[] delimiter = { '|' }; string[] values = QueueKey.Get(context).Split(delimiter, 3); // Lookup the queue and subqueue Queue q = dc.Queues.SingleOrDefault(x => x.QueueName == values[0]); if (q == null) { throw new InvalidProgramException("The specified queue (" + values[0] + ") was not found"); } SubQueue sq = dc.SubQueues.SingleOrDefault(x => x.QueueID == q.QueueID && x.SubQueueName == values[1]); if (sq == null) { throw new InvalidProgramException("The specified subqueue (" + values[0] + " - " + values[1] + ") was not found"); } bool bQC = bool.Parse(values[2]); if (sq.AllowSelection) { // Return all the available instances IEnumerable <QueueInstance> instances; if (bQC) { instances = dc.QueueInstances .Where(x => x.CurrentSubQueueID == sq.SubQueueID && x.QC == bQC && (x.AssignedOperatorID == null || x.AssignedOperatorID == oc.OperatorConfigID)) .OrderBy(x => x.Priority.Value) .OrderBy(x => x.CreateDate); } else { instances = dc.QueueInstances .Where(x => x.CurrentSubQueueID == sq.SubQueueID && x.QC == bQC && (x.AssignedOperatorID == null || x.AssignedOperatorID == oc.OperatorConfigID)) .OrderBy(x => x.CreateDate); } if (instances.Count() > 0) { QueueInstance[] qiList = new QueueInstance[instances.Count()]; int i = 0; foreach (QueueInstance r in instances) { qiList[i++] = r; } QueueInstanceList.Set(context, qiList); } } else { // Return the oldest instance IEnumerable <QueueInstance> instances; if (bQC) { instances = dc.QueueInstances .Where(x => x.CurrentSubQueueID == sq.SubQueueID && x.QC == bQC && (x.AssignedOperatorID == null || x.AssignedOperatorID == oc.OperatorConfigID)) .OrderBy(x => x.Priority.Value) .OrderBy(x => x.CreateDate) .Take(1); } else { instances = dc.QueueInstances .Where(x => x.CurrentSubQueueID == sq.SubQueueID && x.QC == bQC && (x.AssignedOperatorID == null || x.AssignedOperatorID == oc.OperatorConfigID)) .OrderBy(x => (x.Priority.HasValue ? 1 : 0) * x.Priority.Value * (bQC ? 1 : 0)) .OrderBy(x => x.CreateDate) .Take(1); } if (instances.Count() > 0) { QueueInstance qi = instances.First <QueueInstance>(); qi.AssignedOperatorID = oc.OperatorConfigID; dc.SubmitChanges(); QueueInstance[] qiList = new QueueInstance[1]; qiList[0] = qi; QueueInstanceList.Set(context, qiList); } } }