public void DeleteQueue(string topic, int queueId)
{
lock (_lockObj)
{
var key = new QueueKey(topic, queueId);
Queue queue;
if (!_queueDict.TryGetValue(key, out queue))
{
return;
}
//检查队列是否可删除
CheckQueueAllowToDelete(queue);
//删除队列的消费进度信息
_consumeOffsetStore.DeleteConsumeOffset(queue.Key);
//删除队列本身，包括所有的文件
queue.Delete();
//最后将队列从字典中移除
_queueDict.Remove(key);
//如果当前Broker上一个队列都没有了，则清空整个Broker下的所有文件
if (_queueDict.IsEmpty)
{
BrokerController.Instance.Clean();
}
}
}
public void UpdateConsumeOffset(string topic, int queueId, long offset, string group)
{
var queueOffsetDict = _groupConsumeOffsetsDict.GetOrAdd(group, k =>
{
return(new ConcurrentDictionary <QueueKey, long>());
});
var key = new QueueKey(topic, queueId);
queueOffsetDict[key] = offset;
}
private Queue GetQueue(QueueKey key)
{
Queue queue;
if (_queueDict.TryGetValue(key, out queue) && !queue.Setting.IsDeleted)
{
return(queue);
}
return(null);
}
public void SetConsumeNextOffset(string topic, int queueId, string group, long nextOffset)
{
var queueOffsetDict = _groupNextConsumeOffsetsDict.GetOrAdd(group, k =>
{
return(new ConcurrentDictionary <QueueKey, long>());
});
var key = new QueueKey(topic, queueId);
queueOffsetDict[key] = nextOffset;
}
public long GetQueueMinOffset(string topic, int queueId)
{
var key = new QueueKey(topic, queueId);
Queue queue;
if (_queueDict.TryGetValue(key, out queue))
{
return(queue.GetMinQueueOffset());
}
return(-1);
}
public long GetQueueCurrentOffset(string topic, int queueId)
{
var key = new QueueKey(topic, queueId);
Queue queue;
if (_queueDict.TryGetValue(key, out queue))
{
return(queue.NextOffset - 1);
}
return(-1);
}
public Queue(string topic,int queueId)
{
Topic = topic;
QueueId = queueId;
Key = new QueueKey(topic, queueId);
_jsonSerializer = ObjectContainer.Resolve<IJsonSerializer>();
_chunkManager = new ChunkManager(
"QueueChunk-" + Key.ToString(),
BrokerController.Instance.Setting.
}
public Queue(string topic, int queueId)
{
Topic = topic;
QueueId = queueId;
Key = new QueueKey(topic, queueId);
_chunkManager = new ChunkManager("QueueChunk-" + Key.ToString(), BrokerService.Instance.Setting.QueueChunkConfig, BrokerService.Instance.Setting.IsMessageStoreMemoryMode, Topic + @"\" + QueueId);
_chunkWriter = new ChunkWriter(_chunkManager);
_chunkReader = new ChunkReader(_chunkManager, _chunkWriter);
_queueSettingFile = Path.Combine(_chunkManager.ChunkPath, QueueSettingFileName);
}
public void ResetQueue(string queueName, string urlPrefix)
{
if (_serverDelegationFeature is not null)
{
var key = new QueueKey(queueName, urlPrefix);
if (_queues.TryGetValue(key, out var queueWeakRef) && queueWeakRef.TryGetTarget(out var queue))
{
queue.Detatch();
Log.QueueReset(_logger, queueName, urlPrefix);
}
}
}
public void DeleteConsumeOffset(QueueKey queueKey)
{
foreach (var dict in _groupConsumeOffsetsDict.Values)
{
var keys = dict.Keys.Where(x => x == queueKey);
foreach (var key in keys)
{
dict.Remove(key);
}
}
PersistConsumeOffsetInfo();
}
public Queue(string topic, int queueId)
{
Topic = topic;
QueueId = queueId;
Key = new QueueKey(topic, queueId);
_jsonSerializer = ObjectContainer.Resolve<IJsonSerializer>();
_chunkManager = new ChunkManager("QueueChunk-" + Key.ToString(), BrokerController.Instance.Setting.QueueChunkConfig, BrokerController.Instance.Setting.IsMessageStoreMemoryMode, Topic + @"\" + QueueId);
_chunkWriter = new ChunkWriter(_chunkManager);
_chunkReader = new ChunkReader(_chunkManager, _chunkWriter);
_queueSettingFile = Path.Combine(_chunkManager.ChunkPath, QueueSettingFileName);
_logger = ObjectContainer.Resolve<ILoggerFactory>().Create(this.GetType().FullName);
}
public Queue(string topic, int queueId)
{
Topic = topic;
QueueId = queueId;
Key = new QueueKey(topic, queueId);
_jsonSerializer = ObjectContainer.Resolve <IJsonSerializer>();
_chunkManager = new ChunkManager(Key.ToString(), BrokerController.Instance.Setting.QueueChunkConfig, BrokerController.Instance.Setting.IsMessageStoreMemoryMode, Topic + @"\" + QueueId);
_chunkWriter = new ChunkWriter(_chunkManager);
_chunkReader = new ChunkReader(_chunkManager, _chunkWriter);
_queueSettingFile = Path.Combine(_chunkManager.ChunkPath, QueueSettingFileName);
_logger = ObjectContainer.Resolve <ILoggerFactory>().Create(this.GetType().FullName);
}
private void LoadQueue(string topic, int queueId)
{
var queue = new Queue(topic, queueId);
queue.Load();
if (queue.Setting.IsDeleted)
{
return;
}
var key = new QueueKey(topic, queueId);
_queueDict.TryAdd(key, queue);
}
public long GetConsumeOffset(string topic, int queueId, string group)
{
ConcurrentDictionary<QueueKey, long> queueOffsetDict;
if (_groupConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict))
{
long offset;
var key = new QueueKey(topic, queueId);
if (queueOffsetDict.TryGetValue(key, out offset))
{
return offset;
}
}
return -1L;
}
public long GetConsumeOffset(string topic, int queueId, string group)
{
ConcurrentDictionary <QueueKey, long> queueOffsetDict;
if (_groupConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict))
{
long offset;
var key = new QueueKey(topic, queueId);
if (queueOffsetDict.TryGetValue(key, out offset))
{
return(offset);
}
}
return(-1L);
}
public bool TryFetchNextConsumeOffset(string topic, int queueId, string group, out long nextOffset)
{
nextOffset = 0L;
ConcurrentDictionary <QueueKey, long> queueOffsetDict;
if (_groupNextConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict))
{
long offset;
var key = new QueueKey(topic, queueId);
if (queueOffsetDict.TryRemove(key, out offset))
{
nextOffset = offset;
return(true);
}
}
return(false);
}
public long GetMinConsumedOffset(string topic, int queueId)
{
var key = new QueueKey(topic, queueId);
var minOffset = -1L;
foreach (var queueOffsetDict in _groupConsumeOffsetsDict.Values)
{
long offset;
if (queueOffsetDict.TryGetValue(key, out offset))
{
if (minOffset == -1)
{
minOffset = offset;
}
else if (offset < minOffset)
{
minOffset = offset;
}
}
}
return(minOffset);
}
private ConcurrentDictionary <string, ConcurrentDictionary <QueueKey, long> > ConvertDictFrom(ConcurrentDictionary <string, ConcurrentDictionary <string, ConcurrentDictionary <int, long> > > source)
{
var toDict = new ConcurrentDictionary <string, ConcurrentDictionary <QueueKey, long> >();
foreach (var entry1 in source)
{
var key1 = entry1.Key;
var dict = toDict.GetOrAdd(key1, x => new ConcurrentDictionary <QueueKey, long>());
foreach (var entry2 in entry1.Value)
{
var topic = entry2.Key;
foreach (var entry3 in entry2.Value)
{
var queueId = entry3.Key;
var offset = entry3.Value;
var queueKey = new QueueKey(topic, queueId);
dict.TryAdd(queueKey, offset);
}
}
}
return(toDict);
}
public void UpdateConsumeOffset(string topic, int queueId, long offset, string group)
{
var queueOffsetDict = _groupConsumeOffsetsDict.GetOrAdd(group, k =>
{
return new ConcurrentDictionary<QueueKey, long>();
});
var key = new QueueKey(topic, queueId);
queueOffsetDict[key] = offset;
}
private void AddOrUpdateRules(ClusterState cluster)
{
if (_serverDelegationFeature is null)
{
return;
}
// We support multiple destinations referencing the same queue but http.sys only
// allows us to create one handle to the queue, so we keep track of queues two ways.
// 1. We map destination => queue using a ConditionalWeakTable
// This allows us to find the queue to delegate to when processing a request
// 2. We map (queue name + url prefix) => queue using a WeakReference
// This allows us to find the already created queue if more than one destination points to the same queue.
// It also allows us to find the queue by name/url prefix to support reset.
//
// Using weak references means we ensure the queue exist as long as the referencing destinations exist.
// Once all the destinations are gone, GC will eventually finalize the underlying SafeHandle to the http.sys
// queue, which will clean up references in http.sys, allowing us to re-create it again later if needed.
foreach (var destination in cluster.Destinations.Select(kvp => kvp.Value))
{
var queueName = destination.GetHttpSysDelegationQueue();
var urlPrefix = destination.Model?.Config?.Address;
if (queueName is not null && urlPrefix is not null)
{
var queueKey = new QueueKey(queueName, urlPrefix);
if (!_queuesPerDestination.TryGetValue(destination, out var queue) || !queue.Equals(queueKey))
{
if (!_queues.TryGetValue(queueKey, out var queueWeakRef) || !queueWeakRef.TryGetTarget(out queue))
{
// Either the queue hasn't been created or it has been cleaned up.
// Create a new one, and try to add it if someone else didn't beat us to it.
queue = new DelegationQueue(queueName, urlPrefix);
queueWeakRef = new WeakReference <DelegationQueue>(queue);
queueWeakRef = _queues.AddOrUpdate(
queueKey,
(key, newValue) => newValue,
(key, value, newValue) => value.TryGetTarget(out _) ? value : newValue,
queueWeakRef);
queueWeakRef.TryGetTarget(out queue);
}
if (queue is not null)
{
// We call this outside of the above if bock so that if previous
// initialization failed, we will retry it for every new destination added.
var queueState = queue.Initialize(_serverDelegationFeature);
if (!queueState.IsInitialized)
{
Log.QueueInitFailed(
_logger,
destination.DestinationId,
queueName,
urlPrefix,
queueState.InitializationException);
}
_queuesPerDestination.AddOrUpdate(destination, queue);
}
else
{
// This should never happen because we always create a new one above
_queuesPerDestination.Remove(destination);
Log.QueueInitFailed(
_logger,
destination.DestinationId,
queueName,
urlPrefix,
new Exception("Delegation queue is null after adding a new one. This shouldn't happen."));
}
}
}
public long GetMinConsumedOffset(string topic, int queueId)
{
var key = new QueueKey(topic, queueId);
var minOffset = -1L;
foreach (var queueOffsetDict in _groupConsumeOffsetsDict.Values)
{
long offset;
if (queueOffsetDict.TryGetValue(key, out offset))
{
if (minOffset == -1)
{
minOffset = offset;
}
else if (offset < minOffset)
{
minOffset = offset;
}
}
}
return minOffset;
}
public bool IsQueueExist(QueueKey queueKey)
{
return(GetQueue(queueKey) != null);
}
private ConcurrentDictionary<string, ConcurrentDictionary<QueueKey, long>> ConvertDictFrom(ConcurrentDictionary<string, ConcurrentDictionary<string, ConcurrentDictionary<int, long>>> source)
{
var toDict = new ConcurrentDictionary<string, ConcurrentDictionary<QueueKey, long>>();
foreach (var entry1 in source)
{
var key1 = entry1.Key;
var dict = toDict.GetOrAdd(key1, x => new ConcurrentDictionary<QueueKey, long>());
foreach (var entry2 in entry1.Value)
{
var topic = entry2.Key;
foreach (var entry3 in entry2.Value)
{
var queueId = entry3.Key;
var offset = entry3.Value;
var queueKey = new QueueKey(topic, queueId);
dict.TryAdd(queueKey, offset);
}
}
}
return toDict;
}
public bool TryFetchNextConsumeOffset(string topic, int queueId, string group, out long nextOffset)
{
nextOffset = 0L;
ConcurrentDictionary<QueueKey, long> queueOffsetDict;
if (_groupNextConsumeOffsetsDict.TryGetValue(group, out queueOffsetDict))
{
long offset;
var key = new QueueKey(topic, queueId);
if (queueOffsetDict.TryRemove(key, out offset))
{
nextOffset = offset;
return true;
}
}
return false;
}
public void SetConsumeNextOffset(string topic, int queueId, string group, long nextOffset)
{
var queueOffsetDict = _groupNextConsumeOffsetsDict.GetOrAdd(group, k =>
{
return new ConcurrentDictionary<QueueKey, long>();
});
var key = new QueueKey(topic, queueId);
queueOffsetDict[key] = nextOffset;
}
public void DeleteConsumeOffset(QueueKey queueKey)
{
foreach (var dict in _groupConsumeOffsetsDict.Values)
{
var keys = dict.Keys.Where(x => x == queueKey);
foreach (var key in keys)
{
dict.Remove(key);
}
}
PersistConsumeOffsetInfo();
}
public bool IsConsumerExistForQueue(string topic, int queueId)
{
var key = new QueueKey(topic, queueId);
return(_consumerConsumingQueueDict.Values.Any(x => x.Any(y => y == key)));
}
public void DESFunctionRKTest()
{
QueueKey queueKey = new QueueKey(TestData.ExampleKey);
DesMethods.DESFunctionRK(TestData.ExampleKey, queueKey.GetKey(0));
}
protected override void Execute(CodeActivityContext context)
{
// Get the connection string
DBConnection ext = context.GetExtension <DBConnection>();
if (ext == null)
{
throw new InvalidProgramException("No connection string available");
}
// Lookup the operator
UserTasksDataContext dc = new UserTasksDataContext(ext.ConnectionString);
OperatorConfig oc = dc.OperatorConfigs.SingleOrDefault(x => x.OperatorKey == OperatorKey.Get(context));
if (oc == null)
{
oc = new OperatorConfig();
oc.OperatorKey = OperatorKey.Get(context);
oc.UnderEvaluation = false;
oc.Frequency = 5;
oc.NumberSinceLastEval = 0;
dc.OperatorConfigs.InsertOnSubmit(oc);
dc.SubmitChanges();
}
// Determine the Queue info
char[] delimiter = { '|' };
string[] values = QueueKey.Get(context).Split(delimiter, 3);
// Lookup the queue and subqueue
Queue q = dc.Queues.SingleOrDefault(x => x.QueueName == values[0]);
if (q == null)
{
throw new InvalidProgramException("The specified queue (" + values[0] + ") was not found");
}
SubQueue sq = dc.SubQueues.SingleOrDefault(x => x.QueueID == q.QueueID &&
x.SubQueueName == values[1]);
if (sq == null)
{
throw new InvalidProgramException("The specified subqueue (" +
values[0] + " - " +
values[1] + ") was not found");
}
bool bQC = bool.Parse(values[2]);
if (sq.AllowSelection)
{
// Return all the available instances
IEnumerable <QueueInstance> instances;
if (bQC)
{
instances = dc.QueueInstances
.Where(x => x.CurrentSubQueueID == sq.SubQueueID &&
x.QC == bQC &&
(x.AssignedOperatorID == null ||
x.AssignedOperatorID == oc.OperatorConfigID))
.OrderBy(x => x.Priority.Value)
.OrderBy(x => x.CreateDate);
}
else
{
instances = dc.QueueInstances
.Where(x => x.CurrentSubQueueID == sq.SubQueueID &&
x.QC == bQC &&
(x.AssignedOperatorID == null ||
x.AssignedOperatorID == oc.OperatorConfigID))
.OrderBy(x => x.CreateDate);
}
if (instances.Count() > 0)
{
QueueInstance[] qiList = new QueueInstance[instances.Count()];
int i = 0;
foreach (QueueInstance r in instances)
{
qiList[i++] = r;
}
QueueInstanceList.Set(context, qiList);
}
}
else
{
// Return the oldest instance
IEnumerable <QueueInstance> instances;
if (bQC)
{
instances = dc.QueueInstances
.Where(x => x.CurrentSubQueueID == sq.SubQueueID &&
x.QC == bQC &&
(x.AssignedOperatorID == null ||
x.AssignedOperatorID == oc.OperatorConfigID))
.OrderBy(x => x.Priority.Value)
.OrderBy(x => x.CreateDate)
.Take(1);
}
else
{
instances = dc.QueueInstances
.Where(x => x.CurrentSubQueueID == sq.SubQueueID &&
x.QC == bQC &&
(x.AssignedOperatorID == null ||
x.AssignedOperatorID == oc.OperatorConfigID))
.OrderBy(x => (x.Priority.HasValue ? 1 : 0) * x.Priority.Value * (bQC ? 1 : 0))
.OrderBy(x => x.CreateDate)
.Take(1);
}
if (instances.Count() > 0)
{
QueueInstance qi = instances.First <QueueInstance>();
qi.AssignedOperatorID = oc.OperatorConfigID;
dc.SubmitChanges();
QueueInstance[] qiList = new QueueInstance[1];
qiList[0] = qi;
QueueInstanceList.Set(context, qiList);
}
}
}
