public async Task PullCount(int count)
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start();
HorseQueue queue = server.Server.FindQueue("pull-a");
for (int i = 0; i < 25; i++)
{
await queue.Push("Hello, World");
}
HorseClient client = new HorseClient();
await client.ConnectAsync("hmq://localhost:" + port);
HorseResult joined = await client.Queues.Subscribe("pull-a", true);
Assert.Equal(HorseResultCode.Ok, joined.Code);
PullRequest request = new PullRequest
{
Queue = "pull-a",
Count = count
};
PullContainer container = await client.Queues.Pull(request);
Assert.Equal(count, container.ReceivedCount);
Assert.Equal(PullProcess.Completed, container.Status);
}
public async Task RequestAcknowledge(bool queueAckIsActive)
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start(300, 300);
HorseQueue queue = server.Server.FindQueue("rr-a");
Assert.NotNull(queue);
queue.Options.AcknowledgeTimeout = TimeSpan.FromSeconds(3);
queue.Options.Acknowledge = queueAckIsActive ? QueueAckDecision.JustRequest : QueueAckDecision.None;
HorseClient producer = new HorseClient();
await producer.ConnectAsync("hmq://localhost:" + port);
Assert.True(producer.IsConnected);
HorseClient consumer = new HorseClient();
consumer.AutoAcknowledge = true;
consumer.ResponseTimeout = TimeSpan.FromSeconds(4);
consumer.ClientId = "consumer";
await consumer.ConnectAsync("hmq://localhost:" + port);
Assert.True(consumer.IsConnected);
HorseResult joined = await consumer.Queues.Subscribe("rr-a", true);
Assert.Equal(HorseResultCode.Ok, joined.Code);
HorseResult ack = await producer.Queues.Push("rr-a", "Hello, World!", true);
Assert.Equal(queueAckIsActive, ack.Code == HorseResultCode.Ok);
}
public async Task FillData()
{
List <byte[]> items = new List <byte[]>();
for (int i = 0; i < 10; i++)
{
items.Add(Encoding.UTF8.GetBytes("No #" + i));
}
TestHorseMq server = new TestHorseMq();
await server.Initialize();
server.Start(300, 300);
HorseQueue queue = server.Server.FindQueue("push-a");
Assert.NotNull(queue);
QueueFiller filler = new QueueFiller(queue);
filler.FillData(items, false, true);
filler.FillData(items, false, false);
await Task.Delay(500);
Assert.NotEmpty(queue.PriorityMessages);
Assert.NotEmpty(queue.Messages);
}
public async Task MultipleQueue()
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start(300, 300);
Router router = new Router(server.Server, "router", RouteMethod.Distribute);
router.AddBinding(new QueueBinding("qbind-1", "push-a", 0, BindingInteraction.None));
router.AddBinding(new QueueBinding("qbind-2", "push-a-cc", 0, BindingInteraction.None));
server.Server.AddRouter(router);
HorseClient producer = new HorseClient();
await producer.ConnectAsync("hmq://localhost:" + port);
Assert.True(producer.IsConnected);
HorseResult result = await producer.Routers.Publish("router", "Hello, World!", true);
Assert.Equal(HorseResultCode.Ok, result.Code);
HorseQueue queue1 = server.Server.FindQueue("push-a");
HorseQueue queue2 = server.Server.FindQueue("push-a-cc");
Assert.Equal(1, queue1.MessageCount());
Assert.Equal(1, queue2.MessageCount());
}
public async Task CreateWithProperties()
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start();
HorseClient client = new HorseClient();
await client.ConnectAsync("hmq://localhost:" + port);
Assert.True(client.IsConnected);
HorseResult created = await client.Queues.Create("queue-test", o =>
{
o.AcknowledgeTimeout = 33000;
o.Status = MessagingQueueStatus.Pull;
});
Assert.Equal(HorseResultCode.Ok, created.Code);
HorseQueue queue = server.Server.FindQueue("queue-test");
Assert.NotNull(queue);
Assert.Equal(TimeSpan.FromSeconds(33), queue.Options.AcknowledgeTimeout);
Assert.Equal(QueueStatus.Pull, queue.Status);
}
public async Task UnsubscribeToQueue()
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start();
HorseClient client = new HorseClient();
await client.ConnectAsync("hmq://localhost:" + port);
HorseResult joined = await client.Queues.Subscribe("broadcast-a", true);
Assert.Equal(HorseResultCode.Ok, joined.Code);
HorseResult left = await client.Queues.Unsubscribe("broadcast-a", true);
Assert.Equal(HorseResultCode.Ok, left.Code);
HorseQueue queue = server.Server.Queues.FirstOrDefault();
Assert.NotNull(queue);
List <QueueClient> clients = queue.ClientsClone;
Assert.Empty(clients);
}
internal static IQueueState Create(HorseQueue queue, QueueStatus status)
{
switch (status)
{
case QueueStatus.Broadcast:
return(new BroadcastQueueState(queue));
case QueueStatus.Push:
return(new PushQueueState(queue));
case QueueStatus.RoundRobin:
return(new RoundRobinQueueState(queue));
case QueueStatus.Pull:
return(new PullQueueState(queue));
case QueueStatus.Cache:
return(new CacheQueueState(queue));
case QueueStatus.Paused:
return(new PauseQueueState(queue));
case QueueStatus.Stopped:
return(new StopQueueState(queue));
default:
return(new StopQueueState(queue));
}
}
public async Task Handle(MqClient client, HorseMessage message, bool fromNode)
{
try
{
HorseQueue queue = _server.FindQueue(message.Target);
//if auto creation active, try to create queue
if (queue == null && _server.Options.AutoQueueCreation)
{
QueueOptions options = QueueOptions.CloneFrom(_server.Options);
queue = await _server.CreateQueue(message.Target, options, message, _server.DeliveryHandlerFactory, true, true);
}
if (queue == null)
{
if (!string.IsNullOrEmpty(message.MessageId))
{
await client.SendAsync(message.CreateResponse(HorseResultCode.NotFound));
}
return;
}
await HandlePullRequest(client, message, queue);
}
catch (Exception e)
{
_server.SendError("PULL_REQUEST", e, $"QueueName:{message.Target}");
}
}
public async Task Handle(MqClient client, HorseMessage message, bool fromNode)
{
HorseQueue queue = await FindQueue(client, message.Target, message);
if (queue == null)
{
return;
}
//if there is at least one cc header
//we need to create a clone of the message
//clone does not have cc headers but others
HorseMessage clone = null;
List <string> ccList = null;
List <KeyValuePair <string, string> > additionalHeaders = null;
if (message.HasHeader && message.FindHeader(HorseHeaders.CC) != null)
{
additionalHeaders = message.Headers.Where(x => !x.Key.Equals(HorseHeaders.CC, StringComparison.InvariantCultureIgnoreCase)).ToList();
ccList = new List <string>(message.Headers.Where(x => x.Key.Equals(HorseHeaders.CC, StringComparison.InvariantCultureIgnoreCase)).Select(x => x.Value));
clone = message.Clone(false, true, _server.MessageIdGenerator.Create(), additionalHeaders);
}
await HandlePush(client, message, queue, true);
//if there are cc headers, we will push the message to other queues
if (clone != null)
{
await PushOtherQueues(client, clone, ccList, additionalHeaders);
}
}
/// <summary>
/// Gets queue.
/// If it's not cached, finds and caches it before returns.
/// </summary>
/// <returns></returns>
private async Task <HorseQueue> GetQueue(HorseMessage message)
{
if (_targetQueue != null)
{
return(_targetQueue);
}
string queueName = message.FindHeader(HorseHeaders.QUEUE_NAME);
if (queueName == null)
{
return(null);
}
HorseQueue queue = Router.Server.FindQueue(queueName);
if (queue != null)
{
_targetQueue = queue;
return(_targetQueue);
}
_targetQueue = await Router.Server.CreateQueue(queueName, Router.Server.Options, message, null, true, true);
return(_targetQueue);
}
/// <summary>
/// Sends the message to binding receivers
/// </summary>
public override async Task <bool> Send(MqClient sender, HorseMessage message)
{
try
{
HorseQueue queue = await GetQueue(message);
if (queue == null)
{
return(false);
}
string messageId = Interaction == BindingInteraction.None
? Router.Server.MessageIdGenerator.Create()
: message.MessageId;
HorseMessage msg = message.Clone(true, true, messageId);
msg.Type = MessageType.QueueMessage;
msg.SetTarget(Target);
msg.WaitResponse = Interaction == BindingInteraction.Response;
QueueMessage queueMessage = new QueueMessage(msg);
queueMessage.Source = sender;
PushResult result = await queue.Push(queueMessage, sender);
return(result == PushResult.Success);
}
catch (Exception e)
{
Router.Server.SendError("BINDING_SEND", e, $"Type:AutoQueue, Binding:{Name}");
return(false);
}
}
/// <summary>
/// Removes a queue from a server
/// </summary>
private async Task RemoveQueue(MqClient client, HorseMessage message)
{
HorseQueue queue = _server.FindQueue(message.Target);
if (queue == null)
{
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.NotFound));
}
return;
}
foreach (IAdminAuthorization authorization in _server.AdminAuthorizations)
{
bool grant = await authorization.CanRemoveQueue(client, queue);
if (!grant)
{
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.Unauthorized));
}
return;
}
}
await _server.RemoveQueue(queue);
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.Ok));
}
}
private async Task <bool> SendRoundRobin(HorseMessage message)
{
Interlocked.Increment(ref _roundRobinIndex);
int i = _roundRobinIndex;
if (i >= _queues.Length)
{
_roundRobinIndex = 0;
i = 0;
}
if (_queues.Length == 0)
{
return(false);
}
HorseQueue queue = Router.Server.FindQueue(message.Target);
if (queue == null)
{
if (!Router.Server.Options.AutoQueueCreation)
{
return(false);
}
queue = await Router.Server.CreateQueue(message.Target, Router.Server.Options, message, Router.Server.DeliveryHandlerFactory, true, true);
}
QueueMessage queueMessage = new QueueMessage(message);
queue.AddMessage(queueMessage);
return(true);
}
public async Task FillJson()
{
List <QueueMessageA> items = new List <QueueMessageA>();
for (int i = 0; i < 10; i++)
{
items.Add(new QueueMessageA("No #" + i));
}
TestHorseMq server = new TestHorseMq();
await server.Initialize();
server.Start(300, 300);
HorseQueue route = server.Server.FindQueue("broadcast-a");
HorseQueue push = server.Server.FindQueue("push-a");
Assert.NotNull(route);
Assert.NotNull(push);
QueueFiller fillerRouteA = new QueueFiller(route);
QueueFiller fillerPushA = new QueueFiller(push);
fillerRouteA.FillJson(items, false, false);
fillerPushA.FillJson(items, false, false);
await Task.Delay(500);
Assert.NotEmpty(route.Messages);
Assert.NotEmpty(push.Messages);
}
/// <inheritdoc />
public virtual Task <Decision> AcknowledgeTimedOut(HorseQueue queue, MessageDelivery delivery)
{
DeliveryAcknowledgeDecision ack = ProducerAckDecision == ProducerAckDecision.AfterConsumerAckReceived
? DeliveryAcknowledgeDecision.Negative
: DeliveryAcknowledgeDecision.None;
return(Task.FromResult(new Decision(true, false, AckTimeoutPutBack, ack)));
}
/// <inheritdoc />
public virtual Task <Decision> ExceptionThrown(HorseQueue queue, QueueMessage message, Exception exception)
{
if (ConfigurationFactory.Builder.ErrorAction != null)
{
ConfigurationFactory.Builder.ErrorAction(queue, message, exception);
}
return(Task.FromResult(Decision.JustAllow()));
}
public async Task RoundRobin()
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start(300, 300);
Router router = new Router(server.Server, "router", RouteMethod.RoundRobin);
router.AddBinding(new QueueBinding("qbind-1", "push-a", 5, BindingInteraction.None));
router.AddBinding(new QueueBinding("qbind-2", "push-a-cc", 10, BindingInteraction.None));
router.AddBinding(new DirectBinding("dbind-1", "client-1", 20, BindingInteraction.None));
router.AddBinding(new DirectBinding("dbind-2", "client-2", 0, BindingInteraction.None));
server.Server.AddRouter(router);
HorseClient producer = new HorseClient();
await producer.ConnectAsync("hmq://localhost:" + port);
Assert.True(producer.IsConnected);
HorseClient client1 = new HorseClient();
client1.ClientId = "client-1";
await client1.ConnectAsync("hmq://localhost:" + port);
Assert.True(client1.IsConnected);
HorseClient client2 = new HorseClient();
client2.ClientId = "client-2";
await client2.ConnectAsync("hmq://localhost:" + port);
Assert.True(client2.IsConnected);
int client1Received = 0;
int client2Received = 0;
client1.MessageReceived += (c, m) => client1Received++;
client2.MessageReceived += (c, m) => client2Received++;
for (int i = 0; i < 5; i++)
{
HorseResult result = await producer.Routers.Publish("router", "Hello, World!", true);
Assert.Equal(HorseResultCode.Ok, result.Code);
}
await Task.Delay(500);
HorseQueue queue1 = server.Server.FindQueue("push-a");
HorseQueue queue2 = server.Server.FindQueue("push-a-cc");
Assert.Equal(1, queue1.MessageCount());
Assert.Equal(1, queue2.MessageCount());
Assert.Equal(1, client2Received);
Assert.Equal(2, client1Received);
}
/// <summary>
/// Decision: Allow.
/// If AcknowledgeWhen is AfterSent, acknowledge is sent to producer.
/// </summary>
public async Task <Decision> EndSend(HorseQueue queue, QueueMessage message)
{
if (_producerAck == AcknowledgeWhen.AfterSent)
{
return(await Task.FromResult(new Decision(true, false, PutBackDecision.No, DeliveryAcknowledgeDecision.Always)));
}
return(await Task.FromResult(new Decision(true, false)));
}
public async Task Delayed()
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start(300, 300);
HorseQueue queue = server.Server.FindQueue("push-a");
queue.Options.PutBackDelay = 2000;
queue.Options.Acknowledge = QueueAckDecision.WaitForAcknowledge;
server.PutBack = PutBackDecision.Start;
HorseClient producer = new HorseClient();
await producer.ConnectAsync("hmq://localhost:" + port);
Assert.True(producer.IsConnected);
await producer.Queues.Push("push-a", "First", false);
await Task.Delay(100);
await producer.Queues.Push("push-a", "Second", false);
await Task.Delay(200);
Assert.Equal(2, queue.MessageCount());
int receivedMessages = 0;
HorseClient consumer = new HorseClient();
consumer.ClientId = "consumer";
await consumer.ConnectAsync("hmq://localhost:" + port);
Assert.True(consumer.IsConnected);
consumer.MessageReceived += async(c, m) =>
{
receivedMessages++;
await consumer.Queues.Unsubscribe("push-a", true);
await Task.Delay(1000);
await consumer.SendNegativeAck(m);
};
HorseResult joined = await consumer.Queues.Subscribe("push-a", true);
Assert.Equal(HorseResultCode.Ok, joined.Code);
await Task.Delay(1500);
Assert.Equal(1, receivedMessages);
Assert.Equal(1, queue.MessageCount());
await Task.Delay(3000);
Assert.Equal(2, queue.MessageCount());
}
/// <summary>
/// Decision: Allow.
/// If AcknowledgeWhen is AfterReceived, acknowledge is sent to producer.
/// </summary>
public async Task <Decision> ReceivedFromProducer(HorseQueue queue, QueueMessage message, MqClient sender)
{
if (_producerAck == AcknowledgeWhen.AfterReceived)
{
return(await Task.FromResult(new Decision(true, false, PutBackDecision.No, DeliveryAcknowledgeDecision.Always)));
}
return(await Task.FromResult(new Decision(true, false)));
}
/// <summary>
/// Finds and subscribes to the queue and sends response
/// </summary>
private async Task Subscribe(MqClient client, HorseMessage message)
{
HorseQueue queue = _server.FindQueue(message.Target);
//if auto creation active, try to create queue
if (queue == null && _server.Options.AutoQueueCreation)
{
QueueOptions options = QueueOptions.CloneFrom(_server.Options);
queue = await _server.CreateQueue(message.Target, options, message, _server.DeliveryHandlerFactory, true, true);
}
if (queue == null)
{
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.NotFound));
}
return;
}
QueueClient found = queue.FindClient(client.UniqueId);
if (found != null)
{
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.Ok));
}
return;
}
QueueSubscriptionResult result = await queue.AddClient(client);
if (message.WaitResponse)
{
switch (result)
{
case QueueSubscriptionResult.Success:
await client.SendAsync(message.CreateResponse(HorseResultCode.Ok));
break;
case QueueSubscriptionResult.Unauthorized:
await client.SendAsync(message.CreateResponse(HorseResultCode.Unauthorized));
break;
case QueueSubscriptionResult.Full:
await client.SendAsync(message.CreateResponse(HorseResultCode.LimitExceeded));
break;
}
}
}
/// <summary>
/// Removes a queue from the server
/// </summary>
public async Task RemoveQueue(string name)
{
HorseQueue queue = FindQueue(name);
if (queue == null)
{
return;
}
await RemoveQueue(queue);
}
public Task <Decision> ReceivedFromProducer(HorseQueue queue, QueueMessage message, MqClient sender)
{
_mq.OnReceived++;
if (_mq.SendAcknowledgeFromMQ)
{
return(Task.FromResult(new Decision(true, false, PutBackDecision.No, DeliveryAcknowledgeDecision.Always)));
}
return(Task.FromResult(new Decision(true, false)));
}
public Task <Decision> AcknowledgeReceived(HorseQueue queue, HorseMessage acknowledgeMessage, MessageDelivery delivery, bool success)
{
_mq.OnAcknowledge++;
if (!success)
{
return(Task.FromResult(new Decision(true, false, _mq.PutBack, DeliveryAcknowledgeDecision.Always)));
}
return(Task.FromResult(new Decision(true, false, PutBackDecision.No, DeliveryAcknowledgeDecision.Always)));
}
/// <summary>
/// Clears messages in a queue
/// </summary>
private async Task ClearMessages(MqClient client, HorseMessage message)
{
HorseQueue queue = _server.FindQueue(message.Target);
if (queue == null)
{
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.NotFound));
}
return;
}
string prio = message.FindHeader(HorseHeaders.PRIORITY_MESSAGES);
string msgs = message.FindHeader(HorseHeaders.MESSAGES);
bool clearPrio = !string.IsNullOrEmpty(prio) && prio.Equals("yes", StringComparison.InvariantCultureIgnoreCase);
bool clearMsgs = !string.IsNullOrEmpty(msgs) && msgs.Equals("yes", StringComparison.InvariantCultureIgnoreCase);
foreach (IAdminAuthorization authorization in _server.AdminAuthorizations)
{
bool grant = await authorization.CanClearQueueMessages(client, queue, clearPrio, clearMsgs);
if (!grant)
{
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.Unauthorized));
}
return;
}
}
if (clearPrio && clearMsgs)
{
queue.ClearAllMessages();
}
else if (clearPrio)
{
queue.ClearHighPriorityMessages();
}
else if (clearMsgs)
{
queue.ClearRegularMessages();
}
//if creation successful, sends response
if (message.WaitResponse)
{
await client.SendAsync(message.CreateResponse(HorseResultCode.Ok));
}
}
/// <summary>
/// Triggers queue created, updated or deleted events
/// </summary>
public void Trigger(HorseQueue queue, string node = null)
{
base.Trigger(new QueueEvent
{
Name = queue.Name,
Topic = queue.Topic,
Status = queue.Status.ToString(),
Messages = queue.MessageCount(),
PriorityMessages = queue.PriorityMessageCount(),
Node = node
});
}
/// <summary>
/// Removes queue from configurations
/// </summary>
public void Remove(HorseQueue queue)
{
lock (_optionsLock)
{
QueueConfiguration queueConfiguration = Config.Queues.FirstOrDefault(x => x.Name == queue.Name);
if (queueConfiguration != null)
{
Config.Queues.Remove(queueConfiguration);
}
}
}
public async Task RequestAcknowledge()
{
TestHorseMq server = new TestHorseMq();
await server.Initialize();
int port = server.Start(300, 300);
HorseQueue queue = server.Server.FindQueue("pull-a");
Assert.NotNull(queue);
queue.Options.Acknowledge = QueueAckDecision.JustRequest;
queue.Options.AcknowledgeTimeout = TimeSpan.FromSeconds(15);
HorseClient consumer = new HorseClient();
consumer.AutoAcknowledge = true;
consumer.ClientId = "consumer";
await consumer.ConnectAsync("hmq://localhost:" + port);
Assert.True(consumer.IsConnected);
bool msgReceived = false;
consumer.MessageReceived += (c, m) => msgReceived = true;
HorseResult joined = await consumer.Queues.Subscribe("pull-a", true);
Assert.Equal(HorseResultCode.Ok, joined.Code);
HorseClient producer = new HorseClient();
producer.ResponseTimeout = TimeSpan.FromSeconds(15);
await producer.ConnectAsync("hmq://localhost:" + port);
Assert.True(producer.IsConnected);
Task <HorseResult> taskAck = producer.Queues.Push("pull-a", "Hello, World!", true);
await Task.Delay(500);
Assert.False(taskAck.IsCompleted);
Assert.False(msgReceived);
Assert.Single(queue.Messages);
consumer.PullTimeout = TimeSpan.FromDays(1);
PullContainer pull = await consumer.Queues.Pull(PullRequest.Single("pull-a"));
Assert.Equal(PullProcess.Completed, pull.Status);
Assert.Equal(1, pull.ReceivedCount);
Assert.NotEmpty(pull.ReceivedMessages);
}
/// <summary>
/// Creates new persistent queue
/// </summary>
/// <param name="mq">Horse MQ Server</param>
/// <param name="queueName">Queue name</param>
/// <param name="options">Queue Options</param>
/// <param name="factory">Delivery handler instance creator factory</param>
/// <returns></returns>
public static async Task <HorseQueue> CreatePersistentQueue(this HorseMq mq,
string queueName,
QueueOptions options,
Func <DatabaseOptions, IPersistentDeliveryHandler> factory)
{
HorseQueue queue = await CreateQueue(mq, queueName, options, factory);
IPersistentDeliveryHandler deliveryHandler = (IPersistentDeliveryHandler)queue.DeliveryHandler;
ConfigurationFactory.Manager.Add(queue, deliveryHandler.DbFilename);
ConfigurationFactory.Manager.Save();
return(queue);
}
/// <summary>
/// Creates new DatabaseOptions using predefined options
/// </summary>
internal DatabaseOptions CreateOptions(HorseQueue queue)
{
return(new DatabaseOptions
{
Filename = GenerateQueueFilename(queue),
AutoFlush = _autoFlush,
AutoShrink = _autoShrink,
FlushInterval = _flushInteval,
InstantFlush = _instantFlush,
ShrinkInterval = _shrinkInteval,
CreateBackupOnShrink = _createBackup
});
}
