async Task SendMessagesPump()
{
try
{
Events.StartSendMessagesPump(this);
IMessageIterator iterator = this.messageStore.GetMessageIterator(this.Endpoint.Id, this.checkpointer.Offset + 1);
while (!this.cts.IsCancellationRequested)
{
try
{
this.hasMessagesInQueue.WaitOne(this.options.BatchTimeout);
IMessage[] messages = (await iterator.GetNext(this.options.BatchSize)).ToArray();
await this.ProcessMessages(messages);
Events.SendMessagesSuccess(this, messages);
Metrics.DrainedCountIncrement(this.Endpoint.Id, messages.Length);
// If store has no messages, then reset the hasMessagesInQueue flag.
if (messages.Length == 0)
{
this.hasMessagesInQueue.Reset();
}
}
catch (Exception ex)
{
Events.SendMessagesError(this, ex);
// Swallow exception and keep trying.
}
}
}
catch (Exception ex)
{
Events.SendMessagesPumpFailure(this, ex);
}
}
public async Task CleanupTestMessageStoreMissingValue(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 5);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 10; i++)
{
IMessage input = this.GetMessage(i);
await messageStore.Add("module1", input, 0);
if (i == 1)
{
// remove a message to simulate messageStore not in sync with sequential store
await result.inMemoryDbStore.Remove(input.SystemProperties[SystemProperties.EdgeMessageId].ToBytes());
}
}
await Task.Delay(TimeSpan.FromSeconds(30));
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
}
}
public StoreMessagesProvider(IMessageIterator iterator, TimeSpan timeout, int batchSize)
{
this.iterator = iterator;
this.batchSize = batchSize;
this.timeout = timeout;
this.messagesList = new List <IMessage>(this.batchSize);
this.populateTask = this.PopulatePump();
}
public async Task CleanupTestTimeoutMultipleTTLs(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 10);
using (IMessageStore messageStore = result.messageStore)
{
var messageIdsAlive = new List <string>();
var messageIdsExpired = new List <string>();
for (int i = 0; i < 200; i++)
{
IMessage input = this.GetMessage(i);
string edgeMessageId = input.SystemProperties[SystemProperties.EdgeMessageId];
uint timeToLiveSecs = 1000;
if (i % 2 == 0)
{
messageIdsExpired.Add(edgeMessageId);
timeToLiveSecs = 10;
}
else
{
messageIdsAlive.Add(edgeMessageId);
}
IMessage updatedMessage = await messageStore.Add("module1", input, timeToLiveSecs);
CompareUpdatedMessageWithOffset(input, i, updatedMessage);
}
for (int i = 0; i < messageIdsExpired.Count; i++)
{
if (checkEntireQueueOnCleanup || i == 0)
{
int retryAttempts = 0;
while (await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes()))
{
Assert.True(retryAttempts < 10, "Test is taking too long and is considered a failure.");
retryAttempts++;
await Task.Delay(TimeSpan.FromSeconds(10));
}
Assert.False(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes()));
}
else
{
Assert.True(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes()));
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(200);
Assert.Equal(100, batch.Count());
foreach (string edgeMessageId in messageIdsAlive)
{
Assert.True(await result.inMemoryDbStore.Contains(edgeMessageId.ToBytes()));
}
}
}
public async Task BasicTest(long initialCheckpointOffset, bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset, checkEntireQueueOnCleanup);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 10000; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 0);
CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 0);
CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
Assert.NotNull(module1Iterator);
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
Assert.NotNull(module2Iterator);
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[SystemProperties.MessageId]);
}
}
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList2.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[SystemProperties.MessageId]);
}
}
}
}
public async Task CleanupTestCheckpointed(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20);
ICheckpointStore checkpointStore = result.checkpointStore;
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 0);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 0);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None);
await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(100));
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
}
}
public async Task BasicTest(long initialCheckpointOffset)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 10000; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
Assert.NotNull(module1Iterator);
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
Assert.NotNull(module2Iterator);
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]);
}
}
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList2.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]);
}
}
}
}
public async Task CleanupTestCheckpointed()
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20);
ICheckpointStore checkpointStore = result.checkpointStore;
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None);
await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(100));
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(0, batch.Count());
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Equal(0, batch.Count());
}
}
public async Task CleanupTestTimeout()
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
}
}
async Task SendMessagesPump()
{
// TODO: 6099918 - Update StoringAsyncEndpointExecutor message pump to process messages by priority
try
{
Events.StartSendMessagesPump(this);
IMessageIterator iterator = this.messageStore.GetMessageIterator(this.Endpoint.Id, this.checkpointer.Offset + 1);
int batchSize = this.options.BatchSize * this.Endpoint.FanOutFactor;
var storeMessagesProvider = new StoreMessagesProvider(iterator, batchSize);
while (!this.cts.IsCancellationRequested)
{
try
{
await this.hasMessagesInQueue.WaitAsync(this.options.BatchTimeout);
IMessage[] messages = await storeMessagesProvider.GetMessages();
if (messages.Length > 0)
{
await this.ProcessMessages(messages);
Events.SendMessagesSuccess(this, messages);
MetricsV0.DrainedCountIncrement(this.Endpoint.Id, messages.Length);
}
else
{
// If store has no messages, then reset the hasMessagesInQueue flag.
this.hasMessagesInQueue.Reset();
}
}
catch (Exception ex)
{
Events.SendMessagesError(this, ex);
// Swallow exception and keep trying.
}
}
}
catch (Exception ex)
{
Events.SendMessagesPumpFailure(this, ex);
}
}
public StoreMessagesProvider(IMessageIterator iterator, int batchSize)
{
this.iterator = iterator;
this.batchSize = batchSize;
this.getMessagesTask = Task.Run(this.GetMessagesFromStore);
}
public async Task MessageStoreAddRemoveEndpointTest(bool checkEntireQueueOnCleanup)
{
// Arrange
var dbStoreProvider = new InMemoryDbStoreProvider();
IStoreProvider storeProvider = new StoreProvider(dbStoreProvider);
ICheckpointStore checkpointStore = CheckpointStore.Create(storeProvider);
IMessageStore messageStore = new MessageStore(storeProvider, checkpointStore, TimeSpan.FromHours(1), checkEntireQueueOnCleanup, 1800);
// Act
await messageStore.AddEndpoint("module1");
for (int i = 0; i < 10; i++)
{
await messageStore.Add("module1", this.GetMessage(i), 0);
}
// Assert
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
Assert.NotNull(module1Iterator);
IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000);
List <IMessage> batchItemsAsList = batch.ToList();
Assert.Equal(10, batchItemsAsList.Count);
for (int i = 0; i < 10; i++)
{
Assert.Equal($"{i}", batchItemsAsList.ElementAt(i).SystemProperties[SystemProperties.MessageId]);
}
// Remove
await messageStore.RemoveEndpoint("module1");
// Assert
await Assert.ThrowsAsync <InvalidOperationException>(() => messageStore.Add("module1", this.GetMessage(0), 0));
Assert.Throws <InvalidOperationException>(() => messageStore.GetMessageIterator("module1"));
// Act
await messageStore.AddEndpoint("module1");
for (int i = 20; i < 30; i++)
{
await messageStore.Add("module1", this.GetMessage(i), 0);
}
// Assert
module1Iterator = messageStore.GetMessageIterator("module1");
Assert.NotNull(module1Iterator);
batch = await module1Iterator.GetNext(1000);
batchItemsAsList = batch.ToList();
Assert.Equal(10, batchItemsAsList.Count);
for (int i = 20; i < 30; i++)
{
Assert.Equal($"{i}", batchItemsAsList.ElementAt(i - 20).SystemProperties[SystemProperties.MessageId]);
}
}
public async Task CleanupTestTimeoutUpdateIndividualMessageTimeToLive(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20);
result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20));
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 20);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 20);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
// By setting the global TTL for the MessageStore to 20, the CleanupProcessor will run every 10 seconds
// But it won't clean up any messages, since the individual messages are set to have TTL of 2000 seconds
result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20));
await Task.Delay(TimeSpan.FromSeconds(50));
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 2000);
CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 50);
CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage);
}
}
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
module1Iterator = messageStore.GetMessageIterator("module1", 100);
batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
module2Iterator = messageStore.GetMessageIterator("module2", 100);
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
}
}
public async Task CleanupTestTimeoutWithReadd()
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
for (int i = 200; i < 250; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
}
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Equal(25, batch.Count());
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(25, batch.Count());
}
}
