public async Task BasicTest(long initialCheckpointOffset, bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset, checkEntireQueueOnCleanup);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 10000; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 0);
CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 0);
CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
Assert.NotNull(module1Iterator);
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
Assert.NotNull(module2Iterator);
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[SystemProperties.MessageId]);
}
}
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList2.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[SystemProperties.MessageId]);
}
}
}
}
public async Task CleanupTestCheckpointed(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20);
ICheckpointStore checkpointStore = result.checkpointStore;
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 0);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 0);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None);
await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(100));
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
}
}
public async Task BasicTest(long initialCheckpointOffset)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 10000; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
Assert.NotNull(module1Iterator);
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
Assert.NotNull(module2Iterator);
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]);
}
}
for (int i = 0; i < 5; i++)
{
IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000);
IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList();
Assert.Equal(1000, batchItemsAsList2.Count());
for (int j = 0; j < 1000; j++)
{
Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]);
}
}
}
}
public async Task CleanupTestCheckpointed()
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20);
ICheckpointStore checkpointStore = result.checkpointStore;
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None);
await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(100));
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(0, batch.Count());
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Equal(0, batch.Count());
}
}
private void SeedChatMessages(User user)
{
if (_chatRepository.Any())
{
var chats = _chatRepository.GetAll();
foreach (var chat in chats)
{
var message = GenerateRandomMessage(user);
message.ChatId = chat.Id;
_messageStore.Add(message);
}
}
}
public async Task CleanupTestMessageStoreMissingValue(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 5);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 10; i++)
{
IMessage input = this.GetMessage(i);
await messageStore.Add("module1", input, 0);
if (i == 1)
{
// remove a message to simulate messageStore not in sync with sequential store
await result.inMemoryDbStore.Remove(input.SystemProperties[SystemProperties.EdgeMessageId].ToBytes());
}
}
await Task.Delay(TimeSpan.FromSeconds(30));
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
}
}
public async Task CleanupTestTimeout()
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
}
}
public async Task CleanupTestTimeoutMultipleTTLs(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 10);
using (IMessageStore messageStore = result.messageStore)
{
var messageIdsAlive = new List <string>();
var messageIdsExpired = new List <string>();
for (int i = 0; i < 200; i++)
{
IMessage input = this.GetMessage(i);
string edgeMessageId = input.SystemProperties[SystemProperties.EdgeMessageId];
uint timeToLiveSecs = 1000;
if (i % 2 == 0)
{
messageIdsExpired.Add(edgeMessageId);
timeToLiveSecs = 10;
}
else
{
messageIdsAlive.Add(edgeMessageId);
}
IMessage updatedMessage = await messageStore.Add("module1", input, timeToLiveSecs);
CompareUpdatedMessageWithOffset(input, i, updatedMessage);
}
for (int i = 0; i < messageIdsExpired.Count; i++)
{
if (checkEntireQueueOnCleanup || i == 0)
{
int retryAttempts = 0;
while (await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes()))
{
Assert.True(retryAttempts < 10, "Test is taking too long and is considered a failure.");
retryAttempts++;
await Task.Delay(TimeSpan.FromSeconds(10));
}
Assert.False(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes()));
}
else
{
Assert.True(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes()));
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(200);
Assert.Equal(100, batch.Count());
foreach (string edgeMessageId in messageIdsAlive)
{
Assert.True(await result.inMemoryDbStore.Contains(edgeMessageId.ToBytes()));
}
}
}
/// <summary>
/// 接收消息。
/// </summary>
public void Receive(IEnumerable <TMessage> messages)
{
var datas = messages.Select(message => new MessageData {
MessageId = message.Id,
MessageType = _messageTypeName,
Body = _serializer.Serialize(message),
DeliveryDate = message.Timestamp
});
_messageStore.Add(datas);
messages.ForEach(AddToQueue);
}
public void OnMessage(Message message)
{
ThrowIfDisposed();
_logger.LogInformation($"Topic {Name} received message with id: {message.Id}");
// create TopicMessage from message
var queueMessage = message.ToTopicMessage(Name);
// persist the message
_messageStore.Add(queueMessage);
// add the message to queue chan
_queueChannel.Writer.TryWrite(queueMessage.Id);
}
public Task <Documents.Message> AddMessageToChatAsync(Guid chatId, string content, Models.User user)
{
RequireUserIsChatMember(user.Id, chatId);
var message = new Documents.Message
{
Id = Guid.NewGuid(),
Content = content,
Created = DateTime.Now,
ChatId = chatId,
UserId = user.Id,
Username = user.Username
};
_messageStore.Add(message);
return(Task.FromResult(message));
}
public async Task CleanupTestTimeoutUpdateIndividualMessageTimeToLive(bool checkEntireQueueOnCleanup)
{
(IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20);
result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20));
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 20);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 20);
CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Empty(batch);
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
// By setting the global TTL for the MessageStore to 20, the CleanupProcessor will run every 10 seconds
// But it won't clean up any messages, since the individual messages are set to have TTL of 2000 seconds
result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20));
await Task.Delay(TimeSpan.FromSeconds(50));
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module1", input, 2000);
CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage);
}
else
{
IMessage input = this.GetMessage(i);
IMessage updatedMessage = await messageStore.Add("module2", input, 50);
CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage);
}
}
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
module1Iterator = messageStore.GetMessageIterator("module1", 100);
batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
module2Iterator = messageStore.GetMessageIterator("module2", 100);
batch = await module2Iterator.GetNext(100);
Assert.Empty(batch);
}
}
public async Task CleanupTestTimeoutWithReadd()
{
(IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20);
using (IMessageStore messageStore = result.messageStore)
{
for (int i = 0; i < 200; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
}
IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1");
IEnumerable <IMessage> batch = await module1Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(100, batch.Count());
await Task.Delay(TimeSpan.FromSeconds(100));
for (int i = 200; i < 250; i++)
{
if (i % 2 == 0)
{
long offset = await messageStore.Add("module1", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
else
{
long offset = await messageStore.Add("module2", this.GetMessage(i));
Assert.Equal(i / 2, offset);
}
}
module1Iterator = messageStore.GetMessageIterator("module1");
batch = await module1Iterator.GetNext(100);
Assert.Equal(25, batch.Count());
module2Iterator = messageStore.GetMessageIterator("module2");
batch = await module2Iterator.GetNext(100);
Assert.Equal(25, batch.Count());
}
}
