public async Task BasicTest(long initialCheckpointOffset, bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset, checkEntireQueueOnCleanup); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 10000; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 0); CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 0); CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); Assert.NotNull(module1Iterator); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); Assert.NotNull(module2Iterator); for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[SystemProperties.MessageId]); } } for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList2.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[SystemProperties.MessageId]); } } } }
public async Task CleanupTestCheckpointed(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20); ICheckpointStore checkpointStore = result.checkpointStore; using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 0); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 0); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None); await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None); await Task.Delay(TimeSpan.FromSeconds(100)); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Empty(batch); } }
public async Task BasicTest(long initialCheckpointOffset) { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 10000; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); Assert.NotNull(module1Iterator); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); Assert.NotNull(module2Iterator); for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]); } } for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList2.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]); } } } }
public async Task CleanupTestCheckpointed() { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20); ICheckpointStore checkpointStore = result.checkpointStore; using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(i / 2, offset); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None); await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None); await Task.Delay(TimeSpan.FromSeconds(100)); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(0, batch.Count()); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Equal(0, batch.Count()); } }
private void SeedChatMessages(User user) { if (_chatRepository.Any()) { var chats = _chatRepository.GetAll(); foreach (var chat in chats) { var message = GenerateRandomMessage(user); message.ChatId = chat.Id; _messageStore.Add(message); } } }
public async Task CleanupTestMessageStoreMissingValue(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 5); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 10; i++) { IMessage input = this.GetMessage(i); await messageStore.Add("module1", input, 0); if (i == 1) { // remove a message to simulate messageStore not in sync with sequential store await result.inMemoryDbStore.Remove(input.SystemProperties[SystemProperties.EdgeMessageId].ToBytes()); } } await Task.Delay(TimeSpan.FromSeconds(30)); IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Empty(batch); } }
public async Task CleanupTestTimeout() { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Empty(batch); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); } }
public async Task CleanupTestTimeoutMultipleTTLs(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 10); using (IMessageStore messageStore = result.messageStore) { var messageIdsAlive = new List <string>(); var messageIdsExpired = new List <string>(); for (int i = 0; i < 200; i++) { IMessage input = this.GetMessage(i); string edgeMessageId = input.SystemProperties[SystemProperties.EdgeMessageId]; uint timeToLiveSecs = 1000; if (i % 2 == 0) { messageIdsExpired.Add(edgeMessageId); timeToLiveSecs = 10; } else { messageIdsAlive.Add(edgeMessageId); } IMessage updatedMessage = await messageStore.Add("module1", input, timeToLiveSecs); CompareUpdatedMessageWithOffset(input, i, updatedMessage); } for (int i = 0; i < messageIdsExpired.Count; i++) { if (checkEntireQueueOnCleanup || i == 0) { int retryAttempts = 0; while (await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes())) { Assert.True(retryAttempts < 10, "Test is taking too long and is considered a failure."); retryAttempts++; await Task.Delay(TimeSpan.FromSeconds(10)); } Assert.False(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes())); } else { Assert.True(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes())); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(200); Assert.Equal(100, batch.Count()); foreach (string edgeMessageId in messageIdsAlive) { Assert.True(await result.inMemoryDbStore.Contains(edgeMessageId.ToBytes())); } } }
/// <summary> /// 接收消息。 /// </summary> public void Receive(IEnumerable <TMessage> messages) { var datas = messages.Select(message => new MessageData { MessageId = message.Id, MessageType = _messageTypeName, Body = _serializer.Serialize(message), DeliveryDate = message.Timestamp }); _messageStore.Add(datas); messages.ForEach(AddToQueue); }
public void OnMessage(Message message) { ThrowIfDisposed(); _logger.LogInformation($"Topic {Name} received message with id: {message.Id}"); // create TopicMessage from message var queueMessage = message.ToTopicMessage(Name); // persist the message _messageStore.Add(queueMessage); // add the message to queue chan _queueChannel.Writer.TryWrite(queueMessage.Id); }
public Task <Documents.Message> AddMessageToChatAsync(Guid chatId, string content, Models.User user) { RequireUserIsChatMember(user.Id, chatId); var message = new Documents.Message { Id = Guid.NewGuid(), Content = content, Created = DateTime.Now, ChatId = chatId, UserId = user.Id, Username = user.Username }; _messageStore.Add(message); return(Task.FromResult(message)); }
public async Task CleanupTestTimeoutUpdateIndividualMessageTimeToLive(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20); result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20)); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 20); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 20); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Empty(batch); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); // By setting the global TTL for the MessageStore to 20, the CleanupProcessor will run every 10 seconds // But it won't clean up any messages, since the individual messages are set to have TTL of 2000 seconds result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20)); await Task.Delay(TimeSpan.FromSeconds(50)); for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 2000); CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 50); CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage); } } module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); module1Iterator = messageStore.GetMessageIterator("module1", 100); batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); module2Iterator = messageStore.GetMessageIterator("module2", 100); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); } }
public async Task CleanupTestTimeoutWithReadd() { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(i / 2, offset); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); for (int i = 200; i < 250; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(i / 2, offset); } } module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Equal(25, batch.Count()); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(25, batch.Count()); } }