async Task SendMessagesPump() { try { Events.StartSendMessagesPump(this); IMessageIterator iterator = this.messageStore.GetMessageIterator(this.Endpoint.Id, this.checkpointer.Offset + 1); while (!this.cts.IsCancellationRequested) { try { this.hasMessagesInQueue.WaitOne(this.options.BatchTimeout); IMessage[] messages = (await iterator.GetNext(this.options.BatchSize)).ToArray(); await this.ProcessMessages(messages); Events.SendMessagesSuccess(this, messages); Metrics.DrainedCountIncrement(this.Endpoint.Id, messages.Length); // If store has no messages, then reset the hasMessagesInQueue flag. if (messages.Length == 0) { this.hasMessagesInQueue.Reset(); } } catch (Exception ex) { Events.SendMessagesError(this, ex); // Swallow exception and keep trying. } } } catch (Exception ex) { Events.SendMessagesPumpFailure(this, ex); } }
public async Task CleanupTestMessageStoreMissingValue(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 5); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 10; i++) { IMessage input = this.GetMessage(i); await messageStore.Add("module1", input, 0); if (i == 1) { // remove a message to simulate messageStore not in sync with sequential store await result.inMemoryDbStore.Remove(input.SystemProperties[SystemProperties.EdgeMessageId].ToBytes()); } } await Task.Delay(TimeSpan.FromSeconds(30)); IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Empty(batch); } }
public StoreMessagesProvider(IMessageIterator iterator, TimeSpan timeout, int batchSize) { this.iterator = iterator; this.batchSize = batchSize; this.timeout = timeout; this.messagesList = new List <IMessage>(this.batchSize); this.populateTask = this.PopulatePump(); }
public async Task CleanupTestTimeoutMultipleTTLs(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore inMemoryDbStore)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 10); using (IMessageStore messageStore = result.messageStore) { var messageIdsAlive = new List <string>(); var messageIdsExpired = new List <string>(); for (int i = 0; i < 200; i++) { IMessage input = this.GetMessage(i); string edgeMessageId = input.SystemProperties[SystemProperties.EdgeMessageId]; uint timeToLiveSecs = 1000; if (i % 2 == 0) { messageIdsExpired.Add(edgeMessageId); timeToLiveSecs = 10; } else { messageIdsAlive.Add(edgeMessageId); } IMessage updatedMessage = await messageStore.Add("module1", input, timeToLiveSecs); CompareUpdatedMessageWithOffset(input, i, updatedMessage); } for (int i = 0; i < messageIdsExpired.Count; i++) { if (checkEntireQueueOnCleanup || i == 0) { int retryAttempts = 0; while (await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes())) { Assert.True(retryAttempts < 10, "Test is taking too long and is considered a failure."); retryAttempts++; await Task.Delay(TimeSpan.FromSeconds(10)); } Assert.False(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes())); } else { Assert.True(await result.inMemoryDbStore.Contains(messageIdsExpired[i].ToBytes())); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(200); Assert.Equal(100, batch.Count()); foreach (string edgeMessageId in messageIdsAlive) { Assert.True(await result.inMemoryDbStore.Contains(edgeMessageId.ToBytes())); } } }
public async Task BasicTest(long initialCheckpointOffset, bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset, checkEntireQueueOnCleanup); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 10000; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 0); CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 0); CompareUpdatedMessageWithOffset(input, initialCheckpointOffset + 1 + i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); Assert.NotNull(module1Iterator); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); Assert.NotNull(module2Iterator); for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[SystemProperties.MessageId]); } } for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList2.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[SystemProperties.MessageId]); } } } }
public async Task CleanupTestCheckpointed(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20); ICheckpointStore checkpointStore = result.checkpointStore; using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 0); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 0); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None); await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None); await Task.Delay(TimeSpan.FromSeconds(100)); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Empty(batch); } }
public async Task BasicTest(long initialCheckpointOffset) { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(initialCheckpointOffset); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 10000; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(initialCheckpointOffset + 1 + i / 2, offset); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); Assert.NotNull(module1Iterator); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); Assert.NotNull(module2Iterator); for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2).ToString(), batchItemsAsList.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]); } } for (int i = 0; i < 5; i++) { IEnumerable <IMessage> batch = await module2Iterator.GetNext(1000); IEnumerable <IMessage> batchItemsAsList2 = batch as IList <IMessage> ?? batch.ToList(); Assert.Equal(1000, batchItemsAsList2.Count()); for (int j = 0; j < 1000; j++) { Assert.Equal((((i * 1000) + j) * 2 + 1).ToString(), batchItemsAsList2.ElementAt(j).SystemProperties[Core.SystemProperties.MessageId]); } } } }
public async Task CleanupTestCheckpointed() { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20); ICheckpointStore checkpointStore = result.checkpointStore; using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(i / 2, offset); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await checkpointStore.SetCheckpointDataAsync("module1", new CheckpointData(198), CancellationToken.None); await checkpointStore.SetCheckpointDataAsync("module2", new CheckpointData(199), CancellationToken.None); await Task.Delay(TimeSpan.FromSeconds(100)); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(0, batch.Count()); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Equal(0, batch.Count()); } }
public async Task CleanupTestTimeout() { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Empty(batch); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); } }
async Task SendMessagesPump() { // TODO: 6099918 - Update StoringAsyncEndpointExecutor message pump to process messages by priority try { Events.StartSendMessagesPump(this); IMessageIterator iterator = this.messageStore.GetMessageIterator(this.Endpoint.Id, this.checkpointer.Offset + 1); int batchSize = this.options.BatchSize * this.Endpoint.FanOutFactor; var storeMessagesProvider = new StoreMessagesProvider(iterator, batchSize); while (!this.cts.IsCancellationRequested) { try { await this.hasMessagesInQueue.WaitAsync(this.options.BatchTimeout); IMessage[] messages = await storeMessagesProvider.GetMessages(); if (messages.Length > 0) { await this.ProcessMessages(messages); Events.SendMessagesSuccess(this, messages); MetricsV0.DrainedCountIncrement(this.Endpoint.Id, messages.Length); } else { // If store has no messages, then reset the hasMessagesInQueue flag. this.hasMessagesInQueue.Reset(); } } catch (Exception ex) { Events.SendMessagesError(this, ex); // Swallow exception and keep trying. } } } catch (Exception ex) { Events.SendMessagesPumpFailure(this, ex); } }
public StoreMessagesProvider(IMessageIterator iterator, int batchSize) { this.iterator = iterator; this.batchSize = batchSize; this.getMessagesTask = Task.Run(this.GetMessagesFromStore); }
public async Task MessageStoreAddRemoveEndpointTest(bool checkEntireQueueOnCleanup) { // Arrange var dbStoreProvider = new InMemoryDbStoreProvider(); IStoreProvider storeProvider = new StoreProvider(dbStoreProvider); ICheckpointStore checkpointStore = CheckpointStore.Create(storeProvider); IMessageStore messageStore = new MessageStore(storeProvider, checkpointStore, TimeSpan.FromHours(1), checkEntireQueueOnCleanup, 1800); // Act await messageStore.AddEndpoint("module1"); for (int i = 0; i < 10; i++) { await messageStore.Add("module1", this.GetMessage(i), 0); } // Assert IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); Assert.NotNull(module1Iterator); IEnumerable <IMessage> batch = await module1Iterator.GetNext(1000); List <IMessage> batchItemsAsList = batch.ToList(); Assert.Equal(10, batchItemsAsList.Count); for (int i = 0; i < 10; i++) { Assert.Equal($"{i}", batchItemsAsList.ElementAt(i).SystemProperties[SystemProperties.MessageId]); } // Remove await messageStore.RemoveEndpoint("module1"); // Assert await Assert.ThrowsAsync <InvalidOperationException>(() => messageStore.Add("module1", this.GetMessage(0), 0)); Assert.Throws <InvalidOperationException>(() => messageStore.GetMessageIterator("module1")); // Act await messageStore.AddEndpoint("module1"); for (int i = 20; i < 30; i++) { await messageStore.Add("module1", this.GetMessage(i), 0); } // Assert module1Iterator = messageStore.GetMessageIterator("module1"); Assert.NotNull(module1Iterator); batch = await module1Iterator.GetNext(1000); batchItemsAsList = batch.ToList(); Assert.Equal(10, batchItemsAsList.Count); for (int i = 20; i < 30; i++) { Assert.Equal($"{i}", batchItemsAsList.ElementAt(i - 20).SystemProperties[SystemProperties.MessageId]); } }
public async Task CleanupTestTimeoutUpdateIndividualMessageTimeToLive(bool checkEntireQueueOnCleanup) { (IMessageStore messageStore, ICheckpointStore checkpointStore, InMemoryDbStore _)result = await this.GetMessageStore(checkEntireQueueOnCleanup, 20); result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20)); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 20); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 20); CompareUpdatedMessageWithOffset(input, i / 2, updatedMessage); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Empty(batch); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); // By setting the global TTL for the MessageStore to 20, the CleanupProcessor will run every 10 seconds // But it won't clean up any messages, since the individual messages are set to have TTL of 2000 seconds result.messageStore.SetTimeToLive(TimeSpan.FromSeconds(20)); await Task.Delay(TimeSpan.FromSeconds(50)); for (int i = 0; i < 200; i++) { if (i % 2 == 0) { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module1", input, 2000); CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage); } else { IMessage input = this.GetMessage(i); IMessage updatedMessage = await messageStore.Add("module2", input, 50); CompareUpdatedMessageWithOffset(input, 100 + i / 2, updatedMessage); } } module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); module1Iterator = messageStore.GetMessageIterator("module1", 100); batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); module2Iterator = messageStore.GetMessageIterator("module2", 100); batch = await module2Iterator.GetNext(100); Assert.Empty(batch); } }
public async Task CleanupTestTimeoutWithReadd() { (IMessageStore messageStore, ICheckpointStore checkpointStore)result = await this.GetMessageStore(20); using (IMessageStore messageStore = result.messageStore) { for (int i = 0; i < 200; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(i / 2, offset); } } IMessageIterator module1Iterator = messageStore.GetMessageIterator("module1"); IEnumerable <IMessage> batch = await module1Iterator.GetNext(100); Assert.Equal(100, batch.Count()); IMessageIterator module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(100, batch.Count()); await Task.Delay(TimeSpan.FromSeconds(100)); for (int i = 200; i < 250; i++) { if (i % 2 == 0) { long offset = await messageStore.Add("module1", this.GetMessage(i)); Assert.Equal(i / 2, offset); } else { long offset = await messageStore.Add("module2", this.GetMessage(i)); Assert.Equal(i / 2, offset); } } module1Iterator = messageStore.GetMessageIterator("module1"); batch = await module1Iterator.GetNext(100); Assert.Equal(25, batch.Count()); module2Iterator = messageStore.GetMessageIterator("module2"); batch = await module2Iterator.GetNext(100); Assert.Equal(25, batch.Count()); } }