public async Task TestStatus() { var checkpointerStore = new Mock <ICheckpointStore>(); checkpointerStore.Setup(c => c.GetCheckpointDataAsync(It.IsAny <string>(), It.IsAny <CancellationToken>())).ReturnsAsync(new CheckpointData(0L)); checkpointerStore.Setup(c => c.SetCheckpointDataAsync(It.IsAny <string>(), new CheckpointData(It.IsAny <long>()), It.IsAny <CancellationToken>())).Returns(TaskEx.Done); ICheckpointer checkpointer = await Checkpointer.CreateAsync("checkpoint.id1", checkpointerStore.Object); var endpoint = new TestEndpoint("endpoint1"); using (var executor = new AsyncEndpointExecutor(endpoint, checkpointer, MaxConfig, new AsyncEndpointExecutorOptions(1))) { await executor.Invoke(Message1, 0, 3600); await Task.Delay(TimeSpan.FromSeconds(2)); await executor.CloseAsync(); Assert.Equal(1, endpoint.N); EndpointExecutorStatus status = executor.Status; Assert.Equal("endpoint1", status.Id); Assert.Equal(0, status.RetryAttempts); Assert.Equal(State.Closed, status.State); Assert.Equal("checkpoint.id1", status.CheckpointerStatus.Id); Assert.Equal(1, status.CheckpointerStatus.Offset); } }
public async Task CloseAsyncWillCallCloseAsyncOfAllFsmsTest() { var endpoint = new TestEndpoint("endpoint1"); endpoint.CanProcess = false; var storingAsyncEndpointExecutor = new StoringAsyncEndpointExecutor( endpoint, new NullCheckpointerFactory(), new EndpointExecutorConfig(TimeSpan.FromHours(1), RetryStrategy.DefaultFixed, TimeSpan.FromHours(1)), new AsyncEndpointExecutorOptions(10, TimeSpan.FromMilliseconds(1)), new TestMessageStore()); await storingAsyncEndpointExecutor.UpdatePriorities(new List <uint>() { 0 }, Option.None <Endpoint>()); var message = GetNewMessages(1, 0).First(); await storingAsyncEndpointExecutor.Invoke(message, 0, 3600); // await Task.Delay(1000); Assert.Equal(0, endpoint.N); Task closeTask = storingAsyncEndpointExecutor.CloseAsync(); Task timeoutTask = Task.Delay(5000); var firstCompletedTask = await Task.WhenAny(closeTask, timeoutTask); Assert.True(firstCompletedTask == closeTask, "storingAsyncEndpointExecutor can't close when processing a message"); }
public async Task TestCheckpoint() { var endpoint1 = new TestEndpoint("id1"); var checkpointer = new Mock <ICheckpointer>(); checkpointer.Setup(c => c.Admit(It.IsAny <IMessage>())).Returns(true); checkpointer.Setup(c => c.CommitAsync(It.IsAny <IMessage[]>(), It.IsAny <IMessage[]>(), It.IsAny <Option <DateTime> >(), It.IsAny <Option <DateTime> >(), It.IsAny <CancellationToken>())).Returns(TaskEx.Done); checkpointer.Setup(c => c.CommitAsync(It.IsAny <IMessage[]>(), It.IsAny <IMessage[]>(), It.IsAny <Option <DateTime> >(), It.IsAny <Option <DateTime> >(), It.IsAny <CancellationToken>())).Returns(TaskEx.Done); IEndpointExecutor executor1 = new SyncEndpointExecutor(endpoint1, checkpointer.Object); await executor1.Invoke(Message1); checkpointer.Verify(c => c.CommitAsync(new [] { Message1 }, new IMessage[0], It.IsAny <Option <DateTime> >(), It.IsAny <Option <DateTime> >(), It.IsAny <CancellationToken>()), Times.Exactly(1)); var endpoint2 = new NullEndpoint("id2"); var checkpointer2 = new Mock <ICheckpointer>(); checkpointer2.Setup(c => c.Admit(It.IsAny <IMessage>())).Returns(false); checkpointer2.Setup(c => c.CommitAsync(It.IsAny <IMessage[]>(), It.IsAny <IMessage[]>(), It.IsAny <Option <DateTime> >(), It.IsAny <Option <DateTime> >(), It.IsAny <CancellationToken>())).Returns(TaskEx.Done); checkpointer2.Setup(c => c.CommitAsync(It.IsAny <IMessage[]>(), It.IsAny <IMessage[]>(), It.IsAny <Option <DateTime> >(), It.IsAny <Option <DateTime> >(), It.IsAny <CancellationToken>())).Returns(TaskEx.Done); IEndpointExecutor executor2 = new SyncEndpointExecutor(endpoint2, checkpointer2.Object); await executor2.Invoke(Message1); checkpointer2.Verify(c => c.CommitAsync(It.IsAny <IMessage[]>(), It.IsAny <IMessage[]>(), It.IsAny <Option <DateTime> >(), It.IsAny <Option <DateTime> >(), It.IsAny <CancellationToken>()), Times.Never); await executor1.CloseAsync(); await executor2.CloseAsync(); }
public async Task SmokeTest() { var endpoint = new TestEndpoint("id"); Checkpointer checkpointer = await Checkpointer.CreateAsync("exec1", NullCheckpointStore.Instance); var executor = new AsyncEndpointExecutor(endpoint, checkpointer, MaxConfig, new AsyncEndpointExecutorOptions(1)); var expected = new List <IMessage> { Message1, Message2, Message3 }; Assert.Equal(State.Idle, executor.Status.State); Assert.Equal(Checkpointer.InvalidOffset, executor.Status.CheckpointerStatus.Offset); Assert.Equal(Checkpointer.InvalidOffset, executor.Status.CheckpointerStatus.Proposed); foreach (IMessage msg in expected) { await executor.Invoke(msg, 0, 3600); } await Task.Delay(TimeSpan.FromSeconds(2)); await executor.CloseAsync(); Assert.Equal(3, endpoint.N); Assert.Equal(expected, endpoint.Processed); Assert.Equal(3, executor.Status.CheckpointerStatus.Offset); Assert.Equal(3, executor.Status.CheckpointerStatus.Proposed); }
public async Task SmokeTest() { var endpoint = new TestEndpoint("id"); Checkpointer checkpointer = await Checkpointer.CreateAsync("exec1", NullCheckpointStore.Instance); var executor = new SyncEndpointExecutor(endpoint, checkpointer); var expected = new List <IMessage> { Message1, Message2, Message3 }; Assert.Equal(State.Idle, executor.Status.State); Assert.Equal(Checkpointer.InvalidOffset, executor.Status.CheckpointerStatus.Offset); Assert.Equal(Checkpointer.InvalidOffset, executor.Status.CheckpointerStatus.Proposed); foreach (IMessage msg in expected) { await executor.Invoke(msg); } await executor.CloseAsync(); Assert.Equal(3, endpoint.N); Assert.Equal(expected, endpoint.Processed); Assert.Equal(3, executor.Status.CheckpointerStatus.Offset); Assert.Equal(3, executor.Status.CheckpointerStatus.Proposed); }
public async Task PumpMessagesTest() { // Arrange const int MessagesCount = 10; const string EndpointId = "endpoint1"; var endpoint = new TestEndpoint(EndpointId); ICheckpointer checkpointer = await Checkpointer.CreateAsync(EndpointId, new CheckpointStore()); var endpointExecutorConfig = new EndpointExecutorConfig(TimeSpan.FromHours(1), RetryStrategy.NoRetry, TimeSpan.FromHours(1)); var asyncEndpointExecutorOptions = new AsyncEndpointExecutorOptions(4, TimeSpan.FromSeconds(2)); var messageStore = new TestMessageStore(); var storingAsyncEndpointExecutor = new StoringAsyncEndpointExecutor(endpoint, checkpointer, endpointExecutorConfig, asyncEndpointExecutorOptions, messageStore); IEnumerable <IMessage> messages = GetNewMessages(MessagesCount, 0); // Act - Send messages to invoke foreach (IMessage message in messages) { await storingAsyncEndpointExecutor.Invoke(message); } await Task.Delay(TimeSpan.FromSeconds(10)); // Assert - Make sure the endpoint received all the messages. Assert.Equal(MessagesCount, endpoint.N); for (int i = 0; i < MessagesCount; i++) { IMessage message = endpoint.Processed[i]; Assert.True(message.Properties.ContainsKey($"key{i}")); Assert.Equal($"value{i}", message.Properties[$"key{i}"]); } }
public async Task TestSetEndpoint() { var endpoint1 = new TestEndpoint("id"); var endpoint2 = new NullEndpoint("id"); var endpoint3 = new TestEndpoint("id1"); ICheckpointer checkpointer = await Checkpointer.CreateAsync("cid", new CheckpointStore()); var endpointExecutorConfig = new EndpointExecutorConfig(TimeSpan.FromHours(1), RetryStrategy.NoRetry, TimeSpan.FromHours(1)); var asyncEndpointExecutorOptions = new AsyncEndpointExecutorOptions(1, TimeSpan.FromSeconds(1)); var messageStore = new TestMessageStore(); var executor = new StoringAsyncEndpointExecutor(endpoint1, checkpointer, endpointExecutorConfig, asyncEndpointExecutorOptions, messageStore); Assert.Equal(endpoint1, executor.Endpoint); await Assert.ThrowsAsync <ArgumentNullException>(() => executor.SetEndpoint(null)); await Assert.ThrowsAsync <ArgumentException>(() => executor.SetEndpoint(endpoint3)); await executor.SetEndpoint(endpoint2); Assert.Equal(endpoint2, executor.Endpoint); await executor.CloseAsync(); await Assert.ThrowsAsync <InvalidOperationException>(() => executor.SetEndpoint(endpoint1)); }
public async Task TestBatchTimeout() { var endpoint1 = new TestEndpoint("id1"); var executor = new AsyncEndpointExecutor(endpoint1, new NullCheckpointer(), MaxConfig, new AsyncEndpointExecutorOptions(100, TimeSpan.FromMilliseconds(50))); await executor.Invoke(Message1, 0, 3600); await executor.Invoke(Message1, 0, 3600); await Task.Delay(TimeSpan.FromMilliseconds(500)); var expected = new List <IMessage> { Message1, Message1 }; Assert.Equal(expected, endpoint1.Processed); await executor.Invoke(Message2, 0, 3600); expected.Add(Message2); await Task.Delay(TimeSpan.FromMilliseconds(500)); Assert.Equal(expected, endpoint1.Processed); await executor.Invoke(Message2, 0, 3600); expected.Add(Message2); await Task.Delay(TimeSpan.FromMilliseconds(500)); Assert.Equal(expected, endpoint1.Processed); await executor.CloseAsync(); }
public async Task TestClose() { var endpoint = new TestEndpoint("id"); IEndpointExecutor executor = await Factory.CreateAsync(endpoint, null); Task running = executor.Invoke(Default, 0, 3600); await executor.CloseAsync(); await running; Assert.True(running.IsCompleted); // ensure this doesn't throw await executor.CloseAsync(); }
public async Task TestSetEndpoint() { var endpoint1 = new TestEndpoint("id1"); var endpoint2 = new TestEndpoint("id1"); var endpoint3 = new TestEndpoint("id3"); IEndpointExecutor executor = await Factory.CreateAsync(endpoint1); Assert.Equal(new List <IMessage>(), endpoint1.Processed); Assert.Equal(new List <IMessage>(), endpoint2.Processed); Assert.Equal(new List <IMessage>(), endpoint3.Processed); await Assert.ThrowsAsync <ArgumentException>(() => executor.SetEndpoint(endpoint3)); await executor.Invoke(Message1); await executor.Invoke(Message1); Assert.Equal(new List <IMessage> { Message1, Message1 }, endpoint1.Processed); Assert.Equal(new List <IMessage>(), endpoint2.Processed); Assert.Equal(new List <IMessage>(), endpoint3.Processed); await executor.SetEndpoint(endpoint2); Assert.Equal(new List <IMessage> { Message1, Message1 }, endpoint1.Processed); Assert.Equal(new List <IMessage>(), endpoint2.Processed); Assert.Equal(new List <IMessage>(), endpoint3.Processed); await executor.Invoke(Message2); await executor.Invoke(Message3); Assert.Equal(new List <IMessage> { Message1, Message1 }, endpoint1.Processed); Assert.Equal(new List <IMessage> { Message2, Message3 }, endpoint2.Processed); Assert.Equal(new List <IMessage>(), endpoint3.Processed); await executor.CloseAsync(); }
public async Task TestSetEndpoint() { var endpoint1 = new TestEndpoint("id"); var endpoint2 = new NullEndpoint("id"); var endpoint3 = new TestEndpoint("id1"); IEndpointExecutor executor = await Factory.CreateAsync(endpoint1, null); Assert.Equal(endpoint1, executor.Endpoint); await Assert.ThrowsAsync <ArgumentNullException>(() => executor.SetEndpoint(null, null)); await Assert.ThrowsAsync <ArgumentException>(() => executor.SetEndpoint(endpoint3, null)); await executor.SetEndpoint(endpoint2, null); Assert.Equal(endpoint2, executor.Endpoint); await executor.CloseAsync(); await Assert.ThrowsAsync <InvalidOperationException>(() => executor.SetEndpoint(endpoint1, null)); }
public async Task TestClose() { var endpoint = new TestEndpoint("id"); IEndpointExecutor executor = await Factory.CreateAsync(endpoint); Task running = executor.Invoke(Default); await executor.CloseAsync(); await running; Assert.True(running.IsCompleted); // ensure this doesn't throw await executor.CloseAsync(); await Assert.ThrowsAsync <InvalidOperationException>(() => executor.Invoke(Default)); await Assert.ThrowsAsync <InvalidOperationException>(() => executor.SetEndpoint(endpoint)); }
public async Task TestSetEndpoint() { var endpoint1 = new TestEndpoint("id"); var endpoint2 = new NullEndpoint("id"); var endpoint3 = new TestEndpoint("id1"); var priorities = new List <uint>() { 100, 101, 102, 0, 1, 2 }; var checkpointerFactory = new NullCheckpointerFactory(); var endpointExecutorConfig = new EndpointExecutorConfig(TimeSpan.FromHours(1), RetryStrategy.NoRetry, TimeSpan.FromHours(1)); var asyncEndpointExecutorOptions = new AsyncEndpointExecutorOptions(1, TimeSpan.FromSeconds(1)); var messageStore = new TestMessageStore(); var executor = new StoringAsyncEndpointExecutor(endpoint1, checkpointerFactory, endpointExecutorConfig, asyncEndpointExecutorOptions, messageStore); await executor.UpdatePriorities(priorities, Option.None <Endpoint>()); Assert.Equal(endpoint1, executor.Endpoint); await Assert.ThrowsAsync <ArgumentNullException>(() => executor.SetEndpoint(null, new List <uint>() { 0 })); await Assert.ThrowsAsync <ArgumentNullException>(() => executor.SetEndpoint(endpoint1, null)); await Assert.ThrowsAsync <ArgumentException>(() => executor.SetEndpoint(endpoint3, new List <uint>() { 0 })); await executor.SetEndpoint(endpoint2, new List <uint>() { 103, 104, 105, 0, 1, 2, 2 }); Assert.Equal(endpoint2, executor.Endpoint); await executor.CloseAsync(); await Assert.ThrowsAsync <InvalidOperationException>(() => executor.SetEndpoint(endpoint1, new List <uint>() { 0 })); }
public async Task PumpMessagesWithLargeIncomingBatchTest() { // Arrange const int MessagesCount = 200; const string EndpointId = "endpoint1"; const int RoutingPumpBatchSize = 10; var endpoint = new TestEndpoint(EndpointId); var priorities = new List <uint>() { 0, 1, 2, 100, 101, 102 }; var checkpointerFactory = new NullCheckpointerFactory(); var endpointExecutorConfig = new EndpointExecutorConfig(TimeSpan.FromHours(1), RetryStrategy.NoRetry, TimeSpan.FromHours(1)); var asyncEndpointExecutorOptions = new AsyncEndpointExecutorOptions(RoutingPumpBatchSize, TimeSpan.FromMilliseconds(1)); var messageStore = new TestMessageStore(); var storingAsyncEndpointExecutor = new StoringAsyncEndpointExecutor(endpoint, checkpointerFactory, endpointExecutorConfig, asyncEndpointExecutorOptions, messageStore); await storingAsyncEndpointExecutor.UpdatePriorities(priorities, Option.None <Endpoint>()); IEnumerable <IMessage> messages = GetNewMessages(MessagesCount, 0); // Act - Send messages to invoke foreach (IMessage message in messages) { await storingAsyncEndpointExecutor.Invoke(message, 0, 3600); } await Task.Delay(TimeSpan.FromSeconds(10)); // Assert - Make sure the endpoint received all the messages. Assert.Equal(MessagesCount, endpoint.N); for (int i = 0; i < MessagesCount; i++) { IMessage message = endpoint.Processed[i]; Assert.True(message.Properties.ContainsKey($"key{i}")); Assert.Equal($"value{i}", message.Properties[$"key{i}"]); } }
public async Task SmokeTest() { var endpoint = new TestEndpoint("id1", "name1", "hub1"); IProcessor processor = endpoint.CreateProcessor(); Assert.Equal(endpoint, processor.Endpoint); Assert.True(processor.ErrorDetectionStrategy.IsTransient(new Exception())); Assert.Equal(new List <IMessage>(), endpoint.Processed); Assert.Equal("name1", endpoint.Name); Assert.Equal("hub1", endpoint.IotHubName); ISinkResult <IMessage> result = await processor.ProcessAsync(new IMessage[0], CancellationToken.None); Assert.Equal(new IMessage[0], result.Succeeded); Assert.Equal(new List <IMessage>(), endpoint.Processed); IMessage[] messages = new[] { Message1, Message2, Message3 }; ISinkResult <IMessage> result2 = await processor.ProcessAsync(messages, CancellationToken.None); Assert.Equal(new[] { Message1, Message2, Message3 }, result2.Succeeded); Assert.Equal(new List <IMessage> { Message1, Message2, Message3 }, endpoint.Processed); }
public Processor(TestEndpoint endpoint) { this.endpoint = endpoint; }
public async Task MessagePrioritiesTest() { // Arrange const string EndpointId = "endpoint1"; const uint HighPri = 0; const uint NormalPri = 5; const uint LowPri = 10; var endpoint = new TestEndpoint(EndpointId); var priorities = new List <uint>() { HighPri, NormalPri, LowPri }; var checkpointerFactory = new NullCheckpointerFactory(); var endpointExecutorConfig = new EndpointExecutorConfig(TimeSpan.FromHours(1), RetryStrategy.DefaultFixed, TimeSpan.FromHours(1)); var asyncEndpointExecutorOptions = new AsyncEndpointExecutorOptions(4, TimeSpan.FromSeconds(2)); var messageStore = new TestMessageStore(); var storingAsyncEndpointExecutor = new StoringAsyncEndpointExecutor(endpoint, checkpointerFactory, endpointExecutorConfig, asyncEndpointExecutorOptions, messageStore); await storingAsyncEndpointExecutor.UpdatePriorities(priorities, Option.None <Endpoint>()); var normalPriMsg1 = new Message(TelemetryMessageSource.Instance, new byte[] { 1 }, new Dictionary <string, string> { { "normalPriority", string.Empty } }, 0L); var normalPriMsg2 = new Message(TelemetryMessageSource.Instance, new byte[] { 2 }, new Dictionary <string, string> { { "normalPriority", string.Empty } }, 1L); var normalPriMsg3 = new Message(TelemetryMessageSource.Instance, new byte[] { 3 }, new Dictionary <string, string> { { "normalPriority", string.Empty } }, 2L); var lowPriMsg1 = new Message(TelemetryMessageSource.Instance, new byte[] { 4 }, new Dictionary <string, string> { { "lowPriority", string.Empty } }, 3L); var lowPriMsg2 = new Message(TelemetryMessageSource.Instance, new byte[] { 5 }, new Dictionary <string, string> { { "lowPriority", string.Empty } }, 4L); var highPriMsg1 = new Message(TelemetryMessageSource.Instance, new byte[] { 6 }, new Dictionary <string, string> { { "highPriority", string.Empty } }, 5L); var normalPriMsg4 = new Message(TelemetryMessageSource.Instance, new byte[] { 7 }, new Dictionary <string, string> { { "normalPriority", string.Empty } }, 6L); var highPriMsg2 = new Message(TelemetryMessageSource.Instance, new byte[] { 8 }, new Dictionary <string, string> { { "highPriority", string.Empty } }, 7L); const int HighPriCount = 2; const int NormalPriCount = 4; const int LowPriCount = 2; // Disable the endpoint so messages are stuck in queue endpoint.CanProcess = false; // Send normal priority messages await storingAsyncEndpointExecutor.Invoke(normalPriMsg1, NormalPri, 3600); await storingAsyncEndpointExecutor.Invoke(normalPriMsg2, NormalPri, 3600); await storingAsyncEndpointExecutor.Invoke(normalPriMsg3, NormalPri, 3600); // Send low priority messages await storingAsyncEndpointExecutor.Invoke(lowPriMsg1, LowPri, 3600); await storingAsyncEndpointExecutor.Invoke(lowPriMsg2, LowPri, 3600); // Send the remaining messages mixed priority await storingAsyncEndpointExecutor.Invoke(highPriMsg1, HighPri, 3600); await storingAsyncEndpointExecutor.Invoke(normalPriMsg4, NormalPri, 3600); await storingAsyncEndpointExecutor.Invoke(highPriMsg2, HighPri, 3600); // Message store should have the messages in the corresponding queues var highPriQueue = messageStore.GetReceivedMessagesForEndpoint($"{endpoint.Id}_Pri{HighPri}"); Assert.Equal(2, highPriQueue.Count); Assert.Contains(highPriMsg1, highPriQueue); Assert.Contains(highPriMsg2, highPriQueue); var normalPriQueue = messageStore.GetReceivedMessagesForEndpoint($"{endpoint.Id}_Pri{NormalPri}"); Assert.Equal(4, normalPriQueue.Count); Assert.Contains(normalPriMsg1, normalPriQueue); Assert.Contains(normalPriMsg2, normalPriQueue); Assert.Contains(normalPriMsg3, normalPriQueue); Assert.Contains(normalPriMsg4, normalPriQueue); var lowPriQueue = messageStore.GetReceivedMessagesForEndpoint($"{endpoint.Id}_Pri{LowPri}"); Assert.Equal(2, lowPriQueue.Count); Assert.Contains(lowPriMsg1, lowPriQueue); Assert.Contains(lowPriMsg2, lowPriQueue); // Re-enable the endpoint and let the queues drain endpoint.CanProcess = true; await Task.Delay(TimeSpan.FromSeconds(10)); // Assert - Make sure the endpoint received all the messages // in the right priority order: // - HighPri messages should finish processing before others // - NormalPri messages should finish processing before LowPri Assert.Equal(8, endpoint.Processed.Count()); int highPriMessagesProcessed = 0; int normalPriMessagesProcessed = 0; int lowPriMessagesProcessed = 0; for (int i = 0; i < endpoint.Processed.Count(); i++) { IMessage message = endpoint.Processed[i]; if (message.Properties.ContainsKey($"highPriority")) { if (++highPriMessagesProcessed == HighPriCount) { // Found all the high-pri messages, // normal and low pri at this point // must not have completed yet Assert.True(normalPriMessagesProcessed < NormalPriCount); Assert.True(lowPriMessagesProcessed < LowPriCount); } } else if (message.Properties.ContainsKey($"normalPriority")) { if (++normalPriMessagesProcessed == NormalPriCount) { // Found all the normal-pri messages, // low pri messages at this point must // not have completed yet Assert.True(lowPriMessagesProcessed < LowPriCount); // High pri messages should have completed Assert.True(highPriMessagesProcessed == HighPriCount); } } else if (message.Properties.ContainsKey($"lowPriority")) { if (++lowPriMessagesProcessed == LowPriCount) { // Found all the low-pri messages, // high-pri and normal-pri should also // have completed before this Assert.True(highPriMessagesProcessed == HighPriCount); Assert.True(normalPriMessagesProcessed == NormalPriCount); } } else { // Bad test setup Assert.True(false, "Bad test setup, processed a message with unexpected priority"); } } }