public async Task ProcessAsync_SendThrowsNonRetryable_Test()
{
var throwingDeviceProxy = ThrowingDeviceProxyBuilder
.Create()
.WithThrow <Exception>()
.Build();
var messageProcessor = this.CreateMessageProcessor(throwingDeviceProxy);
var message1 = GetMessage();
var message2 = GetMessage();
ISinkResult <IRoutingMessage> result = await messageProcessor.ProcessAsync(message1, CancellationToken.None);
Assert.NotNull(result);
Assert.Empty(result.Succeeded);
Assert.Empty(result.Failed);
Assert.Equal(1, result.InvalidDetailsList.Count);
Assert.False(result.SendFailureDetails.HasValue);
ISinkResult <IRoutingMessage> resultBatch = await messageProcessor.ProcessAsync(new[] { message1, message2 }, CancellationToken.None);
Assert.NotNull(resultBatch);
Assert.Empty(resultBatch.Succeeded);
Assert.Empty(resultBatch.Failed);
Assert.Equal(2, resultBatch.InvalidDetailsList.Count);
Assert.False(resultBatch.SendFailureDetails.HasValue);
}
public static void SendFailure(EndpointExecutorFsm fsm, ISinkResult <IMessage> result, Stopwatch stopwatch)
{
long latencyInMs = stopwatch.ElapsedMilliseconds;
if (!Routing.PerfCounter.LogExternalWriteLatency(fsm.Endpoint.IotHubName, fsm.Endpoint.Name, fsm.Endpoint.Type, false, latencyInMs, out string error))
{
Log.LogError((int)EventIds.CounterFailure, "[LogExternalWriteLatencyCounterFailed] {0}", error);
}
SendFailureDetails failureDetails = result.SendFailureDetails.GetOrElse(DefaultSendFailureDetails);
foreach (InvalidDetails <IMessage> invalidDetails in result.InvalidDetailsList)
{
Routing.UserAnalyticsLogger.LogInvalidMessage(fsm.Endpoint.IotHubName, invalidDetails.Item, invalidDetails.FailureKind);
}
Log.LogWarning(
(int)EventIds.SendFailure,
failureDetails.RawException,
"[SendFailure] Sending failed. SuccessfulSize: {0}, FailedSize: {1}, InvalidSize: {2}, {3}",
result.Succeeded.Count,
result.Failed.Count,
result.InvalidDetailsList.Count,
GetContextString(fsm));
LogUnhealthyEndpointOpMonError(fsm, failureDetails.FailureKind);
}
public async Task ModuleMessageProcessor_ProcessAsyncTest_InactiveDeviceProxy()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
var deviceProxy = new Mock <IDeviceProxy>();
deviceProxy.Setup(d => d.SendMessageAsync(It.IsAny <IMessage>(), It.IsAny <string>())).Returns(Task.CompletedTask);
deviceProxy.Setup(d => d.IsActive).Returns(false);
var routingMessage = Mock.Of <IRoutingMessage>();
string moduleId = "device1/module1";
string moduleEndpointAddress = "in1";
IReadOnlyDictionary <DeviceSubscription, bool> deviceSubscriptions = new ReadOnlyDictionary <DeviceSubscription, bool>(
new Dictionary <DeviceSubscription, bool>
{
[DeviceSubscription.ModuleMessages] = true
});
var connectionManager = new Mock <IConnectionManager>();
connectionManager.Setup(c => c.GetDeviceConnection(It.IsAny <string>())).Returns(Option.Some(deviceProxy.Object));
connectionManager.Setup(c => c.GetSubscriptions(It.IsAny <string>())).Returns(Option.Some(deviceSubscriptions));
var moduleEndpoint = new ModuleEndpoint($"{moduleId}/{moduleEndpointAddress}", moduleId, moduleEndpointAddress, connectionManager.Object, routingMessageConverter);
IProcessor moduleMessageProcessor = moduleEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> sinkResult = await moduleMessageProcessor.ProcessAsync(routingMessage, CancellationToken.None);
Assert.True(sinkResult.Failed.Contains(routingMessage));
Assert.Equal(FailureKind.None, sinkResult.SendFailureDetails.Map(x => x.FailureKind).GetOrElse(FailureKind.None));
}
static async Task EnterDeadCheckpointingAsync(EndpointExecutorFsm thisPtr)
{
ICommand next;
try
{
Preconditions.CheckNotNull(thisPtr.currentCheckpointCommand);
using (var cts = new CancellationTokenSource(thisPtr.config.Timeout))
{
ISinkResult <IMessage> result = thisPtr.currentCheckpointCommand.Result;
Events.Checkpoint(thisPtr, result);
await thisPtr.Checkpointer.CommitAsync(result.Succeeded, EmptyMessages, thisPtr.lastFailedRevivalTime, thisPtr.unhealthySince, cts.Token);
Events.CheckpointSuccess(thisPtr, result);
}
next = Commands.DeadSucceed;
Events.DeadSuccess(thisPtr, thisPtr.currentCheckpointCommand.Result.Succeeded);
}
catch (Exception ex)
{
Events.CheckpointFailure(thisPtr, ex);
next = thisPtr.config.ThrowOnDead
? (ICommand)Commands.Throw(ex)
: Commands.DeadSucceed;
}
await RunInternalAsync(thisPtr, next);
}
public async Task ProcessAsync_NoConnection_ShoulFailTest()
{
var throwingDeviceProxy = ThrowingDeviceProxyBuilder
.Create()
.WithActiveStatus(false)
.Build();
var messageProcessor = this.CreateMessageProcessor(throwingDeviceProxy);
var message1 = GetMessage();
var message2 = GetMessage();
ISinkResult <IRoutingMessage> result = await messageProcessor.ProcessAsync(message1, CancellationToken.None);
Assert.NotNull(result);
Assert.Empty(result.Succeeded);
Assert.Equal(1, result.Failed.Count);
Assert.Empty(result.InvalidDetailsList);
Assert.True(result.SendFailureDetails.HasValue);
ISinkResult <IRoutingMessage> resultBatch = await messageProcessor.ProcessAsync(new[] { message1, message2 }, CancellationToken.None);
Assert.NotNull(resultBatch);
Assert.Empty(resultBatch.Succeeded);
Assert.Equal(2, resultBatch.Failed.Count);
Assert.Empty(resultBatch.InvalidDetailsList);
Assert.True(resultBatch.SendFailureDetails.HasValue);
}
public async Task ProcessAsyncTest()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
const string Mod1Id = "device1/module1";
const string ModEndpointId = "in1";
var deviceProxyMock = new Mock <IDeviceProxy>();
deviceProxyMock.Setup(c => c.SendMessageAsync(It.IsAny <IMessage>(), It.Is <string>((ep) => ep.Equals(ModEndpointId, StringComparison.OrdinalIgnoreCase))))
.Returns(Task.CompletedTask);
deviceProxyMock.SetupGet(p => p.IsActive).Returns(true);
IReadOnlyDictionary <DeviceSubscription, bool> deviceSubscriptions = new ReadOnlyDictionary <DeviceSubscription, bool>(
new Dictionary <DeviceSubscription, bool>
{
[DeviceSubscription.ModuleMessages] = true
});
var connectionManager = new Mock <IConnectionManager>();
connectionManager.Setup(c => c.GetDeviceConnection(It.IsAny <string>())).Returns(Option.Some(deviceProxyMock.Object));
connectionManager.Setup(c => c.GetSubscriptions(It.IsAny <string>())).Returns(Option.Some(deviceSubscriptions));
byte[] messageBody = Encoding.UTF8.GetBytes("Message body");
var properties = new Dictionary <string, string>()
{
{ "Prop1", "Val1" },
{ "Prop2", "Val2" },
};
var systemProperties = new Dictionary <string, string>
{
{ SystemProperties.DeviceId, Mod1Id }
};
var message1 = new RoutingMessage(TelemetryMessageSource.Instance, messageBody, properties, systemProperties);
var message2 = new RoutingMessage(TelemetryMessageSource.Instance, messageBody, properties, systemProperties);
var moduleEndpoint = new ModuleEndpoint($"{Mod1Id}/{ModEndpointId}", Mod1Id, ModEndpointId, connectionManager.Object, routingMessageConverter);
IProcessor moduleMessageProcessor = moduleEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> result = await moduleMessageProcessor.ProcessAsync(message1, CancellationToken.None);
Assert.NotNull(result);
Assert.NotEmpty(result.Succeeded);
Assert.Empty(result.Failed);
Assert.Empty(result.InvalidDetailsList);
Assert.False(result.SendFailureDetails.HasValue);
ISinkResult <IRoutingMessage> resultBatch = await moduleMessageProcessor.ProcessAsync(new[] { message1, message2 }, CancellationToken.None);
Assert.NotNull(resultBatch);
Assert.NotEmpty(resultBatch.Succeeded);
Assert.Empty(resultBatch.Failed);
Assert.Empty(resultBatch.InvalidDetailsList);
Assert.False(resultBatch.SendFailureDetails.HasValue);
}
public static void Checkpoint(EndpointExecutorFsm fsm, ISinkResult <IMessage> result)
{
Log.LogDebug(
(int)EventIds.Checkpoint,
"[Checkpoint] Checkpointing began. CheckpointOffset: {0}, SuccessfulSize: {1}, RemainingSize: {2}, {3}",
fsm.Status.CheckpointerStatus.Offset,
result.Succeeded.Count + result.InvalidDetailsList.Count,
result.Failed.Count,
GetContextString(fsm));
}
public void Merge(ISinkResult <T> other)
{
this.Succeeded.AddRange(other.Succeeded);
this.Failed.AddRange(other.Failed);
this.InvalidDetailsList.AddRange(other.InvalidDetailsList);
if (IsMoreSignificant(this.SendFailureDetails, other.SendFailureDetails))
{
this.SendFailureDetails = other.SendFailureDetails;
}
}
public async Task TestSendCompletes()
{
var retryPolicy = new RetryPolicy(new ErrorDetectionStrategy(_ => true), new FixedInterval(3, TimeSpan.FromMilliseconds(10)));
var items = new[] { 1, 2, 3, 4, 5, 6 };
var testSink = new PartialFailureSink <int>(new Exception());
var sink = new RetryingSink <int>(testSink, retryPolicy);
ISinkResult <int> result = await sink.ProcessAsync(items, CancellationToken.None);
await sink.CloseAsync(CancellationToken.None);
Assert.True(result.IsSuccessful);
Assert.Equal(new List <int>(items), result.Succeeded);
}
public static void CheckpointSuccess(EndpointExecutorFsm fsm, ISinkResult <IMessage> result)
{
Log.LogInformation((int)EventIds.CheckpointSuccess, "[CheckpointSuccess] Checkpointing succeeded. CheckpointOffset: {0}, {1}",
fsm.Status.CheckpointerStatus.Offset, GetContextString(fsm));
IList <IMessage> invalidMessages = result.InvalidDetailsList.Select(d => d.Item).ToList();
SetProcessingInternalCounters(fsm, "Success", result.Succeeded);
SetProcessingInternalCounters(fsm, "Failure", result.Failed);
SetProcessingInternalCounters(fsm, "Invalid", invalidMessages);
SetSuccessfulEgressUserMetricCounter(fsm, result.Succeeded);
SetInvalidEgressUserMetricCounter(fsm, invalidMessages);
}
public async Task SmokeTask()
{
var factory = new RetryingSinkFactory <int>(new TestSinkFactory <int>(), RetryPolicy.NoRetry);
ISink <int> sink = await factory.CreateAsync("hub");
ISinkResult <int> result = await sink.ProcessAsync(1, CancellationToken.None);
Assert.True(result.IsSuccessful);
Assert.Equal(new List <int> {
1
}, result.Succeeded);
Assert.False(result.Failed.Any());
Assert.False(result.InvalidDetailsList.Any());
await sink.CloseAsync(CancellationToken.None);
}
public async Task Events_CloudProxyNotFoundWithDeviceInvalidStateException_DropMessage_Test()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
var routingMessage = Mock.Of <IRoutingMessage>();
string cloudEndpointId = Guid.NewGuid().ToString();
Mock.Get(routingMessage).Setup(rm => rm.SystemProperties).Returns(new Dictionary <string, string> {
{ "connectionDeviceId", "myConnectionDeviceId" }
});
var cloudEndpoint = new CloudEndpoint(cloudEndpointId, id => Task.FromResult(Try <ICloudProxy> .Failure(new DeviceInvalidStateException("Device removed from scope"))), routingMessageConverter, trackDeviceState: true);
IProcessor cloudMessageProcessor = cloudEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> sinkResult = await cloudMessageProcessor.ProcessAsync(routingMessage, CancellationToken.None);
Assert.Equal(FailureKind.InvalidInput, sinkResult.SendFailureDetails.OrDefault().FailureKind);
Assert.Equal(typeof(DeviceInvalidStateException), sinkResult.SendFailureDetails.OrDefault().RawException.GetType());
}
public async Task Events_CloudProxyNotFoundTest()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
var routingMessage = Mock.Of <IRoutingMessage>();
string cloudEndpointId = Guid.NewGuid().ToString();
Mock.Get(routingMessage).Setup(rm => rm.SystemProperties).Returns(new Dictionary <string, string> {
{ "connectionDeviceId", "myConnectionDeviceId" }
});
var cloudEndpoint = new CloudEndpoint(cloudEndpointId, id => Task.FromResult(Option.None <ICloudProxy>()), routingMessageConverter);
IProcessor cloudMessageProcessor = cloudEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> sinkResult = await cloudMessageProcessor.ProcessAsync(routingMessage, CancellationToken.None);
Assert.Equal(FailureKind.None, sinkResult.SendFailureDetails.OrDefault().FailureKind);
Assert.Equal(typeof(EdgeHubConnectionException), sinkResult.SendFailureDetails.OrDefault().RawException.GetType());
}
public async Task ModuleMessageProcessor_RetryableExceptionTest(Exception exception, bool isRetryable)
{
// Arrange
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
var deviceProxy = new Mock <IDeviceProxy>();
deviceProxy.Setup(d => d.SendMessageAsync(It.IsAny <IMessage>(), It.IsAny <string>()))
.ThrowsAsync(exception);
deviceProxy.Setup(d => d.IsActive).Returns(true);
IReadOnlyDictionary <DeviceSubscription, bool> deviceSubscriptions = new ReadOnlyDictionary <DeviceSubscription, bool>(
new Dictionary <DeviceSubscription, bool>
{
[DeviceSubscription.ModuleMessages] = true
});
var connectionManager = new Mock <IConnectionManager>();
connectionManager.Setup(c => c.GetDeviceConnection(It.IsAny <string>())).Returns(Option.Some(deviceProxy.Object));
connectionManager.Setup(c => c.GetSubscriptions(It.IsAny <string>())).Returns(Option.Some(deviceSubscriptions));
var routingMessage = Mock.Of <IRoutingMessage>();
string moduleId = "device1/module1";
string moduleEndpointAddress = "in1";
var moduleEndpoint = new ModuleEndpoint($"{moduleId}/{moduleEndpointAddress}", moduleId, moduleEndpointAddress, connectionManager.Object, routingMessageConverter);
// Act
IProcessor moduleMessageProcessor = moduleEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> result = await moduleMessageProcessor.ProcessAsync(routingMessage, CancellationToken.None);
// Assert
Assert.NotNull(result);
if (isRetryable)
{
Assert.Equal(1, result.Failed.Count);
Assert.Equal(0, result.Succeeded.Count);
Assert.Equal(0, result.InvalidDetailsList.Count);
Assert.Equal(routingMessage, result.Failed.First());
}
else
{
Assert.Equal(1, result.InvalidDetailsList.Count);
Assert.Equal(0, result.Succeeded.Count);
Assert.Equal(0, result.Failed.Count);
Assert.Equal(routingMessage, result.InvalidDetailsList.First().Item);
Assert.Equal(FailureKind.InvalidInput, result.InvalidDetailsList.First().FailureKind);
}
}
public async Task TestFailure()
{
var retryPolicy = new RetryPolicy(new ErrorDetectionStrategy(_ => true), new FixedInterval(2, TimeSpan.FromMilliseconds(10)));
var items = new[] { 1, 2, 3, 4, 5, 6 };
var testSink = new PartialFailureSink <int>(new Exception());
var sink = new RetryingSink <int>(testSink, retryPolicy);
ISinkResult <int> result = await sink.ProcessAsync(items, CancellationToken.None);
Assert.False(result.IsSuccessful);
Assert.Equal(new List <int> {
1, 2, 3, 4, 5
}, result.Succeeded);
Assert.Equal(new List <int> {
6
}, result.Failed);
Assert.Equal(new List <InvalidDetails <int> >(), result.InvalidDetailsList);
}
public async Task CloudMessageProcessor_ProcessAsyncTest()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
var routingMessage = Mock.Of <IRoutingMessage>();
var cloudProxy = Mock.Of <ICloudProxy>(c => c.IsActive);
string cloudEndpointId = Guid.NewGuid().ToString();
Mock.Get(routingMessage).Setup(rm => rm.SystemProperties).Returns(new Dictionary <string, string> {
{ "connectionDeviceId", "myConnectionDeviceId" }
});
Mock.Get(cloudProxy).Setup(cp => cp.SendMessageAsync(It.IsAny <IMessage>())).Returns(Task.FromResult(true));
var cloudEndpoint = new CloudEndpoint(cloudEndpointId, id => Task.FromResult(Option.Some(cloudProxy)), routingMessageConverter);
IProcessor cloudMessageProcessor = cloudEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> sinkResult = await cloudMessageProcessor.ProcessAsync(routingMessage, CancellationToken.None);
Assert.True(sinkResult.Succeeded.Contains(routingMessage));
}
public async Task SmokeTest()
{
var endpoint = new PartialFailureEndpoint("id1");
IProcessor processor = endpoint.CreateProcessor();
Assert.True(processor.ErrorDetectionStrategy.IsTransient(new Exception()));
Assert.Equal(endpoint, processor.Endpoint);
ISinkResult <IMessage> result = await processor.ProcessAsync(new IMessage[] {}, CancellationToken.None);
Assert.Equal(new IMessage[0], result.Succeeded);
IMessage[] messages = new[] { Message1, Message2, Message3, Message4 };
ISinkResult <IMessage> result2 = await processor.ProcessAsync(messages, CancellationToken.None);
Assert.Equal(new[] { Message1, Message2 }, result2.Succeeded);
Assert.Equal(new[] { Message3, Message4 }, result2.Failed);
}
public async Task SmokeTest()
{
var endpoint = new RevivableEndpoint("id1");
IProcessor processor = endpoint.CreateProcessor();
Assert.True(processor.ErrorDetectionStrategy.IsTransient(new Exception()));
Assert.Equal(endpoint, processor.Endpoint);
Assert.Equal(new List <IMessage>(), endpoint.Processed);
Assert.Equal(string.Empty, endpoint.IotHubName);
ISinkResult <IMessage> result = await processor.ProcessAsync(new IMessage[0], CancellationToken.None);
Assert.Equal(new IMessage[0], result.Succeeded);
Assert.Equal(new List <IMessage>(), endpoint.Processed);
IMessage[] messages = new[] { Message1, Message2, Message3 };
ISinkResult <IMessage> result2 = await processor.ProcessAsync(messages, CancellationToken.None);
Assert.Equal(new[] { Message1, Message2, Message3 }, result2.Succeeded);
Assert.Equal(new List <IMessage> {
Message1, Message2, Message3
}, endpoint.Processed);
// set to failing
endpoint.Failing = true;
ISinkResult <IMessage> result3 = await processor.ProcessAsync(messages, CancellationToken.None);
Assert.True(result3.SendFailureDetails.HasValue);
Assert.Equal(new List <IMessage> {
Message1, Message2, Message3
}, endpoint.Processed);
// revive
endpoint.Failing = false;
ISinkResult <IMessage> result4 = await processor.ProcessAsync(messages, CancellationToken.None);
Assert.Equal(new[] { Message1, Message2, Message3 }, result4.Succeeded);
Assert.Equal(new List <IMessage> {
Message1, Message2, Message3, Message1, Message2, Message3
}, endpoint.Processed);
await processor.CloseAsync(CancellationToken.None);
}
public async Task Events_DeviceIdNotFoundTest()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
var routingMessage = Mock.Of <IRoutingMessage>();
string cloudEndpointId = Guid.NewGuid().ToString();
Mock.Get(routingMessage).Setup(rm => rm.SystemProperties).Returns(new Dictionary <string, string> {
{ "messageId", "myConnectionDeviceId" }
});
var cloudProxy = Mock.Of <ICloudProxy>();
var cloudEndpoint = new CloudEndpoint(cloudEndpointId, id => Task.FromResult(Try.Success(cloudProxy)), routingMessageConverter, trackDeviceState: false);
IProcessor cloudMessageProcessor = cloudEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> sinkResult = await cloudMessageProcessor.ProcessAsync(routingMessage, CancellationToken.None);
Assert.Equal(FailureKind.InvalidInput, sinkResult.SendFailureDetails.OrDefault().FailureKind);
Assert.Equal(typeof(InvalidOperationException), sinkResult.SendFailureDetails.OrDefault().RawException.GetType());
Assert.Equal(1, sinkResult.InvalidDetailsList.Count);
Assert.Equal(0, sinkResult.Failed.Count);
Assert.Equal(0, sinkResult.Succeeded.Count);
}
public async Task TestCancellation()
{
var cts = new CancellationTokenSource();
var retryPolicy = new RetryPolicy(new ErrorDetectionStrategy(_ => true), new FixedInterval(2, TimeSpan.FromMilliseconds(10)));
var items = new[] { 1, 2, 3, 4, 5, 6 };
var testSink = new FailedSink <int>(new Exception());
var sink = new RetryingSink <int>(testSink, retryPolicy);
Task <ISinkResult <int> > task = sink.ProcessAsync(items, cts.Token);
cts.Cancel();
ISinkResult <int> result = await task;
Assert.False(result.IsSuccessful);
Assert.Equal(new List <int>(), result.Succeeded);
Assert.Equal(new List <int> {
1, 2, 3, 4, 5, 6
}, result.Failed);
Assert.Equal(new List <InvalidDetails <int> >(), result.InvalidDetailsList);
Assert.True(result.SendFailureDetails.HasValue);
result.SendFailureDetails.ForEach(sfd => Assert.IsType <TaskCanceledException>(sfd.RawException));
}
public async Task SmokeTest()
{
var endpoint = new NullEndpoint("endpoint1");
IProcessor processor = endpoint.CreateProcessor();
Assert.True(processor.ErrorDetectionStrategy.IsTransient(new Exception()));
Assert.Equal(endpoint, processor.Endpoint);
ISinkResult <IMessage> result = await processor.ProcessAsync(new IMessage[] { }, CancellationToken.None);
Assert.Equal(new IMessage[0], result.Succeeded);
ISinkResult <IMessage> result2 = await processor.ProcessAsync(new[] { Message1, Message2, Message3 }, CancellationToken.None);
Assert.Equal(new[] { Message1, Message2, Message3 }, result2.Succeeded);
var endpoint2 = new NullEndpoint("id2", "name2", "hub2");
Assert.Equal("id2", endpoint2.Id);
Assert.Equal("name2", endpoint2.Name);
Assert.Equal("hub2", endpoint2.IotHubName);
}
public async Task SmokeTest()
{
var cts = new CancellationTokenSource();
var endpoint = new FailedEndpoint("id1", "name1", "hub1", new InvalidOperationException());
IProcessor processor = endpoint.CreateProcessor();
Assert.True(processor.ErrorDetectionStrategy.IsTransient(new Exception()));
Assert.Equal(endpoint, processor.Endpoint);
ISinkResult <IMessage> result = await processor.ProcessAsync(new IMessage[0], cts.Token);
Assert.True(result.SendFailureDetails.HasValue);
result.SendFailureDetails.ForEach(ex => Assert.IsType <InvalidOperationException>(ex.RawException));
Assert.True(processor.CloseAsync(CancellationToken.None).IsCompleted);
var endpoint2 = new FailedEndpoint("id2");
IProcessor processor2 = endpoint2.CreateProcessor();
ISinkResult <IMessage> result2 = await processor2.ProcessAsync(new IMessage[0], cts.Token);
Assert.True(result2.SendFailureDetails.HasValue);
result2.SendFailureDetails.ForEach(ex => Assert.IsType <Exception>(ex.RawException));
}
public async Task TestNonTransient()
{
var retryPolicy = new RetryPolicy(new ErrorDetectionStrategy(_ => false), new FixedInterval(int.MaxValue, TimeSpan.FromMilliseconds(10)));
var items = new[] { 1, 2, 3, 4, 5, 6 };
var testSink = new FailedSink <int>(new Exception("non-transient"));
var sink = new RetryingSink <int>(testSink, retryPolicy);
using (var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)))
{
ISinkResult <int> result = await sink.ProcessAsync(items, cts.Token);
Assert.False(result.IsSuccessful);
Assert.Equal(new List <int>(), result.Succeeded);
Assert.Equal(new List <int> {
1, 2, 3, 4, 5, 6
}, result.Failed);
Assert.Equal(new List <InvalidDetails <int> >(), result.InvalidDetailsList);
Assert.True(result.SendFailureDetails.HasValue);
result.SendFailureDetails.ForEach(sfd => Assert.IsType <Exception>(sfd.RawException));
result.SendFailureDetails.ForEach(sfd => Assert.Equal("non-transient", sfd.RawException.Message));
}
}
static async Task EnterCheckpointingAsync(EndpointExecutorFsm thisPtr)
{
ICommand next;
try
{
Preconditions.CheckNotNull(thisPtr.currentCheckpointCommand);
using (var cts = new CancellationTokenSource(thisPtr.config.Timeout))
{
ISinkResult <IMessage> result = thisPtr.currentCheckpointCommand.Result;
if (result.Succeeded.Any() || result.InvalidDetailsList.Any())
{
ICollection <IMessage> toCheckpoint = result.InvalidDetailsList.Count > 0
? result.Succeeded.Concat(result.InvalidDetailsList.Select(i => i.Item)).ToList()
: result.Succeeded;
ICollection <IMessage> remaining = result.Failed;
Events.Checkpoint(thisPtr, result);
await thisPtr.Checkpointer.CommitAsync(toCheckpoint, remaining, Option.None <DateTime>(), thisPtr.unhealthySince, cts.Token);
Events.CheckpointSuccess(thisPtr, result);
}
}
next = EnterCheckpointingHelper(thisPtr);
}
catch (Exception ex)
{
Events.CheckpointFailure(thisPtr, ex);
next = thisPtr.config.ThrowOnDead
? Commands.Throw(ex)
: EnterCheckpointingHelper(thisPtr);
}
await RunInternalAsync(thisPtr, next);
}
public async Task SmokeTest()
{
var endpoint = new TestEndpoint("id1", "name1", "hub1");
IProcessor processor = endpoint.CreateProcessor();
Assert.Equal(endpoint, processor.Endpoint);
Assert.True(processor.ErrorDetectionStrategy.IsTransient(new Exception()));
Assert.Equal(new List <IMessage>(), endpoint.Processed);
Assert.Equal("name1", endpoint.Name);
Assert.Equal("hub1", endpoint.IotHubName);
ISinkResult <IMessage> result = await processor.ProcessAsync(new IMessage[0], CancellationToken.None);
Assert.Equal(new IMessage[0], result.Succeeded);
Assert.Equal(new List <IMessage>(), endpoint.Processed);
IMessage[] messages = new[] { Message1, Message2, Message3 };
ISinkResult <IMessage> result2 = await processor.ProcessAsync(messages, CancellationToken.None);
Assert.Equal(new[] { Message1, Message2, Message3 }, result2.Succeeded);
Assert.Equal(new List <IMessage> {
Message1, Message2, Message3
}, endpoint.Processed);
}
public async Task RetryableExceptionsTest(Exception exception, bool isRetryable)
{
// Arrange
string id = "d1";
var cloudProxy = new Mock <ICloudProxy>();
cloudProxy.Setup(c => c.SendMessageAsync(It.IsAny <IMessage>()))
.ThrowsAsync(exception);
var cloudEndpoint = new CloudEndpoint(Guid.NewGuid().ToString(), _ => Task.FromResult(Option.Some(cloudProxy.Object)), new RoutingMessageConverter());
IProcessor processor = cloudEndpoint.CreateProcessor();
var message = new RoutingMessage(TelemetryMessageSource.Instance, new byte[0], ImmutableDictionary <string, string> .Empty, new Dictionary <string, string>
{
[Core.SystemProperties.ConnectionDeviceId] = id
});
// Act
ISinkResult <IRoutingMessage> result = await processor.ProcessAsync(message, CancellationToken.None);
// Assert
Assert.NotNull(result);
if (isRetryable)
{
Assert.Equal(1, result.Failed.Count);
Assert.Equal(0, result.Succeeded.Count);
Assert.Equal(0, result.InvalidDetailsList.Count);
Assert.Equal(message, result.Failed.First());
}
else
{
Assert.Equal(1, result.InvalidDetailsList.Count);
Assert.Equal(0, result.Succeeded.Count);
Assert.Equal(0, result.Failed.Count);
Assert.Equal(message, result.InvalidDetailsList.First().Item);
Assert.Equal(FailureKind.InvalidInput, result.InvalidDetailsList.First().FailureKind);
}
}
public async Task ProcessAsyncTest()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
string cloudEndpointId = Guid.NewGuid().ToString();
var cloudProxyMock = new Mock <ICloudProxy>();
cloudProxyMock.Setup(c => c.SendMessageAsync(It.IsAny <IMessage>()))
.Callback <IMessage>(
msg =>
{
if (msg.Properties.ContainsKey("Delay"))
{
Task.Delay(TimeSpan.FromSeconds(10)).Wait();
}
})
.Returns(Task.CompletedTask);
cloudProxyMock.SetupGet(p => p.IsActive).Returns(true);
string device1Id = "device1";
string device2Id = "device2";
byte[] messageBody = Encoding.UTF8.GetBytes("Message body");
var properties = new Dictionary <string, string>()
{
{ "Prop1", "Val1" },
{ "Prop2", "Val2" }
};
var device1SystemProperties = new Dictionary <string, string>
{
{ SystemProperties.DeviceId, device1Id }
};
var device2SystemProperties = new Dictionary <string, string>
{
{ SystemProperties.DeviceId, device2Id }
};
var cancelProperties = new Dictionary <string, string>()
{
{ "Delay", "true" },
{ "Prop2", "Val2" }
};
var message1 = new RoutingMessage(TelemetryMessageSource.Instance, messageBody, properties, device1SystemProperties);
var message2 = new RoutingMessage(TelemetryMessageSource.Instance, messageBody, properties, device2SystemProperties);
var message3 = new RoutingMessage(TelemetryMessageSource.Instance, messageBody, cancelProperties, device1SystemProperties);
Task <Option <ICloudProxy> > GetCloudProxy(string id)
{
return(Task.FromResult(
id.Equals(device1Id)
? Option.Some(cloudProxyMock.Object)
: Option.None <ICloudProxy>()));
}
var cloudEndpoint = new CloudEndpoint(cloudEndpointId, GetCloudProxy, routingMessageConverter, maxBatchSize: 1);
IProcessor cloudMessageProcessor = cloudEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> result1 = await cloudMessageProcessor.ProcessAsync(message1, CancellationToken.None);
Assert.NotNull(result1);
Assert.NotEmpty(result1.Succeeded);
Assert.Empty(result1.Failed);
Assert.Empty(result1.InvalidDetailsList);
Assert.False(result1.SendFailureDetails.HasValue);
ISinkResult <IRoutingMessage> result2 = await cloudMessageProcessor.ProcessAsync(message2, CancellationToken.None);
Assert.NotNull(result2);
Assert.Empty(result2.InvalidDetailsList);
Assert.NotEmpty(result2.Failed);
Assert.Empty(result2.Succeeded);
Assert.True(result2.SendFailureDetails.HasValue);
ISinkResult <IRoutingMessage> resultBatch = await cloudMessageProcessor.ProcessAsync(new[] { message1, message2 }, CancellationToken.None);
Assert.NotNull(resultBatch);
Assert.Equal(1, resultBatch.Succeeded.Count);
Assert.Equal(1, resultBatch.Failed.Count);
Assert.Empty(resultBatch.InvalidDetailsList);
Assert.True(resultBatch.SendFailureDetails.HasValue);
ISinkResult <IRoutingMessage> resultBatchCancelled = await cloudMessageProcessor.ProcessAsync(new[] { message1, message2 }, new CancellationToken(true));
Assert.NotNull(resultBatchCancelled);
Assert.Empty(resultBatchCancelled.Succeeded);
Assert.NotEmpty(resultBatchCancelled.Failed);
Assert.Empty(resultBatchCancelled.InvalidDetailsList);
Assert.True(resultBatchCancelled.SendFailureDetails.HasValue);
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
ISinkResult <IRoutingMessage> resultBatchCancelled2 = await cloudMessageProcessor.ProcessAsync(new[] { message1, message3, message1 }, cts.Token);
Assert.NotNull(resultBatchCancelled2);
Assert.Equal(2, resultBatchCancelled2.Succeeded.Count);
Assert.Equal(1, resultBatchCancelled2.Failed.Count);
Assert.Empty(resultBatchCancelled2.InvalidDetailsList);
Assert.True(resultBatchCancelled2.SendFailureDetails.HasValue);
}
public async Task ProcessInBatchesWithBatchSizeTest()
{
// Arrange
string device1 = "d1";
string device2 = "d2";
string device3 = "d3";
IList <IRoutingMessage> device1Messages = GetMessages(device1, 45);
IList <IRoutingMessage> device2Messages = GetMessages(device2, 25);
IList <IRoutingMessage> device3Messages = GetMessages(device3, 30);
IList <IRoutingMessage> messagesToProcess = device1Messages
.Concat(device2Messages)
.Concat(device3Messages)
.ToList();
Mock <ICloudProxy> InitCloudProxy(List <int> receivedMsgCountList)
{
var cp = new Mock <ICloudProxy>();
cp.Setup(c => c.SendMessageBatchAsync(It.IsAny <IEnumerable <IMessage> >()))
.Callback <IEnumerable <IMessage> >(b => receivedMsgCountList.Add(b.Count()))
.Returns(Task.CompletedTask);
cp.SetupGet(p => p.IsActive).Returns(true);
return(cp);
}
var device1CloudReceivedMessagesCountList = new List <int>();
Mock <ICloudProxy> device1CloudProxy = InitCloudProxy(device1CloudReceivedMessagesCountList);
var device2CloudReceivedMessagesCountList = new List <int>();
Mock <ICloudProxy> device2CloudProxy = InitCloudProxy(device2CloudReceivedMessagesCountList);
var device3CloudReceivedMessagesCountList = new List <int>();
Mock <ICloudProxy> device3CloudProxy = InitCloudProxy(device3CloudReceivedMessagesCountList);
Task <Option <ICloudProxy> > GetCloudProxy(string id)
{
ICloudProxy cp = null;
if (id == device1)
{
cp = device1CloudProxy.Object;
}
else if (id == device2)
{
cp = device2CloudProxy.Object;
}
else if (id == device3)
{
cp = device3CloudProxy.Object;
}
return(Task.FromResult(Option.Maybe(cp)));
}
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
string cloudEndpointId = Guid.NewGuid().ToString();
var cloudEndpoint = new CloudEndpoint(cloudEndpointId, GetCloudProxy, routingMessageConverter, 30);
// Act
IProcessor cloudMessageProcessor = cloudEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> sinkResult = await cloudMessageProcessor.ProcessAsync(messagesToProcess, CancellationToken.None);
// Assert
Assert.Equal(messagesToProcess, sinkResult.Succeeded);
Assert.Equal(device1CloudReceivedMessagesCountList, new[] { 30, 15 });
Assert.Equal(device2CloudReceivedMessagesCountList, new[] { 25 });
Assert.Equal(device3CloudReceivedMessagesCountList, new[] { 30 });
}
public async Task ProcessAsync_SendThrows_Test()
{
Core.IMessageConverter <IRoutingMessage> routingMessageConverter = new RoutingMessageConverter();
string cloudEndpointId = Guid.NewGuid().ToString();
var cloudProxyMock = new Mock <ICloudProxy>();
cloudProxyMock.Setup(c => c.SendMessageAsync(It.IsAny <IMessage>()))
.Throws <TimeoutException>();
cloudProxyMock.SetupGet(p => p.IsActive).Returns(true);
string device1Id = "device1";
string device2Id = "device2";
byte[] messageBody = Encoding.UTF8.GetBytes("Message body");
var properties = new Dictionary <string, string>()
{
{ "Prop1", "Val1" },
{ "Prop2", "Val2" }
};
var device1SystemProperties = new Dictionary <string, string>
{
{ SystemProperties.DeviceId, device1Id }
};
var device2SystemProperties = new Dictionary <string, string>
{
{ SystemProperties.DeviceId, device2Id }
};
var message1 = new RoutingMessage(TelemetryMessageSource.Instance, messageBody, properties, device1SystemProperties);
var message2 = new RoutingMessage(TelemetryMessageSource.Instance, messageBody, properties, device2SystemProperties);
Task <Option <ICloudProxy> > GetCloudProxy(string id)
{
return(Task.FromResult(Option.Some(cloudProxyMock.Object)));
}
var cloudEndpoint = new CloudEndpoint(cloudEndpointId, GetCloudProxy, routingMessageConverter);
IProcessor cloudMessageProcessor = cloudEndpoint.CreateProcessor();
ISinkResult <IRoutingMessage> result = await cloudMessageProcessor.ProcessAsync(new[] { message1, message2 }, CancellationToken.None);
Assert.NotNull(result);
Assert.Empty(result.Succeeded);
Assert.Equal(2, result.Failed.Count);
Assert.Empty(result.InvalidDetailsList);
Assert.True(result.SendFailureDetails.HasValue);
// throw non-retryable
cloudProxyMock.Setup(c => c.SendMessageAsync(It.IsAny <IMessage>()))
.Throws <Exception>();
ISinkResult <IRoutingMessage> result1 = await cloudMessageProcessor.ProcessAsync(new[] { message1, message2 }, CancellationToken.None);
Assert.NotNull(result1);
Assert.Empty(result1.Succeeded);
Assert.Empty(result1.Failed);
Assert.Equal(2, result1.InvalidDetailsList.Count);
Assert.True(result1.SendFailureDetails.HasValue);
}
public Checkpoint(ISinkResult <IMessage> result)
{
this.Result = Preconditions.CheckNotNull(result);
}
