static async Task EnterProcessDeadAsync(EndpointExecutorFsm thisPtr)
{
Preconditions.CheckArgument(thisPtr.lastFailedRevivalTime.HasValue);
TimeSpan deadFor = thisPtr.systemTime.UtcNow - thisPtr.lastFailedRevivalTime.GetOrElse(thisPtr.systemTime.UtcNow);
if (deadFor >= thisPtr.config.RevivePeriod)
{
await RunInternalAsync(thisPtr, Commands.Revive);
}
else
{
try
{
// In the dead state, checkpoint all "sent" messages
// This effectively drops the messages, allowing the other endpoints in
// this router to make progress.
ICollection <IMessage> tocheckpoint = thisPtr.currentSendCommand.Messages;
Events.Dead(thisPtr, tocheckpoint);
SendFailureDetails persistingFailureDetails = thisPtr.currentCheckpointCommand?.Result?.SendFailureDetails.GetOrElse(default(SendFailureDetails));
await RunInternalAsync(thisPtr, Commands.Checkpoint(new SinkResult <IMessage>(tocheckpoint, persistingFailureDetails)));
}
catch (Exception ex)
{
Events.DeadFailure(thisPtr, ex);
}
}
}
public static void SendFailure(EndpointExecutorFsm fsm, ISinkResult <IMessage> result, Stopwatch stopwatch)
{
long latencyInMs = stopwatch.ElapsedMilliseconds;
if (!Routing.PerfCounter.LogExternalWriteLatency(fsm.Endpoint.IotHubName, fsm.Endpoint.Name, fsm.Endpoint.Type, false, latencyInMs, out string error))
{
Log.LogError((int)EventIds.CounterFailure, "[LogExternalWriteLatencyCounterFailed] {0}", error);
}
SendFailureDetails failureDetails = result.SendFailureDetails.GetOrElse(DefaultSendFailureDetails);
foreach (InvalidDetails <IMessage> invalidDetails in result.InvalidDetailsList)
{
Routing.UserAnalyticsLogger.LogInvalidMessage(fsm.Endpoint.IotHubName, invalidDetails.Item, invalidDetails.FailureKind);
}
Log.LogWarning(
(int)EventIds.SendFailure,
failureDetails.RawException,
"[SendFailure] Sending failed. SuccessfulSize: {0}, FailedSize: {1}, InvalidSize: {2}, {3}",
result.Succeeded.Count,
result.Failed.Count,
result.InvalidDetailsList.Count,
GetContextString(fsm));
LogUnhealthyEndpointOpMonError(fsm, failureDetails.FailureKind);
}
static ISinkResult GetSyncResultForInvalidMessages(Exception ex, List <IRoutingMessage> routingMessages)
{
List <InvalidDetails <IRoutingMessage> > invalid = routingMessages
.Select(m => new InvalidDetails <IRoutingMessage>(m, FailureKind.InvalidInput))
.ToList();
var sendFailureDetails = new SendFailureDetails(FailureKind.InvalidInput, ex);
return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, ImmutableList <IRoutingMessage> .Empty, invalid, sendFailureDetails));
}
Task <ISinkResult> ProcessNoConnection(ICollection <IRoutingMessage> routingMessages)
{
var failed = new List <IRoutingMessage>();
failed.AddRange(routingMessages);
var sendFailureDetails = new SendFailureDetails(FailureKind.None, new EdgeHubConnectionException($"Target module {this.moduleEndpoint.moduleId} is not connected"));
return(Task.FromResult((ISinkResult <IRoutingMessage>) new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails)));
}
public async Task <ISinkResult> ProcessAsync(ICollection <IRoutingMessage> routingMessages, CancellationToken token)
{
Preconditions.CheckNotNull(routingMessages, nameof(routingMessages));
// TODO - figure out if we can use cancellation token to cancel send
var succeeded = new List <IRoutingMessage>();
var failed = new List <IRoutingMessage>();
var invalid = new List <InvalidDetails <IRoutingMessage> >();
SendFailureDetails sendFailureDetails = null;
Events.ProcessingMessages(this.moduleEndpoint, routingMessages);
Util.Option <IDeviceProxy> deviceProxy = this.GetDeviceProxy();
if (!deviceProxy.HasValue)
{
failed.AddRange(routingMessages);
sendFailureDetails = new SendFailureDetails(FailureKind.None, new EdgeHubConnectionException($"Target module {this.moduleEndpoint.moduleId} is not connected"));
}
else
{
foreach (IRoutingMessage routingMessage in routingMessages)
{
IMessage message = this.moduleEndpoint.messageConverter.ToMessage(routingMessage);
await deviceProxy.ForEachAsync(
async dp =>
{
try
{
await dp.SendMessageAsync(message, this.moduleEndpoint.Input);
succeeded.Add(routingMessage);
}
catch (Exception ex)
{
if (IsRetryable(ex))
{
failed.Add(routingMessage);
}
else
{
Events.InvalidMessage(ex);
invalid.Add(new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.None));
}
Events.ErrorSendingMessages(this.moduleEndpoint, ex);
}
});
}
if (failed.Count > 0)
{
Events.RetryingMessages(failed.Count, this.moduleEndpoint.Id);
sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending message to module {this.moduleEndpoint.moduleId}"));
}
}
return(new SinkResult <IRoutingMessage>(succeeded, failed, invalid, sendFailureDetails));
}
Task <ISinkResult> ProcessNoIdentity(IRoutingMessage routingMessage)
{
Events.InvalidMessageNoIdentity();
var invalid = new List <InvalidDetails <IRoutingMessage> > {
new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.InvalidInput)
};
var sendFailureDetails = new SendFailureDetails(FailureKind.InvalidInput, new InvalidOperationException("Message does not contain client identity"));
return(Task.FromResult((ISinkResult <IRoutingMessage>) new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, ImmutableList <IRoutingMessage> .Empty, invalid, sendFailureDetails)));
}
Task <ISinkResult> ProcessNoConnection(string identity, IRoutingMessage routingMessage)
{
Events.IoTHubNotConnected(identity);
var failed = new List <IRoutingMessage> {
routingMessage
};
var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubConnectionException("IoT Hub is not connected"));
return(Task.FromResult((ISinkResult <IRoutingMessage>) new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails)));
}
public async Task <ISinkResult> ProcessAsync(IRoutingMessage routingMessage, CancellationToken token)
{
Preconditions.CheckNotNull(routingMessage, nameof(routingMessage));
var succeeded = new List <IRoutingMessage>();
var failed = new List <IRoutingMessage>();
var invalid = new List <InvalidDetails <IRoutingMessage> >();
SendFailureDetails sendFailureDetails = null;
IMessage message = this.cloudEndpoint.messageConverter.ToMessage(routingMessage);
Util.Option <ICloudProxy> cloudProxy = await this.GetCloudProxy(routingMessage);
if (!cloudProxy.HasValue)
{
sendFailureDetails = new SendFailureDetails(FailureKind.None, new EdgeHubConnectionException("IoT Hub is not connected"));
failed.Add(routingMessage);
}
else
{
await cloudProxy.ForEachAsync(async cp =>
{
try
{
string id = this.GetIdentity(routingMessage).Expect(() => new InvalidOperationException("Could not retrieve identity of message"));
using (Metrics.CloudLatency(id))
{
await cp.SendMessageAsync(message);
}
succeeded.Add(routingMessage);
Metrics.MessageCount(id);
}
catch (Exception ex)
{
if (IsRetryable(ex))
{
failed.Add(routingMessage);
}
else
{
Events.InvalidMessage(ex);
invalid.Add(new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.None));
}
if (failed.Count > 0)
{
Events.RetryingMessage(routingMessage, ex);
sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending messages to IotHub for device {this.cloudEndpoint.Id}"));
}
}
});
}
return(new SinkResult <IRoutingMessage>(succeeded, failed, invalid, sendFailureDetails));
}
async Task <ISinkResult> ProcessAsync(ICollection <IRoutingMessage> routingMessages, IDeviceProxy dp, CancellationToken token)
{
// TODO - figure out if we can use cancellation token to cancel send
var succeeded = new List <IRoutingMessage>();
var failed = new List <IRoutingMessage>();
var invalid = new List <InvalidDetails <IRoutingMessage> >();
SendFailureDetails sendFailureDetails = null;
foreach (IRoutingMessage routingMessage in routingMessages)
{
IMessage message = this.moduleEndpoint.messageConverter.ToMessage(routingMessage);
try
{
if (failed.Count == 0)
{
await dp.SendMessageAsync(message, this.moduleEndpoint.Input);
succeeded.Add(routingMessage);
}
else
{
// if one failed, fail the rest, so retry will keep message order
failed.Add(routingMessage);
}
}
catch (Exception ex)
{
if (IsRetryable(ex))
{
failed.Add(routingMessage);
}
else
{
Events.InvalidMessage(ex);
invalid.Add(new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.InvalidInput));
}
Events.ErrorSendingMessages(this.moduleEndpoint, ex);
}
}
if (failed.Count > 0)
{
Events.RetryingMessages(failed.Count, this.moduleEndpoint.Id);
sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending message to module {this.moduleEndpoint.moduleId}"));
}
return(new SinkResult <IRoutingMessage>(succeeded, failed, invalid, sendFailureDetails));
}
public async Task <ISinkResult> ProcessAsync(IRoutingMessage routingMessage, CancellationToken token)
{
Preconditions.CheckNotNull(routingMessage, nameof(routingMessage));
if (token.IsCancellationRequested)
{
Events.CancelledProcessingMessage(routingMessage);
var failed = new List <IRoutingMessage> {
routingMessage
};
var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubConnectionException($"Cancelled sending messages to IotHub for device {this.cloudEndpoint.Id}"));
return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails));
}
Util.Option <string> identity = this.GetIdentity(routingMessage);
ISinkResult result = await identity.Match(
id => this.ProcessAsync(routingMessage, id),
() => this.ProcessNoIdentity(routingMessage));
return(result);
}
ISinkResult HandleException(Exception ex, IRoutingMessage routingMessage)
{
if (IsRetryable(ex))
{
var failed = new List <IRoutingMessage> {
routingMessage
};
Events.RetryingMessage(routingMessage, ex);
var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending messages to IotHub for device {this.cloudEndpoint.Id}"));
return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails));
}
else
{
Events.InvalidMessage(ex);
var invalid = new List <InvalidDetails <IRoutingMessage> > {
new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.InvalidInput)
};
var sendFailureDetails = new SendFailureDetails(FailureKind.InvalidInput, ex);
return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, ImmutableList <IRoutingMessage> .Empty, invalid, sendFailureDetails));
}
}
public async Task <ISinkResult <T> > ProcessAsync(ICollection <T> ts, CancellationToken token)
{
var succeeded = new List <T>();
var failed = new List <T>();
var invalid = new List <InvalidDetails <T> >();
ICollection <T> messages = ts;
ISinkResult <T> rv;
ShouldRetry shouldRetry = this.retryPolicy.RetryStrategy.GetShouldRetry();
try
{
int i = 0;
SendFailureDetails failureDetails = null;
bool finished = false;
while (!finished)
{
token.ThrowIfCancellationRequested();
ISinkResult <T> result = await this.underlying.ProcessAsync(messages, token);
succeeded.AddRange(result.Succeeded);
invalid.AddRange(result.InvalidDetailsList);
if (result.IsSuccessful)
{
failureDetails = null;
finished = true;
}
else
{
TimeSpan retryAfter;
failureDetails = result.SendFailureDetails.OrDefault();
if (failureDetails != null && this.retryPolicy.ErrorDetectionStrategy.IsTransient(failureDetails.RawException) && shouldRetry(i, failureDetails.RawException, out retryAfter))
{
// if we should retry, set the next group of messages to send
// to the failed messages from the previous attempt
messages = result.Failed;
Preconditions.CheckRange(retryAfter.TotalMilliseconds, 0.0);
await Task.Delay(retryAfter, token);
i++;
}
else
{
// do not retry, return failed messages as failed
failed.AddRange(result.Failed);
finished = true;
}
}
}
rv = new SinkResult <T>(succeeded, failed, invalid, failureDetails);
}
catch (OperationCanceledException ex)
{
failed.AddRange(messages);
rv = new SinkResult <T>(succeeded, failed, invalid, new SendFailureDetails(FailureKind.InternalError, ex));
}
catch (Exception ex)
{
failed.AddRange(messages);
rv = new SinkResult <T>(succeeded, failed, invalid, new SendFailureDetails(FailureKind.InternalError, ex));
}
return(rv);
}
static ISinkResult GetSyncResultForFailedMessages(Exception ex, List <IRoutingMessage> routingMessages)
{
var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, ex);
return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, routingMessages, sendFailureDetails));
}