static async Task EnterProcessDeadAsync(EndpointExecutorFsm thisPtr) { Preconditions.CheckArgument(thisPtr.lastFailedRevivalTime.HasValue); TimeSpan deadFor = thisPtr.systemTime.UtcNow - thisPtr.lastFailedRevivalTime.GetOrElse(thisPtr.systemTime.UtcNow); if (deadFor >= thisPtr.config.RevivePeriod) { await RunInternalAsync(thisPtr, Commands.Revive); } else { try { // In the dead state, checkpoint all "sent" messages // This effectively drops the messages, allowing the other endpoints in // this router to make progress. ICollection <IMessage> tocheckpoint = thisPtr.currentSendCommand.Messages; Events.Dead(thisPtr, tocheckpoint); SendFailureDetails persistingFailureDetails = thisPtr.currentCheckpointCommand?.Result?.SendFailureDetails.GetOrElse(default(SendFailureDetails)); await RunInternalAsync(thisPtr, Commands.Checkpoint(new SinkResult <IMessage>(tocheckpoint, persistingFailureDetails))); } catch (Exception ex) { Events.DeadFailure(thisPtr, ex); } } }
public static void SendFailure(EndpointExecutorFsm fsm, ISinkResult <IMessage> result, Stopwatch stopwatch) { long latencyInMs = stopwatch.ElapsedMilliseconds; if (!Routing.PerfCounter.LogExternalWriteLatency(fsm.Endpoint.IotHubName, fsm.Endpoint.Name, fsm.Endpoint.Type, false, latencyInMs, out string error)) { Log.LogError((int)EventIds.CounterFailure, "[LogExternalWriteLatencyCounterFailed] {0}", error); } SendFailureDetails failureDetails = result.SendFailureDetails.GetOrElse(DefaultSendFailureDetails); foreach (InvalidDetails <IMessage> invalidDetails in result.InvalidDetailsList) { Routing.UserAnalyticsLogger.LogInvalidMessage(fsm.Endpoint.IotHubName, invalidDetails.Item, invalidDetails.FailureKind); } Log.LogWarning( (int)EventIds.SendFailure, failureDetails.RawException, "[SendFailure] Sending failed. SuccessfulSize: {0}, FailedSize: {1}, InvalidSize: {2}, {3}", result.Succeeded.Count, result.Failed.Count, result.InvalidDetailsList.Count, GetContextString(fsm)); LogUnhealthyEndpointOpMonError(fsm, failureDetails.FailureKind); }
static ISinkResult GetSyncResultForInvalidMessages(Exception ex, List <IRoutingMessage> routingMessages) { List <InvalidDetails <IRoutingMessage> > invalid = routingMessages .Select(m => new InvalidDetails <IRoutingMessage>(m, FailureKind.InvalidInput)) .ToList(); var sendFailureDetails = new SendFailureDetails(FailureKind.InvalidInput, ex); return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, ImmutableList <IRoutingMessage> .Empty, invalid, sendFailureDetails)); }
Task <ISinkResult> ProcessNoConnection(ICollection <IRoutingMessage> routingMessages) { var failed = new List <IRoutingMessage>(); failed.AddRange(routingMessages); var sendFailureDetails = new SendFailureDetails(FailureKind.None, new EdgeHubConnectionException($"Target module {this.moduleEndpoint.moduleId} is not connected")); return(Task.FromResult((ISinkResult <IRoutingMessage>) new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails))); }
public async Task <ISinkResult> ProcessAsync(ICollection <IRoutingMessage> routingMessages, CancellationToken token) { Preconditions.CheckNotNull(routingMessages, nameof(routingMessages)); // TODO - figure out if we can use cancellation token to cancel send var succeeded = new List <IRoutingMessage>(); var failed = new List <IRoutingMessage>(); var invalid = new List <InvalidDetails <IRoutingMessage> >(); SendFailureDetails sendFailureDetails = null; Events.ProcessingMessages(this.moduleEndpoint, routingMessages); Util.Option <IDeviceProxy> deviceProxy = this.GetDeviceProxy(); if (!deviceProxy.HasValue) { failed.AddRange(routingMessages); sendFailureDetails = new SendFailureDetails(FailureKind.None, new EdgeHubConnectionException($"Target module {this.moduleEndpoint.moduleId} is not connected")); } else { foreach (IRoutingMessage routingMessage in routingMessages) { IMessage message = this.moduleEndpoint.messageConverter.ToMessage(routingMessage); await deviceProxy.ForEachAsync( async dp => { try { await dp.SendMessageAsync(message, this.moduleEndpoint.Input); succeeded.Add(routingMessage); } catch (Exception ex) { if (IsRetryable(ex)) { failed.Add(routingMessage); } else { Events.InvalidMessage(ex); invalid.Add(new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.None)); } Events.ErrorSendingMessages(this.moduleEndpoint, ex); } }); } if (failed.Count > 0) { Events.RetryingMessages(failed.Count, this.moduleEndpoint.Id); sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending message to module {this.moduleEndpoint.moduleId}")); } } return(new SinkResult <IRoutingMessage>(succeeded, failed, invalid, sendFailureDetails)); }
Task <ISinkResult> ProcessNoIdentity(IRoutingMessage routingMessage) { Events.InvalidMessageNoIdentity(); var invalid = new List <InvalidDetails <IRoutingMessage> > { new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.InvalidInput) }; var sendFailureDetails = new SendFailureDetails(FailureKind.InvalidInput, new InvalidOperationException("Message does not contain client identity")); return(Task.FromResult((ISinkResult <IRoutingMessage>) new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, ImmutableList <IRoutingMessage> .Empty, invalid, sendFailureDetails))); }
Task <ISinkResult> ProcessNoConnection(string identity, IRoutingMessage routingMessage) { Events.IoTHubNotConnected(identity); var failed = new List <IRoutingMessage> { routingMessage }; var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubConnectionException("IoT Hub is not connected")); return(Task.FromResult((ISinkResult <IRoutingMessage>) new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails))); }
public async Task <ISinkResult> ProcessAsync(IRoutingMessage routingMessage, CancellationToken token) { Preconditions.CheckNotNull(routingMessage, nameof(routingMessage)); var succeeded = new List <IRoutingMessage>(); var failed = new List <IRoutingMessage>(); var invalid = new List <InvalidDetails <IRoutingMessage> >(); SendFailureDetails sendFailureDetails = null; IMessage message = this.cloudEndpoint.messageConverter.ToMessage(routingMessage); Util.Option <ICloudProxy> cloudProxy = await this.GetCloudProxy(routingMessage); if (!cloudProxy.HasValue) { sendFailureDetails = new SendFailureDetails(FailureKind.None, new EdgeHubConnectionException("IoT Hub is not connected")); failed.Add(routingMessage); } else { await cloudProxy.ForEachAsync(async cp => { try { string id = this.GetIdentity(routingMessage).Expect(() => new InvalidOperationException("Could not retrieve identity of message")); using (Metrics.CloudLatency(id)) { await cp.SendMessageAsync(message); } succeeded.Add(routingMessage); Metrics.MessageCount(id); } catch (Exception ex) { if (IsRetryable(ex)) { failed.Add(routingMessage); } else { Events.InvalidMessage(ex); invalid.Add(new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.None)); } if (failed.Count > 0) { Events.RetryingMessage(routingMessage, ex); sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending messages to IotHub for device {this.cloudEndpoint.Id}")); } } }); } return(new SinkResult <IRoutingMessage>(succeeded, failed, invalid, sendFailureDetails)); }
async Task <ISinkResult> ProcessAsync(ICollection <IRoutingMessage> routingMessages, IDeviceProxy dp, CancellationToken token) { // TODO - figure out if we can use cancellation token to cancel send var succeeded = new List <IRoutingMessage>(); var failed = new List <IRoutingMessage>(); var invalid = new List <InvalidDetails <IRoutingMessage> >(); SendFailureDetails sendFailureDetails = null; foreach (IRoutingMessage routingMessage in routingMessages) { IMessage message = this.moduleEndpoint.messageConverter.ToMessage(routingMessage); try { if (failed.Count == 0) { await dp.SendMessageAsync(message, this.moduleEndpoint.Input); succeeded.Add(routingMessage); } else { // if one failed, fail the rest, so retry will keep message order failed.Add(routingMessage); } } catch (Exception ex) { if (IsRetryable(ex)) { failed.Add(routingMessage); } else { Events.InvalidMessage(ex); invalid.Add(new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.InvalidInput)); } Events.ErrorSendingMessages(this.moduleEndpoint, ex); } } if (failed.Count > 0) { Events.RetryingMessages(failed.Count, this.moduleEndpoint.Id); sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending message to module {this.moduleEndpoint.moduleId}")); } return(new SinkResult <IRoutingMessage>(succeeded, failed, invalid, sendFailureDetails)); }
public async Task <ISinkResult> ProcessAsync(IRoutingMessage routingMessage, CancellationToken token) { Preconditions.CheckNotNull(routingMessage, nameof(routingMessage)); if (token.IsCancellationRequested) { Events.CancelledProcessingMessage(routingMessage); var failed = new List <IRoutingMessage> { routingMessage }; var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubConnectionException($"Cancelled sending messages to IotHub for device {this.cloudEndpoint.Id}")); return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails)); } Util.Option <string> identity = this.GetIdentity(routingMessage); ISinkResult result = await identity.Match( id => this.ProcessAsync(routingMessage, id), () => this.ProcessNoIdentity(routingMessage)); return(result); }
ISinkResult HandleException(Exception ex, IRoutingMessage routingMessage) { if (IsRetryable(ex)) { var failed = new List <IRoutingMessage> { routingMessage }; Events.RetryingMessage(routingMessage, ex); var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, new EdgeHubIOException($"Error sending messages to IotHub for device {this.cloudEndpoint.Id}")); return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, failed, sendFailureDetails)); } else { Events.InvalidMessage(ex); var invalid = new List <InvalidDetails <IRoutingMessage> > { new InvalidDetails <IRoutingMessage>(routingMessage, FailureKind.InvalidInput) }; var sendFailureDetails = new SendFailureDetails(FailureKind.InvalidInput, ex); return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, ImmutableList <IRoutingMessage> .Empty, invalid, sendFailureDetails)); } }
public async Task <ISinkResult <T> > ProcessAsync(ICollection <T> ts, CancellationToken token) { var succeeded = new List <T>(); var failed = new List <T>(); var invalid = new List <InvalidDetails <T> >(); ICollection <T> messages = ts; ISinkResult <T> rv; ShouldRetry shouldRetry = this.retryPolicy.RetryStrategy.GetShouldRetry(); try { int i = 0; SendFailureDetails failureDetails = null; bool finished = false; while (!finished) { token.ThrowIfCancellationRequested(); ISinkResult <T> result = await this.underlying.ProcessAsync(messages, token); succeeded.AddRange(result.Succeeded); invalid.AddRange(result.InvalidDetailsList); if (result.IsSuccessful) { failureDetails = null; finished = true; } else { TimeSpan retryAfter; failureDetails = result.SendFailureDetails.OrDefault(); if (failureDetails != null && this.retryPolicy.ErrorDetectionStrategy.IsTransient(failureDetails.RawException) && shouldRetry(i, failureDetails.RawException, out retryAfter)) { // if we should retry, set the next group of messages to send // to the failed messages from the previous attempt messages = result.Failed; Preconditions.CheckRange(retryAfter.TotalMilliseconds, 0.0); await Task.Delay(retryAfter, token); i++; } else { // do not retry, return failed messages as failed failed.AddRange(result.Failed); finished = true; } } } rv = new SinkResult <T>(succeeded, failed, invalid, failureDetails); } catch (OperationCanceledException ex) { failed.AddRange(messages); rv = new SinkResult <T>(succeeded, failed, invalid, new SendFailureDetails(FailureKind.InternalError, ex)); } catch (Exception ex) { failed.AddRange(messages); rv = new SinkResult <T>(succeeded, failed, invalid, new SendFailureDetails(FailureKind.InternalError, ex)); } return(rv); }
static ISinkResult GetSyncResultForFailedMessages(Exception ex, List <IRoutingMessage> routingMessages) { var sendFailureDetails = new SendFailureDetails(FailureKind.Transient, ex); return(new SinkResult <IRoutingMessage>(ImmutableList <IRoutingMessage> .Empty, routingMessages, sendFailureDetails)); }