/// <summary>
/// A previously identified poisoned <see cref="EventHubs.EventData"/> has either successfully processed or can be skipped and should be removed.
/// </summary>
/// <param name="event">The corresponding <see cref="EventHubs.EventData"/>.</param>
/// <param name="action">The corresponding reason (<see cref="PoisonMessageAction"/>) for removal.</param>
/// <remarks>The corresponding <see cref="PoisonMessage"/> will be removed/deleted from Azure table storage.</remarks>
public async Task RemoveAsync(EventHubs.EventData @event, PoisonMessageAction action)
{
if (@event == null)
{
throw new ArgumentNullException(nameof(@event));
}
var msg = await GetPoisonMessageAsync().ConfigureAwait(false);
if (msg == null)
{
return;
}
// Audit the skipped record.
if (action == PoisonMessageAction.PoisonSkip)
{
msg.SkippedTimeUtc = DateTime.UtcNow;
msg.RowKey = msg.SkippedTimeUtc.Value.ToString("o", System.Globalization.CultureInfo.InvariantCulture) + "-" + msg.RowKey;
await _skippedTable.ExecuteAsync(TableOperation.InsertOrReplace(msg)).ConfigureAwait(false);
}
// Remove.
await _poisonTable.ExecuteAsync(TableOperation.Delete(new PoisonMessage(_storagePartitionKey, _storageRowKey)
{
ETag = "*"
})).ConfigureAwait(false);
}
public void SetResult(PoisonMessageAction pma)
{
lock (_lock)
{
Result = pma;
}
}
public Task RemoveAsync(EventHubs.EventData @event, PoisonMessageAction action)
{
lock (_lock)
{
RemovedEvents.Add(@event);
Result = PoisonMessageAction.NotPoison;
return(Task.CompletedTask);
}
}
/// <summary>
/// Executes the "current" event.
/// </summary>
private async Task <FunctionResult> ExecuteCurrentEvent(PartitionContext context, CancellationToken ct)
{
// Where the poison action state is unknown or retry then check to see what the current state is; if skip, then bypass current.
if (_currPoisonAction == PoisonMessageAction.Undetermined || _currPoisonAction == PoisonMessageAction.PoisonRetry)
{
_currPoisonAction = await _poisonOrchestrator.CheckAsync(_currEventData).ConfigureAwait(false);
if (_currPoisonAction == PoisonMessageAction.PoisonSkip)
{
await _poisonOrchestrator.RemoveAsync(_currEventData, PoisonMessageAction.PoisonSkip).ConfigureAwait(false);
_logger.LogWarning($"EventData that was previously identified as Poison is being skipped (not processed) {GetEventDataLogInfo(context, _currEventData)}.");
_currPoisonAction = PoisonMessageAction.NotPoison;
return(new FunctionResult(true));
}
else if (_currPoisonAction == PoisonMessageAction.Undetermined)
{
throw new InvalidOperationException("The IPoisonMessageOrchestrator.CheckAsync must not return PoisonMessageAction.Undetermined.");
}
}
// Execute the function (maybe again if previously failed).
var data = new TriggeredFunctionData {
TriggerValue = new ResilientEventHubData {
EventData = _currEventData
}
};
var fr = await _executor.TryExecuteAsync(data, ct).ConfigureAwait(false);
// Where we have a failure then checkpoint the last so we will at least restart back at this point.
if (fr.Succeeded)
{
if (_currPoisonAction != PoisonMessageAction.NotPoison)
{
await _poisonOrchestrator.RemoveAsync(_currEventData, PoisonMessageAction.NotPoison).ConfigureAwait(false);
_currPoisonAction = PoisonMessageAction.NotPoison;
}
}
else
{
await CheckpointAsync(context, _lastEventData).ConfigureAwait(false);
}
return(fr);
}
/// <summary>
/// Starts processing.
/// </summary>
public Task OpenAsync(PartitionContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_logger.LogInformation($"Processor starting. {GetPartitionContextLogInfo(context)}");
// Configure the retry policy for use.
_policyAsync = Policy
.HandleResult <FunctionResult>(fr => !fr.Succeeded)
.WaitAndRetryForeverAsync(
(count, ctx) =>
{
if (OverrideRetryTimespan.HasValue)
{
return(OverrideRetryTimespan.Value);
}
if (count > 16) // 2^16 is 65,536 which is the biggest allowed within our 1 day (86,400s) constraint therefore no need to calculate.
{
return(_options.MaxRetryTimespan);
}
else
{
// Use a jitterer to randomise the retrys to limit retry concurrency across the underlying threads (key for the early retries).
var ts = TimeSpan.FromSeconds(Math.Pow(2, count)) + TimeSpan.FromMilliseconds(jitterer.Next(0, 100));
return(ts < _options.MaxRetryTimespan ? ts : _options.MaxRetryTimespan);
}
},
async(dr, count, timespan, ctx) =>
{
var isPoisoned = _currPoisonAction == PoisonMessageAction.PoisonRetry || count >= _options.LogPoisonMessageAfterRetryCount;
var msg = $"Failure retry{(isPoisoned ? " (Poisoned)" : "")} in {timespan.TotalSeconds}s (attempt {count}) {GetEventDataLogInfo(context, _currEventData)}.";
switch (count)
{
case var val when val == _options.LogPoisonMessageAfterRetryCount:
// Set the poison message now that we have (possibly) attempted enough times that it may not be transient in nature and some needs to be alerted.
await _poisonOrchestrator.SetAsync(_currEventData, dr.Result.Exception).ConfigureAwait(false);
_currPoisonAction = PoisonMessageAction.PoisonRetry;
_logger.LogError(dr.Result.Exception, msg);
break;
case var val when val > _options.LogPoisonMessageAfterRetryCount:
// Keep logging advising the error is still in play.
_logger.LogError(dr.Result.Exception, msg);
break;
default:
// It could be a transient error, so report as a warning until identified as poison.
if (isPoisoned)
{
_logger.LogError(dr.Result.Exception, msg);
}
else
{
_logger.LogWarning(dr.Result.Exception, msg);
}
break;
}
});
// Instantiate the poison message orchestration.
_poisonOrchestrator = PoisonMessagePersistence.Create(new PoisonMessageCreatePersistenceArgs {
Config = _config, Context = context, Logger = _logger, Options = _options
});
return(Task.CompletedTask);
}