private static PoisonPesist CreatePoisonPersistence()
{
var pp = new PoisonPesist();
PoisonMessagePersistence.Register((_) => pp);
return(pp);
}
public async Task A120_SkipAudit()
{
var tn = GetConfig().GetValue <string>("EventHubPoisonMessageTable");
PoisonMessagePersistence.DefaultTableName = tn;
var cs = GetConfig().GetValue <string>("AzureWebJobsStorage");
var cst = await PoisonMessagePersistence.GetPoisonMessageSkippedTable(cs);
var smc = (await GetSkippedMessages(cst)).Count;
var pp = new PoisonMessagePersistence(new PoisonMessageCreatePersistenceArgs
{
Config = GetConfig(),
Context = ResilientEventHubProcessorTest.CreatePartitionContext(),
Logger = TestSetUp.CreateLogger(),
Options = ResilientEventHubProcessorTest.CreateOptions()
});
var ed = CreateEventData("200", 2);
await pp.SkipAuditAsync(ed, "Explicit audit.");
var smca = (await GetSkippedMessages(cst)).Count;
Assert.AreEqual(smc + 1, smca);
var msgs = await PoisonMessagePersistence.GetAllMessagesAsync(cst);
var msg = msgs.Last();
Assert.AreEqual("path-eventhub", msg.PartitionKey);
Assert.IsTrue(msg.RowKey.EndsWith("consumergroup-0"));
Assert.AreEqual("200", msg.Body);
Assert.AreEqual("Explicit audit.", msg.Exception);
Assert.AreEqual("200", msg.Offset);
Assert.AreEqual(2, msg.SequenceNumber);
Assert.AreEqual(false, msg.SkipMessage);
Assert.AreEqual("ns.class", msg.FunctionType);
Assert.AreEqual("testfunc", msg.FunctionName);
Assert.IsNotNull(msg.SkippedTimeUtc);
}
/// <summary>
/// Starts processing.
/// </summary>
public Task OpenAsync(PartitionContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_logger.LogInformation($"Processor starting. {GetPartitionContextLogInfo(context)}");
// Configure the retry policy for use.
_policyAsync = Policy
.HandleResult <FunctionResult>(fr => !fr.Succeeded)
.WaitAndRetryForeverAsync(
(count, ctx) =>
{
if (OverrideRetryTimespan.HasValue)
{
return(OverrideRetryTimespan.Value);
}
if (count > 16) // 2^16 is 65,536 which is the biggest allowed within our 1 day (86,400s) constraint therefore no need to calculate.
{
return(_options.MaxRetryTimespan);
}
else
{
// Use a jitterer to randomise the retrys to limit retry concurrency across the underlying threads (key for the early retries).
var ts = TimeSpan.FromSeconds(Math.Pow(2, count)) + TimeSpan.FromMilliseconds(jitterer.Next(0, 100));
return(ts < _options.MaxRetryTimespan ? ts : _options.MaxRetryTimespan);
}
},
async(dr, count, timespan, ctx) =>
{
var isPoisoned = _currPoisonAction == PoisonMessageAction.PoisonRetry || count >= _options.LogPoisonMessageAfterRetryCount;
var msg = $"Failure retry{(isPoisoned ? " (Poisoned)" : "")} in {timespan.TotalSeconds}s (attempt {count}) {GetEventDataLogInfo(context, _currEventData!)}.";
switch (count)
{
case var val when val == _options.LogPoisonMessageAfterRetryCount:
// Set the poison message now that we have (possibly) attempted enough times that it may not be transient in nature and some needs to be alerted.
await _poisonOrchestrator !.SetAsync(_currEventData !, dr.Result.Exception).ConfigureAwait(false);
_currPoisonAction = PoisonMessageAction.PoisonRetry;
_logger.LogError(dr.Result.Exception, msg);
break;
case var val when val > _options.LogPoisonMessageAfterRetryCount:
// Keep logging advising the error is still in play.
_logger.LogError(dr.Result.Exception, msg);
break;
default:
// It could be a transient error, so report as a warning until identified as poison.
if (isPoisoned)
{
_logger.LogError(dr.Result.Exception, msg);
}
else
{
_logger.LogWarning(dr.Result.Exception, msg);
}
break;
}
});
// Instantiate the poison message orchestration.
_poisonOrchestrator = PoisonMessagePersistence.Create(new PoisonMessageCreatePersistenceArgs {
Config = _config, Context = context, Logger = _logger, Options = _options
});
return(Task.CompletedTask);
}
public async Task A100_EndToEnd()
{
var tn = GetConfig().GetValue <string>("EventHubPoisonMessageTable");
PoisonMessagePersistence.DefaultTableName = tn;
var cs = GetConfig().GetValue <string>("AzureWebJobsStorage");
var ct = await PoisonMessagePersistence.GetPoisonMessageTable(cs);
var cst = await PoisonMessagePersistence.GetPoisonMessageSkippedTable(cs);
var smc = (await GetSkippedMessages(cst)).Count;
// Make sure there are no messages to begin with.
var msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct);
foreach (var m in msgs)
{
await ct.ExecuteAsync(TableOperation.Delete(m));
}
var pp = new PoisonMessagePersistence(new PoisonMessageCreatePersistenceArgs
{
Config = GetConfig(),
Context = ResilientEventHubProcessorTest.CreatePartitionContext(),
Logger = TestSetUp.CreateLogger(),
Options = ResilientEventHubProcessorTest.CreateOptions()
});
var ed = CreateEventData("100", 1);
// Checking and removing with unknown is a-ok.
Assert.AreEqual(PoisonMessageAction.NotPoison, await pp.CheckAsync(ed));
await pp.RemoveAsync(ed, PoisonMessageAction.NotPoison);
Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count);
// Add an event as poison.
await pp.SetAsync(ed, new DivideByZeroException("My bad."));
Assert.AreEqual(PoisonMessageAction.PoisonRetry, await pp.CheckAsync(ed));
Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count);
msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct);
Assert.AreEqual(1, msgs.Count());
var msg = msgs.First();
Assert.AreEqual("path-eventhub", msg.PartitionKey);
Assert.AreEqual("consumergroup-0", msg.RowKey);
Assert.AreEqual("100", msg.Body);
Assert.AreEqual("System.DivideByZeroException: My bad.", msg.Exception);
Assert.AreEqual("100", msg.Offset);
Assert.AreEqual(1, msg.SequenceNumber);
Assert.AreEqual(false, msg.SkipMessage);
Assert.AreEqual("ns.class", msg.FunctionType);
Assert.AreEqual("testfunc", msg.FunctionName);
// Update to skip.
await PoisonMessagePersistence.SkipMessageAsync(ct, msg.PartitionKey, msg.RowKey);
Assert.AreEqual(PoisonMessageAction.PoisonSkip, await pp.CheckAsync(ed));
Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count);
msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct);
Assert.AreEqual(1, msgs.Count());
msg = msgs.First();
Assert.AreEqual("path-eventhub", msg.PartitionKey);
Assert.AreEqual("consumergroup-0", msg.RowKey);
Assert.AreEqual("100", msg.Body);
Assert.AreEqual("System.DivideByZeroException: My bad.", msg.Exception);
Assert.AreEqual("100", msg.Offset);
Assert.AreEqual(1, msg.SequenceNumber);
Assert.AreEqual(true, msg.SkipMessage);
Assert.AreEqual("ns.class", msg.FunctionType);
Assert.AreEqual("testfunc", msg.FunctionName);
Assert.IsNull(msg.SkippedTimeUtc);
// Remove the poison as no longer poison.
await pp.RemoveAsync(ed, PoisonMessageAction.NotPoison);
Assert.AreEqual(PoisonMessageAction.NotPoison, await pp.CheckAsync(ed));
Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count);
msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct);
Assert.AreEqual(0, msgs.Count());
// Create a new poison message.
ed = CreateEventData("200", 2);
await pp.SetAsync(ed, new DivideByZeroException("My bad."));
Assert.AreEqual(PoisonMessageAction.PoisonRetry, await pp.CheckAsync(ed));
Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count);
msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct);
Assert.AreEqual(1, msgs.Count());
msg = msgs.First();
// Remove the poison as skipped (poison).
await pp.RemoveAsync(ed, PoisonMessageAction.PoisonSkip);
Assert.AreEqual(PoisonMessageAction.NotPoison, await pp.CheckAsync(ed));
var sms = await GetSkippedMessages(cst);
Assert.AreEqual(smc + 1, sms.Count);
var sm = sms.Where(pm => pm.PartitionKey == msg.PartitionKey && pm.RowKey.EndsWith(msg.RowKey)).OrderByDescending(pm => pm.RowKey).FirstOrDefault();
Assert.IsNotNull(sm.SkippedTimeUtc);
}
