private static PoisonPesist CreatePoisonPersistence() { var pp = new PoisonPesist(); PoisonMessagePersistence.Register((_) => pp); return(pp); }
public async Task A120_SkipAudit() { var tn = GetConfig().GetValue <string>("EventHubPoisonMessageTable"); PoisonMessagePersistence.DefaultTableName = tn; var cs = GetConfig().GetValue <string>("AzureWebJobsStorage"); var cst = await PoisonMessagePersistence.GetPoisonMessageSkippedTable(cs); var smc = (await GetSkippedMessages(cst)).Count; var pp = new PoisonMessagePersistence(new PoisonMessageCreatePersistenceArgs { Config = GetConfig(), Context = ResilientEventHubProcessorTest.CreatePartitionContext(), Logger = TestSetUp.CreateLogger(), Options = ResilientEventHubProcessorTest.CreateOptions() }); var ed = CreateEventData("200", 2); await pp.SkipAuditAsync(ed, "Explicit audit."); var smca = (await GetSkippedMessages(cst)).Count; Assert.AreEqual(smc + 1, smca); var msgs = await PoisonMessagePersistence.GetAllMessagesAsync(cst); var msg = msgs.Last(); Assert.AreEqual("path-eventhub", msg.PartitionKey); Assert.IsTrue(msg.RowKey.EndsWith("consumergroup-0")); Assert.AreEqual("200", msg.Body); Assert.AreEqual("Explicit audit.", msg.Exception); Assert.AreEqual("200", msg.Offset); Assert.AreEqual(2, msg.SequenceNumber); Assert.AreEqual(false, msg.SkipMessage); Assert.AreEqual("ns.class", msg.FunctionType); Assert.AreEqual("testfunc", msg.FunctionName); Assert.IsNotNull(msg.SkippedTimeUtc); }
/// <summary> /// Starts processing. /// </summary> public Task OpenAsync(PartitionContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } _logger.LogInformation($"Processor starting. {GetPartitionContextLogInfo(context)}"); // Configure the retry policy for use. _policyAsync = Policy .HandleResult <FunctionResult>(fr => !fr.Succeeded) .WaitAndRetryForeverAsync( (count, ctx) => { if (OverrideRetryTimespan.HasValue) { return(OverrideRetryTimespan.Value); } if (count > 16) // 2^16 is 65,536 which is the biggest allowed within our 1 day (86,400s) constraint therefore no need to calculate. { return(_options.MaxRetryTimespan); } else { // Use a jitterer to randomise the retrys to limit retry concurrency across the underlying threads (key for the early retries). var ts = TimeSpan.FromSeconds(Math.Pow(2, count)) + TimeSpan.FromMilliseconds(jitterer.Next(0, 100)); return(ts < _options.MaxRetryTimespan ? ts : _options.MaxRetryTimespan); } }, async(dr, count, timespan, ctx) => { var isPoisoned = _currPoisonAction == PoisonMessageAction.PoisonRetry || count >= _options.LogPoisonMessageAfterRetryCount; var msg = $"Failure retry{(isPoisoned ? " (Poisoned)" : "")} in {timespan.TotalSeconds}s (attempt {count}) {GetEventDataLogInfo(context, _currEventData!)}."; switch (count) { case var val when val == _options.LogPoisonMessageAfterRetryCount: // Set the poison message now that we have (possibly) attempted enough times that it may not be transient in nature and some needs to be alerted. await _poisonOrchestrator !.SetAsync(_currEventData !, dr.Result.Exception).ConfigureAwait(false); _currPoisonAction = PoisonMessageAction.PoisonRetry; _logger.LogError(dr.Result.Exception, msg); break; case var val when val > _options.LogPoisonMessageAfterRetryCount: // Keep logging advising the error is still in play. _logger.LogError(dr.Result.Exception, msg); break; default: // It could be a transient error, so report as a warning until identified as poison. if (isPoisoned) { _logger.LogError(dr.Result.Exception, msg); } else { _logger.LogWarning(dr.Result.Exception, msg); } break; } }); // Instantiate the poison message orchestration. _poisonOrchestrator = PoisonMessagePersistence.Create(new PoisonMessageCreatePersistenceArgs { Config = _config, Context = context, Logger = _logger, Options = _options }); return(Task.CompletedTask); }
public async Task A100_EndToEnd() { var tn = GetConfig().GetValue <string>("EventHubPoisonMessageTable"); PoisonMessagePersistence.DefaultTableName = tn; var cs = GetConfig().GetValue <string>("AzureWebJobsStorage"); var ct = await PoisonMessagePersistence.GetPoisonMessageTable(cs); var cst = await PoisonMessagePersistence.GetPoisonMessageSkippedTable(cs); var smc = (await GetSkippedMessages(cst)).Count; // Make sure there are no messages to begin with. var msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct); foreach (var m in msgs) { await ct.ExecuteAsync(TableOperation.Delete(m)); } var pp = new PoisonMessagePersistence(new PoisonMessageCreatePersistenceArgs { Config = GetConfig(), Context = ResilientEventHubProcessorTest.CreatePartitionContext(), Logger = TestSetUp.CreateLogger(), Options = ResilientEventHubProcessorTest.CreateOptions() }); var ed = CreateEventData("100", 1); // Checking and removing with unknown is a-ok. Assert.AreEqual(PoisonMessageAction.NotPoison, await pp.CheckAsync(ed)); await pp.RemoveAsync(ed, PoisonMessageAction.NotPoison); Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count); // Add an event as poison. await pp.SetAsync(ed, new DivideByZeroException("My bad.")); Assert.AreEqual(PoisonMessageAction.PoisonRetry, await pp.CheckAsync(ed)); Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count); msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct); Assert.AreEqual(1, msgs.Count()); var msg = msgs.First(); Assert.AreEqual("path-eventhub", msg.PartitionKey); Assert.AreEqual("consumergroup-0", msg.RowKey); Assert.AreEqual("100", msg.Body); Assert.AreEqual("System.DivideByZeroException: My bad.", msg.Exception); Assert.AreEqual("100", msg.Offset); Assert.AreEqual(1, msg.SequenceNumber); Assert.AreEqual(false, msg.SkipMessage); Assert.AreEqual("ns.class", msg.FunctionType); Assert.AreEqual("testfunc", msg.FunctionName); // Update to skip. await PoisonMessagePersistence.SkipMessageAsync(ct, msg.PartitionKey, msg.RowKey); Assert.AreEqual(PoisonMessageAction.PoisonSkip, await pp.CheckAsync(ed)); Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count); msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct); Assert.AreEqual(1, msgs.Count()); msg = msgs.First(); Assert.AreEqual("path-eventhub", msg.PartitionKey); Assert.AreEqual("consumergroup-0", msg.RowKey); Assert.AreEqual("100", msg.Body); Assert.AreEqual("System.DivideByZeroException: My bad.", msg.Exception); Assert.AreEqual("100", msg.Offset); Assert.AreEqual(1, msg.SequenceNumber); Assert.AreEqual(true, msg.SkipMessage); Assert.AreEqual("ns.class", msg.FunctionType); Assert.AreEqual("testfunc", msg.FunctionName); Assert.IsNull(msg.SkippedTimeUtc); // Remove the poison as no longer poison. await pp.RemoveAsync(ed, PoisonMessageAction.NotPoison); Assert.AreEqual(PoisonMessageAction.NotPoison, await pp.CheckAsync(ed)); Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count); msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct); Assert.AreEqual(0, msgs.Count()); // Create a new poison message. ed = CreateEventData("200", 2); await pp.SetAsync(ed, new DivideByZeroException("My bad.")); Assert.AreEqual(PoisonMessageAction.PoisonRetry, await pp.CheckAsync(ed)); Assert.AreEqual(smc, (await GetSkippedMessages(cst)).Count); msgs = await PoisonMessagePersistence.GetAllMessagesAsync(ct); Assert.AreEqual(1, msgs.Count()); msg = msgs.First(); // Remove the poison as skipped (poison). await pp.RemoveAsync(ed, PoisonMessageAction.PoisonSkip); Assert.AreEqual(PoisonMessageAction.NotPoison, await pp.CheckAsync(ed)); var sms = await GetSkippedMessages(cst); Assert.AreEqual(smc + 1, sms.Count); var sm = sms.Where(pm => pm.PartitionKey == msg.PartitionKey && pm.RowKey.EndsWith(msg.RowKey)).OrderByDescending(pm => pm.RowKey).FirstOrDefault(); Assert.IsNotNull(sm.SkippedTimeUtc); }