public override void Configure(IFunctionsHostBuilder builder) { var config = builder.GetBeefConfiguration <Startup>("Beef_"); // Add the core beef services. builder.Services.AddBeefExecutionContext() .AddBeefSystemTime() .AddBeefRequestCache() .AddBeefCachePolicyManager(config.GetSection("BeefCaching").Get <CachePolicyConfig>(), new System.TimeSpan(0, 0, 30), new System.TimeSpan(0, 0, 30), useCachePolicyManagerTimer: true) .AddBeefBusinessServices(); // Add event subscriber host with auto-discovered subscribers and set the audit writer to use azure storage; plus use the poison event orchestrator/invoker. var ehasr = new EventHubAzureStorageRepository(config.GetConnectionString("AzureStorage")); builder.Services.AddBeefEventHubConsumerHost( EventSubscriberHostArgs.Create <Startup>().UseAuditWriter(ehasr).UseMaxAttempts(10), additional: (_, ehsh) => ehsh.UseInvoker(new EventHubConsumerHostPoisonInvoker(ehasr))); var sbasr = new ServiceBusAzureStorageRepository(config.GetConnectionString("AzureStorage")); builder.Services.AddBeefServiceBusReceiverHost( EventSubscriberHostArgs.Create <Startup>().UseAuditWriter(sbasr).UseMaxAttempts(10), additional: (_, ehsh) => ehsh.UseInvoker(new ServiceBusReceiverHostPoisonInvoker(sbasr))); // Add the data sources as singletons for dependency injection requirements. var ccs = config.GetSection("CosmosDb"); builder.Services.AddScoped <Data.Database.IDatabase>(_ => new Database(config.GetConnectionString("BeefDemo"))) .AddDbContext <EfDbContext>() .AddScoped <Data.EntityFrameworkCore.IEfDb, EfDb>() .AddSingleton <Data.Cosmos.ICosmosDb>(_ => new CosmosDb(new Cosmos.CosmosClient(ccs.GetValue <string>("EndPoint"), ccs.GetValue <string>("AuthKey")), ccs.GetValue <string>("Database"))); // Add the generated reference data services for dependency injection requirements. builder.Services.AddGeneratedReferenceDataManagerServices() .AddGeneratedReferenceDataDataSvcServices() .AddGeneratedReferenceDataDataServices(); // Add the generated entity services for dependency injection requirements. builder.Services.AddGeneratedManagerServices() .AddGeneratedValidationServices() .AddGeneratedDataSvcServices() .AddGeneratedDataServices(); // Add identifier generator services. builder.Services.AddSingleton <IGuidIdentifierGenerator, GuidIdentifierGenerator>() .AddSingleton <IStringIdentifierGenerator, StringIdentifierGenerator>(); // Add event publishing. builder.Services.AddBeefEventHubEventProducer(new EventHubProducerClient(config.GetValue <string>("EventHubConnectionString"))); // Add the AutoMapper profiles. builder.Services.AddAutoMapper(Mapper.AutoMapperProfile.Assembly, typeof(ContactData).Assembly); // Add logging. builder.Services.AddLogging(); }
public async Task A100_EndToEnd() { var cfg = GetConfig(); var lgr = TestSetUp.CreateLogger(); var asr = new EventHubAzureStorageRepository(cfg.GetWebJobsConnectionString(ConnectionStringNames.Storage)) { PoisonTableName = cfg.GetValue <string>("EventHubPoisonMessagesTable"), AuditTableName = cfg.GetValue <string>("EventHubAuditMessagesTable") }; ((IUseLogger)asr).UseLogger(lgr); // Make sure there are no AuditRecords to begin with. var pmt = await asr.GetPoisonMessageTableAsync().ConfigureAwait(false); await DeleteAuditRecords(pmt).ConfigureAwait(false); var amt = await asr.GetAuditMessageTableAsync().ConfigureAwait(false); await DeleteAuditRecords(amt).ConfigureAwait(false); var ed = await CreateEventDataAsync(100, 1); // Checking and removing with unknown is a-ok. var ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.NotPoison, ar.Action); Assert.AreEqual(0, ar.Attempts); await asr.RemovePoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(0, (await GetAuditRecords(pmt)).Count); Assert.AreEqual(0, (await GetAuditRecords(amt)).Count); // Add an event as poison. await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found."))).ConfigureAwait(false); ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action); Assert.AreEqual(1, ar.Attempts); Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count); Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count); var ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First(); Assert.AreEqual("testhub-$Default", ear.PartitionKey); Assert.AreEqual("0", ear.RowKey); Assert.NotNull(ear.Body); Assert.NotNull(ear.PoisonedTimeUtc); Assert.IsNull(ear.SkippedTimeUtc); Assert.AreEqual("DataNotFound", ear.Status); Assert.AreEqual("Data not found.", ear.Reason); Assert.IsNull(ear.OriginatingStatus); Assert.IsNull(ear.OriginatingReason); Assert.AreEqual(100, ear.Offset); Assert.AreEqual(1, ear.SequenceNumber); Assert.AreEqual(false, ear.SkipProcessing); Assert.AreEqual(1, ear.Attempts); // Update again an event as poison. await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.InvalidData("Bad data."))).ConfigureAwait(false); ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action); Assert.AreEqual(2, ar.Attempts); Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count); Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count); ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First(); Assert.AreEqual("testhub-$Default", ear.PartitionKey); Assert.AreEqual("0", ear.RowKey); Assert.NotNull(ear.Body); Assert.NotNull(ear.PoisonedTimeUtc); Assert.IsNull(ear.SkippedTimeUtc); Assert.AreEqual("InvalidData", ear.Status); Assert.AreEqual("Bad data.", ear.Reason); Assert.IsNull(ear.OriginatingStatus); Assert.IsNull(ear.OriginatingReason); Assert.AreEqual(100, ear.Offset); Assert.AreEqual(1, ear.SequenceNumber); Assert.AreEqual(false, ear.SkipProcessing); Assert.AreEqual(2, ear.Attempts); // Update to skip. await asr.SkipPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count); Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count); ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First(); Assert.AreEqual("testhub-$Default", ear.PartitionKey); Assert.AreEqual("0", ear.RowKey); Assert.NotNull(ear.Body); Assert.NotNull(ear.PoisonedTimeUtc); Assert.IsNull(ear.SkippedTimeUtc); Assert.AreEqual("InvalidData", ear.Status); Assert.AreEqual("Bad data.", ear.Reason); Assert.IsNull(ear.OriginatingStatus); Assert.IsNull(ear.OriginatingReason); Assert.AreEqual(100, ear.Offset); Assert.AreEqual(1, ear.SequenceNumber); Assert.AreEqual(true, ear.SkipProcessing); Assert.AreEqual(2, ear.Attempts); // Check poisoned which will remove and audit. ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.PoisonSkip, ar.Action); Assert.AreEqual(2, ar.Attempts); Assert.AreEqual(0, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count); Assert.AreEqual(1, (await GetAuditRecords(amt).ConfigureAwait(false)).Count); ear = (await GetAuditRecords(amt).ConfigureAwait(false)).Last(); Assert.AreEqual("testhub-$Default", ear.PartitionKey); Assert.IsTrue(ear.RowKey.EndsWith("-0")); Assert.NotNull(ear.Body); Assert.NotNull(ear.PoisonedTimeUtc); Assert.NotNull(ear.SkippedTimeUtc); Assert.AreEqual("PoisonSkipped", ear.Status); Assert.AreEqual("EventData was identified as Poison and was marked as SkipMessage; this event is skipped (i.e. not processed).", ear.Reason); Assert.AreEqual("InvalidData", ear.OriginatingStatus); Assert.AreEqual("Bad data.", ear.OriginatingReason); Assert.AreEqual(100, ear.Offset); Assert.AreEqual(1, ear.SequenceNumber); Assert.AreEqual(true, ear.SkipProcessing); Assert.AreEqual(2, ear.Attempts); }
public async Task A120_PoisonMismatch() { var cfg = GetConfig(); var lgr = TestSetUp.CreateLogger(); var asr = new EventHubAzureStorageRepository(cfg.GetWebJobsConnectionString(ConnectionStringNames.Storage)) { PoisonTableName = cfg.GetValue <string>("EventHubPoisonMessagesTable"), AuditTableName = cfg.GetValue <string>("EventHubAuditMessagesTable") }; ((IUseLogger)asr).UseLogger(lgr); // Make sure there are no AuditRecords to begin with. var pmt = await asr.GetPoisonMessageTableAsync().ConfigureAwait(false); await DeleteAuditRecords(pmt).ConfigureAwait(false); var amt = await asr.GetAuditMessageTableAsync().ConfigureAwait(false); await DeleteAuditRecords(amt).ConfigureAwait(false); var ed = await CreateEventDataAsync(100, 1); // Add an event as poison. await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found."))).ConfigureAwait(false); var ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action); Assert.AreEqual(1, ar.Attempts); Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count); Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count); var ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First(); Assert.AreEqual("testhub-$Default", ear.PartitionKey); Assert.AreEqual("0", ear.RowKey); Assert.NotNull(ear.Body); Assert.NotNull(ear.PoisonedTimeUtc); Assert.IsNull(ear.SkippedTimeUtc); Assert.AreEqual("DataNotFound", ear.Status); Assert.AreEqual("Data not found.", ear.Reason); Assert.IsNull(ear.OriginatingStatus); Assert.IsNull(ear.OriginatingReason); Assert.AreEqual(100, ear.Offset); Assert.AreEqual(1, ear.SequenceNumber); Assert.AreEqual(false, ear.SkipProcessing); Assert.AreEqual(1, ear.Attempts); // Pretend to check a different event to that poisoned. ed = await CreateEventDataAsync(200, 2); ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.NotPoison, ar.Action); Assert.AreEqual(0, ar.Attempts); Assert.AreEqual(0, (await GetAuditRecords(pmt)).Count); Assert.AreEqual(1, (await GetAuditRecords(amt)).Count); ear = (await GetAuditRecords(amt).ConfigureAwait(false)).First(); Assert.AreEqual("testhub-$Default", ear.PartitionKey); Assert.IsTrue(ear.RowKey.EndsWith("-0")); Assert.NotNull(ear.Body); Assert.NotNull(ear.PoisonedTimeUtc); Assert.IsNull(ear.SkippedTimeUtc); Assert.AreEqual("PoisonMismatch", ear.Status); Assert.AreEqual("Current EventData (Seq#: '2' Offset#: '200') being processed is out of sync with previous Poison (Seq#: '1' Offset#: '100'); current assumed correct with previous Poison now deleted.", ear.Reason); Assert.AreEqual("DataNotFound", ear.OriginatingStatus); Assert.AreEqual("Data not found.", ear.OriginatingReason); Assert.AreEqual(100, ear.Offset); Assert.AreEqual(1, ear.SequenceNumber); Assert.AreEqual(false, ear.SkipProcessing); Assert.AreEqual(1, ear.Attempts); await Task.CompletedTask; }
public async Task A110_PoisonMaxAttempts() { var cfg = GetConfig(); var lgr = TestSetUp.CreateLogger(); var asr = new EventHubAzureStorageRepository(cfg.GetWebJobsConnectionString(ConnectionStringNames.Storage)) { PoisonTableName = cfg.GetValue <string>("EventHubPoisonMessagesTable"), AuditTableName = cfg.GetValue <string>("EventHubAuditMessagesTable") }; ((IUseLogger)asr).UseLogger(lgr); // Make sure there are no AuditRecords to begin with. var pmt = await asr.GetPoisonMessageTableAsync().ConfigureAwait(false); await DeleteAuditRecords(pmt).ConfigureAwait(false); var amt = await asr.GetAuditMessageTableAsync().ConfigureAwait(false); await DeleteAuditRecords(amt).ConfigureAwait(false); var ed = await CreateEventDataAsync(100, 1); // Add events as poison. await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found.")), 3).ConfigureAwait(false); var ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action); Assert.AreEqual(1, ar.Attempts); await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found.")), 3).ConfigureAwait(false); ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action); Assert.AreEqual(2, ar.Attempts); await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found.")), 3).ConfigureAwait(false); ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false); Assert.AreEqual(PoisonMessageAction.NotPoison, ar.Action); Assert.AreEqual(0, ar.Attempts); var ear = (await GetAuditRecords(amt).ConfigureAwait(false)).Last(); Assert.AreEqual("testhub-$Default", ear.PartitionKey); Assert.IsTrue(ear.RowKey.EndsWith("-0")); Assert.NotNull(ear.Body); Assert.NotNull(ear.PoisonedTimeUtc); Assert.NotNull(ear.SkippedTimeUtc); Assert.AreEqual("PoisonMaxAttempts", ear.Status); Assert.AreEqual("EventData was identified as Poison and has been configured to automatically SkipMessage after 3 attempts; this event is skipped (i.e. not processed).", ear.Reason); Assert.AreEqual("DataNotFound", ear.OriginatingStatus); Assert.AreEqual("Data not found.", ear.OriginatingReason); Assert.AreEqual(100, ear.Offset); Assert.AreEqual(1, ear.SequenceNumber); Assert.AreEqual(false, ear.SkipProcessing); Assert.AreEqual(3, ear.Attempts); }