public override void Configure(IFunctionsHostBuilder builder)
{
var config = builder.GetBeefConfiguration <Startup>("Beef_");
// Add the core beef services.
builder.Services.AddBeefExecutionContext()
.AddBeefSystemTime()
.AddBeefRequestCache()
.AddBeefCachePolicyManager(config.GetSection("BeefCaching").Get <CachePolicyConfig>(), new System.TimeSpan(0, 0, 30), new System.TimeSpan(0, 0, 30), useCachePolicyManagerTimer: true)
.AddBeefBusinessServices();
// Add event subscriber host with auto-discovered subscribers and set the audit writer to use azure storage; plus use the poison event orchestrator/invoker.
var ehasr = new EventHubAzureStorageRepository(config.GetConnectionString("AzureStorage"));
builder.Services.AddBeefEventHubConsumerHost(
EventSubscriberHostArgs.Create <Startup>().UseAuditWriter(ehasr).UseMaxAttempts(10), additional: (_, ehsh) => ehsh.UseInvoker(new EventHubConsumerHostPoisonInvoker(ehasr)));
var sbasr = new ServiceBusAzureStorageRepository(config.GetConnectionString("AzureStorage"));
builder.Services.AddBeefServiceBusReceiverHost(
EventSubscriberHostArgs.Create <Startup>().UseAuditWriter(sbasr).UseMaxAttempts(10), additional: (_, ehsh) => ehsh.UseInvoker(new ServiceBusReceiverHostPoisonInvoker(sbasr)));
// Add the data sources as singletons for dependency injection requirements.
var ccs = config.GetSection("CosmosDb");
builder.Services.AddScoped <Data.Database.IDatabase>(_ => new Database(config.GetConnectionString("BeefDemo")))
.AddDbContext <EfDbContext>()
.AddScoped <Data.EntityFrameworkCore.IEfDb, EfDb>()
.AddSingleton <Data.Cosmos.ICosmosDb>(_ => new CosmosDb(new Cosmos.CosmosClient(ccs.GetValue <string>("EndPoint"), ccs.GetValue <string>("AuthKey")), ccs.GetValue <string>("Database")));
// Add the generated reference data services for dependency injection requirements.
builder.Services.AddGeneratedReferenceDataManagerServices()
.AddGeneratedReferenceDataDataSvcServices()
.AddGeneratedReferenceDataDataServices();
// Add the generated entity services for dependency injection requirements.
builder.Services.AddGeneratedManagerServices()
.AddGeneratedValidationServices()
.AddGeneratedDataSvcServices()
.AddGeneratedDataServices();
// Add identifier generator services.
builder.Services.AddSingleton <IGuidIdentifierGenerator, GuidIdentifierGenerator>()
.AddSingleton <IStringIdentifierGenerator, StringIdentifierGenerator>();
// Add event publishing.
builder.Services.AddBeefEventHubEventProducer(new EventHubProducerClient(config.GetValue <string>("EventHubConnectionString")));
// Add the AutoMapper profiles.
builder.Services.AddAutoMapper(Mapper.AutoMapperProfile.Assembly, typeof(ContactData).Assembly);
// Add logging.
builder.Services.AddLogging();
}
public async Task A100_EndToEnd()
{
var cfg = GetConfig();
var lgr = TestSetUp.CreateLogger();
var asr = new EventHubAzureStorageRepository(cfg.GetWebJobsConnectionString(ConnectionStringNames.Storage))
{
PoisonTableName = cfg.GetValue <string>("EventHubPoisonMessagesTable"),
AuditTableName = cfg.GetValue <string>("EventHubAuditMessagesTable")
};
((IUseLogger)asr).UseLogger(lgr);
// Make sure there are no AuditRecords to begin with.
var pmt = await asr.GetPoisonMessageTableAsync().ConfigureAwait(false);
await DeleteAuditRecords(pmt).ConfigureAwait(false);
var amt = await asr.GetAuditMessageTableAsync().ConfigureAwait(false);
await DeleteAuditRecords(amt).ConfigureAwait(false);
var ed = await CreateEventDataAsync(100, 1);
// Checking and removing with unknown is a-ok.
var ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.NotPoison, ar.Action);
Assert.AreEqual(0, ar.Attempts);
await asr.RemovePoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(0, (await GetAuditRecords(pmt)).Count);
Assert.AreEqual(0, (await GetAuditRecords(amt)).Count);
// Add an event as poison.
await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found."))).ConfigureAwait(false);
ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action);
Assert.AreEqual(1, ar.Attempts);
Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count);
Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count);
var ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First();
Assert.AreEqual("testhub-$Default", ear.PartitionKey);
Assert.AreEqual("0", ear.RowKey);
Assert.NotNull(ear.Body);
Assert.NotNull(ear.PoisonedTimeUtc);
Assert.IsNull(ear.SkippedTimeUtc);
Assert.AreEqual("DataNotFound", ear.Status);
Assert.AreEqual("Data not found.", ear.Reason);
Assert.IsNull(ear.OriginatingStatus);
Assert.IsNull(ear.OriginatingReason);
Assert.AreEqual(100, ear.Offset);
Assert.AreEqual(1, ear.SequenceNumber);
Assert.AreEqual(false, ear.SkipProcessing);
Assert.AreEqual(1, ear.Attempts);
// Update again an event as poison.
await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.InvalidData("Bad data."))).ConfigureAwait(false);
ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action);
Assert.AreEqual(2, ar.Attempts);
Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count);
Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count);
ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First();
Assert.AreEqual("testhub-$Default", ear.PartitionKey);
Assert.AreEqual("0", ear.RowKey);
Assert.NotNull(ear.Body);
Assert.NotNull(ear.PoisonedTimeUtc);
Assert.IsNull(ear.SkippedTimeUtc);
Assert.AreEqual("InvalidData", ear.Status);
Assert.AreEqual("Bad data.", ear.Reason);
Assert.IsNull(ear.OriginatingStatus);
Assert.IsNull(ear.OriginatingReason);
Assert.AreEqual(100, ear.Offset);
Assert.AreEqual(1, ear.SequenceNumber);
Assert.AreEqual(false, ear.SkipProcessing);
Assert.AreEqual(2, ear.Attempts);
// Update to skip.
await asr.SkipPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count);
Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count);
ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First();
Assert.AreEqual("testhub-$Default", ear.PartitionKey);
Assert.AreEqual("0", ear.RowKey);
Assert.NotNull(ear.Body);
Assert.NotNull(ear.PoisonedTimeUtc);
Assert.IsNull(ear.SkippedTimeUtc);
Assert.AreEqual("InvalidData", ear.Status);
Assert.AreEqual("Bad data.", ear.Reason);
Assert.IsNull(ear.OriginatingStatus);
Assert.IsNull(ear.OriginatingReason);
Assert.AreEqual(100, ear.Offset);
Assert.AreEqual(1, ear.SequenceNumber);
Assert.AreEqual(true, ear.SkipProcessing);
Assert.AreEqual(2, ear.Attempts);
// Check poisoned which will remove and audit.
ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.PoisonSkip, ar.Action);
Assert.AreEqual(2, ar.Attempts);
Assert.AreEqual(0, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count);
Assert.AreEqual(1, (await GetAuditRecords(amt).ConfigureAwait(false)).Count);
ear = (await GetAuditRecords(amt).ConfigureAwait(false)).Last();
Assert.AreEqual("testhub-$Default", ear.PartitionKey);
Assert.IsTrue(ear.RowKey.EndsWith("-0"));
Assert.NotNull(ear.Body);
Assert.NotNull(ear.PoisonedTimeUtc);
Assert.NotNull(ear.SkippedTimeUtc);
Assert.AreEqual("PoisonSkipped", ear.Status);
Assert.AreEqual("EventData was identified as Poison and was marked as SkipMessage; this event is skipped (i.e. not processed).", ear.Reason);
Assert.AreEqual("InvalidData", ear.OriginatingStatus);
Assert.AreEqual("Bad data.", ear.OriginatingReason);
Assert.AreEqual(100, ear.Offset);
Assert.AreEqual(1, ear.SequenceNumber);
Assert.AreEqual(true, ear.SkipProcessing);
Assert.AreEqual(2, ear.Attempts);
}
public async Task A120_PoisonMismatch()
{
var cfg = GetConfig();
var lgr = TestSetUp.CreateLogger();
var asr = new EventHubAzureStorageRepository(cfg.GetWebJobsConnectionString(ConnectionStringNames.Storage))
{
PoisonTableName = cfg.GetValue <string>("EventHubPoisonMessagesTable"),
AuditTableName = cfg.GetValue <string>("EventHubAuditMessagesTable")
};
((IUseLogger)asr).UseLogger(lgr);
// Make sure there are no AuditRecords to begin with.
var pmt = await asr.GetPoisonMessageTableAsync().ConfigureAwait(false);
await DeleteAuditRecords(pmt).ConfigureAwait(false);
var amt = await asr.GetAuditMessageTableAsync().ConfigureAwait(false);
await DeleteAuditRecords(amt).ConfigureAwait(false);
var ed = await CreateEventDataAsync(100, 1);
// Add an event as poison.
await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found."))).ConfigureAwait(false);
var ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action);
Assert.AreEqual(1, ar.Attempts);
Assert.AreEqual(1, (await GetAuditRecords(pmt).ConfigureAwait(false)).Count);
Assert.AreEqual(0, (await GetAuditRecords(amt).ConfigureAwait(false)).Count);
var ear = (await GetAuditRecords(pmt).ConfigureAwait(false)).First();
Assert.AreEqual("testhub-$Default", ear.PartitionKey);
Assert.AreEqual("0", ear.RowKey);
Assert.NotNull(ear.Body);
Assert.NotNull(ear.PoisonedTimeUtc);
Assert.IsNull(ear.SkippedTimeUtc);
Assert.AreEqual("DataNotFound", ear.Status);
Assert.AreEqual("Data not found.", ear.Reason);
Assert.IsNull(ear.OriginatingStatus);
Assert.IsNull(ear.OriginatingReason);
Assert.AreEqual(100, ear.Offset);
Assert.AreEqual(1, ear.SequenceNumber);
Assert.AreEqual(false, ear.SkipProcessing);
Assert.AreEqual(1, ear.Attempts);
// Pretend to check a different event to that poisoned.
ed = await CreateEventDataAsync(200, 2);
ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.NotPoison, ar.Action);
Assert.AreEqual(0, ar.Attempts);
Assert.AreEqual(0, (await GetAuditRecords(pmt)).Count);
Assert.AreEqual(1, (await GetAuditRecords(amt)).Count);
ear = (await GetAuditRecords(amt).ConfigureAwait(false)).First();
Assert.AreEqual("testhub-$Default", ear.PartitionKey);
Assert.IsTrue(ear.RowKey.EndsWith("-0"));
Assert.NotNull(ear.Body);
Assert.NotNull(ear.PoisonedTimeUtc);
Assert.IsNull(ear.SkippedTimeUtc);
Assert.AreEqual("PoisonMismatch", ear.Status);
Assert.AreEqual("Current EventData (Seq#: '2' Offset#: '200') being processed is out of sync with previous Poison (Seq#: '1' Offset#: '100'); current assumed correct with previous Poison now deleted.", ear.Reason);
Assert.AreEqual("DataNotFound", ear.OriginatingStatus);
Assert.AreEqual("Data not found.", ear.OriginatingReason);
Assert.AreEqual(100, ear.Offset);
Assert.AreEqual(1, ear.SequenceNumber);
Assert.AreEqual(false, ear.SkipProcessing);
Assert.AreEqual(1, ear.Attempts);
await Task.CompletedTask;
}
public async Task A110_PoisonMaxAttempts()
{
var cfg = GetConfig();
var lgr = TestSetUp.CreateLogger();
var asr = new EventHubAzureStorageRepository(cfg.GetWebJobsConnectionString(ConnectionStringNames.Storage))
{
PoisonTableName = cfg.GetValue <string>("EventHubPoisonMessagesTable"),
AuditTableName = cfg.GetValue <string>("EventHubAuditMessagesTable")
};
((IUseLogger)asr).UseLogger(lgr);
// Make sure there are no AuditRecords to begin with.
var pmt = await asr.GetPoisonMessageTableAsync().ConfigureAwait(false);
await DeleteAuditRecords(pmt).ConfigureAwait(false);
var amt = await asr.GetAuditMessageTableAsync().ConfigureAwait(false);
await DeleteAuditRecords(amt).ConfigureAwait(false);
var ed = await CreateEventDataAsync(100, 1);
// Add events as poison.
await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found.")), 3).ConfigureAwait(false);
var ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action);
Assert.AreEqual(1, ar.Attempts);
await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found.")), 3).ConfigureAwait(false);
ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.PoisonRetry, ar.Action);
Assert.AreEqual(2, ar.Attempts);
await asr.MarkAsPoisonedAsync(ed, HandleResult(Result.DataNotFound("Data not found.")), 3).ConfigureAwait(false);
ar = await asr.CheckPoisonedAsync(ed).ConfigureAwait(false);
Assert.AreEqual(PoisonMessageAction.NotPoison, ar.Action);
Assert.AreEqual(0, ar.Attempts);
var ear = (await GetAuditRecords(amt).ConfigureAwait(false)).Last();
Assert.AreEqual("testhub-$Default", ear.PartitionKey);
Assert.IsTrue(ear.RowKey.EndsWith("-0"));
Assert.NotNull(ear.Body);
Assert.NotNull(ear.PoisonedTimeUtc);
Assert.NotNull(ear.SkippedTimeUtc);
Assert.AreEqual("PoisonMaxAttempts", ear.Status);
Assert.AreEqual("EventData was identified as Poison and has been configured to automatically SkipMessage after 3 attempts; this event is skipped (i.e. not processed).", ear.Reason);
Assert.AreEqual("DataNotFound", ear.OriginatingStatus);
Assert.AreEqual("Data not found.", ear.OriginatingReason);
Assert.AreEqual(100, ear.Offset);
Assert.AreEqual(1, ear.SequenceNumber);
Assert.AreEqual(false, ear.SkipProcessing);
Assert.AreEqual(3, ear.Attempts);
}
