public async Task HandleCallback(IEnumerable <EmailEventNotification> emailEvents, CancellationToken cancellation)
{
// Group events by tenant Id
var emailEventsByTenant = emailEvents
.Where(e => e.TenantId != null && e.TenantId != 0) // Right now we do not handle null or zero tenant Ids, those were probably sent from identity or admin servers
.GroupBy(e => e.TenantId.Value);
// Run all tenants in parallel
await Task.WhenAll(emailEventsByTenant.Select(async emailEventsOfTenant =>
{
var tenantId = emailEventsOfTenant.Key;
var stateUpdatesOfTenant = emailEventsOfTenant.Select(emailEvent => new IdStateErrorTimestamp
{
Id = emailEvent.EmailId,
Error = emailEvent.Error,
Timestamp = emailEvent.Timestamp,
State = emailEvent.Event switch
{
EmailEvent.Dropped => EmailState.DispatchFailed,
EmailEvent.Delivered => EmailState.Delivered,
EmailEvent.Bounce => EmailState.DeliveryFailed,
EmailEvent.Open => EmailState.Opened,
EmailEvent.Click => EmailState.Clicked,
EmailEvent.SpamReport => EmailState.ReportedSpam,
_ => throw new InvalidOperationException($"[Bug] Unknown {nameof(EmailEvent)} = {emailEvent.Event}"), // Future proofing
}
});
// Update the state in the database (should we make it serializable?)
await _repo.Notifications_Emails__UpdateState(tenantId, stateUpdatesOfTenant, cancellation);
}));
protected override async Task ExecuteAsync(CancellationToken cancellation)
{
while (!cancellation.IsCancellationRequested)
{
try // To make sure the background service keeps running
{
// When the queue is empty, this goes to sleep until an item is enqueued
var(emails, scheduledAt) = await _queue.DequeueAsync(cancellation);
if (emails == null || !emails.Any())
{
continue; // Empty emails collection for some reason
}
// These Emails spent too long in the queue they're considered stale,.
// EmailPollingJob will (and might have already) pick them up and send them again, so ignore them
if (IsStale(scheduledAt))
{
var firstEmail = emails.First();
_logger.LogWarning($"Stale Email remained in the {nameof(EmailQueue)} for {(DateTimeOffset.Now - scheduledAt).TotalSeconds} seconds. First Email TenantId = {firstEmail.TenantId}, EmailId = {firstEmail.EmailId}.");
continue;
}
foreach (var emailsOfTenant in emails.GroupBy(e => e.TenantId))
{
var tenantId = emailsOfTenant.Key;
var stateUpdates = emailsOfTenant.Select(e => new IdStateErrorTimestamp {
Id = e.EmailId, State = EmailState.Dispatched, Timestamp = DateTimeOffset.Now
});
// Begin serializable transaction
using var trx = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable }, TransactionScopeAsyncFlowOption.Enabled);
// Update the state first (since this action can be rolled back)
await _repo.Notifications_Emails__UpdateState(tenantId, stateUpdates); // actions that modify state should not use cancellationToken
try
{
// Send the emails after you update the state in the DB, since sending emails
// is non-transactional and therefore cannot be rolled back
await _emailSender.SendBulkAsync(emailsOfTenant);
}
catch (Exception ex)
{
_logger.LogWarning(ex, $"Failed to Dispatch Emails. TenantId = {tenantId}, First EmailId = {emailsOfTenant.Select(e => e.EmailId).First()}");
// If sending the Email fails, update the state to DispatchFailed together with the error message
stateUpdates = emailsOfTenant.Select(e => new IdStateErrorTimestamp {
Id = e.EmailId, State = EmailState.DeliveryFailed, Timestamp = DateTimeOffset.Now, Error = ex.Message
});
await _repo.Notifications_Emails__UpdateState(tenantId, stateUpdates);
}
trx.Complete();
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error in {nameof(EmailJob)}.");
}
}
}
