private static AggregateReportInfo Create(Action <AggregateReportDomain> setter)
{
AggregateReportInfo aggregateReportInfo = Create();
setter(aggregateReportInfo.AggregateReport);
return(aggregateReportInfo);
}
private async Task Process(FileInfo fileInfo)
{
_logger.LogDebug(
$"Processing file {fileInfo.FullName}.");
try
{
EmailMessageInfo emailMessageInfo = CreateEmailMessageInfo(fileInfo);
_logger.LogDebug(
$"Successfully retrieved report in file {fileInfo.FullName}.");
AggregateReportInfo aggregateReportInfo = _parser.Parse(emailMessageInfo);
emailMessageInfo.EmailStream.Dispose();
_logger.LogDebug(
$"Successfully parsed report in file {fileInfo.FullName}.");
await _persistor.Persist(aggregateReportInfo);
_logger.LogDebug(
$"Successfully persisted report in file {fileInfo.FullName}.");
aggregateReportInfo.AttachmentInfo.AttachmentStream.Dispose();
}
catch (Exception e)
{
Console.WriteLine($"Failed to process {fileInfo.Name} with error: {e.Message}{Environment.NewLine}{e.StackTrace}");
}
}
public static List <DenormalisedRecord> ToDenormalisedRecord(this AggregateReportInfo aggregateReportInfo)
{
Domain.Dmarc.AggregateReport aggregateReport = aggregateReportInfo.AggregateReport;
string originalUri = aggregateReportInfo.EmailMetadata.OriginalUri;
return(aggregateReport.Records.Select(record => new DenormalisedRecord(
originalUri,
aggregateReport.ReportMetadata?.OrgName,
aggregateReport.ReportMetadata?.Email,
aggregateReport.ReportMetadata?.ExtraContactInfo,
aggregateReport.ReportMetadata.Range.Begin.ToDateTime(),
aggregateReport.ReportMetadata.Range.End.ToDateTime(),
aggregateReport.PolicyPublished?.Domain,
aggregateReport.PolicyPublished?.Adkim,
aggregateReport.PolicyPublished?.Aspf,
aggregateReport.PolicyPublished.P,
aggregateReport.PolicyPublished?.Sp,
aggregateReport.PolicyPublished?.Pct,
record.Row?.SourceIp,
record.Row.Count,
record.Row?.PolicyEvaluated?.Disposition,
record.Row?.PolicyEvaluated?.Dkim,
record.Row.PolicyEvaluated.Spf,
record.Row?.PolicyEvaluated?.Reasons != null ? string.Join(",", record.Row?.PolicyEvaluated?.Reasons.Select(_ => _.PolicyOverride.ToString())) : null,
record.Row?.PolicyEvaluated?.Reasons != null ? string.Join(",", record.Row?.PolicyEvaluated?.Reasons.Where(_ => _.Comment != null).Select(_ => _.Comment.ToString())) : null,
record.Identifiers?.EnvelopeTo,
record.Identifiers?.HeaderFrom,
record.AuthResults?.Dkim != null ? string.Join(",", record.AuthResults?.Dkim.Where(_ => _.Domain != null).Select(_ => _.Domain)) : null,
record.AuthResults?.Dkim != null ? string.Join(",", record.AuthResults?.Dkim.Select(_ => _.Result)) : null,
record.AuthResults?.Dkim != null ? string.Join(",", record.AuthResults?.Dkim.Where(_ => _.HumanResult != null).Select(_ => _.HumanResult)) : null,
record.AuthResults?.Spf != null ? string.Join(",", record.AuthResults?.Spf.Where(_ => _.Domain != null).Select(_ => _.Domain)) : null,
record.AuthResults?.Spf != null ? string.Join(",", record.AuthResults?.Spf.Select(_ => _.Result)) : null)).ToList());
}
public AggregateReportInfo Parse(EmailMessageInfo messageInfo)
{
List <AttachmentInfo> attachments = _mimeMessageFactory
.Create(messageInfo.EmailStream)
.BodyParts.OfType <MimePart>()
.Select(_attachmentStreamNormaliser.Normalise)
.Where(_ => !_.Equals(AttachmentInfo.EmptyAttachmentInfo))
.ToList();
string logString = $"{messageInfo.EmailMetadata.OriginalUri}, message Id: {messageInfo.EmailMetadata.MessageId}, request Id: {messageInfo.EmailMetadata.RequestId}";
if (attachments.Count == 0)
{
throw new ApplicationException(
$"Failed to parse: no attachment found where one was expected for {logString}.");
}
if (attachments.Count > 1)
{
var attachmentFilenames = attachments.Select(attachment => attachment.AttachmentMetadata.Filename).ToArray();
var attachmentFilenamesString = string.Join(", ", attachmentFilenames);
throw new ApplicationException(
$"Failed to parse: multiple attachments found where only one was expected for {logString}. {Environment.NewLine} Attachment filenames: {attachmentFilenamesString}");
}
AttachmentInfo attachmentInfo = attachments[0];
AggregateReportInfo aggregateReportInfo = _aggregateReportDeserialiser.Deserialise(attachmentInfo, messageInfo.EmailMetadata);
_log.LogInformation(
$"Successfully processed attachment for {logString}.");
return(aggregateReportInfo);
}
public override List <object> Create(AggregateReportInfo aggregateReportInfo)
{
return(aggregateReportInfo.AggregateReport.Records.SelectMany(_ => _.AuthResults.Dkim.Where(x => !string.IsNullOrEmpty(x.Selector)))
.GroupBy(_ => _.Domain, _ => _.Selector)
.Select(_ => new DkimSelectorsSeen(aggregateReportInfo.EmailMetadata.RequestId, aggregateReportInfo.EmailMetadata.MessageId, _.Key, _.Distinct().ToList()))
.Cast <object>()
.ToList());
}
public async Task <AggregateReportInfo> Persist(AggregateReportInfo aggregateReportInfo)
{
List <DenormalisedRecord> records = aggregateReportInfo.ToDenormalisedRecord();
await _composite.Persist(records);
return(aggregateReportInfo);
}
public void AggregateReportInfoReturned()
{
A.CallTo(() => _mimeMessageFactory.Create(A <Stream> ._)).Returns(CreateEmailMessage());
A.CallTo(() => _aggregateReportDeserialiser.Deserialise(A <AttachmentInfo> ._, A <EmailMetadata> ._)).Returns(CreateAggregateReportInfo());
AggregateReportInfo aggregateReportInfo = _aggregateReportParser.Parse(new EmailMessageInfo(new EmailMetadata("", "", "", "", 100), Stream.Null));
Assert.That(aggregateReportInfo, Is.Not.Null);
}
public async Task <AggregateReportInfo> Persist(AggregateReportInfo aggregateReportInfo)
{
foreach (IAggregateReportPersistor persistor in _pesistors)
{
aggregateReportInfo = await persistor.Persist(aggregateReportInfo);
}
return(aggregateReportInfo);
}
public void CorrectlyFormedReportNoDeclarationGeneratesAggregateReportInfo()
{
AttachmentInfo attachmentInfo = CreateAttachmentInfo(AggregateReportDeserialiserTestsResource.CorrectlyFormedReportNoDeclaration);
EmailMetadata emailMetadata = CreateEmailMetadata();
AggregateReportInfo aggregateReportInfo = _aggregateReportDeserialiser.Deserialise(attachmentInfo, emailMetadata);
Assert.That(aggregateReportInfo, Is.Not.Null);
}
public void RecordTagsCanOccurrMultipleTimes()
{
AttachmentInfo attachmentInfo = CreateAttachmentInfo(AggregateReportDeserialiserTestsResource.CorrectlyFormedReportNoDeclaration);
EmailMetadata emailMetadata = CreateEmailMetadata();
AggregateReportInfo aggregateReportInfo = _aggregateReportDeserialiser.Deserialise(attachmentInfo, emailMetadata);
Assert.That(aggregateReportInfo, Is.Not.Null);
}
public override List <object> Create(AggregateReportInfo aggregateReportInfo)
{
DateTime effectiveDate = ConversionUtils.UnixTimeStampToDateTime(aggregateReportInfo.AggregateReport.ReportMetadata.Range.EffectiveDate);
List <string> ipAddresses = aggregateReportInfo.AggregateReport.Records.Select(_ => _.Row.SourceIp).Distinct().ToList();
return(ipAddresses
.Batch(IpBatchSize)
.Select(_ => new AggregateReportIpAddresses(aggregateReportInfo.EmailMetadata.RequestId, aggregateReportInfo.EmailMetadata.MessageId, effectiveDate, _.ToList()))
.Cast <object>()
.ToList());
}
public async Task Publish(AggregateReportInfo aggregateReportInfo)
{
List <Message> dkimSelectorsSeenMessages = aggregateReportInfo.ToDkimSelectorsSeenMessages();
_log.LogDebug($"Publishing {dkimSelectorsSeenMessages.Count} dkim selector seen messages.");
foreach (IEnumerable <Message> dkimSelectorsSeenMessageBatch in dkimSelectorsSeenMessages.Batch(Concurrency))
{
await Task.WhenAll(dkimSelectorsSeenMessageBatch.Select(_ => _publisher.Publish(_, _config.DkimSelectorsTopicArn)));
}
}
public async Task Publish(AggregateReportInfo aggregateReportInfo)
{
List <object> messages = Create(aggregateReportInfo);
_log.Info(messages.Any()
? $"Publishing {messages.Count} {messages.First().GetType().Name} messages."
: $"No messages to publish for publisher: {GetType()}");
foreach (var message in messages)
{
await _publisher.Publish(message, _publisherConnectionString);
}
}
public Task <AggregateReportInfo> Persist(AggregateReportInfo aggregateReportInfo)
{
if (!_commandLineArgs.XmlDirectory.Exists)
{
_commandLineArgs.XmlDirectory.Create();
}
using (FileStream fileStream = System.IO.File.Create($"{_commandLineArgs.XmlDirectory.FullName}/{aggregateReportInfo.AttachmentInfo.AttachmentMetadata.Filename}.xml"))
{
aggregateReportInfo.AttachmentInfo.AttachmentStream.Seek(0, SeekOrigin.Begin);
aggregateReportInfo.AttachmentInfo.AttachmentStream.CopyTo(fileStream);
}
return(Task.FromResult(aggregateReportInfo));
}
public async Task <AggregateReportInfo> Persist(AggregateReportInfo aggregateReport)
{
string connectionString = await _connectionInfoAsync.GetConnectionStringAsync();
using (MySqlConnection connection = new MySqlConnection(connectionString))
{
await connection.OpenAsync().ConfigureAwait(false);
using (MySqlCommand command =
new MySqlCommand(AggregateReportDaoResources.InsertAggregateReport, connection))
{
command.Parameters.AddWithValue("request_id", aggregateReport.EmailMetadata.RequestId);
command.Parameters.AddWithValue("original_uri", aggregateReport.EmailMetadata.OriginalUri);
command.Parameters.AddWithValue("attachment_filename", aggregateReport.AttachmentInfo.AttachmentMetadata.Filename);
command.Parameters.AddWithValue("version", aggregateReport.AggregateReport.Version);
command.Parameters.AddWithValue("org_name", aggregateReport.AggregateReport?.ReportMetadata?.OrgName);
command.Parameters.AddWithValue("email", aggregateReport.AggregateReport?.ReportMetadata?.Email);
command.Parameters.AddWithValue("report_id", aggregateReport.AggregateReport?.ReportMetadata?.ReportId);
command.Parameters.AddWithValue("extra_contact_info", aggregateReport.AggregateReport?.ReportMetadata?.ExtraContactInfo);
command.Parameters.AddWithValue("effective_date", aggregateReport.AggregateReport?.ReportMetadata?.Range.EffectiveDate.ToDateTime());
command.Parameters.AddWithValue("begin_date", aggregateReport.AggregateReport?.ReportMetadata?.Range.Begin.ToDateTime());
command.Parameters.AddWithValue("end_date", aggregateReport.AggregateReport?.ReportMetadata?.Range.End.ToDateTime());
command.Parameters.AddWithValue("domain", aggregateReport.AggregateReport.PolicyPublished?.Domain);
command.Parameters.AddWithValue("adkim", aggregateReport.AggregateReport.PolicyPublished?.Adkim.ToString());
command.Parameters.AddWithValue("aspf", aggregateReport.AggregateReport.PolicyPublished?.Aspf?.ToString());
command.Parameters.AddWithValue("p", aggregateReport.AggregateReport.PolicyPublished?.P.ToString());
command.Parameters.AddWithValue("sp", aggregateReport.AggregateReport.PolicyPublished?.Sp?.ToString());
command.Parameters.AddWithValue("pct", aggregateReport.AggregateReport.PolicyPublished?.Pct);
command.Parameters.AddWithValue("fo", aggregateReport.AggregateReport.PolicyPublished?.Fo);
command.Parameters.AddWithValue("created_date", DateTime.UtcNow);
int numberOfUpdates = await command.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
if (numberOfUpdates == 0)
{
throw new DuplicateAggregateReportException($"Aggregate report {aggregateReport.EmailMetadata.OriginalUri} is a duplicate.");
}
aggregateReport.Id = command.LastInsertedId;
}
await _recordDao.Persist(aggregateReport, connection);
connection.Close();
}
return(aggregateReport);
}
public async Task Persist(AggregateReportInfo aggregateReport, MySqlConnection connection)
{
_log.LogDebug($"Persisting {aggregateReport.AggregateReport.Records.Length} records.");
foreach (IEnumerable <Record> batch in aggregateReport.AggregateReport.Records.Batch(BatchSize))
{
Record[] recordBatch = batch.ToArray();
using (MySqlCommand command = new MySqlCommand())
{
StringBuilder stringBuilder = new StringBuilder(AggregateReportDaoResources.InsertRecord);
for (int i = 0; i < recordBatch.Length; i++)
{
stringBuilder.Append(string.Format(AggregateReportDaoResources.InsertRecordValueFormatString, i));
stringBuilder.Append(i < recordBatch.Length - 1 ? "," : ";");
command.Parameters.AddWithValue($"a{i}", aggregateReport.Id);
command.Parameters.AddWithValue($"b{i}", recordBatch[i].Row.SourceIp);
command.Parameters.AddWithValue($"c{i}", recordBatch[i].Row.Count);
command.Parameters.AddWithValue($"d{i}", recordBatch[i].Row.PolicyEvaluated.Disposition?.ToString());
command.Parameters.AddWithValue($"e{i}", recordBatch[i].Row.PolicyEvaluated.Dkim?.ToString());
command.Parameters.AddWithValue($"f{i}", recordBatch[i].Row.PolicyEvaluated.Spf.ToString());
command.Parameters.AddWithValue($"g{i}", recordBatch[i].Identifiers.EnvelopeTo);
command.Parameters.AddWithValue($"h{i}", recordBatch[i].Identifiers.EnvelopeFrom);
command.Parameters.AddWithValue($"i{i}", recordBatch[i].Identifiers.HeaderFrom);
}
command.Connection = connection;
command.CommandText = stringBuilder.ToString();
await command.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
long lastInsertId = command.LastInsertedId;
foreach (Record record in recordBatch)
{
record.Id = lastInsertId++;
}
}
}
await _spfAuthResultDao.Persist(aggregateReport.AggregateReport.Records.ToList(), connection).ConfigureAwait(false);
await _dkimAuthResultDao.Persist(aggregateReport.AggregateReport.Records.ToList(), connection).ConfigureAwait(false);
await _policyOverrideReasonDao.Persist(aggregateReport.AggregateReport.Records.ToList(), connection).ConfigureAwait(false);
}
public void DenormalisedRecordPersistorsActiveProcessingOccursAggregateReportsPersisted()
{
A.CallTo(() => _multiAttachmentPersistor.Active).Returns(false);
A.CallTo(() => _multiDenormalisedRecordPersistor.Active).Returns(true);
A.CallTo(() => _mulitAggregateReportPersistor.Active).Returns(false);
A.CallTo(() => _mimeMessageFactory.Create(A <Stream> ._)).Returns(CreateEmailMessage());
AggregateReportInfo aggregateReportInfo = new AggregateReportInfo(new Domain.Dmarc.AggregateReport(null, null, new Record[0]), null, new AttachmentMetadata(""));
A.CallTo(() => _aggregateReportDeserialiser.Deserialise(A <AttachmentInfo> ._, A <EmailMetadata> ._)).ReturnsNextFromSequence(aggregateReportInfo);
_aggregateReportParser.Parse(new EmailMessageInfo(new EmailMetadata("", "", "", 100), Stream.Null));
A.CallTo(() => _attachmentStreamNormaliser.Normalise(A <MimePart> ._)).MustHaveHappened();
A.CallTo(() => _aggregateReportDeserialiser.Deserialise(A <AttachmentInfo> ._, A <EmailMetadata> ._)).MustHaveHappened();
A.CallTo(() => _log.Warn(A <string> ._)).MustNotHaveHappened();
A.CallTo(() => _denormalisedRecordConverter.ToDenormalisedRecord(A <Domain.Dmarc.AggregateReport> ._, A <string> ._)).MustHaveHappened();
A.CallTo(() => _multiAttachmentPersistor.Persist(A <List <AttachmentInfo> > ._)).MustNotHaveHappened();
A.CallTo(() => _multiDenormalisedRecordPersistor.Persist(A <IEnumerable <DenormalisedRecord> > ._)).MustHaveHappened();
A.CallTo(() => _mulitAggregateReportPersistor.Persist(A <IEnumerable <AggregateReportInfo> > ._)).MustNotHaveHappened();
}
public AggregateReportEntity ConvertToEntity(AggregateReportInfo aggregateReport)
{
return(new AggregateReportEntity
{
RequestId = aggregateReport.EmailMetadata.RequestId,
OrginalUri = aggregateReport.EmailMetadata.OriginalUri,
AttachmentFilename = aggregateReport.AttachmentMetadata.Filename,
OrgName = aggregateReport.AggregateReport.ReportMetadata?.OrgName,
ReportId = aggregateReport.AggregateReport.ReportMetadata.ReportId,
Email = aggregateReport.AggregateReport.ReportMetadata?.Email,
ExtraContactInfo = aggregateReport.AggregateReport.ReportMetadata?.ExtraContactInfo,
BeginDate = ConversionUtils.UnixTimeStampToDateTime(aggregateReport.AggregateReport.ReportMetadata.Range.Begin),
EndDate = ConversionUtils.UnixTimeStampToDateTime(aggregateReport.AggregateReport.ReportMetadata.Range.End),
Domain = aggregateReport.AggregateReport.PolicyPublished?.Domain,
Adkim = Convert(aggregateReport.AggregateReport.PolicyPublished?.Adkim),
Aspf = Convert(aggregateReport.AggregateReport.PolicyPublished?.Aspf),
P = Convert(aggregateReport.AggregateReport.PolicyPublished.P),
Sp = Convert(aggregateReport.AggregateReport.PolicyPublished?.Sp),
Pct = aggregateReport.AggregateReport.PolicyPublished?.Pct,
Records = aggregateReport.AggregateReport.Records?.Select(ConvertToEntity).ToList()
});
}
private static AggregateReportRecord ToAggregateReportRecord(this Record record,
AggregateReportInfo aggregateReportInfo)
{
return(new AggregateReportRecord(
record.Id.ToString(),
aggregateReportInfo.AggregateReport.ReportMetadata.OrgName,
aggregateReportInfo.AggregateReport.ReportMetadata.ReportId,
aggregateReportInfo.AggregateReport.ReportMetadata.Range.EffectiveDate.ToDateTime(),
aggregateReportInfo.AggregateReport.PolicyPublished?.Domain,
aggregateReportInfo.AggregateReport.PolicyPublished?.Adkim?.ToAlignment(),
aggregateReportInfo.AggregateReport.PolicyPublished?.Aspf?.ToAlignment(),
aggregateReportInfo.AggregateReport.PolicyPublished?.P.ToPolicy() ?? ContractPolicy.none,
aggregateReportInfo.AggregateReport.PolicyPublished?.Sp?.ToPolicy(),
aggregateReportInfo.AggregateReport.PolicyPublished?.Pct,
aggregateReportInfo.AggregateReport.PolicyPublished?.Fo,
record.Row.SourceIp,
record.Row.Count,
record.Row.PolicyEvaluated.Disposition?.ToPolicy(),
record.Row.PolicyEvaluated.Dkim?.ToDmarcResult(),
record.Row.PolicyEvaluated.Spf?.ToDmarcResult(),
record.Identifiers.EnvelopeTo,
record.Identifiers.EnvelopeFrom,
record.Identifiers.HeaderFrom,
record.AuthResults.Spf.Select(x => $"{x.Domain}:{x.Result}").ToList(),
record.AuthResults.SpfPassCount,
record.AuthResults.SpfFailCount,
record.AuthResults.Dkim.Select(x => $"{x.Domain}:{x.Selector}:{x.Result}").ToList(),
record.AuthResults.DkimPassCount,
record.AuthResults.DkimFailCount,
record.Row.PolicyEvaluated.HasForwardedReason,
record.Row.PolicyEvaluated.HasSampledOutReason,
record.Row.PolicyEvaluated.HasTrustedForwarderReason,
record.Row.PolicyEvaluated.HasMailingListReason,
record.Row.PolicyEvaluated.HasLocalPolicyReason,
record.Row.PolicyEvaluated.HasArcReason,
record.Row.PolicyEvaluated.HasOtherReason));
}
public async Task Publish(AggregateReportInfo aggregateReportInfo)
{
List <AggregateReportRecord> aggregateReportRecords = aggregateReportInfo.ToAggregateReportRecords();
if (aggregateReportRecords.Any(x => string.IsNullOrEmpty(x.HostSourceIp)))
{
List <AggregateReportRecord> records = aggregateReportRecords.Where(x => string.IsNullOrEmpty(x.HostSourceIp)).ToList();
List <string> recordIds = records.Select(x => x.Id).ToList();
_log.LogError($"Error validating AggregateReportInfo. {recordIds.Count} row(s) missing Host IP with Record Ids: {string.Join(", ", recordIds)}. Publish aborted.");
return;
}
List <AggregateReportRecordBatch> aggregateReportRecordBatches = aggregateReportRecords
.BatchByJsonBytes(MaxBatchSizeBytes)
.Select(_ => new AggregateReportRecordBatch(Guid.NewGuid().ToString(), _.ToList()))
.ToList();
_log.LogDebug($"Publishing {aggregateReportRecordBatches.Count} aggregate report record processed batch messages.");
foreach (IEnumerable <AggregateReportRecordBatch> reportRecordBatches in aggregateReportRecordBatches.Batch(Concurrency))
{
await Task.WhenAll(reportRecordBatches.Select(_ => _messagePublisher.Publish(_, _config.SnsTopicArn)));
}
}
public Task Publish(AggregateReportInfo aggregateReportInfo)
{
return(Task.WhenAll(_publishers.Select(_ => _.Publish(aggregateReportInfo))));
}
public static List <AggregateReportRecord> ToAggregateReportRecords(this AggregateReportInfo aggregateReportInfo)
{
return(aggregateReportInfo.AggregateReport.Records
.Select(_ => _.ToAggregateReportRecord(aggregateReportInfo))
.ToList());
}
private async Task Process(S3SourceInfo s3SourceInfo)
{
using (_logger.BeginScope(new Dictionary <string, object> {
["MessageId"] = s3SourceInfo.MessageId,
["RequestId"] = s3SourceInfo.RequestId,
["S3ObjectPath"] = $"{ s3SourceInfo.BucketName }/{ s3SourceInfo.ObjectName }"
}))
{
_logger.LogDebug(
$"Processing report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " +
$"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}.");
try
{
EmailMessageInfo emailMessageInfo = await _s3Client.GetEmailMessage(s3SourceInfo);
using (_logger.BeginScope(new Dictionary <string, object>
{
["EmailAttachmentFileName"] = emailMessageInfo?.EmailMetadata?.Filename,
}))
{
_logger.LogDebug(
$"Successfully retrieved report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " +
$"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}.");
if (emailMessageInfo.EmailMetadata.FileSizeKb > _config.MaxS3ObjectSizeKilobytes)
{
_logger.LogWarning(
$"Didnt process report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName} " +
$" as MaxS3ObjectSizeKilobytes of {_config.MaxS3ObjectSizeKilobytes} Kb was exceeded, " +
$"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}.");
}
else
{
AggregateReportInfo aggregateReportInfo = null;
try
{
aggregateReportInfo = _parser.Parse(emailMessageInfo);
}
finally
{
emailMessageInfo.EmailStream.Dispose();
aggregateReportInfo?.AttachmentInfo.AttachmentStream.Dispose();
}
_logger.LogDebug(
$"Successfully parsed report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " +
$"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}.");
using (_logger.BeginScope(new Dictionary <string, object>
{
["AggregateReportInfoId"] = aggregateReportInfo?.Id,
["AggregateReportId"] = aggregateReportInfo?.AggregateReport?.ReportMetadata?.ReportId,
["AggregateReportOrgName"] = aggregateReportInfo?.AggregateReport?.ReportMetadata?.OrgName,
["AggregateReportDomain"] = aggregateReportInfo?.AggregateReport?.PolicyPublished?.Domain,
}))
using (TransactionScope transactionScope = new TransactionScope(
TransactionScopeOption.Required,
new TransactionOptions
{
IsolationLevel = IsolationLevel.ReadCommitted,
Timeout = TimeSpan.FromSeconds(300)
},
TransactionScopeAsyncFlowOption.Enabled))
{
aggregateReportInfo = await _persistor.Persist(aggregateReportInfo);
_logger.LogDebug(
$"Successfully persisted report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " +
$"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}.");
await _publisher.Publish(aggregateReportInfo);
_logger.LogDebug(
$"Successfully published report/s in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " +
$"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}.");
transactionScope.Complete();
}
}
}
}
catch (DuplicateAggregateReportException)
{
_logger.LogInformation(
$"Duplicate report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " +
$"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId} ignored.");
}
catch (Exception e)
{
_logger.LogError(e,
$"Failed to process s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName} message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}");
throw;
}
}
}
public void MappingTests <TOut>(TOut expected, AggregateReportInfo aggregateReportInfo, Func <AggregateReportEntity, TOut> actualGetter)
{
AggregateReportEntity aggregateReportEntity = _aggregateReportToEntityConverter.Convert(aggregateReportInfo);
Assert.That(expected, Is.EqualTo(actualGetter(aggregateReportEntity)));
}
public abstract List <object> Create(AggregateReportInfo aggregateReportInfo);
