public async Task <EmailMessageInfo> GetEmailMessage(S3SourceInfo s3SourceInfo) { Stopwatch stopwatch = Stopwatch.StartNew(); GetObjectResponse response = await _s3Client.GetObjectAsync(s3SourceInfo.BucketName, s3SourceInfo.ObjectName) .TimeoutAfter(_config.TimeoutS3) .ConfigureAwait(false); _log.LogDebug($"Retrieving aggregate report from bucket took {stopwatch.Elapsed}"); stopwatch.Stop(); string originalUri = $"{s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}"; return(new EmailMessageInfo(new EmailMetadata(s3SourceInfo.RequestId, s3SourceInfo.MessageId, originalUri, s3SourceInfo.ObjectName, s3SourceInfo.ObjectSize / 1024), response.ResponseStream)); }
private async Task Process(S3SourceInfo s3SourceInfo) { using (_logger.BeginScope(new Dictionary <string, object> { ["MessageId"] = s3SourceInfo.MessageId, ["RequestId"] = s3SourceInfo.RequestId, ["S3ObjectPath"] = $"{ s3SourceInfo.BucketName }/{ s3SourceInfo.ObjectName }" })) { _logger.LogDebug( $"Processing report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " + $"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}."); try { EmailMessageInfo emailMessageInfo = await _s3Client.GetEmailMessage(s3SourceInfo); using (_logger.BeginScope(new Dictionary <string, object> { ["EmailAttachmentFileName"] = emailMessageInfo?.EmailMetadata?.Filename, })) { _logger.LogDebug( $"Successfully retrieved report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " + $"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}."); if (emailMessageInfo.EmailMetadata.FileSizeKb > _config.MaxS3ObjectSizeKilobytes) { _logger.LogWarning( $"Didnt process report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName} " + $" as MaxS3ObjectSizeKilobytes of {_config.MaxS3ObjectSizeKilobytes} Kb was exceeded, " + $"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}."); } else { AggregateReportInfo aggregateReportInfo = null; try { aggregateReportInfo = _parser.Parse(emailMessageInfo); } finally { emailMessageInfo.EmailStream.Dispose(); aggregateReportInfo?.AttachmentInfo.AttachmentStream.Dispose(); } _logger.LogDebug( $"Successfully parsed report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " + $"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}."); using (_logger.BeginScope(new Dictionary <string, object> { ["AggregateReportInfoId"] = aggregateReportInfo?.Id, ["AggregateReportId"] = aggregateReportInfo?.AggregateReport?.ReportMetadata?.ReportId, ["AggregateReportOrgName"] = aggregateReportInfo?.AggregateReport?.ReportMetadata?.OrgName, ["AggregateReportDomain"] = aggregateReportInfo?.AggregateReport?.PolicyPublished?.Domain, })) using (TransactionScope transactionScope = new TransactionScope( TransactionScopeOption.Required, new TransactionOptions { IsolationLevel = IsolationLevel.ReadCommitted, Timeout = TimeSpan.FromSeconds(300) }, TransactionScopeAsyncFlowOption.Enabled)) { aggregateReportInfo = await _persistor.Persist(aggregateReportInfo); _logger.LogDebug( $"Successfully persisted report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " + $"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}."); await _publisher.Publish(aggregateReportInfo); _logger.LogDebug( $"Successfully published report/s in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " + $"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}."); transactionScope.Complete(); } } } } catch (DuplicateAggregateReportException) { _logger.LogInformation( $"Duplicate report in s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName}, " + $"message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId} ignored."); } catch (Exception e) { _logger.LogError(e, $"Failed to process s3 object {s3SourceInfo.BucketName}/{s3SourceInfo.ObjectName} message Id: {s3SourceInfo.MessageId}, request Id: {s3SourceInfo.RequestId}"); throw; } } }