public void Sending_to_non_existent_queue_sets_exception_in_result()
{
var sqs = new InMemorySqsClient();
var r = sqs.Send("foo", AwsSqsMessage.FromBody("foo"), new Result <AwsSqsSendResponse>()).Block();
Assert.IsTrue(r.HasException);
}
public void CanProcessEmailBodyLogMessage()
{
//Given
var logger = new Mock <ILogger <AppService> >();
var config = new Mock <IOptions <AppConfig> >();
var saveEmailBodyService = new Mock <IEmailBodyService>();
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "EmailMessageId", "0000000000000000-00000000-0000-0000-0000-000000000000-000000" },
{ "EmailBody", "EmailTestBody" },
{ "EmailSubject", "EmailTestBodySubject" },
{ "EmailTopicArn", "EmailTestBodyTopicARN" },
{ "EmailTimestamp", "1561095094474" },
{ "EmailResponseMetaDataHTTPCode", "200" },
{ "EmailResponseMetaDataRequestId", "AAAABBBBCCCCDDDDEEEE1" },
{ "EmailResponseMetaDataRetryAttempts", "1" },
};
saveEmailBodyService.Setup(sebs => sebs.SaveEmailBodyMessage(dataDictToBeStored)).Returns(true);
//When
var awsSqsMessage = new AwsSqsMessage()
{
Body = JsonConvert.SerializeObject(new {
Type = "Notification",
Timestamp = "1561095094474",
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
TopicArn = "EmailTestBodyTopicARN",
Message = JsonConvert.SerializeObject(new {
Email = JsonConvert.SerializeObject(new {
Body = "EmailTestBody",
Subject = "EmailTestBodySubject",
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
notificationType = "Email_Body_Log",
EmailResponseMetaData = new {
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
ResponseMetadata = new {
RetryAttempts = 1,
HTTPStatusCode = 200,
RequestId = "AAAABBBBCCCCDDDDEEEE1",
}
}
})
}),
}),
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000"
};
var emailBodyLogProcessor = new EmailBodyLogProcessor(logger.Object, config.Object, saveEmailBodyService.Object);
saveEmailBodyService.Setup(sebs => sebs.SaveEmailBodyMessage(dataDictToBeStored)).Returns(true);
var result = emailBodyLogProcessor.Process(awsSqsMessage);
// //Then
saveEmailBodyService.Verify((sebs) => sebs.SaveEmailBodyMessage(dataDictToBeStored), Times.Exactly(1));
Assert.True(result, "created should return true");
}
public void LIVE_Send_one_for_inspection()
{
var client = CreateLiveClient();
var queue = "encoding-test";
client.CreateQueue(queue, new Result <AwsSqsResponse>()).Wait();
var msg = AwsSqsMessage.FromBody("<deki-event wikiid=\"default\" event-time=\"2011-06-30T16:16:36Z\"><channel>event://default/deki/pages/dependentschanged/properties/update</channel><uri>http://ariel.mindtouch.com/@api/deki/pages/22?redirects=0</uri><pageid>22</pageid><user id=\"1\" anonymous=\"false\"><uri>http://ariel.mindtouch.com/@api/deki/users/1</uri></user><content.uri type=\"application/xml\">http://ariel.mindtouch.com/@api/deki/pages/22/contents?redirects=0&revision=45&format=xhtml</content.uri><revision.uri>http://ariel.mindtouch.com/@api/deki/pages/22/revisions?redirects=0&revision=45</revision.uri><tags.uri>http://ariel.mindtouch.com/@api/deki/pages/22/tags?redirects=0</tags.uri><comments.uri>http://ariel.mindtouch.com/@api/deki/pages/22/comments?redirects=0</comments.uri><path></path><frommove>false</frommove></deki-event>");
var r = client.Send(queue, msg, new Result <AwsSqsSendResponse>()).Block();
var v = r.Value;
}
public void Cannot_receive_a_message_to_a_non_existent_queue()
{
var sqs = new InMemorySqsClient();
var sent = AwsSqsMessage.FromBody("foo");
try {
sqs.Receive("bar", new Result <IEnumerable <AwsSqsMessage> >()).Wait();
Assert.Fail("didn't throw");
} catch (AwsSqsRequestException e) {
Assert.AreEqual("AWS.SimpleQueueService.NonExistentQueue", e.Error.Code);
}
}
private async void DoBodyWork(object state)
{
var receiveBodyMessageRequest = new ReceiveMessageRequest()
{
AttributeNames = new List <string>()
{
"All"
},
QueueUrl = _config.Value.AwsSqsBodyQueueUrl,
WaitTimeSeconds = _config.Value.AwsSqsLongPollTimeInSeconds,
MaxNumberOfMessages = 10
};
_bodyLogger.LogDebug("Heartbeat: {Now}", DateTime.Now);
var receiveBodyMessageResponse = await _sqsBodyClient.ReceiveMessageAsync(receiveBodyMessageRequest);
if (receiveBodyMessageResponse.Messages.Count > 0)
{
foreach (var message in receiveBodyMessageResponse.Messages)
{
_bodyLogger.LogDebug("Message Id: {MessageId}", message.MessageId);
var awsSqsMessage = AwsSqsMessage.FromJson(message.Body);
awsSqsMessage.Body = message.Body;
if (message.Body.Contains("Email_Body_Log"))
{
var emailLogProcessor = _emailLogProcessor;
if (emailLogProcessor == null)
{
_bodyLogger.LogDebug("Email processor not found for {@AwsSqsMessage}. Deleting message", awsSqsMessage);
await DeleteBodyMessage(message);
continue;
}
var result = emailLogProcessor.ProcessMessage(awsSqsMessage);
if (result == true)
{
await DeleteBodyMessage(message);
}
}
else
{
_bodyLogger.LogDebug("Not an email body message for {@AwsSqsMessage}. Deleting message", awsSqsMessage);
await DeleteBodyMessage(message);
continue;
}
}
}
}
public void Can_see_messages_with_visibility_zero()
{
var sqs = new InMemorySqsClient();
sqs.CreateQueue("bar", new Result <AwsSqsResponse>()).Wait();
var sent = sqs.Send("bar", AwsSqsMessage.FromBody("foo"), new Result <AwsSqsSendResponse>()).Wait();
var received1 = sqs.Receive("bar", 10, TimeSpan.Zero, new Result <IEnumerable <AwsSqsMessage> >()).Wait();
var received2 = sqs.Receive("bar", 10, TimeSpan.Zero, new Result <IEnumerable <AwsSqsMessage> >()).Wait();
Assert.AreEqual(1, received1.Count());
Assert.AreEqual(1, received2.Count());
Assert.AreEqual(sent.MessageId, received1.First().MessageId);
Assert.AreEqual(sent.MessageId, received2.First().MessageId);
}
public void Can_delete_message_from_queue()
{
var sqs = new InMemorySqsClient();
sqs.CreateQueue("bar", new Result <AwsSqsResponse>()).Wait();
var sent = AwsSqsMessage.FromBody("foo");
sqs.Send("bar", sent, new Result <AwsSqsSendResponse>()).Wait();
var received = sqs.Receive("bar", 10, TimeSpan.Zero, new Result <IEnumerable <AwsSqsMessage> >()).Wait().First();
sqs.Delete(received, new Result <AwsSqsResponse>()).Wait();
var remaining = sqs.Receive("bar", 10, TimeSpan.Zero, new Result <IEnumerable <AwsSqsMessage> >()).Wait();
Assert.IsFalse(remaining.Any());
}
public void Can_round_trip_message_through_queue()
{
var sqs = new InMemorySqsClient();
sqs.CreateQueue("bar", new Result <AwsSqsResponse>()).Wait();
var sent = AwsSqsMessage.FromBody("foo");
sqs.Send("bar", sent, new Result <AwsSqsSendResponse>()).Wait();
var msgs = sqs.Receive("bar", new Result <IEnumerable <AwsSqsMessage> >()).Wait();
Assert.AreEqual(1, msgs.Count());
var received = msgs.First();
Assert.AreEqual(sent.Body, received.Body);
}
public void LIVE_Can_send_and_receive()
{
var client = CreateLiveClient();
var queue = "test-" + StringUtil.CreateAlphaNumericKey(8);
client.CreateQueue(queue, new Result <AwsSqsResponse>()).Wait();
try {
var body1 = "\"" %20+<>:?";
var r1 = client.Send(queue, AwsSqsMessage.FromBody(body1), new Result <AwsSqsSendResponse>()).Wait();
_log.DebugFormat("created message {0}", r1.MessageId);
var doc = new XDoc("doc")
.Elem("foo", StringUtil.CreateAlphaNumericKey(100))
.Start("baz")
.Elem("bar", StringUtil.CreateAlphaNumericKey(100))
.Elem("baz", StringUtil.CreateAlphaNumericKey(100))
.End()
.Elem("bing", StringUtil.CreateAlphaNumericKey(100));
var body2Str = "<deki-event wikiid=\"default\" event-time=\"2011-06-30T16:16:36Z\"><channel>event://default/deki/pages/dependentschanged/properties/update</channel><uri>http://ariel.mindtouch.com/@api/deki/pages/22?redirects=0</uri><pageid>22</pageid><user id=\"1\" anonymous=\"false\"><uri>http://ariel.mindtouch.com/@api/deki/users/1</uri></user><content.uri type=\"application/xml\">http://ariel.mindtouch.com/@api/deki/pages/22/contents?redirects=0&revision=45&format=xhtml</content.uri><revision.uri>http://ariel.mindtouch.com/@api/deki/pages/22/revisions?redirects=0&revision=45</revision.uri><tags.uri>http://ariel.mindtouch.com/@api/deki/pages/22/tags?redirects=0</tags.uri><comments.uri>http://ariel.mindtouch.com/@api/deki/pages/22/comments?redirects=0</comments.uri><path>Announcements/2008-10-03_-_Slide_Show_of_T2's_Big_Give</path><frommove>false</frommove></deki-event>";
var body2Doc = XDocFactory.From(body2Str, MimeType.XML);
var body2 = body2Doc.ToCompactString();
var r2 = client.Send(queue, AwsSqsMessage.FromBodyDocument(body2Doc), new Result <AwsSqsSendResponse>()).Wait();
_log.DebugFormat("created message {0}", r2.MessageId);
var messages = new List <AwsSqsMessage>();
Assert.IsTrue(
Wait.For(() => {
var r = client.Receive(queue, new Result <IEnumerable <AwsSqsMessage> >()).Wait();
foreach (var m in r)
{
_log.DebugFormat("retrieved message {0}", m.MessageId);
messages.Add(m);
}
return(messages.Count == 2);
},
10.Seconds()),
string.Format("only received {0} messages from queue before timeout", messages.Count)
);
var msg1 = messages.Where(x => x.MessageId == r1.MessageId).FirstOrDefault();
var msg2 = messages.Where(x => x.MessageId == r2.MessageId).FirstOrDefault();
Assert.IsNotNull(msg1, "message 1 was not in response");
Assert.IsNotNull(msg2, "message 2 was not in response");
Assert.AreEqual(body1, msg1.Body, "msg 1 body didn't match");
Assert.AreEqual(body2, msg2.Body, "msg 2 body didn't match");
} finally {
_log.DebugFormat("cleaning up queue '{0}'", queue);
client.DeleteQueue(queue, new Result <AwsSqsResponse>()).Wait();
}
}
private async void DoWork(object state)
{
var receiveMessageRequest = new ReceiveMessageRequest()
{
AttributeNames = new List <string>()
{
"All"
},
QueueUrl = _config.Value.AwsSqsQueueUrl,
WaitTimeSeconds = _config.Value.AwsSqsLongPollTimeInSeconds,
MaxNumberOfMessages = 10
};
_logger.LogDebug("Heartbeat: {Now}", DateTime.Now);
var receiveMessageResponse = await _sqsClient.ReceiveMessageAsync(receiveMessageRequest);
if (receiveMessageResponse.Messages.Count > 0)
{
foreach (var message in receiveMessageResponse.Messages)
{
_logger.LogDebug("Message Id: {MessageId}", message.MessageId);
var notificationLogBodyAnon2 = JsonConvert.DeserializeAnonymousType(message.Body, new {
Message = "",
});
var notificationLogBodyMessageAnon2 = JsonConvert.DeserializeAnonymousType(notificationLogBodyAnon2.Message, new {
notificationType = "",
});
var awsSqsMessage = AwsSqsMessage.FromJson(message.Body);
awsSqsMessage.Body = message.Body;
var emailProcessor = _emailLogProcessor(notificationLogBodyMessageAnon2.notificationType);
if (emailProcessor == null)
{
_logger.LogDebug("Email processor not found for {@AwsSqsMessage}. Deleting message", awsSqsMessage);
await DeleteMessage(message);
continue;
}
var result = emailProcessor.ProcessMessage(awsSqsMessage);
if (result == true)
{
await DeleteMessage(message);
}
}
}
}
public Result <Production> Produce(int messages)
{
var production = new Production();
var final = new Result <Production>();
AsyncUtil.Fork(() => {
try {
_log.DebugFormat("{0}: Producing {1} messages", production.Id, messages);
var responses = new List <Result <string> >();
var client = new AwsSqsClient(_clientConfig);
for (var i = 0; i < messages; i++)
{
var result = new Result <string>();
responses.Add(result);
var msg = production.Id + ":" + messages;
client.Send(_queue, AwsSqsMessage.FromBody(msg), new Result <AwsSqsSendResponse>()).WhenDone(r => {
if (r.HasException)
{
result.Throw(r.Exception);
return;
}
result.Return(msg);
});
}
responses.Join(new Result()).WhenDone(r => {
if (r.HasException)
{
final.Throw(r.Exception);
return;
}
production.Stopwatch.Stop();
production.Sent.AddRange(responses.Select(x => x.Value));
_log.DebugFormat("{0}: Sent {1} messages in {2:0.00}s @ {3:0.00}msg/sec",
production.Id,
production.Sent.Count,
production.Stopwatch.Elapsed.TotalSeconds,
production.Sent.Count / production.Stopwatch.Elapsed.TotalSeconds
);
final.Return(production);
});
} catch (Exception e) {
final.Throw(e);
}
});
return(final);
}
public override bool Process(AwsSqsMessage awsSqsMessage)
{
var emailLogBodyAnon = JsonConvert.DeserializeAnonymousType(awsSqsMessage.Body, new {
Type = "",
Timestamp = "",
MessageId = "",
TopicArn = "",
Message = "",
});
var emailLogBodySubAnon = JsonConvert.DeserializeAnonymousType(emailLogBodyAnon.Message, new {
Email = "",
});
var emailLogBodySubTwoAnon = JsonConvert.DeserializeAnonymousType(emailLogBodySubAnon.Email, new {
Body = "",
Subject = "",
MessageId = "",
notificationType = "",
EmailResponseMetaData = new {
MessageId = "",
ResponseMetadata = new {
RetryAttempts = default(int),
HTTPStatusCode = "",
RequestId = "",
}
},
});
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "EmailMessageId", emailLogBodySubTwoAnon.EmailResponseMetaData.MessageId },
{ "EmailBody", emailLogBodySubTwoAnon.Body },
{ "EmailSubject", emailLogBodySubTwoAnon.Subject },
{ "EmailTopicArn", emailLogBodyAnon.TopicArn },
{ "EmailTimestamp", emailLogBodyAnon.Timestamp.ToString() },
{ "EmailResponseMetaDataHTTPCode", emailLogBodySubTwoAnon.EmailResponseMetaData.ResponseMetadata.HTTPStatusCode },
{ "EmailResponseMetaDataRequestId", emailLogBodySubTwoAnon.EmailResponseMetaData.ResponseMetadata.RequestId },
{ "EmailResponseMetaDataRetryAttempts", emailLogBodySubTwoAnon.EmailResponseMetaData.ResponseMetadata.RetryAttempts.ToString() },
};
_saveEmailBodyService.SaveEmailBodyMessage(dataDictToBeStored);
return(true);
}
public override bool Process(AwsSqsMessage awsSqsMessage)
{
var notificationLogBodyAnon = JsonConvert.DeserializeAnonymousType(awsSqsMessage.Body, new {
Type = "",
Timestamp = "",
MessageId = "",
TopicArn = "",
Message = "",
});
var notificationLogBodyMessageAnon = JsonConvert.DeserializeAnonymousType(notificationLogBodyAnon.Message, new {
notificationType = "",
mail = new {
timestamp = "",
Source = "",
messageID = "",
destination = new List <string>(),
commonHeaders = new {
returnPath = "",
from = new List <string>(),
replyTo = new List <string>(),
to = new List <string>(),
subject = ""
}
},
complaint = new {
userAgent = "",
complainedRecipients = new List <dynamic>(),
complaintFeedbackType = "",
arrivalDate = "",
timestamp = "",
feedbackId = ""
}
});
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "NotificationBodyMessageId", notificationLogBodyMessageAnon.mail.messageID },
{ "NotificationBodyTopicARN", notificationLogBodyAnon.TopicArn },
{ "NotificationBodyType", notificationLogBodyMessageAnon.notificationType },
{ "NotificationBodyTimestamp", notificationLogBodyAnon.Timestamp.ToString() },
{ "NotificationBodyMailSource", notificationLogBodyMessageAnon.mail.Source },
{ "NotificationBodyCommonHeadersSubject", notificationLogBodyMessageAnon.mail.commonHeaders.subject },
{ "NotificationBodyComplaintUserAgent", notificationLogBodyMessageAnon.complaint.userAgent },
{ "NotificationBodyComplaintFeedbackType", notificationLogBodyMessageAnon.complaint.complaintFeedbackType },
{ "NotificationBodyComplaintFeedbackId", notificationLogBodyMessageAnon.complaint.feedbackId },
{ "NotificationBodyComplaintTimestamp", notificationLogBodyMessageAnon.complaint.timestamp },
};
for (var index = 0; index < notificationLogBodyMessageAnon.mail.commonHeaders.from.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersFrom" + (index + 1), notificationLogBodyMessageAnon.mail.commonHeaders.from[index]);
}
for (var index = 0; index < notificationLogBodyMessageAnon.mail.commonHeaders.to.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersTo" + (index + 1), notificationLogBodyMessageAnon.mail.commonHeaders.to[index]);
}
if (notificationLogBodyMessageAnon.mail.commonHeaders.replyTo != null)
{
for (var index = 0; index < notificationLogBodyMessageAnon.mail.commonHeaders.replyTo.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersReplyTo" + (index + 1), notificationLogBodyMessageAnon.mail.commonHeaders.replyTo[index]);
}
}
if (notificationLogBodyMessageAnon.mail.destination != null)
{
for (var index = 0; index < notificationLogBodyMessageAnon.mail.destination.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyDestination" + (index + 1), notificationLogBodyMessageAnon.mail.destination[index]);
}
}
var complainedRecipients = "";
for (var index = 0; index < notificationLogBodyMessageAnon.complaint.complainedRecipients.Count; index++)
{
complainedRecipients += $"{notificationLogBodyMessageAnon.complaint.complainedRecipients[index]},";
}
dataDictToBeStored.Add("NotificationBodyComplaintRecipient", complainedRecipients);
var result = _saveEmailNotificationService.SaveEmailMessage(dataDictToBeStored);
return(result);
}
public void CanProcessEmailBounceNotificationMessage()
{
//Given
var bouncedRecipients = new List <dynamic>()
{
new {
emailAddress = "*****@*****.**",
action = "failed",
status = "5.1.1",
diagnosticCode = "smtp: 0-1.1.1"
}
};
var bounceRecipientsToString = "";
for (var index = 0; index < bouncedRecipients.Count; index++)
{
bounceRecipientsToString += $"{bouncedRecipients[index]}";
}
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "NotificationBodyMessageId", "0000000000000000-00000000-0000-0000-0000-000000000000-000000" }, { "NotificationBodyTopicARN", "NotificationTestBodyTopicARN" }, { "NotificationBodyType", "Bounce" }, { "NotificationBodyTimestamp", "1561095094474" }, { "NotificationBodyMailSource", "Example Agency <*****@*****.**>" }, { "NotificationBodyCommonHeadersSubject", "ExampleSubject" }, { "NotificationBodyBounceType", "Permanent" }, { "NotificationBodyBounceSubType", "Suppressed" }, { "NotificationBodyBounceTimestamp", "1561095094474" }, { "NotificationBodyBounceSMTPResponse", "250 2.0.0 Ok:123456789123" }, { "NotificationBodyBounceRemoteMTAIp", "111.22.333.444" }, { "NotificationBodyBounceReportingMTA", "ExampleMTA" }, { "NotificationBounceBouncedRecipients", bounceRecipientsToString }, { "NotificationBodyCommonHeadersFrom", "Example Company Admin <*****@*****.**>" }, { "NotificationBodyCommonHeadersTo", "*****@*****.**" }, { "NotificationBodyCommonHeadersReplyTo", "*****@*****.**" }, { "NotificationBodyDestination", "*****@*****.**" },
};
var logger = new Mock <ILogger <AppService> >();
var config = new Mock <IOptions <AppConfig> >();
var saveEmailNotificationService = new Mock <IEmailService>();
saveEmailNotificationService.Setup(sens => sens.SaveEmailMessage(It.IsAny <Dictionary <string, string> >())).Returns(true);
var emailBounceNotificationLogProcessor = new EmailBounceNotificationProcessor(logger.Object, config.Object, saveEmailNotificationService.Object);
//When
var awsSqsMessage = new AwsSqsMessage()
{
Body = JsonConvert.SerializeObject(new {
Type = "Notification",
Timestamp = "1561095094474",
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
TopicArn = "NotificationTestBodyTopicARN",
Message = JsonConvert.SerializeObject(new {
NotificationType = "Bounce",
Mail = new {
TimeStamp = "1561095094474",
Source = "Example Agency <*****@*****.**",
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
Destination = new List <string>()
{
"*****@*****.**"
},
CommonHeaders = new {
ReturnPath = "",
From = new List <string>()
{
"Example Company Admin <*****@*****.**>",
},
ReplyTo = new List <string>()
{
"*****@*****.**"
},
To = new List <string>()
{
"*****@*****.**"
},
Subject = "ExampleSubject",
}
},
Bounce = new {
BounceType = "Permanent",
BounceSubType = "Suppressed",
BouncedRecipients = new List <dynamic>()
{
new {
EmailAddress = "*****@*****.**",
Action = "failed",
Status = "5.1.1",
DiagnosticCode = "smtp: 0-1.1.1"
}
},
Timestamp = "1561095094474",
smtpResponse = "250 2.0.0 Ok:123456789123",
remoteMtaIp = "111.22.333.444",
reportingMTA = "ExampleMTA"
},
}),
}),
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
};
var result = emailBounceNotificationLogProcessor.Process(awsSqsMessage);
//Then
saveEmailNotificationService.Verify((sens) => sens.SaveEmailMessage(It.IsAny <Dictionary <string, string> >()), Times.Exactly(1));
Assert.True(result, "created should return true");
}
public Result <AwsSqsResponse> Delete(AwsSqsMessage message, Result <AwsSqsResponse> result)
{
_log.DebugFormat("deleting {0}", message.MessageId);
Deleted.Add(message);
return(new Result <AwsSqsResponse>().WithReturn(null));
}
public Result <AwsSqsSendResponse> Send(string queue, AwsSqsMessage message, Result <AwsSqsSendResponse> result)
{
throw new NotImplementedException();
}
public bool ProcessMessage(AwsSqsMessage awsSqsMessage)
{
return(Process(awsSqsMessage));
}
public abstract bool Process(AwsSqsMessage awsSqsMessage);
public void CanProcessEmailDeliveryNotificationMessage()
{
//Given
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "NotificationBodyMessageId", "0000000000000000-00000000-0000-0000-0000-000000000000-000000" },
{ "NotificationBodyTopicARN", "NotificationTestBodyTopicARN" },
{ "NotificationBodyType", "Delivery" },
{ "NotificationBodyTimestamp", "1561095094474" },
{ "NotificationBodyMailSource", "Example Agency <*****@*****.**>" },
{ "NotificationBodyCommonHeadersSubject", "ExampleSubject" },
{ "NotificationBodyDeliveryTimestamp", "Permanent" },
{ "NotificationBodyDeliveryProcessingTimeMs", "1234" },
{ "NotificationBodyDeliverySMTPResponse", "250 2.0.0 Ok: queued as 123456789123" },
{ "NotificationBodyDeliveryRemoteMtaIp", "111.22.333.444" },
{ "NotificationBodyDeliveryReportingMTA", "ExampleMTA" },
{ "Recipients", "*****@*****.**" },
{ "NotificationBodyCommonHeadersFrom", "Example Company Admin <*****@*****.**>" },
{ "NotificationBodyCommonHeadersTo", "*****@*****.**" },
{ "NotificationBodyCommonHeadersReplyTo", "*****@*****.**" },
{ "NotificationBodyDestination", "*****@*****.**" },
};
var logger = new Mock <ILogger <AppService> >();
var config = new Mock <IOptions <AppConfig> >();
var saveEmailNotificationService = new Mock <IEmailService>();
saveEmailNotificationService.Setup(sens => sens.SaveEmailMessage(It.IsAny <Dictionary <string, string> >())).Returns(true);
var emailDeliveryNotificationLogProcessor = new EmailDeliveryNotificationProcessor(logger.Object, config.Object, saveEmailNotificationService.Object);
//When
var awsSqsMessage = new AwsSqsMessage()
{
Body = JsonConvert.SerializeObject(new {
Type = "Notification",
Timestamp = "1561095094474",
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
TopicArn = "NotificationTestBodyTopicARN",
Message = JsonConvert.SerializeObject(new {
NotificationType = "Delivery",
Mail = new {
Timestamp = "1561095094474",
Source = "Example Agency <*****@*****.**>",
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
Destination = new List <string>()
{
"Email_Body_Log"
},
CommonHeaders = new {
ReturnPath = "",
From = new List <string>()
{
"Example Company Admin <*****@*****.**>",
},
ReplyTo = new List <string>()
{
"*****@*****.**"
},
To = new List <string>()
{
"*****@*****.**"
},
Subject = "ExampleSubject",
},
},
Delivery = new {
TimeStamp = "1561095094474",
ProcessingTimeMillis = "1234",
SmtpResponse = "250 2.0.0 Ok: queued as 123456789123",
RemoteMtaIp = "111.22.333.444",
ReportingMTA = "ExampleMTA",
Recipients = new List <dynamic>()
{
"*****@*****.**",
}
}
}),
}),
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000"
};
var result = emailDeliveryNotificationLogProcessor.Process(awsSqsMessage);
//Then
saveEmailNotificationService.Verify((sens) => sens.SaveEmailMessage(It.IsAny <Dictionary <string, string> >()), Times.Exactly(1));
Assert.True(result, "created should return true");
}
public override bool Process(AwsSqsMessage awsSqsMessage)
{
var notificationLogBodyAnon = JsonConvert.DeserializeAnonymousType(awsSqsMessage.Body, new {
Type = "",
Timestamp = "",
MessageId = "",
TopicArn = "",
Message = "",
});
var notificationLogBodyMessageAnon = JsonConvert.DeserializeAnonymousType(notificationLogBodyAnon.Message, new {
notificationType = "",
mail = new {
timestamp = "",
Source = "",
messageID = "",
destination = new List <string>(),
commonHeaders = new {
returnPath = "",
from = new List <string>(),
replyTo = new List <string>(),
to = new List <string>(),
subject = ""
}
},
bounce = new {
bounceType = "",
bounceSubType = "",
bouncedRecipients = new List <dynamic>(),
timestamp = "",
smtpResponse = "",
remoteMtaIp = "",
reportingMTA = ""
},
});
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "NotificationBodyMessageId", notificationLogBodyMessageAnon.mail.messageID },
{ "NotificationBodyTopicARN", notificationLogBodyAnon.TopicArn },
{ "NotificationBodyType", notificationLogBodyMessageAnon.notificationType },
{ "NotificationBodyTimestamp", notificationLogBodyAnon.Timestamp.ToString() },
{ "NotificationBodyMailSource", notificationLogBodyMessageAnon.mail.Source },
{ "NotificationBodyCommonHeadersSubject", notificationLogBodyMessageAnon.mail.commonHeaders.subject },
{ "NotificationBodyBounceType", notificationLogBodyMessageAnon.bounce.bounceType },
{ "NotificationBodyBounceSubType", notificationLogBodyMessageAnon.bounce.bounceSubType },
{ "NotificationBodyBounceTimestamp", notificationLogBodyMessageAnon.bounce.timestamp },
{ "NotificationBodyBounceSMTPResponse", notificationLogBodyMessageAnon.bounce.smtpResponse },
{ "NotificationBodyBounceRemoteMTAIp", notificationLogBodyMessageAnon.bounce.remoteMtaIp },
{ "NotificationBodyBounceReportingMTA", notificationLogBodyMessageAnon.bounce.reportingMTA },
};
var bouncedRecipients = "";
for (var index = 0; index < notificationLogBodyMessageAnon.bounce.bouncedRecipients.Count; index++)
{
bouncedRecipients += $"{notificationLogBodyMessageAnon.bounce.bouncedRecipients[index]},";
}
dataDictToBeStored.Add("NotificationBodyBounceBouncedRecipients", bouncedRecipients);
for (var index = 0; index < notificationLogBodyMessageAnon.mail.commonHeaders.from.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersFrom" + (index + 1), notificationLogBodyMessageAnon.mail.commonHeaders.from[index]);
}
for (var index = 0; index < notificationLogBodyMessageAnon.mail.commonHeaders.to.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersTo" + (index + 1), notificationLogBodyMessageAnon.mail.commonHeaders.to[index]);
}
if (notificationLogBodyMessageAnon.mail.commonHeaders.replyTo != null)
{
for (var index = 0; index < notificationLogBodyMessageAnon.mail.commonHeaders.replyTo.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersReplyTo" + (index + 1), notificationLogBodyMessageAnon.mail.commonHeaders.replyTo[index]);
}
}
if (notificationLogBodyMessageAnon.mail.destination != null)
{
for (var index = 0; index < notificationLogBodyMessageAnon.mail.destination.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyDestination" + (index + 1), notificationLogBodyMessageAnon.mail.destination[index]);
}
}
_saveEmailNotificationService.SaveEmailMessage(dataDictToBeStored);
return(true);
}
public void LIVE_STRESS_TEST_multiple_producers_multiple_queues_one_consumer_per_queue()
{
var messagesPerProducer = 100;
var producerCount = 1;
var client = CreateLiveClient();
var queues = new List <string>();
foreach (var x in new[] { "a", "b", "c", "d" })
{
var queue = "test-" + x + "-" + StringUtil.CreateAlphaNumericKey(4);
client.CreateQueue(queue, new Result <AwsSqsResponse>()).Wait();
queues.Add(queue);
}
try {
var producers = new List <Result <List <string> > >();
for (var i = 0; i < producerCount; i++)
{
var producer = i;
producers.Add(AsyncUtil.Fork(() => {
_log.DebugFormat("producer {0} started", producer);
var c = CreateLiveClient();
var msgs = new List <string>();
for (var j = 0; j < messagesPerProducer; j++)
{
var msg = StringUtil.CreateAlphaNumericKey(1024);
foreach (var queue in queues)
{
c.Send(queue, AwsSqsMessage.FromBody(msg), new Result <AwsSqsSendResponse>()).Wait();
}
msgs.Add(msg);
if (msgs.Count % 10 == 0)
{
_log.DebugFormat("producer {0} sent {1}/{2} msgs", producer, msgs.Count, messagesPerProducer);
}
}
_log.DebugFormat("producer {0} finished", producer);
return(msgs);
}, new Result <List <string> >()));
}
var consumers = queues.ToDictionary(queue => queue, queue => AsyncUtil.Fork(() => {
_log.DebugFormat("consumer {0} started", queue);
var c = CreateLiveClient();
var msgs = new List <string>();
var expected = messagesPerProducer * producerCount;
var lastReport = 0;
while (msgs.Count < expected)
{
var received = c.ReceiveMax(queue, new Result <IEnumerable <AwsSqsMessage> >()).Wait();
var count = 0;
foreach (var msg in received)
{
count++;
msgs.Add(msg.Body);
c.Delete(msg, new Result <AwsSqsResponse>()).Wait();
}
if (count > 0 && msgs.Count > lastReport + 10)
{
_log.DebugFormat("consumer '{0}' received: {1}/{2}", queue, msgs.Count, expected);
lastReport = msgs.Count;
}
}
return(msgs);
}, new Result <List <string> >()));
producers.Join(new Result()).Wait();
consumers.Values.Join(new Result()).Wait();
var allMessages = producers.SelectMany(x => x.Value).OrderBy(x => x).ToArray();
foreach (var consumed in consumers)
{
var queue = consumed.Key;
var messages = consumed.Value.Value.OrderBy(x => x).ToArray();
Assert.AreEqual(allMessages, messages, string.Format("message list for queue '{0}' is wrong", queue));
}
} finally {
foreach (var queue in queues)
{
_log.DebugFormat("cleaning up queue '{0}'", queue);
client.DeleteQueue(queue, new Result <AwsSqsResponse>()).Wait();
}
}
}
public override bool Process(AwsSqsMessage awsSqsMessage)
{
var notificationLogBodyAnon = JsonConvert.DeserializeAnonymousType(awsSqsMessage.Body, new {
Type = "",
Timestamp = "",
MessageId = "",
TopicArn = "",
Message = "",
});
var notificationLogBodyMessageAnon = JsonConvert.DeserializeAnonymousType(notificationLogBodyAnon.Message, new {
NotificationType = "",
Mail = new {
TimeStamp = "",
Source = "",
MessageID = "",
Destination = new List <string>(),
CommonHeaders = new {
ReturnPath = "",
From = new List <string>(),
ReplyTo = new List <string>(),
To = new List <string>(),
Subject = ""
}
},
Delivery = new {
TimeStamp = "",
ProcessingTimeMillis = default(int),
SmtpResponse = "",
RemoteMtaIp = "",
ReportingMTA = "",
Recipients = new List <dynamic>(),
},
});
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "NotificationBodyMessageId", notificationLogBodyMessageAnon.Mail.MessageID },
{ "NotificationBodyTopicARN", notificationLogBodyAnon.TopicArn },
{ "NotificationBodyType", notificationLogBodyMessageAnon.NotificationType },
{ "NotificationBodyTimestamp", notificationLogBodyAnon.Timestamp.ToString() },
{ "NotificationBodyMailSource", notificationLogBodyMessageAnon.Mail.Source },
{ "NotificationBodyCommonHeadersSubject", notificationLogBodyMessageAnon.Mail.CommonHeaders.Subject },
{ "NotificationBodyDeliveryTimestamp", notificationLogBodyMessageAnon.Delivery.TimeStamp },
{ "NotificationBodyDeliveryProcessingTimeMs", notificationLogBodyMessageAnon.Delivery.ProcessingTimeMillis.ToString() },
{ "NotificationBodyDeliverySMTPResponse", notificationLogBodyMessageAnon.Delivery.SmtpResponse },
{ "NotificationBodyDeliveryRemoteMtaIp", notificationLogBodyMessageAnon.Delivery.RemoteMtaIp },
{ "NotificationBodyDeliveryReportingMTA", notificationLogBodyMessageAnon.Delivery.ReportingMTA },
};
var recipients = "";
for (var index = 0; index < notificationLogBodyMessageAnon.Delivery.Recipients.Count; index++)
{
recipients += $"{notificationLogBodyMessageAnon.Delivery.Recipients[index]},";
}
dataDictToBeStored.Add("Recipients", recipients);
for (var index = 0; index < notificationLogBodyMessageAnon.Mail.CommonHeaders.From.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersFrom" + (index + 1), notificationLogBodyMessageAnon.Mail.CommonHeaders.From[index]);
}
for (var index = 0; index < notificationLogBodyMessageAnon.Mail.CommonHeaders.To.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersTo" + (index + 1), notificationLogBodyMessageAnon.Mail.CommonHeaders.To[index]);
}
if (notificationLogBodyMessageAnon.Mail.CommonHeaders.ReplyTo != null)
{
for (var index = 0; index < notificationLogBodyMessageAnon.Mail.CommonHeaders.ReplyTo.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyCommonHeadersReplyTo" + (index + 1), notificationLogBodyMessageAnon.Mail.CommonHeaders.ReplyTo[index]);
}
}
if (notificationLogBodyMessageAnon.Mail.Destination != null)
{
for (var index = 0; index < notificationLogBodyMessageAnon.Mail.Destination.Count - 1; index++)
{
dataDictToBeStored.Add("NotificationBodyDestination" + (index + 1), notificationLogBodyMessageAnon.Mail.Destination[index]);
}
}
_saveEmailNotificaitonService.SaveEmailMessage(dataDictToBeStored);
return(true);
}
public void CanProcessEmailComplaintNotificationMessage()
{
//Given
var complainedRecipients = new List <dynamic>()
{
new {
emailAddress = "*****@*****.**",
}
};
Dictionary <string, string> dataDictToBeStored = new Dictionary <string, string>()
{
{ "NotificationBodyMessageId", "0000000000000000-00000000-0000-0000-0000-000000000000-000000" },
{ "NotificationBodyTopicARN", "NotificationTestBodyTopicARN" },
{ "NotificationBodyType", "Complaint" },
{ "NotificationBodyTimestamp", "1561095094474" },
{ "NotificationBodyMailSource", "Example Agency <*****@*****.**>" },
{ "NotificationBodyCommonHeadersSubject", "ExampleSubject" },
{ "NotificationBodyComplaintUserAgent", "AWS" },
{ "NotificationBodyComplaintFeedbackType", "Suppressed" },
{ "NotificationBodyComplaintFeedbackId", "1561095094474" },
{ "NotificationBodyComplaintTimestamp", "250 2.0.0 Ok:123456789123" },
{ "NotificationBodyCommonHeadersFrom", "Example Company Admin <*****@*****.**>" },
{ "NotificationBodyCommonHeadersTo", "*****@*****.**" },
{ "NotificationBodyCommonHeadersReplyTo", "*****@*****.**" },
{ "NotificationBodyDestination", "*****@*****.**" },
};
var logger = new Mock <ILogger <AppService> >();
var config = new Mock <IOptions <AppConfig> >();
var saveEmailNotificationService = new Mock <IEmailService>();
saveEmailNotificationService.Setup(sens => sens.SaveEmailMessage(It.IsAny <Dictionary <string, string> >())).Returns(true);
var emailComplaintNotificationLogProcessor = new EmailComplaintNotificationProcessor(logger.Object, config.Object, saveEmailNotificationService.Object);
//When
var awsSqsMessage = new AwsSqsMessage()
{
Body = JsonConvert.SerializeObject(new {
Type = "Notification",
Timestamp = "1561095094474",
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
TopicArn = "NotificationTestBodyTopicARN",
Message = JsonConvert.SerializeObject(new {
notificationType = "Bounce",
mail = new {
timestamp = "1561095094474",
source = "Example Agency <*****@*****.**",
messageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
destination = new List <string>()
{
"*****@*****.**"
},
commonHeaders = new {
returnPath = "",
from = new List <string>()
{
"Example Company Admin <*****@*****.**>",
},
replyTo = new List <string>()
{
"*****@*****.**"
},
to = new List <string>()
{
"*****@*****.**"
},
subject = "ExampleSubject",
}
},
complaint = new {
userAgent = "AWS",
complainedRecipients = new List <dynamic>()
{
new {
emailAddress = "*****@*****.**",
}
},
complaintFeedbackType = "abuse",
arrivalDate = "1561095094474",
timeStamp = "1561095094474",
feedbackId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000"
},
}),
}),
MessageId = "0000000000000000-00000000-0000-0000-0000-000000000000-000000",
};
var result = emailComplaintNotificationLogProcessor.Process(awsSqsMessage);
//Then
saveEmailNotificationService.Verify((sens) => sens.SaveEmailMessage(It.IsAny <Dictionary <string, string> >()), Times.Exactly(1));
Assert.True(result, "created should return true");
}
