public async Task PiggyBack_PullRequest_With_Receipt_Operation_Becomes_DeadLettered_When_Retries_Are_Exhausted()
{
// Arrange
string pullSendUrl = RetrievePullingUrlFromConfig();
var user = new UserMessage($"user-{Guid.NewGuid()}", PullRequestMpc);
var receipt = new Receipt($"receipt-{Guid.NewGuid()}", user.MessageId);
InsertUserMessage(user);
long id = InsertReceipt(receipt, pullSendUrl, Operation.ToBePiggyBacked);
// Act
InsertRetryReliability(id, maxRetryCount: 1);
// Assert
await PollUntilPresent(
() => _databaseSpy.GetOutMessageFor(
m => m.EbmsMessageId == receipt.MessageId &&
m.Operation == Operation.DeadLettered),
timeout : TimeSpan.FromSeconds(40));
RetryReliability reliability = await PollUntilPresent(
() => _databaseSpy.GetRetryReliabilityFor(
r => r.RefToOutMessageId == id &&
r.Status == RetryStatus.Completed),
timeout : TimeSpan.FromSeconds(5));
Assert.True(
reliability.CurrentRetryCount > 0,
"RetryReliability.CurrentRetryCount should be greater then zero");
}
private void InsertRetryReliability(long id, int maxRetryCount)
{
_databaseSpy.InsertRetryReliability(
RetryReliability.CreateForOutMessage(
refToOutMessageId: id,
maxRetryCount: maxRetryCount,
retryInterval: TimeSpan.FromMilliseconds(1),
type: RetryType.PiggyBack));
}
public async Task OutMessage_Is_Set_To_Sent_When_Retry_Happen_Within_Allowed_MaxRetry(
HttpStatusCode secondAttempt,
Operation expected)
{
await TestComponentWithSettings(
"sendagent_settings.xml",
async (settings, as4Msh) =>
{
// Arrange
const string url = "http://localhost:7171/business/sending/";
SendingProcessingMode pmode = ReceptionAwarenessSendingPMode(url);
OutMessage im = CreateOutMessageRefStoredAS4Message(as4Msh);
InsertMessageEntityWithRetry(
entity: im,
config: as4Msh.GetConfiguration(),
pmode: pmode,
createRetry: id => RetryReliability.CreateForOutMessage(
id,
pmode.Reliability.ReceptionAwareness.RetryCount,
pmode.Reliability.ReceptionAwareness.RetryInterval.AsTimeSpan(),
RetryType.Send));
// Act
SimulateSendingFailureOnFirstAttempt(url, secondAttempt);
// Assert
var spy = new DatabaseSpy(as4Msh.GetConfiguration());
OutMessage sent = await PollUntilPresent(
() => spy.GetOutMessageFor(m => m.Operation == expected),
timeout: TimeSpan.FromSeconds(10));
Assert.Equal(im.EbmsMessageId, sent.EbmsMessageId);
RetryReliability referenced = await PollUntilPresent(
() => spy.GetRetryReliabilityFor(r => r.RefToOutMessageId == sent.Id),
timeout: TimeSpan.FromSeconds(5));
Assert.Equal(RetryStatus.Completed, referenced.Status);
InMessage deadLetteredError =
spy.GetInMessageFor(m => m.EbmsRefToMessageId == sent.EbmsMessageId);
bool storedDeadLetteredError =
deadLetteredError?.Operation == Operation.ToBeNotified &&
deadLetteredError?.EbmsMessageType == MessageType.Error;
Assert.True(
storedDeadLetteredError == (expected == Operation.DeadLettered),
"Expected to have stored AS4 Error for DeadLettered message");
});
}
public async Task Update_RetryReliability_To_Pending_When_Receiver_Is_Offline()
{
// Arrange
string ebmsMessageId = $"user-{Guid.NewGuid()}";
AS4Message tobeSendMessage = AS4Message.Create(new UserMessage(ebmsMessageId));
var outMessage = new OutMessage(ebmsMessageId);
GetDataStoreContext.InsertOutMessage(outMessage);
GetDataStoreContext.InsertRetryReliability(
RetryReliability.CreateForOutMessage(
refToOutMessageId: outMessage.Id,
maxRetryCount: 2,
retryInterval: TimeSpan.FromSeconds(1),
type: RetryType.Send));
var ctx = new MessagingContext(
tobeSendMessage,
new ReceivedEntityMessage(
outMessage,
tobeSendMessage.ToStream(),
tobeSendMessage.ContentType),
MessagingContextMode.Send)
{
SendingPMode = CreateSendPModeWithPushUrl()
};
var sabotageException = new WebException("Remote host not available");
IStep sut = CreateSendStepWithResponse(
StubHttpClient.ThatThrows(sabotageException));
// Act
StepResult result = await sut.ExecuteAsync(ctx);
// Assert
Assert.False(result.Succeeded);
GetDataStoreContext.AssertRetryRelatedOutMessage(
outMessage.Id,
r =>
{
Assert.NotNull(r);
Assert.Equal(RetryStatus.Pending, r.Status);
});
}
private static void InsertMessageEntityWithRetry(
OutMessage entity,
IConfig config,
SendingProcessingMode pmode,
Func <long, RetryReliability> createRetry)
{
using (var ctx = new DatastoreContext(config))
{
entity.SetPModeInformation(pmode);
entity.Operation = Operation.ToBeSent;
ctx.OutMessages.Add(entity);
ctx.SaveChanges();
RetryReliability r = createRetry(entity.Id);
ctx.RetryReliability.Add(r);
ctx.SaveChanges();
}
}
public async Task Message_Is_Set_To_Exception_If_Delivery_Fails_After_Exhausted_Retries()
{
// Arrang
AS4Message as4Message = CreateAS4Message();
// Act
await TestDeliverRetryByBlockingDeliveryLocationFor(as4Message, TimeSpan.FromSeconds(30));
// Assert
InMessage actual =
await PollUntilPresent(
() => _databaseSpy.GetInMessageFor(m => m.Operation == Operation.DeadLettered),
timeout : TimeSpan.FromSeconds(10));
RetryReliability rr =
_databaseSpy.GetRetryReliabilityFor(r => r.RefToInMessageId == actual.Id);
Assert.Equal(rr.CurrentRetryCount, rr.MaxRetryCount);
Assert.Equal(RetryStatus.Completed, rr.Status);
}
public async Task Reset_InMessage_Operation_ToBeDelivered_When_CurrentRetry_LessThen_MaxRetry(DeliverRetry input)
{
// Arrange
string id = Guid.NewGuid().ToString();
InMessage im = CreateInMessage(id, InStatus.Received, Operation.Delivering);
GetDataStoreContext.InsertInMessage(im);
var r = RetryReliability.CreateForInMessage(
refToInMessageId: im.Id,
maxRetryCount: input.MaxRetryCount,
retryInterval: default(TimeSpan),
type: RetryType.Notification);
r.CurrentRetryCount = input.CurrentRetryCount;
GetDataStoreContext.InsertRetryReliability(r);
DeliverMessageEnvelope envelope = AnonymousDeliverEnvelope(id);
var stub = new Mock <IDeliverSender>();
stub.Setup(s => s.SendAsync(envelope))
.ReturnsAsync(input.SendResult);
IStep sut = CreateSendDeliverStepWithSender(stub.Object);
// Act
await sut.ExecuteAsync(new MessagingContext(envelope) { ReceivingPMode = CreateDefaultReceivingPMode() });
// Assert
GetDataStoreContext.AssertInMessage(id, inMessage =>
{
Assert.NotNull(inMessage);
Assert.Equal(input.ExpectedStatus, inMessage.Status.ToEnum <InStatus>());
Assert.Equal(input.ExpectedOperation, inMessage.Operation);
});
}
public async Task Message_Is_Set_To_Delivered_After_Its_Being_Retried()
{
// Arrang
AS4Message as4Message = CreateAS4Message();
// Act
await TestDeliverRetryByBlockingDeliveryLocationFor(as4Message, TimeSpan.FromSeconds(1));
// Assert
InMessage actual =
await PollUntilPresent(
() => _databaseSpy.GetInMessageFor(m => m.Operation == Operation.Delivered),
timeout : TimeSpan.FromSeconds(10));
// Assert
Assert.Equal(InStatus.Delivered, actual.Status.ToEnum <InStatus>());
Assert.Equal(Operation.Delivered, actual.Operation);
RetryReliability rr = _databaseSpy.GetRetryReliabilityFor(r => r.RefToInMessageId == actual.Id);
Assert.True(0 < rr.CurrentRetryCount, "0 < actualMessage.CurrentRetryCount");
Assert.Equal(RetryStatus.Completed, rr.Status);
}
public async Task Retries_Uploading_When_Uploader_Returns_RetryableFail_Result(UploadRetry input)
{
// Arrange
string id = "deliver-" + Guid.NewGuid();
InMessage im = InsertInMessage(id);
var r = RetryReliability.CreateForInMessage(
refToInMessageId: im.Id,
maxRetryCount: input.MaxRetryCount,
retryInterval: default(TimeSpan),
type: RetryType.Delivery);
r.CurrentRetryCount = input.CurrentRetryCount;
GetDataStoreContext.InsertRetryReliability(r);
var a = new FilledAttachment();
var userMessage = new FilledUserMessage(id, a.Id);
AS4Message as4Msg = AS4Message.Create(userMessage);
as4Msg.AddAttachment(a);
MessagingContext fixture = await PrepareAS4MessageForDeliveryAsync(as4Msg, CreateReceivingPModeWithPayloadMethod());
IAttachmentUploader stub = CreateStubAttachmentUploader(fixture.DeliverMessage.Message.MessageInfo, input.UploadResult);
// Act
await CreateUploadStep(stub).ExecuteAsync(fixture);
// Assert
GetDataStoreContext.AssertInMessage(id, actual =>
{
Assert.NotNull(actual);
Assert.Equal(input.ExpectedStatus, actual.Status.ToEnum <InStatus>());
Assert.Equal(input.ExpectedOperation, actual.Operation);
});
}
/// <summary>
/// Mark the stored <see cref="OutMessage"/> for retry/delayed piggy backing.
/// </summary>
/// <param name="inserts"></param>
/// <param name="reliability"></param>
internal void InsertRetryForPiggyBackedSignalMessages(IEnumerable <OutMessage> inserts, Model.PMode.RetryReliability reliability)
{
if (reliability?.IsEnabled == true)
{
foreach (OutMessage m in inserts.Where(i => i.Operation == Operation.ToBePiggyBacked))
{
var r = RetryReliability.CreateForOutMessage(
refToOutMessageId: m.Id,
maxRetryCount: reliability.RetryCount,
retryInterval: reliability.RetryInterval.AsTimeSpan(),
type: RetryType.PiggyBack);
Logger.Debug(
$"Insert RetryReliability for ToBePiggyBacked SignalMessage OutMessage {m.EbmsMessageId} with "
+ $"{{RetryCount={r.MaxRetryCount}, RetryInterval={r.RetryInterval}}}");
_repository.InsertRetryReliability(r);
}
}
else
{
Logger.Debug("Will not insert RetryReliability because ReceivingPMode.ReplyHandling.Reliability is not enabeld");
}
}
public async Task All_Attachments_Should_Succeed_Or_Fail(UploadRetry input)
{
// Arrange
string id = "deliver-" + Guid.NewGuid();
InMessage im = InsertInMessage(id);
var r = RetryReliability.CreateForInMessage(
refToInMessageId: im.Id,
maxRetryCount: input.MaxRetryCount,
retryInterval: default(TimeSpan),
type: RetryType.Delivery);
r.CurrentRetryCount = input.CurrentRetryCount;
GetDataStoreContext.InsertRetryReliability(r);
var a1 = new FilledAttachment("attachment-1");
var a2 = new FilledAttachment("attachment-2");
var userMessage = new FilledUserMessage(id, a1.Id, a2.Id);
var as4Msg = AS4Message.Create(userMessage);
as4Msg.AddAttachment(a1);
as4Msg.AddAttachment(a2);
MessagingContext fixture = await PrepareAS4MessageForDeliveryAsync(as4Msg, CreateReceivingPModeWithPayloadMethod());
var stub = new Mock <IAttachmentUploader>();
stub.Setup(s => s.UploadAsync(a1, fixture.DeliverMessage.Message.MessageInfo))
.ReturnsAsync(input.UploadResult);
stub.Setup(s => s.UploadAsync(a2, fixture.DeliverMessage.Message.MessageInfo))
.ReturnsAsync(
input.UploadResult.Status == SendResult.Success
? UploadResult.FatalFail
: UploadResult.RetryableFail);
// Act
await CreateUploadStep(stub.Object).ExecuteAsync(fixture);
// Assert
GetDataStoreContext.AssertInMessage(id, actual =>
{
Assert.NotNull(actual);
Operation op = actual.Operation;
Assert.NotEqual(Operation.Delivered, op);
InStatus st = actual.Status.ToEnum <InStatus>();
Assert.NotEqual(InStatus.Delivered, st);
bool operationToBeRetried = Operation.ToBeRetried == op;
bool uploadResultCanBeRetried =
input.UploadResult.Status == SendResult.RetryableFail &&
input.CurrentRetryCount < input.MaxRetryCount;
Assert.True(
operationToBeRetried == uploadResultCanBeRetried,
"InMessage should update Operation=ToBeDelivered");
bool messageSetToException = Operation.DeadLettered == op && InStatus.Exception == st;
bool exhaustRetries =
input.CurrentRetryCount == input.MaxRetryCount ||
input.UploadResult.Status != SendResult.RetryableFail;
Assert.True(
messageSetToException == exhaustRetries,
$"{messageSetToException} != {exhaustRetries} InMessage should update Operation=DeadLettered, Status=Exception");
});
}