public async Task Should_not_dequeue_priority_dates_in_future()
{
string endpointName = "OutboxDequeueTests1.4";
var futureDate = DateTime.UtcNow.AddMinutes(2);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_1", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_2", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_4", OutboxMessageStatus.Ready, futureDate);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_5", OutboxMessageStatus.Ready, futureDate);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_6", OutboxMessageStatus.Ready, futureDate);
//var callback = A.Fake<Func<string, byte[], MessageMetaData, string, Task>>();
var callback = A.Fake <Dispatcher>();
await _fixture.Outbox.Relay(callback);
A.CallTo(() => callback(
A <string> .Ignored,
A <byte[]> .Ignored,
A <MessageMetaData> .Ignored,
A <string> .Ignored,
false)
).MustHaveHappened(2, Times.Exactly);
}
public async Task Should_cleanup_aged_status_to_processed()
{
string endpointName = "OutboxManagementTests_1.2";
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_1", OutboxMessageStatus.Ready, DateTime.UtcNow.AddMinutes(-61));
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "inprogress_2", OutboxMessageStatus.InProgress, DateTime.UtcNow.AddMinutes(-61));
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "failed_3", OutboxMessageStatus.Failed, DateTime.UtcNow.AddMinutes(-61));
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "processed_4", OutboxMessageStatus.Processed);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "processed_5", OutboxMessageStatus.Processed, DateTime.UtcNow.AddMinutes(-55));
var cleanup1 = await IntegrationTestFixture.CreateOutboxMessage(endpointName, "processed_6", OutboxMessageStatus.Processed, DateTime.UtcNow.AddMinutes(-60));
var cleanup2 = await IntegrationTestFixture.CreateOutboxMessage(endpointName, "processed_7", OutboxMessageStatus.Processed, DateTime.UtcNow.AddMinutes(-61));
await _fixture.Outbox.CleanUp(TimeSpan.FromMinutes(58));
using (var scope = NewScope())
{
var db = scope.ServiceProvider.GetService <ApplicationDbContext>();
var outboxMessages = db.Set <OutboxMessage>()
.OrderBy(x => x.CreatedAtUtc)
.AsNoTracking().ToList();
outboxMessages.Count.ShouldBe(5);
//there shoule only be 2 processed message left, the other 2 should be deletedf
outboxMessages.Where(x => x.Status == (int)OutboxMessageStatus.Processed).Count().ShouldBe(2);
}
}
public async Task Should_only_dequeue_ready_messages()
{
string endpointName = "OutboxDequeueTests1.2";
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_1", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_2", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "inprogress_3", OutboxMessageStatus.InProgress);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "processed_4", OutboxMessageStatus.Processed);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "failed_5", OutboxMessageStatus.Failed);
var callback = A.Fake <Dispatcher>();
await _fixture.Outbox.Relay(callback);
A.CallTo(() => callback(
A <string> .Ignored,
A <byte[]> .Ignored,
A <MessageMetaData> .That.Matches(x => x.MessageId.StartsWith("ready_")),
A <string> .Ignored,
false)
).MustHaveHappenedTwiceExactly();
A.CallTo(() => callback(
A <string> .Ignored,
A <byte[]> .Ignored,
A <MessageMetaData> .That.Matches(x => !x.MessageId.StartsWith("ready_")),
A <string> .Ignored,
false)
).MustNotHaveHappened();
using (var scope = NewScope())
{
var db = scope.ServiceProvider.GetService <ApplicationDbContext>();
var outboxMessages = db.Set <OutboxMessage>()
.OrderBy(x => x.CreatedAtUtc)
.AsNoTracking().ToList();
outboxMessages.Count.ShouldBe(5);
outboxMessages[0].Status.ShouldBe((int)OutboxMessageStatus.Processed);
outboxMessages[0].TryCount.ShouldBe(1);
outboxMessages[1].Status.ShouldBe((int)OutboxMessageStatus.Processed);
outboxMessages[1].TryCount.ShouldBe(1);
//these were never in the ready status
outboxMessages[2].TryCount.ShouldBe(0);
outboxMessages[3].TryCount.ShouldBe(0);
outboxMessages[4].TryCount.ShouldBe(0);
}
}
/// <summary>
/// Clear database once per session
/// </summary>
/// <returns></returns>
public virtual async Task InitializeAsync()
{
if (_initialized)
{
return;
}
using (await Mutex.LockAsync())
{
if (_initialized)
{
return;
}
await IntegrationTestFixture.ResetCheckpoint();
_initialized = true;
}
}
public async Task Should_reset_aged_status_to_inprogress()
{
string endpointName = "OutboxManagementTests_1.1";
var readyOrig = await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_1", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_2", OutboxMessageStatus.InProgress, DateTime.UtcNow.AddMinutes(-40));
var reset1 = await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_3", OutboxMessageStatus.InProgress, DateTime.UtcNow.AddHours(-1));
var reset2 = await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_4", OutboxMessageStatus.InProgress, DateTime.UtcNow.AddHours(-1.1));
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_5", OutboxMessageStatus.Processed);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_6", OutboxMessageStatus.Failed);
await _fixture.Outbox.Reset(TimeSpan.FromMinutes(58));
using (var scope = NewScope())
{
var db = scope.ServiceProvider.GetService <ApplicationDbContext>();
var outboxMessages = db.Set <OutboxMessage>().Include(x => x.MessageData)
.Where(x => x.Status == (int)OutboxMessageStatus.Ready)
.OrderBy(x => x.CreatedAtUtc)
.AsNoTracking().ToList();
outboxMessages.Count.ShouldBe(3);
outboxMessages[0].Status.ShouldBe((int)OutboxMessageStatus.Ready);
outboxMessages[0].TryCount.ShouldBe(0);
outboxMessages[0].PriorityDateUtc.ShouldBe(readyOrig.PriorityDateUtc);
outboxMessages[1].Status.ShouldBe((int)OutboxMessageStatus.Ready);
outboxMessages[1].TryCount.ShouldBe(0);
outboxMessages[1].PriorityDateUtc.ShouldBe(reset1.PriorityDateUtc);
outboxMessages[2].Status.ShouldBe((int)OutboxMessageStatus.Ready);
outboxMessages[2].TryCount.ShouldBe(0);
outboxMessages[2].PriorityDateUtc.ShouldBe(reset2.PriorityDateUtc);
}
}
public async Task Should_limit_dequeue_to_batch_size()
{
string endpointName = "OutboxDequeueTests1.3";
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_1", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_2", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_3", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_4", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_5", OutboxMessageStatus.Ready);
await IntegrationTestFixture.CreateOutboxMessage(endpointName, "ready_6", OutboxMessageStatus.Ready);
//var callback = A.Fake<Func<string, byte[], MessageMetaData, string, Task>>();
var callback = A.Fake <Dispatcher>();
await _fixture.Outbox.Relay(callback);
A.CallTo(() => callback(
A <string> .Ignored,
A <byte[]> .Ignored,
A <MessageMetaData> .Ignored,
A <string> .Ignored,
false)
).MustHaveHappened(_fixture.Outbox.BatchSize, Times.Exactly);
A.CallTo(() => callback(
A <string> .Ignored,
A <byte[]> .Ignored,
A <MessageMetaData> .That.Matches(x => !x.MessageId.StartsWith("ready_")),
A <string> .Ignored,
false)
).MustNotHaveHappened();
}
public async Task Should_update_priority_date_and_trycount_on_dispatch_failure_up_to_max()
{
string endpointName = "OutboxRetryTests_1.1";
var message = await IntegrationTestFixture.CreateOutboxMessage(endpointName, "retry_1", OutboxMessageStatus.Ready);
var lastPriorityDate = message.PriorityDateUtc;
var callback = A.Fake <Dispatcher>();
A.CallTo(() => callback(A <string> .Ignored, A <byte[]> .Ignored, A <MessageMetaData> .Ignored, A <string> .Ignored, false))
.Throws(new ApplicationException("Test: foreced vailure in outbox relay"));
await _fixture.Outbox.Relay(callback);
A.CallTo(() => callback(A <string> .Ignored, A <byte[]> .Ignored, A <MessageMetaData> .That.Matches(x => x.MessageId.StartsWith("retry_")), A <string> .Ignored, false))
.MustHaveHappenedOnceExactly();
using (var scope = NewScope())
{
var db = scope.ServiceProvider.GetService <ApplicationDbContext>();
var outboxMessages = db.Set <OutboxMessage>().OrderBy(x => x.CreatedAtUtc)
.AsNoTracking().ToList();
var outboxMessage = outboxMessages[0];
outboxMessage.Status.ShouldBe((int)OutboxMessageStatus.Ready);
outboxMessage.TryCount.ShouldBe(1);
outboxMessage.PriorityDateUtc.ShouldBeGreaterThan(lastPriorityDate);
lastPriorityDate = outboxMessage.PriorityDateUtc;
}
await Task.Delay(500);//wait the retry interval
//callback = A.Fake<Func<string, byte[], MessageMetaData, string, Task>>();
callback = A.Fake <Dispatcher>();
A.CallTo(() => callback(A <string> .Ignored, A <byte[]> .Ignored, A <MessageMetaData> .Ignored, A <string> .Ignored, false))
.Throws(new ApplicationException("Test: foreced vailure in outbox relay"));
await _fixture.Outbox.Relay(callback);
A.CallTo(() => callback(A <string> .Ignored, A <byte[]> .Ignored, A <MessageMetaData> .That.Matches(x => x.MessageId.StartsWith("retry_")), A <string> .Ignored, false))
.MustHaveHappenedOnceExactly();
using (var scope = NewScope())
{
var db = scope.ServiceProvider.GetService <ApplicationDbContext>();
var outboxMessages = db.Set <OutboxMessage>().OrderBy(x => x.CreatedAtUtc).AsNoTracking().ToList();
var outboxMessage = outboxMessages[0];
outboxMessage.Status.ShouldBe((int)OutboxMessageStatus.Ready);
outboxMessage.TryCount.ShouldBe(2);
outboxMessage.PriorityDateUtc.ShouldBeGreaterThan(lastPriorityDate);
lastPriorityDate = outboxMessage.PriorityDateUtc;
}
await Task.Delay(500);//wait the retry interval
//callback = A.Fake<Func<string, byte[], MessageMetaData, string, Task>>();
callback = A.Fake <Dispatcher>();
A.CallTo(() => callback(A <string> .Ignored, A <byte[]> .Ignored, A <MessageMetaData> .Ignored, A <string> .Ignored, false))
.Throws(new ApplicationException("Test: foreced vailure in outbox relay"));
await _fixture.Outbox.Relay(callback);
A.CallTo(() => callback(A <string> .Ignored, A <byte[]> .Ignored, A <MessageMetaData> .That.Matches(x => x.MessageId.StartsWith("retry_")), A <string> .Ignored, false))
.MustHaveHappenedOnceExactly();
using (var scope = NewScope())
{
var db = scope.ServiceProvider.GetService <ApplicationDbContext>();
var outboxMessages = db.Set <OutboxMessage>().OrderBy(x => x.CreatedAtUtc).AsNoTracking().ToList();
var outboxMessage = outboxMessages[0];
outboxMessage.TryCount.ShouldBe(3);
outboxMessage.Status.ShouldBe((int)OutboxMessageStatus.Failed);
outboxMessage.PriorityDateUtc.ShouldBe(lastPriorityDate);//don't incerment after failure
}
}
public async Task Should_hydrate_all_properties()
{
string endpointName = "OutboxDequeueTests1.1";
var messageMetaData = new MessageMetaData();
var serializedMessageMetaData = messageMetaData != null?JsonConvert.SerializeObject(messageMetaData) : null;
var message1 = new OutboxMessage("Test.Message1", _fixture.MessageBytes, "{ meta: 1}", DateTime.UtcNow, endpointName,
skipTransientDispatch: true,
expiresAtUtc: DateTime.UtcNow.AddMinutes(30),
id: Guid.NewGuid())
{
Status = (int)OutboxMessageStatus.Ready, TryCount = 0, CreatedAtUtc = DateTime.UtcNow.AddMinutes(-3)
};
var message2 = new OutboxMessage("Test.Message2", _fixture.MessageBytes, null, DateTime.UtcNow.AddSeconds(2), null,
skipTransientDispatch: true,
expiresAtUtc: null)
{
Status = (int)OutboxMessageStatus.InProgress, TryCount = 1, CreatedAtUtc = DateTime.UtcNow.AddMinutes(-2)
};
;
var message3 = new OutboxMessage("Test.Message3", _fixture.MessageBytes, "{ meta: 3}", DateTime.UtcNow.AddSeconds(3), endpointName,
skipTransientDispatch: false,
expiresAtUtc: null)
{
Status = (int)OutboxMessageStatus.Processed, TryCount = 2, CreatedAtUtc = DateTime.UtcNow.AddMinutes(-1)
};
;
await IntegrationTestFixture.CreateOutboxMessage(message1);
await IntegrationTestFixture.CreateOutboxMessage(message2);
await IntegrationTestFixture.CreateOutboxMessage(message3);
//if the number of parameters changes in the outbox implementation this should throw an error
using (var conn = new SqlConnectionFactory(ConnectionString).Get())
{
var sql = @"
select inserted.Id, inserted.PriorityDateUtc, inserted.[Type], inserted.Endpoint,
inserted.TryCount, inserted.[Status], inserted.ExpiresAtUtc, inserted.CreatedAtUtc,
od.Id, od.[Data], od.MetaData
FROM OutboxMessages inserted
INNER JOIN OutboxMessageData od
ON inserted.Id = od.Id
ORDER BY inserted.CreatedAtUtc asc
";
conn.Open();
DbCommand command = conn.CreateCommand();
command.CommandText = sql;
using (var reader = await command.ExecuteReaderAsync())
{
var outboxMessages = _fixture.Outbox.HydrateOutboxMessages(reader);
outboxMessages.Count().ShouldBe(3);
var o1 = outboxMessages.ElementAt(0);
var o2 = outboxMessages.ElementAt(1);
var o3 = outboxMessages.ElementAt(2);
o1.Id.ShouldBe(message1.Id);
o1.PriorityDateUtc.ShouldBe(message1.PriorityDateUtc);
o1.Type.ShouldBe(message1.Type);
o1.Endpoint.ShouldBe(message1.Endpoint);
o1.TryCount.ShouldBe(message1.TryCount);
o1.Status.ShouldBe(message1.Status);
o1.ExpiresAtUtc.ShouldBe(message1.ExpiresAtUtc);
o1.CreatedAtUtc.ShouldBe(message1.CreatedAtUtc);
o1.MessageData.Id.ShouldBe(message1.MessageData.Id);
o1.MessageData.Data.ShouldBe(message1.MessageData.Data);
o1.MessageData.MetaData.ShouldBe(message1.MessageData.MetaData);
o2.Id.ShouldBe(message2.Id);
o2.PriorityDateUtc.ShouldBe(message2.PriorityDateUtc);
o2.Type.ShouldBe(message2.Type);
o2.Endpoint.ShouldBe(message2.Endpoint);
o2.TryCount.ShouldBe(message2.TryCount);
o2.Status.ShouldBe(message2.Status);
o2.ExpiresAtUtc.ShouldBe(message2.ExpiresAtUtc);
o2.CreatedAtUtc.ShouldBe(message2.CreatedAtUtc);
o2.MessageData.Id.ShouldBe(message2.MessageData.Id);
o2.MessageData.Data.ShouldBe(message2.MessageData.Data);
o2.MessageData.MetaData.ShouldBe(message2.MessageData.MetaData);
o3.Id.ShouldBe(message3.Id);
o3.PriorityDateUtc.ShouldBe(message3.PriorityDateUtc);
o3.Type.ShouldBe(message3.Type);
o3.Endpoint.ShouldBe(message3.Endpoint);
o3.TryCount.ShouldBe(message3.TryCount);
o3.Status.ShouldBe(message3.Status);
o3.ExpiresAtUtc.ShouldBe(message3.ExpiresAtUtc);
o3.CreatedAtUtc.ShouldBe(message3.CreatedAtUtc);
o3.MessageData.Id.ShouldBe(message3.MessageData.Id);
o3.MessageData.Data.ShouldBe(message3.MessageData.Data);
o3.MessageData.MetaData.ShouldBe(message3.MessageData.MetaData);
}
}//using
}