public async Task When_a_group_is_prepared_and_SC_is_started_the_group_is_marked_as_failed()
{
var domainEvents = new FakeDomainEvents();
var retryManager = new RetryingManager(domainEvents);
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
await CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", false, 1);
new RetryBatches_ByStatusAndSession().Execute(documentStore);
new FailedMessageRetries_ByBatch().Execute(documentStore);
documentStore.WaitForIndexing();
var documentManager = new CustomRetryDocumentManager(false, documentStore)
{
OperationManager = retryManager
};
var orphanage = new FailedMessageRetries.AdoptOrphanBatchesFromPreviousSession(documentManager, null, documentStore);
await orphanage.AdoptOrphanedBatchesAsync();
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.True(status.Failed);
}
}
public async Task When_a_group_is_forwarded_the_status_is_Completed()
{
var domainEvents = new FakeDomainEvents();
var retryManager = new RetryingManager(domainEvents);
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
await CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", true, 1);
var sender = new TestSender();
var bodyStorage = new RavenAttachmentsBodyStorage
{
DocumentStore = documentStore
};
var returnToSender = new TestReturnToSenderDequeuer(new ReturnToSender(bodyStorage), documentStore, domainEvents, "TestEndpoint");
var processor = new RetryProcessor(sender, domainEvents, returnToSender, retryManager);
using (var session = documentStore.OpenAsyncSession())
{
await processor.ProcessBatches(session, CancellationToken.None); // mark ready
await session.SaveChangesAsync();
await processor.ProcessBatches(session, CancellationToken.None);
await session.SaveChangesAsync();
}
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.AreEqual(RetryState.Completed, status.RetryState);
}
}
public async Task ArchiveGroup_skips_over_empty_batches_but_still_completes()
{
// Arrange
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
var groupId = "TestGroup";
var previousArchiveBatchId = ArchiveBatch.MakeId(groupId, ArchiveType.FailureGroup, 1);
using (var session = documentStore.OpenAsyncSession())
{
var previousAchiveBatch = new ArchiveBatch {
Id = previousArchiveBatchId
};
await session.StoreAsync(previousAchiveBatch)
.ConfigureAwait(false);
var previousArchiveOperation = new ArchiveOperation
{
Id = ArchiveOperation.MakeId(groupId, ArchiveType.FailureGroup),
RequestId = groupId,
ArchiveType = ArchiveType.FailureGroup,
TotalNumberOfMessages = 2,
NumberOfMessagesArchived = 0,
Started = DateTime.Now,
GroupName = "Test Group",
NumberOfBatches = 3,
CurrentBatch = 0
};
await session.StoreAsync(previousArchiveOperation)
.ConfigureAwait(false);
await session.SaveChangesAsync()
.ConfigureAwait(false);
}
var documentManager = new ArchiveDocumentManager();
var domainEvents = new FakeDomainEvents();
var archivingManager = new ArchivingManager(domainEvents);
var retryingManager = new RetryingManager(domainEvents);
var handler = new ArchiveAllInGroupHandler(documentStore, domainEvents, documentManager, archivingManager, retryingManager);
var context = new TestableMessageHandlerContext();
var message = new ArchiveAllInGroup {
GroupId = groupId
};
// Act
await handler.Handle(message, context)
.ConfigureAwait(false);
// Assert
using (var session = documentStore.OpenSession())
{
var loadedBatch = session.Load <ArchiveBatch>(previousArchiveBatchId);
Assert.IsNull(loadedBatch);
}
}
}
public async Task When_a_group_is_prepared_with_three_batches_and_SC_is_restarted_while_the_first_group_is_being_forwarded_then_the_count_still_matches()
{
var domainEvents = new FakeDomainEvents();
var retryManager = new RetryingManager(domainEvents);
RetryingManager.RetryOperations = new Dictionary <string, InMemoryRetry>();
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", true, 2001);
new RetryBatches_ByStatus_ReduceInitialBatchSize().Execute(documentStore);
var sender = new TestSender();
var bodyStorage = new RavenAttachmentsBodyStorage
{
DocumentStore = documentStore
};
var settingsHolder = new NServiceBus.Settings.SettingsHolder();
settingsHolder.Set("EndpointName", "TestEndpoint");
var configure = new Configure(settingsHolder, new TestContainer(), new List <Action <NServiceBus.ObjectBuilder.IConfigureComponents> >(), new NServiceBus.Pipeline.PipelineSettings(new BusConfiguration()));
var processor = new RetryProcessor(sender, domainEvents, new TestReturnToSenderDequeuer(bodyStorage, sender, documentStore, domainEvents, configure), retryManager);
documentStore.WaitForIndexing();
using (var session = documentStore.OpenAsyncSession())
{
await processor.ProcessBatches(session, CancellationToken.None); // mark ready
await session.SaveChangesAsync();
// Simulate SC restart
retryManager = new RetryingManager(domainEvents);
RetryingManager.RetryOperations = new Dictionary <string, InMemoryRetry>();
var documentManager = new CustomRetryDocumentManager(false, documentStore)
{
OperationManager = retryManager
};
await documentManager.RebuildRetryOperationState(session);
processor = new RetryProcessor(sender, domainEvents, new TestReturnToSenderDequeuer(bodyStorage, sender, documentStore, domainEvents, configure), retryManager);
await processor.ProcessBatches(session, CancellationToken.None);
await session.SaveChangesAsync();
}
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.AreEqual(2001, status.TotalNumberOfMessages);
}
}
public async Task When_there_is_one_poison_message_it_is_removed_from_batch_and_the_status_is_Complete()
{
var domainEvents = new FakeDomainEvents();
var retryManager = new RetryingManager(domainEvents);
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
await CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", true, "A", "B", "C");
var sender = new TestSender();
sender.Callback = operation =>
{
//Always fails staging message B
if (operation.Message.MessageId == "FailedMessages/B")
{
throw new Exception("Simulated");
}
};
var bodyStorage = new RavenAttachmentsBodyStorage
{
DocumentStore = documentStore
};
var returnToSender = new TestReturnToSenderDequeuer(new ReturnToSender(bodyStorage), documentStore, domainEvents, "TestEndpoint");
var processor = new RetryProcessor(documentStore, sender, domainEvents, returnToSender, retryManager);
bool c;
do
{
try
{
using (var session = documentStore.OpenAsyncSession())
{
c = await processor.ProcessBatches(session, CancellationToken.None);
await session.SaveChangesAsync();
}
}
catch (Exception)
{
//Continue trying until there is no exception -> poison message is removed from the batch
c = true;
}
} while (c);
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.AreEqual(RetryState.Completed, status.RetryState);
Assert.AreEqual(3, status.NumberOfMessagesPrepared);
Assert.AreEqual(2, status.NumberOfMessagesForwarded);
Assert.AreEqual(1, status.NumberOfMessagesSkipped);
}
}
public async Task When_a_group_is_processed_it_is_set_to_the_Preparing_state()
{
var domainEvents = new FakeDomainEvents();
var retryManager = new RetryingManager(domainEvents);
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
await CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", true, 1);
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.AreEqual(RetryState.Preparing, status.RetryState);
}
}
public void When_a_group_is_processed_it_is_set_to_the_Preparing_state()
{
var retryManager = new RetryingManager();
RetryingManager.RetryOperations = new Dictionary <string, InMemoryRetry>();
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", true, 1);
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.AreEqual(RetryState.Preparing, status.RetryState);
}
}
public async Task When_a_group_has_one_batch_out_of_two_forwarded_the_status_is_Forwarding()
{
var domainEvents = new FakeDomainEvents();
var retryManager = new RetryingManager(domainEvents);
RetryingManager.RetryOperations = new Dictionary <string, InMemoryRetry>();
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", true, 1001);
var bodyStorage = new RavenAttachmentsBodyStorage
{
DocumentStore = documentStore
};
var sender = new TestSender();
var settingsHolder = new NServiceBus.Settings.SettingsHolder();
settingsHolder.Set("EndpointName", "TestEndpoint");
var configure = new Configure(settingsHolder, new TestContainer(), new List <Action <NServiceBus.ObjectBuilder.IConfigureComponents> >(), new NServiceBus.Pipeline.PipelineSettings(new BusConfiguration()));
var processor = new RetryProcessor(sender, domainEvents, new TestReturnToSenderDequeuer(bodyStorage, sender, documentStore, domainEvents, configure), retryManager);
documentStore.WaitForIndexing();
using (var session = documentStore.OpenAsyncSession())
{
await processor.ProcessBatches(session, CancellationToken.None); // mark ready
await session.SaveChangesAsync();
await processor.ProcessBatches(session, CancellationToken.None);
await session.SaveChangesAsync();
}
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.AreEqual(RetryState.Forwarding, status.RetryState);
}
}
public void When_a_group_is_forwarded_the_status_is_Completed()
{
var retryManager = new RetryingManager();
RetryingManager.RetryOperations = new Dictionary <string, InMemoryRetry>();
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, "Test-group", true, 1);
var testBus = new TestBus();
var sender = new TestSender();
var bodyStorage = new RavenAttachmentsBodyStorage
{
DocumentStore = documentStore
};
var settingsHolder = new NServiceBus.Settings.SettingsHolder();
settingsHolder.Set("EndpointName", "TestEndpoint");
var configure = new Configure(settingsHolder, new TestContainer(), new List <Action <NServiceBus.ObjectBuilder.IConfigureComponents> >(), new NServiceBus.Pipeline.PipelineSettings(new BusConfiguration()));
var returnToSender = new TestReturnToSenderDequeuer(bodyStorage, sender, documentStore, testBus, configure);
var processor = new RetryProcessor(sender, testBus, returnToSender, retryManager);
using (var session = documentStore.OpenSession())
{
processor.ProcessBatches(session, CancellationToken.None); // mark ready
session.SaveChanges();
processor.ProcessBatches(session, CancellationToken.None);
session.SaveChanges();
}
var status = retryManager.GetStatusForRetryOperation("Test-group", RetryType.FailureGroup);
Assert.AreEqual(RetryState.Completed, status.RetryState);
}
}
async Task CreateAFailedMessageAndMarkAsPartOfRetryBatch(IDocumentStore documentStore, RetryingManager retryManager, string groupId, bool progressToStaged, int numberOfMessages)
{
var messages = Enumerable.Range(0, numberOfMessages).Select(i =>
{
var id = Guid.NewGuid().ToString();
return(new FailedMessage
{
Id = FailedMessage.MakeDocumentId(id),
UniqueMessageId = id,
FailureGroups = new List <FailedMessage.FailureGroup>
{
new FailedMessage.FailureGroup
{
Id = groupId,
Title = groupId,
Type = groupId
}
},
Status = FailedMessageStatus.Unresolved,
ProcessingAttempts = new List <FailedMessage.ProcessingAttempt>
{
new FailedMessage.ProcessingAttempt
{
AttemptedAt = DateTime.UtcNow,
MessageMetadata = new Dictionary <string, object>(),
FailureDetails = new FailureDetails(),
Headers = new Dictionary <string, string>()
}
}
});
});
using (var session = documentStore.OpenAsyncSession())
{
foreach (var message in messages)
{
await session.StoreAsync(message);
}
await session.SaveChangesAsync();
}
new FailedMessages_ByGroup().Execute(documentStore);
documentStore.WaitForIndexing();
var documentManager = new CustomRetryDocumentManager(progressToStaged, documentStore);
var gateway = new RetriesGateway(documentStore, documentManager);
documentManager.OperationManager = retryManager;
gateway.OperationManager = retryManager;
gateway.StartRetryForIndex <FailureGroupMessageView, FailedMessages_ByGroup>("Test-group", RetryType.FailureGroup, DateTime.UtcNow, x => x.FailureGroupId == "Test-group", "Test-Context");
documentStore.WaitForIndexing();
await gateway.ProcessNextBulkRetry();
}
Task CreateAFailedMessageAndMarkAsPartOfRetryBatch(IDocumentStore documentStore, RetryingManager retryManager, string groupId, bool progressToStaged, int numberOfMessages)
{
return(CreateAFailedMessageAndMarkAsPartOfRetryBatch(documentStore, retryManager, groupId, progressToStaged, Enumerable.Range(0, numberOfMessages).Select(i => Guid.NewGuid().ToString()).ToArray()));
}
internal FailureGroupsRetryController(Lazy <IEndpointInstance> bus, RetryingManager retryingManager)
{
this.bus = bus;
this.retryingManager = retryingManager;
}