public async Task Update(IEnumerable <ProcessedMsg> msgs)
{
using (var db = new Entities()) {
foreach (var msg in msgs)
{
if (!msg.IsSuccessed)
{
var fm = new FailedMessage()
{
MsgID = msg.MsgID,
MsgType = msg.MsgType,
Log = msg.Error.SafeSubString(1000)
};
this.SetCreateInfo(fm);
db.FailedMessages.Add(fm);
}
var handler = MessageHandlerFactory.GetHandler(msg.MsgType);
if (handler != null)
{
handler.Update(db, msg);
}
}
this.Errors = db.GetErrors();
if (!this.HasError)
{
await db.SaveChangesAsync();
}
}
}
public void It_should_be_able_to_be_retried_successfully()
{
var context = new MyContext {
Succeed = false
};
FailedMessage failure = null;
Define(context)
.WithEndpoint <FailureEndpoint>(b =>
b.Given(bus => bus.SendLocal(new MyMessage()))
.When(ctx => CheckProcessingAttemptsIs(ctx, 1),
(bus, ctx) => IssueRetry(ctx))
.When(ctx => CheckProcessingAttemptsIs(ctx, 2),
(bus, ctx) => IssueRetry(ctx))
.When(ctx => CheckProcessingAttemptsIs(ctx, 3),
(bus, ctx) =>
{
ctx.Succeed = true;
IssueRetry(ctx);
})
)
.Done(ctx => GetFailedMessage(ctx, out failure, f => f.Status == FailedMessageStatus.Resolved))
.Run(TimeSpan.FromMinutes(4));
Assert.IsNotNull(failure, "Failure should not be null");
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
}
public async Task Should_discard_edit_when_different_edit_already_exists()
{
var failedMessageId = Guid.NewGuid().ToString();
var previousEdit = Guid.NewGuid().ToString();
await CreateFailedMessage(failedMessageId);
using (var session = Store.OpenAsyncSession())
{
await session.StoreAsync(new FailedMessageEdit
{
Id = FailedMessageEdit.MakeDocumentId(failedMessageId),
FailedMessageId = failedMessageId,
EditId = previousEdit
});
await session.SaveChangesAsync();
}
var message = CreateEditMessage(failedMessageId);
await Handler.Handle(message, new TestableMessageHandlerContext());
using (var session = Store.OpenAsyncSession())
{
var failedMessage = await session.LoadAsync <FailedMessage>(FailedMessage.MakeDocumentId(failedMessageId));
var editOperation = await session.LoadAsync <FailedMessageEdit>(FailedMessageEdit.MakeDocumentId(failedMessageId));
Assert.AreEqual(FailedMessageStatus.Unresolved, failedMessage.Status);
Assert.AreEqual(previousEdit, editOperation.EditId);
}
Assert.IsEmpty(Dispatcher.DispatchedMessages);
}
public void SendMessage(FailedMessage failedMessage)
{
var props = _channel.CreateBasicProperties();
var deliveredFailedMessageProperties = System.Text.Json.JsonSerializer.Deserialize <BasicPropertiesHolder>(failedMessage.Properties);
props.AppId = deliveredFailedMessageProperties.AppId;
props.ClusterId = deliveredFailedMessageProperties.ClusterId;
props.ContentEncoding = deliveredFailedMessageProperties.ContentEncoding;
props.ContentType = deliveredFailedMessageProperties.ContentType;
props.CorrelationId = deliveredFailedMessageProperties.CorrelationId;
props.DeliveryMode = deliveredFailedMessageProperties.DeliveryMode;
props.Expiration = deliveredFailedMessageProperties.Expiration;
props.Headers = deliveredFailedMessageProperties.Headers.ToDictionary(x => x.Key, x => (object)Convert.FromBase64String(x.Value.ToString()));
props.MessageId = deliveredFailedMessageProperties.MessageId;
props.Persistent = deliveredFailedMessageProperties.Persistent;
props.Priority = deliveredFailedMessageProperties.Priority;
props.ReplyTo = deliveredFailedMessageProperties.ReplyTo;
if (deliveredFailedMessageProperties.ReplyToAddress != null)
{
props.ReplyToAddress = deliveredFailedMessageProperties.ReplyToAddress;
}
props.Timestamp = deliveredFailedMessageProperties.Timestamp;
props.Type = deliveredFailedMessageProperties.Type;
props.AppId = deliveredFailedMessageProperties.UserId;
var queue = failedMessage.Queue.Replace("_error", "");
_channel.BasicPublish("", queue, basicProperties: props, failedMessage.Content);
_logger.LogInformation("Retried {MessageId} into {queue}", props.MessageId, queue);
}
public void Errors_are_not_being_expired()
{
var failedMsg = new FailedMessage
{
Id = "1",
ProcessingAttempts = new List<FailedMessage.ProcessingAttempt>
{
new FailedMessage.ProcessingAttempt
{
AttemptedAt = DateTime.UtcNow.AddHours(-(Settings.HoursToKeepMessagesBeforeExpiring * 3))
},
new FailedMessage.ProcessingAttempt
{
AttemptedAt = DateTime.UtcNow.AddHours(-(Settings.HoursToKeepMessagesBeforeExpiring * 2)),
}
},
Status = FailedMessageStatus.Unresolved,
};
using (var session = documentStore.OpenSession())
{
session.Store(failedMsg);
session.SaveChanges();
}
documentStore.WaitForIndexing();
Thread.Sleep(Settings.ExpirationProcessTimerInSeconds * 1000 * 2);
using (var session = documentStore.OpenSession())
{
var msg = session.Load<FailedMessage>(failedMsg.Id);
Assert.NotNull(msg);
}
}
private static FailedMessageView Map(FailedMessage message, IAsyncDocumentSession session)
{
var processingAttempt = message.ProcessingAttempts.Last();
var metadata = processingAttempt.MessageMetadata;
var failureDetails = processingAttempt.FailureDetails;
var wasEdited = message.ProcessingAttempts.Last().Headers.ContainsKey("ServiceControl.EditOf");
return(new FailedMessageView
{
Id = message.UniqueMessageId,
MessageType = metadata.GetAsStringOrNull("MessageType"),
IsSystemMessage = metadata.GetOrDefault <bool>("IsSystemMessage"),
SendingEndpoint = metadata.GetOrDefault <EndpointDetails>("SendingEndpoint"),
ReceivingEndpoint = metadata.GetOrDefault <EndpointDetails>("ReceivingEndpoint"),
TimeSent = metadata.GetAsNullableDatetime("TimeSent"),
MessageId = metadata.GetAsStringOrNull("MessageId"),
Exception = failureDetails.Exception,
QueueAddress = failureDetails.AddressOfFailingEndpoint,
NumberOfProcessingAttempts = message.ProcessingAttempts.Count,
Status = message.Status,
TimeOfFailure = failureDetails.TimeOfFailure,
LastModified = session.Advanced.GetMetadataFor(message)["Last-Modified"].Value <DateTime>(),
Edited = wasEdited,
EditOf = wasEdited ? message.ProcessingAttempts.Last().Headers["ServiceControl.EditOf"] : ""
});
}
public void Should_show_up_as_resolved_when_doing_a_retry_all()
{
FailedMessage failure = null;
var context = new MyContext();
Scenario.Define(context)
.WithEndpoint <ManagementEndpoint>(c => c.AppConfig(PathToAppConfig))
.WithEndpoint <FailureEndpoint>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(c =>
{
if (!GetFailedMessage(c, out failure))
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
return(true);
}
IssueRetry(c, () => Post <object>(String.Format("/api/errors/retry/all")));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
}
public async Task Should_show_up_as_resolved_when_doing_a_single_retry()
{
FailedMessage failure = null;
await Define <MyContext>()
.WithEndpoint <FailureEndpoint>()
.Done(async c =>
{
var result = await GetFailedMessage(c);
failure = result;
if (!c.RetryIssued && result)
{
await IssueRetry(c, () => this.Post <object>($"/api/errors/{c.UniqueMessageId}/retry"));
return(false);
}
var afterRetryResult = await GetFailedMessage(c, x => x.Status == FailedMessageStatus.Resolved);
failure = afterRetryResult;
return(c.Done && afterRetryResult);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
}
static Task Log(FailedMessage failed)
{
log.Fatal($@"Message sent to error queue.
Body:
{GetMessageString(failed.Body)}");
return(Task.CompletedTask);
}
async Task CreateAFailedMessageAndMarkAsPartOfRetryBatch(IDocumentStore documentStore, RetryingManager retryManager, string groupId, bool progressToStaged, int numberOfMessages)
{
var messages = Enumerable.Range(0, numberOfMessages).Select(i =>
{
var id = Guid.NewGuid().ToString();
return(new FailedMessage
{
Id = FailedMessage.MakeDocumentId(id),
UniqueMessageId = id,
FailureGroups = new List <FailedMessage.FailureGroup>
{
new FailedMessage.FailureGroup
{
Id = groupId,
Title = groupId,
Type = groupId
}
},
Status = FailedMessageStatus.Unresolved,
ProcessingAttempts = new List <FailedMessage.ProcessingAttempt>
{
new FailedMessage.ProcessingAttempt
{
AttemptedAt = DateTime.UtcNow,
MessageMetadata = new Dictionary <string, object>(),
FailureDetails = new FailureDetails(),
Headers = new Dictionary <string, string>()
}
}
});
});
using (var session = documentStore.OpenAsyncSession())
{
foreach (var message in messages)
{
await session.StoreAsync(message);
}
await session.SaveChangesAsync();
}
new FailedMessages_ByGroup().Execute(documentStore);
documentStore.WaitForIndexing();
var documentManager = new CustomRetryDocumentManager(progressToStaged, documentStore);
var gateway = new RetriesGateway(documentStore, documentManager);
documentManager.OperationManager = retryManager;
gateway.OperationManager = retryManager;
gateway.StartRetryForIndex <FailureGroupMessageView, FailedMessages_ByGroup>("Test-group", RetryType.FailureGroup, DateTime.UtcNow, x => x.FailureGroupId == "Test-group", "Test-Context");
documentStore.WaitForIndexing();
await gateway.ProcessNextBulkRetry();
}
public void Handle(ImportFailedMessage message)
{
var documentId = FailedMessage.MakeDocumentId(message.UniqueMessageId);
var failure = Session.Load <FailedMessage>(documentId) ?? new FailedMessage
{
Id = documentId,
UniqueMessageId = message.UniqueMessageId
};
failure.Status = FailedMessageStatus.Unresolved;
var timeOfFailure = message.FailureDetails.TimeOfFailure;
//check for duplicate
if (failure.ProcessingAttempts.Any(a => a.AttemptedAt == timeOfFailure))
{
return;
}
failure.ProcessingAttempts.Add(new FailedMessage.ProcessingAttempt
{
AttemptedAt = timeOfFailure,
FailureDetails = message.FailureDetails,
MessageMetadata = message.Metadata,
MessageId = message.PhysicalMessage.MessageId,
Headers = message.PhysicalMessage.Headers,
ReplyToAddress = message.PhysicalMessage.ReplyToAddress,
Recoverable = message.PhysicalMessage.Recoverable,
CorrelationId = message.PhysicalMessage.CorrelationId,
MessageIntent = message.PhysicalMessage.MessageIntent,
});
Session.Store(failure);
}
public void Should_show_up_as_resolved_when_doing_a_multi_retry()
{
FailedMessage failure = null;
var context = new MyContext();
Define(context)
.WithEndpoint <FailureEndpoint>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(c =>
{
if (!GetFailedMessage(c, out failure))
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
return(true);
}
IssueRetry(c, () => Post("/api/errors/retry", new List <string> {
c.UniqueMessageId
}));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
}
public async Task It_should_be_able_to_be_retried_successfully()
{
var context = new MyContext {
Succeed = false
};
FailedMessage failure = null;
await Define(context)
.WithEndpoint <FailureEndpoint>(b =>
b.Given(bus => bus.SendLocal(new MyMessage()))
.When(async ctx => await CheckProcessingAttemptsIs(ctx, 1),
(bus, ctx) => Post <object>($"/api/errors/{ctx.UniqueMessageId}/retry"))
.When(async ctx => await CheckProcessingAttemptsIs(ctx, 2),
(bus, ctx) => Post <object>($"/api/errors/{ctx.UniqueMessageId}/retry"))
.When(async ctx => await CheckProcessingAttemptsIs(ctx, 3),
async(bus, ctx) =>
{
ctx.Succeed = true;
await Post <object>($"/api/errors/{ctx.UniqueMessageId}/retry");
})
)
.Done(async ctx =>
{
var result = await GetFailedMessage(ctx, f => f.Status == FailedMessageStatus.Resolved);
failure = result;
return(result);
})
.Run(TimeSpan.FromMinutes(4));
Assert.IsNotNull(failure, "Failure should not be null");
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
}
public void Errors_are_not_being_expired()
{
var failedMsg = new FailedMessage
{
Id = "1",
ProcessingAttempts = new List <FailedMessage.ProcessingAttempt>
{
new FailedMessage.ProcessingAttempt
{
AttemptedAt = DateTime.UtcNow.AddHours(-(Settings.HoursToKeepMessagesBeforeExpiring * 3))
},
new FailedMessage.ProcessingAttempt
{
AttemptedAt = DateTime.UtcNow.AddHours(-(Settings.HoursToKeepMessagesBeforeExpiring * 2)),
}
},
Status = FailedMessageStatus.Unresolved,
};
using (var session = documentStore.OpenSession())
{
session.Store(failedMsg);
session.SaveChanges();
}
documentStore.WaitForIndexing();
Thread.Sleep(Settings.ExpirationProcessTimerInSeconds * 1000 * 2);
using (var session = documentStore.OpenSession())
{
var msg = session.Load <FailedMessage>(failedMsg.Id);
Assert.NotNull(msg);
}
}
public async Task Should_show_up_as_resolved_when_doing_a_retry_all()
{
FailedMessage failure = null;
await Define <MyContext>()
.WithEndpoint <FailureEndpoint>(b => b.When(bus => bus.SendLocal(new MyMessage())).DoNotFailOnErrorMessages())
.Done(async c =>
{
var failedMessageResult = await GetFailedMessage(c);
failure = failedMessageResult;
if (!failedMessageResult)
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
return(true);
}
await IssueRetry(c, () => this.Post <object>("/api/errors/retry/all"));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
}
public async Task Should_show_up_as_resolved_in_the_eventlog()
{
FailedMessage failure = null;
List <EventLogItem> eventLogItems = null;
await Define <MyContext>()
.WithEndpoint <FailureEndpoint>(b => b.When(bus => bus.SendLocal(new MyMessage())).DoNotFailOnErrorMessages())
.Done(async c =>
{
var failedMessageResult = await GetFailedMessage(c);
failure = failedMessageResult;
if (!failedMessageResult)
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
var eventLogItemsResult = await this.TryGetMany <EventLogItem>("/api/eventlogitems", item => item.Description.StartsWith("Failed message resolved by retry"));
eventLogItems = eventLogItemsResult;
return(eventLogItemsResult);
}
await IssueRetry(c, () => this.Post <object>($"/api/errors/{c.UniqueMessageId}/retry"));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
Assert.IsTrue(eventLogItems.Any(item => item.Description.Equals("Failed message resolved by retry") && item.RelatedTo.Contains("/message/" + failure.UniqueMessageId)));
}
public async Task Should_be_imported_and_accessible_via_the_rest_api()
{
var context = new MyContext();
FailedMessage failedMessage = null;
await Define(context)
.WithEndpoint <Receiver>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(async c =>
{
var result = await TryGet <FailedMessage>("/api/errors/" + c.UniqueMessageId);
failedMessage = result;
return(c.MessageId != null && result);
})
.Run();
Assert.AreEqual(context.UniqueMessageId, failedMessage.UniqueMessageId);
// The message Ids may contain a \ if they are from older versions.
Assert.AreEqual(context.MessageId, failedMessage.ProcessingAttempts.Last().MessageId,
"The returned message should match the processed one");
Assert.AreEqual(FailedMessageStatus.Unresolved, failedMessage.Status, "Status should be set to unresolved");
Assert.AreEqual(1, failedMessage.ProcessingAttempts.Count(), "Failed count should be 1");
Assert.AreEqual("Simulated exception", failedMessage.ProcessingAttempts.Single().FailureDetails.Exception.Message,
"Exception message should be captured");
}
public void Should_show_up_as_resolved_when_doing_a_retry_all_for_the_given_endpoint()
{
FailedMessage failure = null;
var context = new MyContext();
Define(context)
.WithEndpoint <FailureEndpoint>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(c =>
{
if (!GetFailedMessage(c, out failure))
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
return(true);
}
IssueRetry(c, () => Post <object>($"/api/errors/{c.EndpointNameOfReceivingEndpoint}/retry/all"));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
}
public void Acknowledging_the_retry_should_be_successful()
{
FailedMessage failure = null;
var context = new MyContext();
Define(context)
.WithEndpoint <FailureEndpoint>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(c =>
{
if (!GetFailedMessage(c, out failure))
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
return(true);
}
IssueRetry(c, () => Post <object>($"/api//recoverability/groups/{failure.FailureGroups.First().Id}/errors/retry"));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Delete($"/api/recoverability/unacknowledgedgroups/{failure.FailureGroups.First().Id}"); // Exception will throw if 404
}
public void Should_show_up_as_resolved_when_doing_a_single_retry()
{
FailedMessage failure = null;
MessagesView message = null;
List <EventLogItem> eventLogItems = null;
var context = new MyContext();
Define(context)
.WithEndpoint <FailureEndpoint>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(c =>
{
if (!GetFailedMessage(c, out failure))
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
return(TryGetSingle("/api/messages", out message, m => m.Status == MessageStatus.ResolvedSuccessfully) &&
TryGetMany("/api/eventlogitems", out eventLogItems));
}
IssueRetry(c, () => Post <object>($"/api/errors/{c.UniqueMessageId}/retry"));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
//Assert.AreEqual(failure.UniqueMessageId, message.Id);
Assert.AreEqual(MessageStatus.ResolvedSuccessfully, message.Status);
Assert.IsTrue(eventLogItems.Any(item => item.Description.Equals("Failed message resolved by retry") && item.RelatedTo.Contains("/message/" + failure.UniqueMessageId)));
}
public async Task Should_status_retryissued_after_retry_is_sent()
{
FailedMessage failedMessage = null;
await Define <Context>()
.WithEndpoint <FailingEndpoint>(b => b.When(async ctx =>
{
if (ctx.UniqueMessageId == null)
{
return(false);
}
var result = await this.TryGet <FailedMessage>($"/api/errors/{ctx.UniqueMessageId}");
failedMessage = result;
return(result);
}, async(bus, ctx) =>
{
ctx.AboutToSendRetry = true;
await this.Post <object>($"/api/errors/{ctx.UniqueMessageId}/retry");
}).DoNotFailOnErrorMessages())
.Done(async ctx =>
{
if (ctx.Retried)
{
failedMessage = await this.TryGet <FailedMessage>($"/api/errors/{ctx.UniqueMessageId}");
return(true);
}
return(false);
})
.Run();
Assert.AreEqual(failedMessage.Status, FailedMessageStatus.RetryIssued, "Status was not set to RetryIssued");
}
TransportOperation ToTransportOperation(FailedMessage message, string stagingId)
{
var attempt = message.ProcessingAttempts.Last();
var headersToRetryWith = HeaderFilter.RemoveErrorMessageHeaders(attempt.Headers);
var addressOfFailingEndpoint = attempt.FailureDetails.AddressOfFailingEndpoint;
var redirect = redirects[addressOfFailingEndpoint];
if (redirect != null)
{
addressOfFailingEndpoint = redirect.ToPhysicalAddress;
}
headersToRetryWith["ServiceControl.TargetEndpointAddress"] = addressOfFailingEndpoint;
headersToRetryWith["ServiceControl.Retry.UniqueMessageId"] = message.UniqueMessageId;
headersToRetryWith["ServiceControl.Retry.StagingId"] = stagingId;
headersToRetryWith["ServiceControl.Retry.Attempt.MessageId"] = attempt.MessageId;
corruptedReplyToHeaderStrategy.FixCorruptedReplyToHeader(headersToRetryWith);
var transportMessage = new OutgoingMessage(message.Id, headersToRetryWith, Array.Empty <byte>());
return(new TransportOperation(transportMessage, new UnicastAddressTag(returnToSender.InputAddress)));
}
public ICommandData CreateFailedMessageRetryDocument(string batchDocumentId, string messageUniqueId)
{
var failureRetryId = FailedMessageRetry.MakeDocumentId(messageUniqueId);
return(new PatchCommandData
{
Patches = patchRequestsEmpty,
PatchesIfMissing = new[]
{
new PatchRequest
{
Name = "FailedMessageId",
Type = PatchCommandType.Set,
Value = FailedMessage.MakeDocumentId(messageUniqueId)
},
new PatchRequest
{
Name = "RetryBatchId",
Type = PatchCommandType.Set,
Value = batchDocumentId
}
},
Key = failureRetryId,
Metadata = defaultMetadata
});
}
public void The_import_should_support_it()
{
var context = new MyContext();
SetSettings = settings =>
{
settings.MaximumConcurrencyLevel = 10;
};
CustomConfiguration = config =>
{
config.DefineCriticalErrorAction((s, exception) => context.CriticalErrorExecuted = true);
};
FailedMessage failure = null;
Define(context)
.WithEndpoint <SourceEndpoint>()
.Done(c =>
{
if (c.UniqueId == null)
{
return(false);
}
return(c.CriticalErrorExecuted || TryGet($"/api/errors/{c.UniqueId}", out failure, m => m.ProcessingAttempts.Count == 10));
})
.Run();
Assert.IsFalse(context.CriticalErrorExecuted);
Assert.NotNull(failure);
Assert.AreEqual(10, failure.ProcessingAttempts.Count);
}
async Task <FailedMessage> CreateFailedMessage(string failedMessageId = null, FailedMessageStatus status = FailedMessageStatus.Unresolved)
{
failedMessageId = failedMessageId ?? Guid.NewGuid().ToString();
using (var session = Store.OpenAsyncSession())
{
var failedMessage = new FailedMessage
{
UniqueMessageId = failedMessageId,
Id = FailedMessage.MakeDocumentId(failedMessageId),
Status = status,
ProcessingAttempts = new List <FailedMessage.ProcessingAttempt>
{
new FailedMessage.ProcessingAttempt
{
MessageId = Guid.NewGuid().ToString(),
FailureDetails = new FailureDetails
{
AddressOfFailingEndpoint = "OriginalEndpointAddress"
}
}
}
};
await session.StoreAsync(failedMessage);
await session.SaveChangesAsync();
return(failedMessage);
}
}
public void Should_show_up_as_resolved_when_doing_a_single_retry()
{
FailedMessage failure = null;
MessagesView message = null;
List <EventLogItem> eventLogItems = null;
var context = new MyContext();
Scenario.Define(context)
.WithEndpoint <ManagementEndpoint>(c => c.AppConfig(PathToAppConfig))
.WithEndpoint <FailureEndpoint>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(c =>
{
if (!GetFailedMessage(c, out failure))
{
return(false);
}
if (failure.Status == FailedMessageStatus.Resolved)
{
return(TryGetSingle("/api/messages", out message, m => m.Status == MessageStatus.ResolvedSuccessfully) &&
TryGetMany("/api/eventlogitems", out eventLogItems));
}
IssueRetry(c, () => Post <object>(String.Format("/api/errors/{0}/retry", c.UniqueMessageId)));
return(false);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(FailedMessageStatus.Resolved, failure.Status);
Assert.AreEqual(failure.UniqueMessageId, message.Id);
Assert.AreEqual(MessageStatus.ResolvedSuccessfully, message.Status);
Assert.IsTrue(eventLogItems.Any(item => item.Description.Equals("Failed message ServiceBus.Management.AcceptanceTests.When_a_retry_for_a_failed_message_is_successful+MyMessage resolved by retry")));
}
public async Task <StatusCodeResult> Edit(string failedMessageId, EditMessageModel edit)
{
if (!settings.AllowMessageEditing)
{
logging.Info("Message edit-retry has not been enabled.");
return(StatusCode(HttpStatusCode.NotFound));
}
if (string.IsNullOrEmpty(failedMessageId))
{
return(StatusCode(HttpStatusCode.BadRequest));
}
FailedMessage failedMessage;
using (var session = documentStore.OpenAsyncSession())
{
failedMessage = await session.LoadAsync <FailedMessage>(FailedMessage.MakeDocumentId(failedMessageId)).ConfigureAwait(false);
}
if (failedMessage == null)
{
logging.WarnFormat("The original failed message could not be loaded for id={0}", failedMessageId);
return(StatusCode(HttpStatusCode.BadRequest));
}
//WARN
/*
* failedMessage.ProcessingAttempts.Last() return the lat retry attempt.
* In theory between teh time someone edits a failed message and retry it someone else
* could have retried the same message without editing. If this is the case "Last()" is
* not anymore the same message.
* Instead of using Last() it's probably better to select the processing attempt by looking for
* one with the same MessageID
*/
if (LockedHeaderModificationValidator.Check(GetEditConfiguration().LockedHeaders, edit.MessageHeaders.ToDictionary(x => x.Key, x => x.Value), failedMessage.ProcessingAttempts.Last().Headers))
{
logging.WarnFormat("Locked headers have been modified on the edit-retry for MessageID {0}.", failedMessageId);
return(StatusCode(HttpStatusCode.BadRequest));
}
if (string.IsNullOrWhiteSpace(edit.MessageBody) || edit.MessageHeaders == null)
{
logging.WarnFormat("There is no message body on the edit-retry for MessageID {0}.", failedMessageId);
return(StatusCode(HttpStatusCode.BadRequest));
}
// Encode the body in base64 so that the new body doesn't have to be escaped
var base64String = Convert.ToBase64String(Encoding.UTF8.GetBytes(edit.MessageBody));
await messageSession.SendLocal(new EditAndSend
{
FailedMessageId = failedMessageId,
NewBody = base64String,
NewHeaders = edit.MessageHeaders.ToDictionary(kvp => kvp.Key, kvp => kvp.Value)
}).ConfigureAwait(false);
return(StatusCode(HttpStatusCode.Accepted));
}
public ClassifiableMessageDetails(FailedMessage message)
{
var last = message.ProcessingAttempts.Last();
Details = last.FailureDetails;
MessageType = (string)last.MessageMetadata["MessageType"];
}
async Task TryStageMessage(FailedMessage message, string stagingId, FailedMessageRetry failedMessageRetry)
{
try
{
await StageMessage(message, stagingId).ConfigureAwait(false);
}
catch (Exception e)
{
var incrementedAttempts = failedMessageRetry.StageAttempts + 1;
if (incrementedAttempts < MaxStagingAttempts)
{
Log.Warn($"Attempt {incrementedAttempts} of {MaxStagingAttempts} to stage a retry message {message.UniqueMessageId} failed", e);
await IncrementAttemptCounter(failedMessageRetry)
.ConfigureAwait(false);
}
else
{
Log.Error($"Retry message {message.UniqueMessageId} reached its staging retry limit ({MaxStagingAttempts}) and is going to be removed from the batch.", e);
await store.AsyncDatabaseCommands.DeleteAsync(FailedMessageRetry.MakeDocumentId(message.UniqueMessageId), null)
.ConfigureAwait(false);
await domainEvents.Raise(new MessageFailedInStaging
{
UniqueMessageId = message.UniqueMessageId
}).ConfigureAwait(false);
}
throw new RetryStagingException();
}
}
public void Errors_are_not_being_expired()
{
using (var documentStore = InMemoryStoreBuilder.GetInMemoryStore())
{
var failedMsg = new FailedMessage
{
Id = "1",
};
using (var session = documentStore.OpenSession())
{
session.Store(failedMsg);
session.SaveChanges();
Debug.WriteLine(session.Advanced.GetMetadataFor(failedMsg)["Last-Modified"]);
}
Thread.Sleep(100);
RunExpiry(documentStore, DateTime.UtcNow);
using (var session = documentStore.OpenSession())
{
Assert.NotNull(session.Load <FailedMessage>(failedMsg.Id));
}
}
}
public async Task <ErrorHandleResult> OnError(IErrorHandlingPolicyContext handlingContext, IDispatchMessages dispatcher)
{
try
{
var message = handlingContext.Error.Message;
var destination = message.Headers["ServiceControl.TargetEndpointAddress"];
var messageUniqueId = message.Headers["ServiceControl.Retry.UniqueMessageId"];
Log.Warn($"Failed to send '{messageUniqueId}' message to '{destination}' for retry. Attempting to revert message status to unresolved so it can be tried again.", handlingContext.Error.Exception);
using (var session = store.OpenAsyncSession())
{
var failedMessage = await session.LoadAsync <FailedMessage>(FailedMessage.MakeDocumentId(messageUniqueId))
.ConfigureAwait(false);
if (failedMessage != null)
{
failedMessage.Status = FailedMessageStatus.Unresolved;
}
var failedMessageRetry = await session.LoadAsync <FailedMessageRetry>(FailedMessageRetry.MakeDocumentId(messageUniqueId))
.ConfigureAwait(false);
if (failedMessageRetry != null)
{
session.Delete(failedMessageRetry);
}
await session.SaveChangesAsync()
.ConfigureAwait(false);
}
string reason;
try
{
reason = handlingContext.Error.Exception.GetBaseException().Message;
}
catch (Exception)
{
reason = "Failed to retrieve reason!";
}
await domainEvents.Raise(new MessagesSubmittedForRetryFailed
{
Reason = reason,
FailedMessageId = messageUniqueId,
Destination = destination
}).ConfigureAwait(false);
}
catch (Exception ex)
{
// If something goes wrong here we just ignore, not the end of the world!
Log.Error("A failure occurred when trying to handle a retry failure.", ex);
}
finally
{
executeOnFailure();
}
return(ErrorHandleResult.Handled);
}
public void Should_allow_errors_with_no_metadata()
{
using (var session = documentStore.OpenSession())
{
var processedMessage = new FailedMessage
{
Id = "1",
UniqueMessageId = "xyz",
Status = FailedMessageStatus.Unresolved,
ProcessingAttempts = new List<FailedMessage.ProcessingAttempt> { new FailedMessage.ProcessingAttempt
{
AttemptedAt = DateTime.UtcNow,
MessageMetadata = new Dictionary<string, object>()
}}
};
session.Store(processedMessage);
session.SaveChanges();
}
RavenQueryStatistics stats;
do
{
using (var session = documentStore.OpenSession())
{
var results = session.Advanced.LuceneQuery<FailedMessageViewIndex.SortAndFilterOptions, FailedMessageViewIndex>()
.SetResultTransformer(new FailedMessageViewTransformer().TransformerName)
.Statistics(out stats)
.SelectFields<FailedMessageView>()
.ToList();
if (!stats.IsStale)
{
Console.Out.WriteLine("Checking result");
Assert.AreEqual(1, results.Count);
Assert.AreEqual(null, results.First().TimeSent);
}
}
if (stats.IsStale)
Thread.Sleep(1000);
} while (stats.IsStale);
}
public void RetryMessage(FailedMessage message)
{
using (var scope = new TransactionScope())
{
var m = errorQueue.ReceiveById(message.Id, TimeSpan.FromSeconds(5), MessageQueueTransactionType.Automatic);
//TODO: Fix with new NServiceBus
//var failedQueue = MsmqTransport.GetFailedQueue(m);
//m.Label = MsmqTransport.GetLabelWithoutFailedQueue(m);
//using (var q = new MessageQueue(failedQueue))
//{
// q.Send(m, MessageQueueTransactionType.Automatic);
//}
scope.Complete();
}
}
