public ProcessedMessage(ImportSuccessfullyProcessedMessage message)
{
UniqueMessageId = message.UniqueMessageId;
MessageMetadata = message.Metadata;
Headers = message.PhysicalMessage.Headers;
string processedAt;
if (message.PhysicalMessage.Headers.TryGetValue(NServiceBus.Headers.ProcessingEnded, out processedAt))
{
ProcessedAt = DateTimeExtensions.ToUtcDateTime(processedAt);
}
else
{
ProcessedAt = DateTime.UtcNow;//best guess
}
}
public ProcessedMessage(ImportSuccessfullyProcessedMessage message)
{
UniqueMessageId = message.UniqueMessageId;
MessageMetadata = message.Metadata;
Headers = message.PhysicalMessage.Headers;
string processedAt;
if (message.PhysicalMessage.Headers.TryGetValue(NServiceBus.Headers.ProcessingEnded, out processedAt))
{
ProcessedAt = DateTimeExtensions.ToUtcDateTime(processedAt);
}
else
{
ProcessedAt = DateTime.UtcNow;//best guess
}
}
void InnerHandle(TransportMessage message)
{
var receivedMessage = new ImportSuccessfullyProcessedMessage(message);
using (var childBuilder = Builder.CreateChildBuilder())
{
PipelineExecutor.CurrentContext.Set(childBuilder);
foreach (var enricher in childBuilder.BuildAll<IEnrichImportedMessages>())
{
enricher.Enrich(receivedMessage);
}
var logicalMessage = LogicalMessageFactory.Create(typeof(ImportSuccessfullyProcessedMessage),
receivedMessage);
PipelineExecutor.InvokeLogicalMessagePipeline(logicalMessage);
}
if (Settings.ForwardAuditMessages == true)
{
Forwarder.Send(message, Settings.AuditLogQueue);
}
}
void InnerHandle(TransportMessage message)
{
var receivedMessage = new ImportSuccessfullyProcessedMessage(message);
using (var childBuilder = Builder.CreateChildBuilder())
{
PipelineExecutor.CurrentContext.Set(childBuilder);
foreach (var enricher in childBuilder.BuildAll <IEnrichImportedMessages>())
{
enricher.Enrich(receivedMessage);
}
var logicalMessage = LogicalMessageFactory.Create(typeof(ImportSuccessfullyProcessedMessage),
receivedMessage);
PipelineExecutor.InvokeLogicalMessagePipeline(logicalMessage);
}
if (Settings.ForwardAuditMessages == true)
{
Forwarder.Send(message, Settings.AuditLogQueue);
}
}
void RetryMessageImportById(string messageID)
{
// Try to get the batchErrorLock, if we can't then exit,
// the message will trigger a retry next time on the next batch read.
// Retrymessage may be fired again for the same message until the batches drain so this
// prevents the message being processed twice,
if (Monitor.TryEnter(batchErrorLockObj))
{
try
{
Logger.DebugFormat("Drain stop running batch importers");
stopping = true;
var runningTasks = batchTaskTracker.Active();
Task.WaitAll(runningTasks);
var commitTransaction = false;
using (var queueReceiver = CreateReceiver())
using (var msmqTransaction = new MessageQueueTransaction())
{
msmqTransaction.Begin();
Logger.DebugFormat("Retry import of messageID - {0}", messageID);
try
{
Message message;
TransportMessage transportMessage;
try
{
message = queueReceiver.ReceiveById(messageID);
performanceCounters.MessageDequeued();
}
catch (Exception exception)
{
importFailuresHandler.FailedToReceive(exception); //logs and increments circuit breaker
return;
}
try
{
transportMessage = MsmqUtilities.Convert(message);
}
catch (Exception convertException)
{
importFailuresHandler.FailedToReceive(convertException); //logs and increments circuit breaker
serviceControlErrorQueue.Send(message, msmqTransaction); // Send unconvertable message to SC's ErrorQueue so it's not lost
commitTransaction = true; // Can't convert the messsage, so commit to get message out of the queue
return;
}
try
{
var importSuccessfullyProcessedMessage = new ImportSuccessfullyProcessedMessage(transportMessage);
foreach (var enricher in enrichers)
{
enricher.Enrich(importSuccessfullyProcessedMessage);
}
using (var session = store.OpenSession())
{
var auditMessage = new ProcessedMessage(importSuccessfullyProcessedMessage);
session.Store(auditMessage);
session.SaveChanges();
}
performanceCounters.MessageProcessed();
if (Settings.ForwardAuditMessages == true)
{
Forwarder.Send(transportMessage, Settings.AuditLogQueue);
}
commitTransaction = true;
}
catch (Exception importException)
{
importFailuresHandler.Log(transportMessage, importException); //Logs and Writes failure transport message to Raven
}
}
finally
{
if (commitTransaction)
{
msmqTransaction.Commit();
}
}
}
}
finally
{
Monitor.Exit(batchErrorLockObj);
//Restart Batch mode
stopping = false;
Logger.Debug("Ready to BeginPeek again");
queuePeeker.BeginPeek();
}
}
}
void BatchImporter()
{
String failedMessageID = null;
try
{
Logger.DebugFormat("Batch job started", Task.CurrentId);
var moreMessages = 0;
using (var queueReceiver = CreateReceiver())
{
do
{
if (moreMessages > RampUpConcurrencyMagicNumber)
{
if (TryStartNewBatchImporter())
{
Logger.Debug("We have too many messages, starting another batch importer");
moreMessages = 0; //Reset to 0 so we only ramp up once per BatchImporter
}
}
moreMessages++;
using (var msmqTransaction = new MessageQueueTransaction())
{
msmqTransaction.Begin();
using (var bulkInsert =store.BulkInsert(options:new BulkInsertOptions {CheckForUpdates = true}))
{
for (var idx = 0; idx < BatchSize; idx++)
{
Message message = null;
TransportMessage transportMessage;
try
{
message = queueReceiver.Receive(receiveTimeout, msmqTransaction);
performanceCounters.MessageDequeued();
transportMessage = MsmqUtilities.Convert(message);
}
catch (MessageQueueException mqe)
{
if (mqe.MessageQueueErrorCode == MessageQueueErrorCode.IOTimeout)
{
moreMessages = 0;
break;
}
throw;
}
catch (Exception)
{
if (message != null) {
failedMessageID = message.Id;
}
throw;
}
try
{
var importSuccessfullyProcessedMessage = new ImportSuccessfullyProcessedMessage(transportMessage);
foreach (var enricher in enrichers)
{
enricher.Enrich(importSuccessfullyProcessedMessage);
}
var auditMessage = new ProcessedMessage(importSuccessfullyProcessedMessage);
bulkInsert.Store(auditMessage);
performanceCounters.MessageProcessed();
if (Settings.ForwardAuditMessages == true)
{
Forwarder.Send(transportMessage, Settings.AuditLogQueue);
}
}
catch (Exception)
{
if (message != null)
{
failedMessageID = message.Id;
}
throw;
}
}
}
msmqTransaction.Commit();
}
} while (moreMessages > 0 && !stopping);
}
Logger.Debug("Stopping batch importer");
}
finally
{
if (!String.IsNullOrEmpty(failedMessageID))
{
// Call RetryMessageImportById outside the Task as it checks for running tasks
ThreadPool.QueueUserWorkItem(state => RetryMessageImportById(failedMessageID));
}
countDownEvent.Decrement();
}
}
void RetryMessageImportById(string messageID)
{
// Try to get the batchErrorLock, if we can't then exit,
// the message will trigger a retry next time on the next batch read.
// Retrymessage may be fired again for the same message until the batches drain so this
// prevents the message being processed twice,
if (Monitor.TryEnter(batchErrorLockObj))
{
try
{
Logger.DebugFormat("Drain stop running batch importers");
stopping = true;
var runningTasks = batchTaskTracker.Active();
Task.WaitAll(runningTasks);
var commitTransaction = false;
using (var queueReceiver = CreateReceiver())
using (var msmqTransaction = new MessageQueueTransaction())
{
msmqTransaction.Begin();
Logger.DebugFormat("Retry import of messageID - {0}", messageID);
try
{
Message message;
TransportMessage transportMessage;
try
{
message = queueReceiver.ReceiveById(messageID);
performanceCounters.MessageDequeued();
}
catch (Exception exception)
{
importFailuresHandler.FailedToReceive(exception); //logs and increments circuit breaker
return;
}
try
{
transportMessage = MsmqUtilities.Convert(message);
}
catch (Exception convertException)
{
importFailuresHandler.FailedToReceive(convertException); //logs and increments circuit breaker
serviceControlErrorQueue.Send(message, msmqTransaction); // Send unconvertable message to SC's ErrorQueue so it's not lost
commitTransaction = true; // Can't convert the messsage, so commit to get message out of the queue
return;
}
try
{
var importSuccessfullyProcessedMessage = new ImportSuccessfullyProcessedMessage(transportMessage);
foreach (var enricher in enrichers)
{
enricher.Enrich(importSuccessfullyProcessedMessage);
}
using (var session = store.OpenSession())
{
var auditMessage = new ProcessedMessage(importSuccessfullyProcessedMessage);
session.Store(auditMessage);
session.SaveChanges();
}
performanceCounters.MessageProcessed();
if (Settings.ForwardAuditMessages == true)
{
Forwarder.Send(transportMessage, Settings.AuditLogQueue);
}
commitTransaction = true;
}
catch (Exception importException)
{
importFailuresHandler.Log(transportMessage, importException); //Logs and Writes failure transport message to Raven
}
}
finally
{
if (commitTransaction)
{
msmqTransaction.Commit();
}
}
}
}
finally
{
Monitor.Exit(batchErrorLockObj);
//Restart Batch mode
stopping = false;
Logger.Debug("Ready to BeginPeek again");
queuePeeker.BeginPeek();
}
}
}
void BatchImporter()
{
String failedMessageID = null;
try
{
Logger.DebugFormat("Batch job started", Task.CurrentId);
var moreMessages = 0;
using (var queueReceiver = CreateReceiver())
{
do
{
if (moreMessages > RampUpConcurrencyMagicNumber)
{
if (TryStartNewBatchImporter())
{
Logger.Debug("We have too many messages, starting another batch importer");
moreMessages = 0; //Reset to 0 so we only ramp up once per BatchImporter
}
}
moreMessages++;
using (var msmqTransaction = new MessageQueueTransaction())
{
msmqTransaction.Begin();
using (var bulkInsert = store.BulkInsert(options: new BulkInsertOptions {
CheckForUpdates = true
}))
{
for (var idx = 0; idx < BatchSize; idx++)
{
Message message = null;
TransportMessage transportMessage;
try
{
message = queueReceiver.Receive(receiveTimeout, msmqTransaction);
performanceCounters.MessageDequeued();
transportMessage = MsmqUtilities.Convert(message);
}
catch (MessageQueueException mqe)
{
if (mqe.MessageQueueErrorCode == MessageQueueErrorCode.IOTimeout)
{
moreMessages = 0;
break;
}
throw;
}
catch (Exception)
{
if (message != null)
{
failedMessageID = message.Id;
}
throw;
}
try
{
var importSuccessfullyProcessedMessage = new ImportSuccessfullyProcessedMessage(transportMessage);
foreach (var enricher in enrichers)
{
enricher.Enrich(importSuccessfullyProcessedMessage);
}
var auditMessage = new ProcessedMessage(importSuccessfullyProcessedMessage);
bulkInsert.Store(auditMessage);
performanceCounters.MessageProcessed();
if (Settings.ForwardAuditMessages == true)
{
Forwarder.Send(transportMessage, Settings.AuditLogQueue);
}
}
catch (Exception)
{
if (message != null)
{
failedMessageID = message.Id;
}
throw;
}
}
}
msmqTransaction.Commit();
}
} while (moreMessages > 0 && !stopping);
}
Logger.Debug("Stopping batch importer");
}
finally
{
if (!String.IsNullOrEmpty(failedMessageID))
{
// Call RetryMessageImportById outside the Task as it checks for running tasks
ThreadPool.QueueUserWorkItem(state => RetryMessageImportById(failedMessageID));
}
countDownEvent.Decrement();
}
}
