public void CanMeasureTimeSpentInSteps()
{
var stats = new PipelineStepProfilerStats();
var pipeline = new DefaultPipeline()
.OnReceive(new Step300())
.OnReceive(new Step100())
.OnReceive(new Step200());
var profiler = new PipelineStepProfiler(pipeline, stats);
var receivePipeline = profiler.ReceivePipeline();
var invoker = new DefaultPipelineInvoker();
var transportMessage = new TransportMessage(new Dictionary<string, string>(), new byte[0]);
using (new DefaultTransactionContextScope())
{
var stepContext = new IncomingStepContext(transportMessage, AmbientTransactionContext.Current);
invoker.Invoke(stepContext, receivePipeline).Wait();
var stepStats = stats.GetStats();
Console.WriteLine(string.Join(Environment.NewLine, stepStats));
}
}
public void HandleMessage(TransportMessage transportMessage, ITransactionContext transactionContext)
{
var message = transportMessage.GetMessageLabel();
var options = new List<KeyOption>();
if (DefaultOutputQueue != null)
{
options.Add(KeyOption.New('d',
() => { MoveMessage(transportMessage, transactionContext, DefaultOutputQueue); },
"Move to default queue '{0}'", DefaultOutputQueue));
}
if (transportMessage.Headers.ContainsKey(Headers.SourceQueue))
{
var sourceQueue = transportMessage.Headers[Headers.SourceQueue];
options.Add(KeyOption.New('s',
() => { MoveMessage(transportMessage, transactionContext, sourceQueue); },
"Return to source queue '{0}'", sourceQueue));
}
options.Add(KeyOption.New('c', () =>
{
Console.Write("queue > ");
var queueName = Console.ReadLine();
MoveMessage(transportMessage, transactionContext, queueName);
}, "Enter custom queue name to move message to"));
Prompt(message, options);
Text.PrintLine();
}
/// <summary>
/// Handles the poisonous message by forwarding it to the configured error queue
/// </summary>
public async Task HandlePoisonMessage(TransportMessage transportMessage, ITransactionContext transactionContext, string errorDescription)
{
var headers = transportMessage.Headers;
string messageId ;
if (!headers.TryGetValue(Headers.MessageId, out messageId))
{
messageId = "<unknown>";
}
headers[Headers.ErrorDetails] = errorDescription;
headers[Headers.SourceQueue] = _transport.Address;
var errorQueueAddress = _simpleRetryStrategySettings.ErrorQueueAddress;
try
{
_log.Error("Moving message with ID {0} to error queue '{1}' - reason: {2}", messageId, errorQueueAddress, errorDescription);
await _transport.Send(errorQueueAddress, transportMessage, transactionContext);
}
catch (Exception exception)
{
_log.Error(exception, "Could not move message with ID {0} to error queue '{1}' - will pause {2} to avoid thrashing",
messageId, errorQueueAddress, MoveToErrorQueueFailedPause);
// if we can't move to error queue, we need to avoid thrashing over and over
await Task.Delay(MoveToErrorQueueFailedPause);
}
}
public async Task<Message> Deserialize(TransportMessage transportMessage)
{
string contentEncoding;
if (!transportMessage.Headers.TryGetValue(Headers.ContentEncoding, out contentEncoding))
{
return await _serializer.Deserialize(transportMessage);
}
if (contentEncoding != GzipEncodingHeader)
{
var message = $"The message {transportMessage.GetMessageLabel()} has a '{Headers.ContentEncoding}' with the" +
$" value '{contentEncoding}', but this serializer decorator only knows how to decompress" +
$" '{GzipEncodingHeader}'";
throw new ArgumentException(message);
}
var headers = transportMessage.Headers.Clone();
var compressedBody = transportMessage.Body;
headers.Remove(Headers.ContentEncoding);
var uncompressedBody = _zipper.Unzip(compressedBody);
var uncompressedTransportMessage = new TransportMessage(headers, uncompressedBody);
return await _serializer.Deserialize(uncompressedTransportMessage);
}
/// <summary>
/// Sends the given <see cref="TransportMessage"/> to the queue with the specified globally addressable name
/// </summary>
public async Task Send(string destinationAddress, TransportMessage message, ITransactionContext context)
{
context.OnCommitted(async () =>
{
var headers = message.Headers.Clone();
var queue = GetQueue(destinationAddress);
var messageId = Guid.NewGuid().ToString();
var popReceipt = Guid.NewGuid().ToString();
var timeToBeReceivedOrNull = GetTimeToBeReceivedOrNull(headers);
var queueVisibilityDelayOrNull = GetQueueVisibilityDelayOrNull(headers);
var cloudQueueMessage = Serialize(messageId, popReceipt, headers, message.Body);
try
{
var options = new QueueRequestOptions {RetryPolicy = new ExponentialRetry()};
var operationContext = new OperationContext();
await queue.AddMessageAsync(cloudQueueMessage, timeToBeReceivedOrNull, queueVisibilityDelayOrNull, options, operationContext);
}
catch (Exception exception)
{
throw new RebusApplicationException(exception, $"Could not send message with ID {cloudQueueMessage.Id} to '{destinationAddress}'");
}
});
}
/// <summary>
/// Constructs the step context, initially stashing the given <see cref="TransportMessage"/> and <see cref="ITransactionContext"/> into its bag of objects
/// </summary>
public IncomingStepContext(TransportMessage message, ITransactionContext transactionContext)
{
Save(message);
Save(transactionContext);
transactionContext.Items[StepContextKey] = this;
}
public void DispatchesDynamicMessageWhenDotNetTypeCannotBeFound()
{
var gotTheMessage = new ManualResetEvent(false);
string messageText = null;
_builtinHandlerActivator.Handle<dynamic>(async message =>
{
Console.WriteLine("Received dynamic message: {0}", message);
messageText = message.something.text;
gotTheMessage.Set();
});
var headers = new Dictionary<string, string>
{
{Headers.MessageId, Guid.NewGuid().ToString()},
{Headers.ContentType, "application/json;charset=utf-8"},
};
var transportMessage = new TransportMessage(headers, Encoding.UTF8.GetBytes(@"{
""something"": {
""text"": ""OMG dynamic JSON BABY!!""
}
}"));
_network.Deliver(InputQueueName, new InMemTransportMessage(transportMessage));
gotTheMessage.WaitOrDie(TimeSpan.FromSeconds(2));
Assert.That(messageText, Is.EqualTo("OMG dynamic JSON BABY!!"));
}
public async Task Send(string destinationAddress, TransportMessage message, ITransactionContext context)
{
await _innerTransport.Send(destinationAddress, message, context);
_sentMessages.Add(message);
MessageSent(message);
}
protected string GetStringBody(TransportMessage transportMessage)
{
if (transportMessage == null)
{
throw new InvalidOperationException("Cannot get string body out of null message!");
}
return _defaultEncoding.GetString(transportMessage.Body);
}
public async Task Send(string destinationAddress, TransportMessage message, ITransactionContext context)
{
if (_sendLatencyMs.HasValue)
{
await Task.Delay(_sendLatencyMs.Value);
}
await _innerTransport.Send(destinationAddress, message, context);
}
public void WorksWithPrefetch(int prefetch, int numberOfMessages)
{
var activator = Using(new BuiltinHandlerActivator());
var counter = new SharedCounter(numberOfMessages);
activator.Handle<string>(async str =>
{
counter.Decrement();
});
Console.WriteLine("Sending {0} messages", numberOfMessages);
var transport = GetTransport();
var tasks = Enumerable.Range(0, numberOfMessages)
.Select(i => string.Format("THIS IS MESSAGE # {0}", i))
.Select(async msg =>
{
using (var context = new DefaultTransactionContext())
{
var headers = DefaultHeaders();
var body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(msg));
var transportMessage = new TransportMessage(headers, body);
await transport.Send(_queueName, transportMessage, context);
await context.Complete();
}
})
.ToArray();
Task.WhenAll(tasks).Wait();
Console.WriteLine("Receiving {0} messages", numberOfMessages);
var stopwatch = Stopwatch.StartNew();
Configure.With(activator)
.Transport(t =>
{
t.UseAzureServiceBus(StandardAzureServiceBusTransportFactory.ConnectionString, _queueName, _mode)
.EnablePrefetching(prefetch);
})
.Options(o =>
{
o.SetNumberOfWorkers(5);
o.SetMaxParallelism(10);
})
.Start();
counter.WaitForResetEvent(timeoutSeconds: (int)(numberOfMessages * 0.1 + 3));
var elapsedSeconds = stopwatch.Elapsed.TotalSeconds;
Console.WriteLine("Receiving {0} messages took {1:0.0} s - that's {2:0.0} msg/s",
numberOfMessages, elapsedSeconds, numberOfMessages / elapsedSeconds);
}
async Task Send(IEnumerable<string> destinationAddressesList,
TransportMessage transportMessage,
ITransactionContext currentTransactionContext)
{
var sendTasks = destinationAddressesList
.Select(address => _transport.Send(address, transportMessage, currentTransactionContext))
.ToArray();
await Task.WhenAll(sendTasks);
}
void MutateLegacyTransportMessage(IncomingStepContext context, Dictionary<string, string> headers, TransportMessage transportMessage)
{
var newHeaders = MapTrivialHeaders(headers);
MapSpecialHeaders(newHeaders);
newHeaders[LegacyMessageHeader] = "";
context.Save(new TransportMessage(newHeaders, transportMessage.Body));
}
public async Task<Message> Deserialize(TransportMessage transportMessage)
{
using (var source = new MemoryStream(transportMessage.Body))
{
var headers = transportMessage.Headers.Clone();
var body = _runtimeTypeModel.Deserialize(source, null, GetMessageType(transportMessage.Headers));
return new Message(headers, body);
}
}
public async Task Send(string destinationAddress, TransportMessage message, ITransactionContext context)
{
var outgoingMessages = context.GetOrAdd(OutgoingMessagesItemsKey, () =>
{
context.OnCommitted(async () => SendOutgoingMessages(context));
return new ConcurrentQueue<OutgoingMessage>();
});
outgoingMessages.Enqueue(new OutgoingMessage(destinationAddress, message));
}
public void SetCommonHeaders(TransportMessage transportMessage)
{
var headers = transportMessage.Headers;
if (_transport.Address != null)
{
headers[AuditHeaders.HandleQueue] = _transport.Address;
}
headers[AuditHeaders.AuditTime] = RebusTime.Now.ToString("O");
headers[AuditHeaders.MachineName] = Environment.MachineName;
}
/// <summary>
/// Delivers the given message to the queue identitied by the given <paramref name="destinationAddress"/>
/// </summary>
public async Task Send(string destinationAddress, TransportMessage message, ITransactionContext context)
{
if (destinationAddress == null) throw new ArgumentNullException("destinationAddress");
if (message == null) throw new ArgumentNullException("message");
if (context == null) throw new ArgumentNullException("context");
if (!_network.HasQueue(destinationAddress))
{
throw new ArgumentException(string.Format("Destination queue address '{0}' does not exist!", destinationAddress));
}
context.OnCommitted(async () => _network.Deliver(destinationAddress, message.ToInMemTransportMessage()));
}
public async Task<Message> Deserialize(TransportMessage transportMessage)
{
var contentType = transportMessage.Headers.GetValue(Headers.ContentType);
if (contentType != JsonUtf8ContentType)
{
throw new FormatException(string.Format("Unknown content type: '{0}' - must be '{1}' for the JSON serialier to work", contentType, JsonUtf8ContentType));
}
var bodyString = DefaultEncoding.GetString(transportMessage.Body);
var bodyObject = JsonConvert.DeserializeObject(bodyString, _settings);
var headers = transportMessage.Headers.Clone();
return new Message(headers, bodyObject);
}
/// <summary>
/// Deserializes the given transport message to a logical message using MsgPack. Throws if the
/// required <see cref="Headers.ContentType"/> does not have the required <code>application/x-msgpack</code> value
/// or of the .NET type of the message cannot be resolved from the <see cref="Headers.Type"/> header (which
/// should carry the short assembly-qualified .NET type name of the message body)
/// </summary>
public async Task<Message> Deserialize(TransportMessage transportMessage)
{
var headers = transportMessage.Headers;
var contentType = headers.GetValue(Headers.ContentType);
if (contentType != MsgPackContentType)
{
throw new FormatException($"Unknown content type: '{contentType}' - must be '{MsgPackContentType}' for the MsgPack serialier to work");
}
var messageType = GetMessageType(headers);
var body = _packer.Unpack(messageType, transportMessage.Body);
return new Message(headers.Clone(), body);
}
/// <summary>
/// Deserializes the given <see cref="TransportMessage"/> back into a <see cref="Message"/>. Expects a
/// <see cref="Headers.ContentType"/> header with a value of <see cref="WireContentType"/>, otherwise
/// it will not attempt to deserialize the message.
/// </summary>
public async Task<Message> Deserialize(TransportMessage transportMessage)
{
var contentType = transportMessage.Headers.GetValue(Headers.ContentType);
if (contentType != WireContentType)
{
throw new FormatException($"Unknown content type: '{contentType}' - must be '{WireContentType}' for the JSON serialier to work");
}
using (var source = new MemoryStream(transportMessage.Body))
{
var body = _serializer.Deserialize(source);
return new Message(transportMessage.Headers.Clone(), body);
}
}
void PossiblyCompressTransportMessage(OutgoingStepContext context)
{
var transportMessage = context.Load<TransportMessage>();
if (transportMessage.Body == null) return;
if (transportMessage.Body.Length < _bodySizeThresholdBytes) return;
var headers = transportMessage.Headers.Clone();
var compressedBody = _zipper.Zip(transportMessage.Body);
headers[Headers.ContentEncoding] = GzipEncodingHeader;
var compressedTransportMessage = new TransportMessage(headers, compressedBody);
context.Save(compressedTransportMessage);
}
/// <summary>
/// Deserializes the given <see cref="TransportMessage"/> back into a <see cref="Message"/>
/// </summary>
public async Task<Message> Deserialize(TransportMessage transportMessage)
{
var contentType = transportMessage.Headers.GetValue(Headers.ContentType);
if (contentType == JsonUtf8ContentType)
{
return GetMessage(transportMessage, _encoding);
}
if (contentType.StartsWith(JsonContentType))
{
var encoding = GetEncoding(contentType);
return GetMessage(transportMessage, encoding);
}
throw new FormatException(string.Format("Unknown content type: '{0}' - must be '{1}' for the JSON serialier to work", contentType, JsonUtf8ContentType));
}
/// <summary>
/// Deserializes the given <see cref="TransportMessage"/> back into a <see cref="Message"/>
/// </summary>
public async Task<Message> Deserialize(TransportMessage transportMessage)
{
var contentType = transportMessage.Headers.GetValue(Headers.ContentType);
if (contentType != JsonUtf8ContentType)
{
throw new FormatException(string.Format("Unknown content type: '{0}' - must be '{1}' for the JSON serialier to work", contentType, JsonUtf8ContentType));
}
var headers = transportMessage.Headers.Clone();
var messageType = GetMessageType(headers);
var bodyString = Encoding.GetString(transportMessage.Body);
var bodyObject = messageType != null
? JSON.Deserialize(bodyString, messageType)
: JSON.DeserializeDynamic(bodyString);
return new Message(headers, bodyObject);
}
public async Task<TransportMessage> Serialize(Message message)
{
var transportMessage = await _serializer.Serialize(message);
var body = transportMessage.Body;
if (body.Length < _bodySizeThresholdBytes)
{
return transportMessage;
}
var headers = transportMessage.Headers.Clone();
var compressedBody = _zipper.Zip(transportMessage.Body);
headers[Headers.ContentEncoding] = GzipEncodingHeader;
var compressedTransportMessage = new TransportMessage(headers, compressedBody);
return compressedTransportMessage;
}
/// <summary>
/// Sends the specified message to the logical queue specified by <paramref name="destinationQueueName"/> by writing
/// a JSON serialied text to a file in the corresponding directory. The actual write operation is delayed until
/// the commit phase of the queue transaction unless we're non-transactional, in which case it is written immediately.
/// </summary>
public async Task Send(string destinationQueueName, TransportMessage message, ITransactionContext context)
{
EnsureQueueInitialized(destinationQueueName);
var destinationDirectory = GetDirectoryForQueueNamed(destinationQueueName);
var serializedMessage = Serialize(message);
var fileName = GetNextFileName();
var fullPath = Path.Combine(destinationDirectory, fileName);
context.OnCommitted(async () =>
{
using (var stream = File.OpenWrite(fullPath))
using (var writer = new StreamWriter(stream, FavoriteEncoding))
{
await writer.WriteAsync(serializedMessage);
}
});
}
public static BrokeredMessage CreateBrokeredMessage(TransportMessage message)
{
var headers = message.Headers.Clone();
var brokeredMessage = new BrokeredMessage(new MemoryStream(message.Body), true);
string timeToBeReceivedStr;
if (headers.TryGetValue(Headers.TimeToBeReceived, out timeToBeReceivedStr))
{
timeToBeReceivedStr = headers[Headers.TimeToBeReceived];
var timeToBeReceived = TimeSpan.Parse(timeToBeReceivedStr);
brokeredMessage.TimeToLive = timeToBeReceived;
}
string deferUntilTime;
if (headers.TryGetValue(Headers.DeferredUntil, out deferUntilTime))
{
var deferUntilDateTimeOffset = deferUntilTime.ToDateTimeOffset();
brokeredMessage.ScheduledEnqueueTimeUtc = deferUntilDateTimeOffset.UtcDateTime;
headers.Remove(Headers.DeferredUntil);
}
string contentType;
if (headers.TryGetValue(Headers.ContentType, out contentType))
{
brokeredMessage.ContentType = contentType;
}
string correlationId;
if (headers.TryGetValue(Headers.CorrelationId, out correlationId))
{
brokeredMessage.CorrelationId = correlationId;
}
brokeredMessage.Label = message.GetMessageLabel();
foreach (var kvp in headers)
{
brokeredMessage.Properties[kvp.Key] = PossiblyLimitLength(kvp.Value);
}
return brokeredMessage;
}
public async Task StillWorksWhenIncomingMessageCannotBeDeserialized()
{
const string brokenJsonString = @"{'broken': 'json', // DIE!!1}";
var headers = new Dictionary<string, string>
{
{Headers.MessageId, Guid.NewGuid().ToString()},
{Headers.ContentType, "application/json;charset=utf-8"},
};
var body = Encoding.UTF8.GetBytes(brokenJsonString);
var transportMessage = new TransportMessage(headers, body);
var inMemTransportMessage = new InMemTransportMessage(transportMessage);
_network.Deliver(InputQueueName, inMemTransportMessage);
await Task.Delay(1000);
var failedMessage = _network.GetNextOrNull("error");
Assert.That(failedMessage, Is.Not.Null);
var bodyString = Encoding.UTF8.GetString(failedMessage.Body);
Assert.That(bodyString, Is.EqualTo(brokenJsonString));
}
void MoveMessage(TransportMessage transportMessage, ITransactionContext transactionContext, string destinationQueue)
{
if (!_initializedQueues.Contains(destinationQueue))
{
_transport.CreateQueue(destinationQueue);
_initializedQueues.Add(destinationQueue);
}
Text.Print(" => '{0}' - ", destinationQueue);
try
{
_transport.Send(destinationQueue, transportMessage, transactionContext).Wait();
Text.PrintLine("OK");
}
catch (Exception exception)
{
Text.PrintLine(exception.Message);
throw;
}
}
/// <summary>
/// Sends the given <see cref="TransportMessage"/> to the queue with the specified globally addressable name
/// </summary>
public async Task Send(string destinationAddress, TransportMessage message, ITransactionContext context)
{
context.OnCommitted(async () =>
{
var queue = GetQueue(destinationAddress);
var messageId = Guid.NewGuid().ToString();
var popReceipt = Guid.NewGuid().ToString();
var timeToBeReceived = GetTimeToBeReceivedOrNull(message);
var cloudQueueMessage = Serialize(message, messageId, popReceipt);
try
{
var options = new QueueRequestOptions {RetryPolicy = new ExponentialRetry()};
var operationContext = new OperationContext();
await queue.AddMessageAsync(cloudQueueMessage, timeToBeReceived, null, options, operationContext);
}
catch (Exception exception)
{
throw new RebusApplicationException(string.Format("Could not send message with ID {0} to '{1}'", cloudQueueMessage.Id, destinationAddress), exception);
}
});
}
/// <summary>
/// Receives the next message by querying the messages table for a message with a recipient matching this transport's <see cref="Address"/>
/// </summary>
public async Task<TransportMessage> Receive(ITransactionContext context)
{
using (await _bottleneck.Enter())
{
var connection = await GetConnection(context);
long? idOfMessageToDelete;
TransportMessage receivedTransportMessage;
using (var selectCommand = connection.CreateCommand())
{
selectCommand.CommandText = string.Format(@"
SET NOCOUNT ON
;WITH TopCTE AS (
SELECT TOP 1
[id],
[headers],
[body]
FROM {0} M WITH (ROWLOCK, READPAST)
WHERE M.[recipient] = @recipient
AND M.[visible] < getdate()
AND M.[expiration] > getdate()
ORDER
BY [priority] ASC,
[id] ASC
)
DELETE FROM TopCTE
OUTPUT deleted.[id] as [id],
deleted.[headers] as [headers],
deleted.[body] as [body]
", _tableName);
selectCommand.Parameters.Add("recipient", SqlDbType.NVarChar, RecipientColumnSize).Value = _inputQueueName;
using (var reader = await selectCommand.ExecuteReaderAsync())
{
if (!await reader.ReadAsync()) return null;
var headers = reader["headers"];
var headersDictionary = _headerSerializer.Deserialize((byte[])headers);
idOfMessageToDelete = (long)reader["id"];
var body = (byte[])reader["body"];
receivedTransportMessage = new TransportMessage(headersDictionary, body);
}
}
if (!idOfMessageToDelete.HasValue)
{
return null;
}
return receivedTransportMessage;
}
}