public Task Send(IFullMessage[] messages, string destination)
{
var options = new SendOptions();
options.SetDestination(destination);
var message = new BulkMessage
{
Messages = messages
};
_metrics.Mark("Dispatched Messages", Unit.Message);
return(Bus.Instance.Send(message, options));
}
public Task Publish(IFullMessage[] messages)
{
var options = new PublishOptions();
_metrics.Mark("Dispatched Messages", Unit.Message);
// Todo: publish would only be called for messages on a single stream
// we can set a routing key somehow for BulkMessage so its routed to the same sharded queue
var message = new BulkMessage
{
Messages = messages
};
// Publishing an IMessage normally creates a warning
options.DoNotEnforceBestPractices();
return(Bus.Instance.Publish(message, options));
}
private static async Task ProcessEvents(ThreadParam param, DelayedClient client, ResolvedEvent[] events)
{
// A fake message that will travel through the pipeline in order to bulk process messages from the context bag
var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray();
var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray();
Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events on stream {events.First().Event.EventStreamId}");
var contextBag = new ContextBag();
// Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one
contextBag.Set(Defaults.BulkHeader, delayed);
// Run bulk process on this thread
using (var tokenSource = new CancellationTokenSource())
{
var success = false;
var retry = 0;
do
{
var transportTransaction = new TransportTransaction();
// Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message
var messageId = Guid.NewGuid().ToString();
var headers = new Dictionary <string, string>()
{
[Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName,
[Headers.MessageIntent] = MessageIntentEnum.Send.ToString(),
[Headers.MessageId] = messageId,
[Defaults.BulkHeader] = delayed.Count().ToString(),
};
try
{
// If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
param.Token.ThrowIfCancellationRequested();
// Don't re-use the event id for the message id
var messageContext = new NServiceBus.Transport.MessageContext(messageId,
headers,
bulkMarker, transportTransaction, tokenSource,
contextBag);
await Bus.OnMessage(messageContext).ConfigureAwait(false);//param.Token);
Logger.Write(LogLevel.Debug,
() => $"Scheduling acknowledge of {delayed.Count()} bulk events");
DelayedCount.Increment(delayed.Count());
client.Acknowledge(events);
success = true;
}
catch (ObjectDisposedException)
{
// NSB transport has been disconnected
break;
}
catch (Exception e)
{
DelayedErrors.Mark($"{e.GetType().Name} {e.Message}");
}
retry++;
} while (!success && retry <= param.MaxRetry);
if (!success)
{
// Dont run through NSB's error handler, it expects a single serialized message which
// is not compatible here. Just tell EventStore to retry it
// Todo: ES will park messages that fail - develop something to monitor parked messages
client.Nack(events);
}
}
}
private static void Threaded(object state)
{
var param = (ThreadParam)state;
// A fake message that will travel through the pipeline in order to bulk process messages from the context bag
var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray();
param.Clients.SelectAsync(x => x.Connect()).Wait();
var threadIdle = Metric.Timer("Delayed Events Idle", Unit.None, tags: "debug");
TimerContext?idleContext = threadIdle.NewContext();
while (true)
{
param.Token.ThrowIfCancellationRequested();
var noEvents = true;
for (var i = 0; i < param.Clients.Count(); i++)
{
var client = param.Clients.ElementAt(i);
var events = client.Flush();
if (!events.Any())
{
continue;
}
noEvents = false;
idleContext?.Dispose();
idleContext = null;
var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray();
Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events");
var transportTransaction = new TransportTransaction();
var contextBag = new ContextBag();
// Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one
contextBag.Set(Defaults.BulkHeader, delayed);
// Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message
var messageId = Guid.NewGuid().ToString();
var headers = new Dictionary <string, string>()
{
[Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName,
[Headers.MessageIntent] = MessageIntentEnum.Send.ToString(),
[Headers.MessageId] = messageId,
[Defaults.BulkHeader] = delayed.Count().ToString(),
};
// Run bulk process on this thread
using (var tokenSource = new CancellationTokenSource())
{
var success = false;
var retry = 0;
do
{
using (var ctx = DelayedExecution.NewContext())
{
try
{
// If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
param.Token.ThrowIfCancellationRequested();
// Don't re-use the event id for the message id
var messageContext = new NServiceBus.Transport.MessageContext(messageId,
headers,
bulkMarker, transportTransaction, tokenSource,
contextBag);
Bus.OnMessage(messageContext).Wait(param.Token);
Logger.Write(LogLevel.Debug,
() => $"Scheduling acknowledge of {delayed.Count()} bulk events");
DelayedHandled.Update(delayed.Count());
client.Acknowledge(events);
success = true;
}
catch (ObjectDisposedException)
{
// NSB transport has been disconnected
break;
}
catch (Exception ex)
{
}
if (ctx.Elapsed > TimeSpan.FromSeconds(5))
{
SlowLogger.Warn(
$"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalSeconds} seconds!");
}
Logger.Write(LogLevel.Info,
() =>
$"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalMilliseconds} ms");
}
// Todo: use max retry setting
} while (!success && retry < 10);
if (!success)
{
// Dont run through NSB's error handler, it expects a single serialized message which
// is not compatible here. Just tell EventStore to retry it
// Todo: ES will park messages that fail - develop something to monitor parked messages
client.Nack(events);
}
}
}
if (idleContext == null)
{
idleContext = threadIdle.NewContext();
}
// Cheap hack to not burn cpu incase there are no events
if (noEvents)
{
Thread.Sleep(50);
}
}
}
