protected override async Task Terminate(IInvokeHandlerContext context)
{
var channel = context.Builder.Build <IDelayedChannel>();
var msgType = context.MessageBeingHandled.GetType();
if (!msgType.IsInterface)
{
msgType = _mapper.GetMappedTypeFor(msgType) ?? msgType;
}
context.Extensions.Set(new State
{
ScopeWasPresent = Transaction.Current != null
});
var messageHandler = context.MessageHandler;
var channelKey = $"{messageHandler.HandlerType.FullName}:{msgType.FullName}";
bool contains = false;
lock (Lock) contains = IsNotDelayed.Contains(channelKey);
var contextChannelKey = "";
// Special case for when we are bulk processing messages from DelayedSubscriber, simply process it and return dont check for more bulk
if (channel == null || contains || (context.Extensions.TryGet(Defaults.ChannelKey, out contextChannelKey) && contextChannelKey == channelKey))
{
Logger.Write(LogLevel.Debug, () => $"Invoking handle for message {msgType.FullName} on handler {messageHandler.HandlerType.FullName}");
await messageHandler.Invoke(context.MessageBeingHandled, context).ConfigureAwait(false);
return;
}
// If we are bulk processing and the above check fails it means the current message handler shouldn't be called
if (!string.IsNullOrEmpty(contextChannelKey))
{
return;
}
if (IsDelayed.ContainsKey(channelKey))
{
DelayedAttribute delayed;
IsDelayed.TryGetValue(channelKey, out delayed);
var msgPkg = new DelayedMessage
{
MessageId = context.MessageId,
Headers = context.Headers,
Message = context.MessageBeingHandled,
Received = DateTime.UtcNow,
ChannelKey = channelKey
};
InvokesDelayed.Mark();
var specificKey = "";
if (delayed.KeyPropertyFunc != null)
{
try
{
specificKey = delayed.KeyPropertyFunc(context.MessageBeingHandled);
}
catch
{
Logger.Warn($"Failed to get key properties from message {msgType.FullName}");
}
}
await channel.AddToQueue(channelKey, msgPkg, key : specificKey).ConfigureAwait(false);
bool bulkInvoked;
if (context.Extensions.TryGet <bool>("BulkInvoked", out bulkInvoked) && bulkInvoked)
{
// Prevents a single message from triggering a dozen different bulk invokes
Logger.Write(LogLevel.Info, () => $"Limiting bulk processing for a single message to a single invoke");
return;
}
lock (RecentLock)
{
if (RecentlyInvoked.ContainsKey($"{channelKey}.{specificKey}"))
{
Logger.Write(LogLevel.Info, () => $"Channel [{channel}] specific [{specificKey}] was checked by this instance recently - leaving it alone");
return;
}
RecentlyInvoked.Add($"{channelKey}.{specificKey}", DateTime.UtcNow.AddSeconds(10));
}
int? size = null;
TimeSpan?age = null;
if (delayed.Delay.HasValue)
{
age = await channel.Age(channelKey, key : specificKey).ConfigureAwait(false);
}
if (delayed.Count.HasValue)
{
size = await channel.Size(channelKey, key : specificKey).ConfigureAwait(false);
}
if (!ShouldExecute(delayed, size, age))
{
Logger.Write(LogLevel.Info, () => $"Threshold Count [{delayed.Count}] DelayMs [{delayed.Delay}] Size [{size}] Age [{age?.TotalMilliseconds}] - delaying processing channel [{channelKey}]");
return;
}
context.Extensions.Set <bool>("BulkInvoked", true);
Logger.Write(LogLevel.Info, () => $"Threshold Count [{delayed.Count}] DelayMs [{delayed.Delay}] Size [{size}] Age [{age?.TotalMilliseconds}] - bulk processing channel [{channelKey}]");
await InvokeDelayedChannel(channel, channelKey, specificKey, delayed, messageHandler, context).ConfigureAwait(false);
return;
}
var attrs =
Attribute.GetCustomAttributes(messageHandler.HandlerType, typeof(DelayedAttribute))
.Cast <DelayedAttribute>();
var single = attrs.SingleOrDefault(x => x.Type == msgType);
if (single == null)
{
lock (Lock) IsNotDelayed.Add(channelKey);
}
else
{
Logger.Write(LogLevel.Debug,
() => $"Found delayed handler {messageHandler.HandlerType.FullName} for message {msgType.FullName}");
IsDelayed.TryAdd(channelKey, single);
}
await Terminate(context).ConfigureAwait(false);
}
protected override async Task Terminate(IInvokeHandlerContext context)
{
var channel = _builder.Build <IDelayedChannel>();
var msgType = context.MessageBeingHandled.GetType();
if (!msgType.IsInterface)
{
msgType = _mapper.GetMappedTypeFor(msgType) ?? msgType;
}
context.Extensions.Set(new State
{
ScopeWasPresent = Transaction.Current != null
});
var messageHandler = context.MessageHandler;
var key = $"{messageHandler.HandlerType.FullName}:{msgType.FullName}";
if (IsNotDelayed.Contains(key))
{
Logger.Write(LogLevel.Debug, () => $"Invoking handle for message {msgType.FullName} on handler {messageHandler.HandlerType.FullName}");
await messageHandler.Invoke(context.MessageBeingHandled, context).ConfigureAwait(false);
return;
}
if (IsDelayed.ContainsKey(key))
{
DelayedAttribute delayed;
IsDelayed.TryGetValue(key, out delayed);
var msgPkg = new DelayedMessage
{
MessageId = context.MessageId,
Headers = context.Headers,
Message = context.MessageBeingHandled,
Received = DateTime.UtcNow,
};
InvokesDelayed.Mark();
if (delayed.KeyPropertyFunc != null)
{
try
{
key = $"{key}:{delayed.KeyPropertyFunc(context.MessageBeingHandled)}";
}
catch
{
Logger.Warn($"Failed to get property key {delayed.KeyProperty} on message {msgType.FullName}");
}
}
Logger.Write(LogLevel.Debug, () => $"Delaying message {msgType.FullName} delivery");
var size = await channel.AddToQueue(key, msgPkg).ConfigureAwait(false);
if (_bulkedMessageId == context.MessageId)
{
// Prevents a single message from triggering a dozen different bulk invokes
Logger.Write(LogLevel.Debug, () => $"Limiting bulk processing for a single message to a single invoke");
return;
}
TimeSpan?age = null;
if (delayed.Delay.HasValue)
{
age = await channel.Age(key).ConfigureAwait(false);
}
if ((delayed.Count.HasValue && size < delayed.Count.Value) || (delayed.Delay.HasValue && age < TimeSpan.FromMilliseconds(delayed.Delay.Value)))
{
Logger.Write(LogLevel.Debug, () => $"Threshold Count [{delayed.Count}] DelayMs [{delayed.Delay}] not hit Size [{size}] Age [{age?.TotalMilliseconds}] - delaying processing {msgType.FullName}");
return;
}
_bulkedMessageId = context.MessageId;
Logger.Write(LogLevel.Debug, () => $"Threshold Count [{delayed.Count}] DelayMs [{delayed.Delay}] hit Size [{size}] Age [{age?.TotalMilliseconds}] - bulk processing {msgType.FullName}");
var msgs = await channel.Pull(key).ConfigureAwait(false);
Invokes.Mark();
Logger.Write(LogLevel.Debug, () => $"Invoking handle {msgs.Count()} times for message {msgType.FullName} on handler {messageHandler.HandlerType.FullName}");
foreach (var msg in msgs.Cast <DelayedMessage>())
{
await messageHandler.Invoke(msg.Message, context).ConfigureAwait(false);
}
return;
}
var attrs =
Attribute.GetCustomAttributes(messageHandler.HandlerType, typeof(DelayedAttribute))
.Cast <DelayedAttribute>();
var single = attrs.SingleOrDefault(x => x.Type == msgType);
if (single == null)
{
IsNotDelayed.Add(key);
}
else
{
Logger.Write(LogLevel.Debug,
() => $"Found delayed handler {messageHandler.HandlerType.FullName} for message {msgType.FullName}");
IsDelayed.TryAdd(key, single);
}
await Terminate(context).ConfigureAwait(false);
}
protected override async Task Terminate(IInvokeHandlerContext context)
{
if (context.Extensions.TryGet(out ActiveSagaInstance saga) && saga.NotFound && ((SagaMetadata)saga.GetType().GetProperty("Metadata", BindingFlags.NonPublic).GetValue(saga)).SagaType == context.MessageHandler.Instance.GetType())
{
return;
}
IDelayedChannel channel = null;
try
{
IContainer container;
if (context.Extensions.TryGet <IContainer>(out container))
{
channel = container.Resolve <IDelayedChannel>();
}
}
catch (Exception)
{
// Catch in case IDelayedChannel isn't registered which shouldn't happen unless a user registered Consumer without GetEventStore
}
var msgType = context.MessageBeingHandled.GetType();
if (!msgType.IsInterface)
{
msgType = _mapper.GetMappedTypeFor(msgType) ?? msgType;
}
context.Extensions.Set(new State
{
ScopeWasPresent = Transaction.Current != null
});
var messageHandler = context.MessageHandler;
var channelKey = $"{messageHandler.HandlerType.FullName}:{msgType.FullName}";
bool contains = false;
lock (Lock) contains = IsNotDelayed.Contains(channelKey);
context.Extensions.TryGet(Defaults.ChannelKey, out string contextChannelKey);
// Special case for when we are bulk processing messages from DelayedSubscriber or BulkMessage, simply process it and return dont check for more bulk
if (channel == null || contains || contextChannelKey == channelKey || context.Headers.ContainsKey(Defaults.BulkHeader))
{
await messageHandler.Invoke(context.MessageBeingHandled, context).ConfigureAwait(false);
return;
}
// If we are bulk processing and the above check fails it means the current message handler shouldn't be called
// because if contextChannelKey is set we are using delayed bulk SendLocal
// which should only execute the message on the handlers that were marked delayed, the other handlers were already invoked when the message
// originally came in
if (!string.IsNullOrEmpty(contextChannelKey))
{
return;
}
if (IsDelayed.ContainsKey(channelKey))
{
DelayedAttribute delayed;
IsDelayed.TryGetValue(channelKey, out delayed);
var specificKey = "";
if (delayed.KeyPropertyFunc != null)
{
try
{
specificKey = delayed.KeyPropertyFunc(context.MessageBeingHandled);
}
catch
{
Logger.Warn($"Failed to get key properties from message {msgType.FullName}");
}
}
var msgPkg = new DelayedMessage
{
MessageId = context.MessageId,
Headers = context.Headers,
Message = context.MessageBeingHandled as IMessage,
Received = DateTime.UtcNow,
ChannelKey = channelKey,
SpecificKey = specificKey
};
await channel.AddToQueue(channelKey, msgPkg, key : specificKey).ConfigureAwait(false);
Logger.DebugEvent("QueueAdd", "Message added to delayed queue [{Channel:l}] key [{Key:l}]", channelKey, specificKey);
bool bulkInvoked;
if (context.Extensions.TryGet <bool>("BulkInvoked", out bulkInvoked) && bulkInvoked)
{
// Prevents a single message from triggering a dozen different bulk invokes
return;
}
int? size = null;
TimeSpan?age = null;
if (delayed.Delay.HasValue)
{
age = await channel.Age(channelKey, key : specificKey).ConfigureAwait(false);
}
if (delayed.Count.HasValue)
{
size = await channel.Size(channelKey, key : specificKey).ConfigureAwait(false);
}
if (!ShouldExecute(delayed, size, age))
{
return;
}
context.Extensions.Set <bool>("BulkInvoked", true);
Logger.DebugEvent("Threshold", "Count [{Count}] DelayMs [{DelayMs}] bulk procesing Size [{Size}] Age [{Age}] - [{Channel:l}] key [{Key:l}]", delayed.Count, delayed.Delay, size, age?.TotalMilliseconds, channelKey, specificKey);
await InvokeDelayedChannel(channel, channelKey, specificKey, delayed, messageHandler, context).ConfigureAwait(false);
return;
}
var attrs =
Attribute.GetCustomAttributes(messageHandler.HandlerType, typeof(DelayedAttribute))
.Cast <DelayedAttribute>();
var single = attrs.SingleOrDefault(x => x.Type == msgType);
if (single == null)
{
lock (Lock) IsNotDelayed.Add(channelKey);
}
else
{
IsDelayed.TryAdd(channelKey, single);
}
await Terminate(context).ConfigureAwait(false);
}
public async Task Resolve <T>(T entity, IEnumerable <IFullEvent> uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var sourced = (IEventSourced)entity;
// Store conflicting events in memory
// After 100 or so pile up pull the latest stream and attempt to write them again
var streamName = _streamGen(typeof(T), StreamTypes.Domain, sourced.Stream.Bucket, sourced.Stream.StreamId, sourced.Stream.Parents);
var message = new ConflictingEvents
{
Bucket = sourced.Stream.Bucket,
StreamId = sourced.Stream.StreamId,
EntityType = typeof(T).AssemblyQualifiedName,
Parents = BuildParentList(entity),
Events = uncommitted
};
var package = new DelayedMessage
{
MessageId = Guid.NewGuid().ToString(),
Headers = new Dictionary <string, string>(),
Message = message,
Received = DateTime.UtcNow,
ChannelKey = streamName
};
foreach (var header in commitHeaders)
{
package.Headers[$"Conflict.{header.Key}"] = header.Value;
}
await _delay.AddToQueue(ConcurrencyConflict.ResolveWeakly.ToString(), package, streamName)
.ConfigureAwait(false);
// Todo: make 30 seconds configurable
var age = await _delay.Age(ConcurrencyConflict.ResolveWeakly.ToString(), streamName).ConfigureAwait(false);
if (!age.HasValue || age < TimeSpan.FromSeconds(30))
{
return;
}
var stream = sourced.Stream;
Logger.Write(LogLevel.Info,
() => $"Starting weak conflict resolve for stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]");
try
{
await _store.Freeze <T>(stream).ConfigureAwait(false);
var delayed = await _delay.Pull(streamName, max : 200).ConfigureAwait(false);
// If someone else pulled while we were waiting
if (!delayed.Any())
{
return;
}
Logger.Write(LogLevel.Info,
() => $"Resolving {delayed.Count()} uncommitted events to stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]");
var latestEvents =
await _eventstore.GetEvents(streamName, stream.CommitVersion + 1L)
.ConfigureAwait(false);
Logger.Write(LogLevel.Info,
() => $"Stream [{stream.StreamId}] bucket [{stream.Bucket}] is {latestEvents.Count()} events behind store");
sourced.Hydrate(latestEvents.Select(x => x.Event as IEvent));
Logger.Write(LogLevel.Debug, () => $"Merging {delayed.Count()} conflicted events");
try
{
foreach (var u in delayed.Select(x => x.Message as ConflictingEvents).SelectMany(x => x.Events))
{
sourced.Conflict(u.Event as IEvent,
metadata:
new Dictionary <string, string>
{
{ "ConflictResolution", ConcurrencyConflict.ResolveWeakly.ToString() }
});
}
}
catch (NoRouteException e)
{
Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}");
throw new ConflictResolutionFailedException("Failed to resolve conflict", e);
}
Logger.Write(LogLevel.Info, () => "Successfully merged conflicted events");
if (stream.StreamVersion != stream.CommitVersion && entity is ISnapshotting &&
((ISnapshotting)entity).ShouldTakeSnapshot())
{
Logger.Write(LogLevel.Debug,
() =>
$"Taking snapshot of [{typeof(T).FullName}] id [{entity.Id}] version {stream.StreamVersion}");
var memento = ((ISnapshotting)entity).TakeSnapshot();
stream.AddSnapshot(memento);
}
await _store.WriteStream <T>(commitId, stream, commitHeaders).ConfigureAwait(false);
}
finally
{
await _store.Unfreeze <T>(stream).ConfigureAwait(false);
}
}