private async Task InvokeDelayedChannel(IDelayedChannel channel, string channelKey, string specificKey, DelayedAttribute attr, MessageHandler handler, IInvokeHandlerContext context)
{
var msgs = await channel.Pull(channelKey, key : specificKey, max : attr.Count).ConfigureAwait(false);
if (!msgs.Any())
{
Logger.Write(LogLevel.Debug, () => $"No delayed events found on channel [{channelKey}] specific key [{specificKey}]");
return;
}
var idx = 0;
var count = msgs.Count();
Invokes.Mark();
InvokeSize.Update(msgs.Count());
Logger.Write(LogLevel.Info, () => $"Starting invoke handle {count} times channel key [{channelKey}] specific key [{specificKey}]");
using (var ctx = InvokeTime.NewContext())
{
foreach (var msg in msgs.Cast <DelayedMessage>())
{
idx++;
Logger.Write(LogLevel.Debug,
() => $"Invoking handle {idx}/{count} times channel key [{channelKey}] specific key [{specificKey}]");
await handler.Invoke(msg.Message, context).ConfigureAwait(false);
}
if (ctx.Elapsed > TimeSpan.FromSeconds(5))
{
SlowLogger.Write(LogLevel.Warn, () => $"Bulk invoking {count} times on channel key [{channelKey}] specific key [{specificKey}] took {ctx.Elapsed.TotalSeconds} seconds!");
}
Logger.Write(LogLevel.Info, () => $"Bulk invoking {count} times on channel key [{channelKey}] specific key [{specificKey}] took {ctx.Elapsed.TotalMilliseconds} ms!");
}
Logger.Write(LogLevel.Info, () => $"Finished invoke handle {count} times channel key [{channelKey}] specific key [{specificKey}]");
}
private async Task InvokeDelayedChannel(IDelayedChannel channel, string channelKey, string specificKey, DelayedAttribute attr, MessageHandler handler, IInvokeHandlerContext context)
{
var msgs = await channel.Pull(channelKey, key : specificKey, max : attr.Count).ConfigureAwait(false);
var messages = msgs as IDelayedMessage[] ?? msgs.ToArray();
if (!messages.Any())
{
Logger.Write(LogLevel.Debug, () => $"No delayed events found on channel [{channelKey}] specific key [{specificKey}]");
return;
}
var count = messages.Length;
Logger.Write(LogLevel.Debug, () => $"Starting invoke handle {count} times channel key [{channelKey}] specific key [{specificKey}]");
using (var ctx = _metrics.Begin("Bulk Messages Time"))
{
foreach (var idx in Enumerable.Range(0, messages.Length))
{
Logger.Write(LogLevel.Debug,
() => $"Invoking handle {idx}/{count} times channel key [{channelKey}] specific key [{specificKey}]");
await handler.Invoke(messages[idx].Message, context).ConfigureAwait(false);
}
if (ctx.Elapsed > TimeSpan.FromSeconds(5))
{
SlowLogger.Write(LogLevel.Warn, () => $"Bulk invoking {count} times on channel key [{channelKey}] specific key [{specificKey}] took {ctx.Elapsed.TotalSeconds} seconds!");
}
Logger.Write(LogLevel.Info, () => $"Bulk invoking {count} times on channel key [{channelKey}] specific key [{specificKey}] took {ctx.Elapsed.TotalMilliseconds} ms!");
}
}
private async Task InvokeDelayedChannel(IDelayedChannel channel, string channelKey, string specificKey, DelayedAttribute attr, MessageHandler handler, IInvokeHandlerContext context)
{
var msgs = await channel.Pull(channelKey, key : specificKey, max : attr.Count).ConfigureAwait(false);
var messages = msgs as IDelayedMessage[] ?? msgs.ToArray();
var count = messages.Length;
using (var ctx = _metrics.Begin("Bulk Messages Time"))
{
switch (attr.Mode)
{
case DeliveryMode.Single:
foreach (var idx in Enumerable.Range(0, messages.Length))
{
await handler.Invoke(messages[idx].Message, context).ConfigureAwait(false);
}
break;
case DeliveryMode.First:
await handler.Invoke(messages[0].Message, context).ConfigureAwait(false);
break;
case DeliveryMode.Last:
await handler.Invoke(messages[messages.Length - 1].Message, context).ConfigureAwait(false);
break;
case DeliveryMode.FirstAndLast:
await handler.Invoke(messages[0].Message, context).ConfigureAwait(false);
await handler.Invoke(messages[messages.Length - 1].Message, context).ConfigureAwait(false);
break;
}
if (ctx.Elapsed > TimeSpan.FromSeconds(5))
{
SlowLogger.InfoEvent("Invoked", "{Count} messages channel [{Channel:l}] key [{Key:l}] took {Milliseconds} ms", count, channelKey, specificKey, ctx.Elapsed.TotalMilliseconds);
}
Logger.DebugEvent("Invoked", "{Count} messages channel [{Channel:l}] key [{Key:l}] took {Milliseconds} ms", count, channelKey, specificKey, ctx.Elapsed.TotalMilliseconds);
}
}
public async Task <Guid> Resolve <T>(T entity, IEnumerable <IWritableEvent> uncommitted, Guid commitId, Guid startingEventId, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
// Store conflicting events in memory
// After 100 or so pile up pull the latest stream and attempt to write them again
foreach (var @event in uncommitted)
{
await _delay.AddToQueue(entity.StreamId, @event).ConfigureAwait(false);
}
// Todo: make 30 seconds configurable
if (await _delay.Age(entity.StreamId).ConfigureAwait(false) < TimeSpan.FromSeconds(30))
{
return(startingEventId);
}
var stream = entity.Stream;
Logger.Write(LogLevel.Debug, () => $"Starting weak conflict resolve for stream [{stream.StreamId}] bucket [{stream.Bucket}]");
try
{
try
{
await _store.Freeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false);
}
catch (VersionException)
{
Logger.Write(LogLevel.Debug, () => $"Stopping weak conflict resolve - someone else is processing");
return(startingEventId);
}
uncommitted = (await _delay.Pull(entity.StreamId).ConfigureAwait(false)).Cast <IWritableEvent>();
Logger.Write(LogLevel.Info,
() =>
$"Resolving {uncommitted.Count()} uncommitted events to stream [{stream.StreamId}] bucket [{stream.Bucket}]");
var latestEvents =
await
_store.GetEvents <T>(stream.Bucket, stream.StreamId, stream.CommitVersion + 1)
.ConfigureAwait(false);
Logger.Write(LogLevel.Debug, () => $"Stream is {latestEvents.Count()} events behind store");
var writableEvents = latestEvents as IWritableEvent[] ?? latestEvents.ToArray();
stream.Concat(writableEvents);
entity.Hydrate(writableEvents.Select(x => x.Event as IEvent));
Logger.Write(LogLevel.Debug, () => "Merging conflicted events");
try
{
foreach (var u in uncommitted)
{
entity.Conflict(u.Event as IEvent);
}
}
catch (NoRouteException e)
{
Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}");
throw new ConflictResolutionFailedException("Failed to resolve conflict", e);
}
Logger.Write(LogLevel.Debug, () => "Successfully merged conflicted events");
if (stream.StreamVersion != stream.CommitVersion && entity is ISnapshotting &&
((ISnapshotting)entity).ShouldTakeSnapshot())
{
Logger.Write(LogLevel.Debug,
() =>
$"Taking snapshot of [{typeof(T).FullName}] id [{entity.StreamId}] version {stream.StreamVersion}");
var memento = ((ISnapshotting)entity).TakeSnapshot();
stream.AddSnapshot(memento, commitHeaders);
}
startingEventId = await stream.Commit(commitId, startingEventId, commitHeaders).ConfigureAwait(false);
}
finally
{
await _store.Unfreeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false);
}
return(startingEventId);
}
public async Task Resolve <T>(T entity, IEnumerable <IWritableEvent> uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
// Todo fix delayed channel impl - currently doesn't do Age from store anymore
throw new NotImplementedException("See todo - needs fix");
// Store conflicting events in memory
// After 100 or so pile up pull the latest stream and attempt to write them again
var streamName = _streamGen(typeof(T), StreamTypes.Domain, entity.Bucket, entity.StreamId);
foreach (var @event in uncommitted)
{
await _delay.AddToQueue(streamName, @event).ConfigureAwait(false);
}
// Todo: make 30 seconds configurable
// Todo: if a stream has a single conflict and never conflicts again, the event we delay will never be processed
var age = await _delay.Age(streamName).ConfigureAwait(false);
if (!age.HasValue || age < TimeSpan.FromSeconds(30))
{
return;
}
var stream = entity.Stream;
Logger.Write(LogLevel.Info, () => $"Starting weak conflict resolve for stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]");
try
{
try
{
await _store.Freeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false);
}
catch (VersionException)
{
Logger.Write(LogLevel.Info, () => $"Stopping weak conflict resolve - someone else is processing");
return;
}
uncommitted = (await _delay.Pull(streamName, max: _maxPulledDelayed).ConfigureAwait(false)).Cast <IWritableEvent>();
// If someone else pulled while we were waiting
if (!uncommitted.Any())
{
return;
}
Logger.Write(LogLevel.Info,
() => $"Resolving {uncommitted.Count()} uncommitted events to stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]");
var latestEvents =
await _store.GetEvents <T>(stream.Bucket, stream.StreamId, stream.CommitVersion + 1)
.ConfigureAwait(false);
Logger.Write(LogLevel.Info, () => $"Stream [{stream.StreamId}] bucket [{stream.Bucket}] is {latestEvents.Count()} events behind store");
var writableEvents = latestEvents as IWritableEvent[] ?? latestEvents.ToArray();
stream.Concat(writableEvents);
entity.Hydrate(writableEvents.Select(x => x.Event as IEvent));
Logger.Write(LogLevel.Debug, () => "Merging conflicted events");
try
{
foreach (var u in uncommitted)
{
entity.Conflict(u.Event as IEvent, metadata: new Dictionary <string, string> {
{ "ConflictResolution", ConcurrencyConflict.ResolveWeakly.ToString() }
});
}
}
catch (NoRouteException e)
{
Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}");
throw new ConflictResolutionFailedException("Failed to resolve conflict", e);
}
Logger.Write(LogLevel.Info, () => "Successfully merged conflicted events");
if (stream.StreamVersion != stream.CommitVersion && entity is ISnapshotting &&
((ISnapshotting)entity).ShouldTakeSnapshot())
{
Logger.Write(LogLevel.Debug,
() =>
$"Taking snapshot of [{typeof(T).FullName}] id [{entity.StreamId}] version {stream.StreamVersion}");
var memento = ((ISnapshotting)entity).TakeSnapshot();
stream.AddSnapshot(memento);
}
await stream.Commit(commitId, commitHeaders).ConfigureAwait(false);
}
finally
{
await _store.Unfreeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false);
}
}
public async Task Resolve <T>(T entity, IEnumerable <IFullEvent> uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var sourced = (IEventSourced)entity;
// Store conflicting events in memory
// After 100 or so pile up pull the latest stream and attempt to write them again
var streamName = _streamGen(typeof(T), StreamTypes.Domain, sourced.Stream.Bucket, sourced.Stream.StreamId, sourced.Stream.Parents);
var message = new ConflictingEvents
{
Bucket = sourced.Stream.Bucket,
StreamId = sourced.Stream.StreamId,
EntityType = typeof(T).AssemblyQualifiedName,
Parents = BuildParentList(entity),
Events = uncommitted
};
var package = new DelayedMessage
{
MessageId = Guid.NewGuid().ToString(),
Headers = new Dictionary <string, string>(),
Message = message,
Received = DateTime.UtcNow,
ChannelKey = streamName
};
foreach (var header in commitHeaders)
{
package.Headers[$"Conflict.{header.Key}"] = header.Value;
}
await _delay.AddToQueue(ConcurrencyConflict.ResolveWeakly.ToString(), package, streamName)
.ConfigureAwait(false);
// Todo: make 30 seconds configurable
var age = await _delay.Age(ConcurrencyConflict.ResolveWeakly.ToString(), streamName).ConfigureAwait(false);
if (!age.HasValue || age < TimeSpan.FromSeconds(30))
{
return;
}
var stream = sourced.Stream;
Logger.Write(LogLevel.Info,
() => $"Starting weak conflict resolve for stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]");
try
{
await _store.Freeze <T>(stream).ConfigureAwait(false);
var delayed = await _delay.Pull(streamName, max : 200).ConfigureAwait(false);
// If someone else pulled while we were waiting
if (!delayed.Any())
{
return;
}
Logger.Write(LogLevel.Info,
() => $"Resolving {delayed.Count()} uncommitted events to stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]");
var latestEvents =
await _eventstore.GetEvents(streamName, stream.CommitVersion + 1L)
.ConfigureAwait(false);
Logger.Write(LogLevel.Info,
() => $"Stream [{stream.StreamId}] bucket [{stream.Bucket}] is {latestEvents.Count()} events behind store");
sourced.Hydrate(latestEvents.Select(x => x.Event as IEvent));
Logger.Write(LogLevel.Debug, () => $"Merging {delayed.Count()} conflicted events");
try
{
foreach (var u in delayed.Select(x => x.Message as ConflictingEvents).SelectMany(x => x.Events))
{
sourced.Conflict(u.Event as IEvent,
metadata:
new Dictionary <string, string>
{
{ "ConflictResolution", ConcurrencyConflict.ResolveWeakly.ToString() }
});
}
}
catch (NoRouteException e)
{
Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}");
throw new ConflictResolutionFailedException("Failed to resolve conflict", e);
}
Logger.Write(LogLevel.Info, () => "Successfully merged conflicted events");
if (stream.StreamVersion != stream.CommitVersion && entity is ISnapshotting &&
((ISnapshotting)entity).ShouldTakeSnapshot())
{
Logger.Write(LogLevel.Debug,
() =>
$"Taking snapshot of [{typeof(T).FullName}] id [{entity.Id}] version {stream.StreamVersion}");
var memento = ((ISnapshotting)entity).TakeSnapshot();
stream.AddSnapshot(memento);
}
await _store.WriteStream <T>(commitId, stream, commitHeaders).ConfigureAwait(false);
}
finally
{
await _store.Unfreeze <T>(stream).ConfigureAwait(false);
}
}
