public async Task <Guid> Resolve <T>(T entity, IEnumerable <IWritableEvent> uncommitted, Guid commitId, Guid startingEventId, IDictionary <string, string> commitHeaders) where T : class, IEventSource { // Store conflicting events in memory // After 100 or so pile up pull the latest stream and attempt to write them again foreach (var @event in uncommitted) { await _delay.AddToQueue(entity.StreamId, @event).ConfigureAwait(false); } // Todo: make 30 seconds configurable if (await _delay.Age(entity.StreamId).ConfigureAwait(false) < TimeSpan.FromSeconds(30)) { return(startingEventId); } var stream = entity.Stream; Logger.Write(LogLevel.Debug, () => $"Starting weak conflict resolve for stream [{stream.StreamId}] bucket [{stream.Bucket}]"); try { try { await _store.Freeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false); } catch (VersionException) { Logger.Write(LogLevel.Debug, () => $"Stopping weak conflict resolve - someone else is processing"); return(startingEventId); } uncommitted = (await _delay.Pull(entity.StreamId).ConfigureAwait(false)).Cast <IWritableEvent>(); Logger.Write(LogLevel.Info, () => $"Resolving {uncommitted.Count()} uncommitted events to stream [{stream.StreamId}] bucket [{stream.Bucket}]"); var latestEvents = await _store.GetEvents <T>(stream.Bucket, stream.StreamId, stream.CommitVersion + 1) .ConfigureAwait(false); Logger.Write(LogLevel.Debug, () => $"Stream is {latestEvents.Count()} events behind store"); var writableEvents = latestEvents as IWritableEvent[] ?? latestEvents.ToArray(); stream.Concat(writableEvents); entity.Hydrate(writableEvents.Select(x => x.Event as IEvent)); Logger.Write(LogLevel.Debug, () => "Merging conflicted events"); try { foreach (var u in uncommitted) { entity.Conflict(u.Event as IEvent); } } catch (NoRouteException e) { Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}"); throw new ConflictResolutionFailedException("Failed to resolve conflict", e); } Logger.Write(LogLevel.Debug, () => "Successfully merged conflicted events"); if (stream.StreamVersion != stream.CommitVersion && entity is ISnapshotting && ((ISnapshotting)entity).ShouldTakeSnapshot()) { Logger.Write(LogLevel.Debug, () => $"Taking snapshot of [{typeof(T).FullName}] id [{entity.StreamId}] version {stream.StreamVersion}"); var memento = ((ISnapshotting)entity).TakeSnapshot(); stream.AddSnapshot(memento, commitHeaders); } startingEventId = await stream.Commit(commitId, startingEventId, commitHeaders).ConfigureAwait(false); } finally { await _store.Unfreeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false); } return(startingEventId); }
protected override async Task Terminate(IInvokeHandlerContext context) { IDelayedChannel channel = null; try { channel = context.Builder.Build <IDelayedChannel>(); } // Catch in case IDelayedChannel isn't registered which shouldn't happen unless a user registered Consumer without GetEventStore catch (Exception) { } var msgType = context.MessageBeingHandled.GetType(); if (!msgType.IsInterface) { msgType = _mapper.GetMappedTypeFor(msgType) ?? msgType; } context.Extensions.Set(new State { ScopeWasPresent = Transaction.Current != null }); var messageHandler = context.MessageHandler; var channelKey = $"{messageHandler.HandlerType.FullName}:{msgType.FullName}"; bool contains = false; lock (Lock) contains = IsNotDelayed.Contains(channelKey); var contextChannelKey = ""; if (context.Headers.ContainsKey(Defaults.ChannelKey)) { contextChannelKey = context.Headers[Defaults.ChannelKey]; } // Special case for when we are bulk processing messages from DelayedSubscriber, simply process it and return dont check for more bulk if (channel == null || contains || contextChannelKey == channelKey) { Logger.Write(LogLevel.Debug, () => $"Invoking handle for message {msgType.FullName} on handler {messageHandler.HandlerType.FullName}"); await messageHandler.Invoke(context.MessageBeingHandled, context).ConfigureAwait(false); return; } // If we are bulk processing and the above check fails it means the current message handler shouldn't be called if (!string.IsNullOrEmpty(contextChannelKey)) { return; } if (IsDelayed.ContainsKey(channelKey)) { DelayedAttribute delayed; IsDelayed.TryGetValue(channelKey, out delayed); var msgPkg = new DelayedMessage { MessageId = context.MessageId, Headers = context.Headers, Message = context.MessageBeingHandled, Received = DateTime.UtcNow, ChannelKey = channelKey }; InvokesDelayed.Mark(); var specificKey = ""; if (delayed.KeyPropertyFunc != null) { try { specificKey = delayed.KeyPropertyFunc(context.MessageBeingHandled); } catch { Logger.Warn($"Failed to get key properties from message {msgType.FullName}"); } } await channel.AddToQueue(channelKey, msgPkg, key : specificKey).ConfigureAwait(false); bool bulkInvoked; if (context.Extensions.TryGet <bool>("BulkInvoked", out bulkInvoked) && bulkInvoked) { // Prevents a single message from triggering a dozen different bulk invokes Logger.Write(LogLevel.Debug, () => $"Limiting bulk processing for a single message to a single invoke"); return; } int? size = null; TimeSpan?age = null; if (delayed.Delay.HasValue) { age = await channel.Age(channelKey, key : specificKey).ConfigureAwait(false); } if (delayed.Count.HasValue) { size = await channel.Size(channelKey, key : specificKey).ConfigureAwait(false); } if (!ShouldExecute(delayed, size, age)) { Logger.Write(LogLevel.Debug, () => $"Threshold Count [{delayed.Count}] DelayMs [{delayed.Delay}] Size [{size}] Age [{age?.TotalMilliseconds}] - delaying processing channel [{channelKey}] specific [{specificKey}]"); return; } lock (RecentLock) { if (RecentlyInvoked.ContainsKey($"{channelKey}.{specificKey}")) { Logger.Write(LogLevel.Debug, () => $"Channel [{channelKey}] specific [{specificKey}] was checked by this instance recently - leaving it alone"); return; } RecentlyInvoked.Add($"{channelKey}.{specificKey}", DateTime.UtcNow.AddSeconds(10)); } context.Extensions.Set <bool>("BulkInvoked", true); Logger.Write(LogLevel.Info, () => $"Threshold Count [{delayed.Count}] DelayMs [{delayed.Delay}] Size [{size}] Age [{age?.TotalMilliseconds}] - bulk processing channel [{channelKey}] specific [{specificKey}]"); await InvokeDelayedChannel(channel, channelKey, specificKey, delayed, messageHandler, context).ConfigureAwait(false); return; } var attrs = Attribute.GetCustomAttributes(messageHandler.HandlerType, typeof(DelayedAttribute)) .Cast <DelayedAttribute>(); var single = attrs.SingleOrDefault(x => x.Type == msgType); if (single == null) { lock (Lock) IsNotDelayed.Add(channelKey); } else { Logger.Write(LogLevel.Debug, () => $"Found delayed handler {messageHandler.HandlerType.FullName} for message {msgType.FullName}"); IsDelayed.TryAdd(channelKey, single); } await Terminate(context).ConfigureAwait(false); }
public async Task Resolve <T>(T entity, IEnumerable <IWritableEvent> uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where T : class, IEventSource { // Todo fix delayed channel impl - currently doesn't do Age from store anymore throw new NotImplementedException("See todo - needs fix"); // Store conflicting events in memory // After 100 or so pile up pull the latest stream and attempt to write them again var streamName = _streamGen(typeof(T), StreamTypes.Domain, entity.Bucket, entity.StreamId); foreach (var @event in uncommitted) { await _delay.AddToQueue(streamName, @event).ConfigureAwait(false); } // Todo: make 30 seconds configurable // Todo: if a stream has a single conflict and never conflicts again, the event we delay will never be processed var age = await _delay.Age(streamName).ConfigureAwait(false); if (!age.HasValue || age < TimeSpan.FromSeconds(30)) { return; } var stream = entity.Stream; Logger.Write(LogLevel.Info, () => $"Starting weak conflict resolve for stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]"); try { try { await _store.Freeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false); } catch (VersionException) { Logger.Write(LogLevel.Info, () => $"Stopping weak conflict resolve - someone else is processing"); return; } uncommitted = (await _delay.Pull(streamName, max: _maxPulledDelayed).ConfigureAwait(false)).Cast <IWritableEvent>(); // If someone else pulled while we were waiting if (!uncommitted.Any()) { return; } Logger.Write(LogLevel.Info, () => $"Resolving {uncommitted.Count()} uncommitted events to stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]"); var latestEvents = await _store.GetEvents <T>(stream.Bucket, stream.StreamId, stream.CommitVersion + 1) .ConfigureAwait(false); Logger.Write(LogLevel.Info, () => $"Stream [{stream.StreamId}] bucket [{stream.Bucket}] is {latestEvents.Count()} events behind store"); var writableEvents = latestEvents as IWritableEvent[] ?? latestEvents.ToArray(); stream.Concat(writableEvents); entity.Hydrate(writableEvents.Select(x => x.Event as IEvent)); Logger.Write(LogLevel.Debug, () => "Merging conflicted events"); try { foreach (var u in uncommitted) { entity.Conflict(u.Event as IEvent, metadata: new Dictionary <string, string> { { "ConflictResolution", ConcurrencyConflict.ResolveWeakly.ToString() } }); } } catch (NoRouteException e) { Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}"); throw new ConflictResolutionFailedException("Failed to resolve conflict", e); } Logger.Write(LogLevel.Info, () => "Successfully merged conflicted events"); if (stream.StreamVersion != stream.CommitVersion && entity is ISnapshotting && ((ISnapshotting)entity).ShouldTakeSnapshot()) { Logger.Write(LogLevel.Debug, () => $"Taking snapshot of [{typeof(T).FullName}] id [{entity.StreamId}] version {stream.StreamVersion}"); var memento = ((ISnapshotting)entity).TakeSnapshot(); stream.AddSnapshot(memento); } await stream.Commit(commitId, commitHeaders).ConfigureAwait(false); } finally { await _store.Unfreeze <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false); } }
protected override async Task Terminate(IInvokeHandlerContext context) { if (context.Extensions.TryGet(out ActiveSagaInstance saga) && saga.NotFound && ((SagaMetadata)saga.GetType().GetProperty("Metadata", BindingFlags.NonPublic).GetValue(saga)).SagaType == context.MessageHandler.Instance.GetType()) { return; } IDelayedChannel channel = null; try { IContainer container; if (context.Extensions.TryGet <IContainer>(out container)) { channel = container.Resolve <IDelayedChannel>(); } } catch (Exception) { // Catch in case IDelayedChannel isn't registered which shouldn't happen unless a user registered Consumer without GetEventStore } var msgType = context.MessageBeingHandled.GetType(); if (!msgType.IsInterface) { msgType = _mapper.GetMappedTypeFor(msgType) ?? msgType; } context.Extensions.Set(new State { ScopeWasPresent = Transaction.Current != null }); var messageHandler = context.MessageHandler; var channelKey = $"{messageHandler.HandlerType.FullName}:{msgType.FullName}"; bool contains = false; lock (Lock) contains = IsNotDelayed.Contains(channelKey); context.Extensions.TryGet(Defaults.ChannelKey, out string contextChannelKey); // Special case for when we are bulk processing messages from DelayedSubscriber or BulkMessage, simply process it and return dont check for more bulk if (channel == null || contains || contextChannelKey == channelKey || context.Headers.ContainsKey(Defaults.BulkHeader)) { await messageHandler.Invoke(context.MessageBeingHandled, context).ConfigureAwait(false); return; } // If we are bulk processing and the above check fails it means the current message handler shouldn't be called // because if contextChannelKey is set we are using delayed bulk SendLocal // which should only execute the message on the handlers that were marked delayed, the other handlers were already invoked when the message // originally came in if (!string.IsNullOrEmpty(contextChannelKey)) { return; } if (IsDelayed.ContainsKey(channelKey)) { DelayedAttribute delayed; IsDelayed.TryGetValue(channelKey, out delayed); var specificKey = ""; if (delayed.KeyPropertyFunc != null) { try { specificKey = delayed.KeyPropertyFunc(context.MessageBeingHandled); } catch { Logger.Warn($"Failed to get key properties from message {msgType.FullName}"); } } var msgPkg = new DelayedMessage { MessageId = context.MessageId, Headers = context.Headers, Message = context.MessageBeingHandled as IMessage, Received = DateTime.UtcNow, ChannelKey = channelKey, SpecificKey = specificKey }; await channel.AddToQueue(channelKey, msgPkg, key : specificKey).ConfigureAwait(false); Logger.DebugEvent("QueueAdd", "Message added to delayed queue [{Channel:l}] key [{Key:l}]", channelKey, specificKey); bool bulkInvoked; if (context.Extensions.TryGet <bool>("BulkInvoked", out bulkInvoked) && bulkInvoked) { // Prevents a single message from triggering a dozen different bulk invokes return; } int? size = null; TimeSpan?age = null; if (delayed.Delay.HasValue) { age = await channel.Age(channelKey, key : specificKey).ConfigureAwait(false); } if (delayed.Count.HasValue) { size = await channel.Size(channelKey, key : specificKey).ConfigureAwait(false); } if (!ShouldExecute(delayed, size, age)) { return; } context.Extensions.Set <bool>("BulkInvoked", true); Logger.DebugEvent("Threshold", "Count [{Count}] DelayMs [{DelayMs}] bulk procesing Size [{Size}] Age [{Age}] - [{Channel:l}] key [{Key:l}]", delayed.Count, delayed.Delay, size, age?.TotalMilliseconds, channelKey, specificKey); await InvokeDelayedChannel(channel, channelKey, specificKey, delayed, messageHandler, context).ConfigureAwait(false); return; } var attrs = Attribute.GetCustomAttributes(messageHandler.HandlerType, typeof(DelayedAttribute)) .Cast <DelayedAttribute>(); var single = attrs.SingleOrDefault(x => x.Type == msgType); if (single == null) { lock (Lock) IsNotDelayed.Add(channelKey); } else { IsDelayed.TryAdd(channelKey, single); } await Terminate(context).ConfigureAwait(false); }
public async Task Resolve <T>(T entity, IEnumerable <IFullEvent> uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where T : class, IEventSource { var sourced = (IEventSourced)entity; // Store conflicting events in memory // After 100 or so pile up pull the latest stream and attempt to write them again var streamName = _streamGen(typeof(T), StreamTypes.Domain, sourced.Stream.Bucket, sourced.Stream.StreamId, sourced.Stream.Parents); var message = new ConflictingEvents { Bucket = sourced.Stream.Bucket, StreamId = sourced.Stream.StreamId, EntityType = typeof(T).AssemblyQualifiedName, Parents = BuildParentList(entity), Events = uncommitted }; var package = new DelayedMessage { MessageId = Guid.NewGuid().ToString(), Headers = new Dictionary <string, string>(), Message = message, Received = DateTime.UtcNow, ChannelKey = streamName }; foreach (var header in commitHeaders) { package.Headers[$"Conflict.{header.Key}"] = header.Value; } await _delay.AddToQueue(ConcurrencyConflict.ResolveWeakly.ToString(), package, streamName) .ConfigureAwait(false); // Todo: make 30 seconds configurable var age = await _delay.Age(ConcurrencyConflict.ResolveWeakly.ToString(), streamName).ConfigureAwait(false); if (!age.HasValue || age < TimeSpan.FromSeconds(30)) { return; } var stream = sourced.Stream; Logger.Write(LogLevel.Info, () => $"Starting weak conflict resolve for stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]"); try { await _store.Freeze <T>(stream).ConfigureAwait(false); var delayed = await _delay.Pull(streamName, max : 200).ConfigureAwait(false); // If someone else pulled while we were waiting if (!delayed.Any()) { return; } Logger.Write(LogLevel.Info, () => $"Resolving {delayed.Count()} uncommitted events to stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]"); var latestEvents = await _eventstore.GetEvents(streamName, stream.CommitVersion + 1L) .ConfigureAwait(false); Logger.Write(LogLevel.Info, () => $"Stream [{stream.StreamId}] bucket [{stream.Bucket}] is {latestEvents.Count()} events behind store"); sourced.Hydrate(latestEvents.Select(x => x.Event as IEvent)); Logger.Write(LogLevel.Debug, () => $"Merging {delayed.Count()} conflicted events"); try { foreach (var u in delayed.Select(x => x.Message as ConflictingEvents).SelectMany(x => x.Events)) { sourced.Conflict(u.Event as IEvent, metadata: new Dictionary <string, string> { { "ConflictResolution", ConcurrencyConflict.ResolveWeakly.ToString() } }); } } catch (NoRouteException e) { Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}"); throw new ConflictResolutionFailedException("Failed to resolve conflict", e); } Logger.Write(LogLevel.Info, () => "Successfully merged conflicted events"); if (stream.StreamVersion != stream.CommitVersion && entity is ISnapshotting && ((ISnapshotting)entity).ShouldTakeSnapshot()) { Logger.Write(LogLevel.Debug, () => $"Taking snapshot of [{typeof(T).FullName}] id [{entity.Id}] version {stream.StreamVersion}"); var memento = ((ISnapshotting)entity).TakeSnapshot(); stream.AddSnapshot(memento); } await _store.WriteStream <T>(commitId, stream, commitHeaders).ConfigureAwait(false); } finally { await _store.Unfreeze <T>(stream).ConfigureAwait(false); } }