public async Task When_projections_are_cursors_then_catchup_does_not_replay_previously_seen_events()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
var eventsAggregated = new List <IDomainEvent>();
var catchup = StreamCatchup.All(streamSource.StreamPerAggregate(), batchSize: 100);
catchup.Subscribe(new BalanceProjector()
.Trace((p, es) => eventsAggregated.AddRange(es)), projectionStore);
await catchup.RunUntilCaughtUp();
var streamId = streamIds.First();
store.WriteEvents(streamId, 100m);
var cursor = await catchup.RunUntilCaughtUp();
cursor.Position
.Should()
.Be("101");
var balanceProjection = await projectionStore.Get(streamId);
balanceProjection.Balance.Should().Be(101);
balanceProjection.CursorPosition.Should().Be(2);
eventsAggregated.Count.Should().Be(101);
}
public async Task A_stream_can_be_derived_from_an_index_projection_in_order_to_perform_a_reduce_operation()
{
store.WriteEvents(i => new AccountOpened
{
AccountType = i % 2 == 0
? BankAccountType.Checking
: BankAccountType.Savings
}, howMany: 100);
var eventsByAggregate = streamSource.StreamPerAggregate()
.Trace()
.Map(ss => ss.Select(s => s.Trace()));
var indexCatchup = StreamCatchup.All(eventsByAggregate, batchSize: 1);
var index = new Projection <ConcurrentBag <AccountOpened>, string>
{
Value = new ConcurrentBag <AccountOpened>()
};
// subscribe a catchup to the updates stream to build up an index
indexCatchup.Subscribe(
Aggregator.Create <Projection <ConcurrentBag <AccountOpened> >, IDomainEvent>(async(p, events) =>
{
foreach (var e in events.OfType <AccountOpened>()
.Where(e => e.AccountType == BankAccountType.Savings))
{
p.Value.Add(e);
}
return(p);
}).Trace(),
async(streamId, aggregate) => await aggregate(index));
// create a catchup over the index
var savingsAccounts = Stream.Create <IDomainEvent, string>(
"Savings accounts",
async q => index.Value.SkipWhile(v => q.Cursor.HasReached(v.CheckpointToken))
.Take(q.BatchSize ?? 1000));
var savingsAccountsCatchup = StreamCatchup.Create(savingsAccounts);
var numberOfSavingsAccounts = new Projection <int, int>();
savingsAccountsCatchup.Subscribe <Projection <int, int>, IDomainEvent, string>(
manageProjection: async(streamId, aggregate) =>
{
numberOfSavingsAccounts = await aggregate(numberOfSavingsAccounts);
},
aggregate: async(c, es) =>
{
c.Value += es.Count;
return(c);
});
using (indexCatchup.Poll(TimeSpan.FromMilliseconds(100)))
using (savingsAccountsCatchup.Poll(TimeSpan.FromMilliseconds(50)))
{
await Wait.Until(() => numberOfSavingsAccounts.Value >= 50);
}
}
public async Task Add(string channel, string key, IDelayedMessage[] messages)
{
// If cache grows larger than 150% of max cache size, pause all processing until flush finished
while (Interlocked.CompareExchange(ref _tooLarge, 1, 1) == 1)
{
await Task.Delay(50).ConfigureAwait(false);
}
// Anything without a key bypasses memory cache
if (string.IsNullOrEmpty(key))
{
var translatedEvents = messages.Select(x => (IFullEvent) new FullEvent
{
Descriptor = new EventDescriptor
{
EntityType = "DELAY",
StreamType = $"{_endpoint}.{StreamTypes.Delayed}",
Bucket = Assembly.GetEntryAssembly()?.FullName ?? "UNKNOWN",
StreamId = channel,
Timestamp = DateTime.UtcNow,
Headers = new Dictionary <string, string>()
{
["Expired"] = "true",
["FlushTime"] = DateTime.UtcNow.ToString("s"),
["Instance"] = Defaults.Instance.ToString(),
["Machine"] = Environment.MachineName,
}
},
Event = x,
}).ToArray();
try
{
var streamName = _streamGen(typeof(DelayedCache),
$"{_endpoint}.{StreamTypes.Delayed}",
Assembly.GetEntryAssembly()?.FullName ?? "UNKNOWN", channel, new Id[] { });
await _store.WriteEvents(streamName, translatedEvents, null).ConfigureAwait(false);
return;
}
catch (Exception e)
{
Logger.Write(LogLevel.Warn,
() => $"Failed to write to channel [{channel}]. Exception: {e.GetType().Name}: {e.Message}");
}
}
addToMemCache(channel, key, messages);
}
public async Task WriteSnapshots <T>(string bucket, Id streamId, Id[] parents, long version, IState snapshot, IDictionary <string, string> commitHeaders) where T : IEntity
{
var streamName = _streamGen(typeof(T), StreamTypes.Snapshot, bucket, streamId, parents);
Logger.Write(LogLevel.Debug, () => $"Writing snapshot to stream [{streamName}]");
// We don't need snapshots to store the previous snapshot
// ideally this field would be [JsonIgnore] but we have no dependency on json.net
snapshot.Snapshot = null;
var e = new FullEvent
{
Descriptor = new EventDescriptor
{
EntityType = typeof(T).AssemblyQualifiedName,
StreamType = StreamTypes.Snapshot,
Bucket = bucket,
StreamId = streamId,
Parents = parents,
Timestamp = DateTime.UtcNow,
Version = version + 1,
Headers = new Dictionary <string, string>(),
CommitHeaders = commitHeaders
},
Event = snapshot,
};
await _store.WriteEvents(streamName, new[] { e }, commitHeaders).ConfigureAwait(false);
}
public async Task WriteSnapshots <T>(IState snapshot, IDictionary <string, string> commitHeaders) where T : IEntity
{
var streamName = _streamGen(_registrar.GetVersionedName(typeof(T)), StreamTypes.Snapshot, snapshot.Bucket, snapshot.Id, snapshot.Parents);
Logger.DebugEvent("Write", "[{Stream:l}]", streamName);
// We don't need snapshots to store the previous snapshot
// ideally this field would be [JsonIgnore] but we have no dependency on json.net
snapshot.Snapshot = null;
var e = new FullEvent
{
Descriptor = new EventDescriptor
{
EntityType = _registrar.GetVersionedName(typeof(T)),
StreamType = StreamTypes.Snapshot,
Bucket = snapshot.Bucket,
StreamId = snapshot.Id,
Parents = snapshot.Parents,
Timestamp = DateTime.UtcNow,
Version = snapshot.Version + 1,
Headers = new Dictionary <string, string>(),
CommitHeaders = commitHeaders
},
Event = snapshot,
};
await _store.WriteEvents(streamName, new[] { e }, commitHeaders).ConfigureAwait(false);
}
public async Task Write <T>(T poco, string bucket, string stream, IDictionary <string, string> commitHeaders)
{
var streamName = $"{_streamGen(typeof(T), bucket + ".POCO", stream)}";
Logger.Write(LogLevel.Debug, () => $"Writing poco to stream id [{streamName}]");
var descriptor = new EventDescriptor
{
EntityType = typeof(T).AssemblyQualifiedName,
Timestamp = DateTime.UtcNow,
Version = -1,
Headers = commitHeaders
};
var @event = new WritableEvent
{
Descriptor = descriptor,
Event = poco,
EventId = Guid.NewGuid()
};
Saved.Mark();
if (await _store.WriteEvents(streamName, new[] { @event }, commitHeaders).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 5).ConfigureAwait(false);
}
if (_shouldCache)
{
_cache.Cache(streamName, poco);
}
}
public async Task Resolve <TEntity, TState>(TEntity entity, IFullEvent[] uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity <TState> where TState : IState, new()
{
var state = entity.State;
Logger.Write(LogLevel.Info, () => $"Resolving {uncommitted.Count()} uncommitted events to stream [{entity.Id}] type [{typeof(TEntity).FullName}] bucket [{entity.Bucket}]");
var latestEvents =
await _eventstore.GetEvents <TEntity>(entity.Bucket, entity.Id, entity.Parents, entity.Version).ConfigureAwait(false);
Logger.Write(LogLevel.Info, () => $"Stream is {latestEvents.Count()} events behind store");
for (var i = 0; i < latestEvents.Length; i++)
{
state.Apply(latestEvents[i].Event as IEvent);
}
Logger.Write(LogLevel.Debug, () => "Merging conflicted events");
try
{
foreach (var u in uncommitted)
{
if (u.Descriptor.StreamType == StreamTypes.Domain)
{
entity.Conflict(u.Event as IEvent);
}
else if (u.Descriptor.StreamType == StreamTypes.OOB)
{
// Todo: small hack
string id = "";
bool transient = true;
int daysToLive = -1;
id = u.Descriptor.Headers[Defaults.OobHeaderKey];
if (u.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey))
{
bool.TryParse(u.Descriptor.Headers[Defaults.OobTransientKey], out transient);
}
if (u.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey))
{
int.TryParse(u.Descriptor.Headers[Defaults.OobDaysToLiveKey], out daysToLive);
}
entity.Raise(u.Event as IEvent, id, transient, daysToLive);
}
}
}
catch (NoRouteException e)
{
Logger.Write(LogLevel.Info, () => $"Failed to resolve conflict: {e.Message}");
throw new ConflictResolutionFailedException("Failed to resolve conflict", e);
}
Logger.Write(LogLevel.Debug, () => "Successfully merged conflicted events");
await _eventstore.WriteEvents <TEntity>(entity.Bucket, entity.Id, entity.Parents, entity.Uncommitted, commitHeaders).ConfigureAwait(false);
}
public async Task Publish <T>(string bucket, string streamId, IEnumerable <IWritableEvent> events, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = _streamGen(typeof(T), bucket + ".OOB", streamId);
var writableEvents = events as IWritableEvent[] ?? events.ToArray();
if (await _store.WriteEvents(streamName, writableEvents, commitHeaders).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 200000).ConfigureAwait(false);
}
}
public async Task When_one_projection_fails_its_cursor_is_not_advanced_while_other_projections_cursors_are_advanced()
{
var projections = new InMemoryProjectionStore <BalanceProjection>();
// first catch up all the projections
var stream = streamSource.StreamPerAggregate().Trace();
var catchup = StreamCatchup.All(stream);
var initialSubscription = catchup.Subscribe(new BalanceProjector(), projections);
await catchup.RunUntilCaughtUp();
initialSubscription.Dispose();
// write some additional events
var streamIdsWithoutErrors = streamIds.Take(5).ToList();
var streamIdsWithErrors = streamIds.Skip(5).Take(5).ToList();
foreach (var streamId in streamIdsWithoutErrors.Concat(streamIdsWithErrors))
{
store.WriteEvents(streamId, howMany: 10);
}
// subscribe a flaky projector
catchup.Subscribe(new BalanceProjector()
.Pipeline(async(projection, batch, next) =>
{
var aggregateId = batch.Select(i => i.AggregateId).First();
if (streamIdsWithErrors.Contains(aggregateId))
{
throw new Exception("oops");
}
await next(projection, batch);
}),
projections.AsHandler(),
e => e.Continue());
await catchup.RunSingleBatch();
var projectionsWithoutErrors = streamIdsWithoutErrors.Select(
id => projections.Get(id).Result);
var projectionsWithErrors = streamIdsWithErrors.Select(
id => projections.Get(id).Result);
foreach (var projection in projectionsWithoutErrors)
{
projection.CursorPosition.Should().Be(11);
projection.Balance.Should().Be(11);
}
foreach (var projection in projectionsWithErrors)
{
projection.CursorPosition.Should().Be(1);
projection.Balance.Should().Be(1);
}
}
public void SetUp()
{
// populate the event store
store = TestEventStore.Create();
streamId = Guid.NewGuid().ToString();
store.WriteEvents(streamId);
stream = NEventStoreStream.ByAggregate(store, streamId).DomainEvents();
}
public void SetUp()
{
// populate the event store
store = TestEventStore.Create();
streamId = Guid.NewGuid().ToString();
store.WriteEvents(streamId);
stream = NEventStoreStream.ByAggregate(store, streamId).DomainEvents();
}
public async Task Resolve <TEntity, TState>(TEntity entity, IFullEvent[] uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity <TState> where TState : IState, new()
{
var state = entity.State;
Logger.DebugEvent("Resolver", "Resolving {Events} conflicting events to stream [{Stream:l}] type [{EntityType:l}] bucket [{Bucket:l}]", uncommitted.Count(), entity.Id, typeof(TEntity).FullName, entity.Bucket);
var latestEvents =
await _eventstore.GetEvents <TEntity>(entity.Bucket, entity.Id, entity.Parents, entity.Version).ConfigureAwait(false);
Logger.DebugEvent("Behind", "Stream is {Count} events behind store", latestEvents.Count());
for (var i = 0; i < latestEvents.Length; i++)
{
state.Apply(latestEvents[i].Event as IEvent);
}
try
{
foreach (var u in uncommitted)
{
if (u.Descriptor.StreamType == StreamTypes.Domain)
{
entity.Conflict(u.Event as IEvent);
}
else if (u.Descriptor.StreamType == StreamTypes.OOB)
{
// Todo: small hack
string id = "";
bool transient = true;
int daysToLive = -1;
id = u.Descriptor.Headers[Defaults.OobHeaderKey];
if (u.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey))
{
bool.TryParse(u.Descriptor.Headers[Defaults.OobTransientKey], out transient);
}
if (u.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey))
{
int.TryParse(u.Descriptor.Headers[Defaults.OobDaysToLiveKey], out daysToLive);
}
entity.Raise(u.Event as IEvent, id, transient, daysToLive);
}
}
}
catch (NoRouteException e)
{
Logger.WarnEvent("ResolveFailure", e, "Failed to resolve conflict: {ExceptionType} - {ExceptionMessage}", e.GetType().Name, e.Message);
throw new ConflictResolutionFailedException("Failed to resolve conflict", e);
}
await _eventstore.WriteEvents <TEntity>(entity.Bucket, entity.Id, entity.Parents, entity.Uncommitted, commitHeaders).ConfigureAwait(false);
}
public static void WriteEvents(
this IStoreEvents store,
Func <int, string> streamId,
decimal amount = 1,
int howMany = 1,
string bucketId = null)
{
for (var i = 0; i < howMany; i++)
{
store.WriteEvents(streamId(i), amount, bucketId: bucketId);
}
}
public async Task WriteEvents <T>(string bucket, string streamId, int expectedVersion, IEnumerable <IWritableEvent> events, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = _streamGen(typeof(T), bucket, streamId);
if (await CheckFrozen <T>(bucket, streamId).ConfigureAwait(false))
{
throw new FrozenException();
}
Saved.Mark();
await _store.WriteEvents(streamName, events, commitHeaders, expectedVersion : expectedVersion).ConfigureAwait(false);
}
public async Task WriteStream <T>(IEventStream stream, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = _streamGen(typeof(T), StreamTypes.Domain, stream.Bucket, stream.StreamId);
if (await CheckFrozen <T>(stream.Bucket, stream.StreamId).ConfigureAwait(false))
{
throw new FrozenException();
}
Saved.Mark();
await _store.WriteEvents(streamName, stream.Uncommitted, commitHeaders, expectedVersion : stream.CommitVersion).ConfigureAwait(false);
}
public async Task Resolve <TEntity, TState>(TEntity entity, IFullEvent[] uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity <TState> where TState : IState, new()
{
var state = entity.State;
Logger.Write(LogLevel.Info, () => $"Resolving {uncommitted.Count()} uncommitted events to stream [{entity.Id}] type [{typeof(TEntity).FullName}] bucket [{entity.Bucket}]");
foreach (var u in uncommitted)
{
state.Apply(u.Event as IEvent);
}
await _store.WriteEvents <TEntity>(entity.Bucket, entity.Id, entity.Parents, uncommitted, commitHeaders).ConfigureAwait(false);
}
public async Task End(Exception ex = null)
{
Logger.Write(LogLevel.Debug, () => $"Saving {_inFlight.Count()} {(ex == null ? "ACKs" : "NACKs")}");
await Task.WhenAll(_inFlight.ToList().Select(x => ex == null ? Ack(x.Key) : NAck(x.Key)));
if (ex == null)
{
Logger.Write(LogLevel.Debug, () => $"Saving {_uncommitted.Count()} delayed streams");
await Task.WhenAll(
_uncommitted.GroupBy(x => x.Item1)
.Select(x => _store.WriteEvents(x.Key, x.Select(y => y.Item2), null)));
}
}
public async Task Resolve <TEntity, TState>(TEntity entity, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity <TState> where TState : class, IState, new()
{
var state = entity.State;
Logger.DebugEvent("Resolver", "Resolving {Events} conflicting events to stream [{Stream:l}] type [{EntityType:l}] bucket [{Bucket:l}]", entity.Uncommitted.Count(), entity.Id, typeof(TEntity).FullName, entity.Bucket);
var domainEvents = entity.Uncommitted.Where(x => x.Descriptor.StreamType == StreamTypes.Domain).ToArray();
var oobEvents = entity.Uncommitted.Where(x => x.Descriptor.StreamType == StreamTypes.OOB).ToArray();
await _store.WriteEvents <TEntity>(entity.Bucket, entity.Id, entity.GetParentIds(), domainEvents, commitHeaders).ConfigureAwait(false);
await _oobStore.WriteEvents <TEntity>(entity.Bucket, entity.Id, entity.GetParentIds(), oobEvents, commitId, commitHeaders).ConfigureAwait(false);
}
public async Task Resolve <TEntity, TState>(TEntity entity, IFullEvent[] uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity <TState> where TState : IState, new()
{
var state = entity.State;
Logger.DebugEvent("Resolver", "Resolving {Events} conflicting events to stream [{Stream:l}] type [{EntityType:l}] bucket [{Bucket:l}]", uncommitted.Count(), entity.Id, typeof(TEntity).FullName, entity.Bucket);
foreach (var u in uncommitted)
{
state.Apply(u.Event as IEvent);
}
await _store.WriteEvents <TEntity>(entity.Bucket, entity.Id, entity.Parents, uncommitted, commitHeaders).ConfigureAwait(false);
}
public async Task WriteEvents <TEntity>(string bucket, Id streamId, Id[] parents, IFullEvent[] events, IDictionary <string, string> commitHeaders) where TEntity : IEntity
{
Logger.Write(LogLevel.Debug, $"Writing {events.Length} oob events stream [{streamId}] bucket [{bucket}]");
await events.WhenAllAsync(async @event =>
{
var message = new FullMessage
{
Message = @event.Event,
Headers = @event.Descriptor.Headers
};
var parentsStr = parents?.Any() ?? false ? parents.Aggregate <Id, string>("", (cur, next) => $"{cur},{next}") : "";
var headers = new Dictionary <string, string>()
{
[$"{Defaults.PrefixHeader}.EventId"] = @event.EventId.ToString(),
[$"{Defaults.PrefixHeader}.EntityType"] = @event.Descriptor.EntityType,
[$"{Defaults.PrefixHeader}.Timestamp"] = @event.Descriptor.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture),
[$"{Defaults.PrefixHeader}.Version"] = @event.Descriptor.Version.ToString(),
[$"{Defaults.PrefixHeader}.Bucket"] = bucket,
[$"{Defaults.PrefixHeader}.StreamId"] = streamId,
[$"{Defaults.PrefixHeader}.Parents"] = parentsStr
};
string id = "";
id = @event.Descriptor.Headers[Defaults.OobHeaderKey];
if (@event.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey) && bool.TryParse(@event.Descriptor.Headers[Defaults.OobTransientKey], out var transient) && !transient)
{
var stream = _generator(typeof(TEntity), StreamTypes.OOB, $"{id}.{bucket}", streamId, parents);
var version = await _store.WriteEvents(stream, new[] { @event }, headers).ConfigureAwait(false);
if (@event.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey) && int.TryParse(@event.Descriptor.Headers[Defaults.OobDaysToLiveKey], out var daysToLive) && daysToLive != -1)
{
var key = $"{bucket}.{id}.{streamId}.{parentsStr}";
// Uses the dictionary to keep track of daysToLive data its already saved.
// If an entity saves the same stream with a new daysToLive the stream metadata needs to be rewritten
if (!DaysToLiveKnowns.ContainsKey(key) || DaysToLiveKnowns[key] != daysToLive)
{
DaysToLiveKnowns[key] = daysToLive;
await _store.WriteMetadata(stream, maxAge: TimeSpan.FromDays(daysToLive)).ConfigureAwait(false);
}
}
}
else
{
await _dispatcher.Publish(message, headers).ConfigureAwait(false);
}
}).ConfigureAwait(false);
}
public void SetUp()
{
// populate the event store
store = TestEventStore.Create();
streamIds = Enumerable.Range(1, 100)
.Select(_ => Guid.NewGuid().ToString())
.ToArray();
foreach (var streamId in streamIds)
{
store.WriteEvents(streamId, 1m);
}
streamSource = new NEventStoreStreamSource(store);
}
public void SetUp()
{
// populate the event store
store = TestEventStore.Create();
streamIds = Enumerable.Range(1, 100)
.Select(_ => Guid.NewGuid().ToString())
.ToArray();
foreach (var streamId in streamIds)
{
store.WriteEvents(streamId);
}
streamSource = new NEventStoreStreamSource(store);
}
public async Task Resolve <T>(T entity, IEnumerable <IWritableEvent> uncommitted, Guid commitId, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var stream = entity.Stream;
Logger.Write(LogLevel.Info, () => $"Resolving {uncommitted.Count()} uncommitted events to stream [{stream.StreamId}] type [{typeof(T).FullName}] bucket [{stream.Bucket}]");
foreach (var u in uncommitted)
{
entity.Apply(u.Event as IEvent, metadata: new Dictionary <string, string> {
{ "ConflictResolution", ConcurrencyConflict.Ignore.ToString() }
});
}
var streamName = _streamGen(typeof(T), StreamTypes.Domain, stream.Bucket, stream.StreamId);
await _store.WriteEvents(streamName, uncommitted, commitHeaders).ConfigureAwait(false);
stream.Flush(true);
}
public async Task A_partitioned_stream_can_be_mapped_at_query_time()
{
for (var i = 1; i <= 9; i++)
{
store.WriteEvents(
streamId: Guid.NewGuid().ToString(),
howMany: 10,
bucketId: i.ToString());
}
var partitionedStream = Stream.Partitioned <EventMessage, int, string>(async(q, p) =>
{
var bucketId = ((IStreamQueryValuePartition <string>)p).Value;
var streamsToSnapshot = store.Advanced.GetStreamsToSnapshot(bucketId, 0);
var streamId = streamsToSnapshot.Select(s => s.StreamId).Single();
var stream = NEventStoreStream.ByAggregate(store, streamId, bucketId);
var batch = await stream.CreateQuery(q.Cursor, q.BatchSize).NextBatch();
return(batch);
}).Trace();
var domainEvents = partitionedStream.Map(es => es.Select(e => e.Body).OfType <IDomainEvent>());
// catch up
var catchup = domainEvents.DistributeAmong(Enumerable.Range(1, 10)
.Select(i => Partition.ByValue(i.ToString())),
batchSize: 2);
var receivedEvents = new ConcurrentBag <IDomainEvent>();
catchup.Subscribe(async b =>
{
foreach (var e in b)
{
receivedEvents.Add(e);
}
});
await catchup.RunUntilCaughtUp().Timeout();
receivedEvents.Count().Should().Be(90);
}
public async Task Write <T>(Tuple <long, T> poco, string bucket, Id streamId, IEnumerable <Id> parents, IDictionary <string, string> commitHeaders)
{
var streamName = _streamGen(typeof(T), StreamTypes.Poco, bucket, streamId, parents);
Logger.Write(LogLevel.Debug, () => $"Writing poco to stream id [{streamName}]");
var descriptor = new EventDescriptor
{
EntityType = typeof(T).AssemblyQualifiedName,
StreamType = StreamTypes.Poco,
Bucket = bucket,
StreamId = streamId,
Parents = parents,
Timestamp = DateTime.UtcNow,
// When reading version will be the stream position
Version = 0,
Headers = new Dictionary <string, string>(),
CommitHeaders = commitHeaders
};
var @event = new WritableEvent
{
Descriptor = descriptor,
Event = poco.Item2,
EventId = Guid.NewGuid()
};
Saved.Mark();
if (await _store.WriteEvents(streamName, new IFullEvent[] { @event }, commitHeaders, expectedVersion: poco.Item1).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 5).ConfigureAwait(false);
}
if (_shouldCache)
{
_cache.Cache(streamName, poco);
}
}
public async Task <Guid> Resolve <T>(T entity, IEnumerable <IWritableEvent> uncommitted, Guid commitId, Guid startingEventId, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var stream = entity.Stream;
Logger.Write(LogLevel.Info, () => $"Resolving {uncommitted.Count()} uncommitted events to stream [{stream.StreamId}] bucket [{stream.Bucket}]");
foreach (var u in uncommitted)
{
if (!u.EventId.HasValue)
{
u.EventId = startingEventId;
startingEventId = startingEventId.Increment();
}
entity.Apply(u.Event as IEvent);
}
var streamName = _streamGen(typeof(T), stream.Bucket, stream.StreamId);
await _store.WriteEvents(streamName, uncommitted, commitHeaders).ConfigureAwait(false);
stream.Flush(true);
return(startingEventId);
}
public async Task WriteSnapshots <T>(string bucket, string streamId, IEnumerable <ISnapshot> snapshots, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = $"{_streamGen(typeof(T), bucket + ".SNAP", streamId)}";
Logger.Write(LogLevel.Debug, () => $"Writing {snapshots.Count()} snapshots to stream [{streamName}]");
var translatedEvents = snapshots.Select(e =>
{
var descriptor = new EventDescriptor
{
EntityType = typeof(T).AssemblyQualifiedName,
Timestamp = e.Timestamp,
Version = e.Version,
Headers = commitHeaders
};
return(new WritableEvent
{
Descriptor = descriptor,
Event = e.Payload,
EventId = Guid.NewGuid()
});
}).ToList();
Saved.Mark();
if (await _store.WriteEvents(streamName, translatedEvents, commitHeaders).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 10).ConfigureAwait(false);
}
if (_shouldCache)
{
_cache.Cache(streamName, snapshots.Last());
}
}
public async Task Catchup_can_traverse_all_events()
{
var projectionStore = new InMemoryProjectionStore <BalanceProjection>();
store.WriteEvents(streamId, 999);
var catchup = StreamCatchup.Create(stream);
catchup.Subscribe(new BalanceProjector(), projectionStore);
await catchup.RunSingleBatch();
projectionStore.Sum(b => b.Balance)
.Should()
.Be(1000);
projectionStore.Count()
.Should()
.Be(1);
}
public async Task WriteStream <T>(Guid commitId, IEventStream stream, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = _streamGen(typeof(T), StreamTypes.Domain, stream.Bucket, stream.StreamId, stream.Parents);
Logger.Write(LogLevel.Debug,
() =>
$"Writing {stream.Uncommitted.Count()} events to stream {stream.StreamId} bucket {stream.Bucket} with commit id {commitId}");
if (await CheckFrozen <T>(stream.Bucket, stream.StreamId, stream.Parents).ConfigureAwait(false))
{
throw new FrozenException();
}
Saved.Mark();
var events = stream.Uncommitted.Select(writable =>
{
IMutating mutated = new Mutating(writable.Event, writable.Descriptor.Headers);
foreach (var mutate in _mutators)
{
Logger.Write(LogLevel.Debug,
() => $"Mutating outgoing event {writable.Event.GetType()} with mutator {mutate.GetType().FullName}");
mutated = mutate.MutateOutgoing(mutated);
}
// Todo: have some bool that is set true if they modified headers
if (_mutators.Any())
{
foreach (var header in mutated.Headers)
{
writable.Descriptor.Headers[header.Key] = header.Value;
}
}
return((IFullEvent) new WritableEvent
{
Descriptor = writable.Descriptor,
Event = mutated.Message,
EventId = UnitOfWork.NextEventId(commitId)
});
}).ToList();
var oobs = stream.Oobs.ToDictionary(x => x.Id, x => x);
foreach (var oob in stream.PendingOobs)
{
oobs[oob.Id] = oob;
}
var domainEvents = events.Where(x => x.Descriptor.StreamType == StreamTypes.Domain);
var oobEvents = events.Where(x => x.Descriptor.StreamType == StreamTypes.OOB);
if (domainEvents.Any())
{
_cache.Evict(streamName);
Logger.Write(LogLevel.Debug,
() => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] committing {domainEvents.Count()} events");
await _store.WriteEvents(streamName, domainEvents, commitHeaders, expectedVersion : stream.CommitVersion)
.ConfigureAwait(false);
}
if (stream.PendingOobs.Any())
{
await _store.WriteMetadata(streamName, custom : new Dictionary <string, string>
{
[OobMetadataKey] = JsonConvert.SerializeObject(oobs.Values)
}).ConfigureAwait(false);
}
if (stream.PendingSnapshot != null)
{
Logger.Write(LogLevel.Debug,
() => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] committing snapshot");
await _snapstore.WriteSnapshots <T>(stream.Bucket, stream.StreamId, stream.Parents, stream.StreamVersion, stream.PendingSnapshot, commitHeaders).ConfigureAwait(false);
}
if (oobEvents.Any())
{
Logger.Write(LogLevel.Debug,
() => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] publishing {oobEvents.Count()} out of band events");
foreach (var group in oobEvents.GroupBy(x => x.Descriptor.Headers[Defaults.OobHeaderKey]))
{
// OOB events of the same stream name don't need to all be written to the same stream
// if we parallelize the events into 10 known streams we can take advantage of internal
// ES optimizations and ES sharding
var vary = _random.Next(10) + 1;
var oobstream = $"{streamName}-{group.Key}.{vary}";
var definition = oobs[group.Key];
if (definition.Transient ?? false)
{
await _publisher.Publish <T>(oobstream, group, commitHeaders).ConfigureAwait(false);
}
else if (definition.DaysToLive.HasValue)
{
var version = await _store.WriteEvents(oobstream, group, commitHeaders).ConfigureAwait(false);
// if new stream, write metadata
if (version == (group.Count() - 1))
{
await _store.WriteMetadata(oobstream, maxAge : TimeSpan.FromDays(definition.DaysToLive.Value)).ConfigureAwait(false);
}
}
else
{
await _store.WriteEvents(oobstream, group, commitHeaders).ConfigureAwait(false);
}
}
}
}
private async Task Flush()
{
var memCacheTotalSize = _memCache.Values.Sum(x => x.Count);
_metrics.Update("Delayed Cache Size", Unit.Message, memCacheTotalSize);
Logger.DebugEvent("Flush", "Cache Size: {CacheSize} Total Channels: {TotalChannels}", memCacheTotalSize, _memCache.Keys.Count);
var totalFlushed = 0;
// A list of channels who have expired or have more than 1/5 the max total cache size
var expiredSpecificChannels =
_memCache.Where(x => TimeSpan.FromTicks(_time.Now.Ticks - x.Value.Pulled) > _expiration)
.Select(x => x.Key).Take(Math.Max(1, _memCache.Keys.Count / 5))
.ToArray();
await expiredSpecificChannels.StartEachAsync(3, async expired =>
{
var messages = pullFromMemCache(expired.Channel, expired.Key, max: _flushSize);
if (!messages.Any())
{
return;
}
Logger.InfoEvent("ExpiredFlush", "{Flush} messages channel [{Channel:l}] key [{Key:l}]", messages.Length, expired.Channel, expired.Key);
var translatedEvents = messages.Select(x => (IFullEvent) new FullEvent
{
Descriptor = new EventDescriptor
{
EntityType = "DELAY",
StreamType = $"{_endpoint}.{StreamTypes.Delayed}",
Bucket = Assembly.GetEntryAssembly()?.FullName ?? "UNKNOWN",
StreamId = $"{expired.Channel}.{expired.Key}",
Timestamp = _time.Now,
Headers = new Dictionary <string, string>()
{
["Expired"] = "true",
["FlushTime"] = _time.Now.ToString("s"),
}
},
Event = x,
}).ToArray();
try
{
// Todo: might be a good idea to have a lock here so while writing to eventstore no new events can pile up
// Stream name to contain the channel, specific key, and the instance id
// it doesn't matter whats in the streamname, the category projection will queue it for execution anyway
// and a lot of writers to a single stream makes eventstore slow
var streamName = _streamGen(_registrar.GetVersionedName(typeof(DelayedCache)),
$"{_endpoint}.{StreamTypes.Delayed}", Assembly.GetEntryAssembly()?.FullName ?? "UNKNOWN",
$"{expired.Channel}.{expired.Key}", new Id[] { });
await _store.WriteEvents(streamName, translatedEvents, null).ConfigureAwait(false);
Interlocked.Add(ref totalFlushed, messages.Length);
Interlocked.Add(ref memCacheTotalSize, -messages.Length);
}
catch (Exception e)
{
Logger.WarnEvent("FlushFailure", e, "Channel [{Channel:l}] key [{Key:l}]: {ExceptionType} - {ExceptionMessage}", expired.Channel, expired.Key, e.GetType().Name, e.Message);
// Failed to write to ES - put object back in memcache
addToMemCache(expired.Channel, expired.Key, messages);
}
}).ConfigureAwait(false);
try
{
if (memCacheTotalSize > (_maxSize * 1.5))
{
Logger.WarnEvent("TooLarge", "Pausing message processing");
Interlocked.CompareExchange(ref _tooLarge, 1, 0);
}
var limit = 10;
while (memCacheTotalSize > _maxSize && limit > 0)
{
// Flush the largest channels
var toFlush = _memCache.Where(x => x.Value.Count > _flushSize || (x.Value.Count > (_maxSize / 5))).Select(x => x.Key).Take(Math.Max(1, _memCache.Keys.Count / 5)).ToArray();
// If no large channels, take some of the oldest
if (!toFlush.Any())
{
toFlush = _memCache.OrderBy(x => x.Value.Pulled).Select(x => x.Key).Take(Math.Max(1, _memCache.Keys.Count / 5)).ToArray();
}
await toFlush.StartEachAsync(3, async expired =>
{
var messages = pullFromMemCache(expired.Channel, expired.Key, max: _flushSize);
Logger.InfoEvent("LargeFlush", "{Flush} messages channel [{Channel:l}] key [{Key:l}]", messages.Length, expired.Channel, expired.Key);
var translatedEvents = messages.Select(x => (IFullEvent) new FullEvent
{
Descriptor = new EventDescriptor
{
EntityType = "DELAY",
StreamType = $"{_endpoint}.{StreamTypes.Delayed}",
Bucket = Assembly.GetEntryAssembly()?.FullName ?? "UNKNOWN",
StreamId = $"{expired.Channel}.{expired.Key}",
Timestamp = _time.Now,
Headers = new Dictionary <string, string>()
{
["Expired"] = "false",
["FlushTime"] = _time.Now.ToString("s"),
}
},
Event = x,
}).ToArray();
try
{
// Todo: might be a good idea to have a lock here so while writing to eventstore no new events can pile up
var streamName = _streamGen(_registrar.GetVersionedName(typeof(DelayedCache)),
$"{_endpoint}.{StreamTypes.Delayed}",
Assembly.GetEntryAssembly()?.FullName ?? "UNKNOWN",
$"{expired.Channel}.{expired.Key}", new Id[] { });
await _store.WriteEvents(streamName, translatedEvents, null).ConfigureAwait(false);
Interlocked.Add(ref totalFlushed, messages.Length);
Interlocked.Add(ref memCacheTotalSize, -messages.Length);
}
catch (Exception e)
{
limit--;
Logger.WarnEvent("FlushFailure", e, "Channel [{Channel:l}] key [{Key:l}]: {ExceptionType} - {ExceptionMessage}", expired.Channel, expired.Key, e.GetType().Name, e.Message);
// Failed to write to ES - put object back in memcache
addToMemCache(expired.Channel, expired.Key, messages);
}
}).ConfigureAwait(false);
}
}
catch (Exception e)
{
Logger.ErrorEvent("FlushException", e, "{ExceptionType} - {ExceptionMessage}", e.GetType().Name, e.Message);
}
finally
{
Interlocked.CompareExchange(ref _tooLarge, 0, 1);
}
Logger.DebugEvent("Flushed", "{Flushed} total", totalFlushed);
}
public async Task Commit(Guid commitId, IDictionary <String, String> commitHeaders)
{
Logger.DebugFormat("Event stream {0} commiting events", this.StreamId);
await this._children.Values.ForEachAsync(2, async (child) =>
{
Logger.DebugFormat("Event stream {0} commiting changes to child stream {1}", this.StreamId, child.StreamId);
await child.Commit(commitId, commitHeaders);
});
if (this._uncommitted.Count == 0)
{
ClearChanges();
return;
}
if (commitHeaders == null)
{
commitHeaders = new Dictionary <String, String>();
}
commitHeaders[CommitHeader] = commitId.ToString();
var oldCommits = Events.Select(x =>
{
String temp;
if (!x.Descriptor.Headers.TryGetValue(CommitHeader, out temp))
{
return(Guid.Empty);
}
return(Guid.Parse(temp));
});
if (oldCommits.Any(x => x == commitId))
{
throw new DuplicateCommitException($"Probable duplicate message handled - discarding commit id {commitId}");
}
Logger.DebugFormat("Event stream {0} committing {1} events", this.StreamId, _uncommitted.Count);
try
{
await _store.WriteEvents(this.Bucket, this.StreamId, this._streamVersion, _uncommitted, commitHeaders);
await _snapshots.WriteSnapshots(this.Bucket, this.StreamId, _pendingShots, commitHeaders);
ClearChanges();
}
catch (WrongExpectedVersionException e)
{
throw new VersionException($"Expected version {_streamVersion}", e);
}
catch (CannotEstablishConnectionException e)
{
throw new PersistenceException(e.Message, e);
}
catch (OperationTimedOutException e)
{
throw new PersistenceException(e.Message, e);
}
catch (EventStoreConnectionException e)
{
throw new PersistenceException(e.Message, e);
}
}
public async Task WriteEvents <TEntity>(string bucket, Id streamId, Id[] parents, IFullEvent[] events, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity
{
Logger.DebugEvent("Write", "{Events} stream [{Stream:l}] bucket [{Bucket:l}]", events.Length, streamId, bucket);
var transients = new List <IFullMessage>();
var durables = new Dictionary <string, List <IFullEvent> >();
foreach (var @event in events)
{
var parentsStr = parents?.Any() ?? false?parents.Aggregate <Id, string>("", (cur, next) => $"{cur},{next}") : "";
var headers = new Dictionary <string, string>()
{
[$"{Defaults.PrefixHeader}.{Defaults.MessageIdHeader}"] = @event.EventId.ToString(),
[$"{Defaults.PrefixHeader}.{Defaults.CorrelationIdHeader}"] = commitId.ToString(),
[$"{Defaults.PrefixHeader}.EventId"] = @event.EventId.ToString(),
[$"{Defaults.PrefixHeader}.EntityType"] = @event.Descriptor.EntityType,
[$"{Defaults.PrefixHeader}.Timestamp"] = @event.Descriptor.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture),
[$"{Defaults.PrefixHeader}.Version"] = @event.Descriptor.Version.ToString(),
[$"{Defaults.PrefixHeader}.Bucket"] = bucket,
[$"{Defaults.PrefixHeader}.StreamId"] = streamId,
[$"{Defaults.PrefixHeader}.Parents"] = parentsStr
};
var id = "";
id = @event.Descriptor.Headers[Defaults.OobHeaderKey];
if (@event.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey) &&
bool.TryParse(@event.Descriptor.Headers[Defaults.OobTransientKey], out var transient) && !transient)
{
var stream = _generator(_registrar.GetVersionedName(typeof(TEntity)), StreamTypes.OOB, $"{id}.{bucket}", streamId, parents);
if (!durables.ContainsKey(stream))
{
durables[stream] = new List <IFullEvent>();
}
durables[stream].Add(new FullEvent
{
EventId = @event.EventId,
Event = @event.Event,
Descriptor = new EventDescriptor
{
EventId = @event.Descriptor.EventId,
EntityType = @event.Descriptor.EntityType,
StreamType = @event.Descriptor.StreamType,
Bucket = @event.Descriptor.Bucket,
StreamId = @event.Descriptor.StreamId,
Parents = @event.Descriptor.Parents,
Compressed = @event.Descriptor.Compressed,
Version = @event.Descriptor.Version,
Timestamp = @event.Descriptor.Timestamp,
Headers = @event.Descriptor.Headers.Merge(headers),
CommitHeaders = @event.Descriptor.CommitHeaders.Merge(commitHeaders)
}
});
}
else
{
transients.Add(new FullMessage
{
Message = @event.Event,
Headers = @event.Descriptor.Headers.Merge(headers).Merge(commitHeaders)
});
}
}
await _dispatcher.Publish(transients.ToArray()).ConfigureAwait(false);
foreach (var stream in durables)
{
// Commit headers were already added to descriptor
await _store.WriteEvents(stream.Key, stream.Value.ToArray(), new Dictionary <string, string> {
}).ConfigureAwait(false);
// Update stream's maxAge if oob channel has a DaysToLive parameter
DaysToLiveKnowns.TryGetValue(stream.Key, out var daysToLiveKnown);
var sample = stream.Value.FirstOrDefault(x => x.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey));
if (sample == null)
{
continue;
}
int.TryParse(sample.Descriptor.Headers[Defaults.OobDaysToLiveKey], out var daysToLive);
if (daysToLiveKnown != daysToLive)
{
DaysToLiveKnowns.AddOrUpdate(stream.Key, daysToLive, (key, val) => daysToLive);
await _store.WriteMetadata(stream.Key, maxAge : TimeSpan.FromDays(daysToLive)).ConfigureAwait(false);
}
}
}
