public async Task Freeze <T>(string bucket, string streamId) where T : class, IEventSource { var streamName = _streamGen(typeof(T), StreamTypes.Domain, bucket, streamId); Logger.Write(LogLevel.Info, () => $"Freezing stream [{streamName}]"); try { await _store.WriteMetadata(streamName, frozen : true, owner : Defaults.Instance).ConfigureAwait(false); } catch (VersionException) { Logger.Write(LogLevel.Info, () => $"Freeze: stream [{streamName}] someone froze before us"); throw new FrozenException(); } }
public async Task Write <T>(T poco, string bucket, string stream, IDictionary <string, string> commitHeaders) { var streamName = $"{_streamGen(typeof(T), bucket + ".POCO", stream)}"; Logger.Write(LogLevel.Debug, () => $"Writing poco to stream id [{streamName}]"); var descriptor = new EventDescriptor { EntityType = typeof(T).AssemblyQualifiedName, Timestamp = DateTime.UtcNow, Version = -1, Headers = commitHeaders }; var @event = new WritableEvent { Descriptor = descriptor, Event = poco, EventId = Guid.NewGuid() }; Saved.Mark(); if (await _store.WriteEvents(streamName, new[] { @event }, commitHeaders).ConfigureAwait(false) == 1) { await _store.WriteMetadata(streamName, maxCount : 5).ConfigureAwait(false); } if (_shouldCache) { _cache.Cache(streamName, poco); } }
public async Task Publish <T>(string bucket, string streamId, IEnumerable <IWritableEvent> events, IDictionary <string, string> commitHeaders) where T : class, IEventSource { var streamName = _streamGen(typeof(T), bucket + ".OOB", streamId); var writableEvents = events as IWritableEvent[] ?? events.ToArray(); if (await _store.WriteEvents(streamName, writableEvents, commitHeaders).ConfigureAwait(false) == 1) { await _store.WriteMetadata(streamName, maxCount : 200000).ConfigureAwait(false); } }
public async Task WriteEvents <TEntity>(string bucket, Id streamId, Id[] parents, IFullEvent[] events, IDictionary <string, string> commitHeaders) where TEntity : IEntity { Logger.Write(LogLevel.Debug, $"Writing {events.Length} oob events stream [{streamId}] bucket [{bucket}]"); await events.WhenAllAsync(async @event => { var message = new FullMessage { Message = @event.Event, Headers = @event.Descriptor.Headers }; var parentsStr = parents?.Any() ?? false ? parents.Aggregate <Id, string>("", (cur, next) => $"{cur},{next}") : ""; var headers = new Dictionary <string, string>() { [$"{Defaults.PrefixHeader}.EventId"] = @event.EventId.ToString(), [$"{Defaults.PrefixHeader}.EntityType"] = @event.Descriptor.EntityType, [$"{Defaults.PrefixHeader}.Timestamp"] = @event.Descriptor.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture), [$"{Defaults.PrefixHeader}.Version"] = @event.Descriptor.Version.ToString(), [$"{Defaults.PrefixHeader}.Bucket"] = bucket, [$"{Defaults.PrefixHeader}.StreamId"] = streamId, [$"{Defaults.PrefixHeader}.Parents"] = parentsStr }; string id = ""; id = @event.Descriptor.Headers[Defaults.OobHeaderKey]; if (@event.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey) && bool.TryParse(@event.Descriptor.Headers[Defaults.OobTransientKey], out var transient) && !transient) { var stream = _generator(typeof(TEntity), StreamTypes.OOB, $"{id}.{bucket}", streamId, parents); var version = await _store.WriteEvents(stream, new[] { @event }, headers).ConfigureAwait(false); if (@event.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey) && int.TryParse(@event.Descriptor.Headers[Defaults.OobDaysToLiveKey], out var daysToLive) && daysToLive != -1) { var key = $"{bucket}.{id}.{streamId}.{parentsStr}"; // Uses the dictionary to keep track of daysToLive data its already saved. // If an entity saves the same stream with a new daysToLive the stream metadata needs to be rewritten if (!DaysToLiveKnowns.ContainsKey(key) || DaysToLiveKnowns[key] != daysToLive) { DaysToLiveKnowns[key] = daysToLive; await _store.WriteMetadata(stream, maxAge: TimeSpan.FromDays(daysToLive)).ConfigureAwait(false); } } } else { await _dispatcher.Publish(message, headers).ConfigureAwait(false); } }).ConfigureAwait(false); }
public async Task Write <T>(Tuple <long, T> poco, string bucket, Id streamId, IEnumerable <Id> parents, IDictionary <string, string> commitHeaders) { var streamName = _streamGen(typeof(T), StreamTypes.Poco, bucket, streamId, parents); Logger.Write(LogLevel.Debug, () => $"Writing poco to stream id [{streamName}]"); var descriptor = new EventDescriptor { EntityType = typeof(T).AssemblyQualifiedName, StreamType = StreamTypes.Poco, Bucket = bucket, StreamId = streamId, Parents = parents, Timestamp = DateTime.UtcNow, // When reading version will be the stream position Version = 0, Headers = new Dictionary <string, string>(), CommitHeaders = commitHeaders }; var @event = new WritableEvent { Descriptor = descriptor, Event = poco.Item2, EventId = Guid.NewGuid() }; Saved.Mark(); if (await _store.WriteEvents(streamName, new IFullEvent[] { @event }, commitHeaders, expectedVersion: poco.Item1).ConfigureAwait(false) == 1) { await _store.WriteMetadata(streamName, maxCount : 5).ConfigureAwait(false); } if (_shouldCache) { _cache.Cache(streamName, poco); } }
public async Task WriteSnapshots <T>(string bucket, string streamId, IEnumerable <ISnapshot> snapshots, IDictionary <string, string> commitHeaders) where T : class, IEventSource { var streamName = $"{_streamGen(typeof(T), bucket + ".SNAP", streamId)}"; Logger.Write(LogLevel.Debug, () => $"Writing {snapshots.Count()} snapshots to stream [{streamName}]"); var translatedEvents = snapshots.Select(e => { var descriptor = new EventDescriptor { EntityType = typeof(T).AssemblyQualifiedName, Timestamp = e.Timestamp, Version = e.Version, Headers = commitHeaders }; return(new WritableEvent { Descriptor = descriptor, Event = e.Payload, EventId = Guid.NewGuid() }); }).ToList(); Saved.Mark(); if (await _store.WriteEvents(streamName, translatedEvents, commitHeaders).ConfigureAwait(false) == 1) { await _store.WriteMetadata(streamName, maxCount : 10).ConfigureAwait(false); } if (_shouldCache) { _cache.Cache(streamName, snapshots.Last()); } }
public async Task WriteStream <T>(Guid commitId, IEventStream stream, IDictionary <string, string> commitHeaders) where T : class, IEventSource { var streamName = _streamGen(typeof(T), StreamTypes.Domain, stream.Bucket, stream.StreamId, stream.Parents); Logger.Write(LogLevel.Debug, () => $"Writing {stream.Uncommitted.Count()} events to stream {stream.StreamId} bucket {stream.Bucket} with commit id {commitId}"); if (await CheckFrozen <T>(stream.Bucket, stream.StreamId, stream.Parents).ConfigureAwait(false)) { throw new FrozenException(); } Saved.Mark(); var events = stream.Uncommitted.Select(writable => { IMutating mutated = new Mutating(writable.Event, writable.Descriptor.Headers); foreach (var mutate in _mutators) { Logger.Write(LogLevel.Debug, () => $"Mutating outgoing event {writable.Event.GetType()} with mutator {mutate.GetType().FullName}"); mutated = mutate.MutateOutgoing(mutated); } // Todo: have some bool that is set true if they modified headers if (_mutators.Any()) { foreach (var header in mutated.Headers) { writable.Descriptor.Headers[header.Key] = header.Value; } } return((IFullEvent) new WritableEvent { Descriptor = writable.Descriptor, Event = mutated.Message, EventId = UnitOfWork.NextEventId(commitId) }); }).ToList(); var oobs = stream.Oobs.ToDictionary(x => x.Id, x => x); foreach (var oob in stream.PendingOobs) { oobs[oob.Id] = oob; } var domainEvents = events.Where(x => x.Descriptor.StreamType == StreamTypes.Domain); var oobEvents = events.Where(x => x.Descriptor.StreamType == StreamTypes.OOB); if (domainEvents.Any()) { _cache.Evict(streamName); Logger.Write(LogLevel.Debug, () => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] committing {domainEvents.Count()} events"); await _store.WriteEvents(streamName, domainEvents, commitHeaders, expectedVersion : stream.CommitVersion) .ConfigureAwait(false); } if (stream.PendingOobs.Any()) { await _store.WriteMetadata(streamName, custom : new Dictionary <string, string> { [OobMetadataKey] = JsonConvert.SerializeObject(oobs.Values) }).ConfigureAwait(false); } if (stream.PendingSnapshot != null) { Logger.Write(LogLevel.Debug, () => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] committing snapshot"); await _snapstore.WriteSnapshots <T>(stream.Bucket, stream.StreamId, stream.Parents, stream.StreamVersion, stream.PendingSnapshot, commitHeaders).ConfigureAwait(false); } if (oobEvents.Any()) { Logger.Write(LogLevel.Debug, () => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] publishing {oobEvents.Count()} out of band events"); foreach (var group in oobEvents.GroupBy(x => x.Descriptor.Headers[Defaults.OobHeaderKey])) { // OOB events of the same stream name don't need to all be written to the same stream // if we parallelize the events into 10 known streams we can take advantage of internal // ES optimizations and ES sharding var vary = _random.Next(10) + 1; var oobstream = $"{streamName}-{group.Key}.{vary}"; var definition = oobs[group.Key]; if (definition.Transient ?? false) { await _publisher.Publish <T>(oobstream, group, commitHeaders).ConfigureAwait(false); } else if (definition.DaysToLive.HasValue) { var version = await _store.WriteEvents(oobstream, group, commitHeaders).ConfigureAwait(false); // if new stream, write metadata if (version == (group.Count() - 1)) { await _store.WriteMetadata(oobstream, maxAge : TimeSpan.FromDays(definition.DaysToLive.Value)).ConfigureAwait(false); } } else { await _store.WriteEvents(oobstream, group, commitHeaders).ConfigureAwait(false); } } } }
public async Task WriteEvents <TEntity>(string bucket, Id streamId, Id[] parents, IFullEvent[] events, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity { Logger.DebugEvent("Write", "{Events} stream [{Stream:l}] bucket [{Bucket:l}]", events.Length, streamId, bucket); var transients = new List <IFullMessage>(); var durables = new Dictionary <string, List <IFullEvent> >(); foreach (var @event in events) { var parentsStr = parents?.Any() ?? false?parents.Aggregate <Id, string>("", (cur, next) => $"{cur},{next}") : ""; var headers = new Dictionary <string, string>() { [$"{Defaults.PrefixHeader}.{Defaults.MessageIdHeader}"] = @event.EventId.ToString(), [$"{Defaults.PrefixHeader}.{Defaults.CorrelationIdHeader}"] = commitId.ToString(), [$"{Defaults.PrefixHeader}.EventId"] = @event.EventId.ToString(), [$"{Defaults.PrefixHeader}.EntityType"] = @event.Descriptor.EntityType, [$"{Defaults.PrefixHeader}.Timestamp"] = @event.Descriptor.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture), [$"{Defaults.PrefixHeader}.Version"] = @event.Descriptor.Version.ToString(), [$"{Defaults.PrefixHeader}.Bucket"] = bucket, [$"{Defaults.PrefixHeader}.StreamId"] = streamId, [$"{Defaults.PrefixHeader}.Parents"] = parentsStr }; var id = ""; id = @event.Descriptor.Headers[Defaults.OobHeaderKey]; if (@event.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey) && bool.TryParse(@event.Descriptor.Headers[Defaults.OobTransientKey], out var transient) && !transient) { var stream = _generator(_registrar.GetVersionedName(typeof(TEntity)), StreamTypes.OOB, $"{id}.{bucket}", streamId, parents); if (!durables.ContainsKey(stream)) { durables[stream] = new List <IFullEvent>(); } durables[stream].Add(new FullEvent { EventId = @event.EventId, Event = @event.Event, Descriptor = new EventDescriptor { EventId = @event.Descriptor.EventId, EntityType = @event.Descriptor.EntityType, StreamType = @event.Descriptor.StreamType, Bucket = @event.Descriptor.Bucket, StreamId = @event.Descriptor.StreamId, Parents = @event.Descriptor.Parents, Compressed = @event.Descriptor.Compressed, Version = @event.Descriptor.Version, Timestamp = @event.Descriptor.Timestamp, Headers = @event.Descriptor.Headers.Merge(headers), CommitHeaders = @event.Descriptor.CommitHeaders.Merge(commitHeaders) } }); } else { transients.Add(new FullMessage { Message = @event.Event, Headers = @event.Descriptor.Headers.Merge(headers).Merge(commitHeaders) }); } } await _dispatcher.Publish(transients.ToArray()).ConfigureAwait(false); foreach (var stream in durables) { // Commit headers were already added to descriptor await _store.WriteEvents(stream.Key, stream.Value.ToArray(), new Dictionary <string, string> { }).ConfigureAwait(false); // Update stream's maxAge if oob channel has a DaysToLive parameter DaysToLiveKnowns.TryGetValue(stream.Key, out var daysToLiveKnown); var sample = stream.Value.FirstOrDefault(x => x.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey)); if (sample == null) { continue; } int.TryParse(sample.Descriptor.Headers[Defaults.OobDaysToLiveKey], out var daysToLive); if (daysToLiveKnown != daysToLive) { DaysToLiveKnowns.AddOrUpdate(stream.Key, daysToLive, (key, val) => daysToLive); await _store.WriteMetadata(stream.Key, maxAge : TimeSpan.FromDays(daysToLive)).ConfigureAwait(false); } } }