public async Task Freeze <T>(string bucket, string streamId) where T : class, IEventSource
{
var streamName = _streamGen(typeof(T), StreamTypes.Domain, bucket, streamId);
Logger.Write(LogLevel.Info, () => $"Freezing stream [{streamName}]");
try
{
await _store.WriteMetadata(streamName, frozen : true, owner : Defaults.Instance).ConfigureAwait(false);
}
catch (VersionException)
{
Logger.Write(LogLevel.Info, () => $"Freeze: stream [{streamName}] someone froze before us");
throw new FrozenException();
}
}
public async Task Write <T>(T poco, string bucket, string stream, IDictionary <string, string> commitHeaders)
{
var streamName = $"{_streamGen(typeof(T), bucket + ".POCO", stream)}";
Logger.Write(LogLevel.Debug, () => $"Writing poco to stream id [{streamName}]");
var descriptor = new EventDescriptor
{
EntityType = typeof(T).AssemblyQualifiedName,
Timestamp = DateTime.UtcNow,
Version = -1,
Headers = commitHeaders
};
var @event = new WritableEvent
{
Descriptor = descriptor,
Event = poco,
EventId = Guid.NewGuid()
};
Saved.Mark();
if (await _store.WriteEvents(streamName, new[] { @event }, commitHeaders).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 5).ConfigureAwait(false);
}
if (_shouldCache)
{
_cache.Cache(streamName, poco);
}
}
public async Task Publish <T>(string bucket, string streamId, IEnumerable <IWritableEvent> events, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = _streamGen(typeof(T), bucket + ".OOB", streamId);
var writableEvents = events as IWritableEvent[] ?? events.ToArray();
if (await _store.WriteEvents(streamName, writableEvents, commitHeaders).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 200000).ConfigureAwait(false);
}
}
public async Task WriteEvents <TEntity>(string bucket, Id streamId, Id[] parents, IFullEvent[] events, IDictionary <string, string> commitHeaders) where TEntity : IEntity
{
Logger.Write(LogLevel.Debug, $"Writing {events.Length} oob events stream [{streamId}] bucket [{bucket}]");
await events.WhenAllAsync(async @event =>
{
var message = new FullMessage
{
Message = @event.Event,
Headers = @event.Descriptor.Headers
};
var parentsStr = parents?.Any() ?? false ? parents.Aggregate <Id, string>("", (cur, next) => $"{cur},{next}") : "";
var headers = new Dictionary <string, string>()
{
[$"{Defaults.PrefixHeader}.EventId"] = @event.EventId.ToString(),
[$"{Defaults.PrefixHeader}.EntityType"] = @event.Descriptor.EntityType,
[$"{Defaults.PrefixHeader}.Timestamp"] = @event.Descriptor.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture),
[$"{Defaults.PrefixHeader}.Version"] = @event.Descriptor.Version.ToString(),
[$"{Defaults.PrefixHeader}.Bucket"] = bucket,
[$"{Defaults.PrefixHeader}.StreamId"] = streamId,
[$"{Defaults.PrefixHeader}.Parents"] = parentsStr
};
string id = "";
id = @event.Descriptor.Headers[Defaults.OobHeaderKey];
if (@event.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey) && bool.TryParse(@event.Descriptor.Headers[Defaults.OobTransientKey], out var transient) && !transient)
{
var stream = _generator(typeof(TEntity), StreamTypes.OOB, $"{id}.{bucket}", streamId, parents);
var version = await _store.WriteEvents(stream, new[] { @event }, headers).ConfigureAwait(false);
if (@event.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey) && int.TryParse(@event.Descriptor.Headers[Defaults.OobDaysToLiveKey], out var daysToLive) && daysToLive != -1)
{
var key = $"{bucket}.{id}.{streamId}.{parentsStr}";
// Uses the dictionary to keep track of daysToLive data its already saved.
// If an entity saves the same stream with a new daysToLive the stream metadata needs to be rewritten
if (!DaysToLiveKnowns.ContainsKey(key) || DaysToLiveKnowns[key] != daysToLive)
{
DaysToLiveKnowns[key] = daysToLive;
await _store.WriteMetadata(stream, maxAge: TimeSpan.FromDays(daysToLive)).ConfigureAwait(false);
}
}
}
else
{
await _dispatcher.Publish(message, headers).ConfigureAwait(false);
}
}).ConfigureAwait(false);
}
public async Task Write <T>(Tuple <long, T> poco, string bucket, Id streamId, IEnumerable <Id> parents, IDictionary <string, string> commitHeaders)
{
var streamName = _streamGen(typeof(T), StreamTypes.Poco, bucket, streamId, parents);
Logger.Write(LogLevel.Debug, () => $"Writing poco to stream id [{streamName}]");
var descriptor = new EventDescriptor
{
EntityType = typeof(T).AssemblyQualifiedName,
StreamType = StreamTypes.Poco,
Bucket = bucket,
StreamId = streamId,
Parents = parents,
Timestamp = DateTime.UtcNow,
// When reading version will be the stream position
Version = 0,
Headers = new Dictionary <string, string>(),
CommitHeaders = commitHeaders
};
var @event = new WritableEvent
{
Descriptor = descriptor,
Event = poco.Item2,
EventId = Guid.NewGuid()
};
Saved.Mark();
if (await _store.WriteEvents(streamName, new IFullEvent[] { @event }, commitHeaders, expectedVersion: poco.Item1).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 5).ConfigureAwait(false);
}
if (_shouldCache)
{
_cache.Cache(streamName, poco);
}
}
public async Task WriteSnapshots <T>(string bucket, string streamId, IEnumerable <ISnapshot> snapshots, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = $"{_streamGen(typeof(T), bucket + ".SNAP", streamId)}";
Logger.Write(LogLevel.Debug, () => $"Writing {snapshots.Count()} snapshots to stream [{streamName}]");
var translatedEvents = snapshots.Select(e =>
{
var descriptor = new EventDescriptor
{
EntityType = typeof(T).AssemblyQualifiedName,
Timestamp = e.Timestamp,
Version = e.Version,
Headers = commitHeaders
};
return(new WritableEvent
{
Descriptor = descriptor,
Event = e.Payload,
EventId = Guid.NewGuid()
});
}).ToList();
Saved.Mark();
if (await _store.WriteEvents(streamName, translatedEvents, commitHeaders).ConfigureAwait(false) == 1)
{
await _store.WriteMetadata(streamName, maxCount : 10).ConfigureAwait(false);
}
if (_shouldCache)
{
_cache.Cache(streamName, snapshots.Last());
}
}
public async Task WriteStream <T>(Guid commitId, IEventStream stream, IDictionary <string, string> commitHeaders) where T : class, IEventSource
{
var streamName = _streamGen(typeof(T), StreamTypes.Domain, stream.Bucket, stream.StreamId, stream.Parents);
Logger.Write(LogLevel.Debug,
() =>
$"Writing {stream.Uncommitted.Count()} events to stream {stream.StreamId} bucket {stream.Bucket} with commit id {commitId}");
if (await CheckFrozen <T>(stream.Bucket, stream.StreamId, stream.Parents).ConfigureAwait(false))
{
throw new FrozenException();
}
Saved.Mark();
var events = stream.Uncommitted.Select(writable =>
{
IMutating mutated = new Mutating(writable.Event, writable.Descriptor.Headers);
foreach (var mutate in _mutators)
{
Logger.Write(LogLevel.Debug,
() => $"Mutating outgoing event {writable.Event.GetType()} with mutator {mutate.GetType().FullName}");
mutated = mutate.MutateOutgoing(mutated);
}
// Todo: have some bool that is set true if they modified headers
if (_mutators.Any())
{
foreach (var header in mutated.Headers)
{
writable.Descriptor.Headers[header.Key] = header.Value;
}
}
return((IFullEvent) new WritableEvent
{
Descriptor = writable.Descriptor,
Event = mutated.Message,
EventId = UnitOfWork.NextEventId(commitId)
});
}).ToList();
var oobs = stream.Oobs.ToDictionary(x => x.Id, x => x);
foreach (var oob in stream.PendingOobs)
{
oobs[oob.Id] = oob;
}
var domainEvents = events.Where(x => x.Descriptor.StreamType == StreamTypes.Domain);
var oobEvents = events.Where(x => x.Descriptor.StreamType == StreamTypes.OOB);
if (domainEvents.Any())
{
_cache.Evict(streamName);
Logger.Write(LogLevel.Debug,
() => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] committing {domainEvents.Count()} events");
await _store.WriteEvents(streamName, domainEvents, commitHeaders, expectedVersion : stream.CommitVersion)
.ConfigureAwait(false);
}
if (stream.PendingOobs.Any())
{
await _store.WriteMetadata(streamName, custom : new Dictionary <string, string>
{
[OobMetadataKey] = JsonConvert.SerializeObject(oobs.Values)
}).ConfigureAwait(false);
}
if (stream.PendingSnapshot != null)
{
Logger.Write(LogLevel.Debug,
() => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] committing snapshot");
await _snapstore.WriteSnapshots <T>(stream.Bucket, stream.StreamId, stream.Parents, stream.StreamVersion, stream.PendingSnapshot, commitHeaders).ConfigureAwait(false);
}
if (oobEvents.Any())
{
Logger.Write(LogLevel.Debug,
() => $"Event stream [{stream.StreamId}] in bucket [{stream.Bucket}] publishing {oobEvents.Count()} out of band events");
foreach (var group in oobEvents.GroupBy(x => x.Descriptor.Headers[Defaults.OobHeaderKey]))
{
// OOB events of the same stream name don't need to all be written to the same stream
// if we parallelize the events into 10 known streams we can take advantage of internal
// ES optimizations and ES sharding
var vary = _random.Next(10) + 1;
var oobstream = $"{streamName}-{group.Key}.{vary}";
var definition = oobs[group.Key];
if (definition.Transient ?? false)
{
await _publisher.Publish <T>(oobstream, group, commitHeaders).ConfigureAwait(false);
}
else if (definition.DaysToLive.HasValue)
{
var version = await _store.WriteEvents(oobstream, group, commitHeaders).ConfigureAwait(false);
// if new stream, write metadata
if (version == (group.Count() - 1))
{
await _store.WriteMetadata(oobstream, maxAge : TimeSpan.FromDays(definition.DaysToLive.Value)).ConfigureAwait(false);
}
}
else
{
await _store.WriteEvents(oobstream, group, commitHeaders).ConfigureAwait(false);
}
}
}
}
public async Task WriteEvents <TEntity>(string bucket, Id streamId, Id[] parents, IFullEvent[] events, Guid commitId, IDictionary <string, string> commitHeaders) where TEntity : IEntity
{
Logger.DebugEvent("Write", "{Events} stream [{Stream:l}] bucket [{Bucket:l}]", events.Length, streamId, bucket);
var transients = new List <IFullMessage>();
var durables = new Dictionary <string, List <IFullEvent> >();
foreach (var @event in events)
{
var parentsStr = parents?.Any() ?? false?parents.Aggregate <Id, string>("", (cur, next) => $"{cur},{next}") : "";
var headers = new Dictionary <string, string>()
{
[$"{Defaults.PrefixHeader}.{Defaults.MessageIdHeader}"] = @event.EventId.ToString(),
[$"{Defaults.PrefixHeader}.{Defaults.CorrelationIdHeader}"] = commitId.ToString(),
[$"{Defaults.PrefixHeader}.EventId"] = @event.EventId.ToString(),
[$"{Defaults.PrefixHeader}.EntityType"] = @event.Descriptor.EntityType,
[$"{Defaults.PrefixHeader}.Timestamp"] = @event.Descriptor.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture),
[$"{Defaults.PrefixHeader}.Version"] = @event.Descriptor.Version.ToString(),
[$"{Defaults.PrefixHeader}.Bucket"] = bucket,
[$"{Defaults.PrefixHeader}.StreamId"] = streamId,
[$"{Defaults.PrefixHeader}.Parents"] = parentsStr
};
var id = "";
id = @event.Descriptor.Headers[Defaults.OobHeaderKey];
if (@event.Descriptor.Headers.ContainsKey(Defaults.OobTransientKey) &&
bool.TryParse(@event.Descriptor.Headers[Defaults.OobTransientKey], out var transient) && !transient)
{
var stream = _generator(_registrar.GetVersionedName(typeof(TEntity)), StreamTypes.OOB, $"{id}.{bucket}", streamId, parents);
if (!durables.ContainsKey(stream))
{
durables[stream] = new List <IFullEvent>();
}
durables[stream].Add(new FullEvent
{
EventId = @event.EventId,
Event = @event.Event,
Descriptor = new EventDescriptor
{
EventId = @event.Descriptor.EventId,
EntityType = @event.Descriptor.EntityType,
StreamType = @event.Descriptor.StreamType,
Bucket = @event.Descriptor.Bucket,
StreamId = @event.Descriptor.StreamId,
Parents = @event.Descriptor.Parents,
Compressed = @event.Descriptor.Compressed,
Version = @event.Descriptor.Version,
Timestamp = @event.Descriptor.Timestamp,
Headers = @event.Descriptor.Headers.Merge(headers),
CommitHeaders = @event.Descriptor.CommitHeaders.Merge(commitHeaders)
}
});
}
else
{
transients.Add(new FullMessage
{
Message = @event.Event,
Headers = @event.Descriptor.Headers.Merge(headers).Merge(commitHeaders)
});
}
}
await _dispatcher.Publish(transients.ToArray()).ConfigureAwait(false);
foreach (var stream in durables)
{
// Commit headers were already added to descriptor
await _store.WriteEvents(stream.Key, stream.Value.ToArray(), new Dictionary <string, string> {
}).ConfigureAwait(false);
// Update stream's maxAge if oob channel has a DaysToLive parameter
DaysToLiveKnowns.TryGetValue(stream.Key, out var daysToLiveKnown);
var sample = stream.Value.FirstOrDefault(x => x.Descriptor.Headers.ContainsKey(Defaults.OobDaysToLiveKey));
if (sample == null)
{
continue;
}
int.TryParse(sample.Descriptor.Headers[Defaults.OobDaysToLiveKey], out var daysToLive);
if (daysToLiveKnown != daysToLive)
{
DaysToLiveKnowns.AddOrUpdate(stream.Key, daysToLive, (key, val) => daysToLive);
await _store.WriteMetadata(stream.Key, maxAge : TimeSpan.FromDays(daysToLive)).ConfigureAwait(false);
}
}
}
