public async Task WhenMessageIsAppended()
{
var mapping = new EventMapping(EventMapping.DiscoverEventNamesInAssembly(typeof(RoadNetworkEvents).Assembly));
var settings = EventsJsonSerializerSettingsProvider.CreateSerializerSettings();
var archiveStream = new StreamName("archive-1");
var commandStream = new StreamName("road-network-commands");
var id = Guid.NewGuid();
var reaction = new ReactionScenarioBuilder()
.Given(new RecordedEvent(archiveStream, new Messages.UploadRoadNetworkChangesArchive {
ArchiveId = "123"
}).WithMessageId(id))
.Then(new RecordedEvent(commandStream, new Messages.UploadRoadNetworkChangesArchive {
ArchiveId = "123"
}).WithMessageId(id).WithMetadata(new { Position = 1 }))
.Build();
var sut = new Subscriber(_store, commandStream);
using (sut)
{
sut.Start();
//Act
foreach (var stream in reaction.Givens.GroupBy(given => given.Stream))
{
await _store.AppendToStream(
stream.Key,
ExpectedVersion.NoStream,
stream.Select((given, index) => new NewStreamMessage(
Deterministic.Create(Deterministic.Namespaces.Events,
$"{given.Stream}-{index}"),
mapping.GetEventName(given.Event.GetType()),
JsonConvert.SerializeObject(given.Event, settings),
given.Metadata != null ? JsonConvert.SerializeObject(given.Metadata, settings) : null
)).ToArray());
}
//Await
var page = await _store.ReadStreamForwards(commandStream, StreamVersion.Start, 1);
while (page.Status == PageReadStatus.StreamNotFound)
{
page = await _store.ReadStreamForwards(commandStream, StreamVersion.Start, 1);
}
//Assert
//Assert.Equal(_messageId, page.Messages[0].MessageId);
}
await sut.Disposed;
}
public async Task <T> Load <T>(AggregateId <T> aggregateId) where T : AggregateRoot
{
var streamId = GetStreamId(aggregateId);
IList <IDomainEvent> domainEvents = new List <IDomainEvent>();
ReadStreamPage readStreamPage;
do
{
readStreamPage = await _streamStore.ReadStreamForwards(streamId, StreamVersion.Start, maxCount : 100);
var messages = readStreamPage.Messages;
foreach (var streamMessage in messages)
{
Type type = DomainEventTypeMappings.Dictionary[streamMessage.Type];
var jsonData = await streamMessage.GetJsonData();
var domainEvent = JsonConvert.DeserializeObject(jsonData, type) as IDomainEvent;
domainEvents.Add(domainEvent);
}
} while (!readStreamPage.IsEnd);
if (!domainEvents.Any())
{
return(null);
}
var aggregate = (T)Activator.CreateInstance(typeof(T), true);
aggregate.Load(domainEvents);
return(aggregate);
}
public async Task <IEnumerable <StreamMessage> > GetAsync(string streamName)
{
var start = 0;
const int BatchSize = 500;
StreamMessagesPage messagesPage;
var events = new List <StreamMessage>();
do
{
messagesPage = await _streamStore.ReadStreamForwards(streamName, start, BatchSize);
if (messagesPage.Status == PageReadStatus.StreamNotFound)
{
throw new Exception("Stream not found");
}
events.AddRange(
messagesPage.Messages);
start = messagesPage.NextStreamVersion;
}while (!messagesPage.IsEndOfStream);
return(events);
}
public async Task <IEnumerable <Event> > GetAllAsync(StreamId streamId, CancellationToken cancellationToken)
{
var events = new List <Event>();
var loaded = false;
var readAllPage = await _streamStore.ReadStreamForwards(streamId, 0, 100, cancellationToken);
while (!loaded)
{
foreach (var message in readAllPage.Messages)
{
var json = await message.GetJsonData(cancellationToken);
var data = (Event)_eventSerializer.DeserializeEvent(message.Type, json);
data.ExpectedVersion = message.StreamVersion;
events.Add(data);
}
if (!readAllPage.IsEnd)
{
readAllPage = await readAllPage.ReadNext(cancellationToken);
}
else
{
loaded = true;
}
}
return(events);
}
private async Task <FastForwardResult> FastForward(IStreamStore from, IStreamStore to, string branchId)
{
var fastForwardResult = new FastForwardResult();
var page = await from.ListStreams();
while (page.StreamIds.Length > 0)
{
foreach (var s in page.StreamIds.Where(x => x.StartsWith(branchId)))
{
var localPosition = await from.LastPosition(s);
var remotePosition = await to.LastPosition(s);
if (localPosition == remotePosition)
{
continue;
}
var eventPage = await from.ReadStreamForwards(s, Math.Max(remotePosition + 1, 0), Configuration.BatchSize);
var appendMessages = new List <NewStreamMessage>();
if (eventPage.Messages.Length == 0 && localPosition >= ExpectedVersion.EmptyStream)
{
await to.AppendToStream(s, remotePosition, appendMessages.ToArray());
}
var metadata = await from.GetStreamMetadata(s);
await to.SetStreamMetadata(s, ExpectedVersion.Any, metadata.MaxAge, metadata.MaxCount, metadata.MetadataJson);
while (eventPage.Messages.Length > 0)
{
appendMessages.Clear();
foreach (var m in eventPage.Messages)
{
var payload = await m.GetJsonData();
var message = new NewStreamMessage(m.MessageId, m.Type, payload, m.JsonMetadata);
appendMessages.Add(message);
}
var result = await to.AppendToStream(s, remotePosition, appendMessages.ToArray());
fastForwardResult.NumberOfMessages += result.CurrentVersion - Math.Max(remotePosition, ExpectedVersion.EmptyStream);
eventPage = await eventPage.ReadNext();
}
fastForwardResult.NumberOfStreams++;
}
page = await page.Next();
}
fastForwardResult.ResultStatus = Status.Success;
return(fastForwardResult);
}
public async Task ShouldAddNewAggregate()
{
// Arrange
var aggregate = new TestAggregate();
// Act
await _sut.Save(aggregate);
// Assert
var streamId = _sut.GetStreamId(aggregate);
var streams = await _streamStore.ListStreams(Pattern.StartsWith(streamId));
streams.StreamIds.Length.ShouldBe(1);
var events = (await _streamStore.ReadStreamForwards(streamId, StreamVersion.Start, 1)).Messages;
(await Task.WhenAll(events.Select(@event => @event.ToEvent(CancellationToken.None))))
.All(@event => @event.GetType() == typeof(TestEvent)).ShouldBeTrue();
}
private async Task <bool> StreamExists(PublicServiceId publicServiceId)
{
var pageReadStatus = await _streamStore
.ReadStreamForwards(
streamId : publicServiceId.ToString(),
fromVersionInclusive : StreamVersion.Start,
maxCount : 1,
prefetchJsonData : false);
return(pageReadStatus.Status != PageReadStatus.StreamNotFound);
}
private async IAsyncEnumerable <object> ReadEvents(string streamId, CancellationToken cancellationToken)
{
var page = await _streamStore.ReadStreamForwards(streamId, 0, int.MaxValue, cancellationToken);
foreach (var message in page.Messages)
{
var type = typeof(DoSomething).Assembly.GetType(message.Type);
yield return(JsonSerializer.Deserialize(await message.GetJsonData(cancellationToken), type));
}
}
public async Task <RoadNetworkChangesArchive> Get(ArchiveId id, CancellationToken ct = default)
{
var stream = new StreamName(id);
if (_map.TryGet(stream, out var entry))
{
return((RoadNetworkChangesArchive)entry.Entity);
}
var page = await _store.ReadStreamForwards(stream, StreamVersion.Start, 1024, ct);
if (page.Status == PageReadStatus.StreamNotFound)
{
var network = RoadNetworkChangesArchive.Factory();
_map.Attach(new EventSourcedEntityMapEntry(network, stream, ExpectedVersion.NoStream));
return(network);
}
IEventSourcedEntity entity = RoadNetworkChangesArchive.Factory();
var messages = new List <object>(page.Messages.Length);
foreach (var message in page.Messages)
{
messages.Add(
JsonConvert.DeserializeObject(
await message.GetJsonData(ct),
_mapping.GetEventType(message.Type),
_settings));
}
entity.RestoreFromEvents(messages.ToArray());
while (!page.IsEnd)
{
messages.Clear();
page = await page.ReadNext(ct);
if (page.Status == PageReadStatus.StreamNotFound)
{
var network = RoadNetworkChangesArchive.Factory();
_map.Attach(new EventSourcedEntityMapEntry(network, stream, ExpectedVersion.NoStream));
return(network);
}
foreach (var message in page.Messages)
{
messages.Add(
JsonConvert.DeserializeObject(
await message.GetJsonData(ct),
_mapping.GetEventType(message.Type),
_settings));
}
entity.RestoreFromEvents(messages.ToArray());
}
_map.Attach(new EventSourcedEntityMapEntry(entity, stream, page.LastStreamVersion));
return((RoadNetworkChangesArchive)entity);
}
public async Task <Organization> TryGet(OrganizationId id, CancellationToken ct = default)
{
var stream = StreamNameFactory(id);
if (_map.TryGet(stream, out var entry))
{
return((Organization)entry.Entity);
}
var organization = Organization.Factory();
var page = await _store.ReadStreamForwards(stream, StreamVersion.Start, 100, ct);
if (page.Status == PageReadStatus.StreamNotFound)
{
return(null);
}
IEventSourcedEntity entity = organization;
var messages = new List <object>(page.Messages.Length);
foreach (var message in page.Messages)
{
messages.Add(
JsonConvert.DeserializeObject(
await message.GetJsonData(ct),
_mapping.GetEventType(message.Type),
_settings));
}
entity.RestoreFromEvents(messages.ToArray());
while (!page.IsEnd)
{
messages.Clear();
page = await page.ReadNext(ct);
if (page.Status == PageReadStatus.StreamNotFound)
{
return(null);
}
foreach (var message in page.Messages)
{
messages.Add(
JsonConvert.DeserializeObject(
await message.GetJsonData(ct),
_mapping.GetEventType(message.Type),
_settings));
}
entity.RestoreFromEvents(messages.ToArray());
}
_map.Attach(new EventSourcedEntityMapEntry(entity, stream, page.LastStreamVersion));
return(organization);
}
public async Task <TodoListState> Load(string aggregateId)
{
var stream = await _store.ReadStreamForwards(new StreamId(aggregateId), 0, int.MaxValue, true);
var aggregate = new TodoListState(aggregateId, stream.LastStreamVersion);
foreach (var message in stream.Messages)
{
var @event = await Deserialize(message);
aggregate.Apply(@event);
}
return(aggregate);
}
public async Task <IEnumerable <dynamic> > GetHistory(string persistenceId)
{
var history = new List <dynamic>();
var streamId = new StreamId(persistenceId);
var readStreamPage = await streamStore.ReadStreamForwards(streamId, global::SqlStreamStore.Streams.StreamVersion.Start, int.MaxValue);
foreach (var message in readStreamPage.Messages)
{
var @event = await DeserializeEvent(message);
history.Add(@event);
}
return(history);
}
protected override async Task <List <IDomainEvent> > LoadChangeLog(string streamId)
{
var changeLog = new List <IDomainEvent>();
var page = await _eventStore.ReadStreamForwards(streamId, 0, int.MaxValue);
foreach (var message in page.Messages)
{
var json = await message.GetJsonData();
var evt = (IDomainEvent)JsonConvert.DeserializeObject(json, SqlStreamStoreOutbox.SerializerSettings);
changeLog.Add(evt);
}
return(changeLog);
}
public IEnumerable LoadEventsFor(Guid aggregateId, int fromVersion)
{
var endOfStream = false;
var startVersion = fromVersion;
while (!endOfStream)
{
var stream = _streamStore.ReadStreamForwards(aggregateId.ToString(), startVersion, 10).GetAwaiter().GetResult();
endOfStream = stream.IsEnd;
startVersion = stream.NextStreamVersion;
foreach (var msg in stream.Messages)
{
yield return(JsonConvert.DeserializeObject(msg.GetJsonData().GetAwaiter().GetResult(), type: Type.GetType(msg.Type)));
}
}
}
public async Task <T> Get(Func <T> factory)
{
var instance = factory();
var streamId = new StreamId(instance.PersistenceId);
var readStreamPage = await streamStore.ReadStreamForwards(streamId, StreamVersion.Start, int.MaxValue);
foreach (var message in readStreamPage.Messages)
{
var eventJson = await message.GetJsonData();
var eventType = Type.GetType($"EventSourced.Simple.Aggregate.Events.{message.Type}, EventSourced.Simple");
var @event = JsonConvert.DeserializeObject(eventJson, eventType);
instance.OnRecover(@event);
}
return((T)instance);
}
public IEnumerable <JournalRecord> GetRecords(long fromRecord = 0)
{
var pageSize = 20;
while (true)
{
var page = _streamStore.ReadStreamForwards(
_streamId, (int)fromRecord, pageSize).Result;
foreach (var message in page.Messages)
{
yield return(RecordFromStreamMessage(message));
}
if (page.IsEnd)
{
break;
}
fromRecord += page.Messages.Length;
}
}
private async Task <List <StreamMessage> > ReadStreamMessages(string streamId, int fromVersionInclusive)
{
if (fromVersionInclusive < 0)
{
throw new InvalidOperationException("Cannot get version < 0");
}
var page = await _store.ReadStreamForwards(streamId, fromVersionInclusive, ReadPageSize);
var messages = new List <StreamMessage>(page.Messages);
while (!page.IsEnd)
{
page = await page.ReadNext();
messages.AddRange(page.Messages);
}
return(messages);
}
public async Task <IList <IDomainEvent> > GetEventsByStreamId(EventStreamId eventStreamId)
{
var streamMessages = new List <StreamMessage>();
ReadStreamPage currentStreamPage;
var nextSliceStart = 0;
do
{
currentStreamPage = await _streamStore.ReadStreamForwards(
streamId : new StreamId(eventStreamId.StreamName),
fromVersionInclusive : nextSliceStart,
maxCount : ReadBatchSize);
nextSliceStart = currentStreamPage.NextStreamVersion;
streamMessages.AddRange(currentStreamPage.Messages);
} while (!currentStreamPage.IsEnd);
return(streamMessages.Select(ConvertStreamMessageToDomainEvent).ToList());
}
protected async Task <TAggregate> Load <TAggregate>(string aggregateId, bool isReadOnly = false) where TAggregate : AggregateState, new()
{
var stream = await _store.ReadStreamForwards(new StreamId(aggregateId), 0, int.MaxValue, true);
var events = new List <object>();
foreach (var message in stream.Messages)
{
var @event = await Deserialize(message);
events.Add(@event);
}
//var events = await Task.WhenAll(stream.Messages.Select(Deserialize));
var aggregate = AggregateState.Build <TAggregate>(aggregateId, stream.LastStreamVersion, isReadOnly, events);
if (!isReadOnly)
{
_trackedAggregates.Add(aggregate);
}
return(aggregate);
}
public async IAsyncEnumerable <object> GetEventsFromStream(string streamId)
{
var stream = new StreamId(streamId);
ReadStreamPage page;
var readFrom = StreamVersion.Start;
do
{
page = await _streamStore.ReadStreamForwards(stream, readFrom, 1000);
foreach (var message in page.Messages)
{
var json = await message.GetJsonData();
yield return(Deserialize(TypeCache.GetType(message.Type), json));
}
readFrom = page.NextStreamVersion;
} while (!page.IsEnd);
}
public static async Task <int> DeletedCount(this IStreamStore streamStore, string key)
{
var deleted = 0;
var page = await streamStore.ReadStreamForwards("$deleted", 0, Configuration.BatchSize);
while (page.Messages.Length > 0)
{
foreach (var m in page.Messages)
{
var json = await m.GetJsonData();
if (json.Contains(key))
{
deleted++;
}
}
page = await page.ReadNext();
}
return(deleted);
}
private IObservable <ICommand> GetCommands(IStream stream)
{
var observable = Observable.Create(async(IObserver <ICommand> observer) =>
{
var page = await _streamStore.ReadStreamForwards(stream.Key, ExpectedVersion.EmptyStream + 1, Configuration.BatchSize);
while (page.Messages.Length > 0)
{
foreach (var m in page.Messages)
{
var data = await m.GetJsonData();
var command = _serializer.Deserialize(data);
observer.OnNext(command);
}
page = await page.ReadNext();
}
observer.OnCompleted();
});
return(observable);
}
public async Task <StreamEventsPage> ReadStream(
string streamName, long fromVersion, long maxCount,
StreamReadDirection direction = StreamReadDirection.Forward,
CancellationToken cancellationToken = default(CancellationToken))
{
var page = await _streamStore
.ReadStreamForwards(streamName, (int)fromVersion, (int)maxCount, cancellationToken)
.ConfigureAwait(false);
var events = await Task
.WhenAll(page.Messages.Select(streamMessage => _resolveStreamEvent(streamMessage, cancellationToken)))
.ConfigureAwait(false);
return(new StreamEventsPage(
streamName: page.StreamId,
fromVersion: page.FromStreamVersion,
nextVersion: page.NextStreamVersion,
lastVersion: page.LastStreamVersion,
isEndOfStream: page.IsEnd,
events: events,
direction: direction));
}
public async Task Transactions()
{
decimal balance = 0;
var endOfStream = false;
var startVersion = 0;
while (endOfStream == false)
{
var stream = await _streamStore.ReadStreamForwards(_streamId, startVersion, 10);
endOfStream = stream.IsEnd;
startVersion = stream.NextStreamVersion;
foreach (var msg in stream.Messages)
{
switch (msg.Type)
{
case "Deposited":
var depositedJson = await msg.GetJsonData();
var deposited = JsonConvert.DeserializeObject <Deposited>(depositedJson);
Console.WriteLine($"Deposited: {deposited.Amount:C} @ {deposited.DateTime} ({deposited.TransactionId})");
balance += deposited.Amount;
break;
case "Withdrawn":
var withdrawnJson = await msg.GetJsonData();
var withdrawn = JsonConvert.DeserializeObject <Withdrawn>(withdrawnJson);
Console.WriteLine($"Withdrawn: {withdrawn.Amount:C} @ {withdrawn.DateTime} ({withdrawn.TransactionId})");
balance -= withdrawn.Amount;
break;
}
}
}
Console.WriteLine($"Balance: {balance:C}");
}
public async Task <IActionResult> GetOutStreamPage(
Guid id,
int?start = null,
CancellationToken cancellationToken = default(CancellationToken))
{
var fromVersionInclusive = start ?? StreamVersion.Start;
var webHooks = await _webHooksRepository.Load(cancellationToken);
if (!webHooks.TryGet(id, out var w))
{
return(NotFound());
}
var streamId = w.StreamIds.OutStreamId;
var page = await _streamStore
.ReadStreamForwards(streamId, fromVersionInclusive, PageSize, true, cancellationToken);
if (page.Status == PageReadStatus.StreamNotFound)
{
return(new ObjectResult(new OutEventsPage
{
Next = StreamVersion.Start
}));
}
var items = page.Messages.Select(m => new OutEventItem
{
MessageId = m.MessageId,
CreatedUtc = m.CreatedUtc,
EventName = m.Type,
Sequence = m.StreamVersion
}).ToArray();
return(new ObjectResult(new OutEventsPage
{
Items = items,
Next = items.Last().Sequence
}));
}
public static async Task <Queue <StreamMessage> > LoadEvents(this IStreamStore eventStream, Guid id, int batchSize = 100)
{
var streamId = new StreamId(id.ToString());
var messages = new Queue <StreamMessage>();
bool haveMoreEvents;
var readFromVersion = 0;
do
{
var page = await eventStream.ReadStreamForwards(streamId, readFromVersion, batchSize);
foreach (var message in page.Messages)
{
messages.Enqueue(message);
}
haveMoreEvents = !page.IsEnd;
readFromVersion = page.NextStreamVersion;
} while (haveMoreEvents);
return(messages);
}
public Task <ReadStreamPage> Invoke(IStreamStore streamStore, CancellationToken ct)
=> ReadDirection == Constants.ReadDirection.Forwards
? streamStore.ReadStreamForwards(StreamId, _fromVersionInclusive, _maxCount, EmbedPayload, ct)
: streamStore.ReadStreamBackwards(StreamId, _fromVersionInclusive, _maxCount, EmbedPayload, ct);
public Task <ReadStreamPage> ReadStreamForwards(StreamId streamId, int fromVersionInclusive, int maxCount, bool prefetchJsonData = true,
CancellationToken cancellationToken = new CancellationToken())
{
return(_store.ReadStreamForwards(streamId, fromVersionInclusive, maxCount, prefetchJsonData, cancellationToken));
}
private async Task <bool> Validate(
IStreamStore from,
IStreamStore to,
string branchId,
bool failOnNoStream = false)
{
await _messageQueue.UncompletedMessagesOnBranch(branchId).FirstAsync(s => s == 0)
.Timeout(Configuration.Timeout);
var page = await from.ListStreams();
while (page.StreamIds.Length > 0)
{
foreach (var s in page.StreamIds.Where(x => !x.StartsWith("$")))
{
var source = await from.GetStream(s, _serializer);
if (source.Timeline != branchId)
{
continue;
}
var target = await to.GetStream(s, _serializer);
if (target == null) // stream exists but some of the ancestors do not
{
return(false);
}
if (failOnNoStream && target.Version == ExpectedVersion.NoStream)
{
_log.Warn($"Stream {s} does not exist on target, probably due to missing ancestors", this);
return(false);
}
if (source.Version <= target.Version && source.Version > ExpectedVersion.EmptyStream)
{
var fromMessage = (await from.ReadStreamBackwards(s, StreamVersion.End, 1)).Messages.SingleOrDefault();
var toMessage = (await to.ReadStreamForwards(s, source.ReadPosition(source.Version), 1)).Messages.SingleOrDefault();
if (fromMessage.MessageId == toMessage.MessageId)
{
return(true);
}
_log.Warn(
$"Message doesn't match with remote for stream {s}@{source.Version} : " +
$"{fromMessage.MessageId} != {toMessage.MessageId}", this);
return(false);
}
var parent = source.Parent;
while (parent != null && parent.Version > ExpectedVersion.EmptyStream)
{
if (!await Validate(from, to, parent.Timeline, true))
{
return(false);
}
parent = parent.Parent;
}
}
page = await page.Next();
}
return(true);
}
public async Task <RoadNetwork> Get(CancellationToken ct = default)
{
if (_map.TryGet(Stream, out var entry))
{
return((RoadNetwork)entry.Entity);
}
var roadNetwork = RoadNetwork.Factory();
var(snapshot, version) = await _snapshotReader.ReadSnapshot(ct);
if (version != ExpectedVersion.NoStream)
{
roadNetwork.RestoreFromSnapshot(snapshot);
version += 1;
}
else
{
version = StreamVersion.Start;
}
var page = await _store.ReadStreamForwards(Stream, version, StreamPageSize, ct);
if (page.Status == PageReadStatus.StreamNotFound)
{
var initial = RoadNetwork.Factory();
_map.Attach(new EventSourcedEntityMapEntry(initial, Stream, ExpectedVersion.NoStream));
return(initial);
}
IEventSourcedEntity entity = roadNetwork;
var messages = new List <object>(page.Messages.Length);
foreach (var message in page.Messages)
{
messages.Add(
JsonConvert.DeserializeObject(
await message.GetJsonData(ct),
_mapping.GetEventType(message.Type),
_settings));
}
entity.RestoreFromEvents(messages.ToArray());
while (!page.IsEnd)
{
messages.Clear();
page = await page.ReadNext(ct);
if (page.Status == PageReadStatus.StreamNotFound)
{
var initial = RoadNetwork.Factory();
_map.Attach(new EventSourcedEntityMapEntry(initial, Stream, ExpectedVersion.NoStream));
return(initial);
}
foreach (var message in page.Messages)
{
messages.Add(
JsonConvert.DeserializeObject(
await message.GetJsonData(ct),
_mapping.GetEventType(message.Type),
_settings));
}
entity.RestoreFromEvents(messages.ToArray());
}
_map.Attach(new EventSourcedEntityMapEntry(entity, Stream, page.LastStreamVersion));
return(roadNetwork);
}