public async Task Save( Foo streamDocument, CancellationToken cancellationToken = default(CancellationToken)) { var state = streamDocument.GetState(); var json = JsonConvert.SerializeObject(state, _serializerSettings); var newStreamMessage = new NewStreamMessage(Guid.NewGuid(), "FooDocState", json); var originalVersion = streamDocument.OriginalVersion; if (originalVersion < 0) { await _streamStore.AppendToStream( streamDocument.Id, ExpectedVersion.NoStream, newStreamMessage, cancellationToken); await _streamStore.SetStreamMetadata( streamDocument.Id, maxCount : _maxCount, cancellationToken : cancellationToken); } else { await _streamStore.AppendToStream( streamDocument.Id, originalVersion, newStreamMessage, cancellationToken); } }
public async Task <Guid> Deposit(decimal amount) { var trx = Guid.NewGuid(); var deposit = new Deposited(trx, amount, DateTime.UtcNow); await _streamStore.AppendToStream(_streamId, ExpectedVersion.Any, new NewStreamMessage(trx, "Deposited", JsonConvert.SerializeObject(deposit))); return(trx); }
private async Task <FastForwardResult> FastForward(IStreamStore from, IStreamStore to, string branchId) { var fastForwardResult = new FastForwardResult(); var page = await from.ListStreams(); while (page.StreamIds.Length > 0) { foreach (var s in page.StreamIds.Where(x => x.StartsWith(branchId))) { var localPosition = await from.LastPosition(s); var remotePosition = await to.LastPosition(s); if (localPosition == remotePosition) { continue; } var eventPage = await from.ReadStreamForwards(s, Math.Max(remotePosition + 1, 0), Configuration.BatchSize); var appendMessages = new List <NewStreamMessage>(); if (eventPage.Messages.Length == 0 && localPosition >= ExpectedVersion.EmptyStream) { await to.AppendToStream(s, remotePosition, appendMessages.ToArray()); } var metadata = await from.GetStreamMetadata(s); await to.SetStreamMetadata(s, ExpectedVersion.Any, metadata.MaxAge, metadata.MaxCount, metadata.MetadataJson); while (eventPage.Messages.Length > 0) { appendMessages.Clear(); foreach (var m in eventPage.Messages) { var payload = await m.GetJsonData(); var message = new NewStreamMessage(m.MessageId, m.Type, payload, m.JsonMetadata); appendMessages.Add(message); } var result = await to.AppendToStream(s, remotePosition, appendMessages.ToArray()); fastForwardResult.NumberOfMessages += result.CurrentVersion - Math.Max(remotePosition, ExpectedVersion.EmptyStream); eventPage = await eventPage.ReadNext(); } fastForwardResult.NumberOfStreams++; } page = await page.Next(); } fastForwardResult.ResultStatus = Status.Success; return(fastForwardResult); }
protected override void OnCommandBatch(IEnumerable <Command> commands) { var messages = commands.Select(ToNewStreamMessage).ToArray(); var result = _streamStore.AppendToStream(_streamId, ExpectedVersion.Any, messages) .GetAwaiter() .GetResult(); }
public async Task <StreamWriteResult> SaveEvents(EventStreamId eventStreamId, long streamVersion, List <IDomainEvent> events) { if (events.Any() == false) { return(new StreamWriteResult(-1)); } var commitId = Guid.NewGuid(); var expectedVersion = (int)streamVersion == 0 ? ExpectedVersion.NoStream : (int)streamVersion - 1; var streamMessagesToAppend = new List <NewStreamMessage>(); foreach (var domainEvent in events) { streamMessagesToAppend.Add(ToNewStreamMessage(commitId, domainEvent)); } try { var result = await _streamStore.AppendToStream( new StreamId(eventStreamId.StreamName), expectedVersion, streamMessagesToAppend.ToArray()); return(new StreamWriteResult(result.CurrentPosition)); } catch (WrongExpectedVersionException e) { Console.WriteLine(e); throw; } }
private static async Task GenerateMessages(IStreamStore store, int chunksCount, int chunkSize) { var dictStreamMessage = new Dictionary <string, NewStreamMessage>(); var random = new Random(); var messageJsonDataSize = 50 * 1024; for (int n = 0; n < chunksCount; n++) { var stopwatch = Stopwatch.StartNew(); for (int i = 0; i < chunkSize; i++) //generate chunk of messages { string jsonData = $"message-{n * chunksCount + i}" + new string('m', random.Next(messageJsonDataSize)); var message = new NewStreamMessage(Guid.NewGuid(), jsonData, "{}", $"{i}"); var streamId = $"streamId{random.Next(n * chunksCount + i)}"; dictStreamMessage[streamId] = message; } //await -in-parallel await dictStreamMessage.ForEachAsync(chunkSize, async kvp => { await store.AppendToStream(kvp.Key, ExpectedVersion.Any, kvp.Value); }, t => { //will be called only if t.IsFaulted var exception = t.Exception; var errorMessage = $"Task {t.Id} failed " + exception?.GetBaseException().Message; throw new Exception(errorMessage, exception); }); Console.WriteLine($"Chunk number {n} saved. Elapsed {stopwatch.Elapsed} "); } }
public async Task Bomb() { var current = await _streamStore.ReadHeadPosition(); var amountOfEvents = MaxAmountOfEvents - (current + 1); while (amountOfEvents > 0) { var newBatch = amountOfEvents > 2500 ? 2500: (int)amountOfEvents; var newEvents = Enumerable.Range(0, newBatch).Select((_, i) => new { Index = i, Message = new NewStreamMessage(Guid.NewGuid(), "RandomEvent", JsonSerializer.Serialize(new Event { Random = Rand.NextDouble() })) }).ToArray(); var tasks = newEvents .GroupBy(x => x.Index / 50) .Select(x => x.Select(y => y.Message).ToArray()) .Select(x => _streamStore.AppendToStream($"{StreamName}/{Guid.NewGuid()}", ExpectedVersion.Any, x)); await Task.WhenAll(tasks); amountOfEvents -= newBatch; Console.WriteLine($"Still need to add: {amountOfEvents} events"); } }
/// <inheritdoc /> public async Task AppendCommand(ICommand command) { if (command.EventType == null && !Debugger.IsAttached) { return; } var message = Encode(command); LogCommands(message); await _streamStore.AppendToStream(Key(command.EventType), ExpectedVersion.Any, message); // resolve the command if failed var holder = _failedCommandsSingleHolders.GetOrAdd(command.Timeline, b => new FailedCommandsSingleHolder()); var failedCommands = await holder.FailedCommands().FirstAsync(); if (failedCommands.Count == 0) { return; } await holder.UpdateState(b => { b.Timeline = command.Timeline; b.Commands.RemoveWhere(c => c.MessageId == message.MessageId); return(b); }); }
public async Task <long> Persist(string persistenceId, object @event) { var streamId = new StreamId(persistenceId); var message = new NewStreamMessage(Guid.NewGuid(), @event.GetType().Name, JsonConvert.SerializeObject(@event)); var appendResult = await streamStore.AppendToStream(streamId, ExpectedVersion.Any, message); return(appendResult.CurrentPosition); }
private static async Task AppendMessages(IStreamStore streamStore, string streamId, int numberOfEvents) { for (int i = 0; i < numberOfEvents; i++) { var newmessage = new NewStreamMessage(Guid.NewGuid(), "MyEvent", "{}"); await streamStore.AppendToStream(streamId, ExpectedVersion.Any, newmessage); } }
public async Task <int> Save(TodoListState aggregate) { var toAppend = aggregate.UncommittedEvents.Select(Serialize).ToArray(); var writeResult = await _store.AppendToStream(aggregate.Id, aggregate.LoadedRevision, toAppend); return(writeResult.CurrentVersion); }
public ValueTask Save(TBusinessTransaction transaction, CancellationToken cancellationToken = default) { var streamName = _getStreamName(transaction); var data = JsonSerializer.Serialize(transaction, _serializerOptions); return(new ValueTask(_streamStore.AppendToStream(streamName, transaction.Version ?? ExpectedVersion.NoStream, new NewStreamMessage(Guid.NewGuid(), typeof(TBusinessTransaction).Name, data), cancellationToken))); }
/// <summary> /// Appends a collection of messages to a stream. /// </summary> /// <remarks> /// Idempotency and concurrency handling is dependent on the choice of expected version and the messages /// to append. /// /// 1. When expectedVersion = ExpectedVersion.NoStream and the stream already exists and the collection of /// message IDs are not already in the stream, then <see cref="WrongExpectedVersionException"/> is /// throw. /// 2. When expectedVersion = ExpectedVersion.Any and the collection of messages IDs don't exist in the /// stream, then they are appended /// 3. When expectedVersion = ExpectedVersion.Any and the collection of messages IDs exist in the stream, /// then idempotency is applied and nothing happens. /// 4. When expectedVersion = ExpectedVersion.Any and of the collection of messages Ids some exist in the /// stream and some don't then a <see cref="WrongExpectedVersionException"/> will be throwm. /// 5. When expectedVersion is specified and the stream current version does not match the /// collection of message IDs are are checked against the stream in the correct position then the /// operation is considered idempotent. Otherwise a <see cref="WrongExpectedVersionException"/> will be /// throwm. /// </remarks> /// <param name="store"> /// The stream store instance. /// </param> /// <param name="streamId"> /// The Stream Id of the stream to append the messages. Must not start with a '$'. /// </param> /// <param name="expectedVersion"> /// The version of the stream that is expected. This is used to control concurrency and idempotency /// concerns. See <see cref="ExpectedVersion"/>. /// </param> /// <param name="message"> /// The messages to append. /// </param> /// <param name="cancellationToken"> /// The cancellation instruction. /// </param> /// <returns>A task representing the asynchronous operation.</returns> public static Task AppendToStream( this IStreamStore store, string streamId, int expectedVersion, NewStreamMessage message, CancellationToken cancellationToken = default(CancellationToken)) { return(store.AppendToStream(streamId, expectedVersion, new[] { message }, cancellationToken)); }
public async Task Save(WebHooks webHooks, CancellationToken cancellationToken) { var memento = webHooks.ToMemento(); var json = JsonConvert.SerializeObject(memento, WebHookPublisher.SerializerSettings); var newStreamMessage = new NewStreamMessage(Guid.NewGuid(), "WebHooksMemento", json); // TODO: are we interested in concurrency handling here? Should be low traffic... await _streamStore.AppendToStream(_name, ExpectedVersion.Any, newStreamMessage, cancellationToken); }
/// <summary> /// Appends a collection of messages to a stream. /// </summary> /// <remarks> /// Idempotency and concurrency handling is dependent on the choice of expected version and the messages /// to append. /// /// 1. When expectedVersion = ExpectedVersion.NoStream and the stream already exists and the collection of /// message IDs are not already in the stream, then <see cref="WrongExpectedVersionException"/> is /// throw. /// 2. When expectedVersion = ExpectedVersion.Any and the collection of messages IDs don't exist in the /// stream, then they are appended /// 3. When expectedVersion = ExpectedVersion.Any and the collection of messages IDs exist in the stream, /// then idempotency is applied and nothing happens. /// 4. When expectedVersion = ExpectedVersion.Any and of the collection of messages Ids some exist in the /// stream and some don't then a <see cref="WrongExpectedVersionException"/> will be throwm. /// 5. When expectedVersion is specified and the stream current version does not match the /// collection of message IDs are are checked against the stream in the correct position then the /// operation is considered idempotent. Otherwise a <see cref="WrongExpectedVersionException"/> will be /// throwm. /// </remarks> /// <param name="store"> /// The stream store instance. /// </param> /// <param name="streamId"> /// The Stream Id of the stream to append the messages. Must not start with a '$'. /// </param> /// <param name="expectedVersion"> /// The version of the stream that is expected. This is used to control concurrency and idempotency /// concerns. See <see cref="ExpectedVersion"/>. /// </param> /// <param name="messages"> /// The messages to append. /// </param> /// <param name="cancellationToken"> /// The cancellation instruction. /// </param> /// <returns>A task representing the asynchronous operation.</returns> public static Task AppendToStream( this IStreamStore store, string streamId, int expectedVersion, IEnumerable <NewStreamMessage> messages, CancellationToken cancellationToken = default(CancellationToken)) { return(store.AppendToStream(streamId, expectedVersion, messages.ToArray(), cancellationToken)); }
private static void Write(IStreamStore streamStore, int messageCount, int streamCount) { var streams = Enumerable.Range(0, streamCount).Select(_ => $"test-{Guid.NewGuid():n}").ToList(); Task.Run(() => Task.WhenAll( from streamId in streams select streamStore.AppendToStream( streamId, ExpectedVersion.NoStream, GenerateMessages(messageCount)))); }
public async Task Save() { foreach (var aggregateToSave in _aggregatesToSave) { await _streamStore.AppendToStream( GetStreamId(aggregateToSave.Aggregate), aggregateToSave.Aggregate.Version, aggregateToSave.Messages.ToArray()); } _aggregatesToSave.Clear(); }
public async Task WhenMessageIsAppended() { var mapping = new EventMapping(EventMapping.DiscoverEventNamesInAssembly(typeof(RoadNetworkEvents).Assembly)); var settings = EventsJsonSerializerSettingsProvider.CreateSerializerSettings(); var archiveStream = new StreamName("archive-1"); var commandStream = new StreamName("road-network-commands"); var id = Guid.NewGuid(); var reaction = new ReactionScenarioBuilder() .Given(new RecordedEvent(archiveStream, new Messages.UploadRoadNetworkChangesArchive { ArchiveId = "123" }).WithMessageId(id)) .Then(new RecordedEvent(commandStream, new Messages.UploadRoadNetworkChangesArchive { ArchiveId = "123" }).WithMessageId(id).WithMetadata(new { Position = 1 })) .Build(); var sut = new Subscriber(_store, commandStream); using (sut) { sut.Start(); //Act foreach (var stream in reaction.Givens.GroupBy(given => given.Stream)) { await _store.AppendToStream( stream.Key, ExpectedVersion.NoStream, stream.Select((given, index) => new NewStreamMessage( Deterministic.Create(Deterministic.Namespaces.Events, $"{given.Stream}-{index}"), mapping.GetEventName(given.Event.GetType()), JsonConvert.SerializeObject(given.Event, settings), given.Metadata != null ? JsonConvert.SerializeObject(given.Metadata, settings) : null )).ToArray()); } //Await var page = await _store.ReadStreamForwards(commandStream, StreamVersion.Start, 1); while (page.Status == PageReadStatus.StreamNotFound) { page = await _store.ReadStreamForwards(commandStream, StreamVersion.Start, 1); } //Assert //Assert.Equal(_messageId, page.Messages[0].MessageId); } await sut.Disposed; }
public async Task Save(T aggregate) { var uncomittedEvents = aggregate.UncomittedEvents; var streamId = new StreamId(aggregate.PersistenceId); foreach (var @event in uncomittedEvents) { var message = new NewStreamMessage(Guid.NewGuid(), @event.GetType().Name, JsonConvert.SerializeObject(@event)); var appendResult = await streamStore.AppendToStream(streamId, ExpectedVersion.Any, message); } aggregate.Commit(); }
public async Task ShouldGetAggregateFromEventStream() { // Arrange var aggregateId = Guid.NewGuid(); var events = new[] { new TestEvent(aggregateId) }; await _streamStore.AppendToStream(_sut.GetStreamId(new TestAggregate(events)), ExpectedVersion.NoStream, events.Select(@event => @event.ToMessageData()).ToArray(), CancellationToken.None); // Act var aggregate = await _sut.GetById <TestAggregate>(aggregateId).ConfigureAwait(false); // Assert aggregate.Id.ShouldBe(aggregateId); }
private static void Write(IStreamStore streamStore, string url, int messageCount, int streamCount) { var streams = Enumerable.Range(0, streamCount).Select(_ => $"test-{Guid.NewGuid():n}").ToList(); var streamIds = string.Join("\n", streams.Select(streamid => $"{url}streams/{streamid}")); Console.WriteLine("\nAbout to create the following streams: "); Console.WriteLine(streamIds); Task.Run(() => Task.WhenAll( from streamId in streams select streamStore.AppendToStream(streamId, ExpectedVersion.NoStream, GenerateMessages(messageCount)))); }
private static async Task RunWrites( CancellationToken ct, int numberOfMessagesPerAmend, int numberOfStreams, int offset, string jsonData, IStreamStore streamStore) { var stopwatch = Stopwatch.StartNew(); var messageNumbers = new int[numberOfMessagesPerAmend]; int count = 1; for (int i = 0; i < numberOfStreams; i++) { ct.ThrowIfCancellationRequested(); try { for (int j = 0; j < numberOfMessagesPerAmend; j++) { messageNumbers[j] = count++; } var newmessages = MessageFactory .CreateNewStreamMessages(jsonData, messageNumbers); await streamStore.AppendToStream( $"stream-{i + offset}", ExpectedVersion.Any, newmessages, ct); //Console.Write($"> {messageNumbers[numberOfMessagesPerAmend - 1]}"); } catch (SqlException ex) when(ex.Number == -2) { // just timeout } catch (Exception ex) when(!(ex is TaskCanceledException)) { Output.WriteLine(ex.ToString()); } } stopwatch.Stop(); var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0); Output.WriteLine(""); Output.WriteLine($"> {count - 1} messages written in {stopwatch.Elapsed} ({rate} m/s)"); }
private async Task AppendClockHasTicked( CancellationToken cancellationToken, StreamId clockStreamId, EventMapping eventMapping) { var clockHasTicked = new ClockHasTicked(_clockProvider.Now); await _streamStore.AppendToStream( streamId : clockStreamId, expectedVersion : ExpectedVersion.Any, message : new NewStreamMessage( messageId: Guid.NewGuid(), type: eventMapping.GetEventName(clockHasTicked.GetType()), jsonData: JsonConvert.SerializeObject(clockHasTicked)), cancellationToken : cancellationToken); }
public async Task Save(Appointment appointment) { foreach (var @event in appointment.PendingEvents) { if (!EventTypeMap.ContainsKey(@event.GetType())) { throw new InvalidOperationException($"Unrecognized event type: {@event.GetType().FullName}"); } await _eventStream.AppendToStream( appointment.Id.ToString(), ExpectedVersion.Any, new NewStreamMessage(@event.Id, EventTypeMap[@event.GetType()], @event.ToJson())); } await DispatchEvents(appointment); }
public async Task Test() { var streamId = new StreamId("test"); var @event = new Test { Id = 233, Name = "some name" }; var json = JsonSerializer.Serialize(@event); var newStreamMessage = new NewStreamMessage(Guid.NewGuid(), "TestType", json); await _streamStore.AppendToStream(streamId, ExpectedVersion.Any, newStreamMessage); }
public async Task <int> Append(IStreamStore streamStore, CancellationToken ct) { var numberOfStreams = Input.ReadInt("Number of streams: ", 1, 100000000); int messageJsonDataSize = Input.ReadInt("Size of Json (bytes): ", 1, 1024 * 1024); int numberOfMessagesPerAmend = Input.ReadInt("Number of messages per stream append: ", 1, 1000); string jsonData = new string('a', messageJsonDataSize); var stopwatch = Stopwatch.StartNew(); var messageNumbers = new int[numberOfMessagesPerAmend]; int count = 1; for (int i = 0; i < numberOfStreams; i++) { ct.ThrowIfCancellationRequested(); try { for (int j = 0; j < numberOfMessagesPerAmend; j++) { messageNumbers[j] = count++; } var newmessages = MessageFactory .CreateNewStreamMessages(jsonData, messageNumbers); await streamStore.AppendToStream( $"stream-{i}", ExpectedVersion.Any, newmessages, ct); Console.Write($"\r> {messageNumbers[numberOfMessagesPerAmend - 1]}"); } catch (Exception ex) when(!(ex is TaskCanceledException)) { Log.Logger.Error(ex, ex.Message); } } stopwatch.Stop(); var rate = Math.Round((decimal)count / stopwatch.ElapsedMilliseconds * 1000, 0); Output.WriteLine(""); Output.WriteLine($"> {count-1} messages written in {stopwatch.Elapsed} ({rate} m/s)"); return(count); }
public async Task AppendStream(string streamName, long expectedVersion, IEnumerable <StreamEvent> streamEvents, CancellationToken cancellationToken = default(CancellationToken)) { var events = await Task .WhenAll(streamEvents.Select(streamEvent => _createNewStreamMessage(streamEvent, cancellationToken))) .ConfigureAwait(false); try { await _streamStore .AppendToStream(streamName, (int)expectedVersion, events, cancellationToken) .ConfigureAwait(false); } catch (WrongExpectedVersionException ex) { throw new WrongExpectedStreamVersionException(ex.Message, ex.InnerException); } }
private static async Task RunLoadTest(CancellationTokenSource cts, IStreamStore streamStore) { var tasks = new List <Task>(); int count = 0; for (int i = 0; i < Environment.ProcessorCount; i++) { var random = new Random(); var task = Task.Run(async() => { while (!cts.IsCancellationRequested) { try { int streamNumber = random.Next(0, 100); var eventNumber1 = Interlocked.Increment(ref count); var eventNumber2 = Interlocked.Increment(ref count); var newmessages = StreamStoreAcceptanceTests .CreateNewStreamMessages(eventNumber1, eventNumber2); var info = $"{streamNumber} - {newmessages[0].MessageId}," + $"{newmessages[1].MessageId}"; Log.Logger.Information($"Begin {info}"); await streamStore.AppendToStream( $"stream-{streamNumber}", ExpectedVersion.Any, newmessages, cts.Token); Log.Logger.Information($"End {info}"); Console.Write($"\r{eventNumber2}"); } catch (Exception ex) when(!(ex is TaskCanceledException)) { cts.Cancel(); Log.Logger.Error(ex, ex.Message); Console.WriteLine(ex); Console.ReadKey(); } } }, cts.Token); tasks.Add(task); } await Task.WhenAll(tasks); }
public static ICommandHandlerBuilder <IRoadRegistryContext, TCommand> UseRoadRegistryContext <TCommand>( this ICommandHandlerBuilder <TCommand> builder, IStreamStore store, IRoadNetworkSnapshotReader snapshotReader, EventEnricher enricher) { if (store == null) { throw new ArgumentNullException(nameof(store)); } if (snapshotReader == null) { throw new ArgumentNullException(nameof(snapshotReader)); } if (enricher == null) { throw new ArgumentNullException(nameof(enricher)); } return(builder.Pipe <IRoadRegistryContext>(next => async(message, ct) => { var map = new EventSourcedEntityMap(); var context = new RoadRegistryContext(map, store, snapshotReader, SerializerSettings, EventMapping); await next(context, message, ct); foreach (var entry in map.Entries) { var events = entry.Entity.TakeEvents(); if (events.Length != 0) { var messageId = message.MessageId.ToString("N"); var version = entry.ExpectedVersion; Array.ForEach(events, @event => enricher(@event)); var messages = Array.ConvertAll( events, @event => new NewStreamMessage( Deterministic.Create(Deterministic.Namespaces.Events, $"{messageId}-{version++}"), EventMapping.GetEventName(@event.GetType()), JsonConvert.SerializeObject(@event, SerializerSettings) )); await store.AppendToStream(entry.Stream, entry.ExpectedVersion, messages, ct); } } } )); }
public async Task WriteAsync(IEnumerable <StreamEvent> events) { if (events == null) { throw new ArgumentNullException(nameof(events)); } var expectedVersions = new ConcurrentDictionary <StreamId, int>(); foreach (var batch in events.Batch(1000)) { foreach (var stream in batch.GroupBy(item => item.Stream, item => item.Event)) { if (!expectedVersions.TryGetValue(stream.Key, out var expectedVersion)) { expectedVersion = ExpectedVersion.NoStream; } var watch = Stopwatch.StartNew(); var appendResult = await _streamStore.AppendToStream( stream.Key, expectedVersion, stream .Select(@event => new NewStreamMessage( Deterministic.Create(Deterministic.Namespaces.Events, $"{stream.Key}-{expectedVersion++}"), Mapping.GetEventName(@event.GetType()), JsonConvert.SerializeObject(@event, SerializerSettings), JsonConvert.SerializeObject(new Dictionary <string, string> { { "$version", "0" } }, SerializerSettings) )) .ToArray() ); _logger.LogInformation("Append took {0}ms for stream {1}@{2}", watch.ElapsedMilliseconds, stream.Key, appendResult.CurrentVersion); expectedVersions[stream.Key] = appendResult.CurrentVersion; } } }