public virtual bool AddSnapshot(ISnapshot snapshot) { if (snapshot == null) { return(false); } Logger.Debug(Messages.AddingSnapshot, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision); try { BsonDocument mongoSnapshot = snapshot.ToMongoSnapshot(_serializer); IMongoQuery query = Query.EQ(MongoShapshotFields.Id, mongoSnapshot[MongoShapshotFields.Id]); UpdateBuilder update = Update.Set(MongoShapshotFields.Payload, mongoSnapshot[MongoShapshotFields.Payload]); // Doing an upsert instead of an insert allows us to overwrite an existing snapshot and not get stuck with a // stream that needs to be snapshotted because the insert fails and the SnapshotRevision isn't being updated. PersistedSnapshots.Update(query, update, UpdateFlags.Upsert); // More commits could have been made between us deciding that a snapshot is required and writing it so just // resetting the Unsnapshotted count may be a little off. Adding snapshots should be a separate process so // this is a good chance to make sure the numbers are still in-sync - it only adds a 'read' after all ... BsonDocument streamHeadId = GetStreamHeadId(snapshot.BucketId, snapshot.StreamId); StreamHead streamHead = PersistedStreamHeads.FindOneById(streamHeadId).ToStreamHead(); int unsnapshotted = streamHead.HeadRevision - snapshot.StreamRevision; PersistedStreamHeads.Update( Query.EQ(MongoStreamHeadFields.Id, streamHeadId), Update.Set(MongoStreamHeadFields.SnapshotRevision, snapshot.StreamRevision).Set(MongoStreamHeadFields.Unsnapshotted, unsnapshotted)); return(true); } catch (Exception) { return(false); } }
//public static (IEnumerable<EventData> Events, long Version) LoadBulkEvents( // this DaprClient client, string storeName, // string streamName, long version, Dictionary<string, string> meta, StreamHead head) //{ // var events = client // .LoadAsyncBulkEventsAsync(storeName, streamName, version, meta, head) // .ToEnumerable(); // return (events, events.LastOrDefault()?.Version ?? head.Version); //} public static async IAsyncEnumerable <EventData> LoadAsyncBulkEventsAsync( this DaprClient client, string storeName, string streamName, long version, Dictionary <string, string> meta, StreamHead head, int chunkSize = 20) { var keys = Enumerable .Range(version == default ? 1 : (int)version, (int)(head.Version) + (version == default ? default : 1)) .Where(x => x <= head.Version) .Select(x => Naming.StreamKey(streamName, x)) .ToList(); if (!keys.Any()) { yield break; } foreach (var chunk in keys.BuildChunks(chunkSize)) { var events = (await client.GetBulkStateAsync(storeName, chunk.ToArray(), null, metadata: meta)) .Select(x => JsonSerializer.Deserialize <EventData>(x.Value)) .OrderBy(x => x.Version); foreach (var e in events) { yield return(e); } } }
private async ValueTask <StreamHead> UpdateStreamHeadSnapshotRevisionAsync(StreamHead streamHead, long snapshotRevision, CancellationToken cancellation) { var id = streamHead.Id; var bucketId = streamHead.BucketId; var streamId = streamHead.StreamId; StreamHead desired; while (streamHead.SnapshotRevision < snapshotRevision) { desired = new StreamHead(bucketId, streamId, streamHead.HeadRevision, snapshotRevision, streamHead.DispatchedRevision, streamHead.Version + 1); if (await _database.CompareExchangeAsync(desired, streamHead, (left, right) => left.Version == right.Version, cancellation)) { return(desired); } streamHead = await _database.GetOneAsync <StreamHead>(p => p.Id == id, cancellation); if (streamHead == null && (streamHead = await AddStreamHeadAsync(bucketId, streamId, cancellation)) == null) { return(null); } } return(streamHead); }
public virtual void Commit(Commit attempt) { ThrowWhenDisposed(); Logger.Debug(Resources.AttemptingToCommit, attempt.CommitId, attempt.StreamId, attempt.CommitSequence); lock (_commits) { if (_commits.Contains(attempt)) { throw new DuplicateCommitException(); } if (_commits.Any(c => c.StreamId == attempt.StreamId && c.StreamRevision == attempt.StreamRevision)) { throw new ConcurrencyException(); } _stamps[attempt.CommitId] = attempt.CommitStamp; _commits.Add(attempt); _undispatched.Add(attempt); StreamHead head = _heads.FirstOrDefault(x => x.StreamId == attempt.StreamId); _heads.Remove(head); Logger.Debug(Resources.UpdatingStreamHead, attempt.StreamId); int snapshotRevision = head == null ? 0 : head.SnapshotRevision; _heads.Add(new StreamHead(attempt.StreamId, attempt.StreamRevision, snapshotRevision)); } }
private ValueTask <StreamHead> AddStreamHeadAsync(string bucketId, string streamId, long headRevision, long snapshotRevision, long dispatchedRevision, CancellationToken cancellation) { var streamHead = new StreamHead(bucketId, streamId, headRevision, snapshotRevision, dispatchedRevision, version: 1); return(_database.GetOrAdd(streamHead, cancellation)); }
public static async Task StateTransactionSliceAsync(this DaprClient client, string storeName, string streamHeadKey, StreamHead head, string headetag, Dictionary <string, string> meta, EventData[] versionedEvents, string sliceKey, string sliceetag) { var sliceReq = new StateTransactionRequest(sliceKey, JsonSerializer.SerializeToUtf8Bytes(versionedEvents), Client.StateOperationType.Upsert, string.IsNullOrWhiteSpace(sliceetag) ? null : sliceetag, metadata: meta); var headReq = new StateTransactionRequest(streamHeadKey, JsonSerializer.SerializeToUtf8Bytes(head), Client.StateOperationType.Upsert, etag: string.IsNullOrWhiteSpace(headetag) ? null : headetag, metadata: meta); var reqs = new List <StateTransactionRequest> { sliceReq, headReq }; await client.ExecuteStateTransactionAsync(storeName, reqs, meta); }
public async Task <long> AppendToStreamAsync(string streamName, Action <StreamHead> concurrencyGuard, params EventData[] events) { var streamKey = $"{streamName}|head"; var(head, headetag) = await client.GetStateAndETagAsync <StreamHead>(StoreName, streamKey); if (head == null) { head = new StreamHead(); } if (!events.Any()) { return(head.Version); } concurrencyGuard(head); var newVersion = head.Version + events.Length; var versionedEvents = events .Select((e, i) => new EventData { EventId = e.EventId, EventName = e.EventName, Data = e.Data, Version = head.Version + (i + 1) }) .ToArray(); var sliceKey = $"{streamName}|{newVersion}"; var(slice, sliceetag) = await client.GetStateAndETagAsync <EventData[]>(StoreName, sliceKey); if (slice != null) { throw new DBConcurrencyException($"Event slice {sliceKey} ending with event version {newVersion} already exists"); } var sliceWriteSuccess = await client.TrySaveStateAsync(StoreName, sliceKey, versionedEvents, sliceetag); if (!sliceWriteSuccess) { throw new DBConcurrencyException($"Error writing events. Event slice {sliceKey} ending with event version {newVersion} already exists"); } head.Version = newVersion; var headWriteSuccess = await client.TrySaveStateAsync(StoreName, streamKey, head, headetag); if (!headWriteSuccess) { throw new DBConcurrencyException($"stream head {streamKey} have been updated"); } return(newVersion); }
public bool AddSnapshot(Snapshot snapshot) { lock (_commits) { StreamHead currentHead = _heads.FirstOrDefault(h => h.StreamId == snapshot.StreamId); if (currentHead == null) { return(false); } _snapshots.Add(snapshot); _heads.Remove(currentHead); _heads.Add(new StreamHead(currentHead.BucketId, currentHead.StreamId, currentHead.HeadRevision, snapshot.StreamRevision)); } return(true); }
public async Task <long> AppendToStreamAsync(string streamName, Action <StreamHead> concurrencyGuard, params EventData[] events) { var streamHeadKey = Naming.StreamHead(streamName); var meta = MetaProvider(streamName); var(head, headetag) = await client.GetStateAndETagAsync <StreamHead>(StoreName, streamHeadKey, metadata : meta); if (head == null) { head = new StreamHead(); } if (!events.Any()) { return(head.Version); } concurrencyGuard(head); var newVersion = head.Version + events.Length; var versionedEvents = events .Select((e, i) => new EventData(e.EventId, e.EventName, e.Data, head.Version + (i + 1))) .ToArray(); var sliceKey = Naming.StreamKey(streamName, newVersion); var(slice, sliceetag) = await client.GetStateAndETagAsync <EventData[]>(StoreName, sliceKey, metadata : meta); if (slice != null) { throw new DBConcurrencyException($"Event slice {sliceKey} ending with event version {newVersion} already exists"); } head = new StreamHead(newVersion); var task = Mode switch { SliceMode.Off => client.StateTransactionAsync(StoreName, streamName, streamHeadKey, head, headetag, meta, versionedEvents), SliceMode.Transactional => client.StateTransactionSliceAsync(StoreName, streamHeadKey, head, headetag, meta, versionedEvents, sliceKey, sliceetag), SliceMode.TwoPhased => client.TwoPhasedAsync(StoreName, streamHeadKey, head, headetag, meta, newVersion, versionedEvents, sliceKey, sliceetag), _ => throw new Exception("Mode not supported") }; await task; return(newVersion); }
private async ValueTask <StreamHead> UpdateStreamHeadDispatchedRevisionAsync(StreamHead streamHead, long dispatchedRevision, CancellationToken cancellation) { var id = streamHead.Id; var bucketId = streamHead.BucketId; var streamId = streamHead.StreamId; StreamHead desired; while (streamHead.DispatchedRevision < dispatchedRevision) { var sequentialDispatchedRevision = dispatchedRevision; if (streamHead.DispatchedRevision < dispatchedRevision - 1) { var commits = GetCommitsInternalAsync(bucketId, streamId, streamHead.DispatchedRevision + 1, dispatchedRevision); var latestDispatchedCommit = await LatestDispatchedCommitAsync(commits, cancellation); if (latestDispatchedCommit == null) { return(streamHead); } sequentialDispatchedRevision = latestDispatchedCommit.StreamRevision; } desired = new StreamHead(bucketId, streamId, streamHead.HeadRevision, streamHead.SnapshotRevision, sequentialDispatchedRevision, streamHead.Version + 1); if (await _database.CompareExchangeAsync(desired, streamHead, (left, right) => left.Version == right.Version, cancellation)) { return(desired); } streamHead = await _database.GetOneAsync <StreamHead>(p => p.Id == id, cancellation); if (streamHead == null && (streamHead = await AddStreamHeadAsync(bucketId, streamId, cancellation)) == null) { return(null); } } return(streamHead); }
public static async Task TwoPhasedAsync(this DaprClient client, string storeName, string streamHeadKey, StreamHead head, string headetag, Dictionary <string, string> meta, long newVersion, EventData[] versionedEvents, string sliceKey, string sliceetag) { var sliceWriteSuccess = await client.TrySaveStateAsync(storeName, sliceKey, versionedEvents, sliceetag, metadata : meta); if (!sliceWriteSuccess) { throw new DBConcurrencyException($"Error writing events. Event slice {sliceKey} ending with event version {newVersion} already exists"); } var headWriteSuccess = await client.TrySaveStateAsync(storeName, streamHeadKey, head, headetag, metadata : meta); if (!headWriteSuccess) { throw new DBConcurrencyException($"stream head {streamHeadKey} have been updated"); } }
public virtual bool AddSnapshot(Snapshot snapshot) { ThrowWhenDisposed(); Logger.Debug(Resources.AddingSnapshot, snapshot.StreamId, snapshot.StreamRevision); lock (_commits) { StreamHead currentHead = _heads.FirstOrDefault(h => h.StreamId == snapshot.StreamId); if (currentHead == null) { return(false); } _snapshots.Add(snapshot); _heads.Remove(currentHead); _heads.Add(new StreamHead(currentHead.StreamId, currentHead.HeadRevision, snapshot.StreamRevision)); } return(true); }
public static async Task <(IEnumerable <EventData> Events, long Version)> LoadSlicesAsync( this DaprClient client, string storeName, ILogger logger, string streamName, long version, Dictionary <string, string> meta, StreamHead head) { var eventSlices = new List <EventData[]>(); using (logger.BeginScope("Loading head {streamName}. Starting version {version}", streamName, head.Version)) { var next = head.Version; while (next != 0 && next >= version) { var sliceKey = Naming.StreamKey(streamName, next); var slice = await client.GetStateAsync <EventData[]>(storeName, sliceKey, metadata : meta); logger.LogDebug("Slice {sliceKey} loaded range : {firstVersion} - {lastVersion}", sliceKey, slice.First().Version, slice.Last().Version); next = slice.First().Version - 1; if (next < version) { logger.LogDebug("Version within slice. Next : {next}. Version : {version}", next, version); eventSlices.Add(slice.Where(e => e.Version >= version).ToArray()); break; } logger.LogDebug("Adding slice. Next : {next}. Version : {version}", next, version); eventSlices.Add(slice); } logger.LogDebug("Done reading. Got {sliceCount} slices.", eventSlices.Count); var events = eventSlices .Reverse <EventData[]>() .SelectMany(e => e) .ToArray(); return(events, events.LastOrDefault()?.Version ?? head.Version); } }
public static async Task StateTransactionAsync(this DaprClient client, string storeName, string streamName, string streamHeadKey, StreamHead head, string headetag, Dictionary <string, string> meta, EventData[] versionedEvents) { var eventsReq = versionedEvents.Select(x => new StateTransactionRequest(Naming.StreamKey(streamName, x.Version), JsonSerializer.SerializeToUtf8Bytes(x), StateOperationType.Upsert, metadata: meta)); var headReq = new StateTransactionRequest(streamHeadKey, JsonSerializer.SerializeToUtf8Bytes(head), Client.StateOperationType.Upsert, etag: string.IsNullOrWhiteSpace(headetag) ? null : headetag, metadata: meta); var reqs = new List <StateTransactionRequest>(); reqs.AddRange(eventsReq); reqs.Add(headReq); await client.ExecuteStateTransactionAsync(storeName, reqs, meta); }